github-featured/triggerdotdev--trigger.dev
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Implement manually invokable jobs through the invokeTrigger (#700)

Browse Source 
* Implement manually invokable jobs through the invokeTrigger

Also implemented a job run notification system, that will POST details of a run on completion. This combines with the task callbackUrl system to implement the invokeAndWait

* Document the invoke trigger

* batch invoke and wait

* background fetch timeouts

* Use @whatwg-node/fetch instead of the polyfilled fetch

* Fix some outdated dependencies in webapp

* Improved subtask error propogation messages

* Document the OpenAI changes and the batch invoke stuff

* Fix dequeuing jobs

* Don’t retry the OpenAI completion background task

* Added OpenAI changesets

* Use the new ResumeTaskService in ProcessCallbackTimeout as well
This commit is contained in:
Eric Allam
2023-11-03 11:05:00 +00:00
committed by GitHub
parent c7434125e4
commit 620b83832b
73 changed files with 3366 additions and 551 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/happy-dryers-guess.md
+6
View File
@@ -0,0 +1,6 @@
---
"@trigger.dev/sdk": patch
"@trigger.dev/core": patch
---
Added invokeTrigger(), which allows jobs to be manually invoked
.changeset/lazy-schools-grin.md
+6
View File
@@ -0,0 +1,6 @@
---
"@trigger.dev/integration-kit": patch
"@trigger.dev/openai": patch
---
Allow customizing OpenAI background retries and timeouts
.gitignore
+2
View File
@@ -52,3 +52,5 @@ apps/**/public/build
/test-results/
/playwright-report/
/playwright/.cache/
.cosine
apps/webapp/app/components/primitives/LabelValueStack.tsx
+39 -12
View File
@@ -2,6 +2,7 @@ import { cn } from "~/utils/cn";
import { Paragraph } from "./Paragraph";
import { ArrowTopRightOnSquareIcon } from "@heroicons/react/20/solid";
import { SimpleTooltip } from "./Tooltip";
import { Link } from "@remix-run/react";
const variations = {
primary: {
@@ -45,18 +46,7 @@ export function LabelValueStack({
<Paragraph variant={variation.label}>{label}</Paragraph>
<>
{href ? (
<SimpleTooltip
side="bottom"
button={
<Paragraph variant={variation.value}>
<a href={href} className="underline underline-offset-2" target="_blank">
{value}
<ArrowTopRightOnSquareIcon className="ml-1 inline-block h-4 w-4 text-dimmed" />
</a>
</Paragraph>
}
content={href}
/>
<ValueButton value={value} href={href} variant={variant} />
) : (
<Paragraph variant={variation.value}>{value}</Paragraph>
)}
@@ -64,3 +54,40 @@ export function LabelValueStack({
</div>
);
}
type ValueButtonStackProps = {
value: React.ReactNode;
href: string;
variant?: keyof typeof variations;
};
function ValueButton({ value, href, variant = "secondary" }: ValueButtonStackProps) {
const variation = variations[variant];
const isExternalUrl = href.startsWith("http");
if (!isExternalUrl) {
return (
<Paragraph variant={variation.value}>
<Link to={href} reloadDocument className="underline underline-offset-2">
{value}
</Link>
</Paragraph>
);
}
return (
<SimpleTooltip
side="bottom"
button={
<Paragraph variant={variation.value}>
<a href={href} className="underline underline-offset-2" target="_blank">
{value}
<ArrowTopRightOnSquareIcon className="ml-1 inline-block h-4 w-4 text-dimmed" />
</a>
</Paragraph>
}
content={href}
/>
);
}
apps/webapp/app/db.server.ts
+2 -19
View File
@@ -3,6 +3,7 @@ import invariant from "tiny-invariant";
import { z } from "zod";
import { logger } from "./services/logger.server";
import { env } from "./env.server";
import { singleton } from "./utils/singleton";
export type PrismaTransactionClient = Omit<
PrismaClient,
@@ -66,24 +67,7 @@ export async function $transaction<R>(
export { Prisma };
let prisma: PrismaClient;
declare global {
var __db__: PrismaClient;
}
// this is needed because in development we don't want to restart
// the server with every change, but we want to make sure we don't
// create a new connection to the DB with every change either.
// in production we'll have a single connection to the DB.
if (process.env.NODE_ENV === "production") {
prisma = getClient();
} else {
if (!global.__db__) {
global.__db__ = getClient();
}
prisma = global.__db__;
}
export const prisma = singleton("prisma", getClient);
function getClient() {
const { DATABASE_URL } = process.env;
@@ -143,7 +127,6 @@ function getClient() {
return client;
}
export { prisma };
export type { PrismaClient } from "@trigger.dev/database";
export const PrismaErrorSchema = z.object({
apps/webapp/app/models/jobRunExecution.server.ts
+39 -33
View File
@@ -1,41 +1,47 @@
import { JobRun, JobRunExecution } from "@trigger.dev/database";
import { JobRun } from "@trigger.dev/database";
import { PrismaClientOrTransaction } from "~/db.server";
import { executionWorker } from "~/services/worker.server";
export type EnqueueRunExecutionV2Options = {
runAt?: Date;
resumeTaskId?: string;
isRetry?: boolean;
skipRetrying?: boolean;
executionCount?: number;
};
export async function enqueueRunExecutionV2(
run: JobRun,
tx: PrismaClientOrTransaction,
options: EnqueueRunExecutionV2Options = {}
) {
const job = await executionWorker.enqueue(
"performRunExecutionV2",
{
id: run.id,
reason: run.status === "PREPROCESSING" ? "PREPROCESS" : "EXECUTE_JOB",
resumeTaskId: options.resumeTaskId,
isRetry: typeof options.isRetry === "boolean" ? options.isRetry : false,
},
{
tx,
runAt: options.runAt,
jobKey: `job_run:${run.id}:${options.executionCount ?? 0}${
options.resumeTaskId ? `:task:${options.resumeTaskId}` : ""
}`,
maxAttempts: options.skipRetrying ? 1 : undefined,
}
);
}
export async function dequeueRunExecutionV2(run: JobRun, tx: PrismaClientOrTransaction) {
return await executionWorker.dequeue(`job_run:${run.id}`, {
tx,
});
}
export type EnqueueRunExecutionV3Options = {
runAt?: Date;
skipRetrying?: boolean;
};
export async function enqueueRunExecutionV3(
run: JobRun,
tx: PrismaClientOrTransaction,
options: EnqueueRunExecutionV3Options = {}
) {
const reason = run.status === "PREPROCESSING" ? "PREPROCESS" : "EXECUTE_JOB";
return await executionWorker.enqueue(
"performRunExecutionV3",
{
id: run.id,
reason: reason,
},
{
tx,
runAt: options.runAt,
queueName: `job_run:${run.id}`,
jobKey: `job_run:${reason}:${run.id}`,
maxAttempts: options.skipRetrying ? 1 : undefined,
}
);
}
export async function dequeueRunExecutionV3(run: JobRun, tx: PrismaClientOrTransaction) {
await executionWorker.dequeue(`job_run:EXECUTE_JOB:${run.id}`, {
tx,
});
await executionWorker.dequeue(`job_run:PREPROCESS:${run.id}`, {
tx,
});
}
apps/webapp/app/models/task.server.ts
+1
View File
@@ -25,6 +25,7 @@ export function taskWithAttemptsToServerTask(task: TaskWithAttempts): ServerTask
operation: task.operation,
callbackUrl: task.callbackUrl,
forceYield: task.run.forceYieldImmediately,
childExecutionMode: task.childExecutionMode,
};
}
apps/webapp/app/platform/zodWorker.server.ts
+1 -2
View File
@@ -14,9 +14,8 @@ import { run as graphileRun, parseCronItems } from "graphile-worker";
import omit from "lodash.omit";
import { z } from "zod";
import { PrismaClient, PrismaClientOrTransaction } from "~/db.server";
import { workerLogger as logger } from "~/services/logger.server";
import { PgListenService } from "~/services/db/pgListen.server";
import { safeJsonParse } from "~/utils/json";
import { workerLogger as logger } from "~/services/logger.server";
export interface MessageCatalogSchema {
[key: string]: z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion<any, any>;
apps/webapp/app/routes/api.v1.endpoints.ts
+1 -1
View File
@@ -60,7 +60,7 @@ export async function action({ request }: ActionFunctionArgs) {
return json(endpoint);
} catch (error) {
if (error instanceof Error) {
logger.error("Error creating endpoint", {
logger.debug("Error creating endpoint", {
url: request.url,
error: error.message,
});
apps/webapp/app/routes/api.v1.jobs.$jobSlug.invoke.ts
+88
View File
@@ -0,0 +1,88 @@
import type { ActionFunctionArgs } from "@remix-run/server-runtime";
import { json } from "@remix-run/server-runtime";
import { InvokeJobRequestBodySchema } from "@trigger.dev/core";
import { z } from "zod";
import { PrismaErrorSchema } from "~/db.server";
import { authenticateApiRequest } from "~/services/apiAuth.server";
import { InvokeJobService } from "~/services/jobs/invokeJob.server";
import { logger } from "~/services/logger.server";
const ParamsSchema = z.object({
jobSlug: z.string(),
});
const HeadersSchema = z.object({
"idempotency-key": z.string().optional().nullable(),
"trigger-version": z.string().optional().nullable(),
});
export async function action({ request, params }: ActionFunctionArgs) {
// Ensure this is a POST request
if (request.method.toUpperCase() !== "POST") {
return { status: 405, body: "Method Not Allowed" };
}
// Authenticate the request
const authenticationResult = await authenticateApiRequest(request);
if (!authenticationResult) {
return json({ error: "Invalid or Missing API Key" }, { status: 401 });
}
const parsed = ParamsSchema.safeParse(params);
if (!parsed.success) {
return json({ error: "Invalid or Missing jobSlug" }, { status: 400 });
}
const { jobSlug } = parsed.data;
const headers = HeadersSchema.safeParse(Object.fromEntries(request.headers));
if (!headers.success) {
return json({ error: "Invalid headers" }, { status: 400 });
}
const { "idempotency-key": idempotencyKey, "trigger-version": triggerVersion } = headers.data;
// Now parse the request body
const anyBody = await request.json();
logger.debug("InvokeJobService.call() request body", {
body: anyBody,
jobSlug,
idempotencyKey,
triggerVersion,
});
const body = InvokeJobRequestBodySchema.safeParse(anyBody);
if (!body.success) {
return json({ error: "Invalid request body" }, { status: 400 });
}
const service = new InvokeJobService();
try {
const run = await service.call(
authenticationResult.environment,
jobSlug,
body.data,
idempotencyKey ?? undefined
);
if (!run) {
return json({ error: "Job count not be invoked" }, { status: 500 });
}
return json({ id: run.id });
} catch (error) {
const prismaError = PrismaErrorSchema.safeParse(error);
// Record not found in the database
if (prismaError.success && prismaError.data.code === "P2005") {
return json({ error: "Job not found" }, { status: 404 });
} else {
return json({ error: "Internal Server Error" }, { status: 500 });
}
}
}
apps/webapp/app/routes/api.v1.runs.$runId.tasks.$id.callback.$secret.ts
+5 -4
View File
@@ -3,8 +3,9 @@ import { json } from "@remix-run/server-runtime";
import { RuntimeEnvironmentType } from "@trigger.dev/database";
import { z } from "zod";
import { $transaction, PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server";
import { enqueueRunExecutionV2 } from "~/models/jobRunExecution.server";
import { logger } from "~/services/logger.server";
import { ResumeTaskService } from "~/services/tasks/resumeTask.server";
import { workerQueue } from "~/services/worker.server";
const ParamsSchema = z.object({
runId: z.string(),
@@ -96,14 +97,14 @@ export class CallbackRunTaskService {
},
});
await workerQueue.dequeue(`process-callback:${task.id}`, { tx });
await this.#resumeRunExecution(task, tx);
});
}
async #resumeRunExecution(task: NonNullable<FoundTask>, prisma: PrismaClientOrTransaction) {
await enqueueRunExecutionV2(task.run, prisma, {
skipRetrying: task.run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
});
await ResumeTaskService.enqueue(task.id, undefined, prisma);
}
}
apps/webapp/app/routes/api.v1.runs.$runId.tasks.$id.complete.ts
+111 -67
View File
@@ -1,9 +1,13 @@
import type { ActionFunctionArgs } from "@remix-run/server-runtime";
import { json } from "@remix-run/server-runtime";
import type { CompleteTaskBodyOutput, ServerTask } from "@trigger.dev/core";
import { CompleteTaskBodyInputSchema } from "@trigger.dev/core";
import {
API_VERSIONS,
CompleteTaskBodyInputSchema,
CompleteTaskBodyV2InputSchema,
} from "@trigger.dev/core";
import { z } from "zod";
import { $transaction, PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server";
import { PrismaClientOrTransaction, prisma } from "~/db.server";
import { taskWithAttemptsToServerTask } from "~/models/task.server";
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
import { authenticateApiRequest } from "~/services/apiAuth.server";
@@ -14,6 +18,10 @@ const ParamsSchema = z.object({
id: z.string(),
});
const HeadersSchema = z.object({
"trigger-version": z.string().optional().nullable(),
});
export async function action({ request, params }: ActionFunctionArgs) {
// Ensure this is a POST request
if (request.method.toUpperCase() !== "POST") {
@@ -31,6 +39,14 @@ export async function action({ request, params }: ActionFunctionArgs) {
const { runId, id } = ParamsSchema.parse(params);
const headers = HeadersSchema.safeParse(Object.fromEntries(request.headers));
if (!headers.success) {
return json({ error: "Invalid headers" }, { status: 400 });
}
const { "trigger-version": triggerVersion } = headers.data;
// Now parse the request body
const anyBody = await request.json();
@@ -40,16 +56,48 @@ export async function action({ request, params }: ActionFunctionArgs) {
id,
});
const body = CompleteTaskBodyInputSchema.safeParse(anyBody);
if (triggerVersion === API_VERSIONS.SERIALIZED_TASK_OUTPUT) {
const body = CompleteTaskBodyV2InputSchema.safeParse(anyBody);
if (!body.success) {
return json({ error: "Invalid request body" }, { status: 400 });
if (!body.success) {
return json({ error: "Invalid request body" }, { status: 400 });
}
// Make sure the length of the output is less than 3MB
if (body.data.output && body.data.output.length > 3 * 1024 * 1024) {
return json({ error: "Output must be less than 3MB" }, { status: 400 });
}
return await completeRunTask(authenticatedEnv, runId, id, {
...body.data,
output: body.data.output ? (JSON.parse(body.data.output) as any) : undefined,
});
} else {
const body = CompleteTaskBodyInputSchema.safeParse(anyBody);
if (!body.success) {
return json({ error: "Invalid request body" }, { status: 400 });
}
// Make sure the length of the output is less than 3MB
if (JSON.stringify(body.data.output).length > 3 * 1024 * 1024) {
return json({ error: "Output must be less than 3MB" }, { status: 400 });
}
return await completeRunTask(authenticatedEnv, runId, id, body.data);
}
}
async function completeRunTask(
environment: AuthenticatedEnvironment,
runId: string,
id: string,
taskBody: CompleteTaskBodyOutput
) {
const service = new CompleteRunTaskService();
try {
const task = await service.call(authenticatedEnv, runId, id, body.data);
const task = await service.call(environment, runId, id, taskBody);
logger.debug("CompleteRunTaskService.call() response body", {
runId,
@@ -84,79 +132,75 @@ export class CompleteRunTaskService {
id: string,
taskBody: CompleteTaskBodyOutput
): Promise<ServerTask | undefined> {
// Using a transaction, we'll first check to see if the task already exists and return if if it does
// If it doesn't exist, we'll create it and return it
const task = await $transaction(this.#prismaClient, async (tx) => {
const existingTask = await tx.task.findUnique({
where: {
id,
},
include: {
run: true,
attempts: {
where: {
status: "PENDING",
},
orderBy: {
number: "desc",
},
take: 1,
const existingTask = await this.#prismaClient.task.findUnique({
where: {
id,
},
include: {
run: true,
attempts: {
where: {
status: "PENDING",
},
orderBy: {
number: "desc",
},
take: 1,
},
},
});
if (!existingTask) {
return;
}
if (existingTask.runId !== runId) {
return;
}
if (existingTask.run.environmentId !== environment.id) {
return;
}
if (
existingTask.status === "COMPLETED" ||
existingTask.status === "ERRORED" ||
existingTask.status === "CANCELED"
) {
logger.debug("Task already completed", {
existingTask,
});
if (!existingTask) {
return;
}
return taskWithAttemptsToServerTask(existingTask);
}
if (existingTask.runId !== runId) {
return;
}
if (existingTask.run.environmentId !== environment.id) {
return;
}
if (
existingTask.status === "COMPLETED" ||
existingTask.status === "ERRORED" ||
existingTask.status === "CANCELED"
) {
logger.debug("Task already completed", {
existingTask,
});
return existingTask;
}
if (existingTask.attempts.length === 1) {
await tx.taskAttempt.update({
where: {
id: existingTask.attempts[0].id,
},
data: {
status: "COMPLETED",
},
});
}
return await tx.task.update({
if (existingTask.attempts.length === 1) {
await this.#prismaClient.taskAttempt.update({
where: {
id,
id: existingTask.attempts[0].id,
},
data: {
status: "COMPLETED",
output: taskBody.output ?? undefined,
completedAt: new Date(),
outputProperties: taskBody.properties,
},
include: {
attempts: true,
run: true,
},
});
}
const updatedTask = await this.#prismaClient.task.update({
where: {
id,
},
data: {
status: "COMPLETED",
output: taskBody.output ?? undefined,
completedAt: new Date(),
outputProperties: taskBody.properties,
},
include: {
attempts: true,
run: true,
},
});
return task ? taskWithAttemptsToServerTask(task) : undefined;
return taskWithAttemptsToServerTask(updatedTask);
}
}
apps/webapp/app/routes/api.v1.runs.$runId.tasks.ts
+6 -1
View File
@@ -286,6 +286,7 @@ export class RunTaskService {
operation: taskBody.operation,
callbackUrl,
style: taskBody.style ?? { style: "normal" },
childExecutionMode: taskBody.parallel ? "PARALLEL" : "SEQUENTIAL",
attempts: {
create: {
number: 1,
@@ -316,7 +317,11 @@ export class RunTaskService {
{
id: task.id,
},
{ tx, runAt: new Date(Date.now() + taskBody.callback.timeoutInSeconds * 1000) }
{
tx,
runAt: new Date(Date.now() + taskBody.callback.timeoutInSeconds * 1000),
jobKey: `process-callback:${task.id}`,
}
);
}
}
apps/webapp/app/services/email.server.ts
+11 -6
View File
@@ -7,13 +7,18 @@ import type { User } from "~/models/user.server";
import type { AuthUser } from "./authUser";
import { workerQueue } from "./worker.server";
import { logger } from "./logger.server";
import { singleton } from "~/utils/singleton";
const client = new EmailClient({
apikey: env.RESEND_API_KEY,
imagesBaseUrl: env.APP_ORIGIN,
from: env.FROM_EMAIL ?? "team@email.trigger.dev",
replyTo: env.REPLY_TO_EMAIL ?? "help@email.trigger.dev",
});
const client = singleton(
"email-client",
() =>
new EmailClient({
apikey: env.RESEND_API_KEY,
imagesBaseUrl: env.APP_ORIGIN,
from: env.FROM_EMAIL ?? "team@email.trigger.dev",
replyTo: env.REPLY_TO_EMAIL ?? "help@email.trigger.dev",
})
);
export async function sendMagicLinkEmail(options: SendEmailOptions<AuthUser>): Promise<void> {
// Auto redirect when in development mode
apps/webapp/app/services/endpointApi.server.ts
+2 -5
View File
@@ -1,8 +1,6 @@
import {
API_VERSIONS,
ApiEventLog,
DeliverEventResponseSchema,
DeserializedJson,
ConnectionAuth,
EndpointHeadersSchema,
ErrorWithStackSchema,
HttpSourceRequest,
@@ -19,10 +17,9 @@ import {
ValidateResponse,
ValidateResponseSchema,
} from "@trigger.dev/core";
import { performance } from "node:perf_hooks";
import { safeBodyFromResponse, safeParseBodyFromResponse } from "~/utils/json";
import { logger } from "./logger.server";
import { ConnectionAuth } from "@trigger.dev/core";
import { performance } from "node:perf_hooks";
export class EndpointApiError extends Error {
constructor(message: string, stack?: string) {
apps/webapp/app/services/jobs/invokeJob.server.ts
+143
View File
@@ -0,0 +1,143 @@
import { $transaction, PrismaClient, prisma } from "~/db.server";
import { AuthenticatedEnvironment } from "../apiAuth.server";
import { InvokeJobRequestBody } from "@trigger.dev/core";
import { ulid } from "../ulid.server";
import { CreateRunService } from "../runs/createRun.server";
export class InvokeJobService {
#prismaClient: PrismaClient;
constructor(prismaClient: PrismaClient = prisma) {
this.#prismaClient = prismaClient;
}
public async call(
environment: AuthenticatedEnvironment,
jobSlug: string,
data: InvokeJobRequestBody,
idempotencyKey?: string
) {
return await $transaction(this.#prismaClient, async (tx) => {
// Check if this is an idempotent request
if (idempotencyKey) {
const existingEvent = await tx.eventRecord.findUnique({
where: {
eventId_environmentId: {
eventId: idempotencyKey,
environmentId: environment.id,
},
},
include: {
runs: true,
},
});
if (existingEvent) {
return existingEvent.runs[0];
}
}
const job = await tx.job.findUniqueOrThrow({
where: {
projectId_slug: {
projectId: environment.projectId,
slug: jobSlug,
},
},
include: {
aliases: {
where: {
environmentId: environment.id,
name: "latest",
},
include: {
version: true,
},
take: 1,
},
},
});
const alias = job.aliases[0];
if (!alias) {
throw new Error(`No version found for job ${jobSlug} in environment ${environment.slug}`);
}
const version = alias.version;
if (!version) {
throw new Error(`No version found for job ${jobSlug} in environment ${environment.slug}`);
}
const options = data.options ?? {};
const externalAccount = options.accountId
? await tx.externalAccount.upsert({
where: {
environmentId_identifier: {
environmentId: environment.id,
identifier: options.accountId,
},
},
create: {
environmentId: environment.id,
organizationId: environment.organizationId,
identifier: options.accountId,
},
update: {},
})
: undefined;
const eventLog = await tx.eventRecord.create({
data: {
organization: {
connect: {
id: environment.organizationId,
},
},
project: {
connect: {
id: environment.projectId,
},
},
environment: {
connect: {
id: environment.id,
},
},
externalAccount: externalAccount
? {
connect: {
id: externalAccount.id,
},
}
: undefined,
eventId: idempotencyKey ?? ulid(),
name: "invoke",
timestamp: new Date(),
payload: data.payload ?? {},
context: data.context ?? {},
source: "trigger.dev",
internal: true,
},
});
const createRunService = new CreateRunService(tx);
const run = await createRunService.call(
{
environment,
eventId: eventLog.id,
job: job,
version,
},
{
callbackUrl: options.callbackUrl,
}
);
return run;
});
}
}
apps/webapp/app/services/jobs/testJob.server.ts
+1
View File
@@ -94,6 +94,7 @@ export class TestJobService {
context: {},
source: event.source ?? "trigger.dev",
isTest: true,
internal: true,
},
});
apps/webapp/app/services/logger.server.ts
+19 -10
View File
@@ -1,17 +1,26 @@
import type { LogLevel } from "@trigger.dev/core";
import { Logger } from "@trigger.dev/core";
import { sensitiveDataReplacer } from "./sensitiveDataReplacer";
import { singleton } from "~/utils/singleton";
export const logger = new Logger(
"webapp",
(process.env.APP_LOG_LEVEL ?? "debug") as LogLevel,
["examples", "output", "connectionString", "payload"],
sensitiveDataReplacer
export const logger = singleton(
"logger",
() =>
new Logger(
"webapp",
(process.env.APP_LOG_LEVEL ?? "debug") as LogLevel,
["examples", "output", "connectionString", "payload"],
sensitiveDataReplacer
)
);
export const workerLogger = new Logger(
"worker",
(process.env.APP_LOG_LEVEL ?? "debug") as LogLevel,
["examples", "output", "connectionString"],
sensitiveDataReplacer
export const workerLogger = singleton(
"worker-logger",
() =>
new Logger(
"worker",
(process.env.APP_LOG_LEVEL ?? "debug") as LogLevel,
["examples", "output", "connectionString"],
sensitiveDataReplacer
)
);
apps/webapp/app/services/runs/cancelRun.server.ts
+2 -2
View File
@@ -1,6 +1,6 @@
import { PrismaClient, prisma } from "~/db.server";
import { executionWorker } from "../worker.server";
import { dequeueRunExecutionV2 } from "~/models/jobRunExecution.server";
import { dequeueRunExecutionV3 } from "~/models/jobRunExecution.server";
export class CancelRunService {
#prismaClient: PrismaClient;
@@ -39,7 +39,7 @@ export class CancelRunService {
},
});
await dequeueRunExecutionV2(run, tx);
await dequeueRunExecutionV3(run, tx);
});
} catch (error) {
throw error;
apps/webapp/app/services/runs/continueRun.server.ts
+2 -2
View File
@@ -1,6 +1,6 @@
import { RuntimeEnvironmentType } from "@trigger.dev/database";
import { $transaction, Prisma, PrismaClient, prisma } from "~/db.server";
import { enqueueRunExecutionV2 } from "~/models/jobRunExecution.server";
import { enqueueRunExecutionV3 } from "~/models/jobRunExecution.server";
const RESUMABLE_STATUSES = ["FAILURE", "TIMED_OUT", "UNRESOLVED_AUTH", "ABORTED", "CANCELED"];
@@ -39,7 +39,7 @@ export class ContinueRunService {
},
});
await enqueueRunExecutionV2(run, tx, {
await enqueueRunExecutionV3(run, tx, {
skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
});
},
apps/webapp/app/services/runs/createRun.server.ts
+33 -11
View File
@@ -11,17 +11,20 @@ export class CreateRunService {
this.#prismaClient = prismaClient;
}
public async call({
environment,
eventId,
job,
version,
}: {
environment: AuthenticatedEnvironment;
eventId: string;
job: Job;
version: JobVersion;
}) {
public async call(
{
environment,
eventId,
job,
version,
}: {
environment: AuthenticatedEnvironment;
eventId: string;
job: Job;
version: JobVersion;
},
options: { callbackUrl?: string } = {}
) {
const endpoint = await this.#prismaClient.endpoint.findUniqueOrThrow({
where: {
id: version.endpointId,
@@ -74,6 +77,25 @@ export class CreateRunService {
},
});
if (options.callbackUrl) {
await tx.jobRunSubscription.createMany({
data: [
{
runId: run.id,
recipientMethod: "WEBHOOK",
recipient: options.callbackUrl,
event: "SUCCESS",
},
{
runId: run.id,
recipientMethod: "WEBHOOK",
recipient: options.callbackUrl,
event: "FAILURE",
},
],
});
}
await workerQueue.enqueue(
"startRun",
{
apps/webapp/app/services/runs/deliverRunSubscription.server.ts
+164
View File
@@ -0,0 +1,164 @@
import { RunNotification } from "@trigger.dev/core";
import { subtle } from "node:crypto";
import { PrismaClient, prisma } from "~/db.server";
// Infer the type of the #findSubscription method
type FoundSubscription = NonNullable<
Awaited<ReturnType<DeliverRunSubscriptionService["_findSubscription"]>>
>;
type FoundRun = FoundSubscription["run"];
export class DeliverRunSubscriptionService {
#prismaClient: PrismaClient;
constructor(prismaClient: PrismaClient = prisma) {
this.#prismaClient = prismaClient;
}
public async call(id: string) {
const subscription = await this._findSubscription(id);
if (!subscription) {
return;
}
if (subscription.deliveredAt) {
return;
}
if (subscription.status !== "ACTIVE") {
return;
}
const { run } = subscription;
const payload = this.#getPayload(run);
const delivered = await this.#deliverPayload(subscription, payload);
if (delivered) {
await this.#prismaClient.jobRunSubscription.update({
where: {
id,
},
data: {
deliveredAt: new Date(),
},
});
} else {
throw new Error(`Failed to deliver subscription ${id}`);
}
}
async #deliverPayload(
subscription: FoundSubscription,
payload: RunNotification<any>
): Promise<boolean> {
switch (subscription.recipientMethod) {
case "WEBHOOK": {
const url = subscription.recipient;
const rawPayload = JSON.stringify(payload);
const hashPayload = Buffer.from(rawPayload, "utf-8");
const hmacSecret = Buffer.from(subscription.run.environment.apiKey, "utf-8");
const key = await subtle.importKey(
"raw",
hmacSecret,
{ name: "HMAC", hash: "SHA-256" },
false,
["sign"]
);
const signature = await subtle.sign("HMAC", key, hashPayload);
const signatureHex = Buffer.from(signature).toString("hex");
const response = await fetch(url, {
method: "POST",
headers: {
"Content-Type": "application/json",
"X-Trigger-Signature-256": signatureHex,
},
body: rawPayload,
});
if (!response.ok) {
throw new Error(
`Failed to deliver webhook to ${url}: [${response.status}] ${response.statusText}`
);
}
return true;
}
}
}
private async _findSubscription(id: string) {
return this.#prismaClient.jobRunSubscription.findUnique({
where: {
id,
},
include: {
run: {
include: {
job: true,
version: true,
statuses: true,
environment: true,
organization: true,
project: true,
event: true,
},
},
},
});
}
#getPayload(run: FoundRun): RunNotification<any> {
const { id, job, version, statuses, environment, organization, project, event } = run;
const payload = {
id,
ok: run.status === "SUCCESS",
status: run.status,
startedAt: run.startedAt,
updatedAt: run.updatedAt,
completedAt: run.completedAt,
executionDurationInMs: run.executionDuration,
executionCount: run.executionCount,
job: {
id: job.id,
version: version.version,
},
statuses: statuses.map((status) => ({
key: status.key,
label: status.label,
state: status.state,
data: status.data,
history: status.history,
})),
environment: {
slug: environment.slug,
id: environment.id,
type: environment.type,
},
organization: {
slug: organization.slug,
id: organization.id,
title: organization.title,
},
project: {
slug: project.slug,
id: project.id,
name: project.name,
},
invocation: {
id: event.id,
context: event.context,
timestamp: event.timestamp,
},
...(run.status === "SUCCESS" ? { output: run.output } : { error: run.output }),
};
return payload as RunNotification<any>;
}
}
apps/webapp/app/services/runs/deliverRunSubscriptions.server.ts
+44
View File
@@ -0,0 +1,44 @@
import { JobRun } from "@trigger.dev/database";
import { PrismaClient, prisma } from "~/db.server";
import { workerQueue } from "../worker.server";
export class DeliverRunSubscriptionsService {
#prismaClient: PrismaClient;
constructor(prismaClient: PrismaClient = prisma) {
this.#prismaClient = prismaClient;
}
public async call(id: string) {
const run = await this.#prismaClient.jobRun.findUnique({
where: {
id,
},
});
if (!run) {
return;
}
const subscriptions = await this.#findSubscriptions(run);
for (const subscription of subscriptions) {
await workerQueue.enqueue("deliverRunSubscription", {
id: subscription.id,
});
}
}
async #findSubscriptions(run: JobRun) {
const subscriptions = await this.#prismaClient.jobRunSubscription.findMany({
where: {
runId: run.id,
deliveredAt: null,
status: "ACTIVE",
event: run.status === "SUCCESS" ? "SUCCESS" : "FAILURE",
},
});
return subscriptions;
}
}
apps/webapp/app/services/runs/performRunExecutionV2.server.ts → apps/webapp/app/services/runs/performRunExecutionV3.server.ts
+199 -82
View File
@@ -1,11 +1,13 @@
import {
ApiEventLog,
AutoYieldMetadata,
ConnectionAuth,
EndpointHeadersSchema,
RunJobAutoYieldWithCompletedTaskExecutionError,
RunJobBody,
RunJobError,
RunJobInvalidPayloadError,
RunJobResumeWithParallelTask,
RunJobResumeWithTask,
RunJobRetryWithTask,
RunJobSuccess,
@@ -25,7 +27,7 @@ import {
} from "~/consts";
import { $transaction, PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server";
import { detectResponseIsTimeout } from "~/models/endpoint.server";
import { enqueueRunExecutionV2 } from "~/models/jobRunExecution.server";
import { enqueueRunExecutionV3 } from "~/models/jobRunExecution.server";
import { resolveRunConnections } from "~/models/runConnection.server";
import { prepareTasksForCaching, prepareTasksForCachingLegacy } from "~/models/task.server";
import { CompleteRunTaskService } from "~/routes/api.v1.runs.$runId.tasks.$id.complete";
@@ -34,6 +36,8 @@ import { safeJsonZodParse } from "~/utils/json";
import { EndpointApi } from "../endpointApi.server";
import { logger } from "../logger.server";
import { forceYieldCoordinator } from "./forceYieldCoordinator.server";
import { workerQueue } from "../worker.server";
import { ResumeTaskService } from "../tasks/resumeTask.server";
type FoundRun = NonNullable<Awaited<ReturnType<typeof findRun>>>;
type FoundTask = FoundRun["tasks"][number];
@@ -41,21 +45,29 @@ type FoundTask = FoundRun["tasks"][number];
// We need to limit the cached tasks to not be too large >3.5MB when serialized
const TOTAL_CACHED_TASK_BYTE_LIMIT = 3500000;
export type PerformRunExecutionV2Input = {
export type PerformRunExecutionV3Input = {
id: string;
reason: "PREPROCESS" | "EXECUTE_JOB";
/**
* @deprecated This is no longer used
*/
isRetry: boolean;
/**
* @deprecated Resuming tasks now goes through ResumeTaskService, this is included here for backwards compatibility
*/
resumeTaskId?: string;
};
export class PerformRunExecutionV2Service {
export class PerformRunExecutionV3Service {
#prismaClient: PrismaClient;
constructor(prismaClient: PrismaClient = prisma) {
this.#prismaClient = prismaClient;
}
public async call(input: PerformRunExecutionV2Input) {
public async call(input: PerformRunExecutionV3Input) {
const run = await findRun(this.#prismaClient, input.id);
if (!run) {
@@ -159,13 +171,13 @@ export class PerformRunExecutionV2Service {
},
});
await enqueueRunExecutionV2(run, tx, {
await enqueueRunExecutionV3(run, tx, {
skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
});
});
}
}
async #executeJob(run: FoundRun, input: PerformRunExecutionV2Input) {
async #executeJob(run: FoundRun, input: PerformRunExecutionV3Input) {
try {
const { isRetry, resumeTaskId } = input;
@@ -379,6 +391,11 @@ export class PerformRunExecutionV2Service {
const status = safeBody.data.status;
logger.debug("Endpoint responded with status", {
status,
data: safeBody.data,
});
switch (status) {
case "SUCCESS": {
await this.#completeRunWithSuccess(run, safeBody.data, durationInMs);
@@ -386,7 +403,7 @@ export class PerformRunExecutionV2Service {
break;
}
case "RESUME_WITH_TASK": {
await this.#resumeRunWithTask(run, safeBody.data, isRetry, durationInMs, executionCount);
await this.#resumeRunWithTask(run, safeBody.data, durationInMs);
break;
}
@@ -396,7 +413,7 @@ export class PerformRunExecutionV2Service {
break;
}
case "RETRY_WITH_TASK": {
await this.#retryRunWithTask(run, safeBody.data, isRetry, durationInMs, executionCount);
await this.#retryRunWithTask(run, safeBody.data, durationInMs);
break;
}
@@ -415,33 +432,20 @@ export class PerformRunExecutionV2Service {
break;
}
case "YIELD_EXECUTION": {
await this.#resumeYieldedRun(
run,
safeBody.data.key,
isRetry,
durationInMs,
executionCount
);
await this.#resumeYieldedRun(run, safeBody.data.key, durationInMs);
break;
}
case "AUTO_YIELD_EXECUTION": {
await this.#resumeAutoYieldedRun(
run,
safeBody.data,
isRetry,
durationInMs,
executionCount
);
await this.#resumeAutoYieldedRun(run, safeBody.data, durationInMs);
break;
}
case "AUTO_YIELD_EXECUTION_WITH_COMPLETED_TASK": {
await this.#resumeAutoYieldedRunWithCompletedTask(
run,
safeBody.data,
isRetry,
durationInMs,
executionCount
);
await this.#resumeAutoYieldedRunWithCompletedTask(run, safeBody.data, durationInMs);
break;
}
case "RESUME_WITH_PARALLEL_TASK": {
await this.#resumeParallelRunWithTask(run, safeBody.data, durationInMs);
break;
}
default: {
@@ -488,6 +492,11 @@ export class PerformRunExecutionV2Service {
slug: run.organization.slug,
title: run.organization.title,
},
project: {
id: run.project.id,
slug: run.project.slug,
name: run.project.name,
},
account: run.externalAccount
? {
id: run.externalAccount.identifier,
@@ -534,6 +543,11 @@ export class PerformRunExecutionV2Service {
slug: run.organization.slug,
title: run.organization.title,
},
project: {
id: run.project.id,
slug: run.project.slug,
name: run.project.name,
},
account: run.externalAccount
? {
id: run.externalAccount.identifier,
@@ -547,25 +561,34 @@ export class PerformRunExecutionV2Service {
}
async #completeRunWithSuccess(run: FoundRun, data: RunJobSuccess, durationInMs: number) {
await this.#prismaClient.jobRun.update({
where: { id: run.id },
data: {
completedAt: new Date(),
status: "SUCCESS",
output: data.output ?? undefined,
executionDuration: {
increment: durationInMs,
await $transaction(this.#prismaClient, async (tx) => {
await tx.jobRun.update({
where: { id: run.id },
data: {
completedAt: new Date(),
status: "SUCCESS",
output: data.output ?? undefined,
executionDuration: {
increment: durationInMs,
},
},
},
});
await workerQueue.enqueue(
"deliverRunSubscriptions",
{
id: run.id,
},
{ tx }
);
});
}
async #resumeRunWithTask(
run: FoundRun,
data: RunJobResumeWithTask,
isRetry: boolean,
durationInMs: number,
executionCount: number
executionCount: number = 1
) {
return await $transaction(this.#prismaClient, async (tx) => {
await tx.jobRun.update({
@@ -574,25 +597,126 @@ export class PerformRunExecutionV2Service {
executionDuration: {
increment: durationInMs,
},
executionCount: {
increment: executionCount,
},
},
});
if (data.task.outputProperties) {
await tx.task.update({
where: {
id: data.task.id,
},
data: {
outputProperties: data.task.outputProperties,
},
});
}
// If the task has an operation, then the next performRunExecution will occur
// when that operation has finished
// Tasks with callbacks enabled will also get processed separately, i.e. when
// they time out, or on valid requests to their callbackUrl
if (!data.task.operation && !data.task.callbackUrl) {
await enqueueRunExecutionV2(run, tx, {
runAt: data.task.delayUntil ?? undefined,
resumeTaskId: data.task.id,
isRetry,
skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
executionCount,
});
await ResumeTaskService.enqueue(data.task.id, data.task.delayUntil ?? undefined, tx);
}
});
}
async #resumeParallelRunWithTask(
run: FoundRun,
data: RunJobResumeWithParallelTask,
durationInMs: number
) {
await this.#prismaClient.jobRun.update({
where: { id: run.id },
data: {
executionDuration: {
increment: durationInMs,
},
executionCount: {
increment: 1,
},
forceYieldImmediately: false,
},
});
if (data.task.outputProperties) {
await this.#prismaClient.task.update({
where: {
id: data.task.id,
},
data: {
outputProperties: data.task.outputProperties,
},
});
}
for (const childError of data.childErrors) {
switch (childError.status) {
case "AUTO_YIELD_EXECUTION": {
await this.#resumeAutoYieldedRun(run, childError, 0, 0);
break;
}
case "AUTO_YIELD_EXECUTION_WITH_COMPLETED_TASK": {
await this.#resumeAutoYieldedRunWithCompletedTask(run, childError, 0, 0);
break;
}
case "CANCELED": {
break;
}
case "ERROR": {
return await this.#failRunExecution(
this.#prismaClient,
"EXECUTE_JOB",
run,
childError.error ?? undefined,
"FAILURE",
durationInMs
);
}
case "INVALID_PAYLOAD": {
return await this.#failRunExecution(
this.#prismaClient,
"EXECUTE_JOB",
run,
childError.errors,
"INVALID_PAYLOAD",
durationInMs
);
}
case "RESUME_WITH_TASK": {
await this.#resumeRunWithTask(run, childError, 0, 0);
break;
}
case "RETRY_WITH_TASK": {
await this.#retryRunWithTask(run, childError, 0, 0);
break;
}
case "UNRESOLVED_AUTH_ERROR": {
return await this.#failRunExecution(
this.#prismaClient,
"EXECUTE_JOB",
run,
childError.issues,
"UNRESOLVED_AUTH",
durationInMs
);
}
case "YIELD_EXECUTION": {
await this.#resumeYieldedRun(run, childError.key, 0, 0);
break;
}
}
}
}
async #failRunWithError(execution: FoundRun, data: RunJobError, durationInMs: number) {
return await $transaction(this.#prismaClient, async (tx) => {
if (data.task) {
@@ -656,9 +780,8 @@ export class PerformRunExecutionV2Service {
async #resumeYieldedRun(
run: FoundRun,
key: string,
isRetry: boolean,
durationInMs: number,
executionCount: number
executionCount: number = 1
) {
await $transaction(this.#prismaClient, async (tx) => {
if (run.yieldedExecutions.length + 1 > MAX_RUN_YIELDED_EXECUTIONS) {
@@ -683,7 +806,7 @@ export class PerformRunExecutionV2Service {
increment: durationInMs,
},
executionCount: {
increment: 1,
increment: executionCount,
},
yieldedExecutions: {
push: key,
@@ -696,20 +819,17 @@ export class PerformRunExecutionV2Service {
},
});
await enqueueRunExecutionV2(run, tx, {
isRetry,
await enqueueRunExecutionV3(run, tx, {
skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
executionCount,
});
});
}
async #resumeAutoYieldedRun(
run: FoundRun,
data: { location: string; timeRemaining: number; timeElapsed: number; limit?: number },
isRetry: boolean,
data: AutoYieldMetadata,
durationInMs: number,
executionCount: number
executionCount: number = 1
) {
await $transaction(this.#prismaClient, async (tx) => {
await tx.jobRun.update({
@@ -721,7 +841,7 @@ export class PerformRunExecutionV2Service {
increment: durationInMs,
},
executionCount: {
increment: 1,
increment: executionCount,
},
autoYieldExecution: {
create: [
@@ -740,10 +860,8 @@ export class PerformRunExecutionV2Service {
},
});
await enqueueRunExecutionV2(run, tx, {
isRetry,
await enqueueRunExecutionV3(run, tx, {
skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
executionCount,
});
});
}
@@ -751,9 +869,8 @@ export class PerformRunExecutionV2Service {
async #resumeAutoYieldedRunWithCompletedTask(
run: FoundRun,
data: RunJobAutoYieldWithCompletedTaskExecutionError,
isRetry: boolean,
durationInMs: number,
executionCount: number
executionCount: number = 1
) {
await $transaction(this.#prismaClient, async (tx) => {
await tx.jobRun.update({
@@ -765,7 +882,7 @@ export class PerformRunExecutionV2Service {
increment: durationInMs,
},
executionCount: {
increment: 1,
increment: executionCount,
},
autoYieldExecution: {
create: [
@@ -788,13 +905,11 @@ export class PerformRunExecutionV2Service {
await service.call(run.environment, run.id, data.id, {
properties: data.properties,
output: data.output,
output: data.output ? (JSON.parse(data.output) as any) : undefined,
});
await enqueueRunExecutionV2(run, tx, {
isRetry,
await enqueueRunExecutionV3(run, tx, {
skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
executionCount,
});
});
}
@@ -802,9 +917,8 @@ export class PerformRunExecutionV2Service {
async #retryRunWithTask(
run: FoundRun,
data: RunJobRetryWithTask,
isRetry: boolean,
durationInMs: number,
executionCount: number
executionCount: number = 1
) {
return await $transaction(this.#prismaClient, async (tx) => {
// We need to check for an existing task attempt
@@ -851,25 +965,22 @@ export class PerformRunExecutionV2Service {
executionDuration: {
increment: durationInMs,
},
executionCount: {
increment: executionCount,
},
},
},
},
});
await enqueueRunExecutionV2(run, tx, {
runAt: data.retryAt,
resumeTaskId: data.task.id,
isRetry,
skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
executionCount,
});
await ResumeTaskService.enqueue(data.task.id, data.retryAt, tx);
});
}
async #resumeRunExecutionAfterTimeout(
prisma: PrismaClientOrTransaction,
run: FoundRun,
input: PerformRunExecutionV2Input,
input: PerformRunExecutionV3Input,
durationInMs: number,
executionCount: number
) {
@@ -963,11 +1074,8 @@ export class PerformRunExecutionV2Service {
});
// The run has timed out, so we need to enqueue a new execution
await enqueueRunExecutionV2(run, tx, {
resumeTaskId: input.resumeTaskId,
isRetry: input.isRetry,
await enqueueRunExecutionV3(run, tx, {
skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
executionCount,
});
});
}
@@ -1014,6 +1122,14 @@ export class PerformRunExecutionV2Service {
},
});
await workerQueue.enqueue(
"deliverRunSubscriptions",
{
id: run.id,
},
{ tx }
);
break;
}
case "PREPROCESS": {
@@ -1041,7 +1157,7 @@ export class PerformRunExecutionV2Service {
},
});
await enqueueRunExecutionV2(run, tx, {
await enqueueRunExecutionV3(run, tx, {
skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
});
@@ -1080,6 +1196,7 @@ async function findRun(prisma: PrismaClientOrTransaction, id: string) {
},
endpoint: true,
organization: true,
project: true,
externalAccount: true,
runConnections: {
include: {
apps/webapp/app/services/runs/startRun.server.ts
+2 -2
View File
@@ -6,7 +6,7 @@ import {
} from "@trigger.dev/database";
import type { PrismaClient, PrismaClientOrTransaction } from "~/db.server";
import { prisma } from "~/db.server";
import { enqueueRunExecutionV2 } from "~/models/jobRunExecution.server";
import { enqueueRunExecutionV3 } from "~/models/jobRunExecution.server";
import { workerQueue } from "../worker.server";
type FoundRun = NonNullable<Awaited<ReturnType<typeof findRun>>>;
@@ -88,7 +88,7 @@ export class StartRunService {
const updatedRun = await updateRun();
await enqueueRunExecutionV2(updatedRun, this.#prismaClient, {
await enqueueRunExecutionV3(updatedRun, this.#prismaClient, {
skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
});
}
apps/webapp/app/services/tasks/performTaskOperation.server.ts
+96 -39
View File
@@ -4,15 +4,17 @@ import {
FetchRetryOptions,
FetchRetryStrategy,
RedactString,
RetryOptions,
calculateRetryAt,
} from "@trigger.dev/core";
import { RuntimeEnvironmentType, type Task } from "@trigger.dev/database";
import { type Task } from "@trigger.dev/database";
import { $transaction, PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server";
import { enqueueRunExecutionV2 } from "~/models/jobRunExecution.server";
import { formatUnknownError } from "~/utils/formatErrors.server";
import { safeJsonFromResponse } from "~/utils/json";
import { logger } from "../logger.server";
import { workerQueue } from "../worker.server";
import { ResumeTaskService } from "./resumeTask.server";
import { fetch } from "@whatwg-node/fetch";
type FoundTask = Awaited<ReturnType<typeof findTask>>;
@@ -38,8 +40,6 @@ export class PerformTaskOperationService {
return await this.#resumeTask(task, null, 0);
}
logger.debug("PerformTaskOperationService.call", { task });
switch (task.operation) {
case "fetch": {
const fetchOperation = FetchOperationSchema.safeParse(task.params);
@@ -51,52 +51,100 @@ export class PerformTaskOperationService {
);
}
const { url, requestInit, retry } = fetchOperation.data;
const { url, requestInit, retry, timeout } = fetchOperation.data;
const startTimeInMs = performance.now();
const response = await fetch(url, {
method: requestInit?.method ?? "GET",
headers: normalizeHeaders(requestInit?.headers ?? {}),
body: requestInit?.body,
});
const abortController = new AbortController();
const durationInMs = Math.floor(performance.now() - startTimeInMs);
// calculate the actual timeout. If timeoutInMs is undefined, we use the default of 120s
// Also make sure the timeout is at least 1s, but not bigger than 120s
const actualTimeoutInMs = Math.min(Math.max(timeout?.durationInMs ?? 120000, 1000), 120000);
const jsonBody = await safeJsonFromResponse(response);
const timeoutId = setTimeout(() => {
abortController.abort();
}, actualTimeoutInMs);
logger.debug("PerformTaskOperationService.call.fetch", {
url,
requestInit,
retry,
statusCode: response.status,
headers: Object.fromEntries(response.headers.entries()),
jsonBody,
durationInMs,
});
try {
logger.debug("PerformTaskOperationService.call fetch request", {
task,
actualTimeoutInMs,
url,
retry,
});
if (!response.ok) {
const retryAt = this.#calculateRetryForResponse(task, retry, response);
const response = await fetch(url, {
method: requestInit?.method ?? "GET",
headers: normalizeHeaders(requestInit?.headers ?? {}),
body: requestInit?.body,
signal: abortController.signal,
});
if (retryAt) {
return await this.#retryTaskWithError(
task,
`Fetch failed with status ${response.status}`,
retryAt
);
clearTimeout(timeoutId);
const durationInMs = Math.floor(performance.now() - startTimeInMs);
const jsonBody = await safeJsonFromResponse(response);
logger.debug("PerformTaskOperationService.call fetch response", {
url,
requestInit,
retry,
statusCode: response.status,
headers: Object.fromEntries(response.headers.entries()),
jsonBody,
durationInMs,
});
if (!response.ok) {
const retryAt = this.#calculateRetryForResponse(task, retry, response);
if (retryAt) {
return await this.#retryTaskWithError(
task,
`Fetch failed with status ${response.status}`,
retryAt
);
}
// See if there is a json body
if (jsonBody) {
return await this.#resumeTaskWithError(task, jsonBody);
} else {
return await this.#resumeTaskWithError(task, {
message: `Fetch failed with status ${response.status}`,
});
}
}
// See if there is a json body
if (jsonBody) {
return await this.#resumeTaskWithError(task, jsonBody);
} else {
return await this.#resumeTask(task, jsonBody, durationInMs);
} catch (error) {
if (error instanceof Error && error.name === "AbortError") {
const durationInMs = Math.floor(performance.now() - startTimeInMs);
logger.debug("PerformTaskOperationService.call fetch timed out", {
url,
durationInMs,
error,
});
const retryAt = this.#calculateRetryForTimeout(task, timeout?.retry);
if (retryAt) {
return await this.#retryTaskWithError(
task,
`Fetch timed out after ${actualTimeoutInMs.toFixed(0)}ms`,
retryAt
);
}
return await this.#resumeTaskWithError(task, {
message: `Fetch failed with status ${response.status}`,
message: `Fetch timed out after ${actualTimeoutInMs.toFixed(0)}ms`,
});
}
}
return await this.#resumeTask(task, jsonBody, durationInMs);
throw error;
}
}
default: {
await this.#resumeTaskWithError(task, {
@@ -142,6 +190,17 @@ export class PerformTaskOperationService {
}
}
#calculateRetryForTimeout(
task: NonNullable<FoundTask>,
retry: RetryOptions | undefined
): Date | undefined {
if (!retry) {
return;
}
return calculateRetryAt(retry, task.attempts.length - 1);
}
#getRetryStrategyForStatusCode(
statusCode: number,
retry: FetchRetryOptions
@@ -258,9 +317,7 @@ export class PerformTaskOperationService {
}
async #resumeRunExecution(task: NonNullable<FoundTask>, prisma: PrismaClientOrTransaction) {
await enqueueRunExecutionV2(task.run, prisma, {
skipRetrying: task.run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
});
await ResumeTaskService.enqueue(task.id, undefined, prisma);
}
}
apps/webapp/app/services/tasks/processCallbackTimeout.ts → apps/webapp/app/services/tasks/processCallbackTimeout.server.ts
+3 -6
View File
@@ -1,7 +1,6 @@
import { RuntimeEnvironmentType } from "@trigger.dev/database";
import { $transaction, PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server";
import { enqueueRunExecutionV2 } from "~/models/jobRunExecution.server";
import { logger } from "../logger.server";
import { ResumeTaskService } from "./resumeTask.server";
type FoundTask = Awaited<ReturnType<typeof findTask>>;
@@ -37,7 +36,7 @@ export class ProcessCallbackTimeoutService {
},
data: {
status: "ERRORED",
error
error,
},
});
@@ -55,9 +54,7 @@ export class ProcessCallbackTimeoutService {
}
async #resumeRunExecution(task: NonNullable<FoundTask>, prisma: PrismaClientOrTransaction) {
await enqueueRunExecutionV2(task.run, prisma, {
skipRetrying: task.run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
});
await ResumeTaskService.enqueue(task.id, undefined, prisma);
}
}
apps/webapp/app/services/tasks/resumeTask.server.ts
+107
View File
@@ -0,0 +1,107 @@
import { PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server";
import { workerQueue } from "../worker.server";
import { enqueueRunExecutionV3 } from "~/models/jobRunExecution.server";
import { RuntimeEnvironmentType } from "@trigger.dev/database";
import { logger } from "../logger.server";
type FoundTask = Awaited<ReturnType<typeof findTask>>;
export class ResumeTaskService {
#prismaClient: PrismaClient;
constructor(prismaClient: PrismaClient = prisma) {
this.#prismaClient = prismaClient;
}
public async call(id: string) {
const task = await findTask(this.#prismaClient, id);
if (!task) {
return;
}
if (task.status === "COMPLETED" || task.status === "ERRORED") {
return await this.#resumeRunExecution(task);
}
const updatedTask = await this.#prismaClient.task.update({
where: {
id: task.id,
},
data: {
status: task.noop ? "COMPLETED" : "RUNNING",
completedAt: task.noop ? new Date() : undefined,
},
include: {
attempts: true,
run: {
include: {
environment: true,
},
},
parent: true,
},
});
// This will retry the task if it's not a noop, or just resume
// the run execution if it is a noop.
return await this.#resumeRunExecution(updatedTask);
}
async #resumeRunExecution(task: NonNullable<FoundTask>) {
logger.debug("ResumeTaskService.call resuming run execution", {
parent: task.parent,
taskId: task.id,
});
if (task.parent && task.parent.childExecutionMode === "PARALLEL") {
const children = await this.#prismaClient.task.findMany({
where: {
parentId: task.parent.id,
},
select: {
id: true,
status: true,
},
});
const allChildrenCompleted = children.every(
(child) =>
child.status === "COMPLETED" || child.status === "ERRORED" || child.status === "CANCELED"
);
logger.debug("ResumeTaskService.call parent executing children in parallel", {
parentId: task.parent.id,
allChildrenCompleted,
children,
});
if (!allChildrenCompleted) {
return;
}
}
await enqueueRunExecutionV3(task.run, this.#prismaClient, {
skipRetrying: task.run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
});
}
public static async enqueue(id: string, runAt?: Date, tx?: PrismaClientOrTransaction) {
return await workerQueue.enqueue("resumeTask", { id }, { tx, jobKey: `resume:${id}`, runAt });
}
}
async function findTask(prisma: PrismaClient, id: string) {
return prisma.task.findUnique({
where: { id },
include: {
attempts: true,
run: {
include: {
environment: true,
},
},
parent: true,
},
});
}
apps/webapp/app/services/telemetry.server.ts
+15 -10
View File
@@ -5,6 +5,7 @@ import { env } from "~/env.server";
import type { Organization } from "~/models/organization.server";
import type { Project } from "~/models/project.server";
import type { User } from "~/models/user.server";
import { singleton } from "~/utils/singleton";
type Options = {
postHogApiKey?: string;
@@ -236,13 +237,17 @@ type CaptureEvent = {
userOnceProperties?: Record<string, any>;
};
export const telemetry = new Telemetry({
postHogApiKey: env.POSTHOG_PROJECT_KEY,
trigger:
env.TELEMETRY_TRIGGER_API_KEY && env.TELEMETRY_TRIGGER_API_URL
? {
apiKey: env.TELEMETRY_TRIGGER_API_KEY,
apiUrl: env.TELEMETRY_TRIGGER_API_URL,
}
: undefined,
});
export const telemetry = singleton(
"telemetry",
() =>
new Telemetry({
postHogApiKey: env.POSTHOG_PROJECT_KEY,
trigger:
env.TELEMETRY_TRIGGER_API_KEY && env.TELEMETRY_TRIGGER_API_URL
? {
apiKey: env.TELEMETRY_TRIGGER_API_KEY,
apiUrl: env.TELEMETRY_TRIGGER_API_URL,
}
: undefined,
})
);
apps/webapp/app/services/worker.server.ts
+59 -4
View File
@@ -13,15 +13,17 @@ import { InvokeDispatcherService } from "./events/invokeDispatcher.server";
import { integrationAuthRepository } from "./externalApis/integrationAuthRepository.server";
import { IntegrationConnectionCreatedService } from "./externalApis/integrationConnectionCreated.server";
import { MissingConnectionCreatedService } from "./runs/missingConnectionCreated.server";
import { PerformRunExecutionV2Service } from "./runs/performRunExecutionV2.server";
import { PerformRunExecutionV3Service } from "./runs/performRunExecutionV3.server";
import { StartRunService } from "./runs/startRun.server";
import { DeliverScheduledEventService } from "./schedules/deliverScheduledEvent.server";
import { ActivateSourceService } from "./sources/activateSource.server";
import { DeliverHttpSourceRequestService } from "./sources/deliverHttpSourceRequest.server";
import { PerformTaskOperationService } from "./tasks/performTaskOperation.server";
import { ProcessCallbackTimeoutService } from "./tasks/processCallbackTimeout";
import { ProcessCallbackTimeoutService } from "./tasks/processCallbackTimeout.server";
import { ProbeEndpointService } from "./endpoints/probeEndpoint.server";
import { PgNotifyService } from "./db/pgNotify.server";
import { DeliverRunSubscriptionService } from "./runs/deliverRunSubscription.server";
import { DeliverRunSubscriptionsService } from "./runs/deliverRunSubscriptions.server";
import { ResumeTaskService } from "./tasks/resumeTask.server";
const workerCatalog = {
indexEndpoint: z.object({
@@ -82,6 +84,15 @@ const workerCatalog = {
simulate: z.object({
seconds: z.number(),
}),
deliverRunSubscriptions: z.object({
id: z.string(),
}),
deliverRunSubscription: z.object({
id: z.string(),
}),
resumeTask: z.object({
id: z.string(),
}),
};
const executionWorkerCatalog = {
@@ -91,6 +102,10 @@ const executionWorkerCatalog = {
resumeTaskId: z.string().optional(),
isRetry: z.boolean(),
}),
performRunExecutionV3: z.object({
id: z.string(),
reason: z.enum(["EXECUTE_JOB", "PREPROCESS"]),
}),
};
let workerQueue: ZodWorker<typeof workerCatalog>;
@@ -336,6 +351,33 @@ function getWorkerQueue() {
await new Promise((resolve) => setTimeout(resolve, payload.seconds * 1000));
},
},
deliverRunSubscriptions: {
priority: 1, // smaller number = higher priority
maxAttempts: 5,
handler: async (payload, job) => {
const service = new DeliverRunSubscriptionsService();
await service.call(payload.id);
},
},
deliverRunSubscription: {
priority: 1, // smaller number = higher priority
maxAttempts: 13,
handler: async (payload, job) => {
const service = new DeliverRunSubscriptionService();
await service.call(payload.id);
},
},
resumeTask: {
priority: 0,
maxAttempts: 3,
handler: async (payload, job) => {
const service = new ResumeTaskService();
return await service.call(payload.id);
},
},
},
});
}
@@ -359,7 +401,7 @@ function getExecutionWorkerQueue() {
priority: 0, // smaller number = higher priority
maxAttempts: 12,
handler: async (payload, job) => {
const service = new PerformRunExecutionV2Service();
const service = new PerformRunExecutionV3Service();
await service.call({
id: payload.id,
@@ -369,6 +411,19 @@ function getExecutionWorkerQueue() {
});
},
},
performRunExecutionV3: {
priority: 0, // smaller number = higher priority
maxAttempts: 12,
handler: async (payload, job) => {
const service = new PerformRunExecutionV3Service();
await service.call({
id: payload.id,
reason: payload.reason,
isRetry: false,
});
},
},
},
});
}
apps/webapp/app/utils/singleton.ts
+6
View File
@@ -0,0 +1,6 @@
export function singleton<T>(name: string, getValue: () => T): T {
const thusly = globalThis as any;
thusly.__trigger_singletons ??= {};
thusly.__trigger_singletons[name] ??= getValue();
return thusly.__trigger_singletons[name];
}
apps/webapp/package.json
+4 -4
View File
@@ -68,6 +68,7 @@
"@trigger.dev/sdk": "workspace:*",
"@types/pg": "8.6.6",
"@uiw/react-codemirror": "^4.19.5",
"@whatwg-node/fetch": "^0.9.14",
"class-variance-authority": "^0.5.2",
"clsx": "^1.2.1",
"compression": "^1.7.4",
@@ -95,13 +96,12 @@
"prismjs": "^1.29.0",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-hot-toast": "^2.4.0",
"react-hotkeys-hook": "^4.4.1",
"react-use": "^17.4.0",
"recharts": "^2.8.0",
"remix-auth": "^3.2.2",
"remix-auth-email-link": "^1.4.2",
"remix-auth-github": "^1.1.1",
"remix-auth": "^3.6.0",
"remix-auth-email-link": "^2.1.0",
"remix-auth-github": "^1.6.0",
"remix-typedjson": "0.3.1",
"remix-utils": "^7.1.0",
"semver": "^7.5.0",
apps/webapp/server.ts
+1 -1
View File
@@ -51,7 +51,7 @@ app.all(
}
);
const port = process.env.REMIX_APP_PORT || 3000;
const port = process.env.REMIX_APP_PORT || process.env.PORT || 3000;
if (process.env.HTTP_SERVER_DISABLED !== "true") {
const server = app.listen(port, () => {
docs/_snippets/jobs/options.mdx
+1
View File
@@ -11,6 +11,7 @@
</ParamField>
<ParamField body="trigger" type="object" required>
The `trigger` property is used to define when the Job should run. There are currently the following Trigger types:
- [invokeTrigger](/sdk/invoke-trigger)
- [cronTrigger](/sdk/crontrigger)
- [intervalTrigger](/sdk/intervaltrigger)
- [eventTrigger](/sdk/eventtrigger)
docs/documentation/concepts/limits.mdx
+1
View File
@@ -14,6 +14,7 @@ The following limits apply to the Trigger.dev Cloud service and users of the sel
| Runs (per Month) | 5,000 | Up to 1m | Custom |
| Run Log retention | 24 hours | 7 days | Custom |
| Connected Integrations | Up to 50 | Up to 1000 | Custom |
| Task Output Size | 3MB | 3MB | 3MB |
| [Tasks per Run](#tasks-per-runs) | Up to 250 | Up to 1000 | Custom |
| [Concurrent Run Executions](#concurrent-run-executions) | Up to 10 | Up to 10 | Custom |
| [Maximum Task Duration](#maximum-task-duration) | < 2m | < 2m | < Deployment Grace Period |
docs/documentation/concepts/triggers/introduction.mdx
+10 -7
View File
@@ -1,21 +1,24 @@
---
title: "Triggers: Introduction"
sidebarTitle: "Introduction"
description: "A Trigger is what starts a Job Run. It can be a webhook, a schedule, or an event."
---
We currently support three types of Triggers: Webhooks, Scheduled, and Events. You can use any of these to start a Job Run.
We currently support four types of Triggers: Manual Invoke, Events, Scheduled, and Webhooks. You can use any of these to start a Job Run.
<CardGroup>
<Card title="Webhooks" icon="webhook" href="/documentation/concepts/triggers/webhooks">
Start your Jobs in realtime when events happen in APIs
</Card>
<Card title="Scheduled" icon="calendar" href="/documentation/concepts/triggers/scheduled">
Run a Job on a repeating schedule
<Card title="Invoke" icon="bolt" href="/documentation/concepts/triggers/invoke">
Run your Job manually by using `Job.invoke()`
</Card>
<Card title="Event" icon="brackets-curly" href="/documentation/concepts/triggers/events">
Run your Job when you send events with data
</Card>
<Card title="Scheduled" icon="calendar" href="/documentation/concepts/triggers/scheduled">
Run a Job on a repeating schedule
</Card>
<Card title="Webhooks" icon="webhook" href="/documentation/concepts/triggers/webhooks">
Start your Jobs in realtime when events happen in APIs
</Card>
<Card
title="DynamicTrigger & DynamicSchedule"
icon="code"
docs/documentation/concepts/triggers/invoke.mdx
+382
View File
@@ -0,0 +1,382 @@
---
title: "Invoke triggers"
sidebarTitle: "Manual Invoke"
description: "Invoke Jobs manually using the invoke Trigger"
---
<Warning>This feature is in beta and not yet available to use on the Trigger.dev Cloud.</Warning>
Sometimes it makes sense to be able to invoke a Job manually, without having to specify an event, especially for cases where you want to get notified when the invoked Job Run is complete.
To specify that a job is manually invokable, you can use the `invokeTrigger()` function when defining a job:
```ts exampleJob.ts
import { invokeTrigger } from "@trigger.dev/sdk";
import { client } from "@/trigger";
export const exampleJob = client.defineJob({
id: "example-job",
name: "Example job",
version: "1.0.1",
trigger: invokeTrigger(),
run: async (payload, io, ctx) => {
// do something with the payload
},
});
```
And then you can invoke the job using the `Job.invoke()` method:
```ts example.ts
import { exampleJob } from "./exampleJob";
const jobRun = await exampleJob.invoke({ foo: "bar" });
```
## Payload Schema
You can specify the type of the expected payload by passing a [Zod schema](https://zod.dev) to `invokeTrigger()`:
```ts exampleJob.ts
import { invokeTrigger } from "@trigger.dev/sdk";
import { client } from "@/trigger";
export const exampleJob = client.defineJob({
id: "example-job",
name: "Example job",
version: "1.0.1",
trigger: invokeTrigger({
//the expected payload shape
schema: z.object({
userId: z.string(),
tier: z.enum(["free", "pro"]),
}),
}),
run: async (payload, io, ctx) => {
// payload is typed as { userId: string, tier: "free" | "pro" }
},
});
```
Now when you invoke the job, you will get a type error if the payload does not match the schema:
```ts example.ts
import { exampleJob } from "./exampleJob";
// this will throw a type error because the payload does not match the schema
const jobRun = await exampleJob.invoke({ foo: "bar" });
// this will work
const jobRun = await exampleJob.invoke({ userId: "123", tier: "free" });
```
## Invoking a Job
As you can see in the example above, invoking a job is as simple as calling the `Job.invoke()` method. This method returns a `JobRun` object that you can use to track the progress of the job run, especially in conjunction with our [React hooks](/documentation/guides/react-hooks), like [useRunDetails()](/sdk/react/userundetails)
```ts
import { exampleJob } from "./exampleJob";
// Somewhere in your backend code
const jobRun = await exampleJob.invoke({ userId: "123", tier: "free" });
// Somewhere in your frontend code
const { data: jobRunDetails } = useRunDetails(jobRun.id);
```
<Note>
There are many variations for how you could get the job run information passed from your backend
to your frontend, depending on the framework you are using and which rendering model. We'll leave
that as an exercise for the reader
</Note>
### Deduplicate Invocations
You can pass an optional `idempotencyKey` to the `invoke()` method to deduplicate invocations. This is useful when you want to make sure that a job is only invoked once for a given payload.
```ts
import { exampleJob } from "./exampleJob";
// Somewhere in your backend code
const jobRun = await exampleJob.invoke(
{ userId: "123", tier: "free" },
{ idempotencyKey: "abc123" }
);
// This will not invoke the job again, but return the existing job run
const jobRun2 = await exampleJob.invoke(
{ userId: "123", tier: "free" },
{ idempotencyKey: "abc123" }
);
```
### Callback URL
You can also pass an optional `callbackUrl` to the `invoke()` method to get notified when the job run is complete, either successfully or with an error.
```ts
import { exampleJob } from "./exampleJob";
// Somewhere in your backend code
const jobRun = await exampleJob.invoke(
{ userId: "123", tier: "free" },
{ callbackUrl: `${process.env.VERCEL_URL}/api/trigger/runs` }
);
```
When the run is complete, we will issue a `POST` request to the URL with the [RunNotification payload](/sdk/run-notification).
#### Verifying the callback
You should make sure to verify the payload in your callback route to make sure that the request is coming from Trigger. You can do this by checking the `X-Trigger-Signature-256` header, which contains a HMAC signature of the payload using your [secret API Key](/documentation/concepts/environments-endpoints).
```ts
import { crypto } from "node:crypto"
app.post("/api/trigger/runs", (req, res) => {
// Create digest with payload + hmac secret
const hashPayload = req.rawBody;
const hmac = crypto.createHmac("sha256", process.env.TRIGGER_API_KEY); /
const digest = Buffer.from(
signatureAlgorithm + "=" + hmac.update(hashPayload).digest("hex"),
"utf8"
);
// Get hash sent by the provider
const providerSig = Buffer.from(req.get("X-Trigger-Signature-256") || "", "utf8");
// Compare digest signature with signature sent by provider
if (providerSig.length !== digest.length || !crypto.timingSafeEqual(digest, providerSig)) {
res.status(401).send("Unauthorized");
} else {
// Webhook Authenticated
// process and respond...
res.json({ message: "Success" });
}
});
```
### Additional context
You can pass an optional `context` object to the `invoke()` method, which will be available in the job run context. This is useful for passing additional information to the job run that doesn't make sense in the payload.
```ts
import { exampleJob } from "./exampleJob";
// Somewhere in your backend code
const jobRun = await exampleJob.invoke(
{ userId: "123", tier: "free" },
{
context: {
traceId: "trace_123",
correlationId: "def456",
},
}
);
```
And then you can access the context in the job run:
```ts
export const exampleJob = client.defineJob({
id: "example-job",
name: "Example job",
version: "1.0.1",
trigger: invokeTrigger({
//the expected payload shape
schema: z.object({
userId: z.string(),
tier: z.enum(["free", "pro"]),
}),
}),
run: async (payload, io, ctx) => {
console.log(ctx.event.context); // { traceId: "trace_123", correlationId: "def456" }
},
});
```
## Invoking a job from a job
You can also invoke a job from another job:
```ts
import { exampleJob } from "./exampleJob";
client.defineJob({
id: "example-job2",
name: "Example job 2",
version: "1.0.1",
trigger: intervalTrigger({
seconds: 60,
}),
run: async (payload, io, ctx) => {
const jobRun = await exampleJob.invoke("⚡", { userId: "123", tier: "free" });
},
});
```
Notice how the `invoke()` method takes a string as the first argument. This is because under the hood `invoke()` is automatically creating a [Task](/documentation/concepts/tasks) and the `"⚡"` string is the `cacheKey` for the created task. You can easily see the run created via the Run Dashboard:
![Task](/images/invoke-task.png)
<Warning>
If `Job.invoke()` is called within another job, and you **don't** include the `cacheKey` we'll
throw an error.
</Warning>
### Wait for completion
You can also invoke a job and wait for it to complete before continuing execution of the current job:
```ts
import { exampleJob } from "./exampleJob";
client.defineJob({
id: "example-job2",
name: "Example job 2",
version: "1.0.1",
trigger: intervalTrigger({
seconds: 60,
}),
run: async (payload, io, ctx) => {
const jobRun = await exampleJob.invokeAndWaitForCompletion("⚡", {
userId: "123",
tier: "free",
});
if (jobRun.ok) {
// The job run finished successfully
console.log(jobRun.output);
} else {
// The job run finished with an error
console.log(`The job failed with status ${jobRun.status}`, jobRun.error);
}
},
});
```
By default, `invokeAndWaitForCompletion()` will wait for the job run to complete for up to 60 minutes. You can change this by passing a `timeoutInSeconds` option:
```ts
import { exampleJob } from "./exampleJob";
client.defineJob({
id: "example-job2",
name: "Example job 2",
version: "1.0.1",
trigger: intervalTrigger({
seconds: 60,
}),
run: async (payload, io, ctx) => {
const jobRun = await exampleJob.invokeAndWaitForCompletion(
"⚡",
{
userId: "123",
tier: "free",
},
86_400 // 24 hours
);
if (jobRun.ok) {
// The job run finished successfully
console.log(jobRun.output);
} else {
// The job run finished with an error
console.log(`The job failed with status ${jobRun.status}`, jobRun.error);
}
},
});
```
<Note>
If the invoked job run doesn't complete in the given time, the underlying task will fail, along
with the run
</Note>
The return value of `invokeAndWaitForCompletion` is a [RunNotification](/sdk/run-notification) object.
### Batch invoke and wait for completion
You can also batch invoke jobs and wait for them all to complete before continuing execution of the current job:
```ts
import { exampleJob } from "./exampleJob";
client.defineJob({
id: "example-job2",
name: "Example job 2",
version: "1.0.1",
trigger: intervalTrigger({
seconds: 60,
}),
run: async (payload, io, ctx) => {
const runs = await exampleJob.batchInvokeAndWaitForCompletion("⚡", [
{
payload: {
userId: "123",
tier: "free",
},
timeoutInSeconds: 86_400, // 24 hours
},
{
payload: {
userId: "abc",
tier: "paid",
},
timeoutInSeconds: 86_400, // 24 hours
},
]);
// runs is an array of RunNotification objects
},
});
```
You can batch up to 25 invocations at once, and we will run them in parallel and wait for all of them to complete before continuing execution of the current job.
You can in an optional `options` object to each invocation, if you want to pass `context` and `accountId` to each invocation:
```ts
import { exampleJob } from "./exampleJob";
client.defineJob({
id: "example-job2",
name: "Example job 2",
version: "1.0.1",
trigger: intervalTrigger({
seconds: 60,
}),
run: async (payload, io, ctx) => {
const runs = await exampleJob.batchInvokeAndWaitForCompletion("⚡", [
{
payload: {
userId: "123",
tier: "free",
},
timeoutInSeconds: 86_400, // 24 hours
options: {
context: {
traceId: "trace_123",
correlationId: "def456",
},
accountId: "abc123",
},
},
{
payload: {
userId: "abc",
tier: "paid",
},
timeoutInSeconds: 86_400, // 24 hours
options: {
context: {
traceId: "trace_abc",
correlationId: "defabc",
},
accountId: "abc123",
},
},
]);
// runs is an array of RunNotification objects
},
});
```
docs/images/invoke-task.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 44 KiB

docs/integrations/apis/openai-tasks.mdx
+132
View File
@@ -43,6 +43,57 @@ run: async (payload, io, ctx) => {
},
```
You can also pass an optional third parameter to `backgroundCreateCompletion` to specify OpenAI request options:
```ts requestOptions.ts
run: async (payload, io, ctx) => {
// This code showcases background text completion using the "gpt-3.5-turbo" model.
// It generates text based on the provided programming task and logs the result.
const programmingTask = `Create a function that checks if a string is a palindrome.`;
const response = await io.openai.backgroundCreateCompletion("background-completion", {
model: "gpt-3.5-turbo",
prompt: `Coding task: ${programmingTask}\n\n`,
}, {
headers: {
"User-Agent": "my-user-agent"
}
});
await io.logger.info("codeSnippet", response.choices[0]?.text);
},
```
This task is implemented using [io.backgroundFetch()](/sdk/io/backgroundfetch) and so you can also pass a 4th parameter customizing the retry and timeout options:
```ts fetchOptions.ts
run: async (payload, io, ctx) => {
// This code showcases background text completion using the "gpt-3.5-turbo" model.
// It generates text based on the provided programming task and logs the result.
const programmingTask = `Create a function that checks if a string is a palindrome.`;
const response = await io.openai.backgroundCreateCompletion("background-completion", {
model: "gpt-3.5-turbo",
prompt: `Coding task: ${programmingTask}\n\n`,
}, {
headers: {
"User-Agent": "my-user-agent"
}
}, {
timeout: {
durationInMs: 10000,
retry: {
limit: 3,
minTimeoutInMs: 1000,
factor: 2,
}
}
});
await io.logger.info("codeSnippet", response.choices[0]?.text);
},
```
### `createChatCompletion`
Generates text completions in a conversational context. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/chat/create)
@@ -85,6 +136,87 @@ run: async (payload, io, ctx) => {
},
```
You can also use the more "fluent" pattern used by the OpenAI SDK:
```ts fluent.ts
run: async (payload, io, ctx) => {
// This code showcases background chat completion using the "gpt-3.5-turbo" model.
// It simulates a conversation with a user message and logs the response choices.
const response = await io.openai.chat.completions.backgroundCreate("background-chat-completion", {
model: "gpt-3.5-turbo",
messages: [
{
role: "user",
content: "Create a good programming joke about background jobs",
},
],
});
await io.logger.info("choices", response.choices);
},
```
Additionally, you can pass an optional third parameter to specify OpenAI request options:
```ts requestOptions.ts
run: async (payload, io, ctx) => {
// This code showcases background text completion using the "gpt-3.5-turbo" model.
// It generates text based on the provided programming task and logs the result.
const programmingTask = `Create a function that checks if a string is a palindrome.`;
const response = await io.openai.chat.completions.backgroundCreate("background-completion", {
model: "gpt-3.5-turbo",
messages: [
{
role: "user",
content: "Create a good programming joke about background jobs",
},
],
}, {
headers: {
"User-Agent": "my-user-agent"
}
});
await io.logger.info("choices", response.choices);
},
```
This task is implemented using [io.backgroundFetch()](/sdk/io/backgroundfetch) and so you can also pass a 4th parameter customizing the retry and timeout options:
```ts fetchOptions.ts
run: async (payload, io, ctx) => {
// This code showcases background text completion using the "gpt-3.5-turbo" model.
// It generates text based on the provided programming task and logs the result.
const programmingTask = `Create a function that checks if a string is a palindrome.`;
const response = await io.openai.chat.completions.backgroundCreate("background-completion", {
model: "gpt-3.5-turbo",
messages: [
{
role: "user",
content: "Create a good programming joke about background jobs",
},
],
}, {
headers: {
"User-Agent": "my-user-agent"
}
}, {
timeout: {
durationInMs: 10000,
retry: {
limit: 3,
minTimeoutInMs: 1000,
factor: 2,
}
}
});
await io.logger.info("choices", response.choices);
},
```
### `retrieveModel`
Retrieves a specific model by ID. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/models/retrieve)
docs/mint.json
+4 -1
View File
@@ -117,9 +117,10 @@
"group": "Triggers",
"pages": [
"documentation/concepts/triggers/introduction",
"documentation/concepts/triggers/webhooks",
"documentation/concepts/triggers/invoke",
"documentation/concepts/triggers/events",
"documentation/concepts/triggers/scheduled",
"documentation/concepts/triggers/webhooks",
"documentation/concepts/triggers/dynamic"
]
},
@@ -343,6 +344,8 @@
]
},
"sdk/context",
"sdk/run-notification",
"sdk/invoke-trigger",
"sdk/eventtrigger",
"sdk/crontrigger",
"sdk/intervaltrigger",
docs/sdk/context.mdx
+18 -4
View File
@@ -38,8 +38,7 @@ description: "The third parameter in a Job's `run()` function. An object that co
The Environment's ID
</ResponseField>
<ResponseField name="type" type="string" required>
The Environment's type. One of: "PRODUCTION", "STAGING", "DEVELOPMENT",
"PREVIEW".
The Environment's type. One of: "PRODUCTION", "STAGING", "DEVELOPMENT", "PREVIEW".
</ResponseField>
</Expandable>
</ResponseField>
@@ -74,6 +73,21 @@ description: "The third parameter in a Job's `run()` function. An object that co
</Expandable>
</ResponseField>
<ResponseField name="project" type="object" required>
Metadata about the Project
<Expandable title="properties">
<ResponseField name="slug" type="string" required>
The Project's slug
</ResponseField>
<ResponseField name="id" type="string" required>
The Project's ID
</ResponseField>
<ResponseField name="name" type="string" required>
The Project's title
</ResponseField>
</Expandable>
</ResponseField>
<ResponseField name="event" type="object" required>
Metadata about the Event that triggered the Run.
<Expandable title="properties">
@@ -105,8 +119,8 @@ description: "The third parameter in a Job's `run()` function. An object that co
</ResponseField>
<ResponseField name="account" type="object">
Metadata about the Account that triggered the Run. This is a Trigger.dev
Connect property which is coming soon.
Metadata about the Account that triggered the Run. See the [BYO Auth
docs](/documentation/guides/using-integrations-byo-auth) for more information.
<Expandable title="properties">
<ResponseField name="id" type="string" required>
The Account's ID
docs/sdk/invoke-trigger.mdx
+86
View File
@@ -0,0 +1,86 @@
---
title: "invokeTrigger()"
description: "Use `invokeTrigger()` to allow a Job to be manually triggered using `Job.invoke()`"
---
<Warning>This feature is in beta and not yet available to use on the Trigger.dev Cloud.</Warning>
Setting `invokeTrigger()` on a job allows you to manually trigger the job using `Job.invoke()`, either from your backend or from inside another job. See our [Invoke Trigger guide](/documentation/guides/invoke) for more info.
## Parameters
<ResponseField name="options" type="object" required>
<Expandable title="options" defaultOpen>
<ResponseField name="schema" type="object">
A [Zod](/documentation/guides/zod) schema that defines the shape of the
job payload. The default is `z.any()` which is `any`. This will be used to correctly type the `Job.invoke()` method.
</ResponseField>
<ResponseField name="examples" type="array">
Used to provide example payloads that are accepted by the job.
This will be available in the dashboard and can be used to trigger test runs.
<Expandable title="example object properties" defaultOpen>
<ResponseField name="id" type="string" required>
The example's ID.
</ResponseField>
<ResponseField name="name" type="string" required>
The name that's displayed in the dashboard.
</ResponseField>
<ResponseField name="payload" type="object" required>
The payload that's accepted by the job.
</ResponseField>
<ResponseField name="icon" type="string">
The icon to use for this example in the dashboard.
</ResponseField>
</Expandable>
</ResponseField>
</Expandable>
</ResponseField>
<RequestExample>
```typescript invokeTrigger.ts
//this Job subscribes to an event called new.user
export const exampleJob = client.defineJob({
id: "example-job",
name: "Example job",
version: "1.0.1",
trigger: invokeTrigger({
//the expected payload shape
schema: z.object({
userId: z.string(),
tier: z.union([z.literal("free"), z.literal("pro")]),
}),
//(optional) example payload object
examples: [
{
id: "issue.opened",
name: "Issue opened",
payload: {
userId: "1234",
tier: "free",
},
//optional
icon: "github",
},
],
}),
run: async (payload, io, ctx) => {
// do something with the payload
},
});
```
```typescript usage.ts
import { exampleJob } from "./invokeTrigger";
//invoke the job with the payload
await exampleJob.invoke({
userId: "1234",
tier: "free",
});
```
</RequestExample>
docs/sdk/io/backgroundfetch.mdx
+84 -2
View File
@@ -98,6 +98,44 @@ An individual retrying strategy can be one of two types:
</ResponseField>
<ResponseField name="timeout options" type="object">
Allows you to set timeouts for the request, as well as specific retry strategies for timeouts.
{" "}
<Expandable title="options" defaultOpen>
<ResponseField name="durationInMs" type="number" required>
The amount of time to wait before timing out the request. The minimum value is 1s and max is 2min
</ResponseField>
<ResponseField name="retry" type="RetryOptions">
<Expandable title="options">
{" "}
<ResponseField name="limit" type="number">
The maximum number of times to retry the request.
</ResponseField>
<ResponseField name="minTimeoutInMs" type="number">
The minimum amount of time to wait before retrying the request.
</ResponseField>
<ResponseField name="maxTimeoutInMs" type="number">
The maximum amount of time to wait before retrying the request.
</ResponseField>
<ResponseField name="factor" type="number">
The exponential factor to use when calculating the next retry time.
</ResponseField>
<ResponseField name="randomize" type="boolean">
Whether to randomize the retry time.
</ResponseField>
</Expandable>
</ResponseField>
</Expandable>
</ResponseField>
</ResponseField>
## Returns
@@ -106,7 +144,7 @@ A `Promise` that resolves after the specified amount of time.
<RequestExample>
```typescript backgroundFetch example
```typescript retrying
client.defineJob({
id: "background-fetch-job",
name: "Background fetch Job",
@@ -128,8 +166,52 @@ client.defineJob({
},
{
"429": {
strategy: "headers",
limitHeader: "x-ratelimit-limit",
remainingHeader: "x-ratelimit-remaining",
resetHeader: "x-ratelimit-reset",
},
"5xx": {
strategy: "backoff",
limit: 10,
limit: 5,
minTimeoutInMs: 1000,
maxTimeoutInMs: 30000,
factor: 1.8,
randomize: true,
},
}
);
return response;
},
});
```
```typescript timeouts
client.defineJob({
id: "background-fetch-job",
name: "Background fetch Job",
version: "0.0.1",
trigger: eventTrigger({
name: "example.event",
}),
run: async (payload, io, ctx) => {
const response = io.backgroundFetch<CreateChatCompetionResponseData>(
"fetch-some-data",
"https://example.com/api/endpoint",
{
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: redactString`Bearer ${auth.apiKey}`,
},
body: JSON.stringify({ foo: "bar" }),
},
{},
{
durationInMs: 10000, // 10 seconds
retry: {
limit: 6,
minTimeoutInMs: 1000,
maxTimeoutInMs: 60000,
factor: 2,
docs/sdk/job.mdx
+1 -24
View File
@@ -5,33 +5,10 @@ description: "Job is used to create and configure a Job"
A [Job](/documentation/concepts/jobs) is used to define the [Trigger](/documentation/concepts/triggers), metadata, and what happens when it runs.
You can define a job in one of two ways, using the `new Job` constructor or by using the `TriggerClient.defineJob` instance method.
You can define a job by using the `TriggerClient.defineJob` instance method:
<RequestExample>
```ts constructor
new Job(client, {
id: "slack-kpi-summary",
name: "Slack kpi summary",
version: "0.1.1",
integrations: {
slack,
},
trigger: cronTrigger({
cron: "0 9 * * *", // 9am every day (UTC)
}),
run: async (payload, io, ctx) => {
const { revenue } = await db.getKpiSummary(payload.ts);
const response = await io.slack.postMessage("Slack 📝", {
text: `Yesterday's revenue was $${revenue}`,
channel: "C04GWUTDC3W",
});
return response;
},
});
```
```ts client.defineJob
client.defineJob({
id: "github-integration-on-issue",
docs/sdk/run-notification.mdx
+118
View File
@@ -0,0 +1,118 @@
---
title: "RunNotification"
sidebarTitle: "RunNotification"
description: "The payload of a Run's completion webhook"
---
This document describes the payload of a Run's completion webhook.
## Properties
<ResponseField name="id" type="string" required>
The Run's ID
</ResponseField>
<ResponseField name="ok" type="boolean" required>
Whether the Run completed successfully or failed.
</ResponseField>
<ResponseField name="status" type="JobRunStatus" required>
The Run's status. Will always be `"SUCCESS"` if `ok` is `true`. If `ok` is `false`, will be one of
`"FAILURE" | "TIMED_OUT" | "ABORTED" | "CANCELED" | "UNRESOLVED_AUTH" | "INVALID_PAYLOAD"`
</ResponseField>
<ResponseField name="statuses" type="JobRunStatusRecord[]" required>
An array of the explicit run statuses that were created during the Run. See [Explicit Status
Hooks](/documentation/guides/react-hooks-statuses#useeventrunstatuses) for more.
</ResponseField>
<ResponseField name="output" type="any">
If the Run completed successfully, this will be the output of the Job.
</ResponseField>
<ResponseField name="error" type="any">
If the Run failed, this will be the error that caused the failure.
</ResponseField>
<ResponseField name="startedAt" type="Date" required>
When the Run started
</ResponseField>
<ResponseField name="completedAt" type="Date" required>
When the Run started
</ResponseField>
<ResponseField name="executionDurationInMs" type="number" required>
The duration of the Run in milliseconds
</ResponseField>
<ResponseField name="executionCount" type="number" required>
The number of individual function executions performed to complete the Run
</ResponseField>
<ResponseField name="job" type="object" required>
Metadata about the Job
<Expandable title="properties">
<ResponseField name="id" type="string" required>
The Job's ID
</ResponseField>
<ResponseField name="version" type="string" required>
The Job's version
</ResponseField>
</Expandable>
</ResponseField>
<ResponseField name="environment" type="object" required>
Metadata about the Environment
<Expandable title="properties">
<ResponseField name="slug" type="string" required>
The Environment's slug
</ResponseField>
<ResponseField name="id" type="string" required>
The Environment's ID
</ResponseField>
<ResponseField name="type" type="string" required>
The Environment's type. One of: "PRODUCTION", "STAGING", "DEVELOPMENT", "PREVIEW".
</ResponseField>
</Expandable>
</ResponseField>
<ResponseField name="organization" type="object" required>
Metadata about the Organization
<Expandable title="properties">
<ResponseField name="slug" type="string" required>
The Organization's slug
</ResponseField>
<ResponseField name="id" type="string" required>
The Organization's ID
</ResponseField>
<ResponseField name="title" type="string" required>
The Organization's title
</ResponseField>
</Expandable>
</ResponseField>
<ResponseField name="project" type="object" required>
Metadata about the Project
<Expandable title="properties">
<ResponseField name="slug" type="string" required>
The Project's slug
</ResponseField>
<ResponseField name="id" type="string" required>
The Project's ID
</ResponseField>
<ResponseField name="name" type="string" required>
The Project's title
</ResponseField>
</Expandable>
</ResponseField>
<ResponseField name="account" type="object">
Metadata about the Account that triggered the Run. See the [BYO Auth
docs](/documentation/guides/using-integrations-byo-auth) for more information.
<Expandable title="properties">
<ResponseField name="id" type="string" required>
The Account's ID
</ResponseField>
<ResponseField name="metadata" type="any">
The Account's metadata. This is additional data that been passed through
</ResponseField>
</Expandable>
</ResponseField>
integrations/openai/src/chat.ts
+8 -2
View File
@@ -8,6 +8,7 @@ import {
createTaskUsageProperties,
} from "./taskUtils";
import { OpenAIIntegrationOptions, OpenAIRequestOptions } from "./types";
import { FetchRetryOptions, FetchTimeoutOptions } from "@trigger.dev/integration-kit";
export class Chat {
constructor(
@@ -47,7 +48,8 @@ export class Chat {
backgroundCreate: (
key: IntegrationTaskKey,
params: Prettify<OpenAI.Chat.ChatCompletionCreateParamsNonStreaming>,
options: OpenAIRequestOptions = {}
options: OpenAIRequestOptions = {},
fetchOptions: { retries?: FetchRetryOptions; timeout?: FetchTimeoutOptions } = {}
): Promise<OpenAI.Chat.ChatCompletion> => {
return this.runTask(
key,
@@ -72,7 +74,8 @@ export class Chat {
),
body: JSON.stringify(params),
},
backgroundTaskRetries
fetchOptions.retries ?? backgroundTaskRetries,
fetchOptions.timeout
);
task.outputProperties = createTaskUsageProperties(response.usage);
@@ -88,6 +91,9 @@ export class Chat {
text: params.model,
},
],
retry: {
limit: 0,
},
}
);
},
integrations/openai/src/completions.ts
+8 -2
View File
@@ -8,6 +8,7 @@ import {
createTaskUsageProperties,
} from "./taskUtils";
import { OpenAIIntegrationOptions, OpenAIRequestOptions } from "./types";
import { FetchRetryOptions, FetchTimeoutOptions } from "@trigger.dev/integration-kit";
export class Completions {
constructor(
@@ -46,7 +47,8 @@ export class Completions {
backgroundCreate(
key: IntegrationTaskKey,
params: Prettify<OpenAI.CompletionCreateParamsNonStreaming>,
options: OpenAIRequestOptions = {}
options: OpenAIRequestOptions = {},
fetchOptions: { retries?: FetchRetryOptions; timeout?: FetchTimeoutOptions } = {}
): Promise<OpenAI.Completion> {
return this.runTask(
key,
@@ -71,7 +73,8 @@ export class Completions {
),
body: JSON.stringify(params),
},
backgroundTaskRetries
fetchOptions.retries ?? backgroundTaskRetries,
fetchOptions.timeout
);
task.outputProperties = createTaskUsageProperties(response.usage);
@@ -87,6 +90,9 @@ export class Completions {
text: params.model,
},
],
retry: {
limit: 0,
},
}
);
}
packages/core/src/index.ts
+1
View File
@@ -9,6 +9,7 @@ export * from "./eventFilterMatches";
export const API_VERSIONS = {
LAZY_LOADED_CACHED_TASKS: "2023-09-29",
SERIALIZED_TASK_OUTPUT: "2023-11-01",
} as const;
export const PLATFORM_FEATURES = {
packages/core/src/logger.ts
+7 -5
View File
@@ -43,36 +43,37 @@ export class Logger {
log(message: string, ...args: Array<Record<string, unknown> | undefined>) {
if (this.#level < 0) return;
this.#structuredLog(console.log, message, ...args);
this.#structuredLog(console.log, message, "log", ...args);
}
error(message: string, ...args: Array<Record<string, unknown> | undefined>) {
if (this.#level < 1) return;
this.#structuredLog(console.error, message, ...args);
this.#structuredLog(console.error, message, "error", ...args);
}
warn(message: string, ...args: Array<Record<string, unknown> | undefined>) {
if (this.#level < 2) return;
this.#structuredLog(console.warn, message, ...args);
this.#structuredLog(console.warn, message, "warn", ...args);
}
info(message: string, ...args: Array<Record<string, unknown> | undefined>) {
if (this.#level < 3) return;
this.#structuredLog(console.info, message, ...args);
this.#structuredLog(console.info, message, "info", ...args);
}
debug(message: string, ...args: Array<Record<string, unknown> | undefined>) {
if (this.#level < 4) return;
this.#structuredLog(console.debug, message, ...args);
this.#structuredLog(console.debug, message, "debug", ...args);
}
#structuredLog(
loggerFunction: (message: string, ...args: any[]) => void,
message: string,
level: string,
...args: Array<Record<string, unknown> | undefined>
) {
const structuredLog = {
@@ -80,6 +81,7 @@ export class Logger {
timestamp: new Date(),
name: this.#name,
message,
level,
};
loggerFunction(JSON.stringify(structuredLog, createReplacer(this.#jsonReplacer)));
packages/core/src/schemas/api.ts
+79 -10
View File
@@ -485,6 +485,13 @@ export const RunJobBodySchema = z.object({
title: z.string(),
slug: z.string(),
}),
project: z
.object({
id: z.string(),
name: z.string(),
slug: z.string(),
})
.optional(),
account: z
.object({
id: z.string(),
@@ -518,27 +525,29 @@ export const RunJobYieldExecutionErrorSchema = z.object({
export type RunJobYieldExecutionError = z.infer<typeof RunJobYieldExecutionErrorSchema>;
export const RunJobAutoYieldExecutionErrorSchema = z.object({
status: z.literal("AUTO_YIELD_EXECUTION"),
export const AutoYieldMetadataSchema = z.object({
location: z.string(),
timeRemaining: z.number(),
timeElapsed: z.number(),
limit: z.number().optional(),
});
export type RunJobAutoYieldExecutionError = z.infer<typeof RunJobAutoYieldExecutionErrorSchema>;
export type AutoYieldMetadata = z.infer<typeof AutoYieldMetadataSchema>;
export const RunJobAutoYieldExecutionErrorSchema = AutoYieldMetadataSchema.extend({
status: z.literal("AUTO_YIELD_EXECUTION"),
});
export type RunJobAutoYieldExecutionError = Prettify<
z.infer<typeof RunJobAutoYieldExecutionErrorSchema>
>;
export const RunJobAutoYieldWithCompletedTaskExecutionErrorSchema = z.object({
status: z.literal("AUTO_YIELD_EXECUTION_WITH_COMPLETED_TASK"),
id: z.string(),
properties: z.array(DisplayPropertySchema).optional(),
output: z.any(),
data: z.object({
location: z.string(),
timeRemaining: z.number(),
timeElapsed: z.number(),
limit: z.number().optional(),
}),
output: z.string().optional(),
data: AutoYieldMetadataSchema,
});
export type RunJobAutoYieldWithCompletedTaskExecutionError = z.infer<
@@ -589,6 +598,28 @@ export const RunJobSuccessSchema = z.object({
export type RunJobSuccess = z.infer<typeof RunJobSuccessSchema>;
export const RunJobErrorResponseSchema = z.union([
RunJobAutoYieldExecutionErrorSchema,
RunJobAutoYieldWithCompletedTaskExecutionErrorSchema,
RunJobYieldExecutionErrorSchema,
RunJobErrorSchema,
RunJobUnresolvedAuthErrorSchema,
RunJobInvalidPayloadErrorSchema,
RunJobResumeWithTaskSchema,
RunJobRetryWithTaskSchema,
RunJobCanceledWithTaskSchema,
]);
export type RunJobErrorResponse = z.infer<typeof RunJobErrorResponseSchema>;
export const RunJobResumeWithParallelTaskSchema = z.object({
status: z.literal("RESUME_WITH_PARALLEL_TASK"),
task: TaskSchema,
childErrors: z.array(RunJobErrorResponseSchema),
});
export type RunJobResumeWithParallelTask = z.infer<typeof RunJobResumeWithParallelTaskSchema>;
export const RunJobResponseSchema = z.discriminatedUnion("status", [
RunJobAutoYieldExecutionErrorSchema,
RunJobAutoYieldWithCompletedTaskExecutionErrorSchema,
@@ -597,6 +628,7 @@ export const RunJobResponseSchema = z.discriminatedUnion("status", [
RunJobUnresolvedAuthErrorSchema,
RunJobInvalidPayloadErrorSchema,
RunJobResumeWithTaskSchema,
RunJobResumeWithParallelTaskSchema,
RunJobRetryWithTaskSchema,
RunJobCanceledWithTaskSchema,
RunJobSuccessSchema,
@@ -737,6 +769,7 @@ export const RunTaskOptionsSchema = z.object({
noop: z.boolean().default(false),
redact: RedactSchema.optional(),
trigger: TriggerMetadataSchema.optional(),
parallel: z.boolean().optional(),
});
export type RunTaskOptions = z.input<typeof RunTaskOptionsSchema>;
@@ -793,6 +826,16 @@ export const CompleteTaskBodyInputSchema = RunTaskBodyInputSchema.pick({
export type CompleteTaskBodyInput = Prettify<z.input<typeof CompleteTaskBodyInputSchema>>;
export type CompleteTaskBodyOutput = z.infer<typeof CompleteTaskBodyInputSchema>;
export const CompleteTaskBodyV2InputSchema = RunTaskBodyInputSchema.pick({
properties: true,
description: true,
params: true,
}).extend({
output: z.string().optional(),
});
export type CompleteTaskBodyV2Input = Prettify<z.input<typeof CompleteTaskBodyV2InputSchema>>;
export const FailTaskBodyInputSchema = z.object({
error: ErrorWithStackSchema,
});
@@ -896,3 +939,29 @@ export const GetRunStatusesSchema = z.object({
statuses: z.array(JobRunStatusRecordSchema),
});
export type GetRunStatuses = z.infer<typeof GetRunStatusesSchema>;
export const InvokeJobResponseSchema = z.object({
id: z.string(),
});
export const InvokeJobRequestBodySchema = z.object({
payload: z.any(),
context: z.any().optional(),
options: z
.object({
accountId: z.string().optional(),
callbackUrl: z.string().optional(),
})
.optional(),
});
export type InvokeJobRequestBody = z.infer<typeof InvokeJobRequestBodySchema>;
export const InvokeOptionsSchema = z.object({
accountId: z.string().optional(),
idempotencyKey: z.string().optional(),
context: z.any().optional(),
callbackUrl: z.string().optional(),
});
export type InvokeOptions = z.infer<typeof InvokeOptionsSchema>;
packages/core/src/schemas/fetch.ts
+8
View File
@@ -52,10 +52,18 @@ export const FetchRetryOptionsSchema = z.record(FetchRetryStrategySchema);
*/
export type FetchRetryOptions = z.infer<typeof FetchRetryOptionsSchema>;
export const FetchTimeoutOptionsSchema = z.object({
durationInMs: z.number(),
retry: RetryOptionsSchema.optional(),
});
export type FetchTimeoutOptions = z.infer<typeof FetchTimeoutOptionsSchema>;
export const FetchOperationSchema = z.object({
url: z.string(),
requestInit: FetchRequestInitSchema.optional(),
retry: z.record(FetchRetryStrategySchema).optional(),
timeout: FetchTimeoutOptionsSchema.optional(),
});
export type FetchOperation = z.infer<typeof FetchOperationSchema>;
packages/core/src/schemas/runs.ts
+51 -2
View File
@@ -1,6 +1,8 @@
import { z } from "zod";
import { TaskStatusSchema } from "./tasks";
import { JobRunStatusRecordSchema } from "./statuses";
import { JobRunStatusRecord, JobRunStatusRecordSchema } from "./statuses";
import { Prettify } from "../types";
import { RuntimeEnvironmentType } from "./api";
export const RunStatusSchema = z.union([
z.literal("PENDING"),
@@ -87,7 +89,7 @@ export const GetRunSchema = RunSchema.extend({
nextCursor: z.string().optional(),
});
export type GetRun = z.infer<typeof GetRunSchema>;
export type GetRun = Prettify<z.infer<typeof GetRunSchema>>;
const GetRunsOptionsSchema = z.object({
/** You can use this to get more tasks, if there are more than are returned in a single batch @default undefined */
@@ -104,3 +106,50 @@ export const GetRunsSchema = z.object({
/** If there are more runs, you can use this to get them */
nextCursor: z.string().optional(),
});
type RunNotificationCommon = {
/** The Run id */
id: string;
/** The Run status */
statuses: JobRunStatusRecord[];
/** When the run started */
startedAt: Date;
/** When the run was last updated */
updatedAt: Date;
/** When the run was completed */
completedAt: Date;
executionDurationInMs: number;
executionCount: number;
/** Job metadata */
job: { id: string; version: string };
/** Environment metadata */
environment: { slug: string; id: string; type: RuntimeEnvironmentType };
/** Organization metadata */
organization: { slug: string; id: string; title: string };
/** Project metadata */
project: { slug: string; id: string; name: string };
/** Account metadata */
account?: { id: string; metadata?: any };
/** Invocation metadata */
invocation: { id: string; context: any; timestamp: Date };
};
export type SuccessfulRunNotification<TOutput> = RunNotificationCommon & {
ok: true;
/** The Run status */
status: "SUCCESS";
/** The output of the run */
output: TOutput;
};
export type FailedRunNotification = RunNotificationCommon & {
ok: false;
/** The Run status */
status: "FAILURE" | "TIMED_OUT" | "ABORTED" | "CANCELED" | "UNRESOLVED_AUTH" | "INVALID_PAYLOAD";
/** The error of the run */
error: any;
};
export type RunNotification<TOutput> = SuccessfulRunNotification<TOutput> | FailedRunNotification;
packages/core/src/schemas/statuses.ts
+2
View File
@@ -29,3 +29,5 @@ export const JobRunStatusRecordSchema = InitalStatusUpdateSchema.extend({
key: z.string(),
history: StatusHistorySchema,
});
export type JobRunStatusRecord = z.infer<typeof JobRunStatusRecordSchema>;
packages/core/src/schemas/tasks.ts
+1
View File
@@ -32,6 +32,7 @@ export const TaskSchema = z.object({
style: StyleSchema.optional().nullable(),
operation: z.string().optional().nullable(),
callbackUrl: z.string().optional().nullable(),
childExecutionMode: z.enum(["SEQUENTIAL", "PARALLEL"]).optional().nullable(),
});
export const ServerTaskSchema = TaskSchema.extend({
packages/core/src/schemas/triggers.ts
+5
View File
@@ -35,6 +35,10 @@ export const StaticTriggerMetadataSchema = z.object({
rule: EventRuleSchema,
});
export const InvokeTriggerMetadataSchema = z.object({
type: z.literal("invoke"),
});
export const ScheduledTriggerMetadataSchema = z.object({
type: z.literal("scheduled"),
schedule: ScheduleMetadataSchema,
@@ -44,6 +48,7 @@ export const TriggerMetadataSchema = z.discriminatedUnion("type", [
DynamicTriggerMetadataSchema,
StaticTriggerMetadataSchema,
ScheduledTriggerMetadataSchema,
InvokeTriggerMetadataSchema,
]);
export type TriggerMetadata = z.infer<typeof TriggerMetadataSchema>;
packages/database/prisma/migrations/20231028193441_prepare_for_invoke_trigger/migration.sql
+29
View File
@@ -0,0 +1,29 @@
-- CreateEnum
CREATE TYPE "JobRunSubscriptionRecipientMethod" AS ENUM ('WEBHOOK');
-- CreateEnum
CREATE TYPE "JobRunSubscriptionStatus" AS ENUM ('ACTIVE', 'INACTIVE');
-- CreateEnum
CREATE TYPE "JobRunSubscriptionEvents" AS ENUM ('SUCCESS', 'FAILURE');
-- AlterTable
ALTER TABLE "EventRecord" ADD COLUMN "internal" BOOLEAN NOT NULL DEFAULT false;
-- CreateTable
CREATE TABLE "JobRunSubscription" (
"id" TEXT NOT NULL,
"runId" TEXT NOT NULL,
"recipient" TEXT NOT NULL,
"recipientMethod" "JobRunSubscriptionRecipientMethod" NOT NULL DEFAULT 'WEBHOOK',
"event" "JobRunSubscriptionEvents" NOT NULL,
"status" "JobRunSubscriptionStatus" NOT NULL DEFAULT 'ACTIVE',
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMP(3) NOT NULL,
"deliveredAt" TIMESTAMP(3),
CONSTRAINT "JobRunSubscription_pkey" PRIMARY KEY ("id")
);
-- AddForeignKey
ALTER TABLE "JobRunSubscription" ADD CONSTRAINT "JobRunSubscription_runId_fkey" FOREIGN KEY ("runId") REFERENCES "JobRun"("id") ON DELETE CASCADE ON UPDATE CASCADE;
packages/database/prisma/migrations/20231101105021_add_child_execution_mode_to_tasks/migration.sql
+5
View File
@@ -0,0 +1,5 @@
-- CreateEnum
CREATE TYPE "TaskChildExecutionMode" AS ENUM ('SEQUENTIAL', 'PARALLEL');
-- AlterTable
ALTER TABLE "Task" ADD COLUMN "childExecutionMode" "TaskChildExecutionMode" NOT NULL DEFAULT 'SEQUENTIAL';
packages/database/prisma/schema.prisma
+45 -5
View File
@@ -673,8 +673,9 @@ model EventRecord {
updatedAt DateTime @updatedAt
cancelledAt DateTime?
isTest Boolean @default(false)
runs JobRun[]
isTest Boolean @default(false)
internal Boolean @default(false)
runs JobRun[]
@@unique([eventId, environmentId])
}
@@ -741,6 +742,7 @@ model JobRun {
executions JobRunExecution[]
statuses JobRunStatusRecord[]
autoYieldExecution JobRunAutoYieldExecution[]
subscriptions JobRunSubscription[]
}
enum JobRunStatus {
@@ -772,6 +774,38 @@ model JobRunAutoYieldExecution {
createdAt DateTime @default(now())
}
model JobRunSubscription {
id String @id @default(cuid())
run JobRun @relation(fields: [runId], references: [id], onDelete: Cascade, onUpdate: Cascade)
runId String
recipient String
recipientMethod JobRunSubscriptionRecipientMethod @default(WEBHOOK)
event JobRunSubscriptionEvents
status JobRunSubscriptionStatus @default(ACTIVE)
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
deliveredAt DateTime?
}
enum JobRunSubscriptionRecipientMethod {
WEBHOOK
}
enum JobRunSubscriptionStatus {
ACTIVE
INACTIVE
}
enum JobRunSubscriptionEvents {
SUCCESS
FAILURE
}
model JobRunExecution {
id String @id @default(cuid())
@@ -849,9 +883,10 @@ model Task {
runConnection RunConnection? @relation(fields: [runConnectionId], references: [id], onDelete: Cascade, onUpdate: Cascade)
runConnectionId String?
children Task[] @relation("TaskParent")
executions JobRunExecution[]
attempts TaskAttempt[]
children Task[] @relation("TaskParent")
childExecutionMode TaskChildExecutionMode @default(SEQUENTIAL)
executions JobRunExecution[]
attempts TaskAttempt[]
@@unique([runId, idempotencyKey])
}
@@ -865,6 +900,11 @@ enum TaskStatus {
CANCELED
}
enum TaskChildExecutionMode {
SEQUENTIAL
PARALLEL
}
model TaskAttempt {
id String @id @default(cuid())
packages/integration-kit/package.json
+3 -2
View File
@@ -34,9 +34,10 @@
"typecheck": "tsup --dts-resolve --no-dts"
},
"dependencies": {
"uuid": "^9.0.0"
"uuid": "^9.0.0",
"@trigger.dev/core": "workspace:*"
},
"engines": {
"node": ">=18.0.0"
}
}
}
packages/integration-kit/src/index.ts
+1
View File
@@ -4,3 +4,4 @@ export * from "./omit";
export * from "./properties";
export * from "./file";
export * from "./prettify";
export * from "./types";
packages/integration-kit/src/types.ts
+3
View File
@@ -0,0 +1,3 @@
import type { FetchRetryOptions, FetchTimeoutOptions } from "@trigger.dev/core";
export type { FetchRetryOptions, FetchTimeoutOptions };
packages/integration-kit/tsconfig.json
+4 -1
View File
@@ -4,7 +4,10 @@
"compilerOptions": {
"experimentalDecorators": true,
"emitDecoratorMetadata": true,
"paths": {},
"paths": {
"@trigger.dev/core/*": ["../core/src/*"],
"@trigger.dev/core": ["../core/src/index"]
},
"lib": ["DOM", "DOM.Iterable"],
"declaration": false,
"declarationMap": false
packages/trigger-sdk/src/apiClient.ts
+79 -3
View File
@@ -30,6 +30,10 @@ import {
urlWithSearchParams,
RunTaskResponseWithCachedTasksBodySchema,
API_VERSIONS,
InvokeJobResponseSchema,
InvokeOptions,
InvokeJobRequestBody,
CompleteTaskBodyV2Input,
} from "@trigger.dev/core";
import { z } from "zod";
@@ -139,7 +143,7 @@ export class ApiClient {
);
}
async completeTask(runId: string, id: string, task: CompleteTaskBodyInput) {
async completeTask(runId: string, id: string, task: CompleteTaskBodyV2Input) {
const apiKey = await this.#apiKey();
this.#logger.debug("Complete Task", {
@@ -154,6 +158,7 @@ export class ApiClient {
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${apiKey}`,
"Trigger-Version": API_VERSIONS.SERIALIZED_TASK_OUTPUT,
},
body: JSON.stringify(task),
}
@@ -476,6 +481,33 @@ export class ApiClient {
);
}
async invokeJob(jobId: string, payload: any, options: InvokeOptions = {}) {
const apiKey = await this.#apiKey();
this.#logger.debug("Invoking Job", {
jobId,
});
const body: InvokeJobRequestBody = {
payload,
context: options.context ?? {},
options: {
accountId: options.accountId,
callbackUrl: options.callbackUrl,
},
};
return await zodfetch(InvokeJobResponseSchema, `${this.#apiUrl}/api/v1/jobs/${jobId}/invoke`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${apiKey}`,
...(options.idempotencyKey ? { "Idempotency-Key": options.idempotencyKey } : {}),
},
body: JSON.stringify(body),
});
}
async #apiKey() {
const apiKey = getApiKey(this.#options.apiKey);
@@ -573,7 +605,8 @@ async function zodfetchWithVersions<
options?: {
errorMessage?: string;
optional?: TOptional;
}
},
retryCount = 0
): Promise<
TOptional extends true
? VersionedResponseBody<TVersionedResponseBodyMap, TUnversionedResponseBodySchema> | undefined
@@ -596,6 +629,22 @@ async function zodfetchWithVersions<
throw new Error(body.error);
}
if (response.status >= 500 && retryCount < 6) {
// retry with exponential backoff and jitter
const delay = exponentialBackoff(retryCount + 1, 2, 50, 1150, 50);
await new Promise((resolve) => setTimeout(resolve, delay));
return zodfetchWithVersions(
versionedSchemaMap,
unversionedSchema,
url,
requestInit,
options,
retryCount + 1
);
}
if (response.status !== 200) {
throw new Error(
options?.errorMessage ?? `Failed to fetch ${url}, got status code ${response.status}`
@@ -632,7 +681,8 @@ async function zodfetch<TResponseSchema extends z.ZodTypeAny, TOptional extends
options?: {
errorMessage?: string;
optional?: TOptional;
}
},
retryCount = 0
): Promise<
TOptional extends true ? z.infer<TResponseSchema> | undefined : z.infer<TResponseSchema>
> {
@@ -653,6 +703,15 @@ async function zodfetch<TResponseSchema extends z.ZodTypeAny, TOptional extends
throw new Error(body.error);
}
if (response.status >= 500 && retryCount < 6) {
// retry with exponential backoff and jitter
const delay = exponentialBackoff(retryCount + 1, 2, 50, 1150, 50);
await new Promise((resolve) => setTimeout(resolve, delay));
return zodfetch(schema, url, requestInit, options, retryCount + 1);
}
if (response.status !== 200) {
throw new Error(
options?.errorMessage ?? `Failed to fetch ${url}, got status code ${response.status}`
@@ -663,3 +722,20 @@ async function zodfetch<TResponseSchema extends z.ZodTypeAny, TOptional extends
return schema.parse(jsonBody);
}
function exponentialBackoff(
retryCount: number,
exponential: number,
minDelay: number,
maxDelay: number,
jitter: number
): number {
// Calculate the delay using the exponential backoff formula
const delay = Math.min(Math.pow(exponential, retryCount) * minDelay, maxDelay);
// Calculate the jitter
const jitterValue = Math.random() * jitter;
// Return the calculated delay with jitter
return delay + jitterValue;
}
packages/trigger-sdk/src/errors.ts
+31 -12
View File
@@ -5,6 +5,13 @@ export class ResumeWithTaskError {
constructor(public task: ServerTask) {}
}
export class ResumeWithParallelTaskError {
constructor(
public task: ServerTask,
public childErrors: Array<TriggerInternalError>
) {}
}
export class RetryWithTaskError {
constructor(
public cause: ErrorWithStack,
@@ -33,8 +40,8 @@ export class AutoYieldWithCompletedTaskExecutionError {
constructor(
public id: string,
public properties: DisplayProperty[] | undefined,
public output: any,
public data: { location: string; timeRemaining: number; timeElapsed: number }
public data: { location: string; timeRemaining: number; timeElapsed: number },
public output?: string
) {}
}
@@ -42,27 +49,39 @@ export class ParsedPayloadSchemaError {
constructor(public schemaErrors: SchemaError[]) {}
}
export type TriggerInternalError =
| ResumeWithTaskError
| RetryWithTaskError
| CanceledWithTaskError
| YieldExecutionError
| AutoYieldExecutionError
| AutoYieldWithCompletedTaskExecutionError
| ResumeWithParallelTaskError;
/** Use this function if you're using a `try/catch` block to catch errors.
* It checks if a thrown error is a special internal error that you should ignore.
* If this returns `true` then you must rethrow the error: `throw err;`
* @param err The error to check
* @returns `true` if the error is a Trigger Error, `false` otherwise.
*/
export function isTriggerError(
err: unknown
): err is
| ResumeWithTaskError
| RetryWithTaskError
| CanceledWithTaskError
| YieldExecutionError
| AutoYieldExecutionError
| AutoYieldWithCompletedTaskExecutionError {
export function isTriggerError(err: unknown): err is TriggerInternalError {
return (
err instanceof ResumeWithTaskError ||
err instanceof RetryWithTaskError ||
err instanceof CanceledWithTaskError ||
err instanceof YieldExecutionError ||
err instanceof AutoYieldExecutionError ||
err instanceof AutoYieldWithCompletedTaskExecutionError
err instanceof AutoYieldWithCompletedTaskExecutionError ||
err instanceof ResumeWithParallelTaskError
);
}
// This error isn't an internal error but it can be used by the user to figure out which task caused the error
export class ErrorWithTask extends Error {
constructor(
public cause: ServerTask,
message: string
) {
super(message);
}
}
packages/trigger-sdk/src/index.ts
+1
View File
@@ -6,6 +6,7 @@ export * from "./triggers/externalSource";
export * from "./triggers/dynamic";
export * from "./triggers/scheduled";
export * from "./triggers/notifications";
export * from "./triggers/invokeTrigger";
export * from "./io";
export * from "./types";
packages/trigger-sdk/src/io.ts
+117 -12
View File
@@ -6,6 +6,7 @@ import {
ErrorWithStackSchema,
FetchRequestInit,
FetchRetryOptions,
FetchTimeoutOptions,
InitialStatusUpdate,
IntervalOptions,
LogLevel,
@@ -13,7 +14,6 @@ import {
RunTaskOptions,
SendEvent,
SendEventOptions,
SerializableJsonSchema,
ServerTask,
UpdateTriggerSourceBodyV2,
supportsFeature,
@@ -26,19 +26,22 @@ import {
AutoYieldExecutionError,
AutoYieldWithCompletedTaskExecutionError,
CanceledWithTaskError,
ErrorWithTask,
ResumeWithParallelTaskError,
ResumeWithTaskError,
RetryWithTaskError,
TriggerInternalError,
YieldExecutionError,
isTriggerError,
} from "./errors";
import { IntegrationTaskKey } from "./integrations";
import { calculateRetryAt } from "./retry";
import { TriggerStatus } from "./status";
import { TriggerClient } from "./triggerClient";
import { DynamicTrigger } from "./triggers/dynamic";
import { ExternalSource, ExternalSourceParams } from "./triggers/externalSource";
import { DynamicSchedule } from "./triggers/scheduled";
import { EventSpecification, TaskLogger, TriggerContext } from "./types";
import { IntegrationTaskKey } from "./integrations";
import { TriggerStatus } from "./status";
export type IOTask = ServerTask;
@@ -85,6 +88,21 @@ export type IOStats = {
noopCachedTaskMisses: number;
};
export interface OutputSerializer {
serialize(value: any): string;
deserialize<T>(value: string): T;
}
export class JSONOutputSerializer implements OutputSerializer {
serialize(value: any): string {
return JSON.stringify(value);
}
deserialize(value?: string): any {
return value ? JSON.parse(value) : undefined;
}
}
export class IO {
private _id: string;
private _apiClient: ApiClient;
@@ -102,6 +120,8 @@ export class IO {
private _serverVersion: string;
private _timeOrigin: number;
private _executionTimeout?: number;
private _outputSerializer: OutputSerializer = new JSONOutputSerializer();
private _visitedCacheKeys: Set<string> = new Set();
get stats() {
return this._stats;
@@ -284,7 +304,8 @@ export class IO {
cacheKey: string | any[],
url: string,
requestInit?: FetchRequestInit,
retry?: FetchRetryOptions
retry?: FetchRetryOptions,
timeout?: FetchTimeoutOptions
): Promise<TResponseData> {
const urlObject = new URL(url);
@@ -295,7 +316,7 @@ export class IO {
},
{
name: `fetch ${urlObject.hostname}${urlObject.pathname}`,
params: { url, requestInit, retry },
params: { url, requestInit, retry, timeout },
operation: "fetch",
icon: "background",
noop: false,
@@ -313,7 +334,11 @@ export class IO {
label: "background",
text: "true",
},
...(timeout ? [{ label: "timeout", text: `${timeout.durationInMs}ms` }] : []),
],
retry: {
limit: 0,
},
}
)) as TResponseData;
}
@@ -583,6 +608,53 @@ export class IO {
);
}
async parallel<T extends Json<T> | void, TItem>(
cacheKey: string | any[],
items: Array<TItem>,
callback: (item: TItem, index: number) => Promise<T>,
options?: Pick<RunTaskOptions, "name" | "properties">
): Promise<Array<T>> {
const results = await this.runTask(
cacheKey,
async (task) => {
const outcomes = await Promise.allSettled(
items.map((item, index) => spaceOut(() => callback(item, index), index, 15))
);
// If all the outcomes are fulfilled, return the values
if (outcomes.every((outcome) => outcome.status === "fulfilled")) {
return outcomes.map(
(outcome) => (outcome as PromiseFulfilledResult<T>).value
) as Array<{}>;
}
// If they any of the errors are non internal errors, throw the first one
const nonInternalErrors = outcomes
.filter((outcome) => outcome.status === "rejected" && !isTriggerError(outcome.reason))
.map((outcome) => outcome as PromiseRejectedResult);
if (nonInternalErrors.length > 0) {
throw nonInternalErrors[0].reason;
}
// gather all the internal errors
const internalErrors = outcomes
.filter((outcome) => outcome.status === "rejected" && isTriggerError(outcome.reason))
.map((outcome) => outcome as PromiseRejectedResult)
.map((outcome) => outcome.reason as TriggerInternalError);
throw new ResumeWithParallelTaskError(task, internalErrors);
},
{
name: "parallel",
parallel: true,
...(options ?? {}),
}
);
return results as unknown as Array<T>;
}
/** `io.runTask()` allows you to run a [Task](https://trigger.dev/docs/documentation/concepts/tasks) from inside a Job run. A Task is a resumable unit of a Run that can be retried, resumed and is logged. [Integrations](https://trigger.dev/docs/integrations) use Tasks internally to perform their actions.
*
* @param cacheKey Should be a stable and unique key inside the `run()`. See [resumability](https://trigger.dev/docs/documentation/concepts/resumability) for more information.
@@ -613,6 +685,22 @@ export class IO {
[this._id, parentId ?? "", cacheKey].flat()
);
if (this._visitedCacheKeys.has(idempotencyKey)) {
if (typeof cacheKey === "string") {
throw new Error(
`Task with cacheKey "${cacheKey}" has already been executed in this run. Each task must have a unique cacheKey.`
);
} else {
throw new Error(
`Task with cacheKey "${cacheKey.join(
"-"
)}" has already been executed in this run. Each task must have a unique cacheKey.`
);
}
}
this._visitedCacheKeys.add(idempotencyKey);
const cachedTask = this._cachedTasks.get(idempotencyKey);
if (cachedTask && cachedTask.status === "COMPLETED") {
@@ -714,7 +802,10 @@ export class IO {
task,
});
throw new Error(task.error ?? task?.output ? JSON.stringify(task.output) : "Task errored");
throw new ErrorWithTask(
task,
task.error ?? task?.output ? JSON.stringify(task.output) : "Task errored"
);
}
this.#detectAutoYield("before_execute_task", 1500);
@@ -731,7 +822,7 @@ export class IO {
return {} as T;
}
const output = SerializableJsonSchema.parse(result) as T;
const output = this._outputSerializer.serialize(result);
this._logger.debug("Completing using output", {
idempotencyKey,
@@ -741,7 +832,7 @@ export class IO {
this.#detectAutoYield("before_complete_task", 500, task, output);
const completedTask = await this._apiClient.completeTask(this._id, task.id, {
output: output ?? undefined,
output,
properties: task.outputProperties ?? undefined,
});
@@ -761,7 +852,7 @@ export class IO {
this.#detectAutoYield("after_complete_task", 500);
return output;
return this._outputSerializer.deserialize<T>(output);
} catch (error) {
if (isTriggerError(error)) {
throw error;
@@ -799,6 +890,13 @@ export class IO {
}
}
if (error instanceof ErrorWithTask) {
// This means a subtask errored, so we need to update the parent task and not retry it
await this._apiClient.failTask(this._id, task.id, {
error: error.cause.output as any,
});
}
const parsedError = ErrorWithStackSchema.safeParse(error);
if (options?.retry && !skipRetrying) {
@@ -902,7 +1000,7 @@ export class IO {
this._cachedTasks.set(task.idempotencyKey, task);
}
#detectAutoYield(location: string, threshold: number = 1500, task?: ServerTask, output?: any) {
#detectAutoYield(location: string, threshold: number = 1500, task?: ServerTask, output?: string) {
const timeRemaining = this.#getRemainingTimeInMillis();
if (timeRemaining && timeRemaining < threshold) {
@@ -910,12 +1008,12 @@ export class IO {
throw new AutoYieldWithCompletedTaskExecutionError(
task.id,
task.outputProperties ?? [],
output,
{
location,
timeRemaining,
timeElapsed: this.#getTimeElapsed(),
}
},
output
);
} else {
throw new AutoYieldExecutionError(location, timeRemaining, this.#getTimeElapsed());
@@ -1019,3 +1117,10 @@ export class IOLogger implements TaskLogger {
return this.callback("ERROR", message, properties);
}
}
// Space out the execution of the callback by a delay of index * delay
async function spaceOut<T>(callback: () => Promise<T>, index: number, delay: number): Promise<T> {
await new Promise((resolve) => setTimeout(resolve, index * delay));
return await callback();
}
packages/trigger-sdk/src/job.ts
+213 -5
View File
@@ -1,12 +1,28 @@
import { IntegrationConfig, JobMetadata, LogLevel, QueueOptions } from "@trigger.dev/core";
import {
IntegrationConfig,
InvokeOptions,
JobMetadata,
LogLevel,
QueueOptions,
RunNotification,
} from "@trigger.dev/core";
import { IOWithIntegrations, TriggerIntegration } from "./integrations";
import { TriggerClient } from "./triggerClient";
import type { EventSpecification, Trigger, TriggerContext, TriggerEventType } from "./types";
import type {
EventSpecification,
Trigger,
TriggerContext,
TriggerEventType,
TriggerInvokeType,
} from "./types";
import { slugifyId } from "./utils";
import { runLocalStorage } from "./runLocalStorage";
import { Prettify } from "@trigger.dev/core";
export type JobOptions<
TTrigger extends Trigger<EventSpecification<any>>,
TIntegrations extends Record<string, TriggerIntegration> = {},
TOutput extends any = any,
> = {
/** The `id` property is used to uniquely identify the Job. Only change this if you want to create a new Job. */
id: string;
@@ -57,7 +73,7 @@ export type JobOptions<
payload: TriggerEventType<TTrigger>,
io: IOWithIntegrations<TIntegrations>,
context: TriggerContext
) => Promise<any>;
) => Promise<TOutput>;
// @internal
__internal?: boolean;
@@ -75,8 +91,9 @@ export type JobIO<TJob> = TJob extends Job<any, infer TIntegrations>
export class Job<
TTrigger extends Trigger<EventSpecification<any>>,
TIntegrations extends Record<string, TriggerIntegration> = {},
TOutput extends any = any,
> {
readonly options: JobOptions<TTrigger, TIntegrations>;
readonly options: JobOptions<TTrigger, TIntegrations, TOutput>;
client: TriggerClient;
@@ -84,7 +101,7 @@ export class Job<
/** An instance of [TriggerClient](/sdk/triggerclient) that is used to send events
to the Trigger API. */
client: TriggerClient,
options: JobOptions<TTrigger, TIntegrations>
options: JobOptions<TTrigger, TIntegrations, TOutput>
) {
this.client = client;
this.options = options;
@@ -152,6 +169,197 @@ export class Job<
};
}
async invoke(
cacheKey: string,
payload: TriggerInvokeType<TTrigger>,
options?: InvokeOptions
): Promise<{ id: string }>;
async invoke(
payload: TriggerInvokeType<TTrigger>,
options?: InvokeOptions
): Promise<{ id: string }>;
async invoke(
param1: string | TriggerInvokeType<TTrigger>,
param2: TriggerInvokeType<TTrigger> | InvokeOptions | undefined = undefined,
param3: InvokeOptions | undefined = undefined
): Promise<{ id: string }> {
const runStore = runLocalStorage.getStore();
if (typeof param1 === "string") {
if (!runStore) {
throw new Error(
"Cannot invoke a job from outside of a run when passing a cacheKey. Make sure you are running the job from within a run or use the invoke method without the cacheKey."
);
}
const options = param3 ?? {};
return await runStore.io.runTask(
param1,
async (task) => {
const result = await this.client.invokeJob(this.id, param2, {
idempotencyKey: task.idempotencyKey,
...options,
});
task.outputProperties = [
{
label: "Run",
text: result.id,
url: `/orgs/${runStore.ctx.organization.slug}/projects/${runStore.ctx.project.slug}/jobs/${this.id}/runs/${result.id}/trigger`,
},
];
return result;
},
{
name: `Manually Invoke '${this.name}'`,
params: param2,
properties: [
{
label: "Job",
text: this.id,
url: `/orgs/${runStore.ctx.organization.slug}/projects/${runStore.ctx.project.slug}/jobs/${this.id}`,
},
{
label: "Env",
text: runStore.ctx.environment.slug,
},
],
}
);
}
if (runStore) {
throw new Error("Cannot invoke a job from within a run without a cacheKey.");
}
return await this.client.invokeJob(this.id, param1, param3);
}
async invokeAndWaitForCompletion(
cacheKey: string | string[],
payload: TriggerInvokeType<TTrigger>,
timeoutInSeconds: number = 60 * 60, // 1 hour
options: Prettify<Pick<InvokeOptions, "accountId" | "context">> = {}
): Promise<RunNotification<TOutput>> {
const runStore = runLocalStorage.getStore();
if (!runStore) {
throw new Error(
"Cannot invoke a job from outside of a run using invokeAndWaitForCompletion. Make sure you are running the job from within a run or use the invoke method instead."
);
}
const { io, ctx } = runStore;
return (await io.runTask(
cacheKey,
async (task) => {
const parsedPayload = this.trigger.event.parseInvokePayload
? this.trigger.event.parseInvokePayload(payload)
? payload
: undefined
: payload;
const result = await this.client.invokeJob(this.id, parsedPayload, {
idempotencyKey: task.idempotencyKey,
callbackUrl: task.callbackUrl ?? undefined,
...options,
});
task.outputProperties = [
{
label: "Run",
text: result.id,
url: `/orgs/${ctx.organization.slug}/projects/${ctx.project.slug}/jobs/${this.id}/runs/${result.id}/trigger`,
},
];
return {}; // we don't want to return anything here, we just want to wait for the callback
},
{
name: `Manually Invoke '${this.name}' and wait for completion`,
params: payload,
properties: [
{
label: "Job",
text: this.id,
url: `/orgs/${ctx.organization.slug}/projects/${ctx.project.slug}/jobs/${this.id}`,
},
{
label: "Env",
text: ctx.environment.slug,
},
],
callback: {
enabled: true,
timeoutInSeconds,
},
}
)) as RunNotification<TOutput>;
}
async batchInvokeAndWaitForCompletion(
cacheKey: string | string[],
batch: Array<{
payload: TriggerInvokeType<TTrigger>;
timeoutInSeconds?: number;
options?: Prettify<Pick<InvokeOptions, "accountId" | "context">>;
}>
): Promise<Array<RunNotification<TOutput>>> {
const runStore = runLocalStorage.getStore();
if (!runStore) {
throw new Error(
"Cannot invoke a job from outside of a run using batchInvokeAndWaitForCompletion."
);
}
// If there are no items in the batch, return an empty array
if (batch.length === 0) {
return [];
}
// If there are too many items in the batch, throw an error
if (batch.length > 25) {
throw new Error(
`Cannot batch invoke more than 25 items. You tried to batch invoke ${batch.length} items.`
);
}
const { io, ctx } = runStore;
const results = await io.parallel(
cacheKey,
batch,
async (item, index) => {
return (await this.invokeAndWaitForCompletion(
String(index),
item.payload,
item.timeoutInSeconds ?? 60 * 60,
item.options
)) as RunNotification<{}>;
},
{
name: `Batch Invoke '${this.name}'`,
properties: [
{
label: "Job",
text: this.id,
url: `/orgs/${ctx.organization.slug}/projects/${ctx.project.slug}/jobs/${this.id}`,
},
{
label: "Env",
text: ctx.environment.slug,
},
],
}
);
return results as Array<RunNotification<TOutput>>;
}
// Make sure the id is valid (must only contain alphanumeric characters and dashes)
// Make sure the version is valid (must be a valid semver version)
#validate() {
packages/trigger-sdk/src/triggerClient.ts
+128 -78
View File
@@ -11,6 +11,7 @@ import {
IndexEndpointResponse,
InitializeTriggerBodySchema,
IntegrationConfig,
InvokeOptions,
JobMetadata,
LogLevel,
Logger,
@@ -24,6 +25,7 @@ import {
RegisterTriggerBodyV2,
RunJobBody,
RunJobBodySchema,
RunJobErrorResponse,
RunJobResponse,
ScheduleMetadata,
SendEvent,
@@ -36,7 +38,9 @@ import {
AutoYieldExecutionError,
AutoYieldWithCompletedTaskExecutionError,
CanceledWithTaskError,
ErrorWithTask,
ParsedPayloadSchemaError,
ResumeWithParallelTaskError,
ResumeWithTaskError,
RetryWithTaskError,
YieldExecutionError,
@@ -321,6 +325,13 @@ export class TriggerClient {
const results = await this.#executeJob(execution.data, job, timeOrigin, triggerVersion);
this.#internalLogger.debug("executed job", {
results,
job: job.id,
version: job.version,
triggerVersion,
});
return {
status: 200,
body: results,
@@ -467,8 +478,9 @@ export class TriggerClient {
defineJob<
TTrigger extends Trigger<EventSpecification<any>>,
TIntegrations extends Record<string, TriggerIntegration> = {},
>(options: JobOptions<TTrigger, TIntegrations>) {
return new Job<TTrigger, TIntegrations>(this, options);
TOutput extends any = any,
>(options: JobOptions<TTrigger, TIntegrations, TOutput>) {
return new Job<TTrigger, TIntegrations, TOutput>(this, options);
}
defineAuthResolver(
@@ -685,6 +697,10 @@ export class TriggerClient {
return this.#client.getRunStatuses(runId);
}
async invokeJob(jobId: string, payload: any, options?: InvokeOptions) {
return this.#client.invokeJob(jobId, payload, options);
}
authorized(
apiKey?: string | null
): "authorized" | "unauthorized" | "missing-client" | "missing-header" {
@@ -790,91 +806,124 @@ export class TriggerClient {
this.#logIOStats(io.stats);
}
if (error instanceof AutoYieldExecutionError) {
if (error instanceof ResumeWithParallelTaskError) {
return {
status: "AUTO_YIELD_EXECUTION",
location: error.location,
timeRemaining: error.timeRemaining,
timeElapsed: error.timeElapsed,
limit: body.runChunkExecutionLimit,
};
}
if (error instanceof AutoYieldWithCompletedTaskExecutionError) {
return {
status: "AUTO_YIELD_EXECUTION_WITH_COMPLETED_TASK",
id: error.id,
properties: error.properties,
output: error.output,
data: {
...error.data,
limit: body.runChunkExecutionLimit,
},
};
}
if (error instanceof YieldExecutionError) {
return { status: "YIELD_EXECUTION", key: error.key };
}
if (error instanceof ParsedPayloadSchemaError) {
return { status: "INVALID_PAYLOAD", errors: error.schemaErrors };
}
if (error instanceof ResumeWithTaskError) {
return { status: "RESUME_WITH_TASK", task: error.task };
}
if (error instanceof RetryWithTaskError) {
return {
status: "RETRY_WITH_TASK",
status: "RESUME_WITH_PARALLEL_TASK",
task: error.task,
error: error.cause,
retryAt: error.retryAt,
childErrors: error.childErrors.map((childError) => {
return this.#convertErrorToExecutionResponse(childError, body);
}),
};
}
if (error instanceof CanceledWithTaskError) {
return {
status: "CANCELED",
task: error.task,
};
}
if (error instanceof RetryWithTaskError) {
const errorWithStack = ErrorWithStackSchema.safeParse(error.cause);
if (errorWithStack.success) {
return {
status: "ERROR",
error: errorWithStack.data,
task: error.task,
};
}
return {
status: "ERROR",
error: { message: "Unknown error" },
task: error.task,
};
}
const errorWithStack = ErrorWithStackSchema.safeParse(error);
if (errorWithStack.success) {
return { status: "ERROR", error: errorWithStack.data };
}
const message = typeof error === "string" ? error : JSON.stringify(error);
return {
status: "ERROR",
error: { name: "Unknown error", message },
};
return this.#convertErrorToExecutionResponse(error, body);
}
}
#convertErrorToExecutionResponse(error: any, body: RunJobBody): RunJobErrorResponse {
if (error instanceof AutoYieldExecutionError) {
return {
status: "AUTO_YIELD_EXECUTION",
location: error.location,
timeRemaining: error.timeRemaining,
timeElapsed: error.timeElapsed,
limit: body.runChunkExecutionLimit,
};
}
if (error instanceof AutoYieldWithCompletedTaskExecutionError) {
return {
status: "AUTO_YIELD_EXECUTION_WITH_COMPLETED_TASK",
id: error.id,
properties: error.properties,
output: error.output,
data: {
...error.data,
limit: body.runChunkExecutionLimit,
},
};
}
if (error instanceof YieldExecutionError) {
return { status: "YIELD_EXECUTION", key: error.key };
}
if (error instanceof ParsedPayloadSchemaError) {
return { status: "INVALID_PAYLOAD", errors: error.schemaErrors };
}
if (error instanceof ResumeWithTaskError) {
return { status: "RESUME_WITH_TASK", task: error.task };
}
if (error instanceof RetryWithTaskError) {
return {
status: "RETRY_WITH_TASK",
task: error.task,
error: error.cause,
retryAt: error.retryAt,
};
}
if (error instanceof CanceledWithTaskError) {
return {
status: "CANCELED",
task: error.task,
};
}
if (error instanceof ErrorWithTask) {
const errorWithStack = ErrorWithStackSchema.safeParse(error.cause.output);
if (errorWithStack.success) {
return {
status: "ERROR",
error: errorWithStack.data,
task: error.cause,
};
}
return {
status: "ERROR",
error: { message: JSON.stringify(error.cause.output) },
task: error.cause,
};
}
if (error instanceof RetryWithTaskError) {
const errorWithStack = ErrorWithStackSchema.safeParse(error.cause);
if (errorWithStack.success) {
return {
status: "ERROR",
error: errorWithStack.data,
task: error.task,
};
}
return {
status: "ERROR",
error: { message: "Unknown error" },
task: error.task,
};
}
const errorWithStack = ErrorWithStackSchema.safeParse(error);
if (errorWithStack.success) {
return { status: "ERROR", error: errorWithStack.data };
}
const message = typeof error === "string" ? error : JSON.stringify(error);
return {
status: "ERROR",
error: { name: "Unknown error", message },
};
}
#createRunContext(execution: RunJobBody): TriggerContext {
const { event, organization, environment, job, run, source } = execution;
const { event, organization, project, environment, job, run, source } = execution;
return {
event: {
@@ -884,6 +933,7 @@ export class TriggerClient {
timestamp: event.timestamp,
},
organization,
project: project ?? { id: "unknown", name: "unknown", slug: "unknown" }, // backwards compat with old servers
environment,
job,
run,
packages/trigger-sdk/src/triggers/invokeTrigger.ts
+83
View File
@@ -0,0 +1,83 @@
import { TriggerMetadata } from "@trigger.dev/core";
import { ParsedPayloadSchemaError } from "../errors";
import { Job } from "../job";
import { TriggerClient } from "../triggerClient";
import { EventSpecification, EventSpecificationExample, SchemaParser, Trigger } from "../types";
import { formatSchemaErrors } from "../utils/formatSchemaErrors";
import { TypeOf, ZodType, z } from "zod";
/** Configuration options for an InvokeTrigger */
type InvokeTriggerOptions<TSchema extends ZodType = z.ZodTypeAny> = {
/** A [Zod](https://trigger.dev/docs/documentation/guides/zod) schema that defines the shape of the event payload.
* The default is `z.any()` which is `any`.
* */
schema?: TSchema;
examples?: EventSpecificationExample[];
};
export class InvokeTrigger<TSchema extends ZodType = z.ZodTypeAny>
implements Trigger<EventSpecification<TypeOf<TSchema>, z.input<TSchema>>>
{
#options: InvokeTriggerOptions<TSchema>;
constructor(options: InvokeTriggerOptions<TSchema>) {
this.#options = options;
}
toJSON(): TriggerMetadata {
return {
type: "invoke",
};
}
get event() {
return {
name: "invoke",
title: "Manual Invoke",
source: "trigger.dev",
examples: this.#options.examples ?? [],
icon: "trigger",
parsePayload: (rawPayload: unknown) => {
if (this.#options.schema) {
const results = this.#options.schema.safeParse(rawPayload);
if (!results.success) {
throw new ParsedPayloadSchemaError(formatSchemaErrors(results.error.issues));
}
return results.data;
}
return rawPayload as any;
},
parseInvokePayload: (rawPayload: unknown) => {
if (this.#options.schema) {
const results = this.#options.schema.safeParse(rawPayload);
if (!results.success) {
throw new ParsedPayloadSchemaError(formatSchemaErrors(results.error.issues));
}
return results.data;
}
return rawPayload as any;
},
};
}
attachToJob(
triggerClient: TriggerClient,
job: Job<Trigger<EventSpecification<ZodType<TSchema>>>, any>
): void {}
get preprocessRuns() {
return false;
}
}
export function invokeTrigger<TSchema extends ZodType = z.ZodTypeAny>(
options?: InvokeTriggerOptions<TSchema>
): Trigger<EventSpecification<TypeOf<TSchema>, z.input<TSchema>>> {
return new InvokeTrigger(options ?? {});
}
packages/trigger-sdk/src/types.ts
+12 -1
View File
@@ -32,6 +32,8 @@ export interface TriggerContext {
environment: { slug: string; id: string; type: RuntimeEnvironmentType };
/** Organization metadata */
organization: { slug: string; id: string; title: string };
/** Project metadata */
project: { slug: string; id: string; name: string };
/** Run metadata */
run: { id: string; isTest: boolean; startedAt: Date; isRetry: boolean };
/** Event metadata */
@@ -69,6 +71,14 @@ export type TriggerEventType<TTrigger extends Trigger<any>> = TTrigger extends T
? ReturnType<TEventSpec["parsePayload"]>
: never;
export type TriggerInvokeType<TTrigger extends Trigger<any>> = TTrigger extends Trigger<
infer TEventSpec
>
? TEventSpec["parseInvokePayload"] extends (payload: unknown) => infer TInvoke
? TInvoke
: any
: never;
export interface Trigger<TEventSpec extends EventSpecification<any>> {
event: TEventSpec;
toJSON(): TriggerMetadata;
@@ -90,7 +100,7 @@ export type EventSpecificationExample = {
payload: any;
};
export interface EventSpecification<TEvent extends any> {
export interface EventSpecification<TEvent extends any, TInvoke extends any = TEvent> {
name: string | string[];
title: string;
source: string;
@@ -100,6 +110,7 @@ export interface EventSpecification<TEvent extends any> {
examples?: Array<EventSpecificationExample>;
filter?: EventFilter;
parsePayload: (payload: unknown) => TEvent;
parseInvokePayload?: (payload: unknown) => TInvoke;
runProperties?: (payload: TEvent) => DisplayProperty[];
}
pnpm-lock.yaml
Generated
+70 -49
View File
@@ -134,6 +134,7 @@ importers:
'@typescript-eslint/eslint-plugin': ^5.59.6
'@typescript-eslint/parser': ^5.59.6
'@uiw/react-codemirror': ^4.19.5
'@whatwg-node/fetch': ^0.9.14
autoprefixer: ^10.4.13
class-variance-authority: ^0.5.2
clsx: ^1.2.1
@@ -170,13 +171,12 @@ importers:
prop-types: ^15.8.1
react: ^18.2.0
react-dom: ^18.2.0
react-hot-toast: ^2.4.0
react-hotkeys-hook: ^4.4.1
react-use: ^17.4.0
recharts: ^2.8.0
remix-auth: ^3.2.2
remix-auth-email-link: ^1.4.2
remix-auth-github: ^1.1.1
remix-auth: ^3.6.0
remix-auth-email-link: ^2.1.0
remix-auth-github: ^1.6.0
remix-typedjson: 0.3.1
remix-utils: ^7.1.0
rimraf: ^3.0.2
@@ -241,6 +241,7 @@ importers:
'@trigger.dev/sdk': link:../../packages/trigger-sdk
'@types/pg': 8.6.6
'@uiw/react-codemirror': 4.19.5_th22fcplkuhrqjnlojwclcaim4
'@whatwg-node/fetch': 0.9.14
class-variance-authority: 0.5.2_typescript@5.2.2
clsx: 1.2.1
compression: 1.7.4
@@ -268,13 +269,12 @@ importers:
prismjs: 1.29.0
react: 18.2.0
react-dom: 18.2.0_react@18.2.0
react-hot-toast: 2.4.0_biqbaboplfbrettd7655fr4n2y
react-hotkeys-hook: 4.4.1_biqbaboplfbrettd7655fr4n2y
react-use: 17.4.0_biqbaboplfbrettd7655fr4n2y
recharts: 2.8.0_v2m5e27vhdewzwhryxwfaorcca
remix-auth: 3.4.0_ybjp5xbtg4zthziheradfmei64
remix-auth-email-link: 1.5.2_7ym4eehrzvo4xt5yocgqipdaji
remix-auth-github: 1.3.0_7ym4eehrzvo4xt5yocgqipdaji
remix-auth: 3.6.0_ybjp5xbtg4zthziheradfmei64
remix-auth-email-link: 2.1.0_esw6v6dv7ru6ju3hn3i62b7n2m
remix-auth-github: 1.6.0_esw6v6dv7ru6ju3hn3i62b7n2m
remix-typedjson: 0.3.1_7t5z2pmrpxrx2dh2jpp55scw5q
remix-utils: 7.1.0_nb5lmxjinjv74o775xrkc35gga
semver: 7.5.0
@@ -907,6 +907,7 @@ importers:
packages/integration-kit:
specifiers:
'@trigger.dev/core': workspace:*
'@trigger.dev/tsconfig': workspace:*
'@types/node': '18'
'@types/uuid': ^9.0.0
@@ -916,6 +917,7 @@ importers:
typescript: ^4.8.4
uuid: ^9.0.0
dependencies:
'@trigger.dev/core': link:../core
uuid: 9.0.0
devDependencies:
'@trigger.dev/tsconfig': link:../../config-packages/tsconfig
@@ -14897,6 +14899,30 @@ packages:
'@xtuc/long': 4.2.2
dev: true
/@whatwg-node/events/0.1.1:
resolution: {integrity: sha512-AyQEn5hIPV7Ze+xFoXVU3QTHXVbWPrzaOkxtENMPMuNL6VVHrp4hHfDt9nrQpjO7BgvuM95dMtkycX5M/DZR3w==}
engines: {node: '>=16.0.0'}
dev: false
/@whatwg-node/fetch/0.9.14:
resolution: {integrity: sha512-wurZC82zzZwXRDSW0OS9l141DynaJQh7Yt0FD1xZ8niX7/Et/7RoiLiltbVU1fSF1RR9z6ndEaTUQBAmddTm1w==}
engines: {node: '>=16.0.0'}
dependencies:
'@whatwg-node/node-fetch': 0.5.0
urlpattern-polyfill: 9.0.0
dev: false
/@whatwg-node/node-fetch/0.5.0:
resolution: {integrity: sha512-q76lDAafvHNGWedNAVHrz/EyYTS8qwRLcwne8SJQdRN5P3HydxU6XROFvJfTML6KZXQX2FDdGY4/SnaNyd7M0Q==}
engines: {node: '>=16.0.0'}
dependencies:
'@whatwg-node/events': 0.1.1
busboy: 1.6.0
fast-querystring: 1.1.2
fast-url-parser: 1.1.3
tslib: 2.6.2
dev: false
/@xmldom/xmldom/0.8.6:
resolution: {integrity: sha512-uRjjusqpoqfmRkTaNuLJ2VohVr67Q5YwDATW3VU7PfzTj6IRaihGrYI7zckGZjxQPBIp63nfvJbM+Yu5ICh0Bg==}
engines: {node: '>=10.0.0'}
@@ -20099,7 +20125,6 @@ packages:
/fast-decode-uri-component/1.0.1:
resolution: {integrity: sha512-WKgKWg5eUxvRZGwW8FvfbaH7AXSh2cL+3j5fMGzUMCxWBJ3dV3a7Wz8y2f/uQ0e3B6WmodD3oS54jTQ9HVTIIg==}
dev: true
/fast-deep-equal/3.1.3:
resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}
@@ -20203,7 +20228,6 @@ packages:
resolution: {integrity: sha512-g6KuKWmFXc0fID8WWH0jit4g0AGBoJhCkJMb1RmbsSEUNvQ+ZC8D6CUZ+GtF8nMzSPXnhiePyyqqipzNNEnHjg==}
dependencies:
fast-decode-uri-component: 1.0.1
dev: true
/fast-redact/3.3.0:
resolution: {integrity: sha512-6T5V1QK1u4oF+ATxs1lWUmlEk6P2T9HqJG3e2DnHOdVgZy2rFJBoEnrIedcTXlkAHU/zKC+7KETJ+KGGKwxgMQ==}
@@ -20221,6 +20245,12 @@ packages:
resolution: {integrity: sha512-cIusKBIt/R/oI6z/1nyfe2FvGKVTohVRfvkOhvx0nCEW+xf5NoCXjAHcWp93uOUBchzYcsvPlrapAdX1uW+YGg==}
dev: true
/fast-url-parser/1.1.3:
resolution: {integrity: sha512-5jOCVXADYNuRkKFzNJ0dCCewsZiYo0dz8QNYljkOpFC6r2U4OBmKtvm/Tsuh4w1YYdDqDb31a8TVhBJ2OJKdqQ==}
dependencies:
punycode: 1.4.1
dev: false
/fastest-stable-stringify/2.0.2:
resolution: {integrity: sha512-bijHueCGd0LqqNK9b5oCMHc0MluJAx0cwqASgbWMvkO01lCYgIhacVRLcaDz3QnyYIRNJRDwMb41VuT6pHJ91Q==}
dev: false
@@ -21118,12 +21148,6 @@ packages:
/globrex/0.1.2:
resolution: {integrity: sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg==}
/goober/2.1.11:
resolution: {integrity: sha512-5SS2lmxbhqH0u9ABEWq7WPU69a4i2pYcHeCxqaNq6Cw3mnrF0ghWNM4tEGid4dKy8XNIAUbuThuozDHHKJVh3A==}
peerDependencies:
csstype: ^3.0.10
dev: false
/gopd/1.0.1:
resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==}
dependencies:
@@ -27340,20 +27364,6 @@ packages:
- supports-color
dev: false
/react-hot-toast/2.4.0_biqbaboplfbrettd7655fr4n2y:
resolution: {integrity: sha512-qnnVbXropKuwUpriVVosgo8QrB+IaPJCpL8oBI6Ov84uvHZ5QQcTp2qg6ku2wNfgJl6rlQXJIQU5q+5lmPOutA==}
engines: {node: '>=10'}
peerDependencies:
react: '>=16'
react-dom: '>=16'
dependencies:
goober: 2.1.11
react: 18.2.0
react-dom: 18.2.0_react@18.2.0
transitivePeerDependencies:
- csstype
dev: false
/react-hotkeys-hook/4.4.1_biqbaboplfbrettd7655fr4n2y:
resolution: {integrity: sha512-sClBMBioFEgFGYLTWWRKvhxcCx1DRznd+wkFHwQZspnRBkHTgruKIHptlK/U/2DPX8BhHoRGzpMVWUXMmdZlmw==}
peerDependencies:
@@ -27982,49 +27992,49 @@ packages:
retext-smartypants: 5.2.0
unist-util-visit: 4.1.2
/remix-auth-email-link/1.5.2_7ym4eehrzvo4xt5yocgqipdaji:
resolution: {integrity: sha512-LCbaYkhsENbRK70O36Ul/jBtvNcNdBabjOIXYhrTNwx4qYA/fFkX8Sr3GMuDhoMzHnXQy7zHtsqbg/DTHocX6g==}
/remix-auth-email-link/2.1.0_esw6v6dv7ru6ju3hn3i62b7n2m:
resolution: {integrity: sha512-1Hp4QDqN17tEEfiRd/6g1XDw6o6y+lyeeseDc0EGrR4vZcR7HLPR8ImfEtxplYpXlFSpOO7mzl6GgJprynjgcg==}
peerDependencies:
'@remix-run/server-runtime': ^1.1.1
remix-auth: ^3.2.1
'@remix-run/server-runtime': ^2.0.1
remix-auth: ^3.6.0
dependencies:
'@remix-run/server-runtime': 2.1.0_typescript@5.2.2
crypto-js: 4.1.1
remix-auth: 3.4.0_ybjp5xbtg4zthziheradfmei64
remix-auth: 3.6.0_ybjp5xbtg4zthziheradfmei64
yarn: 1.22.19
dev: false
/remix-auth-github/1.3.0_7ym4eehrzvo4xt5yocgqipdaji:
resolution: {integrity: sha512-dqveIbXc1yXQXiB8UVjKSR3qgHFUZ0VJOu2/GTT5uMKZY0//FSnp1w8gK0btGlbc1HW97gig16uS84et6FBMiA==}
/remix-auth-github/1.6.0_esw6v6dv7ru6ju3hn3i62b7n2m:
resolution: {integrity: sha512-qdQmWVVEDHxnzMPn7XE0s7QX5GYba0uH3MDW+lVJiCoHDZCGymqmGCajLThL5vFJdwSx0C4GpQJ5EgEsIjQ3pA==}
peerDependencies:
'@remix-run/server-runtime': ^1.0.0
remix-auth: ^3.4.0
dependencies:
'@remix-run/server-runtime': 2.1.0_typescript@5.2.2
remix-auth: 3.4.0_ybjp5xbtg4zthziheradfmei64
remix-auth-oauth2: 1.5.0_7ym4eehrzvo4xt5yocgqipdaji
remix-auth: 3.6.0_ybjp5xbtg4zthziheradfmei64
remix-auth-oauth2: 1.11.0_esw6v6dv7ru6ju3hn3i62b7n2m
transitivePeerDependencies:
- supports-color
dev: false
/remix-auth-oauth2/1.5.0_7ym4eehrzvo4xt5yocgqipdaji:
resolution: {integrity: sha512-vv/CX2bCeVf8vzACfS1Fae+i75Cw0/As3RLkp00+9n9IBEvKi07LhUxTbODNA51jRh+wInnmCRIwsHafKQDD6g==}
/remix-auth-oauth2/1.11.0_esw6v6dv7ru6ju3hn3i62b7n2m:
resolution: {integrity: sha512-Yf1LF6NLYPFa7X2Rax/VEhXmYXFjZOi/q+7DmbMeoMHjAfkpqxbvzqqYSKIKDGR51z5TXR5na4to4380mir5bg==}
peerDependencies:
'@remix-run/server-runtime': ^1.0.0
remix-auth: ^3.2.2
'@remix-run/server-runtime': ^1.0.0 || ^2.0.0
remix-auth: ^3.6.0
dependencies:
'@remix-run/server-runtime': 2.1.0_typescript@5.2.2
debug: 4.3.4
remix-auth: 3.4.0_ybjp5xbtg4zthziheradfmei64
uuid: 8.3.2
remix-auth: 3.6.0_ybjp5xbtg4zthziheradfmei64
transitivePeerDependencies:
- supports-color
dev: false
/remix-auth/3.4.0_ybjp5xbtg4zthziheradfmei64:
resolution: {integrity: sha512-VRliJo9VRAS4sSYgMjYbi7rYRixPWU2Tb8PFZb06OIj0nONK/1KRzHf2+Y6VGZeIacepLVgan6g2IUJjayE2rQ==}
/remix-auth/3.6.0_ybjp5xbtg4zthziheradfmei64:
resolution: {integrity: sha512-mxlzLYi+/GKQSaXIqIw15dxAT1wm+93REAeDIft2unrKDYnjaGhhpapyPhdbALln86wt9lNAk21znfRss3fG7Q==}
peerDependencies:
'@remix-run/react': ^1.0.0
'@remix-run/server-runtime': ^1.0.0
'@remix-run/react': ^1.0.0 || ^2.0.0
'@remix-run/server-runtime': ^1.0.0 || ^2.0.0
dependencies:
'@remix-run/react': 2.1.0_vegmnuvoswdxuttpuhvewcem44
'@remix-run/server-runtime': 2.1.0_typescript@5.2.2
@@ -31468,6 +31478,10 @@ packages:
qs: 6.11.0
dev: false
/urlpattern-polyfill/9.0.0:
resolution: {integrity: sha512-WHN8KDQblxd32odxeIgo83rdVDE2bvdkb86it7bMhYZwWKJz0+O0RK/eZiHYnM+zgt/U7hAHOlCQGfjjvSkw2g==}
dev: false
/use-callback-ref/1.3.0_e74vmjybjy5dsfplslbsgtbvvi:
resolution: {integrity: sha512-3FT9PRuRdbB9HfXhEq35u4oZkvpJ5kuYbpqhCfmiZyReuRgpnhDlbr2ZEnnuS0RrJAPn6l23xjFg9kpDM+Ms7w==}
engines: {node: '>=10'}
@@ -32683,6 +32697,13 @@ packages:
y18n: 5.0.8
yargs-parser: 21.1.1
/yarn/1.22.19:
resolution: {integrity: sha512-/0V5q0WbslqnwP91tirOvldvYISzaqhClxzyUKXYxs07yUILIs5jx/k6CFe8bvKSkds5w+eiOqta39Wk3WxdcQ==}
engines: {node: '>=4.0.0'}
hasBin: true
requiresBuild: true
dev: false
/yauzl/2.10.0:
resolution: {integrity: sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==}
dependencies:
references/job-catalog/package.json
+2 -1
View File
@@ -29,6 +29,7 @@
"misconfigured": "nodemon --watch src/misconfigured.ts -r tsconfig-paths/register -r dotenv/config src/misconfigured.ts",
"auto-yield": "nodemon --watch src/auto-yield.ts -r tsconfig-paths/register -r dotenv/config src/auto-yield.ts",
"cli-example": "nodemon --watch src/cli-example.ts -r tsconfig-paths/register -r dotenv/config src/cli-example.ts",
"invoke": "nodemon --watch src/invoke.ts -r tsconfig-paths/register -r dotenv/config src/invoke.ts",
"dev:trigger": "trigger-cli dev --port 8080"
},
"dependencies": {
@@ -63,4 +64,4 @@
"ts-node": "^10.9.1",
"tsconfig-paths": "^3.14.1"
}
}
}
references/job-catalog/src/invoke.ts
+313
View File
@@ -0,0 +1,313 @@
import { createExpressServer } from "@trigger.dev/express";
import { OpenAI } from "@trigger.dev/openai";
import { TriggerClient, invokeTrigger } from "@trigger.dev/sdk";
import { z } from "zod";
export const client = new TriggerClient({
id: "job-catalog",
apiKey: process.env["TRIGGER_API_KEY"],
apiUrl: process.env["TRIGGER_API_URL"],
verbose: true,
ioLogLocalEnabled: true,
});
const invocableJob = client.defineJob({
id: "invoke-example-1",
name: "Invoke Example 1",
version: "1.0.0",
enabled: true,
trigger: invokeTrigger({
schema: z.object({
message: z.string(),
forceError: z.boolean().default(false),
delay: z.number().default(5),
}),
}),
run: async (payload, io, ctx) => {
const generatingMemes = await io.createStatus("status-1", {
//the label is compulsory on this first call
label: "Generating memes",
//state is optional
state: "loading",
//data is an optional object. the values can be any type that is JSON serializable
data: {
progress: 0.1,
},
});
await io.runTask("task-1", async () => {
await new Promise((resolve) => setTimeout(resolve, 1000));
});
await io.wait("wait-1", payload.delay);
await io.runTask("task-2", async () => {
await new Promise((resolve) => setTimeout(resolve, 1000));
return [{ hello: "there", ts: new Date() }];
});
await generatingMemes.update("middle-generation", {
//label isn't specified so will remain the same
//state will be updated to "success"
state: "success",
//set data, this overrides the previous value
data: {
progress: 1,
urls: [
"https://media.giphy.com/media/v1.Y2lkPTc5MGI3NjExZnZoMndsdWh0MmhvY2kyaDF6YjZjZzg1ZGsxdnhhYm13a3Q1Y3lkbyZlcD12MV9pbnRlcm5hbF9naWZfYnlfaWQmY3Q9Zw/13HgwGsXF0aiGY/giphy.gif",
],
},
});
if (payload.forceError) {
throw new Error("Forced error");
}
const response = await io.runTask("fetch-json", async () =>
fetch("https://jsonhero.io/j/PjHo1o5MVeH4.json").then((r) => r.json())
);
return response;
},
});
client.defineJob({
id: "invoke-example-2",
name: "Invoke Example 2",
version: "1.0.0",
enabled: true,
trigger: invokeTrigger(),
run: async (payload, io, ctx) => {
await invocableJob.invoke("invoke", {
message: "Hello World 1",
});
await invocableJob.invoke(
"invoke-with-url",
{
message: "Hello World 2",
},
{
callbackUrl: process.env.REQUEST_BIN_URL,
}
);
const result = await invocableJob.invokeAndWaitForCompletion("invoke-and-wait", {
message: "Hello World 3",
forceError: true,
});
if (result.ok) {
await io.logger.info("Invoking job worked!", { result });
} else {
await io.logger.error("Invoking job failed!", { result });
}
},
});
const simpleInvokableJob = client.defineJob({
id: "simple-invoke-example-1",
name: "Simple Invoke Example 1",
version: "1.0.0",
enabled: true,
trigger: invokeTrigger(),
run: async (payload, io, ctx) => {
return payload;
},
});
const openai = new OpenAI({
id: "openai",
apiKey: process.env["OPENAI_API_KEY"]!,
});
const perplexity = new OpenAI({
id: "perplexity",
apiKey: process.env["PERPLEXITY_API_KEY"]!,
baseURL: "https://api.perplexity.ai",
icon: "brand-open-source",
});
// This job performs a chat completion using either OpenAI or Perplexity, depending on the model passed in
const completionJob = client.defineJob({
id: "openai-job",
name: "OpenAI Job",
version: "1.0.0",
enabled: true,
trigger: invokeTrigger({
schema: z.object({
model: z.string().default("gpt-3.5-turbo"),
prompt: z.string(),
}),
}),
integrations: {
openai,
perplexity,
},
run: async (payload, io, ctx) => {
if (payload.model === "gpt-3.5-turbo") {
return await io.openai.chat.completions.backgroundCreate(
"background-chat-completion",
{
model: payload.model,
messages: [
{
role: "user",
content: payload.prompt,
},
],
},
{},
{
// Set a timeout of 30 seconds, and retry it up to 3 times
timeout: {
durationInMs: 30000,
retry: {
limit: 3,
minTimeoutInMs: 1000,
factor: 2,
},
},
}
);
} else {
return await io.perplexity.completions.backgroundCreate(
"background-completion",
{
model: payload.model,
prompt: payload.prompt,
},
{},
{
// Set a timeout of 30 seconds, and retry it up to 3 times
timeout: {
durationInMs: 30000,
retry: {
limit: 3,
minTimeoutInMs: 1000,
factor: 2,
},
},
}
);
}
},
});
const prompts = [
"Advantages of quantum computing over classical computing",
"The ethical implications of advanced artificial intelligence",
"Design a thought experiment highlighting the paradoxes in quantum mechanics",
"Explain the Fermi Paradox, its potential solutions, and implications for humanity",
"Analyze Shakespeare's use of iambic pentameter in his plays",
];
client.defineJob({
id: "batch-invoke-ai-example",
name: "Batch Invoke OpenAI Example",
version: "1.0.0",
trigger: invokeTrigger(),
run: async (payload, io, ctx) => {
// This will invoke the completionJob in parallel for each prompt, first trying OpenAI
const runs = await completionJob.batchInvokeAndWaitForCompletion(
"batch-invoke-and-wait",
prompts.map((prompt) => ({
payload: {
model: "gpt-3.5-turbo",
prompt,
},
}))
);
// Runs are returned in the same order as the prompts
const failedRuns = runs.map((run, i) => ({ run, prompt: prompts[i] })).filter((r) => !r.run.ok);
// Run the failed runs with Perplexity by specifying the "mistral-7b-instruct" model
const retriedRuns = await completionJob.batchInvokeAndWaitForCompletion(
"batch-invoke-and-wait",
failedRuns.map((failedRun) => ({
payload: {
model: "mistral-7b-instruct",
prompt: failedRun.prompt,
},
}))
);
const failedPerplexityRuns = retriedRuns
.map((run, i) => ({ run, prompt: failedRuns[i].prompt }))
.filter((r) => !r.run.ok);
// And so on and so forth
},
});
client.defineJob({
id: "batch-invoke-example",
name: "Batch Invoke Example",
version: "1.0.0",
trigger: invokeTrigger(),
run: async (payload, io, ctx) => {
await invocableJob.batchInvokeAndWaitForCompletion(
"batch-invoke-and-wait",
Array.from({ length: 2 }).map((_, i) => ({
payload: {
message: `Hello World ${i}`,
delay: i % 2 === 0 ? 7 : 20,
},
}))
);
await simpleInvokableJob.batchInvokeAndWaitForCompletion(
"batch-invoke-and-wait-simple",
Array.from({ length: 25 }).map((_, i) => ({
payload: {
message: `Hello World ${i}`,
},
options: {
context: {
i,
},
accountId: "FB1C6C79-6C82-45B6-A8AA-207ADA9EE838",
},
}))
);
await simpleInvokableJob.batchInvokeAndWaitForCompletion(
"batch-invoke-and-wait-2",
Array.from({ length: 2 }).map((_, i) => ({
payload: {
message: `Hello World ${i}`,
},
}))
);
},
});
export const exampleJob = client.defineJob({
id: "example-job",
name: "Example job",
version: "1.0.1",
trigger: invokeTrigger({
//the expected payload shape
schema: z.object({
userId: z.string(),
tier: z.enum(["free", "pro"]),
}),
}),
run: async (payload, io, ctx) => {
// payload is typed as { userId: string, tier: "free" | "pro" }
},
});
client.defineJob({
id: "example-job2",
name: "Example job 2",
version: "1.0.1",
trigger: invokeTrigger(),
run: async (payload, io, ctx) => {
const jobRun = await exampleJob.invoke("⚡", { userId: "123", tier: "free" });
},
});
createExpressServer(client);