Implement manually invokable jobs through the invokeTrigger (#700)

* Implement manually invokable jobs through the invokeTrigger

Also implemented a job run notification system, that will POST details of a run on completion. This combines with the task callbackUrl system to implement the invokeAndWait

* Document the invoke trigger

* batch invoke and wait

* background fetch timeouts

* Use @whatwg-node/fetch instead of the polyfilled fetch

* Fix some outdated dependencies in webapp

* Improved subtask error propogation messages

* Document the OpenAI changes and the batch invoke stuff

* Fix dequeuing jobs

* Don’t retry the OpenAI completion background task

* Added OpenAI changesets

* Use the new ResumeTaskService in ProcessCallbackTimeout as well

.changeset/happy-dryers-guess.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Added invokeTrigger(), which allows jobs to be manually invoked

.changeset/lazy-schools-grin.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/integration-kit": patch
+"@trigger.dev/openai": patch
+---
+
+Allow customizing OpenAI background retries and timeouts

.gitignore

@@ -52,3 +52,5 @@ apps/**/public/build
 /test-results/
 /playwright-report/
 /playwright/.cache/
+
+.cosine

apps/webapp/app/components/primitives/LabelValueStack.tsx

@@ -2,6 +2,7 @@ import { cn } from "~/utils/cn";
 import { Paragraph } from "./Paragraph";
 import { ArrowTopRightOnSquareIcon } from "@heroicons/react/20/solid";
 import { SimpleTooltip } from "./Tooltip";
+import { Link } from "@remix-run/react";
 
 const variations = {
   primary: {
@@ -45,18 +46,7 @@ export function LabelValueStack({
       <Paragraph variant={variation.label}>{label}</Paragraph>
       <>
         {href ? (
-          <SimpleTooltip
+          <ValueButton value={value} href={href} variant={variant} />
-            side="bottom"
-            button={
-              <Paragraph variant={variation.value}>
-                <a href={href} className="underline underline-offset-2" target="_blank">
-                  {value}
-                  <ArrowTopRightOnSquareIcon className="ml-1 inline-block h-4 w-4 text-dimmed" />
-                </a>
-              </Paragraph>
-            }
-            content={href}
-          />
         ) : (
           <Paragraph variant={variation.value}>{value}</Paragraph>
         )}
@@ -64,3 +54,40 @@ export function LabelValueStack({
     </div>
   );
 }
+
+type ValueButtonStackProps = {
+  value: React.ReactNode;
+  href: string;
+  variant?: keyof typeof variations;
+};
+
+function ValueButton({ value, href, variant = "secondary" }: ValueButtonStackProps) {
+  const variation = variations[variant];
+
+  const isExternalUrl = href.startsWith("http");
+
+  if (!isExternalUrl) {
+    return (
+      <Paragraph variant={variation.value}>
+        <Link to={href} reloadDocument className="underline underline-offset-2">
+          {value}
+        </Link>
+      </Paragraph>
+    );
+  }
+
+  return (
+    <SimpleTooltip
+      side="bottom"
+      button={
+        <Paragraph variant={variation.value}>
+          <a href={href} className="underline underline-offset-2" target="_blank">
+            {value}
+            <ArrowTopRightOnSquareIcon className="ml-1 inline-block h-4 w-4 text-dimmed" />
+          </a>
+        </Paragraph>
+      }
+      content={href}
+    />
+  );
+}

apps/webapp/app/db.server.ts

@@ -3,6 +3,7 @@ import invariant from "tiny-invariant";
 import { z } from "zod";
 import { logger } from "./services/logger.server";
 import { env } from "./env.server";
+import { singleton } from "./utils/singleton";
 
 export type PrismaTransactionClient = Omit<
   PrismaClient,
@@ -66,24 +67,7 @@ export async function $transaction<R>(
 
 export { Prisma };
 
-let prisma: PrismaClient;
+export const prisma = singleton("prisma", getClient);
-
-declare global {
-  var __db__: PrismaClient;
-}
-
-// this is needed because in development we don't want to restart
-// the server with every change, but we want to make sure we don't
-// create a new connection to the DB with every change either.
-// in production we'll have a single connection to the DB.
-if (process.env.NODE_ENV === "production") {
-  prisma = getClient();
-} else {
-  if (!global.__db__) {
-    global.__db__ = getClient();
-  }
-  prisma = global.__db__;
-}
 
 function getClient() {
   const { DATABASE_URL } = process.env;
@@ -143,7 +127,6 @@ function getClient() {
   return client;
 }
 
-export { prisma };
 export type { PrismaClient } from "@trigger.dev/database";
 
 export const PrismaErrorSchema = z.object({

apps/webapp/app/models/jobRunExecution.server.ts

@@ -1,41 +1,47 @@
-import { JobRun, JobRunExecution } from "@trigger.dev/database";
+import { JobRun } from "@trigger.dev/database";
 import { PrismaClientOrTransaction } from "~/db.server";
 import { executionWorker } from "~/services/worker.server";
 
-export type EnqueueRunExecutionV2Options = {
-  runAt?: Date;
-  resumeTaskId?: string;
-  isRetry?: boolean;
-  skipRetrying?: boolean;
-  executionCount?: number;
-};
-
-export async function enqueueRunExecutionV2(
-  run: JobRun,
-  tx: PrismaClientOrTransaction,
-  options: EnqueueRunExecutionV2Options = {}
-) {
-  const job = await executionWorker.enqueue(
-    "performRunExecutionV2",
-    {
-      id: run.id,
-      reason: run.status === "PREPROCESSING" ? "PREPROCESS" : "EXECUTE_JOB",
-      resumeTaskId: options.resumeTaskId,
-      isRetry: typeof options.isRetry === "boolean" ? options.isRetry : false,
-    },
-    {
-      tx,
-      runAt: options.runAt,
-      jobKey: `job_run:${run.id}:${options.executionCount ?? 0}${
-        options.resumeTaskId ? `:task:${options.resumeTaskId}` : ""
-      }`,
-      maxAttempts: options.skipRetrying ? 1 : undefined,
-    }
-  );
-}
-
 export async function dequeueRunExecutionV2(run: JobRun, tx: PrismaClientOrTransaction) {
   return await executionWorker.dequeue(`job_run:${run.id}`, {
     tx,
   });
 }
+
+export type EnqueueRunExecutionV3Options = {
+  runAt?: Date;
+  skipRetrying?: boolean;
+};
+
+export async function enqueueRunExecutionV3(
+  run: JobRun,
+  tx: PrismaClientOrTransaction,
+  options: EnqueueRunExecutionV3Options = {}
+) {
+  const reason = run.status === "PREPROCESSING" ? "PREPROCESS" : "EXECUTE_JOB";
+
+  return await executionWorker.enqueue(
+    "performRunExecutionV3",
+    {
+      id: run.id,
+      reason: reason,
+    },
+    {
+      tx,
+      runAt: options.runAt,
+      queueName: `job_run:${run.id}`,
+      jobKey: `job_run:${reason}:${run.id}`,
+      maxAttempts: options.skipRetrying ? 1 : undefined,
+    }
+  );
+}
+
+export async function dequeueRunExecutionV3(run: JobRun, tx: PrismaClientOrTransaction) {
+  await executionWorker.dequeue(`job_run:EXECUTE_JOB:${run.id}`, {
+    tx,
+  });
+
+  await executionWorker.dequeue(`job_run:PREPROCESS:${run.id}`, {
+    tx,
+  });
+}

apps/webapp/app/models/task.server.ts

@@ -25,6 +25,7 @@ export function taskWithAttemptsToServerTask(task: TaskWithAttempts): ServerTask
     operation: task.operation,
     callbackUrl: task.callbackUrl,
     forceYield: task.run.forceYieldImmediately,
+    childExecutionMode: task.childExecutionMode,
   };
 }
 

apps/webapp/app/platform/zodWorker.server.ts

@@ -14,9 +14,8 @@ import { run as graphileRun, parseCronItems } from "graphile-worker";
 import omit from "lodash.omit";
 import { z } from "zod";
 import { PrismaClient, PrismaClientOrTransaction } from "~/db.server";
-import { workerLogger as logger } from "~/services/logger.server";
 import { PgListenService } from "~/services/db/pgListen.server";
-import { safeJsonParse } from "~/utils/json";
+import { workerLogger as logger } from "~/services/logger.server";
 
 export interface MessageCatalogSchema {
   [key: string]: z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion<any, any>;

apps/webapp/app/routes/api.v1.endpoints.ts

@@ -60,7 +60,7 @@ export async function action({ request }: ActionFunctionArgs) {
     return json(endpoint);
   } catch (error) {
     if (error instanceof Error) {
-      logger.error("Error creating endpoint", {
+      logger.debug("Error creating endpoint", {
         url: request.url,
         error: error.message,
       });

apps/webapp/app/routes/api.v1.jobs.$jobSlug.invoke.ts

@@ -0,0 +1,88 @@
+import type { ActionFunctionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import { InvokeJobRequestBodySchema } from "@trigger.dev/core";
+import { z } from "zod";
+import { PrismaErrorSchema } from "~/db.server";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { InvokeJobService } from "~/services/jobs/invokeJob.server";
+import { logger } from "~/services/logger.server";
+
+const ParamsSchema = z.object({
+  jobSlug: z.string(),
+});
+
+const HeadersSchema = z.object({
+  "idempotency-key": z.string().optional().nullable(),
+  "trigger-version": z.string().optional().nullable(),
+});
+
+export async function action({ request, params }: ActionFunctionArgs) {
+  // Ensure this is a POST request
+  if (request.method.toUpperCase() !== "POST") {
+    return { status: 405, body: "Method Not Allowed" };
+  }
+
+  // Authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API Key" }, { status: 401 });
+  }
+
+  const parsed = ParamsSchema.safeParse(params);
+
+  if (!parsed.success) {
+    return json({ error: "Invalid or Missing jobSlug" }, { status: 400 });
+  }
+
+  const { jobSlug } = parsed.data;
+
+  const headers = HeadersSchema.safeParse(Object.fromEntries(request.headers));
+
+  if (!headers.success) {
+    return json({ error: "Invalid headers" }, { status: 400 });
+  }
+
+  const { "idempotency-key": idempotencyKey, "trigger-version": triggerVersion } = headers.data;
+
+  // Now parse the request body
+  const anyBody = await request.json();
+
+  logger.debug("InvokeJobService.call() request body", {
+    body: anyBody,
+    jobSlug,
+    idempotencyKey,
+    triggerVersion,
+  });
+
+  const body = InvokeJobRequestBodySchema.safeParse(anyBody);
+
+  if (!body.success) {
+    return json({ error: "Invalid request body" }, { status: 400 });
+  }
+
+  const service = new InvokeJobService();
+
+  try {
+    const run = await service.call(
+      authenticationResult.environment,
+      jobSlug,
+      body.data,
+      idempotencyKey ?? undefined
+    );
+
+    if (!run) {
+      return json({ error: "Job count not be invoked" }, { status: 500 });
+    }
+
+    return json({ id: run.id });
+  } catch (error) {
+    const prismaError = PrismaErrorSchema.safeParse(error);
+    // Record not found in the database
+    if (prismaError.success && prismaError.data.code === "P2005") {
+      return json({ error: "Job not found" }, { status: 404 });
+    } else {
+      return json({ error: "Internal Server Error" }, { status: 500 });
+    }
+  }
+}

apps/webapp/app/routes/api.v1.runs.$runId.tasks.$id.callback.$secret.ts

@@ -3,8 +3,9 @@ import { json } from "@remix-run/server-runtime";
 import { RuntimeEnvironmentType } from "@trigger.dev/database";
 import { z } from "zod";
 import { $transaction, PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server";
-import { enqueueRunExecutionV2 } from "~/models/jobRunExecution.server";
 import { logger } from "~/services/logger.server";
+import { ResumeTaskService } from "~/services/tasks/resumeTask.server";
+import { workerQueue } from "~/services/worker.server";
 
 const ParamsSchema = z.object({
   runId: z.string(),
@@ -96,14 +97,14 @@ export class CallbackRunTaskService {
         },
       });
 
+      await workerQueue.dequeue(`process-callback:${task.id}`, { tx });
+
       await this.#resumeRunExecution(task, tx);
     });
   }
 
   async #resumeRunExecution(task: NonNullable<FoundTask>, prisma: PrismaClientOrTransaction) {
-    await enqueueRunExecutionV2(task.run, prisma, {
+    await ResumeTaskService.enqueue(task.id, undefined, prisma);
-      skipRetrying: task.run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
-    });
   }
 }
 

apps/webapp/app/routes/api.v1.runs.$runId.tasks.$id.complete.ts

@@ -1,9 +1,13 @@
 import type { ActionFunctionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
 import type { CompleteTaskBodyOutput, ServerTask } from "@trigger.dev/core";
-import { CompleteTaskBodyInputSchema } from "@trigger.dev/core";
+import {
+  API_VERSIONS,
+  CompleteTaskBodyInputSchema,
+  CompleteTaskBodyV2InputSchema,
+} from "@trigger.dev/core";
 import { z } from "zod";
-import { $transaction, PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server";
+import { PrismaClientOrTransaction, prisma } from "~/db.server";
 import { taskWithAttemptsToServerTask } from "~/models/task.server";
 import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
@@ -14,6 +18,10 @@ const ParamsSchema = z.object({
   id: z.string(),
 });
 
+const HeadersSchema = z.object({
+  "trigger-version": z.string().optional().nullable(),
+});
+
 export async function action({ request, params }: ActionFunctionArgs) {
   // Ensure this is a POST request
   if (request.method.toUpperCase() !== "POST") {
@@ -31,6 +39,14 @@ export async function action({ request, params }: ActionFunctionArgs) {
 
   const { runId, id } = ParamsSchema.parse(params);
 
+  const headers = HeadersSchema.safeParse(Object.fromEntries(request.headers));
+
+  if (!headers.success) {
+    return json({ error: "Invalid headers" }, { status: 400 });
+  }
+
+  const { "trigger-version": triggerVersion } = headers.data;
+
   // Now parse the request body
   const anyBody = await request.json();
 
@@ -40,16 +56,48 @@ export async function action({ request, params }: ActionFunctionArgs) {
     id,
   });
 
-  const body = CompleteTaskBodyInputSchema.safeParse(anyBody);
+  if (triggerVersion === API_VERSIONS.SERIALIZED_TASK_OUTPUT) {
+    const body = CompleteTaskBodyV2InputSchema.safeParse(anyBody);
 
-  if (!body.success) {
+    if (!body.success) {
-    return json({ error: "Invalid request body" }, { status: 400 });
+      return json({ error: "Invalid request body" }, { status: 400 });
+    }
+
+    // Make sure the length of the output is less than 3MB
+    if (body.data.output && body.data.output.length > 3 * 1024 * 1024) {
+      return json({ error: "Output must be less than 3MB" }, { status: 400 });
+    }
+
+    return await completeRunTask(authenticatedEnv, runId, id, {
+      ...body.data,
+      output: body.data.output ? (JSON.parse(body.data.output) as any) : undefined,
+    });
+  } else {
+    const body = CompleteTaskBodyInputSchema.safeParse(anyBody);
+
+    if (!body.success) {
+      return json({ error: "Invalid request body" }, { status: 400 });
+    }
+
+    // Make sure the length of the output is less than 3MB
+    if (JSON.stringify(body.data.output).length > 3 * 1024 * 1024) {
+      return json({ error: "Output must be less than 3MB" }, { status: 400 });
+    }
+
+    return await completeRunTask(authenticatedEnv, runId, id, body.data);
   }
+}
 
+async function completeRunTask(
+  environment: AuthenticatedEnvironment,
+  runId: string,
+  id: string,
+  taskBody: CompleteTaskBodyOutput
+) {
   const service = new CompleteRunTaskService();
 
   try {
-    const task = await service.call(authenticatedEnv, runId, id, body.data);
+    const task = await service.call(environment, runId, id, taskBody);
 
     logger.debug("CompleteRunTaskService.call() response body", {
       runId,
@@ -84,79 +132,75 @@ export class CompleteRunTaskService {
     id: string,
     taskBody: CompleteTaskBodyOutput
   ): Promise<ServerTask | undefined> {
-    // Using a transaction, we'll first check to see if the task already exists and return if if it does
+    const existingTask = await this.#prismaClient.task.findUnique({
-    // If it doesn't exist, we'll create it and return it
+      where: {
-    const task = await $transaction(this.#prismaClient, async (tx) => {
+        id,
-      const existingTask = await tx.task.findUnique({
+      },
-        where: {
+      include: {
-          id,
+        run: true,
-        },
+        attempts: {
-        include: {
+          where: {
-          run: true,
+            status: "PENDING",
-          attempts: {
-            where: {
-              status: "PENDING",
-            },
-            orderBy: {
-              number: "desc",
-            },
-            take: 1,
           },
+          orderBy: {
+            number: "desc",
+          },
+          take: 1,
         },
+      },
+    });
+
+    if (!existingTask) {
+      return;
+    }
+
+    if (existingTask.runId !== runId) {
+      return;
+    }
+
+    if (existingTask.run.environmentId !== environment.id) {
+      return;
+    }
+
+    if (
+      existingTask.status === "COMPLETED" ||
+      existingTask.status === "ERRORED" ||
+      existingTask.status === "CANCELED"
+    ) {
+      logger.debug("Task already completed", {
+        existingTask,
       });
 
-      if (!existingTask) {
+      return taskWithAttemptsToServerTask(existingTask);
-        return;
+    }
-      }
 
-      if (existingTask.runId !== runId) {
+    if (existingTask.attempts.length === 1) {
-        return;
+      await this.#prismaClient.taskAttempt.update({
-      }
-
-      if (existingTask.run.environmentId !== environment.id) {
-        return;
-      }
-
-      if (
-        existingTask.status === "COMPLETED" ||
-        existingTask.status === "ERRORED" ||
-        existingTask.status === "CANCELED"
-      ) {
-        logger.debug("Task already completed", {
-          existingTask,
-        });
-
-        return existingTask;
-      }
-
-      if (existingTask.attempts.length === 1) {
-        await tx.taskAttempt.update({
-          where: {
-            id: existingTask.attempts[0].id,
-          },
-          data: {
-            status: "COMPLETED",
-          },
-        });
-      }
-
-      return await tx.task.update({
         where: {
-          id,
+          id: existingTask.attempts[0].id,
         },
         data: {
           status: "COMPLETED",
-          output: taskBody.output ?? undefined,
-          completedAt: new Date(),
-          outputProperties: taskBody.properties,
-        },
-        include: {
-          attempts: true,
-          run: true,
         },
       });
+    }
+
+    const updatedTask = await this.#prismaClient.task.update({
+      where: {
+        id,
+      },
+      data: {
+        status: "COMPLETED",
+        output: taskBody.output ?? undefined,
+        completedAt: new Date(),
+        outputProperties: taskBody.properties,
+      },
+      include: {
+        attempts: true,
+        run: true,
+      },
     });
 
-    return task ? taskWithAttemptsToServerTask(task) : undefined;
+    return taskWithAttemptsToServerTask(updatedTask);
   }
 }

apps/webapp/app/routes/api.v1.runs.$runId.tasks.ts

@@ -286,6 +286,7 @@ export class RunTaskService {
           operation: taskBody.operation,
           callbackUrl,
           style: taskBody.style ?? { style: "normal" },
+          childExecutionMode: taskBody.parallel ? "PARALLEL" : "SEQUENTIAL",
           attempts: {
             create: {
               number: 1,
@@ -316,7 +317,11 @@ export class RunTaskService {
             {
               id: task.id,
             },
-            { tx, runAt: new Date(Date.now() + taskBody.callback.timeoutInSeconds * 1000) }
+            {
+              tx,
+              runAt: new Date(Date.now() + taskBody.callback.timeoutInSeconds * 1000),
+              jobKey: `process-callback:${task.id}`,
+            }
           );
         }
       }

apps/webapp/app/services/email.server.ts

@@ -7,13 +7,18 @@ import type { User } from "~/models/user.server";
 import type { AuthUser } from "./authUser";
 import { workerQueue } from "./worker.server";
 import { logger } from "./logger.server";
+import { singleton } from "~/utils/singleton";
 
-const client = new EmailClient({
+const client = singleton(
-  apikey: env.RESEND_API_KEY,
+  "email-client",
-  imagesBaseUrl: env.APP_ORIGIN,
+  () =>
-  from: env.FROM_EMAIL ?? "team@email.trigger.dev",
+    new EmailClient({
-  replyTo: env.REPLY_TO_EMAIL ?? "help@email.trigger.dev",
+      apikey: env.RESEND_API_KEY,
-});
+      imagesBaseUrl: env.APP_ORIGIN,
+      from: env.FROM_EMAIL ?? "team@email.trigger.dev",
+      replyTo: env.REPLY_TO_EMAIL ?? "help@email.trigger.dev",
+    })
+);
 
 export async function sendMagicLinkEmail(options: SendEmailOptions<AuthUser>): Promise<void> {
   // Auto redirect when in development mode

apps/webapp/app/services/endpointApi.server.ts

@@ -1,8 +1,6 @@
 import {
   API_VERSIONS,
-  ApiEventLog,
+  ConnectionAuth,
-  DeliverEventResponseSchema,
-  DeserializedJson,
   EndpointHeadersSchema,
   ErrorWithStackSchema,
   HttpSourceRequest,
@@ -19,10 +17,9 @@ import {
   ValidateResponse,
   ValidateResponseSchema,
 } from "@trigger.dev/core";
+import { performance } from "node:perf_hooks";
 import { safeBodyFromResponse, safeParseBodyFromResponse } from "~/utils/json";
 import { logger } from "./logger.server";
-import { ConnectionAuth } from "@trigger.dev/core";
-import { performance } from "node:perf_hooks";
 
 export class EndpointApiError extends Error {
   constructor(message: string, stack?: string) {

apps/webapp/app/services/jobs/invokeJob.server.ts

@@ -0,0 +1,143 @@
+import { $transaction, PrismaClient, prisma } from "~/db.server";
+import { AuthenticatedEnvironment } from "../apiAuth.server";
+import { InvokeJobRequestBody } from "@trigger.dev/core";
+import { ulid } from "../ulid.server";
+import { CreateRunService } from "../runs/createRun.server";
+
+export class InvokeJobService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(
+    environment: AuthenticatedEnvironment,
+    jobSlug: string,
+    data: InvokeJobRequestBody,
+    idempotencyKey?: string
+  ) {
+    return await $transaction(this.#prismaClient, async (tx) => {
+      // Check if this is an idempotent request
+      if (idempotencyKey) {
+        const existingEvent = await tx.eventRecord.findUnique({
+          where: {
+            eventId_environmentId: {
+              eventId: idempotencyKey,
+              environmentId: environment.id,
+            },
+          },
+          include: {
+            runs: true,
+          },
+        });
+
+        if (existingEvent) {
+          return existingEvent.runs[0];
+        }
+      }
+
+      const job = await tx.job.findUniqueOrThrow({
+        where: {
+          projectId_slug: {
+            projectId: environment.projectId,
+            slug: jobSlug,
+          },
+        },
+        include: {
+          aliases: {
+            where: {
+              environmentId: environment.id,
+              name: "latest",
+            },
+            include: {
+              version: true,
+            },
+            take: 1,
+          },
+        },
+      });
+
+      const alias = job.aliases[0];
+
+      if (!alias) {
+        throw new Error(`No version found for job ${jobSlug} in environment ${environment.slug}`);
+      }
+
+      const version = alias.version;
+
+      if (!version) {
+        throw new Error(`No version found for job ${jobSlug} in environment ${environment.slug}`);
+      }
+
+      const options = data.options ?? {};
+
+      const externalAccount = options.accountId
+        ? await tx.externalAccount.upsert({
+            where: {
+              environmentId_identifier: {
+                environmentId: environment.id,
+                identifier: options.accountId,
+              },
+            },
+            create: {
+              environmentId: environment.id,
+              organizationId: environment.organizationId,
+              identifier: options.accountId,
+            },
+            update: {},
+          })
+        : undefined;
+
+      const eventLog = await tx.eventRecord.create({
+        data: {
+          organization: {
+            connect: {
+              id: environment.organizationId,
+            },
+          },
+          project: {
+            connect: {
+              id: environment.projectId,
+            },
+          },
+          environment: {
+            connect: {
+              id: environment.id,
+            },
+          },
+          externalAccount: externalAccount
+            ? {
+                connect: {
+                  id: externalAccount.id,
+                },
+              }
+            : undefined,
+          eventId: idempotencyKey ?? ulid(),
+          name: "invoke",
+          timestamp: new Date(),
+          payload: data.payload ?? {},
+          context: data.context ?? {},
+          source: "trigger.dev",
+          internal: true,
+        },
+      });
+
+      const createRunService = new CreateRunService(tx);
+
+      const run = await createRunService.call(
+        {
+          environment,
+          eventId: eventLog.id,
+          job: job,
+          version,
+        },
+        {
+          callbackUrl: options.callbackUrl,
+        }
+      );
+
+      return run;
+    });
+  }
+}

apps/webapp/app/services/jobs/testJob.server.ts

@@ -94,6 +94,7 @@ export class TestJobService {
             context: {},
             source: event.source ?? "trigger.dev",
             isTest: true,
+            internal: true,
           },
         });
 

apps/webapp/app/services/logger.server.ts

@@ -1,17 +1,26 @@
 import type { LogLevel } from "@trigger.dev/core";
 import { Logger } from "@trigger.dev/core";
 import { sensitiveDataReplacer } from "./sensitiveDataReplacer";
+import { singleton } from "~/utils/singleton";
 
-export const logger = new Logger(
+export const logger = singleton(
-  "webapp",
+  "logger",
-  (process.env.APP_LOG_LEVEL ?? "debug") as LogLevel,
+  () =>
-  ["examples", "output", "connectionString", "payload"],
+    new Logger(
-  sensitiveDataReplacer
+      "webapp",
+      (process.env.APP_LOG_LEVEL ?? "debug") as LogLevel,
+      ["examples", "output", "connectionString", "payload"],
+      sensitiveDataReplacer
+    )
 );
 
-export const workerLogger = new Logger(
+export const workerLogger = singleton(
-  "worker",
+  "worker-logger",
-  (process.env.APP_LOG_LEVEL ?? "debug") as LogLevel,
+  () =>
-  ["examples", "output", "connectionString"],
+    new Logger(
-  sensitiveDataReplacer
+      "worker",
+      (process.env.APP_LOG_LEVEL ?? "debug") as LogLevel,
+      ["examples", "output", "connectionString"],
+      sensitiveDataReplacer
+    )
 );

apps/webapp/app/services/runs/cancelRun.server.ts

@@ -1,6 +1,6 @@
 import { PrismaClient, prisma } from "~/db.server";
 import { executionWorker } from "../worker.server";
-import { dequeueRunExecutionV2 } from "~/models/jobRunExecution.server";
+import { dequeueRunExecutionV3 } from "~/models/jobRunExecution.server";
 
 export class CancelRunService {
   #prismaClient: PrismaClient;
@@ -39,7 +39,7 @@ export class CancelRunService {
           },
         });
 
-        await dequeueRunExecutionV2(run, tx);
+        await dequeueRunExecutionV3(run, tx);
       });
     } catch (error) {
       throw error;

apps/webapp/app/services/runs/continueRun.server.ts

@@ -1,6 +1,6 @@
 import { RuntimeEnvironmentType } from "@trigger.dev/database";
 import { $transaction, Prisma, PrismaClient, prisma } from "~/db.server";
-import { enqueueRunExecutionV2 } from "~/models/jobRunExecution.server";
+import { enqueueRunExecutionV3 } from "~/models/jobRunExecution.server";
 
 const RESUMABLE_STATUSES = ["FAILURE", "TIMED_OUT", "UNRESOLVED_AUTH", "ABORTED", "CANCELED"];
 
@@ -39,7 +39,7 @@ export class ContinueRunService {
           },
         });
 
-        await enqueueRunExecutionV2(run, tx, {
+        await enqueueRunExecutionV3(run, tx, {
           skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
         });
       },

apps/webapp/app/services/runs/createRun.server.ts

@@ -11,17 +11,20 @@ export class CreateRunService {
     this.#prismaClient = prismaClient;
   }
 
-  public async call({
+  public async call(
-    environment,
+    {
-    eventId,
+      environment,
-    job,
+      eventId,
-    version,
+      job,
-  }: {
+      version,
-    environment: AuthenticatedEnvironment;
+    }: {
-    eventId: string;
+      environment: AuthenticatedEnvironment;
-    job: Job;
+      eventId: string;
-    version: JobVersion;
+      job: Job;
-  }) {
+      version: JobVersion;
+    },
+    options: { callbackUrl?: string } = {}
+  ) {
     const endpoint = await this.#prismaClient.endpoint.findUniqueOrThrow({
       where: {
         id: version.endpointId,
@@ -74,6 +77,25 @@ export class CreateRunService {
         },
       });
 
+      if (options.callbackUrl) {
+        await tx.jobRunSubscription.createMany({
+          data: [
+            {
+              runId: run.id,
+              recipientMethod: "WEBHOOK",
+              recipient: options.callbackUrl,
+              event: "SUCCESS",
+            },
+            {
+              runId: run.id,
+              recipientMethod: "WEBHOOK",
+              recipient: options.callbackUrl,
+              event: "FAILURE",
+            },
+          ],
+        });
+      }
+
       await workerQueue.enqueue(
         "startRun",
         {

apps/webapp/app/services/runs/deliverRunSubscription.server.ts

@@ -0,0 +1,164 @@
+import { RunNotification } from "@trigger.dev/core";
+import { subtle } from "node:crypto";
+import { PrismaClient, prisma } from "~/db.server";
+
+// Infer the type of the #findSubscription method
+type FoundSubscription = NonNullable<
+  Awaited<ReturnType<DeliverRunSubscriptionService["_findSubscription"]>>
+>;
+
+type FoundRun = FoundSubscription["run"];
+
+export class DeliverRunSubscriptionService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(id: string) {
+    const subscription = await this._findSubscription(id);
+
+    if (!subscription) {
+      return;
+    }
+
+    if (subscription.deliveredAt) {
+      return;
+    }
+
+    if (subscription.status !== "ACTIVE") {
+      return;
+    }
+
+    const { run } = subscription;
+
+    const payload = this.#getPayload(run);
+
+    const delivered = await this.#deliverPayload(subscription, payload);
+
+    if (delivered) {
+      await this.#prismaClient.jobRunSubscription.update({
+        where: {
+          id,
+        },
+        data: {
+          deliveredAt: new Date(),
+        },
+      });
+    } else {
+      throw new Error(`Failed to deliver subscription ${id}`);
+    }
+  }
+
+  async #deliverPayload(
+    subscription: FoundSubscription,
+    payload: RunNotification<any>
+  ): Promise<boolean> {
+    switch (subscription.recipientMethod) {
+      case "WEBHOOK": {
+        const url = subscription.recipient;
+
+        const rawPayload = JSON.stringify(payload);
+        const hashPayload = Buffer.from(rawPayload, "utf-8");
+        const hmacSecret = Buffer.from(subscription.run.environment.apiKey, "utf-8");
+        const key = await subtle.importKey(
+          "raw",
+          hmacSecret,
+          { name: "HMAC", hash: "SHA-256" },
+          false,
+          ["sign"]
+        );
+        const signature = await subtle.sign("HMAC", key, hashPayload);
+        const signatureHex = Buffer.from(signature).toString("hex");
+
+        const response = await fetch(url, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            "X-Trigger-Signature-256": signatureHex,
+          },
+          body: rawPayload,
+        });
+
+        if (!response.ok) {
+          throw new Error(
+            `Failed to deliver webhook to ${url}: [${response.status}] ${response.statusText}`
+          );
+        }
+
+        return true;
+      }
+    }
+  }
+
+  private async _findSubscription(id: string) {
+    return this.#prismaClient.jobRunSubscription.findUnique({
+      where: {
+        id,
+      },
+      include: {
+        run: {
+          include: {
+            job: true,
+            version: true,
+            statuses: true,
+            environment: true,
+            organization: true,
+            project: true,
+            event: true,
+          },
+        },
+      },
+    });
+  }
+
+  #getPayload(run: FoundRun): RunNotification<any> {
+    const { id, job, version, statuses, environment, organization, project, event } = run;
+
+    const payload = {
+      id,
+      ok: run.status === "SUCCESS",
+      status: run.status,
+      startedAt: run.startedAt,
+      updatedAt: run.updatedAt,
+      completedAt: run.completedAt,
+      executionDurationInMs: run.executionDuration,
+      executionCount: run.executionCount,
+      job: {
+        id: job.id,
+        version: version.version,
+      },
+      statuses: statuses.map((status) => ({
+        key: status.key,
+        label: status.label,
+        state: status.state,
+        data: status.data,
+        history: status.history,
+      })),
+      environment: {
+        slug: environment.slug,
+        id: environment.id,
+        type: environment.type,
+      },
+      organization: {
+        slug: organization.slug,
+        id: organization.id,
+        title: organization.title,
+      },
+      project: {
+        slug: project.slug,
+        id: project.id,
+        name: project.name,
+      },
+      invocation: {
+        id: event.id,
+        context: event.context,
+        timestamp: event.timestamp,
+      },
+      ...(run.status === "SUCCESS" ? { output: run.output } : { error: run.output }),
+    };
+
+    return payload as RunNotification<any>;
+  }
+}

apps/webapp/app/services/runs/deliverRunSubscriptions.server.ts

@@ -0,0 +1,44 @@
+import { JobRun } from "@trigger.dev/database";
+import { PrismaClient, prisma } from "~/db.server";
+import { workerQueue } from "../worker.server";
+
+export class DeliverRunSubscriptionsService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(id: string) {
+    const run = await this.#prismaClient.jobRun.findUnique({
+      where: {
+        id,
+      },
+    });
+
+    if (!run) {
+      return;
+    }
+
+    const subscriptions = await this.#findSubscriptions(run);
+
+    for (const subscription of subscriptions) {
+      await workerQueue.enqueue("deliverRunSubscription", {
+        id: subscription.id,
+      });
+    }
+  }
+
+  async #findSubscriptions(run: JobRun) {
+    const subscriptions = await this.#prismaClient.jobRunSubscription.findMany({
+      where: {
+        runId: run.id,
+        deliveredAt: null,
+        status: "ACTIVE",
+        event: run.status === "SUCCESS" ? "SUCCESS" : "FAILURE",
+      },
+    });
+
+    return subscriptions;
+  }
+}

apps/webapp/app/services/runs/performRunExecutionV3.server.ts

@@ -1,11 +1,13 @@
 import {
   ApiEventLog,
+  AutoYieldMetadata,
   ConnectionAuth,
   EndpointHeadersSchema,
   RunJobAutoYieldWithCompletedTaskExecutionError,
   RunJobBody,
   RunJobError,
   RunJobInvalidPayloadError,
+  RunJobResumeWithParallelTask,
   RunJobResumeWithTask,
   RunJobRetryWithTask,
   RunJobSuccess,
@@ -25,7 +27,7 @@ import {
 } from "~/consts";
 import { $transaction, PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server";
 import { detectResponseIsTimeout } from "~/models/endpoint.server";
-import { enqueueRunExecutionV2 } from "~/models/jobRunExecution.server";
+import { enqueueRunExecutionV3 } from "~/models/jobRunExecution.server";
 import { resolveRunConnections } from "~/models/runConnection.server";
 import { prepareTasksForCaching, prepareTasksForCachingLegacy } from "~/models/task.server";
 import { CompleteRunTaskService } from "~/routes/api.v1.runs.$runId.tasks.$id.complete";
@@ -34,6 +36,8 @@ import { safeJsonZodParse } from "~/utils/json";
 import { EndpointApi } from "../endpointApi.server";
 import { logger } from "../logger.server";
 import { forceYieldCoordinator } from "./forceYieldCoordinator.server";
+import { workerQueue } from "../worker.server";
+import { ResumeTaskService } from "../tasks/resumeTask.server";
 
 type FoundRun = NonNullable<Awaited<ReturnType<typeof findRun>>>;
 type FoundTask = FoundRun["tasks"][number];
@@ -41,21 +45,29 @@ type FoundTask = FoundRun["tasks"][number];
 // We need to limit the cached tasks to not be too large >3.5MB when serialized
 const TOTAL_CACHED_TASK_BYTE_LIMIT = 3500000;
 
-export type PerformRunExecutionV2Input = {
+export type PerformRunExecutionV3Input = {
   id: string;
   reason: "PREPROCESS" | "EXECUTE_JOB";
+
+  /**
+   * @deprecated This is no longer used
+   */
   isRetry: boolean;
+
+  /**
+   * @deprecated Resuming tasks now goes through ResumeTaskService, this is included here for backwards compatibility
+   */
   resumeTaskId?: string;
 };
 
-export class PerformRunExecutionV2Service {
+export class PerformRunExecutionV3Service {
   #prismaClient: PrismaClient;
 
   constructor(prismaClient: PrismaClient = prisma) {
     this.#prismaClient = prismaClient;
   }
 
-  public async call(input: PerformRunExecutionV2Input) {
+  public async call(input: PerformRunExecutionV3Input) {
     const run = await findRun(this.#prismaClient, input.id);
 
     if (!run) {
@@ -159,13 +171,13 @@ export class PerformRunExecutionV2Service {
           },
         });
 
-        await enqueueRunExecutionV2(run, tx, {
+        await enqueueRunExecutionV3(run, tx, {
           skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
         });
       });
     }
   }
-  async #executeJob(run: FoundRun, input: PerformRunExecutionV2Input) {
+  async #executeJob(run: FoundRun, input: PerformRunExecutionV3Input) {
     try {
       const { isRetry, resumeTaskId } = input;
 
@@ -379,6 +391,11 @@ export class PerformRunExecutionV2Service {
 
       const status = safeBody.data.status;
 
+      logger.debug("Endpoint responded with status", {
+        status,
+        data: safeBody.data,
+      });
+
       switch (status) {
         case "SUCCESS": {
           await this.#completeRunWithSuccess(run, safeBody.data, durationInMs);
@@ -386,7 +403,7 @@ export class PerformRunExecutionV2Service {
           break;
         }
         case "RESUME_WITH_TASK": {
-          await this.#resumeRunWithTask(run, safeBody.data, isRetry, durationInMs, executionCount);
+          await this.#resumeRunWithTask(run, safeBody.data, durationInMs);
 
           break;
         }
@@ -396,7 +413,7 @@ export class PerformRunExecutionV2Service {
           break;
         }
         case "RETRY_WITH_TASK": {
-          await this.#retryRunWithTask(run, safeBody.data, isRetry, durationInMs, executionCount);
+          await this.#retryRunWithTask(run, safeBody.data, durationInMs);
 
           break;
         }
@@ -415,33 +432,20 @@ export class PerformRunExecutionV2Service {
           break;
         }
         case "YIELD_EXECUTION": {
-          await this.#resumeYieldedRun(
+          await this.#resumeYieldedRun(run, safeBody.data.key, durationInMs);
-            run,
-            safeBody.data.key,
-            isRetry,
-            durationInMs,
-            executionCount
-          );
           break;
         }
         case "AUTO_YIELD_EXECUTION": {
-          await this.#resumeAutoYieldedRun(
+          await this.#resumeAutoYieldedRun(run, safeBody.data, durationInMs);
-            run,
-            safeBody.data,
-            isRetry,
-            durationInMs,
-            executionCount
-          );
           break;
         }
         case "AUTO_YIELD_EXECUTION_WITH_COMPLETED_TASK": {
-          await this.#resumeAutoYieldedRunWithCompletedTask(
+          await this.#resumeAutoYieldedRunWithCompletedTask(run, safeBody.data, durationInMs);
-            run,
+          break;
-            safeBody.data,
+        }
-            isRetry,
+        case "RESUME_WITH_PARALLEL_TASK": {
-            durationInMs,
+          await this.#resumeParallelRunWithTask(run, safeBody.data, durationInMs);
-            executionCount
+
-          );
           break;
         }
         default: {
@@ -488,6 +492,11 @@ export class PerformRunExecutionV2Service {
           slug: run.organization.slug,
           title: run.organization.title,
         },
+        project: {
+          id: run.project.id,
+          slug: run.project.slug,
+          name: run.project.name,
+        },
         account: run.externalAccount
           ? {
               id: run.externalAccount.identifier,
@@ -534,6 +543,11 @@ export class PerformRunExecutionV2Service {
         slug: run.organization.slug,
         title: run.organization.title,
       },
+      project: {
+        id: run.project.id,
+        slug: run.project.slug,
+        name: run.project.name,
+      },
       account: run.externalAccount
         ? {
             id: run.externalAccount.identifier,
@@ -547,25 +561,34 @@ export class PerformRunExecutionV2Service {
   }
 
   async #completeRunWithSuccess(run: FoundRun, data: RunJobSuccess, durationInMs: number) {
-    await this.#prismaClient.jobRun.update({
+    await $transaction(this.#prismaClient, async (tx) => {
-      where: { id: run.id },
+      await tx.jobRun.update({
-      data: {
+        where: { id: run.id },
-        completedAt: new Date(),
+        data: {
-        status: "SUCCESS",
+          completedAt: new Date(),
-        output: data.output ?? undefined,
+          status: "SUCCESS",
-        executionDuration: {
+          output: data.output ?? undefined,
-          increment: durationInMs,
+          executionDuration: {
+            increment: durationInMs,
+          },
         },
-      },
+      });
+
+      await workerQueue.enqueue(
+        "deliverRunSubscriptions",
+        {
+          id: run.id,
+        },
+        { tx }
+      );
     });
   }
 
   async #resumeRunWithTask(
     run: FoundRun,
     data: RunJobResumeWithTask,
-    isRetry: boolean,
     durationInMs: number,
-    executionCount: number
+    executionCount: number = 1
   ) {
     return await $transaction(this.#prismaClient, async (tx) => {
       await tx.jobRun.update({
@@ -574,25 +597,126 @@ export class PerformRunExecutionV2Service {
           executionDuration: {
             increment: durationInMs,
           },
+          executionCount: {
+            increment: executionCount,
+          },
         },
       });
 
+      if (data.task.outputProperties) {
+        await tx.task.update({
+          where: {
+            id: data.task.id,
+          },
+          data: {
+            outputProperties: data.task.outputProperties,
+          },
+        });
+      }
+
       // If the task has an operation, then the next performRunExecution will occur
       // when that operation has finished
       // Tasks with callbacks enabled will also get processed separately, i.e. when
       // they time out, or on valid requests to their callbackUrl
       if (!data.task.operation && !data.task.callbackUrl) {
-        await enqueueRunExecutionV2(run, tx, {
+        await ResumeTaskService.enqueue(data.task.id, data.task.delayUntil ?? undefined, tx);
-          runAt: data.task.delayUntil ?? undefined,
-          resumeTaskId: data.task.id,
-          isRetry,
-          skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
-          executionCount,
-        });
       }
     });
   }
 
+  async #resumeParallelRunWithTask(
+    run: FoundRun,
+    data: RunJobResumeWithParallelTask,
+    durationInMs: number
+  ) {
+    await this.#prismaClient.jobRun.update({
+      where: { id: run.id },
+      data: {
+        executionDuration: {
+          increment: durationInMs,
+        },
+        executionCount: {
+          increment: 1,
+        },
+        forceYieldImmediately: false,
+      },
+    });
+
+    if (data.task.outputProperties) {
+      await this.#prismaClient.task.update({
+        where: {
+          id: data.task.id,
+        },
+        data: {
+          outputProperties: data.task.outputProperties,
+        },
+      });
+    }
+
+    for (const childError of data.childErrors) {
+      switch (childError.status) {
+        case "AUTO_YIELD_EXECUTION": {
+          await this.#resumeAutoYieldedRun(run, childError, 0, 0);
+
+          break;
+        }
+        case "AUTO_YIELD_EXECUTION_WITH_COMPLETED_TASK": {
+          await this.#resumeAutoYieldedRunWithCompletedTask(run, childError, 0, 0);
+
+          break;
+        }
+        case "CANCELED": {
+          break;
+        }
+        case "ERROR": {
+          return await this.#failRunExecution(
+            this.#prismaClient,
+            "EXECUTE_JOB",
+            run,
+            childError.error ?? undefined,
+            "FAILURE",
+            durationInMs
+          );
+        }
+        case "INVALID_PAYLOAD": {
+          return await this.#failRunExecution(
+            this.#prismaClient,
+            "EXECUTE_JOB",
+            run,
+            childError.errors,
+            "INVALID_PAYLOAD",
+            durationInMs
+          );
+        }
+        case "RESUME_WITH_TASK": {
+          await this.#resumeRunWithTask(run, childError, 0, 0);
+
+          break;
+        }
+        case "RETRY_WITH_TASK": {
+          await this.#retryRunWithTask(run, childError, 0, 0);
+
+          break;
+        }
+        case "UNRESOLVED_AUTH_ERROR": {
+          return await this.#failRunExecution(
+            this.#prismaClient,
+            "EXECUTE_JOB",
+            run,
+            childError.issues,
+            "UNRESOLVED_AUTH",
+            durationInMs
+          );
+        }
+        case "YIELD_EXECUTION": {
+          await this.#resumeYieldedRun(run, childError.key, 0, 0);
+
+          break;
+        }
+      }
+    }
+  }
+
   async #failRunWithError(execution: FoundRun, data: RunJobError, durationInMs: number) {
     return await $transaction(this.#prismaClient, async (tx) => {
       if (data.task) {
@@ -656,9 +780,8 @@ export class PerformRunExecutionV2Service {
   async #resumeYieldedRun(
     run: FoundRun,
     key: string,
-    isRetry: boolean,
     durationInMs: number,
-    executionCount: number
+    executionCount: number = 1
   ) {
     await $transaction(this.#prismaClient, async (tx) => {
       if (run.yieldedExecutions.length + 1 > MAX_RUN_YIELDED_EXECUTIONS) {
@@ -683,7 +806,7 @@ export class PerformRunExecutionV2Service {
             increment: durationInMs,
           },
           executionCount: {
-            increment: 1,
+            increment: executionCount,
           },
           yieldedExecutions: {
             push: key,
@@ -696,20 +819,17 @@ export class PerformRunExecutionV2Service {
         },
       });
 
-      await enqueueRunExecutionV2(run, tx, {
+      await enqueueRunExecutionV3(run, tx, {
-        isRetry,
         skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
-        executionCount,
       });
     });
   }
 
   async #resumeAutoYieldedRun(
     run: FoundRun,
-    data: { location: string; timeRemaining: number; timeElapsed: number; limit?: number },
+    data: AutoYieldMetadata,
-    isRetry: boolean,
     durationInMs: number,
-    executionCount: number
+    executionCount: number = 1
   ) {
     await $transaction(this.#prismaClient, async (tx) => {
       await tx.jobRun.update({
@@ -721,7 +841,7 @@ export class PerformRunExecutionV2Service {
             increment: durationInMs,
           },
           executionCount: {
-            increment: 1,
+            increment: executionCount,
           },
           autoYieldExecution: {
             create: [
@@ -740,10 +860,8 @@ export class PerformRunExecutionV2Service {
         },
       });
 
-      await enqueueRunExecutionV2(run, tx, {
+      await enqueueRunExecutionV3(run, tx, {
-        isRetry,
         skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
-        executionCount,
       });
     });
   }
@@ -751,9 +869,8 @@ export class PerformRunExecutionV2Service {
   async #resumeAutoYieldedRunWithCompletedTask(
     run: FoundRun,
     data: RunJobAutoYieldWithCompletedTaskExecutionError,
-    isRetry: boolean,
     durationInMs: number,
-    executionCount: number
+    executionCount: number = 1
   ) {
     await $transaction(this.#prismaClient, async (tx) => {
       await tx.jobRun.update({
@@ -765,7 +882,7 @@ export class PerformRunExecutionV2Service {
             increment: durationInMs,
           },
           executionCount: {
-            increment: 1,
+            increment: executionCount,
           },
           autoYieldExecution: {
             create: [
@@ -788,13 +905,11 @@ export class PerformRunExecutionV2Service {
 
       await service.call(run.environment, run.id, data.id, {
         properties: data.properties,
-        output: data.output,
+        output: data.output ? (JSON.parse(data.output) as any) : undefined,
       });
 
-      await enqueueRunExecutionV2(run, tx, {
+      await enqueueRunExecutionV3(run, tx, {
-        isRetry,
         skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
-        executionCount,
       });
     });
   }
@@ -802,9 +917,8 @@ export class PerformRunExecutionV2Service {
   async #retryRunWithTask(
     run: FoundRun,
     data: RunJobRetryWithTask,
-    isRetry: boolean,
     durationInMs: number,
-    executionCount: number
+    executionCount: number = 1
   ) {
     return await $transaction(this.#prismaClient, async (tx) => {
       // We need to check for an existing task attempt
@@ -851,25 +965,22 @@ export class PerformRunExecutionV2Service {
               executionDuration: {
                 increment: durationInMs,
               },
+              executionCount: {
+                increment: executionCount,
+              },
             },
           },
         },
       });
 
-      await enqueueRunExecutionV2(run, tx, {
+      await ResumeTaskService.enqueue(data.task.id, data.retryAt, tx);
-        runAt: data.retryAt,
-        resumeTaskId: data.task.id,
-        isRetry,
-        skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
-        executionCount,
-      });
     });
   }
 
   async #resumeRunExecutionAfterTimeout(
     prisma: PrismaClientOrTransaction,
     run: FoundRun,
-    input: PerformRunExecutionV2Input,
+    input: PerformRunExecutionV3Input,
     durationInMs: number,
     executionCount: number
   ) {
@@ -963,11 +1074,8 @@ export class PerformRunExecutionV2Service {
       });
 
       // The run has timed out, so we need to enqueue a new execution
-      await enqueueRunExecutionV2(run, tx, {
+      await enqueueRunExecutionV3(run, tx, {
-        resumeTaskId: input.resumeTaskId,
-        isRetry: input.isRetry,
         skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
-        executionCount,
       });
     });
   }
@@ -1014,6 +1122,14 @@ export class PerformRunExecutionV2Service {
             },
           });
 
+          await workerQueue.enqueue(
+            "deliverRunSubscriptions",
+            {
+              id: run.id,
+            },
+            { tx }
+          );
+
           break;
         }
         case "PREPROCESS": {
@@ -1041,7 +1157,7 @@ export class PerformRunExecutionV2Service {
             },
           });
 
-          await enqueueRunExecutionV2(run, tx, {
+          await enqueueRunExecutionV3(run, tx, {
             skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
           });
 
@@ -1080,6 +1196,7 @@ async function findRun(prisma: PrismaClientOrTransaction, id: string) {
       },
       endpoint: true,
       organization: true,
+      project: true,
       externalAccount: true,
       runConnections: {
         include: {

apps/webapp/app/services/runs/startRun.server.ts

@@ -6,7 +6,7 @@ import {
 } from "@trigger.dev/database";
 import type { PrismaClient, PrismaClientOrTransaction } from "~/db.server";
 import { prisma } from "~/db.server";
-import { enqueueRunExecutionV2 } from "~/models/jobRunExecution.server";
+import { enqueueRunExecutionV3 } from "~/models/jobRunExecution.server";
 import { workerQueue } from "../worker.server";
 
 type FoundRun = NonNullable<Awaited<ReturnType<typeof findRun>>>;
@@ -88,7 +88,7 @@ export class StartRunService {
 
     const updatedRun = await updateRun();
 
-    await enqueueRunExecutionV2(updatedRun, this.#prismaClient, {
+    await enqueueRunExecutionV3(updatedRun, this.#prismaClient, {
       skipRetrying: run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
     });
   }

apps/webapp/app/services/tasks/performTaskOperation.server.ts

@@ -4,15 +4,17 @@ import {
   FetchRetryOptions,
   FetchRetryStrategy,
   RedactString,
+  RetryOptions,
   calculateRetryAt,
 } from "@trigger.dev/core";
-import { RuntimeEnvironmentType, type Task } from "@trigger.dev/database";
+import { type Task } from "@trigger.dev/database";
 import { $transaction, PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server";
-import { enqueueRunExecutionV2 } from "~/models/jobRunExecution.server";
 import { formatUnknownError } from "~/utils/formatErrors.server";
 import { safeJsonFromResponse } from "~/utils/json";
 import { logger } from "../logger.server";
 import { workerQueue } from "../worker.server";
+import { ResumeTaskService } from "./resumeTask.server";
+import { fetch } from "@whatwg-node/fetch";
 
 type FoundTask = Awaited<ReturnType<typeof findTask>>;
 
@@ -38,8 +40,6 @@ export class PerformTaskOperationService {
       return await this.#resumeTask(task, null, 0);
     }
 
-    logger.debug("PerformTaskOperationService.call", { task });
-
     switch (task.operation) {
       case "fetch": {
         const fetchOperation = FetchOperationSchema.safeParse(task.params);
@@ -51,52 +51,100 @@ export class PerformTaskOperationService {
           );
         }
 
-        const { url, requestInit, retry } = fetchOperation.data;
+        const { url, requestInit, retry, timeout } = fetchOperation.data;
 
         const startTimeInMs = performance.now();
 
-        const response = await fetch(url, {
+        const abortController = new AbortController();
-          method: requestInit?.method ?? "GET",
-          headers: normalizeHeaders(requestInit?.headers ?? {}),
-          body: requestInit?.body,
-        });
 
-        const durationInMs = Math.floor(performance.now() - startTimeInMs);
+        // calculate the actual timeout. If timeoutInMs is undefined, we use the default of 120s
+        // Also make sure the timeout is at least 1s, but not bigger than 120s
+        const actualTimeoutInMs = Math.min(Math.max(timeout?.durationInMs ?? 120000, 1000), 120000);
 
-        const jsonBody = await safeJsonFromResponse(response);
+        const timeoutId = setTimeout(() => {
+          abortController.abort();
+        }, actualTimeoutInMs);
 
-        logger.debug("PerformTaskOperationService.call.fetch", {
+        try {
-          url,
+          logger.debug("PerformTaskOperationService.call fetch request", {
-          requestInit,
+            task,
-          retry,
+            actualTimeoutInMs,
-          statusCode: response.status,
+            url,
-          headers: Object.fromEntries(response.headers.entries()),
+            retry,
-          jsonBody,
+          });
-          durationInMs,
-        });
 
-        if (!response.ok) {
+          const response = await fetch(url, {
-          const retryAt = this.#calculateRetryForResponse(task, retry, response);
+            method: requestInit?.method ?? "GET",
+            headers: normalizeHeaders(requestInit?.headers ?? {}),
+            body: requestInit?.body,
+            signal: abortController.signal,
+          });
 
-          if (retryAt) {
+          clearTimeout(timeoutId);
-            return await this.#retryTaskWithError(
+
-              task,
+          const durationInMs = Math.floor(performance.now() - startTimeInMs);
-              `Fetch failed with status ${response.status}`,
+
-              retryAt
+          const jsonBody = await safeJsonFromResponse(response);
-            );
+
+          logger.debug("PerformTaskOperationService.call fetch response", {
+            url,
+            requestInit,
+            retry,
+            statusCode: response.status,
+            headers: Object.fromEntries(response.headers.entries()),
+            jsonBody,
+            durationInMs,
+          });
+
+          if (!response.ok) {
+            const retryAt = this.#calculateRetryForResponse(task, retry, response);
+
+            if (retryAt) {
+              return await this.#retryTaskWithError(
+                task,
+                `Fetch failed with status ${response.status}`,
+                retryAt
+              );
+            }
+
+            // See if there is a json body
+            if (jsonBody) {
+              return await this.#resumeTaskWithError(task, jsonBody);
+            } else {
+              return await this.#resumeTaskWithError(task, {
+                message: `Fetch failed with status ${response.status}`,
+              });
+            }
           }
 
-          // See if there is a json body
+          return await this.#resumeTask(task, jsonBody, durationInMs);
-          if (jsonBody) {
+        } catch (error) {
-            return await this.#resumeTaskWithError(task, jsonBody);
+          if (error instanceof Error && error.name === "AbortError") {
-          } else {
+            const durationInMs = Math.floor(performance.now() - startTimeInMs);
+
+            logger.debug("PerformTaskOperationService.call fetch timed out", {
+              url,
+              durationInMs,
+              error,
+            });
+
+            const retryAt = this.#calculateRetryForTimeout(task, timeout?.retry);
+
+            if (retryAt) {
+              return await this.#retryTaskWithError(
+                task,
+                `Fetch timed out after ${actualTimeoutInMs.toFixed(0)}ms`,
+                retryAt
+              );
+            }
+
             return await this.#resumeTaskWithError(task, {
-              message: `Fetch failed with status ${response.status}`,
+              message: `Fetch timed out after ${actualTimeoutInMs.toFixed(0)}ms`,
             });
           }
-        }
 
-        return await this.#resumeTask(task, jsonBody, durationInMs);
+          throw error;
+        }
       }
       default: {
         await this.#resumeTaskWithError(task, {
@@ -142,6 +190,17 @@ export class PerformTaskOperationService {
     }
   }
 
+  #calculateRetryForTimeout(
+    task: NonNullable<FoundTask>,
+    retry: RetryOptions | undefined
+  ): Date | undefined {
+    if (!retry) {
+      return;
+    }
+
+    return calculateRetryAt(retry, task.attempts.length - 1);
+  }
+
   #getRetryStrategyForStatusCode(
     statusCode: number,
     retry: FetchRetryOptions
@@ -258,9 +317,7 @@ export class PerformTaskOperationService {
   }
 
   async #resumeRunExecution(task: NonNullable<FoundTask>, prisma: PrismaClientOrTransaction) {
-    await enqueueRunExecutionV2(task.run, prisma, {
+    await ResumeTaskService.enqueue(task.id, undefined, prisma);
-      skipRetrying: task.run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
-    });
   }
 }
 

apps/webapp/app/services/tasks/processCallbackTimeout.server.ts

@@ -1,7 +1,6 @@
-import { RuntimeEnvironmentType } from "@trigger.dev/database";
 import { $transaction, PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server";
-import { enqueueRunExecutionV2 } from "~/models/jobRunExecution.server";
 import { logger } from "../logger.server";
+import { ResumeTaskService } from "./resumeTask.server";
 
 type FoundTask = Awaited<ReturnType<typeof findTask>>;
 
@@ -37,7 +36,7 @@ export class ProcessCallbackTimeoutService {
         },
         data: {
           status: "ERRORED",
-          error
+          error,
         },
       });
 
@@ -55,9 +54,7 @@ export class ProcessCallbackTimeoutService {
   }
 
   async #resumeRunExecution(task: NonNullable<FoundTask>, prisma: PrismaClientOrTransaction) {
-    await enqueueRunExecutionV2(task.run, prisma, {
+    await ResumeTaskService.enqueue(task.id, undefined, prisma);
-      skipRetrying: task.run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
-    });
   }
 }
 

apps/webapp/app/services/tasks/resumeTask.server.ts

@@ -0,0 +1,107 @@
+import { PrismaClient, PrismaClientOrTransaction, prisma } from "~/db.server";
+import { workerQueue } from "../worker.server";
+import { enqueueRunExecutionV3 } from "~/models/jobRunExecution.server";
+import { RuntimeEnvironmentType } from "@trigger.dev/database";
+import { logger } from "../logger.server";
+
+type FoundTask = Awaited<ReturnType<typeof findTask>>;
+
+export class ResumeTaskService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(id: string) {
+    const task = await findTask(this.#prismaClient, id);
+
+    if (!task) {
+      return;
+    }
+
+    if (task.status === "COMPLETED" || task.status === "ERRORED") {
+      return await this.#resumeRunExecution(task);
+    }
+
+    const updatedTask = await this.#prismaClient.task.update({
+      where: {
+        id: task.id,
+      },
+      data: {
+        status: task.noop ? "COMPLETED" : "RUNNING",
+        completedAt: task.noop ? new Date() : undefined,
+      },
+      include: {
+        attempts: true,
+        run: {
+          include: {
+            environment: true,
+          },
+        },
+        parent: true,
+      },
+    });
+
+    // This will retry the task if it's not a noop, or just resume
+    // the run execution if it is a noop.
+    return await this.#resumeRunExecution(updatedTask);
+  }
+
+  async #resumeRunExecution(task: NonNullable<FoundTask>) {
+    logger.debug("ResumeTaskService.call resuming run execution", {
+      parent: task.parent,
+      taskId: task.id,
+    });
+
+    if (task.parent && task.parent.childExecutionMode === "PARALLEL") {
+      const children = await this.#prismaClient.task.findMany({
+        where: {
+          parentId: task.parent.id,
+        },
+        select: {
+          id: true,
+          status: true,
+        },
+      });
+
+      const allChildrenCompleted = children.every(
+        (child) =>
+          child.status === "COMPLETED" || child.status === "ERRORED" || child.status === "CANCELED"
+      );
+
+      logger.debug("ResumeTaskService.call parent executing children in parallel", {
+        parentId: task.parent.id,
+        allChildrenCompleted,
+        children,
+      });
+
+      if (!allChildrenCompleted) {
+        return;
+      }
+    }
+
+    await enqueueRunExecutionV3(task.run, this.#prismaClient, {
+      skipRetrying: task.run.environment.type === RuntimeEnvironmentType.DEVELOPMENT,
+    });
+  }
+
+  public static async enqueue(id: string, runAt?: Date, tx?: PrismaClientOrTransaction) {
+    return await workerQueue.enqueue("resumeTask", { id }, { tx, jobKey: `resume:${id}`, runAt });
+  }
+}
+
+async function findTask(prisma: PrismaClient, id: string) {
+  return prisma.task.findUnique({
+    where: { id },
+    include: {
+      attempts: true,
+      run: {
+        include: {
+          environment: true,
+        },
+      },
+      parent: true,
+    },
+  });
+}

apps/webapp/app/services/telemetry.server.ts

@@ -5,6 +5,7 @@ import { env } from "~/env.server";
 import type { Organization } from "~/models/organization.server";
 import type { Project } from "~/models/project.server";
 import type { User } from "~/models/user.server";
+import { singleton } from "~/utils/singleton";
 
 type Options = {
   postHogApiKey?: string;
@@ -236,13 +237,17 @@ type CaptureEvent = {
   userOnceProperties?: Record<string, any>;
 };
 
-export const telemetry = new Telemetry({
+export const telemetry = singleton(
-  postHogApiKey: env.POSTHOG_PROJECT_KEY,
+  "telemetry",
-  trigger:
+  () =>
-    env.TELEMETRY_TRIGGER_API_KEY && env.TELEMETRY_TRIGGER_API_URL
+    new Telemetry({
-      ? {
+      postHogApiKey: env.POSTHOG_PROJECT_KEY,
-          apiKey: env.TELEMETRY_TRIGGER_API_KEY,
+      trigger:
-          apiUrl: env.TELEMETRY_TRIGGER_API_URL,
+        env.TELEMETRY_TRIGGER_API_KEY && env.TELEMETRY_TRIGGER_API_URL
-        }
+          ? {
-      : undefined,
+              apiKey: env.TELEMETRY_TRIGGER_API_KEY,
-});
+              apiUrl: env.TELEMETRY_TRIGGER_API_URL,
+            }
+          : undefined,
+    })
+);

apps/webapp/app/services/worker.server.ts

@@ -13,15 +13,17 @@ import { InvokeDispatcherService } from "./events/invokeDispatcher.server";
 import { integrationAuthRepository } from "./externalApis/integrationAuthRepository.server";
 import { IntegrationConnectionCreatedService } from "./externalApis/integrationConnectionCreated.server";
 import { MissingConnectionCreatedService } from "./runs/missingConnectionCreated.server";
-import { PerformRunExecutionV2Service } from "./runs/performRunExecutionV2.server";
+import { PerformRunExecutionV3Service } from "./runs/performRunExecutionV3.server";
 import { StartRunService } from "./runs/startRun.server";
 import { DeliverScheduledEventService } from "./schedules/deliverScheduledEvent.server";
 import { ActivateSourceService } from "./sources/activateSource.server";
 import { DeliverHttpSourceRequestService } from "./sources/deliverHttpSourceRequest.server";
 import { PerformTaskOperationService } from "./tasks/performTaskOperation.server";
-import { ProcessCallbackTimeoutService } from "./tasks/processCallbackTimeout";
+import { ProcessCallbackTimeoutService } from "./tasks/processCallbackTimeout.server";
 import { ProbeEndpointService } from "./endpoints/probeEndpoint.server";
-import { PgNotifyService } from "./db/pgNotify.server";
+import { DeliverRunSubscriptionService } from "./runs/deliverRunSubscription.server";
+import { DeliverRunSubscriptionsService } from "./runs/deliverRunSubscriptions.server";
+import { ResumeTaskService } from "./tasks/resumeTask.server";
 
 const workerCatalog = {
   indexEndpoint: z.object({
@@ -82,6 +84,15 @@ const workerCatalog = {
   simulate: z.object({
     seconds: z.number(),
   }),
+  deliverRunSubscriptions: z.object({
+    id: z.string(),
+  }),
+  deliverRunSubscription: z.object({
+    id: z.string(),
+  }),
+  resumeTask: z.object({
+    id: z.string(),
+  }),
 };
 
 const executionWorkerCatalog = {
@@ -91,6 +102,10 @@ const executionWorkerCatalog = {
     resumeTaskId: z.string().optional(),
     isRetry: z.boolean(),
   }),
+  performRunExecutionV3: z.object({
+    id: z.string(),
+    reason: z.enum(["EXECUTE_JOB", "PREPROCESS"]),
+  }),
 };
 
 let workerQueue: ZodWorker<typeof workerCatalog>;
@@ -336,6 +351,33 @@ function getWorkerQueue() {
           await new Promise((resolve) => setTimeout(resolve, payload.seconds * 1000));
         },
       },
+      deliverRunSubscriptions: {
+        priority: 1, // smaller number = higher priority
+        maxAttempts: 5,
+        handler: async (payload, job) => {
+          const service = new DeliverRunSubscriptionsService();
+
+          await service.call(payload.id);
+        },
+      },
+      deliverRunSubscription: {
+        priority: 1, // smaller number = higher priority
+        maxAttempts: 13,
+        handler: async (payload, job) => {
+          const service = new DeliverRunSubscriptionService();
+
+          await service.call(payload.id);
+        },
+      },
+      resumeTask: {
+        priority: 0,
+        maxAttempts: 3,
+        handler: async (payload, job) => {
+          const service = new ResumeTaskService();
+
+          return await service.call(payload.id);
+        },
+      },
     },
   });
 }
@@ -359,7 +401,7 @@ function getExecutionWorkerQueue() {
         priority: 0, // smaller number = higher priority
         maxAttempts: 12,
         handler: async (payload, job) => {
-          const service = new PerformRunExecutionV2Service();
+          const service = new PerformRunExecutionV3Service();
 
           await service.call({
             id: payload.id,
@@ -369,6 +411,19 @@ function getExecutionWorkerQueue() {
           });
         },
       },
+      performRunExecutionV3: {
+        priority: 0, // smaller number = higher priority
+        maxAttempts: 12,
+        handler: async (payload, job) => {
+          const service = new PerformRunExecutionV3Service();
+
+          await service.call({
+            id: payload.id,
+            reason: payload.reason,
+            isRetry: false,
+          });
+        },
+      },
     },
   });
 }

apps/webapp/app/utils/singleton.ts

@@ -0,0 +1,6 @@
+export function singleton<T>(name: string, getValue: () => T): T {
+  const thusly = globalThis as any;
+  thusly.__trigger_singletons ??= {};
+  thusly.__trigger_singletons[name] ??= getValue();
+  return thusly.__trigger_singletons[name];
+}

apps/webapp/package.json

@@ -68,6 +68,7 @@
     "@trigger.dev/sdk": "workspace:*",
     "@types/pg": "8.6.6",
     "@uiw/react-codemirror": "^4.19.5",
+    "@whatwg-node/fetch": "^0.9.14",
     "class-variance-authority": "^0.5.2",
     "clsx": "^1.2.1",
     "compression": "^1.7.4",
@@ -95,13 +96,12 @@
     "prismjs": "^1.29.0",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
-    "react-hot-toast": "^2.4.0",
     "react-hotkeys-hook": "^4.4.1",
     "react-use": "^17.4.0",
     "recharts": "^2.8.0",
-    "remix-auth": "^3.2.2",
+    "remix-auth": "^3.6.0",
-    "remix-auth-email-link": "^1.4.2",
+    "remix-auth-email-link": "^2.1.0",
-    "remix-auth-github": "^1.1.1",
+    "remix-auth-github": "^1.6.0",
     "remix-typedjson": "0.3.1",
     "remix-utils": "^7.1.0",
     "semver": "^7.5.0",

apps/webapp/server.ts

@@ -51,7 +51,7 @@ app.all(
       }
 );
 
-const port = process.env.REMIX_APP_PORT || 3000;
+const port = process.env.REMIX_APP_PORT || process.env.PORT || 3000;
 
 if (process.env.HTTP_SERVER_DISABLED !== "true") {
   const server = app.listen(port, () => {

docs/_snippets/jobs/options.mdx

@@ -11,6 +11,7 @@
     </ParamField>
     <ParamField body="trigger" type="object" required>
       The `trigger` property is used to define when the Job should run. There are currently the following Trigger types:
+      - [invokeTrigger](/sdk/invoke-trigger)
       - [cronTrigger](/sdk/crontrigger)
       - [intervalTrigger](/sdk/intervaltrigger)
       - [eventTrigger](/sdk/eventtrigger)

docs/documentation/concepts/limits.mdx

@@ -14,6 +14,7 @@ The following limits apply to the Trigger.dev Cloud service and users of the sel
 | Runs (per Month)                                                        | 5,000     | Up to 1m    | Custom                    |
 | Run Log retention                                                       | 24 hours  | 7 days      | Custom                    |
 | Connected Integrations                                                  | Up to 50  | Up to 1000  | Custom                    |
+| Task Output Size                                                        | 3MB       | 3MB         | 3MB                       |
 | [Tasks per Run](#tasks-per-runs)                                        | Up to 250 | Up to 1000  | Custom                    |
 | [Concurrent Run Executions](#concurrent-run-executions)                 | Up to 10  | Up to 10    | Custom                    |
 | [Maximum Task Duration](#maximum-task-duration)                         | < 2m      | < 2m        | < Deployment Grace Period |

docs/documentation/concepts/triggers/introduction.mdx

@@ -1,21 +1,24 @@
 ---
 title: "Triggers: Introduction"
 sidebarTitle: "Introduction"
-description: "A Trigger is what starts a Job Run. It can be a webhook, a schedule, or an event."
 ---
 
-We currently support three types of Triggers: Webhooks, Scheduled, and Events. You can use any of these to start a Job Run.
+We currently support four types of Triggers: Manual Invoke, Events, Scheduled, and Webhooks. You can use any of these to start a Job Run.
 
 <CardGroup>
-  <Card title="Webhooks" icon="webhook" href="/documentation/concepts/triggers/webhooks">
+  <Card title="Invoke" icon="bolt" href="/documentation/concepts/triggers/invoke">
-    Start your Jobs in realtime when events happen in APIs
+    Run your Job manually by using `Job.invoke()`
-  </Card>
-  <Card title="Scheduled" icon="calendar" href="/documentation/concepts/triggers/scheduled">
-    Run a Job on a repeating schedule
   </Card>
   <Card title="Event" icon="brackets-curly" href="/documentation/concepts/triggers/events">
     Run your Job when you send events with data
   </Card>
+  <Card title="Scheduled" icon="calendar" href="/documentation/concepts/triggers/scheduled">
+    Run a Job on a repeating schedule
+  </Card>
+  <Card title="Webhooks" icon="webhook" href="/documentation/concepts/triggers/webhooks">
+    Start your Jobs in realtime when events happen in APIs
+  </Card>
+
   <Card
     title="DynamicTrigger & DynamicSchedule"
     icon="code"

docs/documentation/concepts/triggers/invoke.mdx

@@ -0,0 +1,382 @@
+---
+title: "Invoke triggers"
+sidebarTitle: "Manual Invoke"
+description: "Invoke Jobs manually using the invoke Trigger"
+---
+
+<Warning>This feature is in beta and not yet available to use on the Trigger.dev Cloud.</Warning>
+
+Sometimes it makes sense to be able to invoke a Job manually, without having to specify an event, especially for cases where you want to get notified when the invoked Job Run is complete.
+
+To specify that a job is manually invokable, you can use the `invokeTrigger()` function when defining a job:
+
+```ts exampleJob.ts
+import { invokeTrigger } from "@trigger.dev/sdk";
+import { client } from "@/trigger";
+
+export const exampleJob = client.defineJob({
+  id: "example-job",
+  name: "Example job",
+  version: "1.0.1",
+  trigger: invokeTrigger(),
+  run: async (payload, io, ctx) => {
+    // do something with the payload
+  },
+});
+```
+
+And then you can invoke the job using the `Job.invoke()` method:
+
+```ts example.ts
+import { exampleJob } from "./exampleJob";
+
+const jobRun = await exampleJob.invoke({ foo: "bar" });
+```
+
+## Payload Schema
+
+You can specify the type of the expected payload by passing a [Zod schema](https://zod.dev) to `invokeTrigger()`:
+
+```ts exampleJob.ts
+import { invokeTrigger } from "@trigger.dev/sdk";
+import { client } from "@/trigger";
+
+export const exampleJob = client.defineJob({
+  id: "example-job",
+  name: "Example job",
+  version: "1.0.1",
+  trigger: invokeTrigger({
+    //the expected payload shape
+    schema: z.object({
+      userId: z.string(),
+      tier: z.enum(["free", "pro"]),
+    }),
+  }),
+  run: async (payload, io, ctx) => {
+    // payload is typed as { userId: string, tier: "free" | "pro" }
+  },
+});
+```
+
+Now when you invoke the job, you will get a type error if the payload does not match the schema:
+
+```ts example.ts
+import { exampleJob } from "./exampleJob";
+// this will throw a type error because the payload does not match the schema
+const jobRun = await exampleJob.invoke({ foo: "bar" });
+// this will work
+const jobRun = await exampleJob.invoke({ userId: "123", tier: "free" });
+```
+
+## Invoking a Job
+
+As you can see in the example above, invoking a job is as simple as calling the `Job.invoke()` method. This method returns a `JobRun` object that you can use to track the progress of the job run, especially in conjunction with our [React hooks](/documentation/guides/react-hooks), like [useRunDetails()](/sdk/react/userundetails)
+
+```ts
+import { exampleJob } from "./exampleJob";
+
+// Somewhere in your backend code
+const jobRun = await exampleJob.invoke({ userId: "123", tier: "free" });
+
+// Somewhere in your frontend code
+const { data: jobRunDetails } = useRunDetails(jobRun.id);
+```
+
+<Note>
+  There are many variations for how you could get the job run information passed from your backend
+  to your frontend, depending on the framework you are using and which rendering model. We'll leave
+  that as an exercise for the reader
+</Note>
+
+### Deduplicate Invocations
+
+You can pass an optional `idempotencyKey` to the `invoke()` method to deduplicate invocations. This is useful when you want to make sure that a job is only invoked once for a given payload.
+
+```ts
+import { exampleJob } from "./exampleJob";
+
+// Somewhere in your backend code
+const jobRun = await exampleJob.invoke(
+  { userId: "123", tier: "free" },
+  { idempotencyKey: "abc123" }
+);
+
+// This will not invoke the job again, but return the existing job run
+const jobRun2 = await exampleJob.invoke(
+  { userId: "123", tier: "free" },
+  { idempotencyKey: "abc123" }
+);
+```
+
+### Callback URL
+
+You can also pass an optional `callbackUrl` to the `invoke()` method to get notified when the job run is complete, either successfully or with an error.
+
+```ts
+import { exampleJob } from "./exampleJob";
+
+// Somewhere in your backend code
+const jobRun = await exampleJob.invoke(
+  { userId: "123", tier: "free" },
+  { callbackUrl: `${process.env.VERCEL_URL}/api/trigger/runs` }
+);
+```
+
+When the run is complete, we will issue a `POST` request to the URL with the [RunNotification payload](/sdk/run-notification).
+
+#### Verifying the callback
+
+You should make sure to verify the payload in your callback route to make sure that the request is coming from Trigger. You can do this by checking the `X-Trigger-Signature-256` header, which contains a HMAC signature of the payload using your [secret API Key](/documentation/concepts/environments-endpoints).
+
+```ts
+import { crypto } from "node:crypto"
+
+app.post("/api/trigger/runs", (req, res) => {
+  // Create digest with payload + hmac secret
+  const hashPayload = req.rawBody;
+  const hmac = crypto.createHmac("sha256", process.env.TRIGGER_API_KEY); /
+  const digest = Buffer.from(
+    signatureAlgorithm + "=" + hmac.update(hashPayload).digest("hex"),
+    "utf8"
+  );
+
+  // Get hash sent by the provider
+  const providerSig = Buffer.from(req.get("X-Trigger-Signature-256") || "", "utf8");
+
+  // Compare digest signature with signature sent by provider
+  if (providerSig.length !== digest.length || !crypto.timingSafeEqual(digest, providerSig)) {
+    res.status(401).send("Unauthorized");
+  } else {
+    // Webhook Authenticated
+    // process and respond...
+    res.json({ message: "Success" });
+  }
+});
+```
+
+### Additional context
+
+You can pass an optional `context` object to the `invoke()` method, which will be available in the job run context. This is useful for passing additional information to the job run that doesn't make sense in the payload.
+
+```ts
+import { exampleJob } from "./exampleJob";
+
+// Somewhere in your backend code
+const jobRun = await exampleJob.invoke(
+  { userId: "123", tier: "free" },
+  {
+    context: {
+      traceId: "trace_123",
+      correlationId: "def456",
+    },
+  }
+);
+```
+
+And then you can access the context in the job run:
+
+```ts
+export const exampleJob = client.defineJob({
+  id: "example-job",
+  name: "Example job",
+  version: "1.0.1",
+  trigger: invokeTrigger({
+    //the expected payload shape
+    schema: z.object({
+      userId: z.string(),
+      tier: z.enum(["free", "pro"]),
+    }),
+  }),
+  run: async (payload, io, ctx) => {
+    console.log(ctx.event.context); // { traceId: "trace_123", correlationId: "def456" }
+  },
+});
+```
+
+## Invoking a job from a job
+
+You can also invoke a job from another job:
+
+```ts
+import { exampleJob } from "./exampleJob";
+
+client.defineJob({
+  id: "example-job2",
+  name: "Example job 2",
+  version: "1.0.1",
+  trigger: intervalTrigger({
+    seconds: 60,
+  }),
+  run: async (payload, io, ctx) => {
+    const jobRun = await exampleJob.invoke("⚡", { userId: "123", tier: "free" });
+  },
+});
+```
+
+Notice how the `invoke()` method takes a string as the first argument. This is because under the hood `invoke()` is automatically creating a [Task](/documentation/concepts/tasks) and the `"⚡"` string is the `cacheKey` for the created task. You can easily see the run created via the Run Dashboard:
+
+![Task](/images/invoke-task.png)
+
+<Warning>
+  If `Job.invoke()` is called within another job, and you **don't** include the `cacheKey` we'll
+  throw an error.
+</Warning>
+
+### Wait for completion
+
+You can also invoke a job and wait for it to complete before continuing execution of the current job:
+
+```ts
+import { exampleJob } from "./exampleJob";
+
+client.defineJob({
+  id: "example-job2",
+  name: "Example job 2",
+  version: "1.0.1",
+  trigger: intervalTrigger({
+    seconds: 60,
+  }),
+  run: async (payload, io, ctx) => {
+    const jobRun = await exampleJob.invokeAndWaitForCompletion("⚡", {
+      userId: "123",
+      tier: "free",
+    });
+
+    if (jobRun.ok) {
+      // The job run finished successfully
+      console.log(jobRun.output);
+    } else {
+      // The job run finished with an error
+      console.log(`The job failed with status ${jobRun.status}`, jobRun.error);
+    }
+  },
+});
+```
+
+By default, `invokeAndWaitForCompletion()` will wait for the job run to complete for up to 60 minutes. You can change this by passing a `timeoutInSeconds` option:
+
+```ts
+import { exampleJob } from "./exampleJob";
+
+client.defineJob({
+  id: "example-job2",
+  name: "Example job 2",
+  version: "1.0.1",
+  trigger: intervalTrigger({
+    seconds: 60,
+  }),
+  run: async (payload, io, ctx) => {
+    const jobRun = await exampleJob.invokeAndWaitForCompletion(
+      "⚡",
+      {
+        userId: "123",
+        tier: "free",
+      },
+      86_400 // 24 hours
+    );
+
+    if (jobRun.ok) {
+      // The job run finished successfully
+      console.log(jobRun.output);
+    } else {
+      // The job run finished with an error
+      console.log(`The job failed with status ${jobRun.status}`, jobRun.error);
+    }
+  },
+});
+```
+
+<Note>
+  If the invoked job run doesn't complete in the given time, the underlying task will fail, along
+  with the run
+</Note>
+
+The return value of `invokeAndWaitForCompletion` is a [RunNotification](/sdk/run-notification) object.
+
+### Batch invoke and wait for completion
+
+You can also batch invoke jobs and wait for them all to complete before continuing execution of the current job:
+
+```ts
+import { exampleJob } from "./exampleJob";
+
+client.defineJob({
+  id: "example-job2",
+  name: "Example job 2",
+  version: "1.0.1",
+  trigger: intervalTrigger({
+    seconds: 60,
+  }),
+  run: async (payload, io, ctx) => {
+    const runs = await exampleJob.batchInvokeAndWaitForCompletion("⚡", [
+      {
+        payload: {
+          userId: "123",
+          tier: "free",
+        },
+        timeoutInSeconds: 86_400, // 24 hours
+      },
+      {
+        payload: {
+          userId: "abc",
+          tier: "paid",
+        },
+        timeoutInSeconds: 86_400, // 24 hours
+      },
+    ]);
+
+    // runs is an array of RunNotification objects
+  },
+});
+```
+
+You can batch up to 25 invocations at once, and we will run them in parallel and wait for all of them to complete before continuing execution of the current job.
+
+You can in an optional `options` object to each invocation, if you want to pass `context` and `accountId` to each invocation:
+
+```ts
+import { exampleJob } from "./exampleJob";
+
+client.defineJob({
+  id: "example-job2",
+  name: "Example job 2",
+  version: "1.0.1",
+  trigger: intervalTrigger({
+    seconds: 60,
+  }),
+  run: async (payload, io, ctx) => {
+    const runs = await exampleJob.batchInvokeAndWaitForCompletion("⚡", [
+      {
+        payload: {
+          userId: "123",
+          tier: "free",
+        },
+        timeoutInSeconds: 86_400, // 24 hours
+        options: {
+          context: {
+            traceId: "trace_123",
+            correlationId: "def456",
+          },
+          accountId: "abc123",
+        },
+      },
+      {
+        payload: {
+          userId: "abc",
+          tier: "paid",
+        },
+        timeoutInSeconds: 86_400, // 24 hours
+        options: {
+          context: {
+            traceId: "trace_abc",
+            correlationId: "defabc",
+          },
+          accountId: "abc123",
+        },
+      },
+    ]);
+
+    // runs is an array of RunNotification objects
+  },
+});
+```

docs/integrations/apis/openai-tasks.mdx

@@ -43,6 +43,57 @@ run: async (payload, io, ctx) => {
 },
 ```
 
+You can also pass an optional third parameter to `backgroundCreateCompletion` to specify OpenAI request options:
+
+```ts requestOptions.ts
+run: async (payload, io, ctx) => {
+    // This code showcases background text completion using the "gpt-3.5-turbo" model.
+    // It generates text based on the provided programming task and logs the result.
+    const programmingTask = `Create a function that checks if a string is a palindrome.`;
+
+    const response = await io.openai.backgroundCreateCompletion("background-completion", {
+      model: "gpt-3.5-turbo",
+      prompt: `Coding task: ${programmingTask}\n\n`,
+    }, {
+      headers: {
+        "User-Agent": "my-user-agent"
+      }
+    });
+
+    await io.logger.info("codeSnippet", response.choices[0]?.text);
+},
+```
+
+This task is implemented using [io.backgroundFetch()](/sdk/io/backgroundfetch) and so you can also pass a 4th parameter customizing the retry and timeout options:
+
+```ts fetchOptions.ts
+run: async (payload, io, ctx) => {
+    // This code showcases background text completion using the "gpt-3.5-turbo" model.
+    // It generates text based on the provided programming task and logs the result.
+    const programmingTask = `Create a function that checks if a string is a palindrome.`;
+
+    const response = await io.openai.backgroundCreateCompletion("background-completion", {
+      model: "gpt-3.5-turbo",
+      prompt: `Coding task: ${programmingTask}\n\n`,
+    }, {
+      headers: {
+        "User-Agent": "my-user-agent"
+      }
+    }, {
+      timeout: {
+        durationInMs: 10000,
+        retry: {
+          limit: 3,
+          minTimeoutInMs: 1000,
+          factor: 2,
+        }
+      }
+    });
+
+    await io.logger.info("codeSnippet", response.choices[0]?.text);
+},
+```
+
 ### `createChatCompletion`
 
 Generates text completions in a conversational context. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/chat/create)
@@ -85,6 +136,87 @@ run: async (payload, io, ctx) => {
 },
 ```
 
+You can also use the more "fluent" pattern used by the OpenAI SDK:
+
+```ts fluent.ts
+run: async (payload, io, ctx) => {
+    // This code showcases background chat completion using the "gpt-3.5-turbo" model.
+    // It simulates a conversation with a user message and logs the response choices.
+    const response = await io.openai.chat.completions.backgroundCreate("background-chat-completion", {
+      model: "gpt-3.5-turbo",
+      messages: [
+        {
+          role: "user",
+          content: "Create a good programming joke about background jobs",
+        },
+      ],
+    });
+
+    await io.logger.info("choices", response.choices);
+},
+```
+
+Additionally, you can pass an optional third parameter to specify OpenAI request options:
+
+```ts requestOptions.ts
+run: async (payload, io, ctx) => {
+    // This code showcases background text completion using the "gpt-3.5-turbo" model.
+    // It generates text based on the provided programming task and logs the result.
+    const programmingTask = `Create a function that checks if a string is a palindrome.`;
+
+    const response = await io.openai.chat.completions.backgroundCreate("background-completion", {
+      model: "gpt-3.5-turbo",
+      messages: [
+        {
+          role: "user",
+          content: "Create a good programming joke about background jobs",
+        },
+      ],
+    }, {
+      headers: {
+        "User-Agent": "my-user-agent"
+      }
+    });
+
+    await io.logger.info("choices", response.choices);
+},
+```
+
+This task is implemented using [io.backgroundFetch()](/sdk/io/backgroundfetch) and so you can also pass a 4th parameter customizing the retry and timeout options:
+
+```ts fetchOptions.ts
+run: async (payload, io, ctx) => {
+    // This code showcases background text completion using the "gpt-3.5-turbo" model.
+    // It generates text based on the provided programming task and logs the result.
+    const programmingTask = `Create a function that checks if a string is a palindrome.`;
+
+    const response = await io.openai.chat.completions.backgroundCreate("background-completion", {
+      model: "gpt-3.5-turbo",
+      messages: [
+        {
+          role: "user",
+          content: "Create a good programming joke about background jobs",
+        },
+      ],
+    }, {
+      headers: {
+        "User-Agent": "my-user-agent"
+      }
+    }, {
+      timeout: {
+        durationInMs: 10000,
+        retry: {
+          limit: 3,
+          minTimeoutInMs: 1000,
+          factor: 2,
+        }
+      }
+    });
+
+    await io.logger.info("choices", response.choices);
+},
+```
+
 ### `retrieveModel`
 
 Retrieves a specific model by ID. [Official OpenAI Docs](https://platform.openai.com/docs/api-reference/models/retrieve)

docs/mint.json

@@ -117,9 +117,10 @@
           "group": "Triggers",
           "pages": [
             "documentation/concepts/triggers/introduction",
-            "documentation/concepts/triggers/webhooks",
+            "documentation/concepts/triggers/invoke",
             "documentation/concepts/triggers/events",
             "documentation/concepts/triggers/scheduled",
+            "documentation/concepts/triggers/webhooks",
             "documentation/concepts/triggers/dynamic"
           ]
         },
@@ -343,6 +344,8 @@
           ]
         },
         "sdk/context",
+        "sdk/run-notification",
+        "sdk/invoke-trigger",
         "sdk/eventtrigger",
         "sdk/crontrigger",
         "sdk/intervaltrigger",

docs/sdk/context.mdx

@@ -38,8 +38,7 @@ description: "The third parameter in a Job's `run()` function. An object that co
       The Environment's ID
     </ResponseField>
     <ResponseField name="type" type="string" required>
-      The Environment's type. One of: "PRODUCTION", "STAGING", "DEVELOPMENT",
+      The Environment's type. One of: "PRODUCTION", "STAGING", "DEVELOPMENT", "PREVIEW".
-      "PREVIEW".
     </ResponseField>
   </Expandable>
 </ResponseField>
@@ -74,6 +73,21 @@ description: "The third parameter in a Job's `run()` function. An object that co
   </Expandable>
 </ResponseField>
 
+<ResponseField name="project" type="object" required>
+  Metadata about the Project
+  <Expandable title="properties">
+    <ResponseField name="slug" type="string" required>
+      The Project's slug
+    </ResponseField>
+    <ResponseField name="id" type="string" required>
+      The Project's ID
+    </ResponseField>
+    <ResponseField name="name" type="string" required>
+      The Project's title
+    </ResponseField>
+  </Expandable>
+</ResponseField>
+
 <ResponseField name="event" type="object" required>
   Metadata about the Event that triggered the Run.
   <Expandable title="properties">
@@ -105,8 +119,8 @@ description: "The third parameter in a Job's `run()` function. An object that co
 </ResponseField>
 
 <ResponseField name="account" type="object">
-  Metadata about the Account that triggered the Run. This is a Trigger.dev
+  Metadata about the Account that triggered the Run. See the [BYO Auth
-  Connect property which is coming soon.
+  docs](/documentation/guides/using-integrations-byo-auth) for more information.
   <Expandable title="properties">
     <ResponseField name="id" type="string" required>
       The Account's ID

docs/sdk/invoke-trigger.mdx

@@ -0,0 +1,86 @@
+---
+title: "invokeTrigger()"
+description: "Use `invokeTrigger()` to allow a Job to be manually triggered using `Job.invoke()`"
+---
+
+<Warning>This feature is in beta and not yet available to use on the Trigger.dev Cloud.</Warning>
+
+Setting `invokeTrigger()` on a job allows you to manually trigger the job using `Job.invoke()`, either from your backend or from inside another job. See our [Invoke Trigger guide](/documentation/guides/invoke) for more info.
+
+## Parameters
+
+<ResponseField name="options" type="object" required>
+  <Expandable title="options" defaultOpen>
+    <ResponseField name="schema" type="object">
+      A [Zod](/documentation/guides/zod) schema that defines the shape of the
+      job payload. The default is `z.any()` which is `any`. This will be used to correctly type the `Job.invoke()` method.
+    </ResponseField>
+    <ResponseField name="examples" type="array">
+       Used to provide example payloads that are accepted by the job.
+
+       This will be available in the dashboard and can be used to trigger test runs.
+
+       <Expandable title="example object properties" defaultOpen>
+        <ResponseField name="id" type="string" required>
+          The example's ID.
+        </ResponseField>
+        <ResponseField name="name" type="string" required>
+          The name that's displayed in the dashboard.
+        </ResponseField>
+        <ResponseField name="payload" type="object" required>
+          The payload that's accepted by the job.
+        </ResponseField>
+        <ResponseField name="icon" type="string">
+          The icon to use for this example in the dashboard.
+        </ResponseField>
+      </Expandable>
+    </ResponseField>
+
+  </Expandable>
+</ResponseField>
+
+<RequestExample>
+
+```typescript invokeTrigger.ts
+//this Job subscribes to an event called new.user
+export const exampleJob = client.defineJob({
+  id: "example-job",
+  name: "Example job",
+  version: "1.0.1",
+  trigger: invokeTrigger({
+    //the expected payload shape
+    schema: z.object({
+      userId: z.string(),
+      tier: z.union([z.literal("free"), z.literal("pro")]),
+    }),
+    //(optional) example payload object
+    examples: [
+      {
+        id: "issue.opened",
+        name: "Issue opened",
+        payload: {
+          userId: "1234",
+          tier: "free",
+        },
+        //optional
+        icon: "github",
+      },
+    ],
+  }),
+  run: async (payload, io, ctx) => {
+    // do something with the payload
+  },
+});
+```
+
+```typescript usage.ts
+import { exampleJob } from "./invokeTrigger";
+
+//invoke the job with the payload
+await exampleJob.invoke({
+  userId: "1234",
+  tier: "free",
+});
+```
+
+</RequestExample>

docs/sdk/io/backgroundfetch.mdx

@@ -98,6 +98,44 @@ An individual retrying strategy can be one of two types:
 
 </ResponseField>
 
+<ResponseField name="timeout options" type="object">
+  Allows you to set timeouts for the request, as well as specific retry strategies for timeouts.
+
+{" "}
+
+<Expandable title="options" defaultOpen>
+  <ResponseField name="durationInMs" type="number" required>
+    The amount of time to wait before timing out the request. The minimum value is 1s and max is 2min
+  </ResponseField>
+  <ResponseField name="retry" type="RetryOptions">
+    
+    <Expandable title="options">
+
+{" "}
+
+<ResponseField name="limit" type="number">
+  The maximum number of times to retry the request.
+</ResponseField>
+<ResponseField name="minTimeoutInMs" type="number">
+  The minimum amount of time to wait before retrying the request.
+</ResponseField>
+<ResponseField name="maxTimeoutInMs" type="number">
+  The maximum amount of time to wait before retrying the request.
+</ResponseField>
+<ResponseField name="factor" type="number">
+  The exponential factor to use when calculating the next retry time.
+</ResponseField>
+<ResponseField name="randomize" type="boolean">
+  Whether to randomize the retry time.
+</ResponseField>
+
+    </Expandable>
+
+  </ResponseField>
+</Expandable>
+
+</ResponseField>
+
 </ResponseField>
 
 ## Returns
@@ -106,7 +144,7 @@ A `Promise` that resolves after the specified amount of time.
 
 <RequestExample>
 
-```typescript backgroundFetch example
+```typescript retrying
 client.defineJob({
   id: "background-fetch-job",
   name: "Background fetch Job",
@@ -128,8 +166,52 @@ client.defineJob({
       },
       {
         "429": {
+          strategy: "headers",
+          limitHeader: "x-ratelimit-limit",
+          remainingHeader: "x-ratelimit-remaining",
+          resetHeader: "x-ratelimit-reset",
+        },
+        "5xx": {
           strategy: "backoff",
-          limit: 10,
+          limit: 5,
+          minTimeoutInMs: 1000,
+          maxTimeoutInMs: 30000,
+          factor: 1.8,
+          randomize: true,
+        },
+      }
+    );
+
+    return response;
+  },
+});
+```
+
+```typescript timeouts
+client.defineJob({
+  id: "background-fetch-job",
+  name: "Background fetch Job",
+  version: "0.0.1",
+  trigger: eventTrigger({
+    name: "example.event",
+  }),
+  run: async (payload, io, ctx) => {
+    const response = io.backgroundFetch<CreateChatCompetionResponseData>(
+      "fetch-some-data",
+      "https://example.com/api/endpoint",
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: redactString`Bearer ${auth.apiKey}`,
+        },
+        body: JSON.stringify({ foo: "bar" }),
+      },
+      {},
+      {
+        durationInMs: 10000, // 10 seconds
+        retry: {
+          limit: 6,
           minTimeoutInMs: 1000,
           maxTimeoutInMs: 60000,
           factor: 2,

docs/sdk/job.mdx

@@ -5,33 +5,10 @@ description: "Job is used to create and configure a Job"
 
 A [Job](/documentation/concepts/jobs) is used to define the [Trigger](/documentation/concepts/triggers), metadata, and what happens when it runs.
 
-You can define a job in one of two ways, using the `new Job` constructor or by using the `TriggerClient.defineJob` instance method.
+You can define a job by using the `TriggerClient.defineJob` instance method:
 
 <RequestExample>
 
-```ts constructor
-new Job(client, {
-  id: "slack-kpi-summary",
-  name: "Slack kpi summary",
-  version: "0.1.1",
-  integrations: {
-    slack,
-  },
-  trigger: cronTrigger({
-    cron: "0 9 * * *", // 9am every day (UTC)
-  }),
-  run: async (payload, io, ctx) => {
-    const { revenue } = await db.getKpiSummary(payload.ts);
-    const response = await io.slack.postMessage("Slack 📝", {
-      text: `Yesterday's revenue was $${revenue}`,
-      channel: "C04GWUTDC3W",
-    });
-
-    return response;
-  },
-});
-```
-
 ```ts client.defineJob
 client.defineJob({
   id: "github-integration-on-issue",

docs/sdk/run-notification.mdx

@@ -0,0 +1,118 @@
+---
+title: "RunNotification"
+sidebarTitle: "RunNotification"
+description: "The payload of a Run's completion webhook"
+---
+
+This document describes the payload of a Run's completion webhook.
+
+## Properties
+
+<ResponseField name="id" type="string" required>
+  The Run's ID
+</ResponseField>
+
+<ResponseField name="ok" type="boolean" required>
+  Whether the Run completed successfully or failed.
+</ResponseField>
+
+<ResponseField name="status" type="JobRunStatus" required>
+  The Run's status. Will always be `"SUCCESS"` if `ok` is `true`. If `ok` is `false`, will be one of
+  `"FAILURE" | "TIMED_OUT" | "ABORTED" | "CANCELED" | "UNRESOLVED_AUTH" | "INVALID_PAYLOAD"`
+</ResponseField>
+
+<ResponseField name="statuses" type="JobRunStatusRecord[]" required>
+  An array of the explicit run statuses that were created during the Run. See [Explicit Status
+  Hooks](/documentation/guides/react-hooks-statuses#useeventrunstatuses) for more.
+</ResponseField>
+
+<ResponseField name="output" type="any">
+  If the Run completed successfully, this will be the output of the Job.
+</ResponseField>
+
+<ResponseField name="error" type="any">
+  If the Run failed, this will be the error that caused the failure.
+</ResponseField>
+
+<ResponseField name="startedAt" type="Date" required>
+  When the Run started
+</ResponseField>
+<ResponseField name="completedAt" type="Date" required>
+  When the Run started
+</ResponseField>
+<ResponseField name="executionDurationInMs" type="number" required>
+  The duration of the Run in milliseconds
+</ResponseField>
+<ResponseField name="executionCount" type="number" required>
+  The number of individual function executions performed to complete the Run
+</ResponseField>
+
+<ResponseField name="job" type="object" required>
+  Metadata about the Job
+  <Expandable title="properties">
+    <ResponseField name="id" type="string" required>
+      The Job's ID
+    </ResponseField>
+    <ResponseField name="version" type="string" required>
+      The Job's version
+    </ResponseField>
+  </Expandable>
+</ResponseField>
+
+<ResponseField name="environment" type="object" required>
+  Metadata about the Environment
+  <Expandable title="properties">
+    <ResponseField name="slug" type="string" required>
+      The Environment's slug
+    </ResponseField>
+    <ResponseField name="id" type="string" required>
+      The Environment's ID
+    </ResponseField>
+    <ResponseField name="type" type="string" required>
+      The Environment's type. One of: "PRODUCTION", "STAGING", "DEVELOPMENT", "PREVIEW".
+    </ResponseField>
+  </Expandable>
+</ResponseField>
+
+<ResponseField name="organization" type="object" required>
+  Metadata about the Organization
+  <Expandable title="properties">
+    <ResponseField name="slug" type="string" required>
+      The Organization's slug
+    </ResponseField>
+    <ResponseField name="id" type="string" required>
+      The Organization's ID
+    </ResponseField>
+    <ResponseField name="title" type="string" required>
+      The Organization's title
+    </ResponseField>
+  </Expandable>
+</ResponseField>
+
+<ResponseField name="project" type="object" required>
+  Metadata about the Project
+  <Expandable title="properties">
+    <ResponseField name="slug" type="string" required>
+      The Project's slug
+    </ResponseField>
+    <ResponseField name="id" type="string" required>
+      The Project's ID
+    </ResponseField>
+    <ResponseField name="name" type="string" required>
+      The Project's title
+    </ResponseField>
+  </Expandable>
+</ResponseField>
+
+<ResponseField name="account" type="object">
+  Metadata about the Account that triggered the Run. See the [BYO Auth
+  docs](/documentation/guides/using-integrations-byo-auth) for more information.
+  <Expandable title="properties">
+    <ResponseField name="id" type="string" required>
+      The Account's ID
+    </ResponseField>
+    <ResponseField name="metadata" type="any">
+      The Account's metadata. This is additional data that been passed through
+    </ResponseField>
+  </Expandable>
+</ResponseField>

integrations/openai/src/chat.ts

@@ -8,6 +8,7 @@ import {
   createTaskUsageProperties,
 } from "./taskUtils";
 import { OpenAIIntegrationOptions, OpenAIRequestOptions } from "./types";
+import { FetchRetryOptions, FetchTimeoutOptions } from "@trigger.dev/integration-kit";
 
 export class Chat {
   constructor(
@@ -47,7 +48,8 @@ export class Chat {
     backgroundCreate: (
       key: IntegrationTaskKey,
       params: Prettify<OpenAI.Chat.ChatCompletionCreateParamsNonStreaming>,
-      options: OpenAIRequestOptions = {}
+      options: OpenAIRequestOptions = {},
+      fetchOptions: { retries?: FetchRetryOptions; timeout?: FetchTimeoutOptions } = {}
     ): Promise<OpenAI.Chat.ChatCompletion> => {
       return this.runTask(
         key,
@@ -72,7 +74,8 @@ export class Chat {
               ),
               body: JSON.stringify(params),
             },
-            backgroundTaskRetries
+            fetchOptions.retries ?? backgroundTaskRetries,
+            fetchOptions.timeout
           );
 
           task.outputProperties = createTaskUsageProperties(response.usage);
@@ -88,6 +91,9 @@ export class Chat {
               text: params.model,
             },
           ],
+          retry: {
+            limit: 0,
+          },
         }
       );
     },

integrations/openai/src/completions.ts

@@ -8,6 +8,7 @@ import {
   createTaskUsageProperties,
 } from "./taskUtils";
 import { OpenAIIntegrationOptions, OpenAIRequestOptions } from "./types";
+import { FetchRetryOptions, FetchTimeoutOptions } from "@trigger.dev/integration-kit";
 
 export class Completions {
   constructor(
@@ -46,7 +47,8 @@ export class Completions {
   backgroundCreate(
     key: IntegrationTaskKey,
     params: Prettify<OpenAI.CompletionCreateParamsNonStreaming>,
-    options: OpenAIRequestOptions = {}
+    options: OpenAIRequestOptions = {},
+    fetchOptions: { retries?: FetchRetryOptions; timeout?: FetchTimeoutOptions } = {}
   ): Promise<OpenAI.Completion> {
     return this.runTask(
       key,
@@ -71,7 +73,8 @@ export class Completions {
             ),
             body: JSON.stringify(params),
           },
-          backgroundTaskRetries
+          fetchOptions.retries ?? backgroundTaskRetries,
+          fetchOptions.timeout
         );
 
         task.outputProperties = createTaskUsageProperties(response.usage);
@@ -87,6 +90,9 @@ export class Completions {
             text: params.model,
           },
         ],
+        retry: {
+          limit: 0,
+        },
       }
     );
   }

packages/core/src/index.ts

@@ -9,6 +9,7 @@ export * from "./eventFilterMatches";
 
 export const API_VERSIONS = {
   LAZY_LOADED_CACHED_TASKS: "2023-09-29",
+  SERIALIZED_TASK_OUTPUT: "2023-11-01",
 } as const;
 
 export const PLATFORM_FEATURES = {

packages/core/src/logger.ts

@@ -43,36 +43,37 @@ export class Logger {
   log(message: string, ...args: Array<Record<string, unknown> | undefined>) {
     if (this.#level < 0) return;
 
-    this.#structuredLog(console.log, message, ...args);
+    this.#structuredLog(console.log, message, "log", ...args);
   }
 
   error(message: string, ...args: Array<Record<string, unknown> | undefined>) {
     if (this.#level < 1) return;
 
-    this.#structuredLog(console.error, message, ...args);
+    this.#structuredLog(console.error, message, "error", ...args);
   }
 
   warn(message: string, ...args: Array<Record<string, unknown> | undefined>) {
     if (this.#level < 2) return;
 
-    this.#structuredLog(console.warn, message, ...args);
+    this.#structuredLog(console.warn, message, "warn", ...args);
   }
 
   info(message: string, ...args: Array<Record<string, unknown> | undefined>) {
     if (this.#level < 3) return;
 
-    this.#structuredLog(console.info, message, ...args);
+    this.#structuredLog(console.info, message, "info", ...args);
   }
 
   debug(message: string, ...args: Array<Record<string, unknown> | undefined>) {
     if (this.#level < 4) return;
 
-    this.#structuredLog(console.debug, message, ...args);
+    this.#structuredLog(console.debug, message, "debug", ...args);
   }
 
   #structuredLog(
     loggerFunction: (message: string, ...args: any[]) => void,
     message: string,
+    level: string,
     ...args: Array<Record<string, unknown> | undefined>
   ) {
     const structuredLog = {
@@ -80,6 +81,7 @@ export class Logger {
       timestamp: new Date(),
       name: this.#name,
       message,
+      level,
     };
 
     loggerFunction(JSON.stringify(structuredLog, createReplacer(this.#jsonReplacer)));

packages/core/src/schemas/api.ts

@@ -485,6 +485,13 @@ export const RunJobBodySchema = z.object({
     title: z.string(),
     slug: z.string(),
   }),
+  project: z
+    .object({
+      id: z.string(),
+      name: z.string(),
+      slug: z.string(),
+    })
+    .optional(),
   account: z
     .object({
       id: z.string(),
@@ -518,27 +525,29 @@ export const RunJobYieldExecutionErrorSchema = z.object({
 
 export type RunJobYieldExecutionError = z.infer<typeof RunJobYieldExecutionErrorSchema>;
 
-export const RunJobAutoYieldExecutionErrorSchema = z.object({
+export const AutoYieldMetadataSchema = z.object({
-  status: z.literal("AUTO_YIELD_EXECUTION"),
   location: z.string(),
   timeRemaining: z.number(),
   timeElapsed: z.number(),
   limit: z.number().optional(),
 });
 
-export type RunJobAutoYieldExecutionError = z.infer<typeof RunJobAutoYieldExecutionErrorSchema>;
+export type AutoYieldMetadata = z.infer<typeof AutoYieldMetadataSchema>;
+
+export const RunJobAutoYieldExecutionErrorSchema = AutoYieldMetadataSchema.extend({
+  status: z.literal("AUTO_YIELD_EXECUTION"),
+});
+
+export type RunJobAutoYieldExecutionError = Prettify<
+  z.infer<typeof RunJobAutoYieldExecutionErrorSchema>
+>;
 
 export const RunJobAutoYieldWithCompletedTaskExecutionErrorSchema = z.object({
   status: z.literal("AUTO_YIELD_EXECUTION_WITH_COMPLETED_TASK"),
   id: z.string(),
   properties: z.array(DisplayPropertySchema).optional(),
-  output: z.any(),
+  output: z.string().optional(),
-  data: z.object({
+  data: AutoYieldMetadataSchema,
-    location: z.string(),
-    timeRemaining: z.number(),
-    timeElapsed: z.number(),
-    limit: z.number().optional(),
-  }),
 });
 
 export type RunJobAutoYieldWithCompletedTaskExecutionError = z.infer<
@@ -589,6 +598,28 @@ export const RunJobSuccessSchema = z.object({
 
 export type RunJobSuccess = z.infer<typeof RunJobSuccessSchema>;
 
+export const RunJobErrorResponseSchema = z.union([
+  RunJobAutoYieldExecutionErrorSchema,
+  RunJobAutoYieldWithCompletedTaskExecutionErrorSchema,
+  RunJobYieldExecutionErrorSchema,
+  RunJobErrorSchema,
+  RunJobUnresolvedAuthErrorSchema,
+  RunJobInvalidPayloadErrorSchema,
+  RunJobResumeWithTaskSchema,
+  RunJobRetryWithTaskSchema,
+  RunJobCanceledWithTaskSchema,
+]);
+
+export type RunJobErrorResponse = z.infer<typeof RunJobErrorResponseSchema>;
+
+export const RunJobResumeWithParallelTaskSchema = z.object({
+  status: z.literal("RESUME_WITH_PARALLEL_TASK"),
+  task: TaskSchema,
+  childErrors: z.array(RunJobErrorResponseSchema),
+});
+
+export type RunJobResumeWithParallelTask = z.infer<typeof RunJobResumeWithParallelTaskSchema>;
+
 export const RunJobResponseSchema = z.discriminatedUnion("status", [
   RunJobAutoYieldExecutionErrorSchema,
   RunJobAutoYieldWithCompletedTaskExecutionErrorSchema,
@@ -597,6 +628,7 @@ export const RunJobResponseSchema = z.discriminatedUnion("status", [
   RunJobUnresolvedAuthErrorSchema,
   RunJobInvalidPayloadErrorSchema,
   RunJobResumeWithTaskSchema,
+  RunJobResumeWithParallelTaskSchema,
   RunJobRetryWithTaskSchema,
   RunJobCanceledWithTaskSchema,
   RunJobSuccessSchema,
@@ -737,6 +769,7 @@ export const RunTaskOptionsSchema = z.object({
   noop: z.boolean().default(false),
   redact: RedactSchema.optional(),
   trigger: TriggerMetadataSchema.optional(),
+  parallel: z.boolean().optional(),
 });
 
 export type RunTaskOptions = z.input<typeof RunTaskOptionsSchema>;
@@ -793,6 +826,16 @@ export const CompleteTaskBodyInputSchema = RunTaskBodyInputSchema.pick({
 export type CompleteTaskBodyInput = Prettify<z.input<typeof CompleteTaskBodyInputSchema>>;
 export type CompleteTaskBodyOutput = z.infer<typeof CompleteTaskBodyInputSchema>;
 
+export const CompleteTaskBodyV2InputSchema = RunTaskBodyInputSchema.pick({
+  properties: true,
+  description: true,
+  params: true,
+}).extend({
+  output: z.string().optional(),
+});
+
+export type CompleteTaskBodyV2Input = Prettify<z.input<typeof CompleteTaskBodyV2InputSchema>>;
+
 export const FailTaskBodyInputSchema = z.object({
   error: ErrorWithStackSchema,
 });
@@ -896,3 +939,29 @@ export const GetRunStatusesSchema = z.object({
   statuses: z.array(JobRunStatusRecordSchema),
 });
 export type GetRunStatuses = z.infer<typeof GetRunStatusesSchema>;
+
+export const InvokeJobResponseSchema = z.object({
+  id: z.string(),
+});
+
+export const InvokeJobRequestBodySchema = z.object({
+  payload: z.any(),
+  context: z.any().optional(),
+  options: z
+    .object({
+      accountId: z.string().optional(),
+      callbackUrl: z.string().optional(),
+    })
+    .optional(),
+});
+
+export type InvokeJobRequestBody = z.infer<typeof InvokeJobRequestBodySchema>;
+
+export const InvokeOptionsSchema = z.object({
+  accountId: z.string().optional(),
+  idempotencyKey: z.string().optional(),
+  context: z.any().optional(),
+  callbackUrl: z.string().optional(),
+});
+
+export type InvokeOptions = z.infer<typeof InvokeOptionsSchema>;

packages/core/src/schemas/fetch.ts

@@ -52,10 +52,18 @@ export const FetchRetryOptionsSchema = z.record(FetchRetryStrategySchema);
   */
 export type FetchRetryOptions = z.infer<typeof FetchRetryOptionsSchema>;
 
+export const FetchTimeoutOptionsSchema = z.object({
+  durationInMs: z.number(),
+  retry: RetryOptionsSchema.optional(),
+});
+
+export type FetchTimeoutOptions = z.infer<typeof FetchTimeoutOptionsSchema>;
+
 export const FetchOperationSchema = z.object({
   url: z.string(),
   requestInit: FetchRequestInitSchema.optional(),
   retry: z.record(FetchRetryStrategySchema).optional(),
+  timeout: FetchTimeoutOptionsSchema.optional(),
 });
 
 export type FetchOperation = z.infer<typeof FetchOperationSchema>;

packages/core/src/schemas/runs.ts

@@ -1,6 +1,8 @@
 import { z } from "zod";
 import { TaskStatusSchema } from "./tasks";
-import { JobRunStatusRecordSchema } from "./statuses";
+import { JobRunStatusRecord, JobRunStatusRecordSchema } from "./statuses";
+import { Prettify } from "../types";
+import { RuntimeEnvironmentType } from "./api";
 
 export const RunStatusSchema = z.union([
   z.literal("PENDING"),
@@ -87,7 +89,7 @@ export const GetRunSchema = RunSchema.extend({
   nextCursor: z.string().optional(),
 });
 
-export type GetRun = z.infer<typeof GetRunSchema>;
+export type GetRun = Prettify<z.infer<typeof GetRunSchema>>;
 
 const GetRunsOptionsSchema = z.object({
   /** You can use this to get more tasks, if there are more than are returned in a single batch @default undefined */
@@ -104,3 +106,50 @@ export const GetRunsSchema = z.object({
   /** If there are more runs, you can use this to get them */
   nextCursor: z.string().optional(),
 });
+
+type RunNotificationCommon = {
+  /** The Run id */
+  id: string;
+  /** The Run status */
+  statuses: JobRunStatusRecord[];
+  /** When the run started */
+  startedAt: Date;
+  /** When the run was last updated */
+  updatedAt: Date;
+  /** When the run was completed */
+  completedAt: Date;
+
+  executionDurationInMs: number;
+  executionCount: number;
+
+  /** Job metadata */
+  job: { id: string; version: string };
+  /** Environment metadata */
+  environment: { slug: string; id: string; type: RuntimeEnvironmentType };
+  /** Organization metadata */
+  organization: { slug: string; id: string; title: string };
+  /** Project metadata */
+  project: { slug: string; id: string; name: string };
+  /** Account metadata */
+  account?: { id: string; metadata?: any };
+  /** Invocation metadata */
+  invocation: { id: string; context: any; timestamp: Date };
+};
+
+export type SuccessfulRunNotification<TOutput> = RunNotificationCommon & {
+  ok: true;
+  /** The Run status */
+  status: "SUCCESS";
+  /** The output of the run */
+  output: TOutput;
+};
+
+export type FailedRunNotification = RunNotificationCommon & {
+  ok: false;
+  /** The Run status */
+  status: "FAILURE" | "TIMED_OUT" | "ABORTED" | "CANCELED" | "UNRESOLVED_AUTH" | "INVALID_PAYLOAD";
+  /** The error of the run */
+  error: any;
+};
+
+export type RunNotification<TOutput> = SuccessfulRunNotification<TOutput> | FailedRunNotification;

packages/core/src/schemas/statuses.ts

@@ -29,3 +29,5 @@ export const JobRunStatusRecordSchema = InitalStatusUpdateSchema.extend({
   key: z.string(),
   history: StatusHistorySchema,
 });
+
+export type JobRunStatusRecord = z.infer<typeof JobRunStatusRecordSchema>;

packages/core/src/schemas/tasks.ts

@@ -32,6 +32,7 @@ export const TaskSchema = z.object({
   style: StyleSchema.optional().nullable(),
   operation: z.string().optional().nullable(),
   callbackUrl: z.string().optional().nullable(),
+  childExecutionMode: z.enum(["SEQUENTIAL", "PARALLEL"]).optional().nullable(),
 });
 
 export const ServerTaskSchema = TaskSchema.extend({

packages/core/src/schemas/triggers.ts

@@ -35,6 +35,10 @@ export const StaticTriggerMetadataSchema = z.object({
   rule: EventRuleSchema,
 });
 
+export const InvokeTriggerMetadataSchema = z.object({
+  type: z.literal("invoke"),
+});
+
 export const ScheduledTriggerMetadataSchema = z.object({
   type: z.literal("scheduled"),
   schedule: ScheduleMetadataSchema,
@@ -44,6 +48,7 @@ export const TriggerMetadataSchema = z.discriminatedUnion("type", [
   DynamicTriggerMetadataSchema,
   StaticTriggerMetadataSchema,
   ScheduledTriggerMetadataSchema,
+  InvokeTriggerMetadataSchema,
 ]);
 
 export type TriggerMetadata = z.infer<typeof TriggerMetadataSchema>;

packages/database/prisma/migrations/20231028193441_prepare_for_invoke_trigger/migration.sql

@@ -0,0 +1,29 @@
+-- CreateEnum
+CREATE TYPE "JobRunSubscriptionRecipientMethod" AS ENUM ('WEBHOOK');
+
+-- CreateEnum
+CREATE TYPE "JobRunSubscriptionStatus" AS ENUM ('ACTIVE', 'INACTIVE');
+
+-- CreateEnum
+CREATE TYPE "JobRunSubscriptionEvents" AS ENUM ('SUCCESS', 'FAILURE');
+
+-- AlterTable
+ALTER TABLE "EventRecord" ADD COLUMN     "internal" BOOLEAN NOT NULL DEFAULT false;
+
+-- CreateTable
+CREATE TABLE "JobRunSubscription" (
+    "id" TEXT NOT NULL,
+    "runId" TEXT NOT NULL,
+    "recipient" TEXT NOT NULL,
+    "recipientMethod" "JobRunSubscriptionRecipientMethod" NOT NULL DEFAULT 'WEBHOOK',
+    "event" "JobRunSubscriptionEvents" NOT NULL,
+    "status" "JobRunSubscriptionStatus" NOT NULL DEFAULT 'ACTIVE',
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+    "deliveredAt" TIMESTAMP(3),
+
+    CONSTRAINT "JobRunSubscription_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "JobRunSubscription" ADD CONSTRAINT "JobRunSubscription_runId_fkey" FOREIGN KEY ("runId") REFERENCES "JobRun"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/database/prisma/migrations/20231101105021_add_child_execution_mode_to_tasks/migration.sql

@@ -0,0 +1,5 @@
+-- CreateEnum
+CREATE TYPE "TaskChildExecutionMode" AS ENUM ('SEQUENTIAL', 'PARALLEL');
+
+-- AlterTable
+ALTER TABLE "Task" ADD COLUMN     "childExecutionMode" "TaskChildExecutionMode" NOT NULL DEFAULT 'SEQUENTIAL';

packages/database/prisma/schema.prisma

@@ -673,8 +673,9 @@ model EventRecord {
   updatedAt   DateTime  @updatedAt
   cancelledAt DateTime?
 
-  isTest Boolean  @default(false)
+  isTest   Boolean  @default(false)
-  runs   JobRun[]
+  internal Boolean  @default(false)
+  runs     JobRun[]
 
   @@unique([eventId, environmentId])
 }
@@ -741,6 +742,7 @@ model JobRun {
   executions         JobRunExecution[]
   statuses           JobRunStatusRecord[]
   autoYieldExecution JobRunAutoYieldExecution[]
+  subscriptions      JobRunSubscription[]
 }
 
 enum JobRunStatus {
@@ -772,6 +774,38 @@ model JobRunAutoYieldExecution {
   createdAt DateTime @default(now())
 }
 
+model JobRunSubscription {
+  id String @id @default(cuid())
+
+  run   JobRun @relation(fields: [runId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  runId String
+
+  recipient       String
+  recipientMethod JobRunSubscriptionRecipientMethod @default(WEBHOOK)
+
+  event  JobRunSubscriptionEvents
+  status JobRunSubscriptionStatus @default(ACTIVE)
+
+  createdAt DateTime @default(now())
+  updatedAt DateTime @updatedAt
+
+  deliveredAt DateTime?
+}
+
+enum JobRunSubscriptionRecipientMethod {
+  WEBHOOK
+}
+
+enum JobRunSubscriptionStatus {
+  ACTIVE
+  INACTIVE
+}
+
+enum JobRunSubscriptionEvents {
+  SUCCESS
+  FAILURE
+}
+
 model JobRunExecution {
   id String @id @default(cuid())
 
@@ -849,9 +883,10 @@ model Task {
   runConnection   RunConnection? @relation(fields: [runConnectionId], references: [id], onDelete: Cascade, onUpdate: Cascade)
   runConnectionId String?
 
-  children   Task[]            @relation("TaskParent")
+  children           Task[]                 @relation("TaskParent")
-  executions JobRunExecution[]
+  childExecutionMode TaskChildExecutionMode @default(SEQUENTIAL)
-  attempts   TaskAttempt[]
+  executions         JobRunExecution[]
+  attempts           TaskAttempt[]
 
   @@unique([runId, idempotencyKey])
 }
@@ -865,6 +900,11 @@ enum TaskStatus {
   CANCELED
 }
 
+enum TaskChildExecutionMode {
+  SEQUENTIAL
+  PARALLEL
+}
+
 model TaskAttempt {
   id String @id @default(cuid())
 

packages/integration-kit/package.json

@@ -34,9 +34,10 @@
     "typecheck": "tsup --dts-resolve --no-dts"
   },
   "dependencies": {
-    "uuid": "^9.0.0"
+    "uuid": "^9.0.0",
+    "@trigger.dev/core": "workspace:*"
   },
   "engines": {
     "node": ">=18.0.0"
   }
-}
+}

packages/integration-kit/src/index.ts

@@ -4,3 +4,4 @@ export * from "./omit";
 export * from "./properties";
 export * from "./file";
 export * from "./prettify";
+export * from "./types";

packages/integration-kit/src/types.ts

@@ -0,0 +1,3 @@
+import type { FetchRetryOptions, FetchTimeoutOptions } from "@trigger.dev/core";
+
+export type { FetchRetryOptions, FetchTimeoutOptions };

packages/integration-kit/tsconfig.json

@@ -4,7 +4,10 @@
   "compilerOptions": {
     "experimentalDecorators": true,
     "emitDecoratorMetadata": true,
-    "paths": {},
+    "paths": {
+      "@trigger.dev/core/*": ["../core/src/*"],
+      "@trigger.dev/core": ["../core/src/index"]
+    },
     "lib": ["DOM", "DOM.Iterable"],
     "declaration": false,
     "declarationMap": false

packages/trigger-sdk/src/apiClient.ts

@@ -30,6 +30,10 @@ import {
   urlWithSearchParams,
   RunTaskResponseWithCachedTasksBodySchema,
   API_VERSIONS,
+  InvokeJobResponseSchema,
+  InvokeOptions,
+  InvokeJobRequestBody,
+  CompleteTaskBodyV2Input,
 } from "@trigger.dev/core";
 
 import { z } from "zod";
@@ -139,7 +143,7 @@ export class ApiClient {
     );
   }
 
-  async completeTask(runId: string, id: string, task: CompleteTaskBodyInput) {
+  async completeTask(runId: string, id: string, task: CompleteTaskBodyV2Input) {
     const apiKey = await this.#apiKey();
 
     this.#logger.debug("Complete Task", {
@@ -154,6 +158,7 @@ export class ApiClient {
         headers: {
           "Content-Type": "application/json",
           Authorization: `Bearer ${apiKey}`,
+          "Trigger-Version": API_VERSIONS.SERIALIZED_TASK_OUTPUT,
         },
         body: JSON.stringify(task),
       }
@@ -476,6 +481,33 @@ export class ApiClient {
     );
   }
 
+  async invokeJob(jobId: string, payload: any, options: InvokeOptions = {}) {
+    const apiKey = await this.#apiKey();
+
+    this.#logger.debug("Invoking Job", {
+      jobId,
+    });
+
+    const body: InvokeJobRequestBody = {
+      payload,
+      context: options.context ?? {},
+      options: {
+        accountId: options.accountId,
+        callbackUrl: options.callbackUrl,
+      },
+    };
+
+    return await zodfetch(InvokeJobResponseSchema, `${this.#apiUrl}/api/v1/jobs/${jobId}/invoke`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${apiKey}`,
+        ...(options.idempotencyKey ? { "Idempotency-Key": options.idempotencyKey } : {}),
+      },
+      body: JSON.stringify(body),
+    });
+  }
+
   async #apiKey() {
     const apiKey = getApiKey(this.#options.apiKey);
 
@@ -573,7 +605,8 @@ async function zodfetchWithVersions<
   options?: {
     errorMessage?: string;
     optional?: TOptional;
-  }
+  },
+  retryCount = 0
 ): Promise<
   TOptional extends true
     ? VersionedResponseBody<TVersionedResponseBodyMap, TUnversionedResponseBodySchema> | undefined
@@ -596,6 +629,22 @@ async function zodfetchWithVersions<
     throw new Error(body.error);
   }
 
+  if (response.status >= 500 && retryCount < 6) {
+    // retry with exponential backoff and jitter
+    const delay = exponentialBackoff(retryCount + 1, 2, 50, 1150, 50);
+
+    await new Promise((resolve) => setTimeout(resolve, delay));
+
+    return zodfetchWithVersions(
+      versionedSchemaMap,
+      unversionedSchema,
+      url,
+      requestInit,
+      options,
+      retryCount + 1
+    );
+  }
+
   if (response.status !== 200) {
     throw new Error(
       options?.errorMessage ?? `Failed to fetch ${url}, got status code ${response.status}`
@@ -632,7 +681,8 @@ async function zodfetch<TResponseSchema extends z.ZodTypeAny, TOptional extends
   options?: {
     errorMessage?: string;
     optional?: TOptional;
-  }
+  },
+  retryCount = 0
 ): Promise<
   TOptional extends true ? z.infer<TResponseSchema> | undefined : z.infer<TResponseSchema>
 > {
@@ -653,6 +703,15 @@ async function zodfetch<TResponseSchema extends z.ZodTypeAny, TOptional extends
     throw new Error(body.error);
   }
 
+  if (response.status >= 500 && retryCount < 6) {
+    // retry with exponential backoff and jitter
+    const delay = exponentialBackoff(retryCount + 1, 2, 50, 1150, 50);
+
+    await new Promise((resolve) => setTimeout(resolve, delay));
+
+    return zodfetch(schema, url, requestInit, options, retryCount + 1);
+  }
+
   if (response.status !== 200) {
     throw new Error(
       options?.errorMessage ?? `Failed to fetch ${url}, got status code ${response.status}`
@@ -663,3 +722,20 @@ async function zodfetch<TResponseSchema extends z.ZodTypeAny, TOptional extends
 
   return schema.parse(jsonBody);
 }
+
+function exponentialBackoff(
+  retryCount: number,
+  exponential: number,
+  minDelay: number,
+  maxDelay: number,
+  jitter: number
+): number {
+  // Calculate the delay using the exponential backoff formula
+  const delay = Math.min(Math.pow(exponential, retryCount) * minDelay, maxDelay);
+
+  // Calculate the jitter
+  const jitterValue = Math.random() * jitter;
+
+  // Return the calculated delay with jitter
+  return delay + jitterValue;
+}

packages/trigger-sdk/src/errors.ts

@@ -5,6 +5,13 @@ export class ResumeWithTaskError {
   constructor(public task: ServerTask) {}
 }
 
+export class ResumeWithParallelTaskError {
+  constructor(
+    public task: ServerTask,
+    public childErrors: Array<TriggerInternalError>
+  ) {}
+}
+
 export class RetryWithTaskError {
   constructor(
     public cause: ErrorWithStack,
@@ -33,8 +40,8 @@ export class AutoYieldWithCompletedTaskExecutionError {
   constructor(
     public id: string,
     public properties: DisplayProperty[] | undefined,
-    public output: any,
+    public data: { location: string; timeRemaining: number; timeElapsed: number },
-    public data: { location: string; timeRemaining: number; timeElapsed: number }
+    public output?: string
   ) {}
 }
 
@@ -42,27 +49,39 @@ export class ParsedPayloadSchemaError {
   constructor(public schemaErrors: SchemaError[]) {}
 }
 
+export type TriggerInternalError =
+  | ResumeWithTaskError
+  | RetryWithTaskError
+  | CanceledWithTaskError
+  | YieldExecutionError
+  | AutoYieldExecutionError
+  | AutoYieldWithCompletedTaskExecutionError
+  | ResumeWithParallelTaskError;
+
 /** Use this function if you're using a `try/catch` block to catch errors.
  * It checks if a thrown error is a special internal error that you should ignore.
  * If this returns `true` then you must rethrow the error: `throw err;`
  * @param err The error to check
  * @returns `true` if the error is a Trigger Error, `false` otherwise.
  */
-export function isTriggerError(
+export function isTriggerError(err: unknown): err is TriggerInternalError {
-  err: unknown
-): err is
-  | ResumeWithTaskError
-  | RetryWithTaskError
-  | CanceledWithTaskError
-  | YieldExecutionError
-  | AutoYieldExecutionError
-  | AutoYieldWithCompletedTaskExecutionError {
   return (
     err instanceof ResumeWithTaskError ||
     err instanceof RetryWithTaskError ||
     err instanceof CanceledWithTaskError ||
     err instanceof YieldExecutionError ||
     err instanceof AutoYieldExecutionError ||
-    err instanceof AutoYieldWithCompletedTaskExecutionError
+    err instanceof AutoYieldWithCompletedTaskExecutionError ||
+    err instanceof ResumeWithParallelTaskError
   );
 }
+
+// This error isn't an internal error but it can be used by the user to figure out which task caused the error
+export class ErrorWithTask extends Error {
+  constructor(
+    public cause: ServerTask,
+    message: string
+  ) {
+    super(message);
+  }
+}

packages/trigger-sdk/src/index.ts

@@ -6,6 +6,7 @@ export * from "./triggers/externalSource";
 export * from "./triggers/dynamic";
 export * from "./triggers/scheduled";
 export * from "./triggers/notifications";
+export * from "./triggers/invokeTrigger";
 export * from "./io";
 export * from "./types";
 

packages/trigger-sdk/src/io.ts

@@ -6,6 +6,7 @@ import {
   ErrorWithStackSchema,
   FetchRequestInit,
   FetchRetryOptions,
+  FetchTimeoutOptions,
   InitialStatusUpdate,
   IntervalOptions,
   LogLevel,
@@ -13,7 +14,6 @@ import {
   RunTaskOptions,
   SendEvent,
   SendEventOptions,
-  SerializableJsonSchema,
   ServerTask,
   UpdateTriggerSourceBodyV2,
   supportsFeature,
@@ -26,19 +26,22 @@ import {
   AutoYieldExecutionError,
   AutoYieldWithCompletedTaskExecutionError,
   CanceledWithTaskError,
+  ErrorWithTask,
+  ResumeWithParallelTaskError,
   ResumeWithTaskError,
   RetryWithTaskError,
+  TriggerInternalError,
   YieldExecutionError,
   isTriggerError,
 } from "./errors";
+import { IntegrationTaskKey } from "./integrations";
 import { calculateRetryAt } from "./retry";
+import { TriggerStatus } from "./status";
 import { TriggerClient } from "./triggerClient";
 import { DynamicTrigger } from "./triggers/dynamic";
 import { ExternalSource, ExternalSourceParams } from "./triggers/externalSource";
 import { DynamicSchedule } from "./triggers/scheduled";
 import { EventSpecification, TaskLogger, TriggerContext } from "./types";
-import { IntegrationTaskKey } from "./integrations";
-import { TriggerStatus } from "./status";
 
 export type IOTask = ServerTask;
 
@@ -85,6 +88,21 @@ export type IOStats = {
   noopCachedTaskMisses: number;
 };
 
+export interface OutputSerializer {
+  serialize(value: any): string;
+  deserialize<T>(value: string): T;
+}
+
+export class JSONOutputSerializer implements OutputSerializer {
+  serialize(value: any): string {
+    return JSON.stringify(value);
+  }
+
+  deserialize(value?: string): any {
+    return value ? JSON.parse(value) : undefined;
+  }
+}
+
 export class IO {
   private _id: string;
   private _apiClient: ApiClient;
@@ -102,6 +120,8 @@ export class IO {
   private _serverVersion: string;
   private _timeOrigin: number;
   private _executionTimeout?: number;
+  private _outputSerializer: OutputSerializer = new JSONOutputSerializer();
+  private _visitedCacheKeys: Set<string> = new Set();
 
   get stats() {
     return this._stats;
@@ -284,7 +304,8 @@ export class IO {
     cacheKey: string | any[],
     url: string,
     requestInit?: FetchRequestInit,
-    retry?: FetchRetryOptions
+    retry?: FetchRetryOptions,
+    timeout?: FetchTimeoutOptions
   ): Promise<TResponseData> {
     const urlObject = new URL(url);
 
@@ -295,7 +316,7 @@ export class IO {
       },
       {
         name: `fetch ${urlObject.hostname}${urlObject.pathname}`,
-        params: { url, requestInit, retry },
+        params: { url, requestInit, retry, timeout },
         operation: "fetch",
         icon: "background",
         noop: false,
@@ -313,7 +334,11 @@ export class IO {
             label: "background",
             text: "true",
           },
+          ...(timeout ? [{ label: "timeout", text: `${timeout.durationInMs}ms` }] : []),
         ],
+        retry: {
+          limit: 0,
+        },
       }
     )) as TResponseData;
   }
@@ -583,6 +608,53 @@ export class IO {
     );
   }
 
+  async parallel<T extends Json<T> | void, TItem>(
+    cacheKey: string | any[],
+    items: Array<TItem>,
+    callback: (item: TItem, index: number) => Promise<T>,
+    options?: Pick<RunTaskOptions, "name" | "properties">
+  ): Promise<Array<T>> {
+    const results = await this.runTask(
+      cacheKey,
+      async (task) => {
+        const outcomes = await Promise.allSettled(
+          items.map((item, index) => spaceOut(() => callback(item, index), index, 15))
+        );
+
+        // If all the outcomes are fulfilled, return the values
+        if (outcomes.every((outcome) => outcome.status === "fulfilled")) {
+          return outcomes.map(
+            (outcome) => (outcome as PromiseFulfilledResult<T>).value
+          ) as Array<{}>;
+        }
+
+        // If they any of the errors are non internal errors, throw the first one
+        const nonInternalErrors = outcomes
+          .filter((outcome) => outcome.status === "rejected" && !isTriggerError(outcome.reason))
+          .map((outcome) => outcome as PromiseRejectedResult);
+
+        if (nonInternalErrors.length > 0) {
+          throw nonInternalErrors[0].reason;
+        }
+
+        // gather all the internal errors
+        const internalErrors = outcomes
+          .filter((outcome) => outcome.status === "rejected" && isTriggerError(outcome.reason))
+          .map((outcome) => outcome as PromiseRejectedResult)
+          .map((outcome) => outcome.reason as TriggerInternalError);
+
+        throw new ResumeWithParallelTaskError(task, internalErrors);
+      },
+      {
+        name: "parallel",
+        parallel: true,
+        ...(options ?? {}),
+      }
+    );
+
+    return results as unknown as Array<T>;
+  }
+
   /** `io.runTask()` allows you to run a [Task](https://trigger.dev/docs/documentation/concepts/tasks) from inside a Job run. A Task is a resumable unit of a Run that can be retried, resumed and is logged. [Integrations](https://trigger.dev/docs/integrations) use Tasks internally to perform their actions.
    *
    * @param cacheKey Should be a stable and unique key inside the `run()`. See [resumability](https://trigger.dev/docs/documentation/concepts/resumability) for more information.
@@ -613,6 +685,22 @@ export class IO {
       [this._id, parentId ?? "", cacheKey].flat()
     );
 
+    if (this._visitedCacheKeys.has(idempotencyKey)) {
+      if (typeof cacheKey === "string") {
+        throw new Error(
+          `Task with cacheKey "${cacheKey}" has already been executed in this run. Each task must have a unique cacheKey.`
+        );
+      } else {
+        throw new Error(
+          `Task with cacheKey "${cacheKey.join(
+            "-"
+          )}" has already been executed in this run. Each task must have a unique cacheKey.`
+        );
+      }
+    }
+
+    this._visitedCacheKeys.add(idempotencyKey);
+
     const cachedTask = this._cachedTasks.get(idempotencyKey);
 
     if (cachedTask && cachedTask.status === "COMPLETED") {
@@ -714,7 +802,10 @@ export class IO {
         task,
       });
 
-      throw new Error(task.error ?? task?.output ? JSON.stringify(task.output) : "Task errored");
+      throw new ErrorWithTask(
+        task,
+        task.error ?? task?.output ? JSON.stringify(task.output) : "Task errored"
+      );
     }
 
     this.#detectAutoYield("before_execute_task", 1500);
@@ -731,7 +822,7 @@ export class IO {
           return {} as T;
         }
 
-        const output = SerializableJsonSchema.parse(result) as T;
+        const output = this._outputSerializer.serialize(result);
 
         this._logger.debug("Completing using output", {
           idempotencyKey,
@@ -741,7 +832,7 @@ export class IO {
         this.#detectAutoYield("before_complete_task", 500, task, output);
 
         const completedTask = await this._apiClient.completeTask(this._id, task.id, {
-          output: output ?? undefined,
+          output,
           properties: task.outputProperties ?? undefined,
         });
 
@@ -761,7 +852,7 @@ export class IO {
 
         this.#detectAutoYield("after_complete_task", 500);
 
-        return output;
+        return this._outputSerializer.deserialize<T>(output);
       } catch (error) {
         if (isTriggerError(error)) {
           throw error;
@@ -799,6 +890,13 @@ export class IO {
           }
         }
 
+        if (error instanceof ErrorWithTask) {
+          // This means a subtask errored, so we need to update the parent task and not retry it
+          await this._apiClient.failTask(this._id, task.id, {
+            error: error.cause.output as any,
+          });
+        }
+
         const parsedError = ErrorWithStackSchema.safeParse(error);
 
         if (options?.retry && !skipRetrying) {
@@ -902,7 +1000,7 @@ export class IO {
     this._cachedTasks.set(task.idempotencyKey, task);
   }
 
-  #detectAutoYield(location: string, threshold: number = 1500, task?: ServerTask, output?: any) {
+  #detectAutoYield(location: string, threshold: number = 1500, task?: ServerTask, output?: string) {
     const timeRemaining = this.#getRemainingTimeInMillis();
 
     if (timeRemaining && timeRemaining < threshold) {
@@ -910,12 +1008,12 @@ export class IO {
         throw new AutoYieldWithCompletedTaskExecutionError(
           task.id,
           task.outputProperties ?? [],
-          output,
           {
             location,
             timeRemaining,
             timeElapsed: this.#getTimeElapsed(),
-          }
+          },
+          output
         );
       } else {
         throw new AutoYieldExecutionError(location, timeRemaining, this.#getTimeElapsed());
@@ -1019,3 +1117,10 @@ export class IOLogger implements TaskLogger {
     return this.callback("ERROR", message, properties);
   }
 }
+
+// Space out the execution of the callback by a delay of index * delay
+async function spaceOut<T>(callback: () => Promise<T>, index: number, delay: number): Promise<T> {
+  await new Promise((resolve) => setTimeout(resolve, index * delay));
+
+  return await callback();
+}

packages/trigger-sdk/src/job.ts

@@ -1,12 +1,28 @@
-import { IntegrationConfig, JobMetadata, LogLevel, QueueOptions } from "@trigger.dev/core";
+import {
+  IntegrationConfig,
+  InvokeOptions,
+  JobMetadata,
+  LogLevel,
+  QueueOptions,
+  RunNotification,
+} from "@trigger.dev/core";
 import { IOWithIntegrations, TriggerIntegration } from "./integrations";
 import { TriggerClient } from "./triggerClient";
-import type { EventSpecification, Trigger, TriggerContext, TriggerEventType } from "./types";
+import type {
+  EventSpecification,
+  Trigger,
+  TriggerContext,
+  TriggerEventType,
+  TriggerInvokeType,
+} from "./types";
 import { slugifyId } from "./utils";
+import { runLocalStorage } from "./runLocalStorage";
+import { Prettify } from "@trigger.dev/core";
 
 export type JobOptions<
   TTrigger extends Trigger<EventSpecification<any>>,
   TIntegrations extends Record<string, TriggerIntegration> = {},
+  TOutput extends any = any,
 > = {
   /** The `id` property is used to uniquely identify the Job. Only change this if you want to create a new Job. */
   id: string;
@@ -57,7 +73,7 @@ export type JobOptions<
     payload: TriggerEventType<TTrigger>,
     io: IOWithIntegrations<TIntegrations>,
     context: TriggerContext
-  ) => Promise<any>;
+  ) => Promise<TOutput>;
 
   // @internal
   __internal?: boolean;
@@ -75,8 +91,9 @@ export type JobIO<TJob> = TJob extends Job<any, infer TIntegrations>
 export class Job<
   TTrigger extends Trigger<EventSpecification<any>>,
   TIntegrations extends Record<string, TriggerIntegration> = {},
+  TOutput extends any = any,
 > {
-  readonly options: JobOptions<TTrigger, TIntegrations>;
+  readonly options: JobOptions<TTrigger, TIntegrations, TOutput>;
 
   client: TriggerClient;
 
@@ -84,7 +101,7 @@ export class Job<
     /** An instance of [TriggerClient](/sdk/triggerclient) that is used to send events
   to the Trigger API. */
     client: TriggerClient,
-    options: JobOptions<TTrigger, TIntegrations>
+    options: JobOptions<TTrigger, TIntegrations, TOutput>
   ) {
     this.client = client;
     this.options = options;
@@ -152,6 +169,197 @@ export class Job<
     };
   }
 
+  async invoke(
+    cacheKey: string,
+    payload: TriggerInvokeType<TTrigger>,
+    options?: InvokeOptions
+  ): Promise<{ id: string }>;
+  async invoke(
+    payload: TriggerInvokeType<TTrigger>,
+    options?: InvokeOptions
+  ): Promise<{ id: string }>;
+  async invoke(
+    param1: string | TriggerInvokeType<TTrigger>,
+    param2: TriggerInvokeType<TTrigger> | InvokeOptions | undefined = undefined,
+    param3: InvokeOptions | undefined = undefined
+  ): Promise<{ id: string }> {
+    const runStore = runLocalStorage.getStore();
+
+    if (typeof param1 === "string") {
+      if (!runStore) {
+        throw new Error(
+          "Cannot invoke a job from outside of a run when passing a cacheKey. Make sure you are running the job from within a run or use the invoke method without the cacheKey."
+        );
+      }
+
+      const options = param3 ?? {};
+
+      return await runStore.io.runTask(
+        param1,
+        async (task) => {
+          const result = await this.client.invokeJob(this.id, param2, {
+            idempotencyKey: task.idempotencyKey,
+            ...options,
+          });
+
+          task.outputProperties = [
+            {
+              label: "Run",
+              text: result.id,
+              url: `/orgs/${runStore.ctx.organization.slug}/projects/${runStore.ctx.project.slug}/jobs/${this.id}/runs/${result.id}/trigger`,
+            },
+          ];
+
+          return result;
+        },
+        {
+          name: `Manually Invoke '${this.name}'`,
+          params: param2,
+          properties: [
+            {
+              label: "Job",
+              text: this.id,
+              url: `/orgs/${runStore.ctx.organization.slug}/projects/${runStore.ctx.project.slug}/jobs/${this.id}`,
+            },
+            {
+              label: "Env",
+              text: runStore.ctx.environment.slug,
+            },
+          ],
+        }
+      );
+    }
+
+    if (runStore) {
+      throw new Error("Cannot invoke a job from within a run without a cacheKey.");
+    }
+
+    return await this.client.invokeJob(this.id, param1, param3);
+  }
+
+  async invokeAndWaitForCompletion(
+    cacheKey: string | string[],
+    payload: TriggerInvokeType<TTrigger>,
+    timeoutInSeconds: number = 60 * 60, // 1 hour
+    options: Prettify<Pick<InvokeOptions, "accountId" | "context">> = {}
+  ): Promise<RunNotification<TOutput>> {
+    const runStore = runLocalStorage.getStore();
+
+    if (!runStore) {
+      throw new Error(
+        "Cannot invoke a job from outside of a run using invokeAndWaitForCompletion. Make sure you are running the job from within a run or use the invoke method instead."
+      );
+    }
+
+    const { io, ctx } = runStore;
+
+    return (await io.runTask(
+      cacheKey,
+      async (task) => {
+        const parsedPayload = this.trigger.event.parseInvokePayload
+          ? this.trigger.event.parseInvokePayload(payload)
+            ? payload
+            : undefined
+          : payload;
+
+        const result = await this.client.invokeJob(this.id, parsedPayload, {
+          idempotencyKey: task.idempotencyKey,
+          callbackUrl: task.callbackUrl ?? undefined,
+          ...options,
+        });
+
+        task.outputProperties = [
+          {
+            label: "Run",
+            text: result.id,
+            url: `/orgs/${ctx.organization.slug}/projects/${ctx.project.slug}/jobs/${this.id}/runs/${result.id}/trigger`,
+          },
+        ];
+
+        return {}; // we don't want to return anything here, we just want to wait for the callback
+      },
+      {
+        name: `Manually Invoke '${this.name}' and wait for completion`,
+        params: payload,
+        properties: [
+          {
+            label: "Job",
+            text: this.id,
+            url: `/orgs/${ctx.organization.slug}/projects/${ctx.project.slug}/jobs/${this.id}`,
+          },
+          {
+            label: "Env",
+            text: ctx.environment.slug,
+          },
+        ],
+        callback: {
+          enabled: true,
+          timeoutInSeconds,
+        },
+      }
+    )) as RunNotification<TOutput>;
+  }
+
+  async batchInvokeAndWaitForCompletion(
+    cacheKey: string | string[],
+    batch: Array<{
+      payload: TriggerInvokeType<TTrigger>;
+      timeoutInSeconds?: number;
+      options?: Prettify<Pick<InvokeOptions, "accountId" | "context">>;
+    }>
+  ): Promise<Array<RunNotification<TOutput>>> {
+    const runStore = runLocalStorage.getStore();
+
+    if (!runStore) {
+      throw new Error(
+        "Cannot invoke a job from outside of a run using batchInvokeAndWaitForCompletion."
+      );
+    }
+
+    // If there are no items in the batch, return an empty array
+    if (batch.length === 0) {
+      return [];
+    }
+
+    // If there are too many items in the batch, throw an error
+    if (batch.length > 25) {
+      throw new Error(
+        `Cannot batch invoke more than 25 items. You tried to batch invoke ${batch.length} items.`
+      );
+    }
+
+    const { io, ctx } = runStore;
+
+    const results = await io.parallel(
+      cacheKey,
+      batch,
+      async (item, index) => {
+        return (await this.invokeAndWaitForCompletion(
+          String(index),
+          item.payload,
+          item.timeoutInSeconds ?? 60 * 60,
+          item.options
+        )) as RunNotification<{}>;
+      },
+      {
+        name: `Batch Invoke '${this.name}'`,
+        properties: [
+          {
+            label: "Job",
+            text: this.id,
+            url: `/orgs/${ctx.organization.slug}/projects/${ctx.project.slug}/jobs/${this.id}`,
+          },
+          {
+            label: "Env",
+            text: ctx.environment.slug,
+          },
+        ],
+      }
+    );
+
+    return results as Array<RunNotification<TOutput>>;
+  }
+
   // Make sure the id is valid (must only contain alphanumeric characters and dashes)
   // Make sure the version is valid (must be a valid semver version)
   #validate() {

packages/trigger-sdk/src/triggerClient.ts

@@ -11,6 +11,7 @@ import {
   IndexEndpointResponse,
   InitializeTriggerBodySchema,
   IntegrationConfig,
+  InvokeOptions,
   JobMetadata,
   LogLevel,
   Logger,
@@ -24,6 +25,7 @@ import {
   RegisterTriggerBodyV2,
   RunJobBody,
   RunJobBodySchema,
+  RunJobErrorResponse,
   RunJobResponse,
   ScheduleMetadata,
   SendEvent,
@@ -36,7 +38,9 @@ import {
   AutoYieldExecutionError,
   AutoYieldWithCompletedTaskExecutionError,
   CanceledWithTaskError,
+  ErrorWithTask,
   ParsedPayloadSchemaError,
+  ResumeWithParallelTaskError,
   ResumeWithTaskError,
   RetryWithTaskError,
   YieldExecutionError,
@@ -321,6 +325,13 @@ export class TriggerClient {
 
         const results = await this.#executeJob(execution.data, job, timeOrigin, triggerVersion);
 
+        this.#internalLogger.debug("executed job", {
+          results,
+          job: job.id,
+          version: job.version,
+          triggerVersion,
+        });
+
         return {
           status: 200,
           body: results,
@@ -467,8 +478,9 @@ export class TriggerClient {
   defineJob<
     TTrigger extends Trigger<EventSpecification<any>>,
     TIntegrations extends Record<string, TriggerIntegration> = {},
-  >(options: JobOptions<TTrigger, TIntegrations>) {
+    TOutput extends any = any,
-    return new Job<TTrigger, TIntegrations>(this, options);
+  >(options: JobOptions<TTrigger, TIntegrations, TOutput>) {
+    return new Job<TTrigger, TIntegrations, TOutput>(this, options);
   }
 
   defineAuthResolver(
@@ -685,6 +697,10 @@ export class TriggerClient {
     return this.#client.getRunStatuses(runId);
   }
 
+  async invokeJob(jobId: string, payload: any, options?: InvokeOptions) {
+    return this.#client.invokeJob(jobId, payload, options);
+  }
+
   authorized(
     apiKey?: string | null
   ): "authorized" | "unauthorized" | "missing-client" | "missing-header" {
@@ -790,91 +806,124 @@ export class TriggerClient {
         this.#logIOStats(io.stats);
       }
 
-      if (error instanceof AutoYieldExecutionError) {
+      if (error instanceof ResumeWithParallelTaskError) {
         return {
-          status: "AUTO_YIELD_EXECUTION",
+          status: "RESUME_WITH_PARALLEL_TASK",
-          location: error.location,
-          timeRemaining: error.timeRemaining,
-          timeElapsed: error.timeElapsed,
-          limit: body.runChunkExecutionLimit,
-        };
-      }
-
-      if (error instanceof AutoYieldWithCompletedTaskExecutionError) {
-        return {
-          status: "AUTO_YIELD_EXECUTION_WITH_COMPLETED_TASK",
-          id: error.id,
-          properties: error.properties,
-          output: error.output,
-          data: {
-            ...error.data,
-            limit: body.runChunkExecutionLimit,
-          },
-        };
-      }
-
-      if (error instanceof YieldExecutionError) {
-        return { status: "YIELD_EXECUTION", key: error.key };
-      }
-
-      if (error instanceof ParsedPayloadSchemaError) {
-        return { status: "INVALID_PAYLOAD", errors: error.schemaErrors };
-      }
-
-      if (error instanceof ResumeWithTaskError) {
-        return { status: "RESUME_WITH_TASK", task: error.task };
-      }
-
-      if (error instanceof RetryWithTaskError) {
-        return {
-          status: "RETRY_WITH_TASK",
           task: error.task,
-          error: error.cause,
+          childErrors: error.childErrors.map((childError) => {
-          retryAt: error.retryAt,
+            return this.#convertErrorToExecutionResponse(childError, body);
+          }),
         };
       }
 
-      if (error instanceof CanceledWithTaskError) {
+      return this.#convertErrorToExecutionResponse(error, body);
-        return {
-          status: "CANCELED",
-          task: error.task,
-        };
-      }
-
-      if (error instanceof RetryWithTaskError) {
-        const errorWithStack = ErrorWithStackSchema.safeParse(error.cause);
-
-        if (errorWithStack.success) {
-          return {
-            status: "ERROR",
-            error: errorWithStack.data,
-            task: error.task,
-          };
-        }
-
-        return {
-          status: "ERROR",
-          error: { message: "Unknown error" },
-          task: error.task,
-        };
-      }
-
-      const errorWithStack = ErrorWithStackSchema.safeParse(error);
-
-      if (errorWithStack.success) {
-        return { status: "ERROR", error: errorWithStack.data };
-      }
-
-      const message = typeof error === "string" ? error : JSON.stringify(error);
-      return {
-        status: "ERROR",
-        error: { name: "Unknown error", message },
-      };
     }
   }
 
+  #convertErrorToExecutionResponse(error: any, body: RunJobBody): RunJobErrorResponse {
+    if (error instanceof AutoYieldExecutionError) {
+      return {
+        status: "AUTO_YIELD_EXECUTION",
+        location: error.location,
+        timeRemaining: error.timeRemaining,
+        timeElapsed: error.timeElapsed,
+        limit: body.runChunkExecutionLimit,
+      };
+    }
+
+    if (error instanceof AutoYieldWithCompletedTaskExecutionError) {
+      return {
+        status: "AUTO_YIELD_EXECUTION_WITH_COMPLETED_TASK",
+        id: error.id,
+        properties: error.properties,
+        output: error.output,
+        data: {
+          ...error.data,
+          limit: body.runChunkExecutionLimit,
+        },
+      };
+    }
+
+    if (error instanceof YieldExecutionError) {
+      return { status: "YIELD_EXECUTION", key: error.key };
+    }
+
+    if (error instanceof ParsedPayloadSchemaError) {
+      return { status: "INVALID_PAYLOAD", errors: error.schemaErrors };
+    }
+
+    if (error instanceof ResumeWithTaskError) {
+      return { status: "RESUME_WITH_TASK", task: error.task };
+    }
+
+    if (error instanceof RetryWithTaskError) {
+      return {
+        status: "RETRY_WITH_TASK",
+        task: error.task,
+        error: error.cause,
+        retryAt: error.retryAt,
+      };
+    }
+
+    if (error instanceof CanceledWithTaskError) {
+      return {
+        status: "CANCELED",
+        task: error.task,
+      };
+    }
+
+    if (error instanceof ErrorWithTask) {
+      const errorWithStack = ErrorWithStackSchema.safeParse(error.cause.output);
+
+      if (errorWithStack.success) {
+        return {
+          status: "ERROR",
+          error: errorWithStack.data,
+          task: error.cause,
+        };
+      }
+
+      return {
+        status: "ERROR",
+        error: { message: JSON.stringify(error.cause.output) },
+        task: error.cause,
+      };
+    }
+
+    if (error instanceof RetryWithTaskError) {
+      const errorWithStack = ErrorWithStackSchema.safeParse(error.cause);
+
+      if (errorWithStack.success) {
+        return {
+          status: "ERROR",
+          error: errorWithStack.data,
+          task: error.task,
+        };
+      }
+
+      return {
+        status: "ERROR",
+        error: { message: "Unknown error" },
+        task: error.task,
+      };
+    }
+
+    const errorWithStack = ErrorWithStackSchema.safeParse(error);
+
+    if (errorWithStack.success) {
+      return { status: "ERROR", error: errorWithStack.data };
+    }
+
+    const message = typeof error === "string" ? error : JSON.stringify(error);
+
+    return {
+      status: "ERROR",
+      error: { name: "Unknown error", message },
+    };
+  }
+
   #createRunContext(execution: RunJobBody): TriggerContext {
-    const { event, organization, environment, job, run, source } = execution;
+    const { event, organization, project, environment, job, run, source } = execution;
 
     return {
       event: {
@@ -884,6 +933,7 @@ export class TriggerClient {
         timestamp: event.timestamp,
       },
       organization,
+      project: project ?? { id: "unknown", name: "unknown", slug: "unknown" }, // backwards compat with old servers
       environment,
       job,
       run,

packages/trigger-sdk/src/triggers/invokeTrigger.ts

@@ -0,0 +1,83 @@
+import { TriggerMetadata } from "@trigger.dev/core";
+import { ParsedPayloadSchemaError } from "../errors";
+import { Job } from "../job";
+import { TriggerClient } from "../triggerClient";
+import { EventSpecification, EventSpecificationExample, SchemaParser, Trigger } from "../types";
+import { formatSchemaErrors } from "../utils/formatSchemaErrors";
+import { TypeOf, ZodType, z } from "zod";
+
+/** Configuration options for an InvokeTrigger */
+type InvokeTriggerOptions<TSchema extends ZodType = z.ZodTypeAny> = {
+  /** A [Zod](https://trigger.dev/docs/documentation/guides/zod) schema that defines the shape of the event payload.
+   * The default is `z.any()` which is `any`.
+   * */
+  schema?: TSchema;
+  examples?: EventSpecificationExample[];
+};
+
+export class InvokeTrigger<TSchema extends ZodType = z.ZodTypeAny>
+  implements Trigger<EventSpecification<TypeOf<TSchema>, z.input<TSchema>>>
+{
+  #options: InvokeTriggerOptions<TSchema>;
+
+  constructor(options: InvokeTriggerOptions<TSchema>) {
+    this.#options = options;
+  }
+
+  toJSON(): TriggerMetadata {
+    return {
+      type: "invoke",
+    };
+  }
+
+  get event() {
+    return {
+      name: "invoke",
+      title: "Manual Invoke",
+      source: "trigger.dev",
+      examples: this.#options.examples ?? [],
+      icon: "trigger",
+      parsePayload: (rawPayload: unknown) => {
+        if (this.#options.schema) {
+          const results = this.#options.schema.safeParse(rawPayload);
+
+          if (!results.success) {
+            throw new ParsedPayloadSchemaError(formatSchemaErrors(results.error.issues));
+          }
+
+          return results.data;
+        }
+
+        return rawPayload as any;
+      },
+      parseInvokePayload: (rawPayload: unknown) => {
+        if (this.#options.schema) {
+          const results = this.#options.schema.safeParse(rawPayload);
+
+          if (!results.success) {
+            throw new ParsedPayloadSchemaError(formatSchemaErrors(results.error.issues));
+          }
+
+          return results.data;
+        }
+
+        return rawPayload as any;
+      },
+    };
+  }
+
+  attachToJob(
+    triggerClient: TriggerClient,
+    job: Job<Trigger<EventSpecification<ZodType<TSchema>>>, any>
+  ): void {}
+
+  get preprocessRuns() {
+    return false;
+  }
+}
+
+export function invokeTrigger<TSchema extends ZodType = z.ZodTypeAny>(
+  options?: InvokeTriggerOptions<TSchema>
+): Trigger<EventSpecification<TypeOf<TSchema>, z.input<TSchema>>> {
+  return new InvokeTrigger(options ?? {});
+}

packages/trigger-sdk/src/types.ts

@@ -32,6 +32,8 @@ export interface TriggerContext {
   environment: { slug: string; id: string; type: RuntimeEnvironmentType };
   /** Organization metadata */
   organization: { slug: string; id: string; title: string };
+  /** Project metadata */
+  project: { slug: string; id: string; name: string };
   /** Run metadata */
   run: { id: string; isTest: boolean; startedAt: Date; isRetry: boolean };
   /** Event metadata */
@@ -69,6 +71,14 @@ export type TriggerEventType<TTrigger extends Trigger<any>> = TTrigger extends T
   ? ReturnType<TEventSpec["parsePayload"]>
   : never;
 
+export type TriggerInvokeType<TTrigger extends Trigger<any>> = TTrigger extends Trigger<
+  infer TEventSpec
+>
+  ? TEventSpec["parseInvokePayload"] extends (payload: unknown) => infer TInvoke
+    ? TInvoke
+    : any
+  : never;
+
 export interface Trigger<TEventSpec extends EventSpecification<any>> {
   event: TEventSpec;
   toJSON(): TriggerMetadata;
@@ -90,7 +100,7 @@ export type EventSpecificationExample = {
   payload: any;
 };
 
-export interface EventSpecification<TEvent extends any> {
+export interface EventSpecification<TEvent extends any, TInvoke extends any = TEvent> {
   name: string | string[];
   title: string;
   source: string;
@@ -100,6 +110,7 @@ export interface EventSpecification<TEvent extends any> {
   examples?: Array<EventSpecificationExample>;
   filter?: EventFilter;
   parsePayload: (payload: unknown) => TEvent;
+  parseInvokePayload?: (payload: unknown) => TInvoke;
   runProperties?: (payload: TEvent) => DisplayProperty[];
 }
 

pnpm-lock.yaml

@@ -134,6 +134,7 @@ importers:
       '@typescript-eslint/eslint-plugin': ^5.59.6
       '@typescript-eslint/parser': ^5.59.6
       '@uiw/react-codemirror': ^4.19.5
+      '@whatwg-node/fetch': ^0.9.14
       autoprefixer: ^10.4.13
       class-variance-authority: ^0.5.2
       clsx: ^1.2.1
@@ -170,13 +171,12 @@ importers:
       prop-types: ^15.8.1
       react: ^18.2.0
       react-dom: ^18.2.0
-      react-hot-toast: ^2.4.0
       react-hotkeys-hook: ^4.4.1
       react-use: ^17.4.0
       recharts: ^2.8.0
-      remix-auth: ^3.2.2
+      remix-auth: ^3.6.0
-      remix-auth-email-link: ^1.4.2
+      remix-auth-email-link: ^2.1.0
-      remix-auth-github: ^1.1.1
+      remix-auth-github: ^1.6.0
       remix-typedjson: 0.3.1
       remix-utils: ^7.1.0
       rimraf: ^3.0.2
@@ -241,6 +241,7 @@ importers:
       '@trigger.dev/sdk': link:../../packages/trigger-sdk
       '@types/pg': 8.6.6
       '@uiw/react-codemirror': 4.19.5_th22fcplkuhrqjnlojwclcaim4
+      '@whatwg-node/fetch': 0.9.14
       class-variance-authority: 0.5.2_typescript@5.2.2
       clsx: 1.2.1
       compression: 1.7.4
@@ -268,13 +269,12 @@ importers:
       prismjs: 1.29.0
       react: 18.2.0
       react-dom: 18.2.0_react@18.2.0
-      react-hot-toast: 2.4.0_biqbaboplfbrettd7655fr4n2y
       react-hotkeys-hook: 4.4.1_biqbaboplfbrettd7655fr4n2y
       react-use: 17.4.0_biqbaboplfbrettd7655fr4n2y
       recharts: 2.8.0_v2m5e27vhdewzwhryxwfaorcca
-      remix-auth: 3.4.0_ybjp5xbtg4zthziheradfmei64
+      remix-auth: 3.6.0_ybjp5xbtg4zthziheradfmei64
-      remix-auth-email-link: 1.5.2_7ym4eehrzvo4xt5yocgqipdaji
+      remix-auth-email-link: 2.1.0_esw6v6dv7ru6ju3hn3i62b7n2m
-      remix-auth-github: 1.3.0_7ym4eehrzvo4xt5yocgqipdaji
+      remix-auth-github: 1.6.0_esw6v6dv7ru6ju3hn3i62b7n2m
       remix-typedjson: 0.3.1_7t5z2pmrpxrx2dh2jpp55scw5q
       remix-utils: 7.1.0_nb5lmxjinjv74o775xrkc35gga
       semver: 7.5.0
@@ -907,6 +907,7 @@ importers:
 
   packages/integration-kit:
     specifiers:
+      '@trigger.dev/core': workspace:*
       '@trigger.dev/tsconfig': workspace:*
       '@types/node': '18'
       '@types/uuid': ^9.0.0
@@ -916,6 +917,7 @@ importers:
       typescript: ^4.8.4
       uuid: ^9.0.0
     dependencies:
+      '@trigger.dev/core': link:../core
       uuid: 9.0.0
     devDependencies:
       '@trigger.dev/tsconfig': link:../../config-packages/tsconfig
@@ -14897,6 +14899,30 @@ packages:
       '@xtuc/long': 4.2.2
     dev: true
 
+  /@whatwg-node/events/0.1.1:
+    resolution: {integrity: sha512-AyQEn5hIPV7Ze+xFoXVU3QTHXVbWPrzaOkxtENMPMuNL6VVHrp4hHfDt9nrQpjO7BgvuM95dMtkycX5M/DZR3w==}
+    engines: {node: '>=16.0.0'}
+    dev: false
+
+  /@whatwg-node/fetch/0.9.14:
+    resolution: {integrity: sha512-wurZC82zzZwXRDSW0OS9l141DynaJQh7Yt0FD1xZ8niX7/Et/7RoiLiltbVU1fSF1RR9z6ndEaTUQBAmddTm1w==}
+    engines: {node: '>=16.0.0'}
+    dependencies:
+      '@whatwg-node/node-fetch': 0.5.0
+      urlpattern-polyfill: 9.0.0
+    dev: false
+
+  /@whatwg-node/node-fetch/0.5.0:
+    resolution: {integrity: sha512-q76lDAafvHNGWedNAVHrz/EyYTS8qwRLcwne8SJQdRN5P3HydxU6XROFvJfTML6KZXQX2FDdGY4/SnaNyd7M0Q==}
+    engines: {node: '>=16.0.0'}
+    dependencies:
+      '@whatwg-node/events': 0.1.1
+      busboy: 1.6.0
+      fast-querystring: 1.1.2
+      fast-url-parser: 1.1.3
+      tslib: 2.6.2
+    dev: false
+
   /@xmldom/xmldom/0.8.6:
     resolution: {integrity: sha512-uRjjusqpoqfmRkTaNuLJ2VohVr67Q5YwDATW3VU7PfzTj6IRaihGrYI7zckGZjxQPBIp63nfvJbM+Yu5ICh0Bg==}
     engines: {node: '>=10.0.0'}
@@ -20099,7 +20125,6 @@ packages:
 
   /fast-decode-uri-component/1.0.1:
     resolution: {integrity: sha512-WKgKWg5eUxvRZGwW8FvfbaH7AXSh2cL+3j5fMGzUMCxWBJ3dV3a7Wz8y2f/uQ0e3B6WmodD3oS54jTQ9HVTIIg==}
-    dev: true
 
   /fast-deep-equal/3.1.3:
     resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}
@@ -20203,7 +20228,6 @@ packages:
     resolution: {integrity: sha512-g6KuKWmFXc0fID8WWH0jit4g0AGBoJhCkJMb1RmbsSEUNvQ+ZC8D6CUZ+GtF8nMzSPXnhiePyyqqipzNNEnHjg==}
     dependencies:
       fast-decode-uri-component: 1.0.1
-    dev: true
 
   /fast-redact/3.3.0:
     resolution: {integrity: sha512-6T5V1QK1u4oF+ATxs1lWUmlEk6P2T9HqJG3e2DnHOdVgZy2rFJBoEnrIedcTXlkAHU/zKC+7KETJ+KGGKwxgMQ==}
@@ -20221,6 +20245,12 @@ packages:
     resolution: {integrity: sha512-cIusKBIt/R/oI6z/1nyfe2FvGKVTohVRfvkOhvx0nCEW+xf5NoCXjAHcWp93uOUBchzYcsvPlrapAdX1uW+YGg==}
     dev: true
 
+  /fast-url-parser/1.1.3:
+    resolution: {integrity: sha512-5jOCVXADYNuRkKFzNJ0dCCewsZiYo0dz8QNYljkOpFC6r2U4OBmKtvm/Tsuh4w1YYdDqDb31a8TVhBJ2OJKdqQ==}
+    dependencies:
+      punycode: 1.4.1
+    dev: false
+
   /fastest-stable-stringify/2.0.2:
     resolution: {integrity: sha512-bijHueCGd0LqqNK9b5oCMHc0MluJAx0cwqASgbWMvkO01lCYgIhacVRLcaDz3QnyYIRNJRDwMb41VuT6pHJ91Q==}
     dev: false
@@ -21118,12 +21148,6 @@ packages:
   /globrex/0.1.2:
     resolution: {integrity: sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg==}
 
-  /goober/2.1.11:
-    resolution: {integrity: sha512-5SS2lmxbhqH0u9ABEWq7WPU69a4i2pYcHeCxqaNq6Cw3mnrF0ghWNM4tEGid4dKy8XNIAUbuThuozDHHKJVh3A==}
-    peerDependencies:
-      csstype: ^3.0.10
-    dev: false
-
   /gopd/1.0.1:
     resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==}
     dependencies:
@@ -27340,20 +27364,6 @@ packages:
       - supports-color
     dev: false
 
-  /react-hot-toast/2.4.0_biqbaboplfbrettd7655fr4n2y:
-    resolution: {integrity: sha512-qnnVbXropKuwUpriVVosgo8QrB+IaPJCpL8oBI6Ov84uvHZ5QQcTp2qg6ku2wNfgJl6rlQXJIQU5q+5lmPOutA==}
-    engines: {node: '>=10'}
-    peerDependencies:
-      react: '>=16'
-      react-dom: '>=16'
-    dependencies:
-      goober: 2.1.11
-      react: 18.2.0
-      react-dom: 18.2.0_react@18.2.0
-    transitivePeerDependencies:
-      - csstype
-    dev: false
-
   /react-hotkeys-hook/4.4.1_biqbaboplfbrettd7655fr4n2y:
     resolution: {integrity: sha512-sClBMBioFEgFGYLTWWRKvhxcCx1DRznd+wkFHwQZspnRBkHTgruKIHptlK/U/2DPX8BhHoRGzpMVWUXMmdZlmw==}
     peerDependencies:
@@ -27982,49 +27992,49 @@ packages:
       retext-smartypants: 5.2.0
       unist-util-visit: 4.1.2
 
-  /remix-auth-email-link/1.5.2_7ym4eehrzvo4xt5yocgqipdaji:
+  /remix-auth-email-link/2.1.0_esw6v6dv7ru6ju3hn3i62b7n2m:
-    resolution: {integrity: sha512-LCbaYkhsENbRK70O36Ul/jBtvNcNdBabjOIXYhrTNwx4qYA/fFkX8Sr3GMuDhoMzHnXQy7zHtsqbg/DTHocX6g==}
+    resolution: {integrity: sha512-1Hp4QDqN17tEEfiRd/6g1XDw6o6y+lyeeseDc0EGrR4vZcR7HLPR8ImfEtxplYpXlFSpOO7mzl6GgJprynjgcg==}
     peerDependencies:
-      '@remix-run/server-runtime': ^1.1.1
+      '@remix-run/server-runtime': ^2.0.1
-      remix-auth: ^3.2.1
+      remix-auth: ^3.6.0
     dependencies:
       '@remix-run/server-runtime': 2.1.0_typescript@5.2.2
       crypto-js: 4.1.1
-      remix-auth: 3.4.0_ybjp5xbtg4zthziheradfmei64
+      remix-auth: 3.6.0_ybjp5xbtg4zthziheradfmei64
+      yarn: 1.22.19
     dev: false
 
-  /remix-auth-github/1.3.0_7ym4eehrzvo4xt5yocgqipdaji:
+  /remix-auth-github/1.6.0_esw6v6dv7ru6ju3hn3i62b7n2m:
-    resolution: {integrity: sha512-dqveIbXc1yXQXiB8UVjKSR3qgHFUZ0VJOu2/GTT5uMKZY0//FSnp1w8gK0btGlbc1HW97gig16uS84et6FBMiA==}
+    resolution: {integrity: sha512-qdQmWVVEDHxnzMPn7XE0s7QX5GYba0uH3MDW+lVJiCoHDZCGymqmGCajLThL5vFJdwSx0C4GpQJ5EgEsIjQ3pA==}
     peerDependencies:
       '@remix-run/server-runtime': ^1.0.0
       remix-auth: ^3.4.0
     dependencies:
       '@remix-run/server-runtime': 2.1.0_typescript@5.2.2
-      remix-auth: 3.4.0_ybjp5xbtg4zthziheradfmei64
+      remix-auth: 3.6.0_ybjp5xbtg4zthziheradfmei64
-      remix-auth-oauth2: 1.5.0_7ym4eehrzvo4xt5yocgqipdaji
+      remix-auth-oauth2: 1.11.0_esw6v6dv7ru6ju3hn3i62b7n2m
     transitivePeerDependencies:
       - supports-color
     dev: false
 
-  /remix-auth-oauth2/1.5.0_7ym4eehrzvo4xt5yocgqipdaji:
+  /remix-auth-oauth2/1.11.0_esw6v6dv7ru6ju3hn3i62b7n2m:
-    resolution: {integrity: sha512-vv/CX2bCeVf8vzACfS1Fae+i75Cw0/As3RLkp00+9n9IBEvKi07LhUxTbODNA51jRh+wInnmCRIwsHafKQDD6g==}
+    resolution: {integrity: sha512-Yf1LF6NLYPFa7X2Rax/VEhXmYXFjZOi/q+7DmbMeoMHjAfkpqxbvzqqYSKIKDGR51z5TXR5na4to4380mir5bg==}
     peerDependencies:
-      '@remix-run/server-runtime': ^1.0.0
+      '@remix-run/server-runtime': ^1.0.0 || ^2.0.0
-      remix-auth: ^3.2.2
+      remix-auth: ^3.6.0
     dependencies:
       '@remix-run/server-runtime': 2.1.0_typescript@5.2.2
       debug: 4.3.4
-      remix-auth: 3.4.0_ybjp5xbtg4zthziheradfmei64
+      remix-auth: 3.6.0_ybjp5xbtg4zthziheradfmei64
-      uuid: 8.3.2
     transitivePeerDependencies:
       - supports-color
     dev: false
 
-  /remix-auth/3.4.0_ybjp5xbtg4zthziheradfmei64:
+  /remix-auth/3.6.0_ybjp5xbtg4zthziheradfmei64:
-    resolution: {integrity: sha512-VRliJo9VRAS4sSYgMjYbi7rYRixPWU2Tb8PFZb06OIj0nONK/1KRzHf2+Y6VGZeIacepLVgan6g2IUJjayE2rQ==}
+    resolution: {integrity: sha512-mxlzLYi+/GKQSaXIqIw15dxAT1wm+93REAeDIft2unrKDYnjaGhhpapyPhdbALln86wt9lNAk21znfRss3fG7Q==}
     peerDependencies:
-      '@remix-run/react': ^1.0.0
+      '@remix-run/react': ^1.0.0 || ^2.0.0
-      '@remix-run/server-runtime': ^1.0.0
+      '@remix-run/server-runtime': ^1.0.0 || ^2.0.0
     dependencies:
       '@remix-run/react': 2.1.0_vegmnuvoswdxuttpuhvewcem44
       '@remix-run/server-runtime': 2.1.0_typescript@5.2.2
@@ -31468,6 +31478,10 @@ packages:
       qs: 6.11.0
     dev: false
 
+  /urlpattern-polyfill/9.0.0:
+    resolution: {integrity: sha512-WHN8KDQblxd32odxeIgo83rdVDE2bvdkb86it7bMhYZwWKJz0+O0RK/eZiHYnM+zgt/U7hAHOlCQGfjjvSkw2g==}
+    dev: false
+
   /use-callback-ref/1.3.0_e74vmjybjy5dsfplslbsgtbvvi:
     resolution: {integrity: sha512-3FT9PRuRdbB9HfXhEq35u4oZkvpJ5kuYbpqhCfmiZyReuRgpnhDlbr2ZEnnuS0RrJAPn6l23xjFg9kpDM+Ms7w==}
     engines: {node: '>=10'}
@@ -32683,6 +32697,13 @@ packages:
       y18n: 5.0.8
       yargs-parser: 21.1.1
 
+  /yarn/1.22.19:
+    resolution: {integrity: sha512-/0V5q0WbslqnwP91tirOvldvYISzaqhClxzyUKXYxs07yUILIs5jx/k6CFe8bvKSkds5w+eiOqta39Wk3WxdcQ==}
+    engines: {node: '>=4.0.0'}
+    hasBin: true
+    requiresBuild: true
+    dev: false
+
   /yauzl/2.10.0:
     resolution: {integrity: sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==}
     dependencies:

references/job-catalog/package.json

@@ -29,6 +29,7 @@
     "misconfigured": "nodemon --watch src/misconfigured.ts -r tsconfig-paths/register -r dotenv/config src/misconfigured.ts",
     "auto-yield": "nodemon --watch src/auto-yield.ts -r tsconfig-paths/register -r dotenv/config src/auto-yield.ts",
     "cli-example": "nodemon --watch src/cli-example.ts -r tsconfig-paths/register -r dotenv/config src/cli-example.ts",
+    "invoke": "nodemon --watch src/invoke.ts -r tsconfig-paths/register -r dotenv/config src/invoke.ts",
     "dev:trigger": "trigger-cli dev --port 8080"
   },
   "dependencies": {
@@ -63,4 +64,4 @@
     "ts-node": "^10.9.1",
     "tsconfig-paths": "^3.14.1"
   }
-}
+}

references/job-catalog/src/invoke.ts

@@ -0,0 +1,313 @@
+import { createExpressServer } from "@trigger.dev/express";
+import { OpenAI } from "@trigger.dev/openai";
+import { TriggerClient, invokeTrigger } from "@trigger.dev/sdk";
+import { z } from "zod";
+
+export const client = new TriggerClient({
+  id: "job-catalog",
+  apiKey: process.env["TRIGGER_API_KEY"],
+  apiUrl: process.env["TRIGGER_API_URL"],
+  verbose: true,
+  ioLogLocalEnabled: true,
+});
+
+const invocableJob = client.defineJob({
+  id: "invoke-example-1",
+  name: "Invoke Example 1",
+  version: "1.0.0",
+  enabled: true,
+  trigger: invokeTrigger({
+    schema: z.object({
+      message: z.string(),
+      forceError: z.boolean().default(false),
+      delay: z.number().default(5),
+    }),
+  }),
+  run: async (payload, io, ctx) => {
+    const generatingMemes = await io.createStatus("status-1", {
+      //the label is compulsory on this first call
+      label: "Generating memes",
+      //state is optional
+      state: "loading",
+      //data is an optional object. the values can be any type that is JSON serializable
+      data: {
+        progress: 0.1,
+      },
+    });
+
+    await io.runTask("task-1", async () => {
+      await new Promise((resolve) => setTimeout(resolve, 1000));
+    });
+
+    await io.wait("wait-1", payload.delay);
+
+    await io.runTask("task-2", async () => {
+      await new Promise((resolve) => setTimeout(resolve, 1000));
+
+      return [{ hello: "there", ts: new Date() }];
+    });
+
+    await generatingMemes.update("middle-generation", {
+      //label isn't specified so will remain the same
+      //state will be updated to "success"
+      state: "success",
+      //set data, this overrides the previous value
+      data: {
+        progress: 1,
+        urls: [
+          "https://media.giphy.com/media/v1.Y2lkPTc5MGI3NjExZnZoMndsdWh0MmhvY2kyaDF6YjZjZzg1ZGsxdnhhYm13a3Q1Y3lkbyZlcD12MV9pbnRlcm5hbF9naWZfYnlfaWQmY3Q9Zw/13HgwGsXF0aiGY/giphy.gif",
+        ],
+      },
+    });
+
+    if (payload.forceError) {
+      throw new Error("Forced error");
+    }
+
+    const response = await io.runTask("fetch-json", async () =>
+      fetch("https://jsonhero.io/j/PjHo1o5MVeH4.json").then((r) => r.json())
+    );
+
+    return response;
+  },
+});
+
+client.defineJob({
+  id: "invoke-example-2",
+  name: "Invoke Example 2",
+  version: "1.0.0",
+  enabled: true,
+  trigger: invokeTrigger(),
+  run: async (payload, io, ctx) => {
+    await invocableJob.invoke("invoke", {
+      message: "Hello World 1",
+    });
+
+    await invocableJob.invoke(
+      "invoke-with-url",
+      {
+        message: "Hello World 2",
+      },
+      {
+        callbackUrl: process.env.REQUEST_BIN_URL,
+      }
+    );
+
+    const result = await invocableJob.invokeAndWaitForCompletion("invoke-and-wait", {
+      message: "Hello World 3",
+      forceError: true,
+    });
+
+    if (result.ok) {
+      await io.logger.info("Invoking job worked!", { result });
+    } else {
+      await io.logger.error("Invoking job failed!", { result });
+    }
+  },
+});
+
+const simpleInvokableJob = client.defineJob({
+  id: "simple-invoke-example-1",
+  name: "Simple Invoke Example 1",
+  version: "1.0.0",
+  enabled: true,
+  trigger: invokeTrigger(),
+  run: async (payload, io, ctx) => {
+    return payload;
+  },
+});
+
+const openai = new OpenAI({
+  id: "openai",
+  apiKey: process.env["OPENAI_API_KEY"]!,
+});
+
+const perplexity = new OpenAI({
+  id: "perplexity",
+  apiKey: process.env["PERPLEXITY_API_KEY"]!,
+  baseURL: "https://api.perplexity.ai",
+  icon: "brand-open-source",
+});
+
+// This job performs a chat completion using either OpenAI or Perplexity, depending on the model passed in
+const completionJob = client.defineJob({
+  id: "openai-job",
+  name: "OpenAI Job",
+  version: "1.0.0",
+  enabled: true,
+  trigger: invokeTrigger({
+    schema: z.object({
+      model: z.string().default("gpt-3.5-turbo"),
+      prompt: z.string(),
+    }),
+  }),
+  integrations: {
+    openai,
+    perplexity,
+  },
+  run: async (payload, io, ctx) => {
+    if (payload.model === "gpt-3.5-turbo") {
+      return await io.openai.chat.completions.backgroundCreate(
+        "background-chat-completion",
+        {
+          model: payload.model,
+          messages: [
+            {
+              role: "user",
+              content: payload.prompt,
+            },
+          ],
+        },
+        {},
+        {
+          // Set a timeout of 30 seconds, and retry it up to 3 times
+          timeout: {
+            durationInMs: 30000,
+            retry: {
+              limit: 3,
+              minTimeoutInMs: 1000,
+              factor: 2,
+            },
+          },
+        }
+      );
+    } else {
+      return await io.perplexity.completions.backgroundCreate(
+        "background-completion",
+        {
+          model: payload.model,
+          prompt: payload.prompt,
+        },
+        {},
+        {
+          // Set a timeout of 30 seconds, and retry it up to 3 times
+          timeout: {
+            durationInMs: 30000,
+            retry: {
+              limit: 3,
+              minTimeoutInMs: 1000,
+              factor: 2,
+            },
+          },
+        }
+      );
+    }
+  },
+});
+
+const prompts = [
+  "Advantages of quantum computing over classical computing",
+  "The ethical implications of advanced artificial intelligence",
+  "Design a thought experiment highlighting the paradoxes in quantum mechanics",
+  "Explain the Fermi Paradox, its potential solutions, and implications for humanity",
+  "Analyze Shakespeare's use of iambic pentameter in his plays",
+];
+
+client.defineJob({
+  id: "batch-invoke-ai-example",
+  name: "Batch Invoke OpenAI Example",
+  version: "1.0.0",
+  trigger: invokeTrigger(),
+  run: async (payload, io, ctx) => {
+    // This will invoke the completionJob in parallel for each prompt, first trying OpenAI
+    const runs = await completionJob.batchInvokeAndWaitForCompletion(
+      "batch-invoke-and-wait",
+      prompts.map((prompt) => ({
+        payload: {
+          model: "gpt-3.5-turbo",
+          prompt,
+        },
+      }))
+    );
+
+    // Runs are returned in the same order as the prompts
+    const failedRuns = runs.map((run, i) => ({ run, prompt: prompts[i] })).filter((r) => !r.run.ok);
+
+    // Run the failed runs with Perplexity by specifying the "mistral-7b-instruct" model
+    const retriedRuns = await completionJob.batchInvokeAndWaitForCompletion(
+      "batch-invoke-and-wait",
+      failedRuns.map((failedRun) => ({
+        payload: {
+          model: "mistral-7b-instruct",
+          prompt: failedRun.prompt,
+        },
+      }))
+    );
+
+    const failedPerplexityRuns = retriedRuns
+      .map((run, i) => ({ run, prompt: failedRuns[i].prompt }))
+      .filter((r) => !r.run.ok);
+
+    // And so on and so forth
+  },
+});
+
+client.defineJob({
+  id: "batch-invoke-example",
+  name: "Batch Invoke Example",
+  version: "1.0.0",
+  trigger: invokeTrigger(),
+  run: async (payload, io, ctx) => {
+    await invocableJob.batchInvokeAndWaitForCompletion(
+      "batch-invoke-and-wait",
+      Array.from({ length: 2 }).map((_, i) => ({
+        payload: {
+          message: `Hello World ${i}`,
+          delay: i % 2 === 0 ? 7 : 20,
+        },
+      }))
+    );
+
+    await simpleInvokableJob.batchInvokeAndWaitForCompletion(
+      "batch-invoke-and-wait-simple",
+      Array.from({ length: 25 }).map((_, i) => ({
+        payload: {
+          message: `Hello World ${i}`,
+        },
+        options: {
+          context: {
+            i,
+          },
+          accountId: "FB1C6C79-6C82-45B6-A8AA-207ADA9EE838",
+        },
+      }))
+    );
+
+    await simpleInvokableJob.batchInvokeAndWaitForCompletion(
+      "batch-invoke-and-wait-2",
+      Array.from({ length: 2 }).map((_, i) => ({
+        payload: {
+          message: `Hello World ${i}`,
+        },
+      }))
+    );
+  },
+});
+
+export const exampleJob = client.defineJob({
+  id: "example-job",
+  name: "Example job",
+  version: "1.0.1",
+  trigger: invokeTrigger({
+    //the expected payload shape
+    schema: z.object({
+      userId: z.string(),
+      tier: z.enum(["free", "pro"]),
+    }),
+  }),
+  run: async (payload, io, ctx) => {
+    // payload is typed as { userId: string, tier: "free" | "pro" }
+  },
+});
+
+client.defineJob({
+  id: "example-job2",
+  name: "Example job 2",
+  version: "1.0.1",
+  trigger: invokeTrigger(),
+  run: async (payload, io, ctx) => {
+    const jobRun = await exampleJob.invoke("⚡", { userId: "123", tier: "free" });
+  },
+});
+
+createExpressServer(client);