Release 3.0.0-beta.45

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.changeset/fast-colts-relax.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Increase dev worker timeout

.changeset/friendly-walls-repair.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Add sox and audiowaveform binaries to worker images

.changeset/modern-stingrays-end.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+v3: Trigger delayed runs and reschedule them

.changeset/nervous-planets-sparkle.md

@@ -0,0 +1,9 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+- Improve non-zero exit code error messages
+- Detect OOM conditions within worker child processes
+- Internal errors can have optional stack traces
+- Docker provider can be set to enforce machine presets

.changeset/polite-pears-grow.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Use global setTimeout to ensure cross-runtime support

.changeset/polite-pots-walk.md

@@ -0,0 +1,7 @@
+---
+"trigger.dev": patch
+---
+
+Add e2e fixtures corresponding to past issues
+Implement e2e suite parallelism
+Enhance log level for specific e2e suite messages

.changeset/pre.json

@@ -65,8 +65,10 @@
     "eight-pumas-float",
     "eleven-paws-join",
     "famous-boats-tease",
+    "fast-colts-relax",
     "few-students-share",
     "five-toes-destroy",
+    "friendly-walls-repair",
     "funny-swans-destroy",
     "gorgeous-gorillas-compete",
     "green-bags-wink",
@@ -90,7 +92,9 @@
     "mighty-camels-joke",
     "mighty-flowers-train",
     "mighty-parrots-sin",
+    "modern-stingrays-end",
     "nasty-jars-pump",
+    "nervous-planets-sparkle",
     "new-pants-beg",
     "new-rivers-tell",
     "nice-bulldogs-turn",
@@ -99,9 +103,12 @@
     "pink-pumas-rhyme",
     "plenty-ducks-beam",
     "polite-ducks-switch",
+    "polite-pears-grow",
+    "polite-pots-walk",
     "polite-rockets-matter",
     "poor-flowers-cross",
     "purple-garlics-shop",
+    "purple-spiders-care",
     "rare-lamps-promise",
     "rare-roses-float",
     "real-planets-stare",
@@ -110,18 +117,25 @@
     "rotten-dryers-exercise",
     "rude-toys-compare",
     "selfish-ducks-sort",
+    "serious-hats-rest",
     "shaggy-spoons-taste",
     "sharp-emus-compare",
     "sharp-zebras-serve",
     "shiny-coats-cry",
+    "silly-buses-obey",
+    "silly-forks-kiss",
     "silly-suits-switch",
     "silver-doors-juggle",
     "six-ligers-exist",
+    "six-rats-hunt",
     "sixty-insects-watch",
     "slow-buses-own",
+    "slow-kiwis-hide",
     "slow-sloths-retire",
     "smart-needles-move",
     "smart-olives-eat",
+    "sour-pugs-teach",
+    "spicy-frogs-remain",
     "spicy-lamps-smoke",
     "spicy-terms-bow",
     "strange-ghosts-matter",
@@ -131,13 +145,17 @@
     "strong-phones-smoke",
     "stupid-adults-sniff",
     "stupid-bulldogs-applaud",
+    "sweet-ducks-remember",
     "sweet-lizards-press",
     "swift-dragons-peel",
     "tall-bees-wave",
+    "tall-masks-repeat",
+    "tame-apricots-clap",
     "tame-guests-know",
     "tender-moose-tell",
     "tender-oranges-rhyme",
     "tender-turkeys-compete",
+    "thick-carrots-sneeze",
     "thin-parents-heal",
     "thirty-islands-kiss",
     "tidy-balloons-suffer",
@@ -148,7 +166,9 @@
     "tricky-bulldogs-heal",
     "tricky-keys-attack",
     "tricky-ladybugs-unite",
+    "twelve-knives-notice",
     "two-pumas-wait",
+    "violet-clocks-notice",
     "warm-olives-provide",
     "warm-planes-taste",
     "young-snails-sell"

.changeset/purple-spiders-care.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Await file watcher cleanup in dev

.changeset/serious-hats-rest.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+v3: [prod] force flush timeout should be 1s

.changeset/silly-buses-obey.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Add callback to checkpoint created message

.changeset/silly-forks-kiss.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+v3: Copy over more of the project's package.json keys into the deployed package.json (support for custom config like zenstack)

.changeset/six-rats-hunt.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Improved ESM module require error detection logic

.changeset/slow-kiwis-hide.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+v3: vercel edge runtime support

.changeset/sour-pugs-teach.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+v3: fix otel flushing causing CLEANUP ack timeout errors by always setting a forceFlushTimeoutMillis value

.changeset/spicy-frogs-remain.md

@@ -0,0 +1,10 @@
+---
+"trigger.dev": patch
+---
+
+- Prevent downgrades during update check and advise to upgrade CLI
+- Detect bun and use npm instead
+- During init, fail early and advise if not a TypeScript project
+- During init, allow specifying custom package manager args
+- Add links to dev worker started message
+- Fix links in unsupported terminals

.changeset/sweet-ducks-remember.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+v3: Adding SDK functions for triggering tasks in a typesafe way, without importing task file

.changeset/tall-masks-repeat.md

@@ -0,0 +1,9 @@
+---
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+- Fix artifact detection logs
+- Fix OOM detection and error messages
+- Add test link to cli deployment completion

.changeset/tame-apricots-clap.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+v3: postInstall config option now replaces the postinstall script found in package.json

.changeset/thick-carrots-sneeze.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/core": patch
+"@trigger.dev/sdk": patch
+---
+
+v3: Include presigned urls for downloading large payloads and outputs when using runs.retrieve

.changeset/twelve-knives-notice.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+v3: fix missing init output in task run function when no middleware is defined

.changeset/violet-clocks-notice.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix jsonc-parser import

.github/workflows/publish.yml

@@ -49,11 +49,11 @@ jobs:
     uses: ./.github/workflows/unit-tests.yml
     secrets: inherit
 
-  e2e:
-    uses: ./.github/workflows/e2e.yml
-    with:
-      package: cli-v3
-    secrets: inherit
+  # e2e:
+  #   uses: ./.github/workflows/e2e.yml
+  #   with:
+  #     package: cli-v3
+  #   secrets: inherit
 
   publish:
     needs: [typecheck, units]

apps/coordinator/src/index.ts

@@ -1162,13 +1162,7 @@ class TaskCoordinator {
             return;
           }
 
-          if (!checkpoint.docker || !willSimulate) {
-            socket.emit("REQUEST_EXIT", {
-              version: "v1",
-            });
-          }
-
-          this.#platformSocket?.send("CHECKPOINT_CREATED", {
+          const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", {
             version: "v1",
             attemptFriendlyId: message.attemptFriendlyId,
             docker: checkpoint.docker,
@@ -1179,6 +1173,17 @@ class TaskCoordinator {
               now: message.now,
             },
           });
+
+          if (ack?.keepRunAlive) {
+            logger.log("keeping run alive after duration checkpoint", { runId: socket.data.runId });
+            return;
+          }
+
+          if (!checkpoint.docker || !willSimulate) {
+            socket.emit("REQUEST_EXIT", {
+              version: "v1",
+            });
+          }
         });
 
         socket.on("WAIT_FOR_TASK", async (message, callback) => {
@@ -1205,13 +1210,7 @@ class TaskCoordinator {
             return;
           }
 
-          if (!checkpoint.docker || !willSimulate) {
-            socket.emit("REQUEST_EXIT", {
-              version: "v1",
-            });
-          }
-
-          this.#platformSocket?.send("CHECKPOINT_CREATED", {
+          const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", {
             version: "v1",
             attemptFriendlyId: message.attemptFriendlyId,
             docker: checkpoint.docker,
@@ -1221,6 +1220,17 @@ class TaskCoordinator {
               friendlyId: message.friendlyId,
             },
           });
+
+          if (ack?.keepRunAlive) {
+            logger.log("keeping run alive after task checkpoint", { runId: socket.data.runId });
+            return;
+          }
+
+          if (!checkpoint.docker || !willSimulate) {
+            socket.emit("REQUEST_EXIT", {
+              version: "v1",
+            });
+          }
         });
 
         socket.on("WAIT_FOR_BATCH", async (message, callback) => {
@@ -1247,13 +1257,7 @@ class TaskCoordinator {
             return;
           }
 
-          if (!checkpoint.docker || !willSimulate) {
-            socket.emit("REQUEST_EXIT", {
-              version: "v1",
-            });
-          }
-
-          this.#platformSocket?.send("CHECKPOINT_CREATED", {
+          const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", {
             version: "v1",
             attemptFriendlyId: message.attemptFriendlyId,
             docker: checkpoint.docker,
@@ -1264,6 +1268,17 @@ class TaskCoordinator {
               runFriendlyIds: message.runFriendlyIds,
             },
           });
+
+          if (ack?.keepRunAlive) {
+            logger.log("keeping run alive after batch checkpoint", { runId: socket.data.runId });
+            return;
+          }
+
+          if (!checkpoint.docker || !willSimulate) {
+            socket.emit("REQUEST_EXIT", {
+              version: "v1",
+            });
+          }
         });
 
         socket.on("INDEX_TASKS", async (message, callback) => {

apps/docker-provider/src/index.ts

@@ -109,22 +109,27 @@ class DockerTaskOperations implements TaskOperations {
 
     const containerName = this.#getRunContainerName(opts.runId);
 
+    const runArgs = [
+      "run",
+      "--network=host",
+      "--detach",
+      `--env=TRIGGER_ENV_ID=${opts.envId}`,
+      `--env=TRIGGER_RUN_ID=${opts.runId}`,
+      `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
+      `--env=POD_NAME=${containerName}`,
+      `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
+      `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
+      `--name=${containerName}`,
+    ];
+
+    if (process.env.ENFORCE_MACHINE_PRESETS) {
+      runArgs.push(`--cpus=${opts.machine.cpu}`, `--memory=${opts.machine.memory}G`);
+    }
+
+    runArgs.push(`${opts.image}`);
+
     try {
-      logger.debug(
-        await execa("docker", [
-          "run",
-          "--network=host",
-          "--detach",
-          `--env=TRIGGER_ENV_ID=${opts.envId}`,
-          `--env=TRIGGER_RUN_ID=${opts.runId}`,
-          `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
-          `--env=POD_NAME=${containerName}`,
-          `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
-          `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
-          `--name=${containerName}`,
-          `${opts.image}`,
-        ])
-      );
+      logger.debug(await execa("docker", runArgs));
     } catch (error) {
       if (!isExecaChildProcess(error)) {
         throw error;

apps/kubernetes-provider/src/index.ts

@@ -529,27 +529,28 @@ provider.listen();
 
 const taskMonitor = new TaskMonitor({
   runtimeEnv: RUNTIME_ENV,
-  onIndexFailure: async (deploymentId, failureInfo) => {
-    logger.log("Indexing failed", { deploymentId, failureInfo });
+  onIndexFailure: async (deploymentId, details) => {
+    logger.log("Indexing failed", { deploymentId, details });
 
     try {
       provider.platformSocket.send("INDEXING_FAILED", {
         deploymentId,
         error: {
-          name: `Crashed with exit code ${failureInfo.exitCode}`,
-          message: failureInfo.reason,
-          stack: failureInfo.logs,
+          name: `Crashed with exit code ${details.exitCode}`,
+          message: details.reason,
+          stack: details.logs,
         },
+        overrideCompletion: details.overrideCompletion,
       });
     } catch (error) {
       logger.error(error);
     }
   },
-  onRunFailure: async (runId, failureInfo) => {
-    logger.log("Run failed:", { runId, failureInfo });
+  onRunFailure: async (runId, details) => {
+    logger.log("Run failed:", { runId, details });
 
     try {
-      provider.platformSocket.send("WORKER_CRASHED", { runId, ...failureInfo });
+      provider.platformSocket.send("WORKER_CRASHED", { runId, ...details });
     } catch (error) {
       logger.error(error);
     }

apps/kubernetes-provider/src/taskMonitor.ts

@@ -1,25 +1,20 @@
 import * as k8s from "@kubernetes/client-node";
 import { SimpleLogger } from "@trigger.dev/core-apps";
+import { EXIT_CODE_ALREADY_HANDLED, EXIT_CODE_CHILD_NONZERO } from "@trigger.dev/core-apps/process";
 import { setTimeout } from "timers/promises";
 import PQueue from "p-queue";
+import type { Prettify } from "@trigger.dev/core/v3";
 
-type IndexFailureHandler = (
-  deploymentId: string,
-  failureInfo: {
-    exitCode: number;
-    reason: string;
-    logs: string;
-  }
-) => Promise<any>;
+type FailureDetails = Prettify<{
+  exitCode: number;
+  reason: string;
+  logs: string;
+  overrideCompletion: boolean;
+}>;
 
-type RunFailureHandler = (
-  runId: string,
-  failureInfo: {
-    exitCode: number;
-    reason: string;
-    logs: string;
-  }
-) => Promise<any>;
+type IndexFailureHandler = (deploymentId: string, details: FailureDetails) => Promise<any>;
+
+type RunFailureHandler = (runId: string, details: FailureDetails) => Promise<any>;
 
 type TaskMonitorOptions = {
   runtimeEnv: "local" | "kubernetes";
@@ -144,8 +139,7 @@ export class TaskMonitor {
     const containerState = this.#getContainerStateSummary(containerStatus.state);
     const exitCode = containerState.exitCode ?? -1;
 
-    // We use this special exit code to signal any errors were already handled elsewhere
-    if (exitCode === 111) {
+    if (exitCode === EXIT_CODE_ALREADY_HANDLED) {
       return;
     }
 
@@ -162,6 +156,7 @@ export class TaskMonitor {
 
     let reason = rawReason || "Unknown error";
     let logs = rawLogs || "";
+    let overrideCompletion = false;
 
     switch (rawReason) {
       case "Error":
@@ -181,7 +176,10 @@ export class TaskMonitor {
         }
         break;
       case "OOMKilled":
-        reason = "Out of memory! Try increasing the memory on this task.";
+        overrideCompletion = true;
+        reason = `${
+          exitCode === EXIT_CODE_CHILD_NONZERO ? "Child process" : "Parent process"
+        } ran out of memory! Try choosing a machine preset with more memory for this task.`;
         break;
       default:
         break;
@@ -191,7 +189,8 @@ export class TaskMonitor {
       exitCode,
       reason,
       logs,
-    };
+      overrideCompletion,
+    } satisfies FailureDetails;
 
     const app = pod.metadata?.labels?.app;
 

apps/webapp/app/components/primitives/TreeView/TreeView.tsx

@@ -528,7 +528,7 @@ export type Tree<TData> = {
 /** A tree but flattened so it can easily be used for DOM elements */
 export type FlatTreeItem<TData> = {
   id: string;
-  parentId: string | undefined;
+  parentId?: string | undefined;
   children: string[];
   hasChildren: boolean;
   /** The indentation level, the root is 0 */

apps/webapp/app/components/runs/v3/RunFilters.tsx

@@ -146,7 +146,6 @@ function FilterMenu(props: RunFiltersProps) {
 
   const filterTrigger = (
     <SelectTrigger
-      autoFocus
       icon={
         <div className="flex size-4 items-center justify-center">
           <ListFilterIcon className="size-3.5" />

apps/webapp/app/components/runs/v3/TaskRunStatus.tsx

@@ -3,10 +3,12 @@ import {
   BoltSlashIcon,
   BugAntIcon,
   CheckCircleIcon,
+  ClockIcon,
   FireIcon,
   NoSymbolIcon,
   PauseCircleIcon,
   RectangleStackIcon,
+  TrashIcon,
   XCircleIcon,
 } from "@heroicons/react/20/solid";
 import { TaskRunStatus } from "@trigger.dev/database";
@@ -16,6 +18,7 @@ import { Spinner } from "~/components/primitives/Spinner";
 import { cn } from "~/utils/cn";
 
 export const allTaskRunStatuses = [
+  "DELAYED",
   "WAITING_FOR_DEPLOY",
   "PENDING",
   "EXECUTING",
@@ -28,10 +31,12 @@ export const allTaskRunStatuses = [
   "PAUSED",
   "INTERRUPTED",
   "SYSTEM_FAILURE",
+  "EXPIRED",
 ] as const satisfies Readonly<Array<TaskRunStatus>>;
 
 export const filterableTaskRunStatuses = [
   "WAITING_FOR_DEPLOY",
+  "DELAYED",
   "PENDING",
   "EXECUTING",
   "RETRYING_AFTER_FAILURE",
@@ -42,9 +47,11 @@ export const filterableTaskRunStatuses = [
   "CRASHED",
   "INTERRUPTED",
   "SYSTEM_FAILURE",
+  "EXPIRED",
 ] as const satisfies Readonly<Array<TaskRunStatus>>;
 
 const taskRunStatusDescriptions: Record<TaskRunStatus, string> = {
+  DELAYED: "Task has been delayed and is waiting to be executed",
   PENDING: "Task is waiting to be executed",
   WAITING_FOR_DEPLOY: "Task needs to be deployed first to start executing",
   EXECUTING: "Task is currently being executed",
@@ -57,9 +64,10 @@ const taskRunStatusDescriptions: Record<TaskRunStatus, string> = {
   SYSTEM_FAILURE: "Task has failed due to a system failure",
   PAUSED: "Task has been paused by the user",
   CRASHED: "Task has crashed and won't be retried",
+  EXPIRED: "Task has surpassed its ttl and won't be executed",
 };
 
-export const QUEUED_STATUSES: TaskRunStatus[] = ["PENDING", "WAITING_FOR_DEPLOY"];
+export const QUEUED_STATUSES: TaskRunStatus[] = ["PENDING", "WAITING_FOR_DEPLOY", "DELAYED"];
 
 export const RUNNING_STATUSES: TaskRunStatus[] = [
   "EXECUTING",
@@ -74,6 +82,7 @@ export const FINISHED_STATUSES: TaskRunStatus[] = [
   "INTERRUPTED",
   "SYSTEM_FAILURE",
   "CRASHED",
+  "EXPIRED",
 ];
 
 export function descriptionForTaskRunStatus(status: TaskRunStatus): string {
@@ -109,6 +118,8 @@ export function TaskRunStatusIcon({
   className: string;
 }) {
   switch (status) {
+    case "DELAYED":
+      return <ClockIcon className={cn(runStatusClassNameColor(status), className)} />;
     case "PENDING":
       return <RectangleStackIcon className={cn(runStatusClassNameColor(status), className)} />;
     case "WAITING_FOR_DEPLOY":
@@ -133,6 +144,8 @@ export function TaskRunStatusIcon({
       return <BugAntIcon className={cn(runStatusClassNameColor(status), className)} />;
     case "CRASHED":
       return <FireIcon className={cn(runStatusClassNameColor(status), className)} />;
+    case "EXPIRED":
+      return <TrashIcon className={cn(runStatusClassNameColor(status), className)} />;
 
     default: {
       assertNever(status);
@@ -143,6 +156,7 @@ export function TaskRunStatusIcon({
 export function runStatusClassNameColor(status: TaskRunStatus): string {
   switch (status) {
     case "PENDING":
+    case "DELAYED":
       return "text-charcoal-500";
     case "WAITING_FOR_DEPLOY":
       return "text-amber-500";
@@ -154,6 +168,7 @@ export function runStatusClassNameColor(status: TaskRunStatus): string {
     case "PAUSED":
       return "text-amber-300";
     case "CANCELED":
+    case "EXPIRED":
       return "text-charcoal-500";
     case "INTERRUPTED":
       return "text-error";
@@ -173,6 +188,8 @@ export function runStatusClassNameColor(status: TaskRunStatus): string {
 
 export function runStatusTitle(status: TaskRunStatus): string {
   switch (status) {
+    case "DELAYED":
+      return "Delayed";
     case "PENDING":
       return "Queued";
     case "WAITING_FOR_DEPLOY":
@@ -197,6 +214,8 @@ export function runStatusTitle(status: TaskRunStatus): string {
       return "System failure";
     case "CRASHED":
       return "Crashed";
+    case "EXPIRED":
+      return "Expired";
     default: {
       assertNever(status);
     }

apps/webapp/app/components/runs/v3/TaskRunsTable.tsx

@@ -118,6 +118,8 @@ export function TaskRunsTable({
           <TableHeaderCell>Duration</TableHeaderCell>
           <TableHeaderCell>Test</TableHeaderCell>
           <TableHeaderCell>Created at</TableHeaderCell>
+          <TableHeaderCell>Delayed until</TableHeaderCell>
+          <TableHeaderCell>TTL</TableHeaderCell>
           <TableHeaderCell>
             <span className="sr-only">Go to page</span>
           </TableHeaderCell>
@@ -125,7 +127,7 @@ export function TaskRunsTable({
       </TableHeader>
       <TableBody>
         {total === 0 && !hasFilters ? (
-          <TableBlankRow colSpan={9}>
+          <TableBlankRow colSpan={10}>
             {!isLoading && <NoRuns title="No runs found" />}
           </TableBlankRow>
         ) : runs.length === 0 ? (
@@ -187,6 +189,10 @@ export function TaskRunsTable({
                 <TableCell to={path}>
                   {run.createdAt ? <DateTime date={run.createdAt} /> : "–"}
                 </TableCell>
+                <TableCell to={path}>
+                  {run.delayUntil ? <DateTime date={run.delayUntil} /> : "–"}
+                </TableCell>
+                <TableCell to={path}>{run.ttl ?? "–"}</TableCell>
                 <RunActionsCell run={run} path={path} />
               </TableRow>
             );

apps/webapp/app/database-types.ts

@@ -40,6 +40,8 @@ export const TaskRunStatus = {
   COMPLETED_WITH_ERRORS: "COMPLETED_WITH_ERRORS",
   SYSTEM_FAILURE: "SYSTEM_FAILURE",
   CRASHED: "CRASHED",
+  DELAYED: "DELAYED",
+  EXPIRED: "EXPIRED",
 } as const satisfies Record<TaskRunStatusType, TaskRunStatusType>;
 
 export const JobRunStatus = {

apps/webapp/app/entry.server.tsx

@@ -186,3 +186,9 @@ export { apiRateLimiter } from "./services/apiRateLimit.server";
 export { socketIo } from "./v3/handleSocketIo.server";
 export { wss } from "./v3/handleWebsockets.server";
 export { registryProxy } from "./v3/registryProxy.server";
+import { eventLoopMonitor } from "./eventLoopMonitor.server";
+import { env } from "./env.server";
+
+if (env.EVENT_LOOP_MONITOR_ENABLED === "1") {
+  eventLoopMonitor.enable();
+}

apps/webapp/app/env.server.ts

@@ -204,6 +204,11 @@ const EnvironmentSchema = z.object({
 
   USAGE_OPEN_METER_API_KEY: z.string().optional(),
   USAGE_OPEN_METER_BASE_URL: z.string().optional(),
+  EVENT_LOOP_MONITOR_ENABLED: z.string().default("1"),
+  MAXIMUM_LIVE_RELOADING_EVENTS: z.coerce.number().int().default(1000),
+  MAXIMUM_TRACE_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(25_000),
+  TASK_PAYLOAD_OFFLOAD_THRESHOLD: z.coerce.number().int().default(524_288), // 512KB
+  TASK_PAYLOAD_MAXIMUM_SIZE: z.coerce.number().int().default(3_145_728), // 3MB
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/eventLoopMonitor.server.ts

@@ -0,0 +1,77 @@
+import { createHook } from "node:async_hooks";
+import { singleton } from "./utils/singleton";
+import { tracer } from "./v3/tracer.server";
+
+const THRESHOLD_NS = 1e8; // 100ms
+
+const cache = new Map<number, { type: string; start?: [number, number] }>();
+
+function init(asyncId: number, type: string, triggerAsyncId: number, resource: any) {
+  cache.set(asyncId, {
+    type,
+  });
+}
+
+function destroy(asyncId: number) {
+  cache.delete(asyncId);
+}
+
+function before(asyncId: number) {
+  const cached = cache.get(asyncId);
+
+  if (!cached) {
+    return;
+  }
+
+  cache.set(asyncId, {
+    ...cached,
+    start: process.hrtime(),
+  });
+}
+
+function after(asyncId: number) {
+  const cached = cache.get(asyncId);
+
+  if (!cached) {
+    return;
+  }
+
+  cache.delete(asyncId);
+
+  if (!cached.start) {
+    return;
+  }
+
+  const diff = process.hrtime(cached.start);
+  const diffNs = diff[0] * 1e9 + diff[1];
+  if (diffNs > THRESHOLD_NS) {
+    const time = diffNs / 1e6; // in ms
+
+    const newSpan = tracer.startSpan("event-loop-blocked", {
+      startTime: new Date(new Date().getTime() - time),
+      attributes: {
+        asyncType: cached.type,
+        label: "EventLoopMonitor",
+      },
+    });
+
+    newSpan.end();
+  }
+}
+
+export const eventLoopMonitor = singleton("eventLoopMonitor", () => {
+  const hook = createHook({ init, before, after, destroy });
+
+  return {
+    enable: () => {
+      console.log("🥸  Initializing event loop monitor");
+
+      hook.enable();
+    },
+    disable: () => {
+      console.log("🥸  Disabling event loop monitor");
+
+      hook.disable();
+    },
+  };
+});

apps/webapp/app/hooks/useEventSource.tsx

@@ -26,7 +26,7 @@ export function useEventSource(
     const eventSource = new EventSource(url, init);
     eventSource.addEventListener(event ?? "message", handler);
 
-    // rest data if dependencies change
+    // reset data if dependencies change
     setData(null);
 
     function handler(event: MessageEvent) {

apps/webapp/app/models/api-key.server.ts

@@ -84,7 +84,9 @@ export function createPkApiKeyForEnv(envType: RuntimeEnvironment["type"]) {
   return `pk_${envSlug(envType)}_${apiKeyId(20)}`;
 }
 
-export function envSlug(environmentType: RuntimeEnvironment["type"]) {
+export type EnvSlug = "dev" | "stg" | "prod" | "prev";
+
+export function envSlug(environmentType: RuntimeEnvironment["type"]): EnvSlug {
   switch (environmentType) {
     case "DEVELOPMENT": {
       return "dev";
@@ -100,3 +102,7 @@ export function envSlug(environmentType: RuntimeEnvironment["type"]) {
     }
   }
 }
+
+export function isEnvSlug(maybeSlug: string): maybeSlug is EnvSlug {
+  return ["dev", "stg", "prod", "prev"].includes(maybeSlug);
+}

apps/webapp/app/models/runtimeEnvironment.server.ts

@@ -65,59 +65,55 @@ export async function findEnvironmentById(id: string) {
 }
 
 export async function createNewSession(environment: RuntimeEnvironment, ipAddress: string) {
-  return prisma.$transaction(async (tx) => {
-    const session = await tx.runtimeEnvironmentSession.create({
-      data: {
-        environmentId: environment.id,
-        ipAddress,
-      },
-    });
-
-    await tx.runtimeEnvironment.update({
-      where: {
-        id: environment.id,
-      },
-      data: {
-        currentSessionId: session.id,
-      },
-    });
-
-    return session;
+  const session = await prisma.runtimeEnvironmentSession.create({
+    data: {
+      environmentId: environment.id,
+      ipAddress,
+    },
   });
+
+  await prisma.runtimeEnvironment.update({
+    where: {
+      id: environment.id,
+    },
+    data: {
+      currentSessionId: session.id,
+    },
+  });
+
+  return session;
 }
 
 export async function disconnectSession(environmentId: string) {
-  return prisma.$transaction(async (tx) => {
-    const environment = await tx.runtimeEnvironment.findUnique({
-      where: {
-        id: environmentId,
-      },
-    });
-
-    if (!environment || !environment.currentSessionId) {
-      return null;
-    }
-
-    const session = await tx.runtimeEnvironmentSession.update({
-      where: {
-        id: environment.currentSessionId,
-      },
-      data: {
-        disconnectedAt: new Date(),
-      },
-    });
-
-    await tx.runtimeEnvironment.update({
-      where: {
-        id: environment.id,
-      },
-      data: {
-        currentSessionId: null,
-      },
-    });
-
-    return session;
+  const environment = await prisma.runtimeEnvironment.findUnique({
+    where: {
+      id: environmentId,
+    },
   });
+
+  if (!environment || !environment.currentSessionId) {
+    return null;
+  }
+
+  const session = await prisma.runtimeEnvironmentSession.update({
+    where: {
+      id: environment.currentSessionId,
+    },
+    data: {
+      disconnectedAt: new Date(),
+    },
+  });
+
+  await prisma.runtimeEnvironment.update({
+    where: {
+      id: environment.id,
+    },
+    data: {
+      currentSessionId: null,
+    },
+  });
+
+  return session;
 }
 
 type DisplayableInputEnvironment = Prisma.RuntimeEnvironmentGetPayload<{

apps/webapp/app/models/taskRun.server.ts

@@ -118,6 +118,7 @@ export function batchTaskRunItemStatusForRunStatus(
     case TaskRunStatus.COMPLETED_WITH_ERRORS:
     case TaskRunStatus.SYSTEM_FAILURE:
     case TaskRunStatus.CRASHED:
+    case TaskRunStatus.EXPIRED:
       return BatchTaskRunItemStatus.FAILED;
     case TaskRunStatus.PENDING:
     case TaskRunStatus.WAITING_FOR_DEPLOY:
@@ -125,6 +126,7 @@ export function batchTaskRunItemStatusForRunStatus(
     case TaskRunStatus.RETRYING_AFTER_FAILURE:
     case TaskRunStatus.EXECUTING:
     case TaskRunStatus.PAUSED:
+    case TaskRunStatus.DELAYED:
       return BatchTaskRunItemStatus.PENDING;
     default:
       assertNever(status);

apps/webapp/app/presenters/RunListPresenter.server.ts

@@ -1,11 +1,9 @@
-import { z } from "zod";
 import {
   Direction,
   FilterableEnvironment,
   FilterableStatus,
   filterableStatuses,
 } from "~/components/runs/RunStatuses";
-import { PrismaClient, prisma } from "~/db.server";
 import { getUsername } from "~/utils/username";
 import { BasePresenter } from "./v3/basePresenter.server";
 
@@ -29,8 +27,6 @@ const DEFAULT_PAGE_SIZE = 20;
 export type RunList = Awaited<ReturnType<RunListPresenter["call"]>>;
 
 export class RunListPresenter extends BasePresenter {
-  
-
   public async call({
     userId,
     eventId,

apps/webapp/app/presenters/ScheduledTriggersPresenter.server.ts

@@ -1,27 +1,52 @@
-import { User } from "@trigger.dev/database";
 import { ScheduleMetadataSchema } from "@trigger.dev/core";
-import { PrismaClient, prisma } from "~/db.server";
+import { User } from "@trigger.dev/database";
 import { Organization } from "~/models/organization.server";
 import { Project } from "~/models/project.server";
 import { calculateNextScheduledEvent } from "~/services/schedules/nextScheduledEvent.server";
+import { BasePresenter } from "./v3/basePresenter.server";
 
-export class ScheduledTriggersPresenter {
-  #prismaClient: PrismaClient;
-
-  constructor(prismaClient: PrismaClient = prisma) {
-    this.#prismaClient = prismaClient;
-  }
+const DEFAULT_PAGE_SIZE = 20;
 
+export class ScheduledTriggersPresenter extends BasePresenter {
   public async call({
     userId,
     projectSlug,
     organizationSlug,
+    direction = "forward",
+    pageSize = DEFAULT_PAGE_SIZE,
+    cursor,
   }: {
     userId: User["id"];
     projectSlug: Project["slug"];
     organizationSlug: Organization["slug"];
+    direction?: "forward" | "backward";
+    pageSize?: number;
+    cursor?: string;
   }) {
-    const scheduled = await this.#prismaClient.scheduleSource.findMany({
+    const organization = await this._replica.organization.findFirstOrThrow({
+      select: {
+        id: true,
+      },
+      where: {
+        slug: organizationSlug,
+        members: { some: { userId } },
+      },
+    });
+
+    // Find the project scoped to the organization
+    const project = await this._replica.project.findFirstOrThrow({
+      select: {
+        id: true,
+      },
+      where: {
+        slug: projectSlug,
+        organizationId: organization.id,
+      },
+    });
+
+    const directionMultiplier = direction === "forward" ? 1 : -1;
+
+    const scheduled = await this._replica.scheduleSource.findMany({
       select: {
         id: true,
         key: true,
@@ -50,23 +75,50 @@ export class ScheduledTriggersPresenter {
               },
             },
           ],
-          organization: {
-            slug: organizationSlug,
-            members: {
-              some: {
-                userId,
-              },
-            },
-          },
-          project: {
-            slug: projectSlug,
-          },
+          projectId: project.id,
         },
       },
+      orderBy: [{ id: "desc" }],
+      //take an extra record to tell if there are more
+      take: directionMultiplier * (pageSize + 1),
+      //skip the cursor if there is one
+      skip: cursor ? 1 : 0,
+      cursor: cursor
+        ? {
+            id: cursor,
+          }
+        : undefined,
     });
 
+    const hasMore = scheduled.length > pageSize;
+
+    //get cursors for next and previous pages
+    let next: string | undefined;
+    let previous: string | undefined;
+    switch (direction) {
+      case "forward":
+        previous = cursor ? scheduled.at(0)?.id : undefined;
+        if (hasMore) {
+          next = scheduled[pageSize - 1]?.id;
+        }
+        break;
+      case "backward":
+        if (hasMore) {
+          previous = scheduled[1]?.id;
+          next = scheduled[pageSize]?.id;
+        } else {
+          next = scheduled[pageSize - 1]?.id;
+        }
+        break;
+    }
+
+    const scheduledToReturn =
+      direction === "backward" && hasMore
+        ? scheduled.slice(1, pageSize + 1)
+        : scheduled.slice(0, pageSize);
+
     return {
-      scheduled: scheduled.map((s) => {
+      scheduled: scheduledToReturn.map((s) => {
         const schedule = ScheduleMetadataSchema.parse(s.schedule);
         const nextEventTimestamp = s.active
           ? calculateNextScheduledEvent(schedule, s.lastEventTimestamp)
@@ -78,6 +130,10 @@ export class ScheduledTriggersPresenter {
           nextEventTimestamp,
         };
       }),
+      pagination: {
+        next,
+        previous,
+      },
     };
   }
 }

apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts

@@ -12,6 +12,7 @@ import {
 import { Prisma, TaskRunAttemptStatus, TaskRunStatus } from "@trigger.dev/database";
 import assertNever from "assert-never";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { generatePresignedUrl } from "~/v3/r2.server";
 import { BasePresenter } from "./basePresenter.server";
 
 export class ApiRetrieveRunPresenter extends BasePresenter {
@@ -44,7 +45,9 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
       }
 
       let $payload: any;
+      let $payloadPresignedUrl: string | undefined;
       let $output: any;
+      let $outputPresignedUrl: string | undefined;
 
       if (showSecretDetails) {
         const payloadPacket = await conditionallyImportPacket({
@@ -52,7 +55,19 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
           dataType: taskRun.payloadType,
         });
 
-        $payload = await parsePacket(payloadPacket);
+        if (
+          payloadPacket.dataType === "application/store" &&
+          typeof payloadPacket.data === "string"
+        ) {
+          $payloadPresignedUrl = await generatePresignedUrl(
+            env.project.externalRef,
+            env.slug,
+            payloadPacket.data,
+            "GET"
+          );
+        } else {
+          $payload = await parsePacket(payloadPacket);
+        }
 
         if (taskRun.status === "COMPLETED_SUCCESSFULLY") {
           const completedAttempt = taskRun.attempts.find(
@@ -65,7 +80,19 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
               dataType: completedAttempt.outputType,
             });
 
-            $output = await parsePacket(outputPacket);
+            if (
+              outputPacket.dataType === "application/store" &&
+              typeof outputPacket.data === "string"
+            ) {
+              $outputPresignedUrl = await generatePresignedUrl(
+                env.project.externalRef,
+                env.slug,
+                outputPacket.data,
+                "GET"
+              );
+            } else {
+              $output = await parsePacket(outputPacket);
+            }
           }
         }
       }
@@ -84,9 +111,14 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
         finishedAt: ApiRetrieveRunPresenter.isStatusFinished(apiStatus)
           ? taskRun.updatedAt
           : undefined,
+        delayedUntil: taskRun.delayUntil ?? undefined,
         payload: $payload,
+        payloadPresignedUrl: $payloadPresignedUrl,
         output: $output,
+        outputPresignedUrl: $outputPresignedUrl,
         isTest: taskRun.isTest,
+        ttl: taskRun.ttl ?? undefined,
+        expiredAt: taskRun.expiredAt ?? undefined,
         schedule: taskRun.schedule
           ? {
               id: taskRun.schedule.friendlyId,
@@ -142,6 +174,9 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
 
   static apiStatusFromRunStatus(status: TaskRunStatus): RunStatus {
     switch (status) {
+      case "DELAYED": {
+        return "DELAYED";
+      }
       case "WAITING_FOR_DEPLOY": {
         return "WAITING_FOR_DEPLOY";
       }
@@ -176,6 +211,9 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
       case "COMPLETED_WITH_ERRORS": {
         return "FAILED";
       }
+      case "EXPIRED": {
+        return "EXPIRED";
+      }
       default: {
         assertNever(status);
       }
@@ -183,7 +221,7 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
   }
 
   static apiBooleanHelpersFromRunStatus(status: RunStatus) {
-    const isQueued = status === "QUEUED" || status === "WAITING_FOR_DEPLOY";
+    const isQueued = status === "QUEUED" || status === "WAITING_FOR_DEPLOY" || status === "DELAYED";
     const isExecuting = status === "EXECUTING" || status === "REATTEMPTING" || status === "FROZEN";
     const isCompleted =
       status === "COMPLETED" ||

apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts

@@ -209,7 +209,10 @@ export class ApiRunListPresenter extends BasePresenter {
           updatedAt: new Date(run.updatedAt),
           startedAt: run.startedAt ? new Date(run.startedAt) : undefined,
           finishedAt: run.finishedAt ? new Date(run.finishedAt) : undefined,
+          delayedUntil: run.delayUntil ? new Date(run.delayUntil) : undefined,
           isTest: run.isTest,
+          ttl: run.ttl ?? undefined,
+          expiredAt: run.expiredAt ? new Date(run.expiredAt) : undefined,
           env: {
             id: run.environment.id,
             name: run.environment.slug,
@@ -233,6 +236,8 @@ export class ApiRunListPresenter extends BasePresenter {
 
   static apiStatusToRunStatuses(status: RunStatus): TaskRunStatus[] | TaskRunStatus {
     switch (status) {
+      case "DELAYED":
+        return "DELAYED";
       case "WAITING_FOR_DEPLOY": {
         return "WAITING_FOR_DEPLOY";
       }
@@ -266,6 +271,9 @@ export class ApiRunListPresenter extends BasePresenter {
       case "FAILED": {
         return "COMPLETED_WITH_ERRORS";
       }
+      case "EXPIRED": {
+        return "EXPIRED";
+      }
       default: {
         assertNever(status);
       }

apps/webapp/app/presenters/v3/RunListPresenter.server.ts

@@ -4,8 +4,8 @@ import { Direction } from "~/components/runs/RunStatuses";
 import { FINISHED_STATUSES } from "~/components/runs/v3/TaskRunStatus";
 import { sqlDatabaseSchema } from "~/db.server";
 import { displayableEnvironment } from "~/models/runtimeEnvironment.server";
-import { CANCELLABLE_STATUSES } from "~/v3/services/cancelTaskRun.server";
 import { BasePresenter } from "./basePresenter.server";
+import { isCancellableRunStatus } from "~/v3/taskStatus";
 
 export type RunListOptions = {
   userId?: string;
@@ -158,10 +158,13 @@ export class RunListPresenter extends BasePresenter {
         createdAt: Date;
         startedAt: Date | null;
         lockedAt: Date | null;
+        delayUntil: Date | null;
         updatedAt: Date;
         isTest: boolean;
         spanId: string;
         idempotencyKey: string | null;
+        ttl: string | null;
+        expiredAt: Date | null;
       }[]
     >`
     SELECT
@@ -174,11 +177,14 @@ export class RunListPresenter extends BasePresenter {
     tr.status AS status,
     tr."createdAt" AS "createdAt",
     tr."startedAt" AS "startedAt",
+    tr."delayUntil" AS "delayUntil",
     tr."lockedAt" AS "lockedAt",
     tr."updatedAt" AS "updatedAt",
     tr."isTest" AS "isTest",
     tr."spanId" AS "spanId",
-    tr."idempotencyKey" AS "idempotencyKey"
+    tr."idempotencyKey" AS "idempotencyKey",
+    tr."ttl" AS "ttl",
+    tr."expiredAt" AS "expiredAt"
   FROM
     ${sqlDatabaseSchema}."TaskRun" tr
   LEFT JOIN
@@ -283,6 +289,7 @@ export class RunListPresenter extends BasePresenter {
           createdAt: run.createdAt.toISOString(),
           updatedAt: run.updatedAt.toISOString(),
           startedAt: startedAt ? startedAt.toISOString() : undefined,
+          delayUntil: run.delayUntil ? run.delayUntil.toISOString() : undefined,
           hasFinished,
           finishedAt: hasFinished ? run.updatedAt.toISOString() : undefined,
           isTest: run.isTest,
@@ -291,9 +298,11 @@ export class RunListPresenter extends BasePresenter {
           taskIdentifier: run.taskIdentifier,
           spanId: run.spanId,
           isReplayable: true,
-          isCancellable: CANCELLABLE_STATUSES.includes(run.status),
+          isCancellable: isCancellableRunStatus(run.status),
           environment: displayableEnvironment(environment, userId),
           idempotencyKey: run.idempotencyKey ? run.idempotencyKey : undefined,
+          ttl: run.ttl ? run.ttl : undefined,
+          expiredAt: run.expiredAt ? run.expiredAt.toISOString() : undefined,
         };
       }),
       pagination: {

apps/webapp/app/presenters/v3/RunStreamPresenter.server.ts

@@ -94,17 +94,24 @@ export class RunStreamPresenter {
 
         eventEmitter.removeAllListeners();
 
-        unsubscribe().catch((error) => {
-          logger.error("RunStreamPresenter.abort.unsubscribe", {
-            runFriendlyId,
-            traceId: run.traceId,
-            error: {
-              name: error.name,
-              message: error.message,
-              stack: error.stack,
-            },
+        unsubscribe()
+          .then(() => {
+            logger.info("RunStreamPresenter.abort.unsubscribe succeeded", {
+              runFriendlyId,
+              traceId: run.traceId,
+            });
+          })
+          .catch((error) => {
+            logger.error("RunStreamPresenter.abort.unsubscribe failed", {
+              runFriendlyId,
+              traceId: run.traceId,
+              error: {
+                name: error.name,
+                message: error.message,
+                stack: error.stack,
+              },
+            });
           });
-        });
       };
     });
   }

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.triggers.scheduled/route.tsx

@@ -2,6 +2,8 @@ import { NoSymbolIcon } from "@heroicons/react/20/solid";
 import { CheckCircleIcon, XCircleIcon } from "@heroicons/react/24/solid";
 import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
+import { z } from "zod";
+import { ListPagination } from "~/components/ListPagination";
 import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
 import { DateTime } from "~/components/primitives/DateTime";
 import { LabelValueStack } from "~/components/primitives/LabelValueStack";
@@ -17,30 +19,38 @@ import {
   TableRow,
 } from "~/components/primitives/Table";
 import { TextLink } from "~/components/primitives/TextLink";
-import { useOrganization } from "~/hooks/useOrganizations";
-import { useProject } from "~/hooks/useProject";
+import { DirectionSchema } from "~/components/runs/RunStatuses";
 import { ScheduledTriggersPresenter } from "~/presenters/ScheduledTriggersPresenter.server";
 import { requireUserId } from "~/services/session.server";
 import { ProjectParamSchema, docsPath } from "~/utils/pathBuilder";
 
+const SearchSchema = z.object({
+  cursor: z.string().optional(),
+  direction: DirectionSchema.optional(),
+});
+
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const userId = await requireUserId(request);
   const { organizationSlug, projectParam } = ProjectParamSchema.parse(params);
 
+  const url = new URL(request.url);
+  const s = Object.fromEntries(url.searchParams.entries());
+  const searchParams = SearchSchema.parse(s);
+
   const presenter = new ScheduledTriggersPresenter();
   const data = await presenter.call({
     userId,
     organizationSlug,
     projectSlug: projectParam,
+    direction: searchParams.direction,
+    cursor: searchParams.cursor,
   });
 
   return typedjson(data);
 };
 
-export default function Integrations() {
-  const { scheduled } = useTypedLoaderData<typeof loader>();
-  const organization = useOrganization();
-  const project = useProject();
+export default function Route() {
+  const { scheduled, pagination } = useTypedLoaderData<typeof loader>();
 
   return (
     <>
@@ -49,6 +59,10 @@ export default function Integrations() {
         expression or an interval.
       </Paragraph>
 
+      {scheduled.length > 0 && (
+        <ListPagination list={{ pagination }} className="mt-2 justify-end" />
+      )}
+
       <Table containerClassName="mt-4">
         <TableHeader>
           <TableRow>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx

@@ -1,12 +1,13 @@
 import {
+  BoltSlashIcon,
   ChevronDownIcon,
   ChevronRightIcon,
   MagnifyingGlassMinusIcon,
   MagnifyingGlassPlusIcon,
 } from "@heroicons/react/20/solid";
 import type { Location } from "@remix-run/react";
-import { useParams, useRevalidator } from "@remix-run/react";
-import { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { useLoaderData, useParams, useRevalidator } from "@remix-run/react";
+import { LoaderFunctionArgs, SerializeFrom } from "@remix-run/server-runtime";
 import { Virtualizer } from "@tanstack/react-virtual";
 import {
   formatDurationMilliseconds,
@@ -17,10 +18,10 @@ import { RuntimeEnvironmentType } from "@trigger.dev/database";
 import { motion } from "framer-motion";
 import { useCallback, useEffect, useRef, useState } from "react";
 import { useHotkeys } from "react-hotkeys-hook";
-import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { ShowParentIcon, ShowParentIconSelected } from "~/assets/icons/ShowParentIcon";
 import tileBgPath from "~/assets/images/error-banner-tile@2x.png";
 import { BlankstateInstructions } from "~/components/BlankstateInstructions";
+import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
 import { InlineCode } from "~/components/code/InlineCode";
 import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
 import { MainCenteredContainer, PageBody } from "~/components/layout/AppLayout";
@@ -32,6 +33,7 @@ import { Input } from "~/components/primitives/Input";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
 import { Popover, PopoverArrowTrigger, PopoverContent } from "~/components/primitives/Popover";
+import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
 import {
   ResizableHandle,
   ResizablePanel,
@@ -54,7 +56,7 @@ import { useProject } from "~/hooks/useProject";
 import { useReplaceLocation } from "~/hooks/useReplaceLocation";
 import { Shortcut, useShortcutKeys } from "~/hooks/useShortcutKeys";
 import { useUser } from "~/hooks/useUser";
-import { RunEvent, RunPresenter } from "~/presenters/v3/RunPresenter.server";
+import { RunPresenter } from "~/presenters/v3/RunPresenter.server";
 import { getResizableRunSettings, setResizableRunSettings } from "~/services/resizablePanel";
 import { requireUserId } from "~/services/session.server";
 import { cn } from "~/utils/cn";
@@ -67,8 +69,10 @@ import {
   v3RunsPath,
 } from "~/utils/pathBuilder";
 import { SpanView } from "../resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route";
-import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
-import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
+import { SimpleTooltip } from "~/components/primitives/Tooltip";
+import { env } from "~/env.server";
+
+type TraceEvent = NonNullable<SerializeFrom<typeof loader>["trace"]>["events"][0];
 
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const userId = await requireUserId(request);
@@ -85,10 +89,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   //resizable settings
   const resizeSettings = await getResizableRunSettings(request);
 
-  return typedjson({
-    ...result,
+  return {
+    run: result.run,
+    trace: result.trace,
+    maximumLiveReloadingSetting: env.MAXIMUM_LIVE_RELOADING_EVENTS,
     resizeSettings,
-  });
+  };
 };
 
 function getSpanId(location: Location<any>): string | undefined {
@@ -97,7 +103,8 @@ function getSpanId(location: Location<any>): string | undefined {
 }
 
 export default function Page() {
-  const { run, trace, resizeSettings } = useTypedLoaderData<typeof loader>();
+  const { run, trace, resizeSettings, maximumLiveReloadingSetting } =
+    useLoaderData<typeof loader>();
   const organization = useOrganization();
   const project = useProject();
   const user = useUser();
@@ -167,6 +174,7 @@ export default function Page() {
   }
 
   const { events, parentRunFriendlyId, duration, rootSpanStatus, rootStartedAt } = trace;
+  const shouldLiveReload = events.length <= maximumLiveReloadingSetting;
 
   const changeToSpan = useDebounce((selectedSpan: string) => {
     replaceSearchParam("span", selectedSpan);
@@ -175,6 +183,7 @@ export default function Page() {
   const revalidator = useRevalidator();
   const streamedEvents = useEventSource(v3RunStreamingPath(organization, project, run), {
     event: "message",
+    disabled: !shouldLiveReload,
   });
   useEffect(() => {
     if (streamedEvents !== null) {
@@ -252,8 +261,10 @@ export default function Page() {
                 }}
                 totalDuration={duration}
                 rootSpanStatus={rootSpanStatus}
-                rootStartedAt={rootStartedAt}
+                rootStartedAt={rootStartedAt ? new Date(rootStartedAt) : undefined}
                 environmentType={run.environment.type}
+                shouldLiveReload={shouldLiveReload}
+                maximumLiveReloadingSetting={maximumLiveReloadingSetting}
               />
             </ResizablePanel>
             <ResizableHandle withHandle />
@@ -274,7 +285,7 @@ export default function Page() {
 }
 
 type TasksTreeViewProps = {
-  events: RunEvent[];
+  events: TraceEvent[];
   selectedId?: string;
   parentRunFriendlyId?: string;
   onSelectedIdChanged: (selectedId: string | undefined) => void;
@@ -282,6 +293,8 @@ type TasksTreeViewProps = {
   rootSpanStatus: "executing" | "completed" | "failed";
   rootStartedAt: Date | undefined;
   environmentType: RuntimeEnvironmentType;
+  shouldLiveReload: boolean;
+  maximumLiveReloadingSetting: number;
 };
 
 function TasksTreeView({
@@ -293,6 +306,8 @@ function TasksTreeView({
   rootSpanStatus,
   rootStartedAt,
   environmentType,
+  shouldLiveReload,
+  maximumLiveReloadingSetting,
 }: TasksTreeViewProps) {
   const [filterText, setFilterText] = useState("");
   const [errorsOnly, setErrorsOnly] = useState(false);
@@ -367,7 +382,11 @@ function TasksTreeView({
                   This is the root task
                 </Paragraph>
               )}
-              <LiveReloadingStatus rootSpanCompleted={rootSpanStatus !== "executing"} />
+              <LiveReloadingStatus
+                rootSpanCompleted={rootSpanStatus !== "executing"}
+                isLiveReloading={shouldLiveReload}
+                settingValue={maximumLiveReloadingSetting}
+              />
             </div>
             <TreeView
               parentRef={parentRef}
@@ -750,7 +769,7 @@ function TimelineView({
   );
 }
 
-function NodeText({ node }: { node: RunEvent }) {
+function NodeText({ node }: { node: TraceEvent }) {
   const className = "truncate";
   return (
     <Paragraph variant="small" className={cn(className)}>
@@ -759,7 +778,7 @@ function NodeText({ node }: { node: RunEvent }) {
   );
 }
 
-function NodeStatusIcon({ node }: { node: RunEvent }) {
+function NodeStatusIcon({ node }: { node: TraceEvent }) {
   if (node.data.level !== "TRACE") return null;
   if (node.data.style.variant !== "primary") return null;
 
@@ -834,16 +853,40 @@ function ShowParentLink({ runFriendlyId }: { runFriendlyId: string }) {
   );
 }
 
-function LiveReloadingStatus({ rootSpanCompleted }: { rootSpanCompleted: boolean }) {
+function LiveReloadingStatus({
+  rootSpanCompleted,
+  isLiveReloading,
+  settingValue,
+}: {
+  rootSpanCompleted: boolean;
+  isLiveReloading: boolean;
+  settingValue: number;
+}) {
   if (rootSpanCompleted) return null;
 
   return (
-    <div className="flex items-center gap-1">
-      <PulsingDot />
-      <Paragraph variant="extra-small" className="whitespace-nowrap text-blue-500">
-        Live reloading
-      </Paragraph>
-    </div>
+    <>
+      {isLiveReloading ? (
+        <div className="flex items-center gap-1">
+          <PulsingDot />
+          <Paragraph variant="extra-small" className="whitespace-nowrap text-blue-500">
+            Live reloading
+          </Paragraph>
+        </div>
+      ) : (
+        <SimpleTooltip
+          content={`Live reloading is disabled because you've exceeded ${settingValue} logs.`}
+          button={
+            <div className="flex items-center gap-1">
+              <BoltSlashIcon className="size-3.5 text-text-dimmed" />
+              <Paragraph variant="extra-small" className="whitespace-nowrap text-text-dimmed">
+                Live reloading disabled
+              </Paragraph>
+            </div>
+          }
+        ></SimpleTooltip>
+      )}
+    </>
   );
 }
 
@@ -862,7 +905,7 @@ function SpanWithDuration({
   showDuration,
   node,
   ...props
-}: Timeline.SpanProps & { node: RunEvent; showDuration: boolean }) {
+}: Timeline.SpanProps & { node: TraceEvent; showDuration: boolean }) {
   return (
     <Timeline.Span {...props}>
       <motion.div

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.schedules.$scheduleParam/route.tsx

@@ -23,6 +23,7 @@ import { Paragraph } from "~/components/primitives/Paragraph";
 import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
 import {
   Table,
+  TableBlankRow,
   TableBody,
   TableCell,
   TableHeader,
@@ -180,6 +181,14 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
   }
 };
 
+function PlaceholderText({ title }: { title: string }) {
+  return (
+    <div className="flex items-center justify-center">
+      <Paragraph className="w-auto">{title}</Paragraph>
+    </div>
+  );
+}
+
 export default function Page() {
   const { schedule } = useTypedLoaderData<typeof loader>();
   const location = useLocation();
@@ -252,18 +261,30 @@ export default function Page() {
                   </TableRow>
                 </TableHeader>
                 <TableBody>
-                  {schedule.nextRuns.map((run, index) => (
-                    <TableRow key={index}>
-                      {!isUtc && (
-                        <TableCell>
-                          <DateTime date={run} timeZone={schedule.timezone} />
-                        </TableCell>
-                      )}
-                      <TableCell>
-                        <DateTime date={run} timeZone="UTC" />
-                      </TableCell>
-                    </TableRow>
-                  ))}
+                  {schedule.active ? (
+                    schedule.nextRuns.length ? (
+                      schedule.nextRuns.map((run, index) => (
+                        <TableRow key={index}>
+                          {!isUtc && (
+                            <TableCell>
+                              <DateTime date={run} timeZone={schedule.timezone} />
+                            </TableCell>
+                          )}
+                          <TableCell>
+                            <DateTime date={run} timeZone="UTC" />
+                          </TableCell>
+                        </TableRow>
+                      ))
+                    ) : (
+                      <TableBlankRow colSpan={1}>
+                        <PlaceholderText title="You found a bug" />
+                      </TableBlankRow>
+                    )
+                  ) : (
+                    <TableBlankRow colSpan={1}>
+                      <PlaceholderText title="Schedule disabled" />
+                    </TableBlankRow>
+                  )}
                 </TableBody>
               </Table>
             </div>

apps/webapp/app/routes/api.v1.packets.$.ts

@@ -1,10 +1,8 @@
 import type { ActionFunctionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
 import { z } from "zod";
-import { env } from "~/env.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
-import { logger } from "~/services/logger.server";
-import { r2 } from "~/v3/r2.server";
+import { generatePresignedUrl } from "~/v3/r2.server";
 
 const ParamsSchema = z.object({
   "*": z.string(),
@@ -26,34 +24,19 @@ export async function action({ request, params }: ActionFunctionArgs) {
   const parsedParams = ParamsSchema.parse(params);
   const filename = parsedParams["*"];
 
-  if (!env.OBJECT_STORE_BASE_URL) {
-    return json({ error: "Object store base URL is not set" }, { status: 500 });
-  }
-
-  if (!r2) {
-    return json({ error: "Object store credentials are not set" }, { status: 500 });
-  }
-
-  const url = new URL(env.OBJECT_STORE_BASE_URL);
-  url.pathname = `/packets/${authenticationResult.environment.project.externalRef}/${authenticationResult.environment.slug}/${filename}`;
-  url.searchParams.set("X-Amz-Expires", "300"); // 5 minutes
-
-  const signed = await r2.sign(
-    new Request(url, {
-      method: "PUT",
-    }),
-    {
-      aws: { signQuery: true },
-    }
+  const presignedUrl = await generatePresignedUrl(
+    authenticationResult.environment.project.externalRef,
+    authenticationResult.environment.slug,
+    filename,
+    "PUT"
   );
 
-  logger.debug("Generated presigned URL", {
-    url: signed.url,
-    headers: Object.fromEntries(signed.headers),
-  });
+  if (!presignedUrl) {
+    return json({ error: "Failed to generate presigned URL" }, { status: 500 });
+  }
 
   // Caller can now use this URL to upload to that object.
-  return json({ presignedUrl: signed.url });
+  return json({ presignedUrl });
 }
 
 export async function loader({ request, params }: ActionFunctionArgs) {
@@ -67,35 +50,17 @@ export async function loader({ request, params }: ActionFunctionArgs) {
   const parsedParams = ParamsSchema.parse(params);
   const filename = parsedParams["*"];
 
-  if (!env.OBJECT_STORE_BASE_URL) {
-    return json({ error: "Object store base URL is not set" }, { status: 500 });
-  }
-
-  if (!r2) {
-    return json({ error: "Object store credentials are not set" }, { status: 500 });
-  }
-
-  const url = new URL(env.OBJECT_STORE_BASE_URL);
-  url.pathname = `/packets/${authenticationResult.environment.project.externalRef}/${authenticationResult.environment.slug}/${filename}`;
-  url.searchParams.set("X-Amz-Expires", "300"); // 5 minutes
-
-  const signed = await r2.sign(
-    new Request(url, {
-      method: request.method,
-    }),
-    {
-      aws: { signQuery: true },
-    }
+  const presignedUrl = await generatePresignedUrl(
+    authenticationResult.environment.project.externalRef,
+    authenticationResult.environment.slug,
+    filename,
+    "GET"
   );
 
-  logger.debug("Generated presigned URL", {
-    url: signed.url,
-    headers: Object.fromEntries(signed.headers),
-  });
+  if (!presignedUrl) {
+    return json({ error: "Failed to generate presigned URL" }, { status: 500 });
+  }
 
-  const getUrl = new URL(url.href);
-  getUrl.searchParams.delete("X-Amz-Expires");
-
-  // Caller can now use this URL to upload to that object.
-  return json({ presignedUrl: signed.url });
+  // Caller can now use this URL to fetch that object.
+  return json({ presignedUrl });
 }

apps/webapp/app/routes/api.v1.runs.$runId.tasks.$id.complete/CompleteRunTaskService.server.ts

@@ -3,6 +3,7 @@ import { PrismaClientOrTransaction, prisma } from "~/db.server";
 import { taskWithAttemptsToServerTask } from "~/models/task.server";
 import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
+import { startActiveSpan } from "~/v3/tracer.server";
 
 export class CompleteRunTaskService {
   #prismaClient: PrismaClientOrTransaction;
@@ -17,76 +18,81 @@ export class CompleteRunTaskService {
     id: string,
     taskBody: CompleteTaskBodyOutput
   ): Promise<ServerTask | undefined> {
-    const existingTask = await this.#prismaClient.task.findUnique({
-      where: {
-        id,
-      },
-      include: {
-        run: true,
-        attempts: {
-          where: {
-            status: "PENDING",
-          },
-          orderBy: {
-            number: "desc",
-          },
-          take: 1,
+    return startActiveSpan("CompleteRunTaskService.call", async (span) => {
+      span.setAttribute("runId", runId);
+      span.setAttribute("taskId", id);
+
+      const existingTask = await this.#prismaClient.task.findUnique({
+        where: {
+          id,
+        },
+        include: {
+          run: true,
+          attempts: {
+            where: {
+              status: "PENDING",
+            },
+            orderBy: {
+              number: "desc",
+            },
+            take: 1,
+          },
         },
-      },
-    });
-
-    if (!existingTask) {
-      return;
-    }
-
-    if (existingTask.runId !== runId) {
-      return;
-    }
-
-    if (existingTask.run.environmentId !== environment.id) {
-      return;
-    }
-
-    if (
-      existingTask.status === "COMPLETED" ||
-      existingTask.status === "ERRORED" ||
-      existingTask.status === "CANCELED"
-    ) {
-      logger.debug("Task already completed", {
-        existingTask,
       });
 
-      return taskWithAttemptsToServerTask(existingTask);
-    }
+      if (!existingTask) {
+        return;
+      }
 
-    if (existingTask.attempts.length === 1) {
-      await this.#prismaClient.taskAttempt.update({
+      if (existingTask.runId !== runId) {
+        return;
+      }
+
+      if (existingTask.run.environmentId !== environment.id) {
+        return;
+      }
+
+      if (
+        existingTask.status === "COMPLETED" ||
+        existingTask.status === "ERRORED" ||
+        existingTask.status === "CANCELED"
+      ) {
+        logger.debug("Task already completed", {
+          taskId: id,
+        });
+
+        return taskWithAttemptsToServerTask(existingTask);
+      }
+
+      if (existingTask.attempts.length === 1) {
+        await this.#prismaClient.taskAttempt.update({
+          where: {
+            id: existingTask.attempts[0].id,
+          },
+          data: {
+            status: "COMPLETED",
+          },
+        });
+      }
+
+      const updatedTask = await this.#prismaClient.task.update({
         where: {
-          id: existingTask.attempts[0].id,
+          id,
         },
         data: {
           status: "COMPLETED",
+          output: taskBody.output as any,
+          outputIsUndefined: typeof taskBody.output === "undefined",
+          completedAt: new Date(),
+          outputProperties: taskBody.properties,
+        },
+        include: {
+          attempts: true,
+          run: true,
         },
       });
-    }
 
-    const updatedTask = await this.#prismaClient.task.update({
-      where: {
-        id,
-      },
-      data: {
-        status: "COMPLETED",
-        output: taskBody.output as any,
-        outputIsUndefined: typeof taskBody.output === "undefined",
-        completedAt: new Date(),
-        outputProperties: taskBody.properties,
-      },
-      include: {
-        attempts: true,
-        run: true,
-      },
+      return taskWithAttemptsToServerTask(updatedTask);
     });
-
-    return taskWithAttemptsToServerTask(updatedTask);
   }
 }

apps/webapp/app/routes/api.v1.runs.$runId.tasks.$id.complete/route.ts

@@ -9,8 +9,10 @@ import {
 import { z } from "zod";
 import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
-import { logger } from "~/services/logger.server";
 import { CompleteRunTaskService } from "./CompleteRunTaskService.server";
+import { startActiveSpan } from "~/v3/tracer.server";
+import { parseRequestJsonAsync } from "~/utils/parseRequestJson.server";
+import { FailRunTaskService } from "../api.v1.runs.$runId.tasks.$id.fail/FailRunTaskService.server";
 
 const ParamsSchema = z.object({
   runId: z.string(),
@@ -44,46 +46,51 @@ export async function action({ request, params }: ActionFunctionArgs) {
     return json({ error: "Invalid headers" }, { status: 400 });
   }
 
+  // Check the content size of the request and make sure it's not too large
+  const contentLength = request.headers.get("content-length");
+
+  if (!contentLength || parseInt(contentLength) > 3 * 1024 * 1024) {
+    const service = new FailRunTaskService();
+
+    await service.call(authenticatedEnv, runId, id, {
+      error: {
+        message: "Task output is too large. The limit is 3MB",
+      },
+    });
+
+    return json({ error: "Task output is too large. The limit is 3MB" }, { status: 413 });
+  }
+
   const { "trigger-version": triggerVersion } = headers.data;
 
   // Now parse the request body
-  const anyBody = await request.json();
-
-  logger.debug("CompleteRunTaskService.call() request body", {
-    body: anyBody,
-    runId,
-    id,
-  });
+  const anyBody = await parseRequestJsonAsync(request, { runId });
 
   if (triggerVersion === API_VERSIONS.SERIALIZED_TASK_OUTPUT) {
-    const body = CompleteTaskBodyV2InputSchema.safeParse(anyBody);
+    const body = await startActiveSpan("CompleteTaskBodyV2InputSchema.safeParse()", async () => {
+      return CompleteTaskBodyV2InputSchema.safeParse(anyBody);
+    });
 
     if (!body.success) {
       return json({ error: "Invalid request body" }, { status: 400 });
     }
 
-    // Make sure the length of the output is less than 3MB
-    if (body.data.output && body.data.output.length > 3 * 1024 * 1024) {
-      return json({ error: "Output must be less than 3MB" }, { status: 400 });
-    }
-
     return await completeRunTask(authenticatedEnv, runId, id, {
       ...body.data,
       output: body.data.output ? (JSON.parse(body.data.output) as any) : undefined,
     });
   } else {
-    const body = CompleteTaskBodyInputSchema.safeParse(anyBody);
+    const body = await startActiveSpan("CompleteTaskBodyInputSchema.safeParse()", async () => {
+      return CompleteTaskBodyInputSchema.omit({ output: true }).safeParse(anyBody);
+    });
 
     if (!body.success) {
       return json({ error: "Invalid request body" }, { status: 400 });
     }
 
-    // Make sure the length of the output is less than 3MB
-    if (JSON.stringify(body.data.output).length > 3 * 1024 * 1024) {
-      return json({ error: "Output must be less than 3MB" }, { status: 400 });
-    }
+    const output = (anyBody as any).output;
 
-    return await completeRunTask(authenticatedEnv, runId, id, body.data);
+    return await completeRunTask(authenticatedEnv, runId, id, { ...body.data, output });
   }
 }
 
@@ -98,12 +105,6 @@ async function completeRunTask(
   try {
     const task = await service.call(environment, runId, id, taskBody);
 
-    logger.debug("CompleteRunTaskService.call() response body", {
-      runId,
-      id,
-      task,
-    });
-
     if (!task) {
       return json({ message: "Task not found" }, { status: 404 });
     }

apps/webapp/app/routes/api.v1.runs.$runId.tasks.$id.fail/FailRunTaskService.server.ts

@@ -57,10 +57,6 @@ export class FailRunTaskService {
         existingTask.status === "ERRORED" ||
         existingTask.status === "CANCELED"
       ) {
-        logger.debug("Task already completed", {
-          existingTask,
-        });
-
         return existingTask;
       }
 

apps/webapp/app/routes/api.v1.runs.$runId.tasks/route.ts

@@ -6,6 +6,8 @@ import { authenticateApiRequest } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { RunTaskService } from "~/services/tasks/runTask.server";
 import { ChangeRequestLazyLoadedCachedTasks } from "./ChangeRequestLazyLoadedCachedTasks.server";
+import { startActiveSpan } from "~/v3/tracer.server";
+import { parseRequestJsonAsync } from "~/utils/parseRequestJson.server";
 
 const ParamsSchema = z.object({
   runId: z.string(),
@@ -17,6 +19,8 @@ const HeadersSchema = z.object({
   "x-cached-tasks-cursor": z.string().optional().nullable(),
 });
 
+const BodySchema = RunTaskBodyOutputSchema.omit({ params: true });
+
 export async function action({ request, params }: ActionFunctionArgs) {
   // Ensure this is a POST request
   if (request.method.toUpperCase() !== "POST") {
@@ -44,18 +48,26 @@ export async function action({ request, params }: ActionFunctionArgs) {
 
   const { runId } = ParamsSchema.parse(params);
 
+  const contentLength = request.headers.get("content-length");
+
+  if (!contentLength || parseInt(contentLength) > 3 * 1024 * 1024) {
+    return json({ error: "Request body too large" }, { status: 413 });
+  }
+
   // Now parse the request body
-  const anyBody = await request.json();
+  const anyBody = await parseRequestJsonAsync(request, { runId });
 
-  logger.debug("RunTaskService.call() request body", {
-    body: anyBody,
-    runId,
-    idempotencyKey,
-    triggerVersion,
-    cachedTasksCursor,
-  });
-
-  const body = RunTaskBodyOutputSchema.safeParse(anyBody);
+  const body = await startActiveSpan(
+    "BodySchema.safeParse",
+    async () => {
+      return BodySchema.safeParse(anyBody);
+    },
+    {
+      attributes: {
+        runId,
+      },
+    }
+  );
 
   if (!body.success) {
     return json({ error: "Invalid request body" }, { status: 400 });
@@ -64,12 +76,9 @@ export async function action({ request, params }: ActionFunctionArgs) {
   const service = new RunTaskService();
 
   try {
-    const task = await service.call(runId, idempotencyKey, body.data);
-
-    logger.debug("RunTaskService.call() response body", {
-      runId,
-      idempotencyKey,
-      task,
+    const task = await service.call(runId, idempotencyKey, {
+      ...body.data,
+      params: (anyBody as any).params,
     });
 
     if (!task) {
@@ -84,7 +93,6 @@ export async function action({ request, params }: ActionFunctionArgs) {
       logger.debug(
         "RunTaskService.call() response migrating with ChangeRequestLazyLoadedCachedTasks",
         {
-          responseBody,
           cachedTasksCursor,
         }
       );

apps/webapp/app/routes/api.v1.runs.$runParam.reschedule.ts

@@ -0,0 +1,85 @@
+import type { ActionFunctionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import { RescheduleRunRequestBody } from "@trigger.dev/core/v3/schemas";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { ApiRetrieveRunPresenter } from "~/presenters/v3/ApiRetrieveRunPresenter.server";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { ServiceValidationError } from "~/v3/services/baseService.server";
+import { RescheduleTaskRunService } from "~/v3/services/rescheduleTaskRun.server";
+
+const ParamsSchema = z.object({
+  runParam: z.string(),
+});
+
+export async function action({ request, params }: ActionFunctionArgs) {
+  // Ensure this is a POST request
+  if (request.method.toUpperCase() !== "POST") {
+    return { status: 405, body: "Method Not Allowed" };
+  }
+
+  // Authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or missing API Key" }, { status: 401 });
+  }
+
+  const parsed = ParamsSchema.safeParse(params);
+
+  if (!parsed.success) {
+    return json({ error: "Invalid or missing run ID" }, { status: 400 });
+  }
+
+  const { runParam } = parsed.data;
+
+  const taskRun = await prisma.taskRun.findUnique({
+    where: {
+      friendlyId: runParam,
+      runtimeEnvironmentId: authenticationResult.environment.id,
+    },
+  });
+
+  if (!taskRun) {
+    return json({ error: "Run not found" }, { status: 404 });
+  }
+
+  const anyBody = await request.json();
+
+  const body = RescheduleRunRequestBody.safeParse(anyBody);
+
+  if (!body.success) {
+    return json({ error: "Invalid request body" }, { status: 400 });
+  }
+
+  const service = new RescheduleTaskRunService();
+
+  try {
+    const updatedRun = await service.call(taskRun, body.data);
+
+    if (!updatedRun) {
+      return json({ error: "An unknown error occurred" }, { status: 500 });
+    }
+
+    const presenter = new ApiRetrieveRunPresenter();
+    const result = await presenter.call(
+      updatedRun.friendlyId,
+      authenticationResult.environment,
+      true
+    );
+
+    if (!result) {
+      return json({ error: "Run not found" }, { status: 404 });
+    }
+
+    return json(result);
+  } catch (error) {
+    if (error instanceof ServiceValidationError) {
+      return json({ error: error.message }, { status: 400 });
+    } else if (error instanceof Error) {
+      return json({ error: error.message }, { status: 500 });
+    } else {
+      return json({ error: "An unknown error occurred" }, { status: 500 });
+    }
+  }
+}

apps/webapp/app/routes/api.v1.tasks.$taskId.batch.ts

@@ -7,6 +7,7 @@ import { authenticateApiRequest } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { BatchTriggerTaskService } from "~/v3/services/batchTriggerTask.server";
 import { HeadersSchema } from "./api.v1.tasks.$taskId.trigger";
+import { env } from "~/env.server";
 
 const ParamsSchema = z.object({
   taskId: z.string(),
@@ -43,6 +44,12 @@ export async function action({ request, params }: ActionFunctionArgs) {
 
   const { taskId } = ParamsSchema.parse(params);
 
+  const contentLength = request.headers.get("content-length");
+
+  if (!contentLength || parseInt(contentLength) > env.TASK_PAYLOAD_MAXIMUM_SIZE) {
+    return json({ error: "Request body too large" }, { status: 413 });
+  }
+
   // Now parse the request body
   const anyBody = await request.json();
 

apps/webapp/app/routes/api.v1.tasks.$taskId.trigger.ts

@@ -2,9 +2,12 @@ import type { ActionFunctionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
 import { TriggerTaskRequestBody } from "@trigger.dev/core/v3";
 import { z } from "zod";
+import { env } from "~/env.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
+import { parseRequestJsonAsync } from "~/utils/parseRequestJson.server";
 import { TriggerTaskService } from "~/v3/services/triggerTask.server";
+import { startActiveSpan } from "~/v3/tracer.server";
 
 const ParamsSchema = z.object({
   taskId: z.string(),
@@ -32,6 +35,12 @@ export async function action({ request, params }: ActionFunctionArgs) {
     return json({ error: "Invalid or Missing API key" }, { status: 401 });
   }
 
+  const contentLength = request.headers.get("content-length");
+
+  if (!contentLength || parseInt(contentLength) > env.TASK_PAYLOAD_MAXIMUM_SIZE) {
+    return json({ error: "Request body too large" }, { status: 413 });
+  }
+
   const rawHeaders = Object.fromEntries(request.headers);
 
   const headers = HeadersSchema.safeParse(rawHeaders);
@@ -52,9 +61,11 @@ export async function action({ request, params }: ActionFunctionArgs) {
   const { taskId } = ParamsSchema.parse(params);
 
   // Now parse the request body
-  const anyBody = await request.json();
+  const anyBody = await parseRequestJsonAsync(request, { taskId });
 
-  const body = TriggerTaskRequestBody.safeParse(anyBody);
+  const body = await startActiveSpan("TriggerTaskRequestBody.safeParse()", async (span) => {
+    return TriggerTaskRequestBody.safeParse(anyBody);
+  });
 
   if (!body.success) {
     return json({ error: "Invalid request body" }, { status: 400 });
@@ -76,17 +87,23 @@ export async function action({ request, params }: ActionFunctionArgs) {
       idempotencyKey,
       triggerVersion,
       headers: Object.fromEntries(request.headers),
-      body: body.data,
+      options: body.data.options,
       isFromWorker,
       traceContext,
     });
 
-    const run = await service.call(taskId, authenticationResult.environment, body.data, {
-      idempotencyKey: idempotencyKey ?? undefined,
-      triggerVersion: triggerVersion ?? undefined,
-      traceContext,
-      spanParentAsLink: spanParentAsLink === 1,
-    });
+    const run = await service.call(
+      taskId,
+      authenticationResult.environment,
+      { ...body.data },
+      // { ...body.data, payload: (anyBody as any).payload },
+      {
+        idempotencyKey: idempotencyKey ?? undefined,
+        triggerVersion: triggerVersion ?? undefined,
+        traceContext,
+        spanParentAsLink: spanParentAsLink === 1,
+      }
+    );
 
     if (!run) {
       return json({ error: "Task not found" }, { status: 404 });

apps/webapp/app/routes/api.v2.runs.$runParam.cancel.ts

@@ -33,6 +33,7 @@ export async function action({ request, params }: ActionFunctionArgs) {
   const taskRun = await prisma.taskRun.findUnique({
     where: {
       friendlyId: runParam,
+      runtimeEnvironmentId: authenticationResult.environment.id,
     },
   });
 

apps/webapp/app/routes/projects.v3.$projectRef.runs.ts

@@ -0,0 +1,74 @@
+import { LoaderFunctionArgs, redirect } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { EnvSlug, isEnvSlug } from "~/models/api-key.server";
+import { requireUserId } from "~/services/session.server";
+
+const ParamsSchema = z.object({
+  projectRef: z.string(),
+});
+
+export async function loader({ params, request }: LoaderFunctionArgs) {
+  const userId = await requireUserId(request);
+
+  const { projectRef } = ParamsSchema.parse(params);
+
+  const project = await prisma.project.findFirst({
+    where: {
+      externalRef: projectRef,
+      organization: {
+        members: {
+          some: {
+            userId,
+          },
+        },
+      },
+    },
+    include: {
+      organization: true,
+    },
+  });
+
+  if (!project) {
+    return new Response("Project not found", { status: 404 });
+  }
+
+  const url = new URL(request.url);
+  const envSlug = url.searchParams.get("envSlug");
+
+  // Get the environment from the slug
+  if (envSlug && isEnvSlug(envSlug)) {
+    const env = await getEnvFromSlug(project.id, userId, envSlug);
+
+    if (env) {
+      url.searchParams.set("environments", env.id);
+    }
+
+    url.searchParams.delete("envSlug");
+  }
+
+  return redirect(
+    `/orgs/${project.organization.slug}/projects/v3/${project.slug}/runs${url.search}`
+  );
+}
+
+async function getEnvFromSlug(projectId: string, userId: string, envSlug: EnvSlug) {
+  if (envSlug === "dev") {
+    return await prisma.runtimeEnvironment.findFirst({
+      where: {
+        projectId,
+        slug: envSlug,
+        orgMember: {
+          userId,
+        },
+      },
+    });
+  }
+
+  return await prisma.runtimeEnvironment.findFirst({
+    where: {
+      projectId,
+      slug: envSlug,
+    },
+  });
+}

apps/webapp/app/routes/projects.v3.$projectRef.test.ts

@@ -0,0 +1,40 @@
+import { LoaderFunctionArgs, redirect } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { requireUserId } from "~/services/session.server";
+
+const ParamsSchema = z.object({
+  projectRef: z.string(),
+});
+
+export async function loader({ params, request }: LoaderFunctionArgs) {
+  const userId = await requireUserId(request);
+
+  const validatedParams = ParamsSchema.parse(params);
+
+  const project = await prisma.project.findFirst({
+    where: {
+      externalRef: validatedParams.projectRef,
+      organization: {
+        members: {
+          some: {
+            userId,
+          },
+        },
+      },
+    },
+    include: {
+      organization: true,
+    },
+  });
+
+  if (!project) {
+    return new Response("Not found", { status: 404 });
+  }
+
+  const url = new URL(request.url);
+
+  return redirect(
+    `/orgs/${project.organization.slug}/projects/v3/${project.slug}/test${url.search}`
+  );
+}

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx

@@ -33,7 +33,13 @@ import { redirectWithErrorMessage } from "~/models/message.server";
 import { Span, SpanPresenter } from "~/presenters/v3/SpanPresenter.server";
 import { requireUserId } from "~/services/session.server";
 import { cn } from "~/utils/cn";
-import { v3RunPath, v3RunSpanPath, v3SpanParamsSchema, v3TraceSpanPath } from "~/utils/pathBuilder";
+import {
+  v3RunDownloadLogsPath,
+  v3RunPath,
+  v3RunSpanPath,
+  v3SpanParamsSchema,
+  v3TraceSpanPath,
+} from "~/utils/pathBuilder";
 import { SpanLink } from "~/v3/eventRepository.server";
 
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
@@ -256,6 +262,15 @@ function RunActionButtons({ span }: { span: Span }) {
   if (span.isPartial) {
     return (
       <Dialog>
+        <LinkButton
+          to={v3RunDownloadLogsPath({ friendlyId: runParam })}
+          LeadingIcon={CloudArrowDownIcon}
+          variant="tertiary/medium"
+          target="_blank"
+          download
+        >
+          Download logs
+        </LinkButton>
         <DialogTrigger asChild>
           <Button variant="danger/medium" LeadingIcon={StopCircleIcon}>
             Cancel run
@@ -276,6 +291,15 @@ function RunActionButtons({ span }: { span: Span }) {
 
   return (
     <Dialog>
+      <LinkButton
+        to={v3RunDownloadLogsPath({ friendlyId: runParam })}
+        LeadingIcon={CloudArrowDownIcon}
+        variant="tertiary/medium"
+        target="_blank"
+        download
+      >
+        Download logs
+      </LinkButton>
       <DialogTrigger asChild>
         <Button variant="tertiary/medium" LeadingIcon={ArrowPathIcon}>
           Replay run

apps/webapp/app/routes/resources.packets.$environmentId.$.ts

@@ -2,9 +2,8 @@ import { LoaderFunctionArgs } from "@remix-run/node";
 import { basename } from "node:path";
 import { z } from "zod";
 import { prisma } from "~/db.server";
-import { env } from "~/env.server";
 import { requireUserId } from "~/services/session.server";
-import { r2 } from "~/v3/r2.server";
+import { generatePresignedRequest } from "~/v3/r2.server";
 
 const ParamSchema = z.object({
   environmentId: z.string(),
@@ -35,27 +34,17 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
     return new Response("Not found", { status: 404 });
   }
 
-  if (!env.OBJECT_STORE_BASE_URL) {
-    return new Response("Object store base URL is not set", { status: 500 });
-  }
-
-  if (!r2) {
-    return new Response("Object store credentials are not set", { status: 500 });
-  }
-
-  const url = new URL(env.OBJECT_STORE_BASE_URL);
-  url.pathname = `/packets/${environment.project.externalRef}/${environment.slug}/${filename}`;
-  url.searchParams.set("X-Amz-Expires", "30"); // 30 seconds
-
-  const signed = await r2.sign(
-    new Request(url, {
-      method: "GET",
-    }),
-    {
-      aws: { signQuery: true },
-    }
+  const signed = await generatePresignedRequest(
+    environment.project.externalRef,
+    environment.slug,
+    filename,
+    "GET"
   );
 
+  if (!signed) {
+    return new Response("Failed to generate presigned URL", { status: 500 });
+  }
+
   const response = await fetch(signed.url, {
     headers: signed.headers,
   });
@@ -64,7 +53,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
     status: 200,
     headers: {
       "Content-Type": "application/octet-stream",
-      "Content-Disposition": `attachment; filename="${basename(url.pathname)}"`,
+      "Content-Disposition": `attachment; filename="${basename(filename)}"`,
     },
   });
 }

apps/webapp/app/routes/resources.runs.$runParam.logs.download.ts

@@ -0,0 +1,112 @@
+import { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { prisma } from "~/db.server";
+import { requireUserId } from "~/services/session.server";
+import { v3RunParamsSchema } from "~/utils/pathBuilder";
+import {
+  PreparedEvent,
+  RunPreparedEvent,
+  eventRepository,
+  getDateFromNanoseconds,
+} from "~/v3/eventRepository.server";
+import { createGzip } from "zlib";
+import { Readable } from "stream";
+import { formatDurationMilliseconds } from "@trigger.dev/core/v3/utils/durations";
+
+export async function loader({ params, request }: LoaderFunctionArgs) {
+  const userId = await requireUserId(request);
+  const parsedParams = v3RunParamsSchema.pick({ runParam: true }).parse(params);
+
+  const run = await prisma.taskRun.findFirst({
+    where: {
+      friendlyId: parsedParams.runParam,
+      project: {
+        organization: {
+          members: {
+            some: {
+              userId,
+            },
+          },
+        },
+      },
+    },
+  });
+
+  if (!run) {
+    return new Response("Not found", { status: 404 });
+  }
+
+  const runEvents = await eventRepository.getRunEvents(run.friendlyId);
+
+  // Create a Readable stream from the runEvents array
+  const readable = new Readable({
+    read() {
+      runEvents.forEach((event) => {
+        try {
+          this.push(formatRunEvent(event) + "\n");
+        } catch {}
+      });
+      this.push(null); // End of stream
+    },
+  });
+
+  // Create a gzip transform stream
+  const gzip = createGzip();
+
+  // Pipe the readable stream into the gzip stream
+  const compressedStream = readable.pipe(gzip);
+
+  // Return the response with the compressed stream
+  return new Response(compressedStream as any, {
+    status: 200,
+    headers: {
+      "Content-Type": "application/octet-stream",
+      "Content-Disposition": `attachment; filename="${parsedParams.runParam}.log"`,
+      "Content-Encoding": "gzip",
+    },
+  });
+}
+
+function formatRunEvent(event: RunPreparedEvent): string {
+  const entries = [];
+  const parts: string[] = [];
+
+  parts.push(getDateFromNanoseconds(event.startTime).toISOString());
+
+  if (event.taskSlug) {
+    parts.push(event.taskSlug);
+  }
+
+  parts.push(event.level);
+  parts.push(event.message);
+
+  if (event.level === "TRACE") {
+    parts.push(`(${formatDurationMilliseconds(event.duration / 1_000_000)})`);
+  }
+
+  entries.push(parts.join(" "));
+
+  if (event.events) {
+    for (const subEvent of event.events) {
+      if (subEvent.name === "exception") {
+        const subEventParts: string[] = [];
+
+        subEventParts.push(subEvent.time as unknown as string);
+
+        if (event.taskSlug) {
+          subEventParts.push(event.taskSlug);
+        }
+
+        subEventParts.push(subEvent.name);
+        subEventParts.push((subEvent.properties as any).exception.message);
+
+        if ((subEvent.properties as any).exception.stack) {
+          subEventParts.push((subEvent.properties as any).exception.stack);
+        }
+
+        entries.push(subEventParts.join(" "));
+      }
+    }
+  }
+
+  return entries.join("\n");
+}

apps/webapp/app/services/endpointApi.server.ts

@@ -136,7 +136,7 @@ export class EndpointApi {
     };
   }
 
-  async executeJobRequest(options: RunJobBody) {
+  async executeJobRequest(options: RunJobBody, timeoutInMs?: number) {
     const startTimeInMs = performance.now();
 
     const response = await safeFetch(this.url, {
@@ -147,8 +147,18 @@ export class EndpointApi {
         "x-trigger-action": "EXECUTE_JOB",
       },
       body: JSON.stringify(options),
+      signal: timeoutInMs ? AbortSignal.timeout(timeoutInMs) : undefined,
     });
 
+    if (response) {
+      logger.debug("executeJobRequest() response from endpoint", {
+        status: response.status,
+        headers: Object.fromEntries(response.headers.entries()),
+      });
+    } else {
+      logger.debug("executeJobRequest() no response from endpoint");
+    }
+
     return {
       response,
       parser: RunJobResponseSchema,
@@ -434,7 +444,10 @@ async function safeFetch(url: string, options: RequestInit) {
   } catch (error) {
     logger.debug("Error while trying to connect to endpoint", {
       url,
-      error,
+      error:
+        error instanceof Error
+          ? { name: error.name, message: error.message, stack: error.stack }
+          : String(error),
     });
   }
 }

apps/webapp/app/services/runs/performRunExecutionV3.server.ts

@@ -269,7 +269,10 @@ export class PerformRunExecutionV3Service {
 
       // TODO: add the ability to abort the execution from any server using Redis pub/sub
       const { response, parser, errorParser, headersParser, durationInMs } =
-        await client.executeJobRequest(executionBody);
+        await client.executeJobRequest(
+          executionBody,
+          run.environment.type === "DEVELOPMENT" ? 60_000 * 5 : undefined
+        );
 
       await createExecutionEvent({
         eventType: "finish",
@@ -929,6 +932,25 @@ export class PerformRunExecutionV3Service {
     executionCount: number = 1
   ) {
     await $transaction(this.#prismaClient, async (tx) => {
+      const service = new CompleteRunTaskService(tx);
+
+      const task = await service.call(run.environment, run.id, data.id, {
+        properties: data.properties,
+        output: data.output ? (JSON.parse(data.output) as any) : undefined,
+      });
+
+      if (!task || task.status === "ERRORED") {
+        return await this.#failRunExecution(
+          tx,
+          run,
+          {
+            message: task ? `Task '${task.name}' failed to complete` : "Task failed to complete",
+          },
+          "FAILURE",
+          durationInMs
+        );
+      }
+
       await tx.jobRun.update({
         where: {
           id: run.id,
@@ -958,13 +980,6 @@ export class PerformRunExecutionV3Service {
         },
       });
 
-      const service = new CompleteRunTaskService(tx);
-
-      await service.call(run.environment, run.id, data.id, {
-        properties: data.properties,
-        output: data.output ? (JSON.parse(data.output) as any) : undefined,
-      });
-
       await ResumeRunService.enqueue(run, tx);
     });
   }

apps/webapp/app/services/tasks/runTask.server.ts

@@ -6,6 +6,7 @@ import { taskWithAttemptsToServerTask } from "~/models/task.server";
 import { generateSecret } from "~/services/sources/utils.server";
 import { ulid } from "~/services/ulid.server";
 import { taskOperationWorker, workerQueue } from "~/services/worker.server";
+import { startActiveSpan } from "~/v3/tracer.server";
 
 export class RunTaskService {
   #prismaClient: PrismaClient;
@@ -19,142 +20,154 @@ export class RunTaskService {
     idempotencyKey: string,
     taskBody: RunTaskBodyOutput
   ): Promise<ServerTask | undefined> {
-    const delayUntilInFuture = taskBody.delayUntil
-      ? taskBody.delayUntil.getTime() > Date.now()
-      : false;
-    const callbackEnabled = taskBody.callback?.enabled ?? false;
+    return startActiveSpan("RunTaskService.call", async (span) => {
+      span.setAttribute("runId", runId);
 
-    // First
-    const existingTask = await this.#handleExistingTask(
-      runId,
-      idempotencyKey,
-      taskBody,
-      delayUntilInFuture,
-      callbackEnabled
-    );
+      const delayUntilInFuture = taskBody.delayUntil
+        ? taskBody.delayUntil.getTime() > Date.now()
+        : false;
+      const callbackEnabled = taskBody.callback?.enabled ?? false;
 
-    if (existingTask) {
-      return taskWithAttemptsToServerTask(existingTask);
-    }
+      // First
+      const existingTask = await this.#handleExistingTask(
+        runId,
+        idempotencyKey,
+        taskBody,
+        delayUntilInFuture,
+        callbackEnabled
+      );
 
-    const run = await this.#prismaClient.jobRun.findUnique({
-      where: {
-        id: runId,
-      },
-      select: {
-        status: true,
-        forceYieldImmediately: true,
-      },
-    });
+      if (existingTask) {
+        span.setAttribute("taskId", existingTask.id);
 
-    if (!run) throw new Error("Run not found");
-
-    const runConnection = taskBody.connectionKey
-      ? await this.#prismaClient.runConnection.findUnique({
-          where: {
-            runId_key: {
-              runId,
-              key: taskBody.connectionKey,
-            },
-          },
-          select: {
-            id: true,
-          },
-        })
-      : undefined;
-
-    const results = await $transaction(this.#prismaClient, async (tx) => {
-      // If task.delayUntil is set and is in the future, we'll set the task's status to "WAITING", else set it to RUNNING
-      let status: TaskStatus;
-
-      if (run.status === "CANCELED") {
-        status = "CANCELED";
-      } else {
-        status =
-          delayUntilInFuture || callbackEnabled
-            ? "WAITING"
-            : taskBody.noop
-            ? "COMPLETED"
-            : "RUNNING";
+        return taskWithAttemptsToServerTask(existingTask);
       }
 
-      const taskId = ulid();
-      const callbackUrl = callbackEnabled
-        ? `${env.APP_ORIGIN}/api/v1/tasks/${taskId}/callback/${generateSecret(12)}`
+      const run = await this.#prismaClient.jobRun.findUnique({
+        where: {
+          id: runId,
+        },
+        select: {
+          status: true,
+          forceYieldImmediately: true,
+        },
+      });
+
+      if (!run) throw new Error("Run not found");
+
+      const runConnection = taskBody.connectionKey
+        ? await this.#prismaClient.runConnection.findUnique({
+            where: {
+              runId_key: {
+                runId,
+                key: taskBody.connectionKey,
+              },
+            },
+            select: {
+              id: true,
+            },
+          })
         : undefined;
 
-      const task = await tx.task.create({
-        data: {
-          id: taskId,
-          idempotencyKey,
-          displayKey: taskBody.displayKey,
-          runConnectionId: runConnection ? runConnection.id : undefined,
-          icon: taskBody.icon,
-          runId,
-          parentId: taskBody.parentId,
-          name: taskBody.name ?? "Task",
-          description: taskBody.description,
-          status,
-          startedAt: new Date(),
-          completedAt: status === "COMPLETED" || status === "CANCELED" ? new Date() : undefined,
-          noop: taskBody.noop,
-          delayUntil: taskBody.delayUntil,
-          params: taskBody.params ?? undefined,
-          properties: this.#filterProperties(taskBody.properties) ?? undefined,
-          redact: taskBody.redact ?? undefined,
-          operation: taskBody.operation,
-          callbackUrl,
-          style: taskBody.style ?? { style: "normal" },
-          childExecutionMode: taskBody.parallel ? "PARALLEL" : "SEQUENTIAL",
-        },
-      });
+      const results = await $transaction(
+        this.#prismaClient,
+        async (tx) => {
+          // If task.delayUntil is set and is in the future, we'll set the task's status to "WAITING", else set it to RUNNING
+          let status: TaskStatus;
 
-      const taskAttempt = await tx.taskAttempt.create({
-        data: {
-          number: 1,
-          taskId: task.id,
-          status: "PENDING",
-        },
-      });
+          if (run.status === "CANCELED") {
+            status = "CANCELED";
+          } else {
+            status =
+              delayUntilInFuture || callbackEnabled
+                ? "WAITING"
+                : taskBody.noop
+                ? "COMPLETED"
+                : "RUNNING";
+          }
 
-      if (task.status === "RUNNING" && typeof taskBody.operation === "string") {
-        // We need to schedule the operation
-        await taskOperationWorker.enqueue(
-          "performTaskOperation",
-          {
-            id: task.id,
-          },
-          { tx, runAt: task.delayUntil ?? undefined, jobKey: `operation:${task.id}` }
-        );
-      } else if (task.status === "WAITING" && callbackUrl && taskBody.callback) {
-        if (taskBody.callback.timeoutInSeconds > 0) {
-          // We need to schedule the callback timeout
-          await workerQueue.enqueue(
-            "processCallbackTimeout",
-            {
-              id: task.id,
+          const taskId = ulid();
+          const callbackUrl = callbackEnabled
+            ? `${env.APP_ORIGIN}/api/v1/tasks/${taskId}/callback/${generateSecret(12)}`
+            : undefined;
+
+          const task = await tx.task.create({
+            data: {
+              id: taskId,
+              idempotencyKey,
+              displayKey: taskBody.displayKey,
+              runConnectionId: runConnection ? runConnection.id : undefined,
+              icon: taskBody.icon,
+              runId,
+              parentId: taskBody.parentId,
+              name: taskBody.name ?? "Task",
+              description: taskBody.description,
+              status,
+              startedAt: new Date(),
+              completedAt: status === "COMPLETED" || status === "CANCELED" ? new Date() : undefined,
+              noop: taskBody.noop,
+              delayUntil: taskBody.delayUntil,
+              params: taskBody.params ?? undefined,
+              properties: this.#filterProperties(taskBody.properties) ?? undefined,
+              redact: taskBody.redact ?? undefined,
+              operation: taskBody.operation,
+              callbackUrl,
+              style: taskBody.style ?? { style: "normal" },
+              childExecutionMode: taskBody.parallel ? "PARALLEL" : "SEQUENTIAL",
             },
-            {
-              tx,
-              runAt: new Date(Date.now() + taskBody.callback.timeoutInSeconds * 1000),
-              jobKey: `process-callback:${task.id}`,
+          });
+
+          span.setAttribute("taskId", task.id);
+
+          const taskAttempt = await tx.taskAttempt.create({
+            data: {
+              number: 1,
+              taskId: task.id,
+              status: "PENDING",
+            },
+          });
+
+          if (task.status === "RUNNING" && typeof taskBody.operation === "string") {
+            // We need to schedule the operation
+            await taskOperationWorker.enqueue(
+              "performTaskOperation",
+              {
+                id: task.id,
+              },
+              { tx, runAt: task.delayUntil ?? undefined, jobKey: `operation:${task.id}` }
+            );
+          } else if (task.status === "WAITING" && callbackUrl && taskBody.callback) {
+            if (taskBody.callback.timeoutInSeconds > 0) {
+              // We need to schedule the callback timeout
+              await workerQueue.enqueue(
+                "processCallbackTimeout",
+                {
+                  id: task.id,
+                },
+                {
+                  tx,
+                  runAt: new Date(Date.now() + taskBody.callback.timeoutInSeconds * 1000),
+                  jobKey: `process-callback:${task.id}`,
+                }
+              );
             }
-          );
-        }
+          }
+
+          return { task, taskAttempt };
+        },
+        { timeout: 10000 }
+      );
+
+      if (!results) {
+        return;
       }
 
-      return { task, taskAttempt };
+      const { task, taskAttempt } = results;
+
+      return task
+        ? taskWithAttemptsToServerTask({ ...task, attempts: [taskAttempt], run })
+        : undefined;
     });
-
-    if (!results) {
-      return;
-    }
-
-    const { task, taskAttempt } = results;
-
-    return task
-      ? taskWithAttemptsToServerTask({ ...task, attempts: [taskAttempt], run })
-      : undefined;
   }
 
   async #handleExistingTask(

apps/webapp/app/services/worker.server.ts

@@ -47,6 +47,8 @@ import { ResumeTaskService } from "./tasks/resumeTask.server";
 import { RequeueV2Message } from "~/v3/marqs/requeueV2Message.server";
 import { MarqsConcurrencyMonitor } from "~/v3/marqs/concurrencyMonitor.server";
 import { reportUsageEvent } from "~/v3/openMeter.server";
+import { EnqueueDelayedRunService } from "~/v3/services/enqueueDelayedRun.server";
+import { ExpireEnqueuedRunService } from "~/v3/services/expireEnqueuedRun.server";
 
 const workerCatalog = {
   indexEndpoint: z.object({
@@ -177,6 +179,12 @@ const workerCatalog = {
     }),
     additionalData: z.record(z.any()).optional(),
   }),
+  "v3.enqueueDelayedRun": z.object({
+    runId: z.string(),
+  }),
+  "v3.expireRun": z.object({
+    runId: z.string(),
+  }),
 };
 
 const executionWorkerCatalog = {
@@ -672,6 +680,24 @@ function getWorkerQueue() {
           });
         },
       },
+      "v3.enqueueDelayedRun": {
+        priority: 0,
+        maxAttempts: 8,
+        handler: async (payload, job) => {
+          const service = new EnqueueDelayedRunService();
+
+          return await service.call(payload.runId);
+        },
+      },
+      "v3.expireRun": {
+        priority: 0,
+        maxAttempts: 8,
+        handler: async (payload, job) => {
+          const service = new ExpireEnqueuedRunService();
+
+          return await service.call(payload.runId);
+        },
+      },
     },
   });
 }

apps/webapp/app/utils/parseRequestJson.server.ts

@@ -0,0 +1,29 @@
+import { Attributes } from "@opentelemetry/api";
+import { startActiveSpan } from "~/v3/tracer.server";
+
+export async function parseRequestJsonAsync(
+  request: Request,
+  attributes?: Attributes
+): Promise<unknown> {
+  return await startActiveSpan(
+    "parseRequestJsonAsync()",
+    async (span) => {
+      span.setAttribute("content-length", parseInt(request.headers.get("content-length") ?? "0"));
+      span.setAttribute("content-type", request.headers.get("content-type") ?? "application/json");
+      span.setAttribute("experiment.async", false);
+
+      const rawText = await startActiveSpan("request.text()", async () => {
+        return await request.text();
+      });
+
+      if (rawText.length === 0) {
+        return;
+      }
+
+      return JSON.parse(rawText);
+    },
+    {
+      attributes,
+    }
+  );
+}

apps/webapp/app/utils/pathBuilder.ts

@@ -375,6 +375,10 @@ export function v3RunPath(organization: OrgForPath, project: ProjectForPath, run
   return `${v3RunsPath(organization, project)}/${run.friendlyId}`;
 }
 
+export function v3RunDownloadLogsPath(run: v3RunForPath) {
+  return `/resources/runs/${run.friendlyId}/logs/download`;
+}
+
 export function v3RunSpanPath(
   organization: OrgForPath,
   project: ProjectForPath,

apps/webapp/app/v3/eventRepository.server.ts

@@ -4,6 +4,7 @@ import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions"
 import {
   ExceptionEventProperties,
   ExceptionSpanEvent,
+  NULL_SENTINEL,
   PRIMARY_VARIANT,
   SemanticInternalAttributes,
   SpanEvent,
@@ -14,7 +15,6 @@ import {
   correctErrorStackTrace,
   createPacketAttributesAsJson,
   flattenAttributes,
-  NULL_SENTINEL,
   isExceptionSpanEvent,
   omit,
   unflattenAttributes,
@@ -23,14 +23,15 @@ import { Prisma, TaskEvent, TaskEventStatus, type TaskEventKind } from "@trigger
 import Redis, { RedisOptions } from "ioredis";
 import { createHash } from "node:crypto";
 import { EventEmitter } from "node:stream";
+import { Gauge } from "prom-client";
 import { $replica, PrismaClient, PrismaReplicaClient, prisma } from "~/db.server";
 import { env } from "~/env.server";
+import { metricsRegister } from "~/metrics.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
-import { DynamicFlushScheduler } from "./dynamicFlushScheduler.server";
 import { singleton } from "~/utils/singleton";
-import { Gauge } from "prom-client";
-import { metricsRegister } from "~/metrics.server";
+import { DynamicFlushScheduler } from "./dynamicFlushScheduler.server";
+import { startActiveSpan } from "./tracer.server";
 
 export type CreatableEvent = Omit<
   Prisma.TaskEventCreateInput,
@@ -128,6 +129,10 @@ export type PreparedEvent = Omit<QueriedEvent, "events" | "style" | "duration">
   style: TaskEventStyle;
 };
 
+export type RunPreparedEvent = PreparedEvent & {
+  taskSlug?: string;
+};
+
 export type SpanLink =
   | {
       type: "run";
@@ -339,7 +344,7 @@ export class EventRepository {
     });
   }
 
-  async queryIncompleteEvents(queryOptions: QueryOptions) {
+  async queryIncompleteEvents(queryOptions: QueryOptions, allowCompleteDuplicate = false) {
     // First we will find all the events that match the query options (selecting minimal data).
     const taskEvents = await this.readReplica.taskEvent.findMany({
       where: queryOptions,
@@ -357,6 +362,10 @@ export class EventRepository {
       // If the event is cancelled, it is not incomplete
       if (event.isCancelled) return false;
 
+      if (allowCompleteDuplicate) {
+        return true;
+      }
+
       // There must not be another complete event with the same spanId
       const hasCompleteDuplicate = taskEvents.some(
         (otherEvent) =>
@@ -374,43 +383,261 @@ export class EventRepository {
   }
 
   public async getTraceSummary(traceId: string): Promise<TraceSummary | undefined> {
-    const events = await this.readReplica.taskEvent.findMany({
-      select: {
-        id: true,
-        spanId: true,
-        parentId: true,
-        runId: true,
-        idempotencyKey: true,
-        message: true,
-        style: true,
-        startTime: true,
-        duration: true,
-        isError: true,
-        isPartial: true,
-        isCancelled: true,
-        level: true,
-        events: true,
-        environmentType: true,
-      },
-      where: {
-        traceId,
-      },
-      orderBy: {
-        startTime: "asc",
-      },
+    return await startActiveSpan("getTraceSummary", async (span) => {
+      const events = await this.readReplica.taskEvent.findMany({
+        select: {
+          id: true,
+          spanId: true,
+          parentId: true,
+          runId: true,
+          idempotencyKey: true,
+          message: true,
+          style: true,
+          startTime: true,
+          duration: true,
+          isError: true,
+          isPartial: true,
+          isCancelled: true,
+          level: true,
+          events: true,
+          environmentType: true,
+        },
+        where: {
+          traceId,
+        },
+        orderBy: {
+          startTime: "asc",
+        },
+        take: env.MAXIMUM_TRACE_SUMMARY_VIEW_COUNT,
+      });
+
+      let preparedEvents: Array<PreparedEvent> = [];
+      let rootSpanId: string | undefined;
+      const eventsBySpanId = new Map<string, PreparedEvent>();
+
+      for (const event of events) {
+        preparedEvents.push(prepareEvent(event));
+
+        if (!rootSpanId && !event.parentId) {
+          rootSpanId = event.spanId;
+        }
+      }
+
+      for (const event of preparedEvents) {
+        const existingEvent = eventsBySpanId.get(event.spanId);
+
+        if (!existingEvent) {
+          eventsBySpanId.set(event.spanId, event);
+          continue;
+        }
+
+        if (event.isCancelled || !event.isPartial) {
+          eventsBySpanId.set(event.spanId, event);
+        }
+      }
+
+      preparedEvents = Array.from(eventsBySpanId.values());
+
+      const spansBySpanId = new Map<string, SpanSummary>();
+
+      const spans = preparedEvents.map((event) => {
+        const ancestorCancelled = isAncestorCancelled(eventsBySpanId, event.spanId);
+        const duration = calculateDurationIfAncestorIsCancelled(
+          eventsBySpanId,
+          event.spanId,
+          event.duration
+        );
+
+        const span = {
+          recordId: event.id,
+          id: event.spanId,
+          parentId: event.parentId ?? undefined,
+          runId: event.runId,
+          idempotencyKey: event.idempotencyKey,
+          data: {
+            message: event.message,
+            style: event.style,
+            duration,
+            isError: event.isError,
+            isPartial: ancestorCancelled ? false : event.isPartial,
+            isCancelled: event.isCancelled === true ? true : event.isPartial && ancestorCancelled,
+            startTime: getDateFromNanoseconds(event.startTime),
+            level: event.level,
+            events: event.events,
+            environmentType: event.environmentType,
+          },
+        };
+
+        spansBySpanId.set(event.spanId, span);
+
+        return span;
+      });
+
+      if (!rootSpanId) {
+        return;
+      }
+
+      const rootSpan = spansBySpanId.get(rootSpanId);
+
+      if (!rootSpan) {
+        return;
+      }
+
+      return {
+        rootSpan,
+        spans,
+      };
     });
+  }
 
-    const preparedEvents = removeDuplicateEvents(events.map(prepareEvent));
+  public async getRunEvents(runId: string): Promise<RunPreparedEvent[]> {
+    return await startActiveSpan("getRunEvents", async (span) => {
+      const events = await this.readReplica.taskEvent.findMany({
+        select: {
+          id: true,
+          spanId: true,
+          parentId: true,
+          runId: true,
+          idempotencyKey: true,
+          message: true,
+          style: true,
+          startTime: true,
+          duration: true,
+          isError: true,
+          isPartial: true,
+          isCancelled: true,
+          level: true,
+          events: true,
+          environmentType: true,
+          taskSlug: true,
+        },
+        where: {
+          runId,
+          isPartial: false,
+        },
+        orderBy: {
+          startTime: "asc",
+        },
+      });
 
-    const spans = preparedEvents.map((event) => {
-      const ancestorCancelled = isAncestorCancelled(preparedEvents, event.spanId);
-      const duration = calculateDurationIfAncestorIsCancelled(
-        preparedEvents,
-        event.spanId,
-        event.duration
+      let preparedEvents: Array<PreparedEvent> = [];
+
+      for (const event of events) {
+        preparedEvents.push(prepareEvent(event));
+      }
+
+      return preparedEvents;
+    });
+  }
+
+  // A Span can be cancelled if it is partial and has a parent that is cancelled
+  // And a span's duration, if it is partial and has a cancelled parent, is the time between the start of the span and the time of the cancellation event of the parent
+  public async getSpan(spanId: string, traceId: string) {
+    return await startActiveSpan("getSpan", async (s) => {
+      const spanEvent = await this.#getSpanEvent(spanId);
+
+      if (!spanEvent) {
+        return;
+      }
+
+      const preparedEvent = prepareEvent(spanEvent);
+
+      const span = await this.#createSpanFromEvent(preparedEvent);
+
+      const output = rehydrateJson(spanEvent.output);
+      const payload = rehydrateJson(spanEvent.payload);
+
+      const show = rehydrateShow(spanEvent.properties);
+
+      const properties = sanitizedAttributes(spanEvent.properties);
+
+      const messagingEvent = SpanMessagingEvent.optional().safeParse(
+        (properties as any)?.messaging
+      );
+
+      const links: SpanLink[] = [];
+
+      if (messagingEvent.success && messagingEvent.data) {
+        if (messagingEvent.data.message && "id" in messagingEvent.data.message) {
+          if (messagingEvent.data.message.id.startsWith("run_")) {
+            links.push({
+              type: "run",
+              icon: "runs",
+              title: `Run ${messagingEvent.data.message.id}`,
+              runId: messagingEvent.data.message.id,
+            });
+          }
+        }
+      }
+
+      const backLinks = spanEvent.links as any as Link[] | undefined;
+
+      if (backLinks && backLinks.length > 0) {
+        backLinks.forEach((l) => {
+          const title = String(
+            l.attributes?.[SemanticInternalAttributes.LINK_TITLE] ?? "Triggered by"
+          );
+
+          links.push({
+            type: "span",
+            icon: "trigger",
+            title,
+            traceId: l.context.traceId,
+            spanId: l.context.spanId,
+          });
+        });
+      }
+
+      const spanEvents = transformEvents(
+        preparedEvent.events,
+        spanEvent.metadata as Attributes,
+        spanEvent.environmentType === "DEVELOPMENT"
       );
 
       return {
+        ...spanEvent,
+        ...span.data,
+        payload,
+        output,
+        properties,
+        events: spanEvents,
+        show,
+        links,
+      };
+    });
+  }
+
+  async #createSpanFromEvent(event: PreparedEvent) {
+    return await startActiveSpan("createSpanFromEvent", async (s) => {
+      let ancestorCancelled = false;
+      let duration = event.duration;
+
+      if (!event.isCancelled && event.isPartial) {
+        await this.#walkSpanAncestors(event, (ancestorEvent, level) => {
+          if (level >= 8) {
+            return { stop: true };
+          }
+
+          if (ancestorEvent.isCancelled) {
+            ancestorCancelled = true;
+
+            // We need to get the cancellation time from the cancellation span event
+            const cancellationEvent = ancestorEvent.events.find(
+              (event) => event.name === "cancellation"
+            );
+
+            if (cancellationEvent) {
+              duration = calculateDurationFromStart(event.startTime, cancellationEvent.time);
+            }
+
+            return { stop: true };
+          }
+
+          return { stop: false };
+        });
+      }
+
+      const span = {
         recordId: event.id,
         id: event.spanId,
         parentId: event.parentId ?? undefined,
@@ -429,104 +656,93 @@ export class EventRepository {
           environmentType: event.environmentType,
         },
       };
+
+      return span;
     });
-
-    const rootSpanId = events.find((event) => !event.parentId);
-    if (!rootSpanId) {
-      return;
-    }
-
-    const rootSpan = spans.find((span) => span.id === rootSpanId.spanId);
-
-    if (!rootSpan) {
-      return;
-    }
-
-    return {
-      rootSpan,
-      spans,
-    };
   }
 
-  // A Span can be cancelled if it is partial and has a parent that is cancelled
-  // And a span's duration, if it is partial and has a cancelled parent, is the time between the start of the span and the time of the cancellation event of the parent
-  public async getSpan(spanId: string, traceId: string) {
-    const traceSummary = await this.getTraceSummary(traceId);
-
-    const span = traceSummary?.spans.find((span) => span.id === spanId);
-
-    if (!span) {
+  async #walkSpanAncestors(
+    event: PreparedEvent,
+    callback: (event: PreparedEvent, level: number) => { stop: boolean }
+  ) {
+    const parentId = event.parentId;
+    if (!parentId) {
       return;
     }
 
-    const fullEvent = await this.readReplica.taskEvent.findUnique({
-      where: {
-        id: span.recordId,
-      },
-    });
+    await startActiveSpan("walkSpanAncestors", async (s) => {
+      let parentEvent = await this.#getSpanEvent(parentId);
+      let level = 1;
 
-    if (!fullEvent) {
-      return;
-    }
+      while (parentEvent) {
+        const preparedParentEvent = prepareEvent(parentEvent);
 
-    const output = rehydrateJson(fullEvent.output);
-    const payload = rehydrateJson(fullEvent.payload);
+        const result = callback(preparedParentEvent, level);
 
-    const show = rehydrateShow(fullEvent.properties);
-
-    const properties = sanitizedAttributes(fullEvent.properties);
-
-    const messagingEvent = SpanMessagingEvent.optional().safeParse((properties as any)?.messaging);
-
-    const links: SpanLink[] = [];
-
-    if (messagingEvent.success && messagingEvent.data) {
-      if (messagingEvent.data.message && "id" in messagingEvent.data.message) {
-        if (messagingEvent.data.message.id.startsWith("run_")) {
-          links.push({
-            type: "run",
-            icon: "runs",
-            title: `Run ${messagingEvent.data.message.id}`,
-            runId: messagingEvent.data.message.id,
-          });
+        if (result.stop) {
+          return;
         }
+
+        if (!preparedParentEvent.parentId) {
+          return;
+        }
+
+        parentEvent = await this.#getSpanEvent(preparedParentEvent.parentId);
+
+        level++;
       }
+    });
+  }
+
+  async #getSpanAncestors(event: PreparedEvent, levels = 1): Promise<Array<PreparedEvent>> {
+    if (levels >= 8) {
+      return [];
     }
 
-    const backLinks = fullEvent.links as any as Link[] | undefined;
+    if (!event.parentId) {
+      return [];
+    }
 
-    if (backLinks && backLinks.length > 0) {
-      backLinks.forEach((l) => {
-        const title = String(
-          l.attributes?.[SemanticInternalAttributes.LINK_TITLE] ?? "Triggered by"
-        );
+    const parentEvent = await this.#getSpanEvent(event.parentId);
 
-        links.push({
-          type: "span",
-          icon: "trigger",
-          title,
-          traceId: l.context.traceId,
-          spanId: l.context.spanId,
-        });
+    if (!parentEvent) {
+      return [];
+    }
+
+    const preparedParentEvent = prepareEvent(parentEvent);
+
+    if (!preparedParentEvent.parentId) {
+      return [preparedParentEvent];
+    }
+
+    const moreAncestors = await this.#getSpanAncestors(preparedParentEvent, levels + 1);
+
+    return [preparedParentEvent, ...moreAncestors];
+  }
+
+  async #getSpanEvent(spanId: string) {
+    return await startActiveSpan("getSpanEvent", async (s) => {
+      const events = await this.readReplica.taskEvent.findMany({
+        where: {
+          spanId,
+        },
+        orderBy: {
+          startTime: "asc",
+        },
       });
-    }
 
-    const events = transformEvents(
-      span.data.events,
-      fullEvent.metadata as Attributes,
-      traceSummary?.rootSpan.data.environmentType === "DEVELOPMENT"
-    );
+      let finalEvent: TaskEvent | undefined;
 
-    return {
-      ...fullEvent,
-      ...span.data,
-      payload,
-      output,
-      properties,
-      events,
-      show,
-      links,
-    };
+      for (const event of events) {
+        if (event.isPartial && finalEvent) {
+          continue;
+        }
+
+        finalEvent = event;
+      }
+
+      return finalEvent;
+    });
   }
 
   public async recordEvent(message: string, options: TraceEventOptions) {
@@ -906,6 +1122,7 @@ export function createExceptionPropertiesFromError(error: TaskRunError): Excepti
       return {
         type: "Internal error",
         message: [error.code, error.message].filter(Boolean).join(": "),
+        stacktrace: error.stackTrace,
       };
     }
     case "STRING_ERROR": {
@@ -973,34 +1190,38 @@ function prepareEvent(event: QueriedEvent): PreparedEvent {
 }
 
 function parseEventsField(events: Prisma.JsonValue): SpanEvents {
-  const eventsUnflattened = events
+  const unsafe = events
     ? (events as any[]).map((e) => ({
         ...e,
         properties: unflattenAttributes(e.properties as Attributes),
       }))
     : undefined;
 
-  const spanEvents = SpanEvents.safeParse(eventsUnflattened);
-
-  if (spanEvents.success) {
-    return spanEvents.data;
-  }
-
-  return [];
+  return unsafe as SpanEvents;
 }
 
 function parseStyleField(style: Prisma.JsonValue): TaskEventStyle {
-  const parsedStyle = TaskEventStyle.safeParse(unflattenAttributes(style as Attributes));
+  const unsafe = unflattenAttributes(style as Attributes);
 
-  if (parsedStyle.success) {
-    return parsedStyle.data;
+  if (!unsafe) {
+    return {};
+  }
+
+  if (typeof unsafe === "object") {
+    return Object.assign(
+      {
+        icon: undefined,
+        variant: undefined,
+      },
+      unsafe
+    ) as TaskEventStyle;
   }
 
   return {};
 }
 
-function isAncestorCancelled(events: PreparedEvent[], spanId: string) {
-  const event = events.find((event) => event.spanId === spanId);
+function isAncestorCancelled(events: Map<string, PreparedEvent>, spanId: string) {
+  const event = events.get(spanId);
 
   if (!event) {
     return false;
@@ -1018,11 +1239,11 @@ function isAncestorCancelled(events: PreparedEvent[], spanId: string) {
 }
 
 function calculateDurationIfAncestorIsCancelled(
-  events: PreparedEvent[],
+  events: Map<string, PreparedEvent>,
   spanId: string,
   defaultDuration: number
 ) {
-  const event = events.find((event) => event.spanId === spanId);
+  const event = events.get(spanId);
 
   if (!event) {
     return defaultDuration;
@@ -1054,8 +1275,9 @@ function calculateDurationIfAncestorIsCancelled(
   return defaultDuration;
 }
 
-function findFirstCancelledAncestor(events: PreparedEvent[], spanId: string) {
-  const event = events.find((event) => event.spanId === spanId);
+function findFirstCancelledAncestor(events: Map<string, PreparedEvent>, spanId: string) {
+  const event = events.get(spanId);
+
   if (!event) {
     return;
   }
@@ -1196,7 +1418,7 @@ function getNowInNanoseconds(): bigint {
   return BigInt(new Date().getTime() * 1_000_000);
 }
 
-function getDateFromNanoseconds(nanoseconds: bigint) {
+export function getDateFromNanoseconds(nanoseconds: bigint) {
   return new Date(Number(nanoseconds) / 1_000_000);
 }
 

apps/webapp/app/v3/handleSocketIo.server.ts

@@ -138,8 +138,19 @@ function createCoordinatorNamespace(io: Server) {
         await sharedQueueTasks.taskRunHeartbeat(message.runId);
       },
       CHECKPOINT_CREATED: async (message) => {
-        const createCheckpoint = new CreateCheckpointService();
-        await createCheckpoint.call(message);
+        try {
+          const createCheckpoint = new CreateCheckpointService();
+          const result = await createCheckpoint.call(message);
+
+          return { keepRunAlive: result?.keepRunAlive ?? false };
+        } catch (error) {
+          logger.error("Error while creating checkpoint", {
+            rawMessage: message,
+            error: error instanceof Error ? error.message : error,
+          });
+
+          return { keepRunAlive: false };
+        }
       },
       CREATE_WORKER: async (message) => {
         try {
@@ -256,7 +267,7 @@ function createProviderNamespace(io: Server) {
         try {
           const service = new DeploymentIndexFailed();
 
-          await service.call(message.deploymentId, message.error);
+          await service.call(message.deploymentId, message.error, message.overrideCompletion);
         } catch (e) {
           logger.error("Error while indexing", { error: e });
         }

apps/webapp/app/v3/marqs/index.server.ts

@@ -1625,7 +1625,7 @@ function getMarQSClient() {
         defaultEnvConcurrency: env.DEFAULT_ENV_EXECUTION_CONCURRENCY_LIMIT,
         defaultOrgConcurrency: env.DEFAULT_ORG_EXECUTION_CONCURRENCY_LIMIT,
         visibilityTimeoutInMs: 120 * 1000, // 2 minutes,
-        enableRebalancing: !env.MARQS_DISABLE_REBALANCING,
+        enableRebalancing: false,
       });
     } else {
       console.warn(

apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts

@@ -17,7 +17,6 @@ import {
   BackgroundWorkerTask,
   RuntimeEnvironment,
   TaskRun,
-  TaskRunAttemptStatus,
   TaskRunStatus,
 } from "@trigger.dev/database";
 import { z } from "zod";
@@ -43,6 +42,7 @@ import { generateJWTTokenForEnvironment } from "~/services/apiAuth.server";
 import { EnvironmentVariable } from "../environmentVariables/repository";
 import { machinePresetFromConfig } from "../machinePresets.server";
 import { env } from "~/env.server";
+import { isFinalAttemptStatus, isFinalRunStatus } from "../taskStatus";
 
 const WithTraceContext = z.object({
   traceparent: z.string().optional(),
@@ -962,19 +962,7 @@ class SharedQueueTasks {
     }
 
     if (setToExecuting) {
-      const FINAL_RUN_STATUSES: TaskRunStatus[] = [
-        "CANCELED",
-        "COMPLETED_SUCCESSFULLY",
-        "COMPLETED_WITH_ERRORS",
-        "INTERRUPTED",
-        "SYSTEM_FAILURE",
-      ];
-      const FINAL_ATTEMPT_STATUSES: TaskRunAttemptStatus[] = ["CANCELED", "COMPLETED", "FAILED"];
-
-      if (
-        FINAL_ATTEMPT_STATUSES.includes(attempt.status) ||
-        FINAL_RUN_STATUSES.includes(attempt.taskRun.status)
-      ) {
+      if (isFinalAttemptStatus(attempt.status) || isFinalRunStatus(attempt.taskRun.status)) {
         logger.error("Status already in final state", {
           attempt: {
             id: attempt.id,

apps/webapp/app/v3/marqs/v2.server.ts

@@ -82,7 +82,7 @@ function getMarQSClient() {
     defaultEnvConcurrency: env.V2_MARQS_DEFAULT_ENV_CONCURRENCY, // this is so we aren't limited by the environment concurrency
     defaultOrgConcurrency: env.DEFAULT_ORG_EXECUTION_CONCURRENCY_LIMIT,
     visibilityTimeoutInMs: env.V2_MARQS_VISIBILITY_TIMEOUT_MS, // 15 minutes
-    enableRebalancing: env.V2_MARQS_CONSUMER_POOL_ENABLED === "1",
+    enableRebalancing: false,
   });
 }
 

apps/webapp/app/v3/r2.server.ts

@@ -3,6 +3,7 @@ import { env } from "~/env.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { singleton } from "~/utils/singleton";
+import { startActiveSpan } from "./tracer.server";
 
 export const r2 = singleton("r2", initializeR2);
 
@@ -23,30 +24,87 @@ export async function uploadToObjectStore(
   contentType: string,
   environment: AuthenticatedEnvironment
 ): Promise<string> {
-  if (!r2) {
-    throw new Error("Object store credentials are not set");
+  return await startActiveSpan("uploadToObjectStore()", async (span) => {
+    if (!r2) {
+      throw new Error("Object store credentials are not set");
+    }
+
+    if (!env.OBJECT_STORE_BASE_URL) {
+      throw new Error("Object store base URL is not set");
+    }
+
+    span.setAttributes({
+      projectRef: environment.project.externalRef,
+      environmentSlug: environment.slug,
+      filename: filename,
+    });
+
+    const url = new URL(env.OBJECT_STORE_BASE_URL);
+    url.pathname = `/packets/${environment.project.externalRef}/${environment.slug}/${filename}`;
+
+    logger.debug("Uploading to object store", { url: url.href });
+
+    const response = await r2.fetch(url.toString(), {
+      method: "PUT",
+      headers: {
+        "Content-Type": contentType,
+      },
+      body: data,
+    });
+
+    if (!response.ok) {
+      throw new Error(`Failed to upload output to ${url}: ${response.statusText}`);
+    }
+
+    return url.href;
+  });
+}
+
+export async function generatePresignedRequest(
+  projectRef: string,
+  envSlug: string,
+  filename: string,
+  method: "PUT" | "GET" = "PUT"
+) {
+  if (!env.OBJECT_STORE_BASE_URL) {
+    return;
   }
 
-  if (!env.OBJECT_STORE_BASE_URL) {
-    throw new Error("Object store base URL is not set");
+  if (!r2) {
+    return;
   }
 
   const url = new URL(env.OBJECT_STORE_BASE_URL);
-  url.pathname = `/packets/${environment.project.externalRef}/${environment.slug}/${filename}`;
+  url.pathname = `/packets/${projectRef}/${envSlug}/${filename}`;
+  url.searchParams.set("X-Amz-Expires", "300"); // 5 minutes
 
-  logger.debug("Uploading to object store", { url: url.href });
+  const signed = await r2.sign(
+    new Request(url, {
+      method,
+    }),
+    {
+      aws: { signQuery: true },
+    }
+  );
 
-  const response = await r2.fetch(url.toString(), {
-    method: "PUT",
-    headers: {
-      "Content-Type": contentType,
-    },
-    body: data,
+  logger.debug("Generated presigned URL", {
+    url: signed.url,
+    headers: Object.fromEntries(signed.headers),
+    projectRef,
+    envSlug,
+    filename,
   });
 
-  if (!response.ok) {
-    throw new Error(`Failed to upload output to ${url}: ${response.statusText}`);
-  }
-
-  return url.href;
+  return signed;
+}
+
+export async function generatePresignedUrl(
+  projectRef: string,
+  envSlug: string,
+  filename: string,
+  method: "PUT" | "GET" = "PUT"
+) {
+  const signed = await generatePresignedRequest(projectRef, envSlug, filename, method);
+
+  return signed?.url;
 }

apps/webapp/app/v3/requeueTaskRun.server.ts

@@ -48,6 +48,7 @@ export class RequeueTaskRunService extends BaseService {
 
         break;
       }
+      case "DELAYED":
       case "WAITING_FOR_DEPLOY": {
         logger.debug("[RequeueTaskRunService] Removing task run from queue", { taskRun });
 
@@ -68,6 +69,7 @@ export class RequeueTaskRunService extends BaseService {
       case "CRASHED":
       case "COMPLETED_WITH_ERRORS":
       case "COMPLETED_SUCCESSFULLY":
+      case "EXPIRED":
       case "CANCELED": {
         logger.debug("[RequeueTaskRunService] Task run is completed", { taskRun });
 

apps/webapp/app/v3/services/cancelAttempt.server.ts

@@ -6,7 +6,7 @@ import { logger } from "~/services/logger.server";
 
 import { PrismaClientOrTransaction, prisma } from "~/db.server";
 import { ResumeTaskRunDependenciesService } from "./resumeTaskRunDependencies.server";
-import { CANCELLABLE_STATUSES } from "./cancelTaskRun.server";
+import { isCancellableRunStatus } from "../taskStatus";
 
 export class CancelAttemptService extends BaseService {
   public async call(
@@ -55,7 +55,7 @@ export class CancelAttemptService extends BaseService {
           taskRun: {
             update: {
               data: {
-                status: CANCELLABLE_STATUSES.includes(taskRunAttempt.taskRun.status)
+                status: isCancellableRunStatus(taskRunAttempt.taskRun.status)
                   ? "INTERRUPTED"
                   : undefined,
               },

apps/webapp/app/v3/services/cancelTaskRun.server.ts

@@ -1,4 +1,4 @@
-import { Prisma, TaskRun, TaskRunAttemptStatus, TaskRunStatus } from "@trigger.dev/database";
+import { Prisma, TaskRun } from "@trigger.dev/database";
 import assertNever from "assert-never";
 import { logger } from "~/services/logger.server";
 import { marqs } from "~/v3/marqs/index.server";
@@ -7,22 +7,7 @@ import { socketIo } from "../handleSocketIo.server";
 import { devPubSub } from "../marqs/devPubSub.server";
 import { BaseService } from "./baseService.server";
 import { CancelAttemptService } from "./cancelAttempt.server";
-
-export const CANCELLABLE_STATUSES: Array<TaskRunStatus> = [
-  "PENDING",
-  "WAITING_FOR_DEPLOY",
-  "EXECUTING",
-  "PAUSED",
-  "WAITING_TO_RESUME",
-  "PAUSED",
-  "RETRYING_AFTER_FAILURE",
-];
-
-const CANCELLABLE_ATTEMPT_STATUSES: Array<TaskRunAttemptStatus> = [
-  "EXECUTING",
-  "PAUSED",
-  "PENDING",
-];
+import { CANCELLABLE_ATTEMPT_STATUSES, isCancellableRunStatus } from "../taskStatus";
 
 type ExtendedTaskRun = Prisma.TaskRunGetPayload<{
   include: {
@@ -53,7 +38,11 @@ export class CancelTaskRunService extends BaseService {
     };
 
     // Make sure the task run is in a cancellable state
-    if (!CANCELLABLE_STATUSES.includes(taskRun.status)) {
+    if (!isCancellableRunStatus(taskRun.status)) {
+      logger.error("Task run is not in a cancellable state", {
+        runId: taskRun.id,
+        status: taskRun.status,
+      });
       return;
     }
 

apps/webapp/app/v3/services/crashTaskRun.server.ts

@@ -1,27 +1,11 @@
-import {
-  TaskRun,
-  TaskRunAttempt,
-  TaskRunAttemptStatus,
-  TaskRunStatus,
-} from "@trigger.dev/database";
+import { TaskRun, TaskRunAttempt } from "@trigger.dev/database";
 import { eventRepository } from "../eventRepository.server";
 import { marqs } from "~/v3/marqs/index.server";
 import { BaseService } from "./baseService.server";
 import { logger } from "~/services/logger.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { ResumeTaskRunDependenciesService } from "./resumeTaskRunDependencies.server";
-
-export const CRASHABLE_RUN_STATUSES: Array<TaskRunStatus> = [
-  "PENDING",
-  "WAITING_FOR_DEPLOY",
-  "EXECUTING",
-  "PAUSED",
-  "WAITING_TO_RESUME",
-  "PAUSED",
-  "RETRYING_AFTER_FAILURE",
-];
-
-const CRASHABLE_ATTEMPT_STATUSES: Array<TaskRunAttemptStatus> = ["EXECUTING", "PAUSED", "PENDING"];
+import { CRASHABLE_ATTEMPT_STATUSES, isCrashableRunStatus } from "../taskStatus";
 
 export type CrashTaskRunServiceOptions = {
   reason?: string;
@@ -29,6 +13,7 @@ export type CrashTaskRunServiceOptions = {
   logs?: string;
   crashAttempts?: boolean;
   crashedAt?: Date;
+  overrideCompletion?: boolean;
 };
 
 export class CrashTaskRunService extends BaseService {
@@ -52,7 +37,8 @@ export class CrashTaskRunService extends BaseService {
     }
 
     // Make sure the task run is in a crashable state
-    if (!CRASHABLE_RUN_STATUSES.includes(taskRun.status)) {
+    if (!opts.overrideCompletion && !isCrashableRunStatus(taskRun.status)) {
+      logger.error("Task run is not in a crashable state", { runId, status: taskRun.status });
       return;
     }
 
@@ -89,9 +75,12 @@ export class CrashTaskRunService extends BaseService {
       },
     });
 
-    const inProgressEvents = await eventRepository.queryIncompleteEvents({
-      runId: taskRun.friendlyId,
-    });
+    const inProgressEvents = await eventRepository.queryIncompleteEvents(
+      {
+        runId: taskRun.friendlyId,
+      },
+      options?.overrideCompletion
+    );
 
     logger.debug("Crashing in-progress events", {
       inProgressEvents: inProgressEvents.map((event) => event.id),

apps/webapp/app/v3/services/createCheckpoint.server.ts

@@ -1,19 +1,13 @@
 import { CoordinatorToPlatformMessages } from "@trigger.dev/core/v3";
 import type { InferSocketMessageSchema } from "@trigger.dev/core/v3/zodSocket";
-import type {
-  CheckpointRestoreEvent,
-  TaskRunAttemptStatus,
-  TaskRunStatus,
-} from "@trigger.dev/database";
+import type { Checkpoint, CheckpointRestoreEvent } from "@trigger.dev/database";
 import { logger } from "~/services/logger.server";
 import { generateFriendlyId } from "../friendlyIdentifiers";
 import { marqs } from "~/v3/marqs/index.server";
 import { CreateCheckpointRestoreEventService } from "./createCheckpointRestoreEvent.server";
 import { BaseService } from "./baseService.server";
 import { CrashTaskRunService } from "./crashTaskRun.server";
-
-const FREEZABLE_RUN_STATUSES: TaskRunStatus[] = ["EXECUTING", "RETRYING_AFTER_FAILURE"];
-const FREEZABLE_ATTEMPT_STATUSES: TaskRunAttemptStatus[] = ["EXECUTING", "FAILED"];
+import { isFinalRunStatus, isFreezableAttemptStatus, isFreezableRunStatus } from "../taskStatus";
 
 export class CreateCheckpointService extends BaseService {
   public async call(
@@ -21,7 +15,14 @@ export class CreateCheckpointService extends BaseService {
       InferSocketMessageSchema<typeof CoordinatorToPlatformMessages, "CHECKPOINT_CREATED">,
       "version"
     >
-  ) {
+  ): Promise<
+    | {
+        checkpoint: Checkpoint;
+        event: CheckpointRestoreEvent;
+        keepRunAlive: boolean;
+      }
+    | undefined
+  > {
     logger.debug(`Creating checkpoint`, params);
 
     const attempt = await this._prisma.taskRunAttempt.findUnique({
@@ -49,8 +50,8 @@ export class CreateCheckpointService extends BaseService {
     }
 
     if (
-      !FREEZABLE_ATTEMPT_STATUSES.includes(attempt.status) ||
-      !FREEZABLE_RUN_STATUSES.includes(attempt.taskRun.status)
+      !isFreezableAttemptStatus(attempt.status) ||
+      !isFreezableRunStatus(attempt.taskRun.status)
     ) {
       logger.error("Unfreezable state", {
         attempt: {
@@ -115,7 +116,9 @@ export class CreateCheckpointService extends BaseService {
     });
 
     const { reason } = params;
+
     let checkpointEvent: CheckpointRestoreEvent | undefined;
+    let keepRunAlive = false;
 
     switch (reason.type) {
       case "WAIT_FOR_DURATION": {
@@ -131,7 +134,12 @@ export class CreateCheckpointService extends BaseService {
           dependencyFriendlyRunId: reason.friendlyId,
         });
 
-        await marqs?.acknowledgeMessage(attempt.taskRunId);
+        keepRunAlive = await this.#isRunCompleted(reason.friendlyId);
+
+        if (!keepRunAlive) {
+          await marqs?.acknowledgeMessage(attempt.taskRunId);
+        }
+
         break;
       }
       case "WAIT_FOR_BATCH": {
@@ -140,7 +148,12 @@ export class CreateCheckpointService extends BaseService {
           batchDependencyFriendlyId: reason.batchFriendlyId,
         });
 
-        await marqs?.acknowledgeMessage(attempt.taskRunId);
+        keepRunAlive = await this.#isBatchCompleted(reason.batchFriendlyId);
+
+        if (!keepRunAlive) {
+          await marqs?.acknowledgeMessage(attempt.taskRunId);
+        }
+
         break;
       }
       case "RETRYING_AFTER_FAILURE": {
@@ -180,6 +193,37 @@ export class CreateCheckpointService extends BaseService {
     return {
       checkpoint,
       event: checkpointEvent,
+      keepRunAlive,
     };
   }
+
+  async #isBatchCompleted(friendlyId: string): Promise<boolean> {
+    const batch = await this._prisma.batchTaskRun.findUnique({
+      where: {
+        friendlyId,
+      },
+    });
+
+    if (!batch) {
+      logger.error("Batch not found", { friendlyId });
+      return false;
+    }
+
+    return batch.status === "COMPLETED";
+  }
+
+  async #isRunCompleted(friendlyId: string): Promise<boolean> {
+    const run = await this._prisma.taskRun.findUnique({
+      where: {
+        friendlyId,
+      },
+    });
+
+    if (!run) {
+      logger.error("Run not found", { friendlyId });
+      return false;
+    }
+
+    return isFinalRunStatus(run.status);
+  }
 }

apps/webapp/app/v3/services/deploymentIndexFailed.server.ts

@@ -18,7 +18,8 @@ export class DeploymentIndexFailed extends BaseService {
       message: string;
       stack?: string;
       stderr?: string;
-    }
+    },
+    overrideCompletion = false
   ) {
     const isFriendlyId = maybeFriendlyId.startsWith("deployment_");
 
@@ -38,6 +39,15 @@ export class DeploymentIndexFailed extends BaseService {
     }
 
     if (FINAL_DEPLOYMENT_STATUSES.includes(deployment.status)) {
+      if (overrideCompletion) {
+        logger.error("No support for overriding final deployment statuses just yet", {
+          id: deployment.id,
+          status: deployment.status,
+          previousError: deployment.errorData,
+          incomingError: error,
+        });
+      }
+
       logger.error("Worker deployment already in final state", {
         id: deployment.id,
         status: deployment.status,

apps/webapp/app/v3/services/enqueueDelayedRun.server.ts

@@ -0,0 +1,72 @@
+import { logger } from "~/services/logger.server";
+import { marqs } from "~/v3/marqs/index.server";
+import { BaseService } from "./baseService.server";
+import { parseNaturalLanguageDuration } from "./triggerTask.server";
+import { workerQueue } from "~/services/worker.server";
+import { $transaction } from "~/db.server";
+
+export class EnqueueDelayedRunService extends BaseService {
+  public async call(runId: string) {
+    const run = await this._prisma.taskRun.findUnique({
+      where: {
+        id: runId,
+      },
+      include: {
+        runtimeEnvironment: {
+          include: {
+            organization: true,
+            project: true,
+          },
+        },
+      },
+    });
+
+    if (!run) {
+      logger.debug("Could not find delayed run to enqueue", {
+        runId,
+      });
+
+      return;
+    }
+
+    if (run.status !== "DELAYED") {
+      logger.debug("Delayed run cannot be enqueued because it's not in DELAYED status", {
+        run,
+      });
+
+      return;
+    }
+
+    await $transaction(this._prisma, async (tx) => {
+      await tx.taskRun.update({
+        where: {
+          id: run.id,
+        },
+        data: {
+          status: "PENDING",
+          queuedAt: new Date(),
+        },
+      });
+
+      if (run.ttl) {
+        const expireAt = parseNaturalLanguageDuration(run.ttl);
+
+        if (expireAt) {
+          await workerQueue.enqueue(
+            "v3.expireRun",
+            { runId: run.id },
+            { tx, runAt: expireAt, jobKey: `v3.expireRun.${run.id}` }
+          );
+        }
+      }
+    });
+
+    await marqs?.enqueueMessage(
+      run.runtimeEnvironment,
+      run.queue,
+      run.id,
+      { type: "EXECUTE", taskIdentifier: run.taskIdentifier },
+      run.concurrencyKey ?? undefined
+    );
+  }
+}

apps/webapp/app/v3/services/expireEnqueuedRun.server.ts

@@ -0,0 +1,53 @@
+import { logger } from "~/services/logger.server";
+import { marqs } from "~/v3/marqs/index.server";
+import { BaseService } from "./baseService.server";
+
+export class ExpireEnqueuedRunService extends BaseService {
+  public async call(runId: string) {
+    const run = await this._prisma.taskRun.findUnique({
+      where: {
+        id: runId,
+      },
+      include: {
+        runtimeEnvironment: {
+          include: {
+            organization: true,
+            project: true,
+          },
+        },
+      },
+    });
+
+    if (!run) {
+      logger.debug("Could not find enqueued run to expire", {
+        runId,
+      });
+
+      return;
+    }
+
+    if (run.status !== "PENDING") {
+      logger.debug("Run cannot be expired because it's not in PENDING status", {
+        run,
+      });
+
+      return;
+    }
+
+    logger.debug("Expiring enqueued run", {
+      run,
+    });
+
+    await this._prisma.taskRun.update({
+      where: {
+        id: run.id,
+      },
+      data: {
+        status: "EXPIRED",
+        expiredAt: new Date(),
+      },
+    });
+
+    await marqs?.acknowledgeMessage(run.id);
+  }
+}

apps/webapp/app/v3/services/rescheduleTaskRun.server.ts

@@ -0,0 +1,39 @@
+import { TaskRun } from "@trigger.dev/database";
+import { BaseService, ServiceValidationError } from "./baseService.server";
+import { RescheduleRunRequestBody } from "@trigger.dev/core/v3";
+import { parseDelay } from "./triggerTask.server";
+import { $transaction } from "~/db.server";
+import { workerQueue } from "~/services/worker.server";
+
+export class RescheduleTaskRunService extends BaseService {
+  public async call(taskRun: TaskRun, body: RescheduleRunRequestBody) {
+    if (taskRun.status !== "DELAYED") {
+      throw new ServiceValidationError("Cannot reschedule a run that is not delayed");
+    }
+
+    const delay = await parseDelay(body.delay);
+
+    if (!delay) {
+      throw new ServiceValidationError(`Invalid delay: ${body.delay}`);
+    }
+
+    return await $transaction(this._prisma, async (tx) => {
+      const updatedRun = await tx.taskRun.update({
+        where: {
+          id: taskRun.id,
+        },
+        data: {
+          delayUntil: delay,
+        },
+      });
+
+      await workerQueue.enqueue(
+        "v3.enqueueDelayedRun",
+        { runId: taskRun.id },
+        { tx, runAt: delay, jobKey: `v3.enqueueDelayedRun.${taskRun.id}` }
+      );
+
+      return updatedRun;
+    });
+  }
+}

apps/webapp/app/v3/services/restoreCheckpoint.server.ts

@@ -1,12 +1,10 @@
-import { TaskRunAttemptStatus, TaskRunStatus, type Checkpoint } from "@trigger.dev/database";
+import { type Checkpoint } from "@trigger.dev/database";
 import { logger } from "~/services/logger.server";
 import { socketIo } from "../handleSocketIo.server";
 import { machinePresetFromConfig } from "../machinePresets.server";
 import { BaseService } from "./baseService.server";
 import { CreateCheckpointRestoreEventService } from "./createCheckpointRestoreEvent.server";
-
-const RESTORABLE_RUN_STATUSES: TaskRunStatus[] = ["WAITING_TO_RESUME"];
-const RESTORABLE_ATTEMPT_STATUSES: TaskRunAttemptStatus[] = ["PAUSED"];
+import { isRestorableAttemptStatus, isRestorableRunStatus } from "../taskStatus";
 
 export class RestoreCheckpointService extends BaseService {
   public async call(params: {
@@ -51,10 +49,7 @@ export class RestoreCheckpointService extends BaseService {
 
     const checkpoint = checkpointEvent.checkpoint;
 
-    const runIsRestorable = RESTORABLE_RUN_STATUSES.includes(checkpoint.run.status);
-    const attemptIsRestorable = RESTORABLE_ATTEMPT_STATUSES.includes(checkpoint.attempt.status);
-
-    if (!runIsRestorable) {
+    if (!isRestorableRunStatus(checkpoint.run.status)) {
       logger.error("Run is unrestorable", {
         eventId: params.eventId,
         runId: checkpoint.runId,
@@ -64,7 +59,7 @@ export class RestoreCheckpointService extends BaseService {
       return;
     }
 
-    if (!attemptIsRestorable && !params.isRetry) {
+    if (!isRestorableAttemptStatus(checkpoint.attempt.status) && !params.isRetry) {
       logger.error("Attempt is unrestorable", {
         eventId: params.eventId,
         runId: checkpoint.runId,

apps/webapp/app/v3/services/triggerTask.server.ts

@@ -4,13 +4,15 @@ import {
   TriggerTaskRequestBody,
   packetRequiresOffloading,
 } from "@trigger.dev/core/v3";
-import { prisma } from "~/db.server";
+import { env } from "~/env.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { autoIncrementCounter } from "~/services/autoIncrementCounter.server";
+import { workerQueue } from "~/services/worker.server";
 import { marqs, sanitizeQueueName } from "~/v3/marqs/index.server";
 import { eventRepository } from "../eventRepository.server";
 import { generateFriendlyId } from "../friendlyIdentifiers";
 import { uploadToObjectStore } from "../r2.server";
+import { startActiveSpan } from "../tracer.server";
 import { BaseService } from "./baseService.server";
 
 export type TriggerTaskServiceOptions = {
@@ -34,6 +36,12 @@ export class TriggerTaskService extends BaseService {
       span.setAttribute("taskId", taskId);
 
       const idempotencyKey = options.idempotencyKey ?? body.options?.idempotencyKey;
+      const delayUntil = await parseDelay(body.options?.delay);
+
+      const ttl =
+        typeof body.options?.ttl === "number"
+          ? stringifyDuration(body.options?.ttl)
+          : body.options?.ttl ?? (environment.type === "DEVELOPMENT" ? "10m" : undefined);
 
       const existingRun = idempotencyKey
         ? await this._prisma.taskRun.findUnique({
@@ -48,9 +56,19 @@ export class TriggerTaskService extends BaseService {
 
       if (existingRun && existingRun.taskIdentifier === taskId) {
         span.setAttribute("runId", existingRun.friendlyId);
+
         return existingRun;
       }
 
+      const runFriendlyId = generateFriendlyId("run");
+
+      const payloadPacket = await this.#handlePayloadPacket(
+        body.payload,
+        body.options?.payloadType ?? "application/json",
+        runFriendlyId,
+        environment
+      );
+
       return await eventRepository.traceEvent(
         taskId,
         {
@@ -75,15 +93,6 @@ export class TriggerTaskService extends BaseService {
           immediate: true,
         },
         async (event, traceContext) => {
-          const runFriendlyId = generateFriendlyId("run");
-
-          const payloadPacket = await this.#handlePayloadPacket(
-            body.payload,
-            body.options?.payloadType ?? "application/json",
-            runFriendlyId,
-            environment
-          );
-
           const run = await autoIncrementCounter.incrementInTransaction(
             `v3-run:${environment.id}:${taskId}`,
             async (num, tx) => {
@@ -111,7 +120,7 @@ export class TriggerTaskService extends BaseService {
 
               const taskRun = await tx.taskRun.create({
                 data: {
-                  status: "PENDING",
+                  status: delayUntil ? "DELAYED" : "PENDING",
                   number: num,
                   friendlyId: runFriendlyId,
                   runtimeEnvironmentId: environment.id,
@@ -128,6 +137,9 @@ export class TriggerTaskService extends BaseService {
                   concurrencyKey: body.options?.concurrencyKey,
                   queue: queueName,
                   isTest: body.options?.test ?? false,
+                  delayUntil,
+                  queuedAt: delayUntil ? undefined : new Date(),
+                  ttl,
                 },
               });
 
@@ -176,10 +188,12 @@ export class TriggerTaskService extends BaseService {
               }
 
               if (body.options?.queue) {
-                const concurrencyLimit = body.options.queue.concurrencyLimit
-                  ? Math.max(0, body.options.queue.concurrencyLimit)
-                  : null;
-                const taskQueue = await prisma.taskQueue.upsert({
+                const concurrencyLimit =
+                  typeof body.options.queue.concurrencyLimit === "number"
+                    ? Math.max(0, body.options.queue.concurrencyLimit)
+                    : undefined;
+
+                const taskQueue = await tx.taskQueue.upsert({
                   where: {
                     runtimeEnvironmentId_name: {
                       runtimeEnvironmentId: environment.id,
@@ -212,6 +226,26 @@ export class TriggerTaskService extends BaseService {
                 }
               }
 
+              if (taskRun.delayUntil) {
+                await workerQueue.enqueue(
+                  "v3.enqueueDelayedRun",
+                  { runId: taskRun.id },
+                  { tx, runAt: delayUntil, jobKey: `v3.enqueueDelayedRun.${taskRun.id}` }
+                );
+              }
+
+              if (!taskRun.delayUntil && taskRun.ttl) {
+                const expireAt = parseNaturalLanguageDuration(taskRun.ttl);
+
+                if (expireAt) {
+                  await workerQueue.enqueue(
+                    "v3.expireRun",
+                    { runId: taskRun.id },
+                    { tx, runAt: expireAt, jobKey: `v3.expireRun.${taskRun.id}` }
+                  );
+                }
+              }
+
               return taskRun;
             },
             async (_, tx) => {
@@ -235,13 +269,15 @@ export class TriggerTaskService extends BaseService {
           }
 
           // We need to enqueue the task run into the appropriate queue. This is done after the tx completes to prevent a race condition where the task run hasn't been created yet by the time we dequeue.
-          await marqs?.enqueueMessage(
-            environment,
-            run.queue,
-            run.id,
-            { type: "EXECUTE", taskIdentifier: taskId },
-            body.options?.concurrencyKey
-          );
+          if (run.status === "PENDING") {
+            await marqs?.enqueueMessage(
+              environment,
+              run.queue,
+              run.id,
+              { type: "EXECUTE", taskIdentifier: taskId },
+              body.options?.concurrencyKey
+            );
+          }
 
           return run;
         }
@@ -255,26 +291,31 @@ export class TriggerTaskService extends BaseService {
     pathPrefix: string,
     environment: AuthenticatedEnvironment
   ) {
-    const packet = this.#createPayloadPacket(payload, payloadType);
+    return await startActiveSpan("handlePayloadPacket()", async (span) => {
+      const packet = this.#createPayloadPacket(payload, payloadType);
 
-    if (!packet.data) {
-      return packet;
-    }
+      if (!packet.data) {
+        return packet;
+      }
 
-    const { needsOffloading, size } = packetRequiresOffloading(packet);
+      const { needsOffloading, size } = packetRequiresOffloading(
+        packet,
+        env.TASK_PAYLOAD_OFFLOAD_THRESHOLD
+      );
 
-    if (!needsOffloading) {
-      return packet;
-    }
+      if (!needsOffloading) {
+        return packet;
+      }
 
-    const filename = `${pathPrefix}/payload.json`;
+      const filename = `${pathPrefix}/payload.json`;
 
-    await uploadToObjectStore(filename, packet.data, packet.dataType, environment);
+      await uploadToObjectStore(filename, packet.data, packet.dataType, environment);
 
-    return {
-      data: filename,
-      dataType: "application/store",
-    };
+      return {
+        data: filename,
+        dataType: "application/store",
+      };
+    });
   }
 
   #createPayloadPacket(payload: any, payloadType: string): IOPacket {
@@ -289,3 +330,104 @@ export class TriggerTaskService extends BaseService {
     return { dataType: payloadType };
   }
 }
+
+export async function parseDelay(value?: string | Date): Promise<Date | undefined> {
+  if (!value) {
+    return;
+  }
+
+  if (value instanceof Date) {
+    return value;
+  }
+
+  try {
+    const date = new Date(value);
+
+    // Check if the date is valid
+    if (isNaN(date.getTime())) {
+      return parseNaturalLanguageDuration(value);
+    }
+
+    if (date.getTime() <= Date.now()) {
+      return;
+    }
+
+    return date;
+  } catch (error) {
+    return parseNaturalLanguageDuration(value);
+  }
+}
+
+export function parseNaturalLanguageDuration(duration: string): Date | undefined {
+  const regexPattern = /^(\d+w)?(\d+d)?(\d+h)?(\d+m)?(\d+s)?$/;
+
+  const result: Date = new Date();
+  let hasMatch = false;
+
+  const elements = duration.match(regexPattern);
+  if (elements) {
+    if (elements[1]) {
+      const weeks = Number(elements[1].slice(0, -1));
+      if (weeks >= 0) {
+        result.setDate(result.getDate() + 7 * weeks);
+        hasMatch = true;
+      }
+    }
+    if (elements[2]) {
+      const days = Number(elements[2].slice(0, -1));
+      if (days >= 0) {
+        result.setDate(result.getDate() + days);
+        hasMatch = true;
+      }
+    }
+    if (elements[3]) {
+      const hours = Number(elements[3].slice(0, -1));
+      if (hours >= 0) {
+        result.setHours(result.getHours() + hours);
+        hasMatch = true;
+      }
+    }
+    if (elements[4]) {
+      const minutes = Number(elements[4].slice(0, -1));
+      if (minutes >= 0) {
+        result.setMinutes(result.getMinutes() + minutes);
+        hasMatch = true;
+      }
+    }
+    if (elements[5]) {
+      const seconds = Number(elements[5].slice(0, -1));
+      if (seconds >= 0) {
+        result.setSeconds(result.getSeconds() + seconds);
+        hasMatch = true;
+      }
+    }
+  }
+
+  if (hasMatch) {
+    return result;
+  }
+
+  return undefined;
+}
+
+function stringifyDuration(seconds: number): string | undefined {
+  if (seconds <= 0) {
+    return;
+  }
+
+  const units = {
+    w: Math.floor(seconds / 604800),
+    d: Math.floor((seconds % 604800) / 86400),
+    h: Math.floor((seconds % 86400) / 3600),
+    m: Math.floor((seconds % 3600) / 60),
+    s: Math.floor(seconds % 60),
+  };
+
+  // Filter the units having non-zero values and join them
+  const result: string = Object.entries(units)
+    .filter(([unit, val]) => val != 0)
+    .map(([unit, val]) => `${val}${unit}`)
+    .join("");
+
+  return result;
+}

apps/webapp/app/v3/taskStatus.ts

@@ -0,0 +1,71 @@
+import type { TaskRunAttemptStatus, TaskRunStatus } from "@trigger.dev/database";
+
+export const CANCELLABLE_RUN_STATUSES: TaskRunStatus[] = [
+  "DELAYED",
+  "PENDING",
+  "WAITING_FOR_DEPLOY",
+  "EXECUTING",
+  "PAUSED",
+  "WAITING_TO_RESUME",
+  "PAUSED",
+  "RETRYING_AFTER_FAILURE",
+];
+export const CANCELLABLE_ATTEMPT_STATUSES: TaskRunAttemptStatus[] = [
+  "EXECUTING",
+  "PAUSED",
+  "PENDING",
+];
+
+export function isCancellableRunStatus(status: TaskRunStatus): boolean {
+  return CANCELLABLE_RUN_STATUSES.includes(status);
+}
+export function isCancellableAttemptStatus(status: TaskRunAttemptStatus): boolean {
+  return CANCELLABLE_ATTEMPT_STATUSES.includes(status);
+}
+
+export const CRASHABLE_RUN_STATUSES: TaskRunStatus[] = CANCELLABLE_RUN_STATUSES;
+export const CRASHABLE_ATTEMPT_STATUSES: TaskRunAttemptStatus[] = CANCELLABLE_ATTEMPT_STATUSES;
+
+export function isCrashableRunStatus(status: TaskRunStatus): boolean {
+  return CRASHABLE_RUN_STATUSES.includes(status);
+}
+export function isCrashableAttemptStatus(status: TaskRunAttemptStatus): boolean {
+  return CRASHABLE_ATTEMPT_STATUSES.includes(status);
+}
+
+export const FINAL_RUN_STATUSES: TaskRunStatus[] = [
+  "CANCELED",
+  "COMPLETED_SUCCESSFULLY",
+  "COMPLETED_WITH_ERRORS",
+  "INTERRUPTED",
+  "SYSTEM_FAILURE",
+  "EXPIRED",
+];
+export const FINAL_ATTEMPT_STATUSES: TaskRunAttemptStatus[] = ["CANCELED", "COMPLETED", "FAILED"];
+
+export const FREEZABLE_RUN_STATUSES: TaskRunStatus[] = ["EXECUTING", "RETRYING_AFTER_FAILURE"];
+export const FREEZABLE_ATTEMPT_STATUSES: TaskRunAttemptStatus[] = ["EXECUTING", "FAILED"];
+
+export function isFreezableRunStatus(status: TaskRunStatus): boolean {
+  return FREEZABLE_RUN_STATUSES.includes(status);
+}
+export function isFreezableAttemptStatus(status: TaskRunAttemptStatus): boolean {
+  return FREEZABLE_ATTEMPT_STATUSES.includes(status);
+}
+
+export function isFinalRunStatus(status: TaskRunStatus): boolean {
+  return FINAL_RUN_STATUSES.includes(status);
+}
+export function isFinalAttemptStatus(status: TaskRunAttemptStatus): boolean {
+  return FINAL_ATTEMPT_STATUSES.includes(status);
+}
+
+export const RESTORABLE_RUN_STATUSES: TaskRunStatus[] = ["WAITING_TO_RESUME"];
+export const RESTORABLE_ATTEMPT_STATUSES: TaskRunAttemptStatus[] = ["PAUSED"];
+
+export function isRestorableRunStatus(status: TaskRunStatus): boolean {
+  return RESTORABLE_RUN_STATUSES.includes(status);
+}
+export function isRestorableAttemptStatus(status: TaskRunAttemptStatus): boolean {
+  return RESTORABLE_ATTEMPT_STATUSES.includes(status);
+}

apps/webapp/app/v3/tracer.server.ts

@@ -4,7 +4,10 @@ import {
   DiagConsoleLogger,
   DiagLogLevel,
   Link,
+  Span,
   SpanKind,
+  SpanOptions,
+  SpanStatusCode,
   diag,
   trace,
 } from "@opentelemetry/api";
@@ -76,6 +79,32 @@ class CustomWebappSampler implements Sampler {
 
 export const tracer = singleton("tracer", getTracer);
 
+export async function startActiveSpan<T>(
+  name: string,
+  fn: (span: Span) => Promise<T>,
+  options?: SpanOptions
+): Promise<T> {
+  return tracer.startActiveSpan(name, options ?? {}, async (span) => {
+    try {
+      return await fn(span);
+    } catch (error) {
+      if (error instanceof Error) {
+        span.recordException(error);
+      } else if (typeof error === "string") {
+        span.recordException(new Error(error));
+      } else {
+        span.recordException(new Error(String(error)));
+      }
+
+      span.setStatus({ code: SpanStatusCode.ERROR });
+
+      throw error;
+    } finally {
+      span.end();
+    }
+  });
+}
+
 function getTracer() {
   if (env.INTERNAL_OTEL_TRACE_DISABLED === "1") {
     console.log(`🔦 Tracer disabled, returning a noop tracer`);
@@ -88,7 +117,7 @@ function getTracer() {
   const samplingRate = 1.0 / Math.max(parseInt(env.INTERNAL_OTEL_TRACE_SAMPLING_RATE, 10), 1);
 
   const provider = new NodeTracerProvider({
-    forceFlushTimeoutMillis: 500,
+    forceFlushTimeoutMillis: 15_000,
     resource: new Resource({
       [SEMRESATTRS_SERVICE_NAME]: env.SERVICE_NAME,
     }),
@@ -100,7 +129,7 @@ function getTracer() {
   if (env.INTERNAL_OTEL_TRACE_EXPORTER_URL) {
     const exporter = new OTLPTraceExporter({
       url: env.INTERNAL_OTEL_TRACE_EXPORTER_URL,
-      timeoutMillis: 10_000,
+      timeoutMillis: 15_000,
       headers:
         env.INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_NAME &&
         env.INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE

docs/mint.json

@@ -1,8 +1,14 @@
 {
   "$schema": "https://mintlify.com/schema.json",
   "name": "Trigger.dev",
-  "openapi": ["/openapi.yml", "/v3-openapi.yaml"],
-  "versions": ["v3 (Developer Preview)", "v2"],
+  "openapi": [
+    "/openapi.yml",
+    "/v3-openapi.yaml"
+  ],
+  "versions": [
+    "v3",
+    "v2"
+  ],
   "api": {
     "playground": {
       "mode": "simple"
@@ -16,15 +22,19 @@
   },
   "favicon": "/images/favicon.png",
   "colors": {
-    "primary": "#818cf8",
-    "light": "#818cf8",
-    "dark": "#818cf8",
+    "primary": "#A8FF53",
+    "light": "#A8FF53",
+    "dark": "#A8FF53",
     "anchors": {
       "from": "#fff",
       "to": "#eee"
+    },
+    "background": {
+      "light": "#fff",
+      "dark": "#121317"
     }
   },
-  "backgroundImage": "/images/docs-background.png",
+  "theme": "quill",
   "modeToggle": {
     "default": "dark",
     "isHidden": true
@@ -44,36 +54,6 @@
       "url": "https://discord.gg/kA47vcd8P6"
     }
   ],
-  "primaryTab": {
-    "name": "Home"
-  },
-  "tabs": [
-    {
-      "name": "Integrations",
-      "url": "integrations",
-      "version": "v2"
-    },
-    {
-      "name": "Reference",
-      "url": "sdk",
-      "version": "v2"
-    },
-    {
-      "name": "Examples",
-      "url": "https://trigger.dev/apis",
-      "version": "v2"
-    },
-    {
-      "name": "v3 Developer Preview",
-      "url": "https://trigger.dev/docs/v3",
-      "version": "v2"
-    },
-    {
-      "name": "v2",
-      "url": "https://trigger.dev/docs/documentation",
-      "version": "v3 (Developer Preview)"
-    }
-  ],
   "redirects": [
     {
       "source": "/documentation/quickstart",
@@ -95,12 +75,14 @@
   "navigation": [
     {
       "group": "",
-      "version": "v3 (Developer Preview)",
-      "pages": ["v3/introduction"]
+      "version": "v3",
+      "pages": [
+        "v3/introduction"
+      ]
     },
     {
       "group": "Getting Started",
-      "version": "v3 (Developer Preview)",
+      "version": "v3",
       "pages": [
         "v3/quick-start",
         "v3/upgrading-from-v2",
@@ -111,7 +93,7 @@
     },
     {
       "group": "Fundamentals",
-      "version": "v3 (Developer Preview)",
+      "version": "v3",
       "pages": [
         "v3/trigger-folder",
         "v3/tasks-overview",
@@ -119,32 +101,40 @@
         "v3/apikeys",
         {
           "group": "Task types",
-          "pages": ["v3/tasks-regular", "v3/tasks-scheduled"]
+          "pages": [
+            "v3/tasks-regular",
+            "v3/tasks-scheduled"
+          ]
         },
         "v3/trigger-config"
       ]
     },
     {
       "group": "Development",
-      "version": "v3 (Developer Preview)",
-      "pages": ["v3/cli-dev", "v3/run-tests"]
+      "version": "v3",
+      "pages": [
+        "v3/cli-dev",
+        "v3/run-tests"
+      ]
     },
     {
       "group": "Deployment",
-      "version": "v3 (Developer Preview)",
+      "version": "v3",
       "pages": [
         "v3/deploy-environment-variables",
         "v3/cli-deploy",
         "v3/github-actions",
         {
           "group": "Deployment integrations",
-          "pages": ["v3/vercel-integration"]
+          "pages": [
+            "v3/vercel-integration"
+          ]
         }
       ]
     },
     {
       "group": "Writing tasks",
-      "version": "v3 (Developer Preview)",
+      "version": "v3",
       "pages": [
         "v3/writing-tasks-introduction",
         "v3/logging",
@@ -169,16 +159,24 @@
     },
     {
       "group": "API reference",
-      "version": "v3 (Developer Preview)",
+      "version": "v3",
       "pages": [
         "v3/management/overview",
+        {
+          "group": "Tasks API",
+          "pages": [
+            "v3/management/tasks/trigger",
+            "v3/management/tasks/batch-trigger"
+          ]
+        },
         {
           "group": "Runs API",
           "pages": [
             "v3/management/runs/list",
             "v3/management/runs/retrieve",
             "v3/management/runs/replay",
-            "v3/management/runs/cancel"
+            "v3/management/runs/cancel",
+            "v3/management/runs/reschedule"
           ]
         },
         {
@@ -207,18 +205,24 @@
         },
         {
           "group": "Projects API",
-          "pages": ["v3/management/projects/runs"]
+          "pages": [
+            "v3/management/projects/runs"
+          ]
         }
       ]
     },
     {
       "group": "Open source",
-      "version": "v3 (Developer Preview)",
-      "pages": ["v3/github-repo", "v3/open-source-self-hosting", "v3/open-source-contributing"]
+      "version": "v3",
+      "pages": [
+        "v3/github-repo",
+        "v3/open-source-self-hosting",
+        "v3/open-source-contributing"
+      ]
     },
     {
       "group": "Troubleshooting",
-      "version": "v3 (Developer Preview)",
+      "version": "v3",
       "pages": [
         "v3/troubleshooting",
         "v3/troubleshooting-alerts",
@@ -229,8 +233,12 @@
     },
     {
       "group": "Help",
-      "version": "v3 (Developer Preview)",
-      "pages": ["v3/community", "v3/help-slack", "v3/help-email"]
+      "version": "v3",
+      "pages": [
+        "v3/community",
+        "v3/help-slack",
+        "v3/help-email"
+      ]
     },
     {
       "group": "Getting Started",
@@ -422,7 +430,10 @@
       "pages": [
         {
           "group": "Airtable",
-          "pages": ["integrations/apis/airtable", "integrations/apis/airtable-tasks"]
+          "pages": [
+            "integrations/apis/airtable",
+            "integrations/apis/airtable-tasks"
+          ]
         },
         {
           "group": "GitHub",
@@ -448,16 +459,25 @@
         },
         {
           "group": "Plain",
-          "pages": ["integrations/apis/plain", "integrations/apis/plain-tasks"]
+          "pages": [
+            "integrations/apis/plain",
+            "integrations/apis/plain-tasks"
+          ]
         },
         "integrations/apis/replicate",
         {
           "group": "SendGrid",
-          "pages": ["integrations/apis/sendgrid", "integrations/apis/sendgrid-tasks"]
+          "pages": [
+            "integrations/apis/sendgrid",
+            "integrations/apis/sendgrid-tasks"
+          ]
         },
         {
           "group": "Resend",
-          "pages": ["integrations/apis/resend", "integrations/apis/resend-tasks"]
+          "pages": [
+            "integrations/apis/resend",
+            "integrations/apis/resend-tasks"
+          ]
         },
         {
           "group": "Shopify",
@@ -469,7 +489,10 @@
         },
         {
           "group": "Slack",
-          "pages": ["integrations/apis/slack", "integrations/apis/slack-tasks"]
+          "pages": [
+            "integrations/apis/slack",
+            "integrations/apis/slack-tasks"
+          ]
         },
         "integrations/apis/stripe",
         {
@@ -495,7 +518,9 @@
             "sdk/triggerclient/constructor",
             {
               "group": "Instance properties",
-              "pages": ["sdk/triggerclient/store"]
+              "pages": [
+                "sdk/triggerclient/store"
+              ]
             },
             {
               "group": "Instance methods",
@@ -558,7 +583,10 @@
             "sdk/dynamictrigger/constructor",
             {
               "group": "Instance methods",
-              "pages": ["sdk/dynamictrigger/register", "sdk/dynamictrigger/unregister"]
+              "pages": [
+                "sdk/dynamictrigger/register",
+                "sdk/dynamictrigger/unregister"
+              ]
             }
           ]
         },
@@ -569,7 +597,10 @@
             "sdk/dynamicschedule/constructor",
             {
               "group": "Instance methods",
-              "pages": ["sdk/dynamicschedule/register", "sdk/dynamicschedule/unregister"]
+              "pages": [
+                "sdk/dynamicschedule/register",
+                "sdk/dynamicschedule/unregister"
+              ]
             }
           ]
         },
@@ -582,7 +613,9 @@
     {
       "group": "HTTP Reference",
       "version": "v2",
-      "pages": ["sdk/api-reference/events/create-an-event"]
+      "pages": [
+        "sdk/api-reference/events/create-an-event"
+      ]
     },
     {
       "group": "React SDK",
@@ -598,7 +631,9 @@
     {
       "group": "Overview",
       "version": "v2",
-      "pages": ["examples/introduction"]
+      "pages": [
+        "examples/introduction"
+      ]
     }
   ],
   "footerSocials": {
@@ -606,4 +641,4 @@
     "github": "https://github.com/triggerdotdev",
     "linkedin": "https://www.linkedin.com/company/triggerdotdev"
   }
-}
+}

docs/v3-openapi.yaml

@@ -269,6 +269,7 @@ paths:
 
   "/api/v1/timezones":
     get:
+      security: []
       operationId: get_timezones_v1
       summary: Get all supported timezones
       description: Get all supported timezones that schedule tasks support.
@@ -424,6 +425,71 @@ paths:
 
             await runs.cancel("run_1234");
 
+  "/api/v1/runs/{runId}/reschedule":
+    parameters:
+      - $ref: "#/components/parameters/runId"
+    post:
+      operationId: reschedule_run_v1
+      summary: Rescheduled a delayed run
+      description: Updates a delayed run with a new delay. Only valid when the run is in the DELAYED state.
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              "$ref": "#/components/schemas/RescheduleRunRequestBody"
+      responses:
+        "200":
+          description: Successful request
+          content:
+            application/json:
+              schema:
+                "$ref": "#/components/schemas/RetrieveRunResponse"
+        "400":
+          description: Invalid request
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  error:
+                    type: string
+                    enum:
+                      - Invalid or missing run ID
+                      - Failed to create new run
+        "401":
+          description: Unauthorized request
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  error:
+                    type: string
+                    enum:
+                      - Invalid or Missing API key
+        "404":
+          description: Resource not found
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  error:
+                    type: string
+                    enum:
+                      - Run not found
+      tags:
+        - runs
+      security:
+        - secretKey: []
+      x-codeSamples:
+        - lang: typescript
+          source: |-
+            import { runs } from "@trigger.dev/sdk/v3";
+
+            const handle = await runs.reschedule("run_1234", { delay: new Date("2024-06-29T20:45:56.340Z") });
+
   "/api/v3/runs/{runId}":
     parameters:
       - $ref: "#/components/parameters/runId"
@@ -811,17 +877,7 @@ paths:
                   description: Whether to override existing variables or not
                   default: false
               required: ["variables"]
-          multipart/form-data:
-            schema:
-              type: object
-              properties:
-                variables:
-                  type: string
-                  format: binary
-                override:
-                  type: boolean
-              required:
-                - variables
+
       responses:
         "200":
           description: Environment variables imported successfully
@@ -864,57 +920,11 @@ paths:
           source: |-
             import { envvars } from "@trigger.dev/sdk/v3";
 
-            // Import variables from an array
             await envvars.upload("proj_yubjwjsfkxnylobaqvqz", "dev", {
-              variables: [
-                {
-                  name: "SLACK_API_KEY",
-                  value: "slack_123456"
-                }
-              ],
+              variables: { SLACK_API_KEY: "slack_key_1234" },
               override: false
             });
-        - lang: typescript
-          label: Import variables from a read stream
-          source: |-
-            import { envvars } from "@trigger.dev/sdk/v3";
-            import { createReadStream } from "node:fs";
 
-            // Import variables in dotenv format from a file
-            await envvars.upload("proj_yubjwjsfkxnylobaqvqz", "dev", {
-              variables: createReadStream(".env"),
-              override: false
-            });
-        - lang: typescript
-          label: Import variables from a response
-          source: |-
-            import { envvars } from "@trigger.dev/sdk/v3";
-
-            // Import variables in dotenv format from a response
-            await envvars.upload("proj_yubjwjsfkxnylobaqvqz", "dev", {
-              variables: await fetch("https://example.com/.env"),
-              override: false
-            });
-        - lang: typescript
-          label: Import variables from a Buffer
-          source: |-
-            import { envvars } from "@trigger.dev/sdk/v3";
-
-            // Import variables in dotenv format from a buffer
-            await envvars.upload("proj_yubjwjsfkxnylobaqvqz", "dev", {
-              variables: Buffer.from("SLACK_API_KEY=slack_1234"),
-              override: false
-            });
-        - lang: typescript
-          label: Import variables from a File
-          source: |-
-            import { envvars } from "@trigger.dev/sdk/v3";
-
-            // Import variables in dotenv format from a file
-            await envvars.upload("proj_yubjwjsfkxnylobaqvqz", "dev", {
-              variables: new File(["SLACK_API_KEY=slack_1234"], ".env"),
-              override: false
-            });
 
   "/api/v1/projects/{projectRef}/envvars/{env}/{name}":
     parameters:
@@ -1095,9 +1105,229 @@ paths:
                 });
               }
             })
+  "/api/v1/tasks/{taskIdentifier}/trigger":
+    parameters:
+      - $ref: "#/components/parameters/taskIdentifier"
+    post:
+      operationId: trigger_task_v1
+      summary: Trigger a task
+      description: Trigger a task by its identifier.
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              "$ref": "#/components/schemas/TriggerTaskRequestBody"
+      responses:
+        "200":
+          description: Task triggered successfully
+          content:
+            application/json:
+              schema:
+                "$ref": "#/components/schemas/TriggerTaskResponse"
+        "400":
+          description: Invalid request parameters or body
+          content:
+            application/json:
+              schema:
+                "$ref": "#/components/schemas/ErrorResponse"
+        "401":
+          description: Unauthorized request
+          content:
+            application/json:
+              schema:
+                "$ref": "#/components/schemas/ErrorResponse"
+        "404":
+          description: Resource not found
+          content:
+            application/json:
+              schema:
+                "$ref": "#/components/schemas/ErrorResponse"
+      tags:
+        - tasks
+      security:
+        - secretKey: []
+      x-codeSamples:
+        - lang: typescript
+          source: |-
+            import { task } from "@trigger.dev/sdk/v3";
+
+            export const myTask = await task({
+              id: "my-task",
+              run: async (payload: { message: string }) => {
+                console.log("Hello, world!");
+              }
+            });
+
+            // Somewhere else in your code
+            await myTask.trigger({ message: "Hello, world!" }, {
+              idempotencyKey: "unique-key-123",
+              concurrencyKey: "user123-task",
+              queue: {
+                name: "my-task-queue",
+                concurrencyLimit: 5
+              },
+            });
+        - lang: curl
+          source: |-
+            curl -X POST "https://api.trigger.dev/api/v1/tasks/my-task/trigger" \
+              -H "Content-Type: application/json" \
+              -H "Authorization: Bearer tr_dev_1234" \
+              -d '{
+                    "payload": {
+                      "message": "Hello, world!"
+                    },
+                    "context": {
+                      "user": "user123"
+                    },
+                    "options": {
+                      "queue": {
+                        "name": "default",
+                        "concurrencyLimit": 5
+                      },
+                      "concurrencyKey": "user123-task",
+                      "idempotencyKey": "unique-key-123"
+                    }
+                  }'
+        - lang: python
+          source: |-
+            import requests
+
+            url = "https://api.trigger.dev/api/v1/tasks/my-task/trigger"
+            headers = {
+                "Content-Type": "application/json",
+                "Authorization": "Bearer tr_dev_1234"
+            }
+            data = {
+                "payload": {
+                    "message": "Hello, world!"
+                },
+                "context": {
+                    "user": "user123"
+                },
+                "options": {
+                    "queue": {
+                        "name": "default",
+                        "concurrencyLimit": 5
+                    },
+                    "concurrencyKey": "user123-task",
+                    "idempotencyKey": "unique-key-123"
+                }
+            }
+
+            response = requests.post(url, headers=headers, json=data)
+            print(response.json())
+
+  "/api/v1/tasks/{taskIdentifier}/batch":
+    parameters:
+      - $ref: "#/components/parameters/taskIdentifier"
+    post:
+      operationId: batch_trigger_task_v1
+      summary: Batch trigger a task
+      description: Batch trigger a task with up to 100 payloads.
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              "$ref": "#/components/schemas/BatchTriggerRequestBody"
+      responses:
+        "200":
+          description: Task batch triggered successfully
+          content:
+            application/json:
+              schema:
+                "$ref": "#/components/schemas/BatchTriggerTaskResponse"
+        "400":
+          description: Invalid request parameters or body
+          content:
+            application/json:
+              schema:
+                "$ref": "#/components/schemas/ErrorResponse"
+        "401":
+          description: Unauthorized request
+          content:
+            application/json:
+              schema:
+                "$ref": "#/components/schemas/ErrorResponse"
+        "404":
+          description: Resource not found
+          content:
+            application/json:
+              schema:
+                "$ref": "#/components/schemas/ErrorResponse"
+      tags:
+        - tasks
+      security:
+        - secretKey: []
+      x-codeSamples:
+        - lang: typescript
+          source: |-
+            import { task } from "@trigger.dev/sdk/v3";
+
+            export const myTask = await task({
+              id: "my-task",
+              run: async (payload: { message: string }) => {
+                console.log("Hello, world!");
+              }
+            });
+
+            // Somewhere else in your code
+            await myTask.batchTrigger({
+              items: [
+                {
+                  payload: { message: "Hello, world!" },
+                  options: {
+                    idempotencyKey: "unique-key-123",
+                    concurrencyKey: "user-123-task",
+                    queue: {
+                      name: "my-task-queue",
+                      concurrencyLimit: 5
+                    }
+                  }
+                }
+              ]
+            });
+        - lang: curl
+          source: |-
+            curl -X POST "https://api.trigger.dev/api/v1/tasks/my-task/batch" \
+              -H "Content-Type: application/json" \
+              -H "Authorization: Bearer tr_dev_1234" \
+              -d '{
+                    "items": [
+                      {
+                        "payload": {
+                          "message": "Hello, world!"
+                        },
+                        "context": {
+                          "user": "user123"
+                        },
+                        "options": {
+                          "queue": {
+                            "name": "default",
+                            "concurrencyLimit": 5
+                          },
+                          "concurrencyKey": "user123-task",
+                          "idempotencyKey": "unique-key-123"
+                        }
+                      }
+                    ]
+                  }'
+
+
+
+
 
 components:
   parameters:
+    taskIdentifier:
+      in: path
+      name: taskIdentifier
+      required: true
+      schema:
+        type: string
+      description: The id of a task
+      example: my-task
     runsFilterWithEnv:
       in: query
       name: filter
@@ -1215,6 +1445,65 @@ components:
         configure({ secretKey: "tr_pat_1234" });
         ```
   schemas:
+    TriggerTaskResponse:
+      type: object
+      properties:
+        id:
+          type: string
+          description: The ID of the run that was triggered.
+          example: run_1234
+    QueueOptions:
+      type: object
+      properties:
+        name:
+          type: string
+          description: You can define a shared queue and then pass the name in to your task.
+        concurrencyLimit:
+          type: integer
+          minimum: 0
+          maximum: 1000
+          description: An optional property that specifies the maximum number of concurrent run executions. If this property is omitted, the task can potentially use up the full concurrency of an environment.
+    BatchTriggerRequestBody:
+      type: object
+      properties:
+        items:
+          type: array
+          items:
+            "$ref": "#/components/schemas/TriggerTaskRequestBody"
+          description: An array of payloads to trigger the task with
+      required: ["items"]
+    BatchTriggerTaskResponse:
+      type: object
+      required: ["batchId", "runs"]
+      properties:
+        batchId:
+          type: string
+          description: The ID of the batch that was triggered
+          example: batch_1234
+        runs:
+          type: array
+          items:
+            type: string
+          description: An array of run IDs that were triggered
+    TriggerTaskRequestBody:
+      type: object
+      properties:
+        payload:
+          description: The payload can include any valid JSON
+        context:
+          description: The context can include any valid JSON
+        options:
+          type: object
+          properties:
+            queue:
+              $ref: '#/components/schemas/QueueOptions'
+            concurrencyKey:
+              type: string
+              description: Scope the concurrency limit to a specific key.
+            idempotencyKey:
+              type: string
+              description: An optional property that specifies the idempotency key used to prevent creating duplicate runs. If you provide an existing idempotency key, we will return the existing run ID.
+      
     EnvFilter:
       type: object
       properties:
@@ -1406,6 +1695,7 @@ components:
       properties:
         error:
           type: string
+          example: Something went wrong
       required: ["error"]
     ErrorWithDetailsResponse:
       type: object
@@ -1457,6 +1747,18 @@ components:
           type: string
           example: slack_123456
       required: ["name", "value"]
+    RescheduleRunRequestBody:
+      type: object
+      properties:
+        delay:
+          oneOf:
+            - type: string
+              description: The duration to delay the run by. The duration should be in the format of `1d`, `6h`, `10m`, `11s`, etc.
+              example: 1hr
+            - type: string
+              format: date-time
+              description: The Date to delay the run until, e.g. `new Date()` or `"2024-06-25T15:45:26Z"`
+              example: 2024-06-25T15:45:26Z
     RetrieveRunResponse:
       type: object
       required:
@@ -1475,6 +1777,7 @@ components:
           type: string
           description: The status of the run
           enum:
+            - DELAYED
             - WAITING_FOR_DEPLOY
             - QUEUED
             - EXECUTING
@@ -1498,10 +1801,18 @@ components:
           type: object
           description: The payload that was sent to the task. Will be omitted if the request was made with a Public API key
           example: { "foo": "bar" }
+        payloadPresignedUrl:
+          type: string
+          description: The presigned URL to download the payload. Will only be included if the payload is too large to be included in the response. Expires in 5 minutes.
+          example: "https://r2.cloudflarestorage.com/packets/yubjwjsfkxnylobaqvqz/dev/run_p4omhh45hgxxnq1re6ovy/payload.json?X-Amz-Expires=300&X-Amz-Date=20240625T154526Z&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=10b064e58a0680db5b5e077be2be3b2a%2F20240625%2Fauto%2Fs3%2Faws4_request&X-Amz-SignedHeaders=host&X-Amz-Signature=88604cb993ffc151b4d73f2439da431d9928488e4b3dcfa4a7c8f1819"
         output:
           type: object
           description: The output of the run. Will be omitted if the request was made with a Public API key
           example: { "foo": "bar" }
+        outputPresignedUrl:
+          type: string
+          description: The presigned URL to download the output. Will only be included if the output is too large to be included in the response. Expires in 5 minutes.
+          example: "https://r2.cloudflarestorage.com/packets/yubjwjsfkxnylobaqvqz/dev/run_p4omhh45hgxxnq1re6ovy/payload.json?X-Amz-Expires=300&X-Amz-Date=20240625T154526Z&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=10b064e58a0680db5b5e077be2be3b2a%2F20240625%2Fauto%2Fs3%2Faws4_request&X-Amz-SignedHeaders=host&X-Amz-Signature=88604cb993ffc151b4d73f2439da431d9928488e4b3dcfa4a7c8f1819"
         idempotencyKey:
           type: string
           description: The idempotency key used to prevent creating duplicate runs, if provided
@@ -1524,6 +1835,10 @@ components:
           type: string
           format: date-time
           description: The time the run finished
+        delayedUntil:
+          type: string
+          format: date-time
+          description: If the run was triggered with a delay, this will be the time the run will be enqueued to execute
         schedule:
           type: object
           description: The schedule that triggered the run. Will be omitted if the run was not triggered by a schedule

docs/v3/introduction.mdx

@@ -15,7 +15,7 @@ Trigger.dev v3 makes it easy to write reliable long-running tasks without timeou
 - We run your tasks with no timeouts. You don't have to manage any infrastructure (unless you [self-host](/v3/open-source-self-hosting)). Workers are automatically scaled and managed for you.
 - We provide a multi-tenant queue that is used when triggering tasks.
 - We provide an SDK and CLI for writing tasks in your existing codebase, inside [/trigger folders](/v3/trigger-folder).
-- We provide different types of tasks: [regular](/v3/tasks-regular), [scheduled](/v3/tasks-scheduled), [zod](/v3/tasks-zod), [webhooks](/v3/tasks-webhooks).
+- We provide different types of tasks: [regular](/v3/tasks-regular) and [scheduled](/v3/tasks-scheduled).
 - We provide a dashboard for monitoring, debugging, and managing your tasks.
 
 We're [open source](https://github.com/triggerdotdev/trigger.dev) and you can choose to use the [Trigger.dev Cloud](https://cloud.trigger.dev) or [Self-host Trigger.dev](/v3/open-source-self-hosting) on your own infrastructure.

docs/v3/limits.mdx

@@ -37,3 +37,13 @@ If you add them dynamically using code make sure you add a `deduplicationKey` so
 If you're creating schedules for your user you will definitely need to request more schedules from us.
 
 <Snippet file="v3/soft-limit.mdx" />
+
+## Task payloads and outputs
+
+| Limit                  | Details                                        |
+| ---------------------- | ---------------------------------------------- |
+| Single trigger payload | Must not exceed 10MB                           |
+| Batch trigger payload  | The total of all payloads must not exceed 10MB |
+| Task outputs           | Must not exceed 10MB                           |
+
+Payloads and outputs that exceed 512KB will be offloaded to object storage and a presigned URL will be provided to download the data when calling `runs.retrieve`. You don't need to do anything to handle this in your tasks however, as we will transparently upload/download these during operation.

docs/v3/management/overview.mdx

@@ -94,6 +94,8 @@ function personalAccessTokenExample() {
 
 | Endpoint               | Secret key | Personal Access Token |
 | ---------------------- | ---------- | --------------------- |
+| `task.trigger`         | ✅         |                       |
+| `task.batchTrigger`    | ✅         |                       |
 | `runs.list`            | ✅         | ✅                    |
 | `runs.retrieve`        | ✅         |                       |
 | `runs.cancel`          | ✅         |                       |

docs/v3/management/runs/reschedule.mdx

@@ -0,0 +1,4 @@
+---
+title: "Reschedule run"
+openapi: "v3-openapi POST /api/v1/runs/{runId}/reschedule"
+---

docs/v3/management/tasks/batch-trigger.mdx

@@ -0,0 +1,4 @@
+---
+title: "Batch trigger"
+openapi: "v3-openapi POST /api/v1/tasks/{taskIdentifier}/batch"
+---

docs/v3/management/tasks/trigger.mdx

@@ -0,0 +1,4 @@
+---
+title: "Trigger"
+openapi: "v3-openapi POST /api/v1/tasks/{taskIdentifier}/trigger"
+---

docs/v3/tasks-overview.mdx

@@ -4,7 +4,7 @@ sidebarTitle: "Tasks"
 description: "Tasks are functions that can run for a long time and provide strong resilience to failure."
 ---
 
-There are different types of tasks including [regular tasks](/v3/tasks-regular), [scheduled tasks](/v3/tasks-scheduled), [zod tasks](/v3/tasks-zod) and [webhook tasks](/v3/tasks-webhooks).
+There are different types of tasks including [regular tasks](/v3/tasks-regular) and [scheduled tasks](/v3/tasks-scheduled).
 
 ## Hello world task and how to trigger it
 

docs/v3/triggering.mdx

@@ -3,7 +3,7 @@ title: "Triggering"
 description: "Tasks need to be triggered to run."
 ---
 
-There are currently four ways you can trigger any task from your own code:
+There are currently six ways you can trigger tasks:
 
 | Function                         | Where does this work? | What it does                                                                                                                       |
 | -------------------------------- | --------------------- | ---------------------------------------------------------------------------------------------------------------------------------- |
@@ -11,6 +11,8 @@ There are currently four ways you can trigger any task from your own code:
 | `yourTask.batchTrigger()`        | Anywhere              | Triggers a task multiple times and gets a handle you can use to monitor and manage the runs. It does not wait for the results.     |
 | `yourTask.triggerAndWait()`      | Inside a task         | Triggers a task and then waits until it's complete. You get the result data to continue with.                                      |
 | `yourTask.batchTriggerAndWait()` | Inside a task         | Triggers a task multiple times in parallel and then waits until they're all complete. You get the resulting data to continue with. |
+| `tasks.trigger()`                | Outside of a task     | Triggers a task and gets a handle you can use to fetch and manage the run.                                                         |
+| `tasks.batchTrigger()`           | Outside of a task     | Triggers a task multiple times and gets a handle you can use to fetch and manage the runs.                                         |
 
 Additionally, [scheduled tasks](/v3/tasks-scheduled) get automatically triggered on their schedule and [webhooks](/v3/tasks-webhooks) when receiving a webhook.
 
@@ -18,25 +20,20 @@ Additionally, [scheduled tasks](/v3/tasks-scheduled) get automatically triggered
 
 You should attach one or more schedules to your `schedules.task()` to trigger it on a recurring schedule. [Read the scheduled tasks docs](/v3/tasks-scheduled).
 
-## From outside of a task
-
-You can trigger any task from your backend code, using either `trigger()` or `batchTrigger()`.
-
-<Note>
-  Do not trigger tasks directly from your frontend. If you do, you will leak your private
-  Trigger.dev API key to the world.
-</Note>
-
-You can use Next.js Server Actions but [you need to be careful with bundling](#next-js-server-actions).
-
-### Authentication
+## Authentication
 
 When you trigger a task from your backend code, you need to set the `TRIGGER_SECRET_KEY` environment variable. You can find the value on the API keys page in the Trigger.dev dashboard. [More info on API keys](/v3/apikeys).
 
-### trigger()
+## Task instance methods
+
+Task instance methods are available on the `Task` object you receive when you define a task. They can be called from your backend code or from inside another task.
+
+### Task.trigger()
 
 Triggers a single run of a task with the payload you pass in, and any options you specify. It does NOT wait for the result, you cannot do that from outside a task.
 
+If called from within a task, you can use the `AndWait` version to pause execution until the triggered run is complete.
+
 <CodeGroup>
 
 ```ts Next.js API route
@@ -74,11 +71,24 @@ export async function action({ request, params }: ActionFunctionArgs) {
 }
 ```
 
+```ts /trigger/my-task.ts
+import { myOtherTask } from "~/trigger/my-other-task";
+
+export const myTask = task({
+  id: "my-task",
+  run: async (payload: string) => {
+    const handle = await myOtherTask.trigger("some data");
+
+    //...do other stuff
+  },
+});
+```
+
 </CodeGroup>
 
-### batchTrigger()
+### Task.batchTrigger()
 
-Triggers multiples runs of a task with the payloads you pass in, and any options you specify. It does NOT wait for the results, you cannot do that from outside a task.
+Triggers multiples runs of a task with the payloads you pass in, and any options you specify.
 
 <CodeGroup>
 
@@ -121,33 +131,6 @@ export async function action({ request, params }: ActionFunctionArgs) {
 }
 ```
 
-</CodeGroup>
-
-## From inside a task
-
-You can trigger tasks from other tasks using `trigger()` or `batchTrigger()`. You can also trigger and wait for the result of triggered tasks using `triggerAndWait()` and `batchTriggerAndWait()`. This is a powerful way to build complex tasks.
-
-### trigger()
-
-This works the same as from outside a task. You call it and you get a handle back, but it does not wait for the result.
-
-```ts /trigger/my-task.ts
-import { myOtherTask } from "~/trigger/my-other-task";
-
-export const myTask = task({
-  id: "my-task",
-  run: async (payload: string) => {
-    const handle = await myOtherTask.trigger("some data");
-
-    //...do other stuff
-  },
-});
-```
-
-### batchTrigger()
-
-This works the same as from outside a task. You call it and you get a handle back, but it does not wait for the results.
-
 ```ts /trigger/my-task.ts
 import { myOtherTask } from "~/trigger/my-other-task";
 
@@ -161,7 +144,9 @@ export const myTask = task({
 });
 ```
 
-### triggerAndWait()
+</CodeGroup>
+
+### Task.triggerAndWait()
 
 This is where it gets interesting. You can trigger a task and then wait for the result. This is useful when you need to call a different task and then use the result to continue with your task.
 
@@ -219,7 +204,7 @@ export const parentTask = task({
 });
 ```
 
-### batchTriggerAndWait()
+### Task.batchTriggerAndWait()
 
 You can batch trigger a task and wait for all the results. This is useful for the fan-out pattern, where you need to call a task multiple times and then wait for all the results to continue with your task.
 
@@ -284,11 +269,213 @@ export const batchParentTask = task({
 });
 ```
 
+## SDK functions
+
+You can trigger any task from your backend code using the `tasks.trigger()` or `tasks.batchTrigger()` SDK functions.
+
+<Note>
+  Do not trigger tasks directly from your frontend. If you do, you will leak your private
+  Trigger.dev API key.
+</Note>
+
+You can use Next.js Server Actions but [you need to be careful with bundling](#next-js-server-actions).
+
+### tasks.trigger()
+
+Triggers a single run of a task with the payload you pass in, and any options you specify, without needing to import the task.
+
+<Note>
+  Why would you use this instead of the `Task.trigger()` instance method? Tasks can import
+  dependencies/modules that you might not want included in your application code or cause problems
+  with building.
+</Note>
+
+<CodeGroup>
+
+```ts Next.js API route
+import { tasks } from "@trigger.dev/sdk/v3";
+import type { emailSequence } from "~/trigger/emails";
+//     👆 **type-only** import
+
+//app/email/route.ts
+export async function POST(request: Request) {
+  //get the JSON from the request
+  const data = await request.json();
+
+  // Pass the task type to `trigger()` as a generic argument, giving you full type checking
+  const handle = await tasks.trigger<typeof emailSequence>("email-sequence", {
+    to: data.email,
+    name: data.name,
+  });
+
+  //return a success response with the handle
+  return Response.json(handle);
+}
+```
+
+```ts Remix
+import { tasks } from "@trigger.dev/sdk/v3";
+
+export async function action({ request, params }: ActionFunctionArgs) {
+  if (request.method.toUpperCase() !== "POST") {
+    return json("Method Not Allowed", { status: 405 });
+  }
+
+  //get the JSON from the request
+  const data = await request.json();
+
+  // The generic argument is optional, but recommended for full type checking
+  const handle = await tasks.trigger("email-sequence", {
+    to: data.email,
+    name: data.name,
+  });
+
+  //return a success response with the handle
+  return json(handle);
+}
+```
+
+</CodeGroup>
+
+<Tip>
+  By importing the task with the type modifier, the import of `"~/trigger/emails"` is a type-only
+  import. This means that the task code is not included in your application at build time.
+</Tip>
+
+### tasks.batchTrigger()
+
+Triggers multiples runs of a task with the payloads you pass in, and any options you specify, without needing to import the task.
+
+<CodeGroup>
+
+```ts Next.js API route
+import { tasks } from "@trigger.dev/sdk/v3";
+import type { emailSequence } from "~/trigger/emails";
+//     👆 **type-only** import
+
+//app/email/route.ts
+export async function POST(request: Request) {
+  //get the JSON from the request
+  const data = await request.json();
+
+  // Pass the task type to `batchTrigger()` as a generic argument, giving you full type checking
+  const batchHandle = await tasks.batchTrigger<typeof emailSequence>(
+    "email-sequence",
+    data.users.map((u) => ({ payload: { to: u.email, name: u.name } }))
+  );
+
+  //return a success response with the handle
+  return Response.json(batchHandle);
+}
+```
+
+</CodeGroup>
+
+### tasks.triggerAndPoll()
+
+Triggers a single run of a task with the payload you pass in, and any options you specify, and then polls the run until it's complete.
+
+<CodeGroup>
+
+```ts Next.js API route
+import { tasks } from "@trigger.dev/sdk/v3";
+import type { emailSequence } from "~/trigger/emails";
+
+//app/email/route.ts
+export async function POST(request: Request) {
+  //get the JSON from the request
+  const data = await request.json();
+
+  // Pass the task type to `triggerAndPoll()` as a generic argument, giving you full type checking
+  const result = await tasks.triggerAndPoll<typeof emailSequence>(
+    "email-sequence",
+    {
+      to: data.email,
+      name: data.name,
+    },
+    { pollIntervalMs: 5000 }
+  );
+
+  //return a success response with the result
+  return Response.json(result);
+}
+```
+
+</CodeGroup>
+
+<Note>
+  The above code is just a demonstration of the API and is not recommended to use in an API route
+  this way as it will block the request until the task is complete.
+</Note>
+
+### runs.retrieve()
+
+You can retrieve a run by its handle using the `runs.retrieve()` function.
+
+<CodeGroup>
+
+```ts Next.js API route
+import { tasks, runs } from "@trigger.dev/sdk/v3";
+import type { emailSequence } from "~/trigger/emails";
+//     👆 **type-only** import
+
+//app/email/route.ts
+export async function POST(request: Request) {
+  //get the JSON from the request
+  const data = await request.json();
+
+  // Pass the task type to `trigger()` as a generic argument, giving you full type checking
+  const handle = await tasks.trigger<typeof emailSequence>("email-sequence", {
+    to: data.email,
+    name: data.name,
+  });
+
+  const run = await runs.retrieve(handle);
+
+  // run.output will be correctly typed as the return value of the task
+  return Response.json(run.output);
+}
+```
+
+</CodeGroup>
+
+### runs.poll()
+
+You can poll a run by its handle using the `runs.poll()` function.
+
+<CodeGroup>
+
+```ts Next.js API route
+import { tasks, runs } from "@trigger.dev/sdk/v3";
+import type { emailSequence } from "~/trigger/emails";
+//     👆 **type-only** import
+
+//app/email/route.ts
+export async function POST(request: Request) {
+  //get the JSON from the request
+  const data = await request.json();
+
+  // Pass the task type to `trigger()` as a generic argument, giving you full type checking
+  const handle = await tasks.trigger<typeof emailSequence>("email-sequence", {
+    to: data.email,
+    name: data.name,
+  });
+
+  // Poll the run until it's complete
+  const run = await runs.poll(handle, { pollIntervalMs: 5000 });
+
+  // run.output will be correctly typed as the return value of the task
+  return Response.json(run.output);
+}
+```
+
+</CodeGroup>
+
 ## Next.js Server Actions
 
 Server Actions allow you to call your backend code without creating API routes. This is very useful for triggering tasks but you need to be careful you don't accidentally bundle the Trigger.dev SDK into your frontend code.
 
-If you see an error like this then you've bundled `@trigger.dev/sdk` into your frontend code:
+If you see an error like this then you've bundled `@trigger.dev/sdk/v3` into your frontend code:
 
 ```bash
 Module build failed: UnhandledSchemeError: Reading from "node:crypto" is not handled by plugins (Unhandled scheme).
@@ -297,7 +484,7 @@ Webpack supports "data:" and "file:" URIs by default.
 You may need an additional plugin to handle "node:" URIs.
 ```
 
-When you use server actions that use `@trigger.dev/sdk`:
+When you use server actions that use `@trigger.dev/sdk/v3`:
 
 - The file can't have any React components in it.
 - The file should have `"use server"` on the first line.
@@ -349,3 +536,177 @@ export async function create() {
 ```
 
 </CodeGroup>
+
+## Large Payloads
+
+We recommend keeping your task payloads as small as possible. We currently have a hard limit on task payloads above 10MB.
+
+If your payload size is larger than 512KB, instead of saving the payload to the database, we will upload it to an S3-compatible object store and store the URL in the database.
+
+When your task runs, we automatically download the payload from the object store and pass it to your task function. We also will return to you a `payloadPresignedUrl` from the `runs.retrieve` SDK function so you can download the payload if needed:
+
+```ts
+import { runs } from "@trigger.dev/sdk/v3";
+
+const run = await runs.retrieve(handle);
+
+if (run.payloadPresignedUrl) {
+  const response = await fetch(run.payloadPresignedUrl);
+  const payload = await response.json();
+
+  console.log("Payload", payload);
+}
+```
+
+<Note>
+  We also use this same system for dealing with large task outputs, and subsequently will return a
+  corresponding `outputPresignedUrl`. Task outputs are limited to 100MB.
+</Note>
+
+If you need to pass larger payloads, you'll need to upload the payload to your own storage and pass a URL to the file in the payload instead. For example, uploading to S3 and then sending a presigned URL that expires in URL:
+
+<CodeGroup>
+
+```ts /yourServer.ts
+import { myTask } from "./trigger/myTasks";
+import { s3Client, getSignedUrl, PutObjectCommand, GetObjectCommand } from "./s3";
+import { createReadStream } from "node:fs";
+
+// Upload file to S3
+await s3Client.send(
+  new PutObjectCommand({
+    Bucket: "my-bucket",
+    Key: "myfile.json",
+    Body: createReadStream("large-payload.json"),
+  })
+);
+
+// Create presigned URL
+const presignedUrl = await getSignedUrl(
+  s3Client,
+  new GetObjectCommand({
+    Bucket: "my-bucket",
+    Key: "my-file.json",
+  }),
+  {
+    expiresIn: 3600, // expires in 1 hour
+  }
+);
+
+// Now send the URL to the task
+const handle = await myTask.trigger({
+  url: presignedUrl,
+});
+```
+
+```ts /trigger/myTasks.ts
+import { task } from "@trigger.dev/sdk/v3";
+
+export const myTask = task({
+  id: "my-task",
+  run: async (payload: { url: string }) => {
+    // Download the file from the URL
+    const response = await fetch(payload.url);
+    const data = await response.json();
+
+    // Do something with the data
+  },
+});
+```
+
+</CodeGroup>
+
+### Batch Triggering
+
+When using `batchTrigger` or `batchTriggerAndWait`, the total size of all payloads cannot exceed 10MB. This means if you are doing a batch of 100 runs, each payload should be less than 100KB.
+
+## Delayed runs
+
+When you want to trigger a task now, but have it run at a later time, you can use the `delay` option:
+
+```ts
+// Delay the task run by 1 hour
+await myTask.trigger({ some: "data" }, { delay: "1h" });
+// Delay the task run by 88 seconds
+await myTask.trigger({ some: "data" }, { delay: "88s" });
+// Delay the task run by 1 hour and 52 minutes and 18 seconds
+await myTask.trigger({ some: "data" }, { delay: "1h52m18s" });
+// Delay until a specific time
+await myTask.trigger({ some: "data" }, { delay: "2024-12-01T00:00:00" });
+// Delay using a Date object
+await myTask.trigger({ some: "data" }, { delay: new Date(Date.now() + 1000 * 60 * 60) });
+```
+
+Runs that are delayed and have not been enqueued yet will display in the dashboard with a "Delayed" status:
+
+![Delayed run in the dashboard](/images/v3/delayed-runs.png)
+
+<Note>
+  Delayed runs will be enqueued at the time specified, and will run as soon as possible after that
+  time, just as a normally triggered run would.
+</Note>
+
+You can cancel a delayed run using the `runs.cancel` SDK function:
+
+```ts
+import { runs } from "@trigger.dev/sdk/v3";
+
+await runs.cancel("run_1234");
+```
+
+You can also reschedule a delayed run using the `runs.reschedule` SDK function:
+
+```ts
+import { runs } from "@trigger.dev/sdk/v3";
+
+// The delay option here takes the same format as the trigger delay option
+await runs.reschedule("run_1234", { delay: "1h" });
+```
+
+The `delay` option is also available when using `batchTrigger`:
+
+```ts
+await myTask.batchTrigger([{ payload: { some: "data" }, options: { delay: "1h" } }]);
+```
+
+## TTL
+
+You can set a TTL (time to live) when triggering a task, which will automatically expire the run if it hasn't started within the specified time. This is useful for ensuring that a run doesn't get stuck in the queue for too long.
+
+<Note>
+  All runs in development have a default `ttl` of 10 minutes. You can disable this by setting the
+  `ttl` option.
+</Note>
+
+```ts
+import { myTask } from "./trigger/myTasks";
+
+// Expire the run if it hasn't started within 1 hour
+await myTask.trigger({ some: "data" }, { ttl: "1h" });
+
+// If you specify a number, it will be treated as seconds
+await myTask.trigger({ some: "data" }, { ttl: 3600 }); // 1 hour
+```
+
+When a run is expired, it will be marked as "Expired" in the dashboard:
+
+![Expired runs in the dashboard](/images/v3/expired-runs.png)
+
+### Delayed runs and TTL
+
+When you use both `delay` and `ttl`, the TTL will start counting down from the time the run is enqueued, not from the time the run is triggered.
+
+So for example, when using the following code:
+
+```ts
+await myTask.trigger({ some: "data" }, { delay: "10m", ttl: "1h" });
+```
+
+The timeline would look like this:
+
+1. The run is created at 12:00:00
+2. The run is enqueued at 12:10:00
+3. The TTL starts counting down from 12:10:00
+4. If the run hasn't started by 13:10:00, it will be expired
+
+For this reason, the `ttl` option only accepts durations and not absolute timestamps.

integrations/airtable/CHANGELOG.md

@@ -1,5 +1,58 @@
 # @trigger.dev/airtable
 
+## 3.0.0-beta.45
+
+### Patch Changes
+
+- Updated dependencies [0e77e7ef7]
+  - @trigger.dev/sdk@3.0.0-beta.45
+  - @trigger.dev/integration-kit@3.0.0-beta.45
+
+## 3.0.0-beta.44
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.44
+- @trigger.dev/sdk@3.0.0-beta.44
+
+## 3.0.0-beta.43
+
+### Patch Changes
+
+- Updated dependencies [34ca7667d]
+  - @trigger.dev/sdk@3.0.0-beta.43
+  - @trigger.dev/integration-kit@3.0.0-beta.43
+
+## 3.0.0-beta.42
+
+### Patch Changes
+
+- Updated dependencies [ecef19966]
+  - @trigger.dev/sdk@3.0.0-beta.42
+  - @trigger.dev/integration-kit@3.0.0-beta.42
+
+## 3.0.0-beta.41
+
+### Patch Changes
+
+- Updated dependencies [7c36a1a4b]
+  - @trigger.dev/sdk@3.0.0-beta.41
+  - @trigger.dev/integration-kit@3.0.0-beta.41
+
+## 3.0.0-beta.40
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.40
+- @trigger.dev/sdk@3.0.0-beta.40
+
+## 3.0.0-beta.39
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.39
+- @trigger.dev/sdk@3.0.0-beta.39
+
 ## 3.0.0-beta.38
 
 ### Patch Changes

integrations/airtable/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/airtable",
-  "version": "3.0.0-beta.38",
+  "version": "3.0.0-beta.45",
   "description": "Trigger.dev integration for airtable",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -25,8 +25,8 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.38",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.38",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.45",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.45",
     "airtable": "^0.12.1",
     "zod": "3.22.3"
   },