Latest lockfile

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.changeset/few-students-share.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix permissions inside node_modules

.changeset/late-icons-lie.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Display errors for runs and deployments

.changeset/light-dragons-complain.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Correctly handle self-hosted deploy command errors

.changeset/loud-actors-remember.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Default to retrying enabled in dev when running init

.changeset/new-rivers-tell.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Added replayRun function to the SDK

.changeset/polite-ducks-switch.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix post start hooks

.changeset/pre.json

@@ -49,19 +49,29 @@
     "clean-pianos-listen",
     "cool-glasses-bake",
     "eight-pumas-float",
+    "few-students-share",
     "green-bags-wink",
     "khaki-apricots-design",
+    "late-icons-lie",
     "lemon-jobs-repair",
     "light-bulldogs-press",
+    "light-dragons-complain",
+    "loud-actors-remember",
     "many-ligers-pump",
     "mighty-camels-joke",
+    "new-rivers-tell",
     "odd-poets-own",
+    "polite-ducks-switch",
+    "rare-roses-float",
     "real-planets-stare",
     "sharp-zebras-serve",
     "smart-olives-eat",
+    "spicy-lamps-smoke",
     "strange-ghosts-matter",
     "stupid-bulldogs-applaud",
     "sweet-lizards-press",
-    "tall-bees-wave"
+    "tall-bees-wave",
+    "tame-guests-know",
+    "tiny-doors-type"
   ]
 }

.changeset/rare-roses-float.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Add openssl to prod worker image and allow passing auth token via env var for deploy

.changeset/spicy-lamps-smoke.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fixing an issue with bundling @trigger.dev/core/v3 in dev when using pnpm

.changeset/tame-guests-know.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Changed the binary name from trigger.dev to triggerdev to fix a Windows issue

.changeset/tiny-doors-type.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Added cancelRun to the SDK

.vscode/launch.json

@@ -33,7 +33,7 @@
       "type": "node-terminal",
       "request": "launch",
       "name": "Debug V3 Dev CLI",
-      "command": "pnpm exec trigger.dev dev --log-level debug",
+      "command": "pnpm exec triggerdev dev --log-level debug",
       "cwd": "${workspaceFolder}/references/v3-catalog",
       "sourceMaps": true
     },
@@ -41,7 +41,7 @@
       "type": "node-terminal",
       "request": "launch",
       "name": "Debug V3 Deploy CLI",
-      "command": "pnpm exec trigger.dev deploy",
+      "command": "pnpm exec triggerdev deploy",
       "cwd": "${workspaceFolder}/references/v3-catalog",
       "sourceMaps": true
     },

apps/kubernetes-provider/package.json

@@ -19,6 +19,7 @@
     "@kubernetes/client-node": "^0.20.0",
     "@trigger.dev/core": "workspace:*",
     "@trigger.dev/core-apps": "workspace:*",
+    "p-queue": "^8.0.1",
     "socket.io-client": "^4.7.4"
   },
   "devDependencies": {

apps/kubernetes-provider/src/index.ts

@@ -9,6 +9,7 @@ import {
 } from "@trigger.dev/core-apps";
 import { Machine, PostStartCauses, PreStopCauses, EnvironmentType } from "@trigger.dev/core/v3";
 import { randomUUID } from "crypto";
+import { TaskMonitor } from "./taskMonitor";
 
 const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
 const NODE_NAME = process.env.NODE_NAME || "local";
@@ -24,6 +25,10 @@ type Namespace = {
   };
 };
 
+type ComputeResources = {
+  [K in "cpu" | "memory" | "ephemeral-storage"]?: string;
+};
+
 class KubernetesTaskOperations implements TaskOperations {
   #namespace: Namespace;
   #k8sApi: {
@@ -55,25 +60,15 @@ class KubernetesTaskOperations implements TaskOperations {
           template: {
             metadata: {
               labels: {
+                ...this.#getSharedLabels(opts),
                 app: "task-index",
                 "app.kubernetes.io/part-of": "trigger-worker",
                 "app.kubernetes.io/component": "index",
-                env: opts.envId,
-                envtype: this.#envTypeToLabelValue(opts.envType),
-                org: opts.orgId,
-                project: opts.projectId,
+                deployment: opts.deploymentId,
               },
             },
             spec: {
-              restartPolicy: "Never",
-              imagePullSecrets: [
-                {
-                  name: "registry-trigger",
-                },
-              ],
-              nodeSelector: {
-                nodetype: "worker",
-              },
+              ...this.#defaultPodSpec,
               containers: [
                 {
                   name: this.#getIndexContainerName(opts.shortCode),
@@ -83,12 +78,13 @@ class KubernetesTaskOperations implements TaskOperations {
                       containerPort: 8000,
                     },
                   ],
-                  // resources: {
-                  //   limits: {
-                  //     cpu: "100m",
-                  //     memory: "50Mi",
-                  //   },
-                  // },
+                  resources: {
+                    limits: {
+                      cpu: "250m",
+                      memory: "0.5G",
+                      "ephemeral-storage": "2Gi",
+                    },
+                  },
                   lifecycle: {
                     preStop: {
                       exec: {
@@ -97,10 +93,7 @@ class KubernetesTaskOperations implements TaskOperations {
                     },
                   },
                   env: [
-                    {
-                      name: "DEBUG",
-                      value: "true",
-                    },
+                    ...this.#getSharedEnv(opts.envId),
                     {
                       name: "INDEX_TASKS",
                       value: "true",
@@ -113,42 +106,6 @@ class KubernetesTaskOperations implements TaskOperations {
                       name: "TRIGGER_API_URL",
                       value: opts.apiUrl,
                     },
-                    {
-                      name: "TRIGGER_ENV_ID",
-                      value: opts.envId,
-                    },
-                    {
-                      name: "OTEL_EXPORTER_OTLP_ENDPOINT",
-                      value: OTEL_EXPORTER_OTLP_ENDPOINT,
-                    },
-                    {
-                      name: "HTTP_SERVER_PORT",
-                      value: "8000",
-                    },
-                    {
-                      name: "POD_NAME",
-                      valueFrom: {
-                        fieldRef: {
-                          fieldPath: "metadata.name",
-                        },
-                      },
-                    },
-                    {
-                      name: "COORDINATOR_HOST",
-                      valueFrom: {
-                        fieldRef: {
-                          fieldPath: "status.hostIP",
-                        },
-                      },
-                    },
-                    {
-                      name: "MACHINE_NAME",
-                      valueFrom: {
-                        fieldRef: {
-                          fieldPath: "spec.nodeName",
-                        },
-                      },
-                    },
                   ],
                 },
               ],
@@ -167,26 +124,15 @@ class KubernetesTaskOperations implements TaskOperations {
           name: this.#getRunContainerName(opts.runId),
           namespace: this.#namespace.metadata.name,
           labels: {
+            ...this.#getSharedLabels(opts),
             app: "task-run",
             "app.kubernetes.io/part-of": "trigger-worker",
             "app.kubernetes.io/component": "create",
-            env: opts.envId,
-            envtype: this.#envTypeToLabelValue(opts.envType),
-            org: opts.orgId,
-            project: opts.projectId,
             run: opts.runId,
           },
         },
         spec: {
-          restartPolicy: "Never",
-          imagePullSecrets: [
-            {
-              name: "registry-trigger",
-            },
-          ],
-          nodeSelector: {
-            nodetype: "worker",
-          },
+          ...this.#defaultPodSpec,
           containers: [
             {
               name: this.#getRunContainerName(opts.runId),
@@ -197,14 +143,15 @@ class KubernetesTaskOperations implements TaskOperations {
                 },
               ],
               resources: {
-                limits: this.#getResourcesFromMachineConfig(opts.machine),
+                requests: {
+                  ...this.#defaultResourceRequests,
+                },
+                limits: {
+                  ...this.#defaultResourceLimits,
+                  ...this.#getResourcesFromMachineConfig(opts.machine),
+                },
               },
               lifecycle: {
-                postStart: {
-                  exec: {
-                    command: this.#getLifecycleCommand("postStart", "create"),
-                  },
-                },
                 preStop: {
                   exec: {
                     command: this.#getLifecycleCommand("preStop", "terminate"),
@@ -212,54 +159,11 @@ class KubernetesTaskOperations implements TaskOperations {
                 },
               },
               env: [
-                {
-                  name: "DEBUG",
-                  value: "true",
-                },
-                {
-                  name: "HTTP_SERVER_PORT",
-                  value: "8000",
-                },
-                {
-                  name: "TRIGGER_ENV_ID",
-                  value: opts.envId,
-                },
+                ...this.#getSharedEnv(opts.envId),
                 {
                   name: "TRIGGER_RUN_ID",
                   value: opts.runId,
                 },
-                {
-                  name: "TRIGGER_WORKER_VERSION",
-                  value: opts.version,
-                },
-                {
-                  name: "OTEL_EXPORTER_OTLP_ENDPOINT",
-                  value: OTEL_EXPORTER_OTLP_ENDPOINT,
-                },
-                {
-                  name: "POD_NAME",
-                  valueFrom: {
-                    fieldRef: {
-                      fieldPath: "metadata.name",
-                    },
-                  },
-                },
-                {
-                  name: "COORDINATOR_HOST",
-                  valueFrom: {
-                    fieldRef: {
-                      fieldPath: "status.hostIP",
-                    },
-                  },
-                },
-                {
-                  name: "NODE_NAME",
-                  valueFrom: {
-                    fieldRef: {
-                      fieldPath: "spec.nodeName",
-                    },
-                  },
-                },
               ],
               volumeMounts: [
                 {
@@ -288,27 +192,16 @@ class KubernetesTaskOperations implements TaskOperations {
           name: `${this.#getRunContainerName(opts.runId)}-${randomUUID().slice(0, 8)}`,
           namespace: this.#namespace.metadata.name,
           labels: {
+            ...this.#getSharedLabels(opts),
             app: "task-run",
             "app.kubernetes.io/part-of": "trigger-worker",
             "app.kubernetes.io/component": "restore",
-            env: opts.envId,
-            envtype: this.#envTypeToLabelValue(opts.envType),
-            org: opts.orgId,
-            project: opts.projectId,
             run: opts.runId,
             checkpoint: opts.checkpointId,
           },
         },
         spec: {
-          restartPolicy: "Never",
-          imagePullSecrets: [
-            {
-              name: "registry-trigger",
-            },
-          ],
-          nodeSelector: {
-            nodetype: "worker",
-          },
+          ...this.#defaultPodSpec,
           initContainers: [
             {
               name: "pull-base-image",
@@ -348,7 +241,13 @@ class KubernetesTaskOperations implements TaskOperations {
                 },
               ],
               resources: {
-                limits: this.#getResourcesFromMachineConfig(opts.machine),
+                requests: {
+                  ...this.#defaultResourceRequests,
+                },
+                limits: {
+                  ...this.#defaultResourceLimits,
+                  ...this.#getResourcesFromMachineConfig(opts.machine),
+                },
               },
               lifecycle: {
                 postStart: {
@@ -406,7 +305,90 @@ class KubernetesTaskOperations implements TaskOperations {
     }
   }
 
-  #getResourcesFromMachineConfig(config: Machine) {
+  get #defaultPodSpec(): Omit<k8s.V1PodSpec, "containers"> {
+    return {
+      restartPolicy: "Never",
+      automountServiceAccountToken: false,
+      imagePullSecrets: [
+        {
+          name: "registry-trigger",
+        },
+      ],
+      nodeSelector: {
+        nodetype: "worker",
+      },
+    };
+  }
+
+  get #defaultResourceRequests(): ComputeResources {
+    return {
+      "ephemeral-storage": "2Gi",
+    };
+  }
+
+  get #defaultResourceLimits(): ComputeResources {
+    return {
+      "ephemeral-storage": "10Gi",
+    };
+  }
+
+  #getSharedEnv(envId: string): k8s.V1EnvVar[] {
+    return [
+      {
+        name: "TRIGGER_ENV_ID",
+        value: envId,
+      },
+      {
+        name: "DEBUG",
+        value: process.env.DEBUG ? "1" : "0",
+      },
+      {
+        name: "HTTP_SERVER_PORT",
+        value: "8000",
+      },
+      {
+        name: "OTEL_EXPORTER_OTLP_ENDPOINT",
+        value: OTEL_EXPORTER_OTLP_ENDPOINT,
+      },
+      {
+        name: "POD_NAME",
+        valueFrom: {
+          fieldRef: {
+            fieldPath: "metadata.name",
+          },
+        },
+      },
+      {
+        name: "COORDINATOR_HOST",
+        valueFrom: {
+          fieldRef: {
+            fieldPath: "status.hostIP",
+          },
+        },
+      },
+      {
+        name: "MACHINE_NAME",
+        valueFrom: {
+          fieldRef: {
+            fieldPath: "spec.nodeName",
+          },
+        },
+      },
+    ];
+  }
+
+  #getSharedLabels(
+    opts: TaskOperationsIndexOptions | TaskOperationsCreateOptions | TaskOperationsRestoreOptions
+  ): Record<string, string> {
+    return {
+      env: opts.envId,
+      envtype: this.#envTypeToLabelValue(opts.envType),
+      org: opts.orgId,
+      project: opts.projectId,
+    };
+  }
+
+  #getResourcesFromMachineConfig(config: Machine): ComputeResources {
     return {
       cpu: `${config.cpu}`,
       memory: `${config.memory}G`,
@@ -417,7 +399,19 @@ class KubernetesTaskOperations implements TaskOperations {
     type: THookType,
     cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses
   ) {
-    return ["/bin/sh", "-c", `sleep 1; wget -q -O- 127.0.0.1:8000/${type}?cause=${cause}`];
+    const retries = 5;
+
+    // This will retry sending the lifecycle hook up to `retries` times
+    // The sleep is required as this may start running before the HTTP server is up
+    const exec = [
+      "/bin/sh",
+      "-c",
+      `for i in $(seq ${retries}); do sleep 1; busybox wget -q -O- 127.0.0.1:8000/${type}?cause=${cause} && break; done`,
+    ];
+
+    logger.log("getLifecycleCommand()", { exec });
+
+    return exec;
   }
 
   #getIndexContainerName(suffix: string) {
@@ -516,3 +510,34 @@ const provider = new ProviderShell({
 });
 
 provider.listen();
+
+const taskMonitor = new TaskMonitor({
+  runtimeEnv: RUNTIME_ENV,
+  onIndexFailure: async (deploymentId, failureInfo) => {
+    logger.log("Indexing failed", { deploymentId, failureInfo });
+
+    try {
+      provider.platformSocket.send("INDEXING_FAILED", {
+        deploymentId,
+        error: {
+          name: `Crashed with exit code ${failureInfo.exitCode}`,
+          message: failureInfo.reason,
+          stack: failureInfo.logs,
+        },
+      });
+    } catch (error) {
+      logger.error(error);
+    }
+  },
+  onRunFailure: async (runId, failureInfo) => {
+    logger.log("Run failed:", { runId, failureInfo });
+
+    try {
+      provider.platformSocket.send("WORKER_CRASHED", { runId, ...failureInfo });
+    } catch (error) {
+      logger.error(error);
+    }
+  },
+});
+
+taskMonitor.start();

apps/kubernetes-provider/src/taskMonitor.ts

@@ -0,0 +1,436 @@
+import * as k8s from "@kubernetes/client-node";
+import { SimpleLogger } from "@trigger.dev/core-apps";
+import { setTimeout } from "timers/promises";
+import PQueue from "p-queue";
+
+type IndexFailureHandler = (
+  deploymentId: string,
+  failureInfo: {
+    exitCode: number;
+    reason: string;
+    logs: string;
+  }
+) => Promise<any>;
+
+type RunFailureHandler = (
+  runId: string,
+  failureInfo: {
+    exitCode: number;
+    reason: string;
+    logs: string;
+  }
+) => Promise<any>;
+
+type TaskMonitorOptions = {
+  runtimeEnv: "local" | "kubernetes";
+  onIndexFailure?: IndexFailureHandler;
+  onRunFailure?: RunFailureHandler;
+  namespace?: string;
+};
+
+export class TaskMonitor {
+  #enabled = false;
+  #logger = new SimpleLogger("[TaskMonitor]");
+  #taskInformer: ReturnType<typeof k8s.makeInformer<k8s.V1Pod>>;
+  #processedPods = new Map<string, number>();
+  #queue = new PQueue({ concurrency: 10 });
+  #k8sClient: {
+    core: k8s.CoreV1Api;
+    kubeConfig: k8s.KubeConfig;
+  };
+
+  private namespace = "default";
+  private fieldSelector = "status.phase=Failed";
+  private labelSelector = "app in (task-index, task-run)";
+
+  constructor(private opts: TaskMonitorOptions) {
+    this.#k8sClient = this.#createK8sClient();
+
+    this.#taskInformer = this.#createTaskInformer();
+    this.#taskInformer.on("connect", this.#onInformerConnected.bind(this));
+    this.#taskInformer.on("error", this.#onInformerError.bind(this));
+    this.#taskInformer.on("update", this.#enqueueOnPodUpdated.bind(this));
+  }
+
+  #createTaskInformer() {
+    const listTasks = () =>
+      this.#k8sClient.core.listNamespacedPod(
+        this.namespace,
+        undefined,
+        undefined,
+        undefined,
+        this.fieldSelector,
+        this.labelSelector
+      );
+
+    // Uses watch with local caching
+    // https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes
+    const informer = k8s.makeInformer(
+      this.#k8sClient.kubeConfig,
+      `/api/v1/namespaces/${this.namespace}/pods`,
+      listTasks,
+      this.labelSelector,
+      this.fieldSelector
+    );
+
+    return informer;
+  }
+
+  async #onInformerConnected() {
+    this.#logger.log("Connected");
+  }
+
+  async #onInformerError(error: any) {
+    this.#logger.error("Error:", error);
+
+    // Automatic reconnect
+    await setTimeout(2_000);
+    this.#taskInformer.start();
+  }
+
+  #enqueueOnPodUpdated(pod: k8s.V1Pod) {
+    this.#queue.add(async () => {
+      try {
+        // It would be better to only pass the cache key, but the pod may already be removed from the cache by the time we process it
+        await this.#onPodUpdated(pod);
+      } catch (error) {
+        this.#logger.error("Caught onPodUpdated() error:", error);
+      }
+    });
+  }
+
+  async #onPodUpdated(pod: k8s.V1Pod) {
+    this.#logger.debug(`Updated: ${pod.metadata?.name}`);
+    this.#logger.debug("Updated", JSON.stringify(pod, null, 2));
+
+    // We only care about failures
+    if (pod.status?.phase !== "Failed") {
+      return;
+    }
+
+    const podName = pod.metadata?.name;
+
+    if (!podName) {
+      this.#logger.error("Pod is nameless", { pod });
+      return;
+    }
+
+    const containerStatus = pod.status.containerStatuses?.[0];
+
+    if (!containerStatus?.state) {
+      this.#logger.error("Pod failed, but container status doesn't have state", {
+        status: pod.status,
+      });
+      return;
+    }
+
+    if (this.#processedPods.has(podName)) {
+      this.#logger.debug("Pod update already processed", {
+        podName,
+        timestamp: this.#processedPods.get(podName),
+      });
+      return;
+    }
+
+    this.#processedPods.set(podName, Date.now());
+
+    const podStatus = this.#getPodStatusSummary(pod.status);
+    const containerState = this.#getContainerStateSummary(containerStatus.state);
+    const rawLogs = await this.#getLogTail(podName);
+
+    this.#logger.log(`${podName} failed with:`, {
+      podStatus,
+      containerState,
+      rawLogs,
+    });
+
+    const exitCode = containerState.exitCode ?? -1;
+    const rawReason = podStatus.reason ?? containerState.reason ?? "";
+    const message = podStatus.message ?? containerState.message ?? "";
+
+    let reason = rawReason || "Unknown error";
+    let logs = rawLogs || "";
+
+    switch (rawReason) {
+      case "Error":
+        reason = "Unknown error.";
+        break;
+      case "Evicted":
+        if (message.startsWith("Pod ephemeral local storage usage")) {
+          reason = "Storage limit exceeded.";
+        } else if (message) {
+          reason = `Evicted: ${message}`;
+        } else {
+          reason = "Evicted for unknown reason.";
+        }
+
+        if (logs.startsWith("failed to try resolving symlinks")) {
+          logs = "";
+        }
+        break;
+      case "OOMKilled":
+        reason = "Out of memory! Try increasing the memory on this task.";
+        break;
+      default:
+        break;
+    }
+
+    const failureInfo = {
+      exitCode,
+      reason,
+      logs,
+    };
+
+    const app = pod.metadata?.labels?.app;
+
+    switch (app) {
+      case "task-index":
+        const deploymentId = pod.metadata?.labels?.deployment;
+
+        if (!deploymentId) {
+          this.#logger.error("Index is missing ID", { pod });
+          return;
+        }
+
+        if (this.opts.onIndexFailure) {
+          await this.opts.onIndexFailure(deploymentId, failureInfo);
+        }
+        break;
+      case "task-run":
+        const runId = pod.metadata?.labels?.run;
+
+        if (!runId) {
+          this.#logger.error("Run is missing ID", { pod });
+          return;
+        }
+
+        if (this.opts.onRunFailure) {
+          await this.opts.onRunFailure(runId, failureInfo);
+        }
+        break;
+      default:
+        this.#logger.error("Pod has invalid app label", { pod });
+        return;
+    }
+
+    await this.#deletePod(podName);
+  }
+
+  async #getLogTail(podName: string) {
+    try {
+      const logs = await this.#k8sClient.core.readNamespacedPodLog(
+        podName,
+        this.namespace,
+        undefined,
+        undefined,
+        undefined,
+        1024, // limitBytes
+        undefined,
+        undefined,
+        undefined,
+        20 // tailLines
+      );
+
+      const responseBody = logs.body ?? "";
+
+      if (responseBody.startsWith("unable to retrieve container logs")) {
+        return "";
+      }
+
+      // Type is wrong, body may be undefined
+      return responseBody;
+    } catch (error) {
+      this.#logger.error("Log tail error:", error instanceof Error ? error.message : "unknown");
+      return "";
+    }
+  }
+
+  #getPodStatusSummary(status: k8s.V1PodStatus) {
+    return {
+      reason: status.reason,
+      message: status.message,
+    };
+  }
+
+  #getContainerStateSummary(state: k8s.V1ContainerState) {
+    return {
+      reason: state.terminated?.reason,
+      exitCode: state.terminated?.exitCode,
+      message: state.terminated?.message,
+    };
+  }
+
+  #createK8sClient() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (this.opts.runtimeEnv === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (this.opts.runtimeEnv === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      kubeConfig: kubeConfig,
+    };
+  }
+
+  #isRecord(candidate: unknown): candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  #logK8sError(err: unknown, debugOnly = false) {
+    if (debugOnly) {
+      this.#logger.debug("K8s API Error", err);
+    } else {
+      this.#logger.error("K8s API Error", err);
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
+      this.#logK8sError(err);
+      return;
+    }
+
+    this.#logK8sError(err, true);
+
+    if (typeof err.body.message === "string") {
+      this.#logK8sError({ message: err.body.message });
+      return;
+    }
+
+    this.#logK8sError({ body: err.body });
+  }
+
+  #printStats(includeMoreDetails = false) {
+    this.#logger.log("Stats:", {
+      cacheSize: this.#taskInformer.list().length,
+      totalProcessed: this.#processedPods.size,
+      ...(includeMoreDetails && {
+        processedPods: this.#processedPods,
+      }),
+    });
+  }
+
+  async #deletePod(name: string) {
+    this.#logger.debug("Deleting pod:", name);
+
+    await this.#k8sClient.core
+      .deleteNamespacedPod(name, this.namespace)
+      .catch(this.#handleK8sError.bind(this));
+  }
+
+  async start() {
+    this.#enabled = true;
+
+    const interval = setInterval(() => {
+      if (!this.#enabled) {
+        clearInterval(interval);
+        return;
+      }
+
+      this.#printStats();
+    }, 300_000);
+
+    await this.#taskInformer.start();
+
+    // this.#launchTests();
+  }
+
+  async stop() {
+    if (!this.#enabled) {
+      return;
+    }
+
+    this.#enabled = false;
+    this.#logger.log("Shutting down..");
+
+    await this.#taskInformer.stop();
+
+    this.#printStats(true);
+  }
+
+  async #launchTests() {
+    const createPod = async (
+      container: k8s.V1Container,
+      name: string,
+      labels?: Record<string, string>
+    ) => {
+      this.#logger.log("Creating pod:", name);
+
+      const pod = {
+        metadata: {
+          name,
+          labels,
+        },
+        spec: {
+          restartPolicy: "Never",
+          automountServiceAccountToken: false,
+          terminationGracePeriodSeconds: 1,
+          containers: [container],
+        },
+      } satisfies k8s.V1Pod;
+
+      await this.#k8sClient.core
+        .createNamespacedPod(this.namespace, pod)
+        .catch(this.#handleK8sError.bind(this));
+    };
+
+    const createOomPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "polinux/stress",
+        resources: {
+          limits: {
+            memory: "100Mi",
+          },
+        },
+        command: ["stress"],
+        args: ["--vm", "1", "--vm-bytes", "150M", "--vm-hang", "1"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createNonZeroExitPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "busybox",
+        command: ["sh"],
+        args: ["-c", "exit 1"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createOoDiskPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "busybox",
+        command: ["sh"],
+        args: [
+          "-c",
+          "echo creating huge-file..; head -c 1000m /dev/zero > huge-file; ls -lh huge-file; sleep infinity",
+        ],
+        resources: {
+          limits: {
+            "ephemeral-storage": "500Mi",
+          },
+        },
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    await createNonZeroExitPod("non-zero-exit-task", { app: "task-run", run: "123" });
+    await createOomPod("oom-task", { app: "task-index", deployment: "456" });
+    await createOoDiskPod("ood-task", { app: "task-run", run: "abc" });
+  }
+}

apps/webapp/app/components/navigation/SideMenu.tsx

@@ -57,6 +57,7 @@ import { UserProfilePhoto } from "../UserProfilePhoto";
 import { FreePlanUsage } from "../billing/FreePlanUsage";
 import { Badge } from "../primitives/Badge";
 import { Button } from "../primitives/Buttons";
+import { Callout } from "../primitives/Callout";
 import { ClipboardField } from "../primitives/ClipboardField";
 import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "../primitives/Dialog";
 import { Icon } from "../primitives/Icon";
@@ -187,7 +188,20 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
             />
           </div>
         </div>
+        <div className="m-2">
+          <Callout variant={"info"}>
+            {`This is a ${project.version === "V2" ? "v2" : "v3"} project`}
+          </Callout>
+        </div>
         <div className="flex flex-col gap-1 border-t border-grid-bright p-1">
+          {project.version === "V2" && (
+            <SideMenuItem
+              to="https://trigger.dev/v3-early-access"
+              target="_blank"
+              name="Request access to v3"
+              icon={V3Icon}
+            />
+          )}
           {currentPlan?.subscription?.isPaying === true && (
             <Dialog>
               <DialogTrigger asChild>
@@ -586,3 +600,15 @@ function V3ProjectSideMenu({
     </>
   );
 }
+
+function V3Icon() {
+  return (
+    <svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <rect width="16" height="16" rx="8" fill="#A8FF53" />
+      <path
+        d="M7.7488 6.17L5.5818 12H3.6678L1.5008 6.17H3.2058L4.6248 10.339L6.0328 6.17H7.7488ZM11.0527 7.292C11.6357 7.303 12.2297 7.006 12.2297 6.28C12.2297 5.774 11.7787 5.433 11.0527 5.433C10.4147 5.433 9.98567 5.741 9.91967 6.214L8.22567 6.126C8.36867 4.861 9.51267 4.014 11.0857 4.014C12.8457 4.014 13.9567 4.806 13.9567 6.049C13.9567 6.951 13.3847 7.534 12.3067 7.776C13.5387 8.04 14.2207 8.777 14.2207 9.855C14.2207 11.274 13.0107 12.176 11.0857 12.176C9.32567 12.176 8.12667 11.197 8.04967 9.712L9.75467 9.646C9.83167 10.405 10.4917 10.757 11.0967 10.757C11.8007 10.757 12.4937 10.394 12.4937 9.591C12.4937 8.81 11.7897 8.425 11.0527 8.447L10.3817 8.458V7.281L11.0527 7.292Z"
+        fill="#15171A"
+      />
+    </svg>
+  );
+}

apps/webapp/app/components/primitives/Dialog.tsx

@@ -43,7 +43,7 @@ const DialogContent = React.forwardRef<
     <DialogPrimitive.Content
       ref={ref}
       className={cn(
-        "fixed z-50 grid w-full gap-4 rounded-b-lg border bg-background-dimmed px-4 pb-6 pt-3.5 shadow-lg animate-in data-[state=open]:fade-in-90 data-[state=open]:slide-in-from-bottom-10 sm:max-w-lg sm:rounded-lg sm:zoom-in-90 data-[state=open]:sm:slide-in-from-bottom-0",
+        "fixed z-50 grid w-full gap-4 rounded-b-lg border bg-background-dimmed px-4 pb-4 pt-3.5 shadow-lg animate-in data-[state=open]:fade-in-90 data-[state=open]:slide-in-from-bottom-10 sm:max-w-lg sm:rounded-lg sm:zoom-in-90 data-[state=open]:sm:slide-in-from-bottom-0",
         className
       )}
       {...props}

apps/webapp/app/components/primitives/PageHeader.tsx

@@ -15,6 +15,7 @@ import { Tabs, TabsProps } from "./Tabs";
 
 type WithChildren = {
   children: React.ReactNode;
+  className?: string;
 };
 
 export function NavBar({ children }: WithChildren) {
@@ -72,8 +73,8 @@ export function PageAccessories({ children }: WithChildren) {
   return <div className="flex items-center gap-3">{children}</div>;
 }
 
-export function PageInfoRow({ children }: WithChildren) {
-  return <div className="flex w-full items-center gap-2">{children}</div>;
+export function PageInfoRow({ children, className }: WithChildren) {
+  return <div className={cn("flex w-full items-center gap-2", className)}>{children}</div>;
 }
 
 export function PageInfoGroup({

apps/webapp/app/components/run/RunOverview.tsx

@@ -124,9 +124,9 @@ export function RunOverview({ run, trigger, showRerun, paths, currentUser }: Run
           {!run.isFinished && <CancelRun runId={run.id} />}
         </PageAccessories>
       </NavBar>
-      <PageBody scrollable={false}>
-        <div className="px-4 pt-4">
-          <PageInfoRow>
+      <PageBody scrollable={false} className="grid grid-rows-[auto_1fr] overflow-hidden">
+        <div className="border-b border-grid-dimmed px-4 py-4">
+          <PageInfoRow className="overflow-hidden">
             <PageInfoGroup>
               <PageInfoProperty
                 icon={<RunStatusIcon status={run.status} className="h-4 w-4" />}
@@ -166,7 +166,7 @@ export function RunOverview({ run, trigger, showRerun, paths, currentUser }: Run
             </PageInfoGroup>
           </PageInfoRow>
         </div>
-        <div className="grid h-full grid-cols-2 gap-2">
+        <div className="grid h-full grid-cols-2 gap-2 overflow-hidden">
           <div className="flex flex-col gap-6 overflow-y-auto py-4 pl-4 pr-2 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600">
             <div>
               {run.status === "SUCCESS" &&

apps/webapp/app/components/runs/v3/CancelRunDialog.tsx

@@ -1,5 +1,5 @@
 import { StopCircleIcon } from "@heroicons/react/20/solid";
-import { useFetcher } from "@remix-run/react";
+import { Form, useFetcher, useNavigation } from "@remix-run/react";
 import { Button } from "~/components/primitives/Buttons";
 import {
   DialogContent,
@@ -14,29 +14,32 @@ type CancelRunDialogProps = {
 };
 
 export function CancelRunDialog({ runFriendlyId, redirectPath }: CancelRunDialogProps) {
-  const cancelFetcher = useFetcher();
+  const navigation = useNavigation();
+
+  const formAction = `/resources/taskruns/${runFriendlyId}/cancel`;
+  const isLoading = navigation.formAction === formAction;
 
   return (
-    <DialogContent>
+    <DialogContent key="cancel">
       <DialogHeader>Cancel this run?</DialogHeader>
       <DialogDescription>
         Canceling a run will stop execution. If you want to run this later you will have to replay
         the entire run with the original payload.
       </DialogDescription>
       <DialogFooter>
-        <cancelFetcher.Form action={`/resources/taskruns/${runFriendlyId}/cancel`} method="post">
+        <Form action={`/resources/taskruns/${runFriendlyId}/cancel`} method="post">
           <Button
             type="submit"
             name="redirectUrl"
             value={redirectPath}
             variant="danger/small"
-            LeadingIcon={cancelFetcher.state === "idle" ? StopCircleIcon : "spinner-white"}
-            disabled={cancelFetcher.state !== "idle"}
+            LeadingIcon={isLoading ? "spinner-white" : StopCircleIcon}
+            disabled={isLoading}
             shortcut={{ modifiers: ["meta"], key: "enter" }}
           >
-            {cancelFetcher.state === "idle" ? "Cancel run" : "Canceling..."}
+            {isLoading ? "Canceling..." : "Cancel run"}
           </Button>
-        </cancelFetcher.Form>
+        </Form>
       </DialogFooter>
     </DialogContent>
   );

apps/webapp/app/components/runs/v3/ReplayRunDialog.tsx

@@ -0,0 +1,44 @@
+import { ArrowPathIcon } from "@heroicons/react/20/solid";
+import { Form, useFetcher, useNavigation } from "@remix-run/react";
+import { Button } from "~/components/primitives/Buttons";
+import {
+  DialogContent,
+  DialogDescription,
+  DialogFooter,
+  DialogHeader,
+} from "~/components/primitives/Dialog";
+
+type ReplayRunDialogProps = {
+  runFriendlyId: string;
+  failedRedirect: string;
+};
+
+export function ReplayRunDialog({ runFriendlyId, failedRedirect }: ReplayRunDialogProps) {
+  const navigation = useNavigation();
+
+  const formAction = `/resources/taskruns/${runFriendlyId}/replay`;
+  const isLoading = navigation.formAction === formAction;
+
+  return (
+    <DialogContent key="replay">
+      <DialogHeader>Replay this run?</DialogHeader>
+      <DialogDescription>
+        Replaying a run will create a new run with the same payload and environment as the original.
+      </DialogDescription>
+      <DialogFooter>
+        <Form action={formAction} method="post">
+          <input type="hidden" name="failedRedirect" value={failedRedirect} />
+          <Button
+            type="submit"
+            variant="primary/small"
+            LeadingIcon={isLoading ? "spinner-white" : ArrowPathIcon}
+            disabled={isLoading}
+            shortcut={{ modifiers: ["meta"], key: "enter" }}
+          >
+            {isLoading ? "Replaying..." : "Replay run"}
+          </Button>
+        </Form>
+      </DialogFooter>
+    </DialogContent>
+  );
+}

apps/webapp/app/components/runs/v3/RunFilters.tsx

@@ -35,6 +35,7 @@ export const allTaskRunStatuses = [
   "COMPLETED_WITH_ERRORS",
   "INTERRUPTED",
   "SYSTEM_FAILURE",
+  "CRASHED",
 ] as TaskRunStatusType[];
 
 export const TaskAttemptStatus = z.nativeEnum(TaskRunStatus);

apps/webapp/app/components/runs/v3/SpanEvents.tsx

@@ -66,7 +66,11 @@ function SpanEventError({
 }) {
   return (
     <div className="flex flex-col gap-2 rounded-sm border border-rose-500/50 p-3">
-      <SpanEventHeader title={"Error"} time={spanEvent.time} titleClassName="text-rose-500" />
+      <SpanEventHeader
+        title={exception.type ?? "Error"}
+        time={spanEvent.time}
+        titleClassName="text-rose-500"
+      />
       {exception.message && <Callout variant="error">{exception.message}</Callout>}
       {exception.stacktrace && (
         <CodeBlock

apps/webapp/app/components/runs/v3/TaskRunStatus.tsx

@@ -3,7 +3,7 @@ import {
   BoltSlashIcon,
   BugAntIcon,
   CheckCircleIcon,
-  ClockIcon,
+  FireIcon,
   NoSymbolIcon,
   PauseCircleIcon,
   RectangleStackIcon,
@@ -25,6 +25,7 @@ const taskRunStatusDescriptions: Record<TaskRunStatus, string> = {
   INTERRUPTED: "Task has failed because it was interrupted",
   SYSTEM_FAILURE: "Task has failed due to a system failure",
   PAUSED: "Task has been paused by the user",
+  CRASHED: "Task has crashed and won't be retried",
 };
 
 export function descriptionForTaskRunStatus(status: TaskRunStatus): string {
@@ -80,6 +81,8 @@ export function TaskRunStatusIcon({
       return <XCircleIcon className={cn(runStatusClassNameColor(status), className)} />;
     case "SYSTEM_FAILURE":
       return <BugAntIcon className={cn(runStatusClassNameColor(status), className)} />;
+    case "CRASHED":
+      return <FireIcon className={cn(runStatusClassNameColor(status), className)} />;
 
     default: {
       const _exhaustiveCheck: never = status;
@@ -109,6 +112,8 @@ export function runStatusClassNameColor(status: TaskRunStatus): string {
       return "text-error";
     case "SYSTEM_FAILURE":
       return "text-error";
+    case "CRASHED":
+      return "text-error";
     default: {
       const _exhaustiveCheck: never = status;
       throw new Error(`Non-exhaustive match for value: ${status}`);
@@ -138,6 +143,8 @@ export function runStatusTitle(status: TaskRunStatus): string {
       return "Failed";
     case "SYSTEM_FAILURE":
       return "System failure";
+    case "CRASHED":
+      return "Crashed";
     default: {
       const _exhaustiveCheck: never = status;
       throw new Error(`Non-exhaustive match for value: ${status}`);

apps/webapp/app/components/runs/v3/TaskRunsTable.tsx

@@ -24,10 +24,11 @@ import { formatDuration } from "@trigger.dev/core/v3";
 import { TaskRunStatusCombo } from "./TaskRunStatus";
 import { useEnvironments } from "~/hooks/useEnvironments";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
-import { StopCircleIcon } from "@heroicons/react/20/solid";
+import { ArrowPathIcon, StopCircleIcon } from "@heroicons/react/20/solid";
 import { Dialog, DialogTrigger } from "~/components/primitives/Dialog";
 import { CancelRunDialog } from "./CancelRunDialog";
 import { useLocation } from "@remix-run/react";
+import { ReplayRunDialog } from "./ReplayRunDialog";
 
 type RunsTableProps = {
   total: number;
@@ -49,7 +50,6 @@ export function TaskRunsTable({
 }: RunsTableProps) {
   const organization = useOrganization();
   const project = useProject();
-  const location = useLocation();
 
   return (
     <Table>
@@ -110,23 +110,7 @@ export function TaskRunsTable({
                 <TableCell to={path}>
                   {run.createdAt ? <DateTime date={run.createdAt} /> : "–"}
                 </TableCell>
-                {run.isCancellable ? (
-                  <TableCellMenu isSticky>
-                    <Dialog>
-                      <DialogTrigger asChild>
-                        <Button variant="small-menu-item" LeadingIcon={StopCircleIcon}>
-                          Cancel run
-                        </Button>
-                      </DialogTrigger>
-                      <CancelRunDialog
-                        runFriendlyId={run.friendlyId}
-                        redirectPath={`${location.pathname}${location.search}`}
-                      />
-                    </Dialog>
-                  </TableCellMenu>
-                ) : (
-                  <TableCell to={path}>{""}</TableCell>
-                )}
+                <RunActionsCell run={run} path={path} />
               </TableRow>
             );
           })
@@ -144,6 +128,43 @@ export function TaskRunsTable({
   );
 }
 
+function RunActionsCell({ run, path }: { run: RunListItem; path: string }) {
+  const location = useLocation();
+
+  if (!run.isCancellable && !run.isReplayable) return <TableCell to={path}>{""}</TableCell>;
+
+  return (
+    <TableCellMenu isSticky>
+      {run.isCancellable && (
+        <Dialog>
+          <DialogTrigger asChild>
+            <Button variant="small-menu-item" LeadingIcon={StopCircleIcon}>
+              Cancel run
+            </Button>
+          </DialogTrigger>
+          <CancelRunDialog
+            runFriendlyId={run.friendlyId}
+            redirectPath={`${location.pathname}${location.search}`}
+          />
+        </Dialog>
+      )}
+      {run.isReplayable && (
+        <Dialog>
+          <DialogTrigger asChild>
+            <Button variant="small-menu-item" LeadingIcon={ArrowPathIcon}>
+              Replay run
+            </Button>
+          </DialogTrigger>
+          <ReplayRunDialog
+            runFriendlyId={run.friendlyId}
+            failedRedirect={`${location.pathname}${location.search}`}
+          />
+        </Dialog>
+      )}
+    </TableCellMenu>
+  );
+}
+
 function NoRuns({ title }: { title: string }) {
   return (
     <div className="flex items-center justify-center">

apps/webapp/app/env.server.ts

@@ -137,7 +137,7 @@ const EnvironmentSchema = z.object({
   INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE: z.string().optional(),
   INTERNAL_OTEL_TRACE_LOGGING_ENABLED: z.string().default("1"),
   // this means 1/20 traces or 5% of traces will be sampled (sampled = recorded)
-  INTERNAL_OTEL_TRACE_SAMPING_RATE: z.string().default("20"),
+  INTERNAL_OTEL_TRACE_SAMPLING_RATE: z.string().default("20"),
   INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED: z.string().default("0"),
 });
 

apps/webapp/app/presenters/v3/RunListPresenter.server.ts

@@ -222,6 +222,7 @@ export class RunListPresenter {
           version: run.version,
           taskIdentifier: run.taskIdentifier,
           attempts: Number(run.attempts),
+          isReplayable: true,
           isCancellable: CANCELLABLE_STATUSES.includes(run.status),
           environment: {
             type: environment.type,

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.triggers._index/route.tsx

@@ -44,7 +44,7 @@ export default function Integrations() {
 
   return (
     <>
-      <Paragraph variant="small" spacing>
+      <Paragraph variant="small" spacing className="pt-2">
         External Triggers get registered with external APIs, for example a webhook.
       </Paragraph>
       <Table containerClassName="mt-4">

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.triggers.scheduled/route.tsx

@@ -44,7 +44,7 @@ export default function Integrations() {
 
   return (
     <>
-      <Paragraph variant="small" spacing>
+      <Paragraph variant="small" spacing className="pt-2">
         A Scheduled Trigger runs a Job on a repeated schedule. The schedule can use a CRON
         expression or an interval.
       </Paragraph>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.triggers.webhooks/route.tsx

@@ -44,7 +44,7 @@ export default function Integrations() {
 
   return (
     <>
-      <Paragraph variant="small" spacing>
+      <Paragraph variant="small" spacing className="pt-2">
         A Webhook Trigger runs a Job when it receives a matching payload at a registered HTTP
         Endpoint.
       </Paragraph>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.triggers/route.tsx

@@ -32,7 +32,7 @@ export default function Page() {
       </NavBar>
 
       <PageBody scrollable={false}>
-        <div className="grid grid-rows-[auto_1fr] gap-y-4 px-4">
+        <div className="grid max-h-full grid-rows-[auto_1fr] overflow-hidden px-4">
           <PageTabs
             layoutId="triggers"
             tabs={[

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.environment-variables.new/route.tsx

@@ -16,6 +16,7 @@ import { FormError } from "~/components/primitives/FormError";
 import { Input } from "~/components/primitives/Input";
 import { InputGroup } from "~/components/primitives/InputGroup";
 import { Label } from "~/components/primitives/Label";
+import { Switch } from "~/components/primitives/Switch";
 import { prisma } from "~/db.server";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
@@ -134,6 +135,8 @@ export default function Page() {
     shouldRevalidate: "onSubmit",
   });
 
+  const [revealAll, setRevealAll] = useState(false);
+
   useEffect(() => {
     setIsOpen(true);
   }, []);
@@ -169,7 +172,15 @@ export default function Page() {
               />
             </InputGroup>
             <InputGroup fullWidth>
-              <Label>Values</Label>
+              <div className="flex items-center justify-between">
+                <Label>Values</Label>
+                <Switch
+                  variant="small"
+                  label="Reveal values"
+                  checked={revealAll}
+                  onCheckedChange={(e) => setRevealAll(e.valueOf())}
+                />
+              </div>
               <div className="grid grid-cols-[auto_1fr] gap-x-2 gap-y-2">
                 {environments.map((environment, index) => {
                   return (
@@ -186,7 +197,7 @@ export default function Page() {
                         <EnvironmentLabel environment={environment} className="h-5 px-2" />
                       </label>
                       <Input
-                        type="password"
+                        type={revealAll ? "text" : "password"}
                         name={`values[${index}].value`}
                         placeholder="Not set"
                       />

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx

@@ -1,4 +1,9 @@
-import { CloudArrowDownIcon, QueueListIcon, StopCircleIcon } from "@heroicons/react/20/solid";
+import {
+  ArrowPathIcon,
+  CloudArrowDownIcon,
+  QueueListIcon,
+  StopCircleIcon,
+} from "@heroicons/react/20/solid";
 import { useParams } from "@remix-run/react";
 import { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { formatDurationNanoseconds, nanosecondsToMilliseconds } from "@trigger.dev/core/v3";
@@ -14,6 +19,7 @@ import { Paragraph } from "~/components/primitives/Paragraph";
 import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
 import { CancelRunDialog } from "~/components/runs/v3/CancelRunDialog";
 import { LiveTimer } from "~/components/runs/v3/LiveTimer";
+import { ReplayRunDialog } from "~/components/runs/v3/ReplayRunDialog";
 import { RunIcon } from "~/components/runs/v3/RunIcon";
 import { SpanEvents } from "~/components/runs/v3/SpanEvents";
 import { SpanTitle } from "~/components/runs/v3/SpanTitle";
@@ -22,7 +28,7 @@ import { TaskRunAttemptStatusCombo } from "~/components/runs/v3/TaskRunAttemptSt
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
 import { redirectWithErrorMessage } from "~/models/message.server";
-import { SpanPresenter } from "~/presenters/v3/SpanPresenter.server";
+import { Span, SpanPresenter } from "~/presenters/v3/SpanPresenter.server";
 import { requireUserId } from "~/services/session.server";
 import { cn } from "~/utils/cn";
 import { v3RunPath, v3RunSpanPath, v3SpanParamsSchema, v3TraceSpanPath } from "~/utils/pathBuilder";
@@ -188,24 +194,7 @@ export default function Page() {
             )}
           </div>
           <div className="flex items-center gap-4">
-            {event.isPartial && runParam && (
-              <Dialog>
-                <DialogTrigger asChild>
-                  <Button variant="danger/small" LeadingIcon={StopCircleIcon}>
-                    Cancel run
-                  </Button>
-                </DialogTrigger>
-                <CancelRunDialog
-                  runFriendlyId={event.runId}
-                  redirectPath={v3RunSpanPath(
-                    organization,
-                    project,
-                    { friendlyId: runParam },
-                    { spanId: event.spanId }
-                  )}
-                />
-              </Dialog>
-            )}
+            <RunActionButtons span={event} />
           </div>
         </div>
       ) : null}
@@ -213,6 +202,54 @@ export default function Page() {
   );
 }
 
+function RunActionButtons({ span }: { span: Span }) {
+  const organization = useOrganization();
+  const project = useProject();
+  const { runParam } = useParams();
+
+  if (!runParam) return null;
+
+  if (span.isPartial) {
+    return (
+      <Dialog>
+        <DialogTrigger asChild>
+          <Button variant="danger/small" LeadingIcon={StopCircleIcon}>
+            Cancel run
+          </Button>
+        </DialogTrigger>
+        <CancelRunDialog
+          runFriendlyId={span.runId}
+          redirectPath={v3RunSpanPath(
+            organization,
+            project,
+            { friendlyId: runParam },
+            { spanId: span.spanId }
+          )}
+        />
+      </Dialog>
+    );
+  }
+
+  return (
+    <Dialog>
+      <DialogTrigger asChild>
+        <Button variant="tertiary/small" LeadingIcon={ArrowPathIcon}>
+          Replay run
+        </Button>
+      </DialogTrigger>
+      <ReplayRunDialog
+        runFriendlyId={span.runId}
+        failedRedirect={v3RunSpanPath(
+          organization,
+          project,
+          { friendlyId: runParam },
+          { spanId: span.spanId }
+        )}
+      />
+    </Dialog>
+  );
+}
+
 function PacketDisplay({
   data,
   dataType,

apps/webapp/app/routes/api.v1.runs.$runParam.replay.ts

@@ -0,0 +1,72 @@
+import type { ActionFunctionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import { PrismaErrorSchema, prisma } from "~/db.server";
+import { z } from "zod";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { CancelRunService } from "~/services/runs/cancelRun.server";
+import { ApiRunPresenter } from "~/presenters/ApiRunPresenter.server";
+import { ReplayTaskRunService } from "~/v3/services/replayTaskRun.server";
+import { logger } from "~/services/logger.server";
+
+const ParamsSchema = z.object({
+  /* This is the run friendly ID */
+  runParam: z.string(),
+});
+
+export async function action({ request, params }: ActionFunctionArgs) {
+  // Ensure this is a POST request
+  if (request.method.toUpperCase() !== "POST") {
+    return { status: 405, body: "Method Not Allowed" };
+  }
+
+  // Authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API Key" }, { status: 401 });
+  }
+
+  const parsed = ParamsSchema.safeParse(params);
+  if (!parsed.success) {
+    return json({ error: "Invalid or missing run ID" }, { status: 400 });
+  }
+
+  const { runParam } = parsed.data;
+
+  try {
+    const taskRun = await prisma.taskRun.findUnique({
+      where: {
+        friendlyId: runParam,
+      },
+    });
+
+    if (!taskRun) {
+      return json({ error: "Run not found" }, { status: 404 });
+    }
+
+    const service = new ReplayTaskRunService();
+    const newRun = await service.call(taskRun);
+
+    if (!newRun) {
+      return json({ error: "Failed to create new run" }, { status: 400 });
+    }
+
+    return json({
+      id: newRun?.friendlyId,
+    });
+  } catch (error) {
+    if (error instanceof Error) {
+      logger.error("Failed to replay run", {
+        error: {
+          name: error.name,
+          message: error.message,
+          stack: error.stack,
+        },
+        run: runParam,
+      });
+      return json({ error: error.message }, { status: 400 });
+    } else {
+      logger.error("Failed to replay run", { error: JSON.stringify(error), run: runParam });
+      return json({ error: JSON.stringify(error) }, { status: 400 });
+    }
+  }
+}

apps/webapp/app/routes/api.v2.runs.$runParam.cancel.ts

@@ -25,7 +25,7 @@ export async function action({ request, params }: ActionFunctionArgs) {
   const parsed = ParamsSchema.safeParse(params);
 
   if (!parsed.success) {
-    return json({ error: "Invalid or Missing runId" }, { status: 400 });
+    return json({ error: "Invalid or Missing run id" }, { status: 400 });
   }
 
   const { runParam } = parsed.data;

apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts

@@ -0,0 +1,85 @@
+import { parse } from "@conform-to/zod";
+import { ActionFunction, json } from "@remix-run/node";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
+import { logger } from "~/services/logger.server";
+import { v3RunPath } from "~/utils/pathBuilder";
+import { ReplayTaskRunService } from "~/v3/services/replayTaskRun.server";
+
+const FormSchema = z.object({
+  failedRedirect: z.string(),
+});
+
+const ParamSchema = z.object({
+  runParam: z.string(),
+});
+
+export const action: ActionFunction = async ({ request, params }) => {
+  const { runParam } = ParamSchema.parse(params);
+
+  const formData = await request.formData();
+  const submission = parse(formData, { schema: FormSchema });
+
+  if (!submission.value) {
+    return json(submission);
+  }
+
+  try {
+    const taskRun = await prisma.taskRun.findUnique({
+      where: {
+        friendlyId: runParam,
+      },
+      include: {
+        project: {
+          include: {
+            organization: true,
+          },
+        },
+      },
+    });
+
+    if (!taskRun) {
+      return redirectWithErrorMessage(submission.value.failedRedirect, request, "Run not found");
+    }
+
+    const replayRunService = new ReplayTaskRunService();
+    const newRun = await replayRunService.call(taskRun);
+
+    if (!newRun) {
+      return redirectWithErrorMessage(
+        submission.value.failedRedirect,
+        request,
+        "Failed to replay run"
+      );
+    }
+
+    const runPath = v3RunPath(
+      {
+        slug: taskRun.project.organization.slug,
+      },
+      { slug: taskRun.project.slug },
+      { friendlyId: newRun.friendlyId }
+    );
+
+    return redirectWithSuccessMessage(runPath, request, `Replaying run`);
+  } catch (error) {
+    if (error instanceof Error) {
+      logger.error("Failed to replay run", {
+        error: {
+          name: error.name,
+          message: error.message,
+          stack: error.stack,
+        },
+      });
+      return redirectWithErrorMessage(submission.value.failedRedirect, request, error.message);
+    } else {
+      logger.error("Failed to replay run", { error });
+      return redirectWithErrorMessage(
+        submission.value.failedRedirect,
+        request,
+        JSON.stringify(error)
+      );
+    }
+  }
+};

apps/webapp/app/services/apiRateLimit.server.ts

@@ -67,6 +67,11 @@ export function authorizationRateLimitMiddleware({
       logger.info(`RateLimiter (${keyPrefix}): request to ${req.path}`);
     }
 
+    // allow OPTIONS requests
+    if (req.method.toUpperCase() === "OPTIONS") {
+      return next();
+    }
+
     //first check if any of the pathMatchers match the request path
     const path = req.path;
     if (
@@ -109,6 +114,7 @@ export function authorizationRateLimitMiddleware({
             status: 401,
             type: "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/401",
             detail: "No authorization header provided",
+            error: "No authorization header provided",
           },
           null,
           2
@@ -160,6 +166,7 @@ export function authorizationRateLimitMiddleware({
           detail: `Rate limit exceeded ${remaining}/${limit} requests remaining. Retry after ${reset} seconds.`,
           reset: reset,
           limit: limit,
+          error: `Rate limit exceeded ${remaining}/${limit} requests remaining. Retry after ${reset} seconds.`,
         },
         null,
         2
@@ -182,7 +189,16 @@ export const apiRateLimiter = authorizationRateLimitMiddleware({
   },
   limiter: Ratelimit.slidingWindow(env.API_RATE_LIMIT_MAX, env.API_RATE_LIMIT_WINDOW as Duration),
   pathMatchers: [/^\/api/],
-  pathWhiteList: ["/api/v1/authorization-code", "/api/v1/token"],
+  // Allow /api/v1/tasks/:id/callback/:secret
+  pathWhiteList: [
+    "/api/v1/authorization-code",
+    "/api/v1/token",
+    /^\/api\/v1\/tasks\/[^\/]+\/callback\/[^\/]+$/, // /api/v1/tasks/$id/callback/$secret
+    /^\/api\/v1\/runs\/[^\/]+\/tasks\/[^\/]+\/callback\/[^\/]+$/, // /api/v1/runs/$runId/tasks/$id/callback/$secret
+    /^\/api\/v1\/http-endpoints\/[^\/]+\/env\/[^\/]+\/[^\/]+$/, // /api/v1/http-endpoints/$httpEndpointId/env/$envType/$shortcode
+    /^\/api\/v1\/sources\/http\/[^\/]+$/, // /api/v1/sources/http/$id
+    /^\/api\/v1\/endpoints\/[^\/]+\/[^\/]+\/index\/[^\/]+$/, // /api/v1/endpoints/$environmentId/$endpointSlug/index/$indexHookIdentifier
+  ],
   log: {
     rejections: env.API_RATE_LIMIT_REJECTION_LOGS_ENABLED === "1",
     requests: env.API_RATE_LIMIT_REQUEST_LOGS_ENABLED === "1",

apps/webapp/app/v3/eventRepository.server.ts

@@ -3,6 +3,7 @@ import { RandomIdGenerator } from "@opentelemetry/sdk-trace-base";
 import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions";
 import {
   ExceptionEventProperties,
+  ExceptionSpanEvent,
   PRIMARY_VARIANT,
   SemanticInternalAttributes,
   SpanEvent,
@@ -255,6 +256,47 @@ export class EventRepository {
     });
   }
 
+  async crashEvent({
+    event,
+    crashedAt,
+    exception,
+  }: {
+    event: TaskEventRecord;
+    crashedAt: Date;
+    exception: ExceptionEventProperties;
+  }) {
+    if (!event.isPartial) {
+      return;
+    }
+
+    await this.insertImmediate({
+      ...omit(event, "id"),
+      isPartial: false,
+      isError: true,
+      isCancelled: false,
+      status: "ERROR",
+      links: event.links ?? [],
+      events: [
+        {
+          name: "exception",
+          time: crashedAt,
+          properties: {
+            exception,
+          },
+        } satisfies ExceptionSpanEvent,
+        ...((event.events as any[]) ?? []),
+      ],
+      duration: calculateDurationFromStart(event.startTime, crashedAt),
+      properties: event.properties as Attributes,
+      metadata: event.metadata as Attributes,
+      style: event.style as Attributes,
+      output: event.output as Attributes,
+      outputType: event.outputType,
+      payload: event.payload as Attributes,
+      payloadType: event.payloadType,
+    });
+  }
+
   async queryEvents(queryOptions: QueryOptions): Promise<TaskEventRecord[]> {
     return await this.db.taskEvent.findMany({
       where: queryOptions,

apps/webapp/app/v3/handleSocketIo.server.ts

@@ -21,6 +21,7 @@ import { ResumeAttemptService } from "./services/resumeAttempt.server";
 import { DeploymentIndexFailed } from "./services/deploymentIndexFailed.server";
 import { Redis } from "ioredis";
 import { createAdapter } from "@socket.io/redis-adapter";
+import { CrashTaskRunService } from "./services/crashTaskRun.server";
 
 export const socketIo = singleton("socketIo", initalizeIoServer);
 
@@ -135,7 +136,7 @@ function createCoordinatorNamespace(io: Server) {
 
           await service.call(message.deploymentId, message.error);
         } catch (e) {
-          logger.error("Error while indexing failed", { error: e });
+          logger.error("Error while indexing", { error: e });
         }
       },
     },
@@ -151,6 +152,28 @@ function createProviderNamespace(io: Server) {
     authToken: env.PROVIDER_SECRET,
     clientMessages: ProviderToPlatformMessages,
     serverMessages: PlatformToProviderMessages,
+    handlers: {
+      WORKER_CRASHED: async (message) => {
+        try {
+          const service = new CrashTaskRunService();
+
+          await service.call(message.runId, {
+            ...message,
+          });
+        } catch (error) {
+          logger.error("Error while handling crashed worker", { error });
+        }
+      },
+      INDEXING_FAILED: async (message) => {
+        try {
+          const service = new DeploymentIndexFailed();
+
+          await service.call(message.deploymentId, message.error);
+        } catch (e) {
+          logger.error("Error while indexing", { error: e });
+        }
+      },
+    },
   });
 
   return provider.namespace;

apps/webapp/app/v3/marqs/devQueueConsumer.server.ts

@@ -16,12 +16,14 @@ import { generateFriendlyId } from "../friendlyIdentifiers";
 import { marqs } from "~/v3/marqs/index.server";
 import { CancelAttemptService } from "../services/cancelAttempt.server";
 import { CompleteAttemptService } from "../services/completeAttempt.server";
-import { attributesFromAuthenticatedEnv } from "../tracer.server";
+import {
+  SEMINTATTRS_FORCE_RECORDING,
+  attributesFromAuthenticatedEnv,
+  tracer,
+} from "../tracer.server";
 import { DevSubscriber, devPubSub } from "./devPubSub.server";
 import { CancelTaskRunService } from "../services/cancelTaskRun.server";
 
-const tracer = trace.getTracer("devQueueConsumer");
-
 const MessageBody = z.discriminatedUnion("type", [
   z.object({
     type: z.literal("EXECUTE"),
@@ -165,6 +167,11 @@ export class DevQueueConsumer {
 
       logger.debug("Unsubscribed from background worker channel", { id });
     }
+
+    // We need to end the current span
+    if (this._currentSpan) {
+      this._currentSpan.end();
+    }
   }
 
   async #cancelInProgressRunsAndAttempts(reason: string) {
@@ -284,6 +291,10 @@ export class DevQueueConsumer {
         this._currentSpan.setAttribute("tasks.period.failures", this._taskFailures);
         this._currentSpan.setAttribute("tasks.period.successes", this._taskSuccesses);
 
+        logger.debug("Ending DevQueueConsumer.doWork() trace", {
+          isRecording: this._currentSpan.isRecording(),
+        });
+
         this._currentSpan.end();
       }
 
@@ -294,6 +305,7 @@ export class DevQueueConsumer {
           kind: SpanKind.CONSUMER,
           attributes: {
             ...attributesFromAuthenticatedEnv(this.env),
+            [SEMINTATTRS_FORCE_RECORDING]: true,
           },
         },
         ROOT_CONTEXT

apps/webapp/app/v3/marqs/index.server.ts

@@ -139,7 +139,7 @@ export class MarQS {
   public async dequeueMessageInEnv(env: AuthenticatedEnvironment) {
     return this.#trace(
       "dequeueMessageInEnv",
-      async (span, abort) => {
+      async (span) => {
         const parentQueue = this.keys.envSharedQueueKey(env);
 
         // Read the parent queue for matching queues
@@ -150,7 +150,6 @@ export class MarQS {
         );
 
         if (!messageQueue) {
-          abort();
           return;
         }
 
@@ -167,7 +166,6 @@ export class MarQS {
         });
 
         if (!messageData) {
-          abort();
           return;
         }
 
@@ -181,8 +179,6 @@ export class MarQS {
             [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey,
             [SemanticAttributes.PARENT_QUEUE]: message.parentQueue,
           });
-        } else {
-          abort();
         }
 
         return message;
@@ -204,7 +200,7 @@ export class MarQS {
   public async dequeueMessageInSharedQueue() {
     return this.#trace(
       "dequeueMessageInSharedQueue",
-      async (span, abort) => {
+      async (span) => {
         const parentQueue = constants.SHARED_QUEUE;
 
         // Read the parent queue for matching queues
@@ -215,7 +211,6 @@ export class MarQS {
         );
 
         if (!messageQueue) {
-          abort();
           return;
         }
 
@@ -233,7 +228,6 @@ export class MarQS {
         });
 
         if (!messageData) {
-          abort();
           return;
         }
 
@@ -247,8 +241,6 @@ export class MarQS {
             [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey,
             [SemanticAttributes.PARENT_QUEUE]: message.parentQueue,
           });
-        } else {
-          abort();
         }
 
         return message;
@@ -355,17 +347,12 @@ export class MarQS {
 
   async #trace<T>(
     name: string,
-    fn: (span: Span, abort: () => void) => Promise<T>,
-    options?: SpanOptions
+    fn: (span: Span) => Promise<T>,
+    options?: SpanOptions & { sampleRate?: number }
   ): Promise<T> {
     return tracer.startActiveSpan(name, options ?? {}, async (span) => {
-      let _abort = false;
-      let aborter = () => {
-        _abort = true;
-      };
-
       try {
-        return await fn(span, aborter);
+        return await fn(span);
       } catch (e) {
         if (e instanceof Error) {
           span.recordException(e);
@@ -375,9 +362,7 @@ export class MarQS {
 
         throw e;
       } finally {
-        if (!_abort) {
-          span.end();
-        }
+        span.end();
       }
     });
   }
@@ -480,7 +465,7 @@ export class MarQS {
   ) {
     return this.#trace(
       "getRandomQueueFromParentQueue",
-      async (span, abort) => {
+      async (span) => {
         const { range, selectionId } = await queuePriorityStrategy.nextCandidateSelection(
           parentQueue
         );
@@ -497,7 +482,6 @@ export class MarQS {
         );
 
         if (typeof choice !== "string") {
-          abort();
           return;
         }
 

apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts

@@ -1,12 +1,4 @@
-import {
-  Context,
-  ROOT_CONTEXT,
-  Span,
-  SpanKind,
-  context,
-  propagation,
-  trace,
-} from "@opentelemetry/api";
+import { Context, ROOT_CONTEXT, Span, SpanKind, context, trace } from "@opentelemetry/api";
 import {
   Machine,
   ProdTaskRunExecution,
@@ -28,16 +20,15 @@ import {
 import { z } from "zod";
 import { prisma } from "~/db.server";
 import { logger } from "~/services/logger.server";
-import { generateFriendlyId } from "../friendlyIdentifiers";
+import { singleton } from "~/utils/singleton";
 import { marqs } from "~/v3/marqs/index.server";
 import { EnvironmentVariablesRepository } from "../environmentVariables/environmentVariablesRepository.server";
-import { CancelAttemptService } from "../services/cancelAttempt.server";
+import { generateFriendlyId } from "../friendlyIdentifiers";
 import { socketIo } from "../handleSocketIo.server";
-import { singleton } from "~/utils/singleton";
-import { RestoreCheckpointService } from "../services/restoreCheckpoint.server";
 import { findCurrentWorkerDeployment } from "../models/workerDeployment.server";
-
-const tracer = trace.getTracer("sharedQueueConsumer");
+import { CancelAttemptService } from "../services/cancelAttempt.server";
+import { RestoreCheckpointService } from "../services/restoreCheckpoint.server";
+import { tracer } from "../tracer.server";
 
 const WithTraceContext = z.object({
   traceparent: z.string().optional(),
@@ -154,6 +145,10 @@ export class SharedQueueConsumer {
 
     logger.debug("Stopping shared queue consumer");
     this._enabled = false;
+
+    if (this._currentSpan) {
+      this._currentSpan.end();
+    }
   }
 
   async #cancelInProgressAttempts(reason: string) {

apps/webapp/app/v3/services/cancelTaskRun.server.ts

@@ -1,4 +1,4 @@
-import { TaskRun, TaskRunAttemptStatus, TaskRunStatus } from "@trigger.dev/database";
+import { Prisma, TaskRun, TaskRunAttemptStatus, TaskRunStatus } from "@trigger.dev/database";
 import { eventRepository } from "../eventRepository.server";
 import { marqs } from "~/v3/marqs/index.server";
 import { devPubSub } from "../marqs/devPubSub.server";
@@ -23,6 +23,13 @@ const CANCELLABLE_ATTEMPT_STATUSES: Array<TaskRunAttemptStatus> = [
   "PENDING",
 ];
 
+type ExtendedTaskRunAttempt = Prisma.TaskRunAttemptGetPayload<{
+  include: {
+    runtimeEnvironment: true;
+    backgroundWorker: true;
+  };
+}>;
+
 export type CancelTaskRunServiceOptions = {
   reason?: string;
   cancelAttempts?: boolean;
@@ -87,56 +94,60 @@ export class CancelTaskRunService extends BaseService {
 
     // Cancel any in progress attempts
     if (opts.cancelAttempts) {
-      for (const attempt of cancelledTaskRun.attempts) {
-        if (attempt.runtimeEnvironment.type === "DEVELOPMENT") {
-          // Signal the task run attempt to stop
-          await devPubSub.publish(
-            `backgroundWorker:${attempt.backgroundWorkerId}:${attempt.id}`,
-            "CANCEL_ATTEMPT",
-            {
-              attemptId: attempt.friendlyId,
-              backgroundWorkerId: attempt.backgroundWorker.friendlyId,
-              taskRunId: cancelledTaskRun.friendlyId,
-            }
-          );
-        } else {
-          switch (attempt.status) {
-            case "EXECUTING": {
-              // We need to send a cancel message to the coordinator
-              socketIo.coordinatorNamespace.emit("REQUEST_ATTEMPT_CANCELLATION", {
-                version: "v1",
-                attemptId: attempt.id,
-                attemptFriendlyId: attempt.friendlyId,
-              });
+      await this.#cancelPotentiallyRunningAttempts(cancelledTaskRun, cancelledTaskRun.attempts);
+    }
+  }
 
-              break;
-            }
-            case "PENDING":
-            case "PAUSED": {
-              logger.debug("Cancelling pending or paused attempt", {
-                attempt,
-              });
+  async #cancelPotentiallyRunningAttempts(run: TaskRun, attempts: ExtendedTaskRunAttempt[]) {
+    for (const attempt of attempts) {
+      if (attempt.runtimeEnvironment.type === "DEVELOPMENT") {
+        // Signal the task run attempt to stop
+        await devPubSub.publish(
+          `backgroundWorker:${attempt.backgroundWorkerId}:${attempt.id}`,
+          "CANCEL_ATTEMPT",
+          {
+            attemptId: attempt.friendlyId,
+            backgroundWorkerId: attempt.backgroundWorker.friendlyId,
+            taskRunId: run.friendlyId,
+          }
+        );
+      } else {
+        switch (attempt.status) {
+          case "EXECUTING": {
+            // We need to send a cancel message to the coordinator
+            socketIo.coordinatorNamespace.emit("REQUEST_ATTEMPT_CANCELLATION", {
+              version: "v1",
+              attemptId: attempt.id,
+              attemptFriendlyId: attempt.friendlyId,
+            });
 
-              const service = new CancelAttemptService();
+            break;
+          }
+          case "PENDING":
+          case "PAUSED": {
+            logger.debug("Cancelling pending or paused attempt", {
+              attempt,
+            });
 
-              await service.call(
-                attempt.friendlyId,
-                taskRun.id,
-                new Date(),
-                "Task run was cancelled by user"
-              );
+            const service = new CancelAttemptService();
 
-              break;
-            }
-            case "CANCELED":
-            case "COMPLETED":
-            case "FAILED": {
-              // Do nothing
-              break;
-            }
-            default: {
-              assertUnreachable(attempt.status);
-            }
+            await service.call(
+              attempt.friendlyId,
+              run.id,
+              new Date(),
+              "Task run was cancelled by user"
+            );
+
+            break;
+          }
+          case "CANCELED":
+          case "COMPLETED":
+          case "FAILED": {
+            // Do nothing
+            break;
+          }
+          default: {
+            assertUnreachable(attempt.status);
           }
         }
       }

apps/webapp/app/v3/services/crashTaskRun.server.ts

@@ -0,0 +1,157 @@
+import {
+  TaskRun,
+  TaskRunAttempt,
+  TaskRunAttemptStatus,
+  TaskRunStatus,
+} from "@trigger.dev/database";
+import { eventRepository } from "../eventRepository.server";
+import { marqs } from "~/v3/marqs/index.server";
+import { BaseService } from "./baseService.server";
+import { logger } from "~/services/logger.server";
+import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { ResumeTaskRunDependenciesService } from "./resumeTaskRunDependencies.server";
+
+export const CRASHABLE_RUN_STATUSES: Array<TaskRunStatus> = [
+  "PENDING",
+  "EXECUTING",
+  "PAUSED",
+  "WAITING_TO_RESUME",
+  "PAUSED",
+  "RETRYING_AFTER_FAILURE",
+];
+
+const CRASHABLE_ATTEMPT_STATUSES: Array<TaskRunAttemptStatus> = ["EXECUTING", "PAUSED", "PENDING"];
+
+export type CrashTaskRunServiceOptions = {
+  reason?: string;
+  exitCode?: number;
+  logs?: string;
+  crashAttempts?: boolean;
+  crashedAt?: Date;
+};
+
+export class CrashTaskRunService extends BaseService {
+  public async call(runId: string, options?: CrashTaskRunServiceOptions) {
+    const opts = {
+      reason: "Worker crashed",
+      crashAttempts: true,
+      crashedAt: new Date(),
+      ...options,
+    };
+
+    const taskRun = await this._prisma.taskRun.findFirst({
+      where: {
+        id: runId,
+      },
+    });
+
+    if (!taskRun) {
+      logger.error("Task run not found", { runId });
+      return;
+    }
+
+    // Make sure the task run is in a crashable state
+    if (!CRASHABLE_RUN_STATUSES.includes(taskRun.status)) {
+      return;
+    }
+
+    // Remove the task run from the queue if it's there for some reason
+    await marqs?.acknowledgeMessage(taskRun.id);
+
+    // Set the task run status to crashed
+    const crashedTaskRun = await this._prisma.taskRun.update({
+      where: {
+        id: taskRun.id,
+      },
+      data: {
+        status: "CRASHED",
+      },
+      include: {
+        attempts: {
+          where: {
+            status: {
+              in: CRASHABLE_ATTEMPT_STATUSES,
+            },
+          },
+          include: {
+            backgroundWorker: true,
+            runtimeEnvironment: true,
+          },
+        },
+        dependency: true,
+        runtimeEnvironment: {
+          include: {
+            organization: true,
+            project: true,
+          },
+        },
+      },
+    });
+
+    const inProgressEvents = await eventRepository.queryIncompleteEvents({
+      runId: taskRun.friendlyId,
+    });
+
+    logger.debug("Crashing in-progress events", {
+      inProgressEvents: inProgressEvents.map((event) => event.id),
+    });
+
+    await Promise.all(
+      inProgressEvents.map((event) => {
+        return eventRepository.crashEvent({
+          event: event,
+          crashedAt: opts.crashedAt,
+          exception: {
+            type: "Worker crashed",
+            message: opts.reason,
+            stacktrace: opts.logs,
+          },
+        });
+      })
+    );
+
+    if (!opts.crashAttempts) {
+      return;
+    }
+
+    // Cancel any in progress attempts
+    for (const attempt of crashedTaskRun.attempts) {
+      await this.#failAttempt(
+        attempt,
+        crashedTaskRun,
+        new Date(),
+        crashedTaskRun.runtimeEnvironment
+      );
+    }
+  }
+
+  async #failAttempt(
+    attempt: TaskRunAttempt,
+    run: TaskRun,
+    failedAt: Date,
+    environment: AuthenticatedEnvironment
+  ) {
+    return await this.traceWithEnv("failAttempt()", environment, async (span) => {
+      span.setAttribute("taskRunId", run.id);
+      span.setAttribute("attemptId", attempt.id);
+
+      await marqs?.acknowledgeMessage(run.id);
+
+      await this._prisma.taskRunAttempt.update({
+        where: {
+          id: attempt.id,
+        },
+        data: {
+          status: "FAILED",
+          completedAt: failedAt,
+        },
+      });
+
+      if (environment.type === "DEVELOPMENT") {
+        return;
+      }
+
+      await ResumeTaskRunDependenciesService.enqueue(attempt.id, this._prisma);
+    });
+  }
+}

apps/webapp/app/v3/services/deploymentIndexFailed.server.ts

@@ -2,13 +2,19 @@ import { BaseService } from "./baseService.server";
 
 export class DeploymentIndexFailed extends BaseService {
   public async call(
-    deploymentId: string,
+    maybeFriendlyId: string,
     error: { name: string; message: string; stack?: string }
   ) {
+    const isFriendlyId = maybeFriendlyId.startsWith("deployment_");
+
     const deployment = await this._prisma.workerDeployment.update({
-      where: {
-        friendlyId: deploymentId,
-      },
+      where: isFriendlyId
+        ? {
+            friendlyId: maybeFriendlyId,
+          }
+        : {
+            id: maybeFriendlyId,
+          },
       data: {
         status: "FAILED",
         failedAt: new Date(),

apps/webapp/app/v3/services/indexDeployment.server.ts

@@ -56,6 +56,7 @@ export class IndexDeploymentService extends BaseService {
         envType: deployment.environment.type,
         projectId: deployment.projectId,
         orgId: deployment.environment.organizationId,
+        deploymentId: deployment.id,
       });
 
       logger.debug("Index ACK received", { responses });

apps/webapp/app/v3/services/replayTaskRun.server.ts

@@ -0,0 +1,53 @@
+import { conditionallyImportPacket, parsePacket } from "@trigger.dev/core/v3";
+import { Prisma, TaskRun } from "@trigger.dev/database";
+import { findEnvironmentById } from "~/models/runtimeEnvironment.server";
+import { logger } from "~/services/logger.server";
+import { BaseService } from "./baseService.server";
+import { TriggerTaskService } from "./triggerTask.server";
+
+type ExtendedTaskRunAttempt = Prisma.TaskRunAttemptGetPayload<{
+  include: {
+    runtimeEnvironment: true;
+    backgroundWorker: true;
+  };
+}>;
+
+export class ReplayTaskRunService extends BaseService {
+  public async call(existingTaskRun: TaskRun) {
+    const authenticatedEnvironment = await findEnvironmentById(
+      existingTaskRun.runtimeEnvironmentId
+    );
+    if (!authenticatedEnvironment) {
+      return;
+    }
+
+    logger.info("Replaying task run", {
+      taskRunId: existingTaskRun.id,
+      taskRunFriendlyId: existingTaskRun.friendlyId,
+    });
+
+    const payloadPacket = await conditionallyImportPacket({
+      data: existingTaskRun.payload,
+      dataType: existingTaskRun.payloadType,
+    });
+    const parsedPayload = await parsePacket(payloadPacket);
+
+    logger.info("Replaying task run payload", {
+      taskRunId: existingTaskRun.id,
+      taskRunFriendlyId: existingTaskRun.friendlyId,
+      payloadPacketType: payloadPacket.dataType,
+    });
+
+    const triggerTaskService = new TriggerTaskService();
+    return await triggerTaskService.call(existingTaskRun.taskIdentifier, authenticatedEnvironment, {
+      payload: parsedPayload,
+      options: {
+        queue: {
+          name: existingTaskRun.queue,
+        },
+        concurrencyKey: existingTaskRun.concurrencyKey ?? undefined,
+        test: existingTaskRun.isTest,
+      },
+    });
+  }
+}

apps/webapp/app/v3/tracer.server.ts

@@ -30,6 +30,8 @@ import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { singleton } from "~/utils/singleton";
 import { LoggerSpanExporter } from "./telemetry/loggerExporter.server";
 
+export const SEMINTATTRS_FORCE_RECORDING = "forceRecording";
+
 class CustomWebappSampler implements Sampler {
   constructor(private readonly _baseSampler: Sampler) {}
 
@@ -49,8 +51,22 @@ class CustomWebappSampler implements Sampler {
       return { decision: SamplingDecision.NOT_RECORD };
     }
 
+    // If the span has the forceRecording attribute, always record it
+    if (attributes[SEMINTATTRS_FORCE_RECORDING]) {
+      return { decision: SamplingDecision.RECORD_AND_SAMPLED };
+    }
+
     // For all other spans, defer to the base sampler
-    return this._baseSampler.shouldSample(context, traceId, name, spanKind, attributes, links);
+    const result = this._baseSampler.shouldSample(
+      context,
+      traceId,
+      name,
+      spanKind,
+      attributes,
+      links
+    );
+
+    return result;
   }
 
   toString(): string {
@@ -63,7 +79,7 @@ export const tracer = singleton("tracer", getTracer);
 function getTracer() {
   diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.ERROR);
 
-  const samplingRate = 1.0 / Math.max(parseInt(env.INTERNAL_OTEL_TRACE_SAMPING_RATE, 10), 1);
+  const samplingRate = 1.0 / Math.max(parseInt(env.INTERNAL_OTEL_TRACE_SAMPLING_RATE, 10), 1);
 
   const provider = new NodeTracerProvider({
     forceFlushTimeoutMillis: 500,
@@ -71,8 +87,8 @@ function getTracer() {
       [SEMRESATTRS_SERVICE_NAME]: env.SERVICE_NAME,
     }),
     sampler: new ParentBasedSampler({
-      root: new CustomWebappSampler(new TraceIdRatioBasedSampler(samplingRate)), // 5% sampling
-    }), // 5% sampling
+      root: new CustomWebappSampler(new TraceIdRatioBasedSampler(samplingRate)),
+    }),
   });
 
   if (env.INTERNAL_OTEL_TRACE_EXPORTER_URL) {
@@ -92,13 +108,15 @@ function getTracer() {
     provider.addSpanProcessor(
       new BatchSpanProcessor(exporter, {
         maxExportBatchSize: 512,
-        scheduledDelayMillis: 200,
+        scheduledDelayMillis: 1000,
         exportTimeoutMillis: 30000,
         maxQueueSize: 2048,
       })
     );
 
-    console.log(`🔦 Tracer: OTLP exporter enabled to ${env.INTERNAL_OTEL_TRACE_EXPORTER_URL}`);
+    console.log(
+      `🔦 Tracer: OTLP exporter enabled to ${env.INTERNAL_OTEL_TRACE_EXPORTER_URL} (sampling = ${samplingRate})`
+    );
   } else {
     if (env.INTERNAL_OTEL_TRACE_LOGGING_ENABLED === "1") {
       console.log(`🔦 Tracer: Logger exporter enabled`);

apps/webapp/server.ts

@@ -120,6 +120,10 @@ if (process.env.HTTP_SERVER_DISABLED !== "true") {
       )}`
     );
 
+    socket.on("error", (err) => {
+      console.error("Connection upgrade error:", err);
+    });
+
     const url = new URL(req.url ?? "", "http://localhost");
 
     // Upgrade socket.io connection
@@ -133,6 +137,7 @@ if (process.env.HTTP_SERVER_DISABLED !== "true") {
 
     // Only upgrade the connecting if the path is `/ws`
     if (url.pathname !== "/ws") {
+      // Setting the socket.destroy() error param causes an error event to be emitted which needs to be handled with socket.on("error") to prevent uncaught exceptions.
       socket.destroy(
         new Error(
           "Cannot connect because of invalid path: Please include `/ws` in the path of your upgrade request."

docs/_snippets/quickstart-whats-next.mdx

@@ -11,7 +11,7 @@
   >
     Learn more about how Trigger.dev works and how it can help you.
   </Card>
-  <Card title="Examples" icon="slot-machine" href="https:/trigger.dev/apis">
+  <Card title="Examples" icon="slot-machine" href="https://trigger.dev/apis">
     Find code examples for many popular APIs. These can be copied / modified for use in your own
     projects.
   </Card>

docs/_snippets/v3/step-cli-dev.mdx

@@ -0,0 +1,21 @@
+<Step title="Run the CLI `dev` command">
+
+The CLI `dev` command runs a server for your tasks. It will watches for changes in your `/trigger` directory and communicates with the Trigger.dev platform to register your tasks, perform runs, and send data back and forth.
+
+<CodeGroup>
+
+```bash npm
+npx trigger.dev@beta dev
+```
+
+```bash pnpm
+pnpm dlx trigger.dev@beta dev
+```
+
+```bash yarn
+yarn dlx trigger.dev@beta dev
+```
+
+</CodeGroup>
+
+</Step>

docs/_snippets/v3/step-cli-init.mdx

@@ -0,0 +1,30 @@
+<Step title="Run the CLI `init` command">
+
+The easiest way to get started it to use the CLI. It will add Trigger.dev to your existing project, create a `/trigger` folder and give you an example task.
+
+Run this command in the root of your project to get started:
+
+<CodeGroup>
+
+```bash npm
+npx trigger.dev@beta init
+```
+
+```bash pnpm
+pnpm dlx trigger.dev@beta init
+```
+
+```bash yarn
+yarn dlx trigger.dev@beta init
+```
+
+</CodeGroup>
+
+It will do a few things:
+
+1. Log you into the CLI if you're not already logged in.
+2. Create a `trigger.config.ts` file in the root of your project.
+3. Ask where you'd like to create the `/trigger` directory.
+4. Create the `/trigger` directory with an example task, `/trigger/example.[ts/js]`.
+
+</Step>

docs/_snippets/v3/step-run-test.mdx

@@ -0,0 +1,9 @@
+<Step title="Perform a test run using the dashboard">
+
+The CLI `dev` command spits out various useful URLs. Right now we want to visit the Test page it provided.
+
+You should see our Example task in the list, select it. Most tasks have a "payload" which you enter in the JSON editor, but our example task doesn't need any input.
+
+Press the "Run test" button.
+
+</Step>

docs/_snippets/v3/step-view-run.mdx

@@ -0,0 +1,7 @@
+<Step title="View your run">
+
+Congratulations, you should see the run page which will live reload showing you the current state of the run.
+
+If you go back to your terminal you'll see that the dev command also shows tasks that are running and links to the run log.
+
+</Step>

docs/documentation/introduction.mdx

@@ -1,9 +1,14 @@
 ---
-title: "Getting Started: Introduction"
+title: "v2: Introduction"
 sidebarTitle: "Introduction"
-description: "Welcome to the Trigger.dev documentation."
+description: "Welcome to the Trigger.dev (v2) documentation."
 ---
 
+<Warning>
+  These are the docs for Trigger.dev v2. You probably want [the v3
+  docs](https://trigger.dev/docs/v3).
+</Warning>
+
 Trigger.dev is an open source framework for creating long-running Jobs directly in your app with API Integrations, webhooks, scheduling and delays. You can reliably run Jobs that wouldn't normally work in serverless environments (like Vercel) because of timeouts.
 
 You can use [Trigger.dev Cloud](https://cloud.trigger.dev) or [Self-host Trigger.dev](/documentation/guides/self-hosting) on your own infrastructure.

docs/mint.json

@@ -1,7 +1,8 @@
 {
   "$schema": "https://mintlify.com/schema.json",
   "name": "Trigger.dev",
-  "openapi": ["/openapi.yml"],
+  "openapi": ["/openapi.yml", "/v3-openapi.json"],
+  "versions": ["v3 (Developer Preview)", "v2"],
   "logo": {
     "dark": "/logo/dark.png",
     "light": "/logo/light.png",
@@ -37,22 +38,34 @@
       "url": "https://discord.gg/kA47vcd8P6"
     }
   ],
+  "primaryTab": {
+    "name": "Home"
+  },
   "tabs": [
     {
       "name": "Integrations",
-      "url": "integrations"
+      "url": "integrations",
+      "version": "v2"
     },
     {
       "name": "Reference",
-      "url": "sdk"
+      "url": "sdk",
+      "version": "v2"
     },
     {
       "name": "Examples",
-      "url": "https://trigger.dev/apis"
+      "url": "https://trigger.dev/apis",
+      "version": "v2"
     },
     {
       "name": "v3 Developer Preview",
-      "url": "v3"
+      "url": "https://trigger.dev/docs/v3",
+      "version": "v2"
+    },
+    {
+      "name": "v2",
+      "url": "https://trigger.dev/docs",
+      "version": "v3 (Developer Preview)"
     }
   ],
   "redirects": [
@@ -76,6 +89,7 @@
   "navigation": [
     {
       "group": "Getting Started",
+      "version": "v2",
       "pages": [
         "documentation/introduction",
         {
@@ -107,6 +121,7 @@
     },
     {
       "group": "Concepts",
+      "version": "v2",
       "pages": [
         "documentation/concepts/what-is-triggerdotdev",
         "documentation/concepts/limits",
@@ -142,6 +157,7 @@
     },
     {
       "group": "Guides",
+      "version": "v2",
       "pages": [
         {
           "group": "Frameworks",
@@ -226,6 +242,7 @@
     },
     {
       "group": "Community & Support",
+      "version": "v2",
       "pages": [
         "documentation/get-help",
         "documentation/faq",
@@ -430,24 +447,29 @@
     },
     {
       "group": "Overview",
+      "version": "v2",
       "pages": ["examples/introduction"]
     },
     {
       "group": "",
+      "version": "v3 (Developer Preview)",
       "pages": ["v3/introduction"]
     },
     {
       "group": "Getting Started",
+      "version": "v3 (Developer Preview)",
       "pages": [
         "v3/quick-start",
         "v3/upgrading-from-v2",
         "v3/changelog",
         "v3/feature-matrix",
-        "v3/limits"
+        "v3/known-issues",
+        "v3/limits-performance"
       ]
     },
     {
       "group": "Fundamentals",
+      "version": "v3 (Developer Preview)",
       "pages": [
         "v3/trigger-folder",
         "v3/tasks-overview",
@@ -456,15 +478,18 @@
         {
           "group": "Task types",
           "pages": ["v3/tasks-regular", "v3/tasks-scheduled", "v3/tasks-zod", "v3/tasks-webhooks"]
-        }
+        },
+        "v3/trigger-config"
       ]
     },
     {
       "group": "Development",
+      "version": "v3 (Developer Preview)",
       "pages": ["v3/cli-dev", "v3/run-tests"]
     },
     {
       "group": "Deployment",
+      "version": "v3 (Developer Preview)",
       "pages": [
         "v3/deploy-environment-variables",
         "v3/cli-deploy",
@@ -477,6 +502,7 @@
     },
     {
       "group": "Writing tasks",
+      "version": "v3 (Developer Preview)",
       "pages": [
         "v3/writing-tasks-introduction",
         "v3/logging",
@@ -506,6 +532,7 @@
     },
     {
       "group": "Dashboard",
+      "version": "v3 (Developer Preview)",
       "pages": [
         "v3/dashboard-overview",
         "v3/dashboard-runs",
@@ -515,6 +542,7 @@
     },
     {
       "group": "API reference",
+      "version": "v3 (Developer Preview)",
       "pages": [
         {
           "group": "Functions",
@@ -559,11 +587,21 @@
             "v3/reference-cli-who-am-i"
           ]
         },
-        "v3/reference-trigger-config"
+        {
+          "group": "Management API",
+          "pages": [
+            "v3/management-start-run",
+            "v3/management-get-run",
+            "v3/management-get-runs",
+            "v3/management-replay-run",
+            "v3/management-cancel-run"
+          ]
+        }
       ]
     },
     {
       "group": "Architecture",
+      "version": "v3 (Developer Preview)",
       "pages": [
         "v3/architecture-how-it-works",
         "v3/architecture-multi-tenant-queue",
@@ -572,10 +610,12 @@
     },
     {
       "group": "Open source",
+      "version": "v3 (Developer Preview)",
       "pages": ["v3/github-repo", "v3/open-source-self-hosting", "v3/open-source-contributing"]
     },
     {
       "group": "Help",
+      "version": "v3 (Developer Preview)",
       "pages": [
         "v3/help-faqs",
         "v3/community",
@@ -589,10 +629,5 @@
     "twitter": "https://twitter.com/triggerdotdev",
     "github": "https://github.com/triggerdotdev",
     "linkedin": "https://www.linkedin.com/company/triggerdotdev"
-  },
-  "analytics": {
-    "posthog": {
-      "apiKey": "phc_9aSDbJCaDUMdZdHxxMPTvcj7A9fsl3mCgM1RBPmPsl7"
-    }
   }
 }

docs/v3-openapi.json

@@ -0,0 +1,216 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "Trigger.dev v3 REST API",
+    "description": "The REST API lets you trigger and manage runs on Trigger.dev. You can trigger a run, get the status of a run, and get the results of a run. ",
+    "version": "2024-04"
+  },
+  "servers": [
+    {
+      "url": "https://api.trigger.dev",
+      "description": "Trigger.dev API"
+    }
+  ],
+  "paths": {
+    "/api/v1/runs/{run_id}/replay": {
+      "post": {
+        "description": "Creates a new run with the same payload and options as the original run.",
+        "parameters": [
+          {
+            "in": "path",
+            "name": "run_id",
+            "required": true,
+            "schema": {
+              "type": "string"
+            },
+            "description": "The ID of an existing run. When you trigger a run you will get an id in the response."
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Successful request",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "id": {
+                      "type": "string",
+                      "description": "The ID of the new run."
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "400": {
+            "description": "Invalid request",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "error": {
+                      "type": "string",
+                      "enum": ["Invalid or missing run ID", "Failed to create new run"]
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "401": {
+            "description": "Unauthorized request",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "error": {
+                      "type": "string",
+                      "enum": ["Invalid or Missing API key"]
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "404": {
+            "description": "Resource not found",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "error": {
+                      "type": "string",
+                      "enum": ["Run not found"]
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        "tags": ["run"],
+        "security": [{ "bearerAuth": [] }],
+        "operationId": "replay_run_v1",
+        "x-codeSamples": [
+          {
+            "lang": "typescript",
+            "source": "const handle = await runs.replay(existingRun.id);"
+          },
+          {
+            "lang": "sh",
+            "source": "curl --request POST \\\n\t--url https://api.trigger.dev/api/v1/runs/{run_id}/replay \\\n\t--header 'Authorization: Bearer <token>'"
+          }
+        ]
+      }
+    },
+    "/api/v1/runs/{run_id}/cancel": {
+      "post": {
+        "description": "Cancels a run.",
+        "parameters": [
+          {
+            "in": "path",
+            "name": "run_id",
+            "required": true,
+            "schema": {
+              "type": "string"
+            },
+            "description": "The ID of an existing run. When you trigger a run you will get an id in the response."
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Successful request",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "message": {
+                      "type": "string",
+                      "description": "Confirmation message that the run was canceled."
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "400": {
+            "description": "Invalid request",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "error": {
+                      "type": "string",
+                      "enum": ["Invalid or missing run ID", "Failed to create new run"]
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "401": {
+            "description": "Unauthorized request",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "error": {
+                      "type": "string",
+                      "enum": ["Invalid or Missing API key"]
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "404": {
+            "description": "Resource not found",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "error": {
+                      "type": "string",
+                      "enum": ["Run not found"]
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        "tags": ["run"],
+        "security": [{ "bearerAuth": [] }],
+        "operationId": "replay_run_v1",
+        "x-codeSamples": [
+          {
+            "lang": "typescript",
+            "source": "await runs.cancel(existingRun.id);"
+          },
+          {
+            "lang": "sh",
+            "source": "curl --request POST \\\n\t--url https://api.trigger.dev/api/v1/runs/{run_id}/cancel \\\n\t--header 'Authorization: Bearer <token>'"
+          }
+        ]
+      }
+    }
+  },
+  "components": {
+    "securitySchemes": {
+      "bearerAuth": {
+        "type": "http",
+        "scheme": "bearer",
+        "description": "Use your Secret API key in the form 'Bearer <SECRET KEY>' (without the quotation marks)"
+      }
+    }
+  },
+  "security": [{ "bearerAuth": [] }]
+}

docs/v3/errors-retrying.mdx

@@ -7,12 +7,12 @@ When an uncaught error is thrown inside your task, that task attempt will fail.
 
 You can configure retrying in two ways:
 
-1. In your [trigger.config file](/v3/reference-trigger-config) you can set the default retrying behavior for all tasks.
+1. In your [trigger.config file](/v3/trigger-config) you can set the default retrying behavior for all tasks.
 2. On each task you can set the retrying behavior.
 
 <Note>
   By default when you create your project using the CLI init command we disabled retrying in the DEV
-  environment. You can enable it in your [trigger.config file](/v3/reference-trigger-config).
+  environment. You can enable it in your [trigger.config file](/v3/trigger-config).
 </Note>
 
 ## A simple example with OpenAI

docs/v3/feature-matrix.mdx

@@ -3,8 +3,24 @@ title: "Feature matrix"
 description: "What features are currently available in the Developer Preview"
 ---
 
-| Feature                                | Description                                | Status |
-| -------------------------------------- | ------------------------------------------ | ------ |
-| [Regular tasks](/v3/tasks-regular)     | A task that can be triggered from anywhere | ✅     |
-| [Scheduled tasks](/v3/tasks-scheduled) | A task that can be triggered on a schedule | ⏳     |
-| [Webhook tasks](v3/tasks-webhooks)     | A task that can be triggered by a webhook  | ⏳     |
+| Feature                                                                             | Description                                        | Status |
+| ----------------------------------------------------------------------------------- | -------------------------------------------------- | ------ |
+| [Regular tasks](/v3/tasks-regular)                                                  | A task that can be triggered from anywhere         | ✅     |
+| [Triggering](/v3/triggering)                                                        | Triggering and batch triggering tasks              | ✅     |
+| [Testing from the dashboard](/v3/run-tests)                                         | Test your tasks from the dashboard                 | ✅     |
+| [Queues and concurrency controls](/v3/queue-concurrency)                            | Queues and concurrency controls                    | ✅     |
+| [Per-tenant queuing](/v3/queue-concurrency#concurrency-keys-and-per-tenant-queuing) | Separate queues for each of your users             | ✅     |
+| [Reattempts and retrying](/v3/errors-retrying)                                      | Write reliable tasks using retries                 | ✅     |
+| [Atomic versioning](/v3/versioning)                                                 | Each deploy creates a new version                  | ✅     |
+| [Deploy via CLI](/v3/cli-deploy)                                                    | Deploy from the command line                       | ✅     |
+| [Deploy via GitHub Actions](/v3/github-actions)                                     | Deploy using GitHub Actions                        | ✅     |
+| [Scheduled tasks](/v3/tasks-scheduled)                                              | A task that can be triggered on a schedule         | ⏳     |
+| [Zod tasks](/v3/tasks-zod)                                                          | Define tasks using Zod schemas                     | ⏳     |
+| [Webhook tasks](/v3/tasks-webhooks)                                                 | A task that can be triggered by a webhook          | ⏳     |
+| Full text search of runs                                                            | Find a run by searching the payload and output     | ⏳     |
+| Logs view with search                                                               | All logs view with filtering and full text search  | ⏳     |
+| Alerts                                                                              | Add alerts in the UI for errors and queue backlogs | ⏳     |
+| Notifications                                                                       | Send data to your web app from a run               | ⏳     |
+| Rollbacks                                                                           | Easily rollback changes when errors happen         | ⏳     |
+
+[Let us know](https://trigger.dev/discord) what we should prioritize and what we are missing.

docs/v3/known-issues.mdx

@@ -0,0 +1,12 @@
+---
+title: "Known issues"
+description: "Please let us know if you encounter an issue not on this list."
+---
+
+#### `pnpm` is not currently supported
+
+By default, pnpm has a more complicated package structure than npm. If you see this error when running the CLI then you're experiencing this bug:
+
+```bash
+✘ [ERROR] Could not resolve "@trigger.dev/core/v3"
+```

docs/v3/limits-performance.mdx

@@ -1,5 +1,5 @@
 ---
-title: "Limits"
+title: "Limits & Performance"
 ---
 
 <Snippet file="incomplete-docs.mdx" />

docs/v3/logging.mdx

@@ -38,18 +38,20 @@ Tracing is a way to follow the flow of your code. It's very useful for debugging
 
 Trigger.dev uses OpenTelemetry tracing under the hood. With automatic tracing for many things like task triggering, task attempts, HTTP requests, and more.
 
-### Automatic instrumentation
-
 | Name          | Description                      |
 | ------------- | -------------------------------- |
 | Task triggers | Task triggers.                   |
 | Task attempts | Task attempts.                   |
 | HTTP requests | HTTP requests made by your code. |
-| OpenAI        | OpenAI SDK calls.                |
 
-We want to provide automatic instrumentation for as many things as possible. Please do [request any automatic instrumentation](https://github.com/triggerdotdev/trigger.dev/issues/new?template=instrumentation_request.yml) you would like to see.
+### Adding instrumentations
 
-## Add custom traces
+![The run log
+  ](/images/v3/auto-instrumentation.png)
+
+You can [add instrumentations](/v3/trigger-config#instrumentations). The Prisma one above will automatically trace all Prisma queries.
+
+### Add custom traces
 
 If you want to add custom traces to your code, you can use the `logger.trace` function. It will create a new OTEL trace and you can set attributes on it.
 

docs/v3/management-cancel-run.mdx

@@ -0,0 +1,4 @@
+---
+title: "Cancel run"
+openapi: "v3-openapi POST /api/v1/runs/{run_id}/cancel"
+---

docs/v3/management-get-run.mdx

@@ -0,0 +1,6 @@
+---
+title: "Get run"
+description: "Get a run using the Task id."
+---
+
+<Snippet file="incomplete-docs.mdx" />

docs/v3/management-get-runs.mdx

@@ -0,0 +1,6 @@
+---
+title: "Get runs"
+description: "Get runs using a Task id."
+---
+
+<Snippet file="incomplete-docs.mdx" />

docs/v3/management-replay-run.mdx

@@ -0,0 +1,4 @@
+---
+title: "Replay run"
+openapi: "v3-openapi POST /api/v1/runs/{run_id}/replay"
+---

docs/v3/management-start-run.mdx

@@ -0,0 +1,6 @@
+---
+title: "Start run"
+description: "Start a run using the Task id, payload and options."
+---
+
+<Snippet file="incomplete-docs.mdx" />

docs/v3/migration-defer.mdx

@@ -0,0 +1,261 @@
+---
+title: "Migrating from Defer.run"
+description: "A guide to migrating from Defer to Trigger.dev v3"
+---
+
+First of all, you should use the [Trigger.dev v3 Developer Preview](https://trigger.dev/blog/v3-developer-preview-launch/), not version 2. Version 2 is not a good choice if you're coming from Defer as the execution model is very different. [Let us know](https://) if you need v3 access.
+
+This guide highlights the differences and should help you migrate your project.
+
+## Features coming very soon
+
+Here are some features you might be using in Defer that are coming this month to v3:
+
+- [Scheduled tasks (including CRON)](/v3/tasks-scheduled) will be available in mid-April.
+- Triggering a task with a delay (like `assignOptions` delay in Defer) will be available soon – there is [an alternative](#delay) you can use for now.
+
+You can view the full feature matrix [here](/v3/feature-matrix).
+
+## Differences
+
+#### Local development
+
+In Defer you run your tasks locally using `npm run dev` (or other package manager). This is simple but it means dev behaved differently from production for your background tasks.
+
+With Trigger.dev you need to use our CLI to run a local server that behaves like the deployed production environment. It also means you will see your runs in the dashboard.
+
+#### Multiple tasks in a single file
+
+In Defer you needed to use a default export in a file inside your `/defer` directory.
+
+```ts /defer/longRunningTask.ts
+import { defer } from "@defer/client";
+
+async function longRunningTask() {
+  // runs a fake task for 30s
+  await performLongRunningTask();
+}
+
+export default defer(longRunningTask);
+```
+
+In Trigger.dev **you use named exports** so you can have multiple tasks in a single file.
+
+```ts /trigger/someTasks.ts
+export const longRunningTask = task({
+  id: "longRunningTask",
+  run: async (payload: any) => {
+    //...do stuff
+  },
+});
+
+export const otherTask = task({
+  id: "otherTask",
+  run: async (payload: any) => {
+    //...do different stuff
+  },
+});
+```
+
+#### Triggering your tasks
+
+In Defer, you wrapped your existing function in `defer()`. Then for simple cases you could just call the function. In other cases, like when you wanted to have a delay you needed to use `assignOptions` to create a new function.
+
+```ts /app/actions/actions.ts
+"use server";
+
+import longRunningTask from "@/defer/longRunningTask";
+
+export async function runLongRunningTask() {
+  return await longRunningTask();
+}
+```
+
+In Trigger.dev your logic goes in the `run` function of a task. You can then `trigger` and `batchTrigger` that task, with a payload and options.
+
+```ts /app/actions/actions.ts
+"use server";
+
+import { longRunningTask } from "@/trigger/someTasks";
+
+export async function runLongRunningTask() {
+  return await longRunningTask.trigger({ payload: { foo: "bar" } });
+}
+```
+
+#### `wait`
+
+In Trigger.dev you can use the [wait](/v3/wait) functions to freeze execution of your code until a later date (it can be months later). You won't pay while it's frozen and the state is restored exactly when it wakes (using a technology called CRIU).
+
+```ts
+//In Defer you could use "sleep" but that would keep your function running.
+await sleep(1000 * 60 * 5); // 5 minutes but you'd pay for it.
+
+//In Trigger.dev you can use wait. We freeze execution if it's more than 30s
+await wait.for({ seconds: 5 });
+await wait.for({ minutes: 10 });
+await wait.for({ hours: 1 });
+await wait.for({ days: 1 });
+await wait.for({ weeks: 1 });
+await wait.for({ months: 1 });
+await wait.for({ years: 1 });
+
+//you can wait for a date too
+await wait.until({ date: aFutureDate });
+```
+
+#### delay the start of a run
+
+In Defer you can do this:
+
+```ts
+const delayedRun = assignOptions(someTask, { delay: "10s" });
+await delayedRun();
+```
+
+There will be a nice way to do this soon when you call `trigger()` but for now you can use `wait` to get the same behavior:
+
+```ts
+export const helloWorld = task({
+  id: "hello-world",
+  run: async (payload: { delayUntil?: string; delayForSeconds?: number }) => {
+    if (payload.delayUntil) {
+      await wait.until({ date: new Date(payload.delayUntil) });
+    }
+
+    if (payload.delayForSeconds) {
+      await wait.for({ seconds: payload.delayForSeconds });
+    }
+
+    //do stuff
+  },
+});
+```
+
+## How to migrate
+
+### 1. Get Trigger.dev working in your project
+
+<Steps>
+
+<Step title="Create an organization on Trigger.dev">
+
+1. Go to the [Trigger.dev Cloud](https://cloud.trigger.dev)
+2. Create an account
+3. Create an organization with a project (this will be a version 2 project)
+4. [DM us on Discord](https://trigger.dev/discord) or [fill in this form](https://trigger.dev/v3-early-access) and mention Defer in the company name.
+
+We will grant you v3 access.
+
+</Step>
+
+<Step title="Create a v3 project">
+
+1. Go to the Projects page
+
+![Click the Projects page](/images/v3/sidemenu-projects.png)
+
+2. Click "Create a new project"
+
+![Click "Create a new project"](/images/v3/projects-new-button.png)
+
+3. Make sure you select "Version 3" from the dropdown!
+
+![Select "Version 3"](/images/v3/projects-new-v3.png)
+
+<Warning>
+  If you don't see a dropdown then you don't have v3 access. [Fill in this
+  form](https://trigger.dev/v3-early-access) and mention Defer in the company name.
+</Warning>
+
+</Step>
+
+<Snippet file="v3/step-cli-init.mdx" />
+<Snippet file="v3/step-cli-dev.mdx" />
+<Snippet file="v3/step-run-test.mdx" />
+<Snippet file="v3/step-view-run.mdx" />
+
+</Steps>
+
+### 2. Migrate your Defer functions to Trigger.dev tasks
+
+#### Example 1: Simple function
+
+In Defer you might have a function like this.
+
+<CodeGroup>
+
+```ts /defer/longRunningTask.ts
+import { performLongRunningTask } from "@/utils/performLongRunningTask";
+import { defer } from "@defer/client";
+
+async function longRunningTask() {
+  // runs a fake task for 30s
+  await performLongRunningTask();
+}
+
+export default defer(longRunningTask, {
+  concurrency: 2, // want maximum 2 executions of this function in parallel
+  retry: 5, // adding retry to recover from potential network issues or rate limiting
+});
+```
+
+```ts /app/actions/actions.ts
+"use server";
+
+import longRunningTask from "@/defer/longRunningTask";
+
+export async function runLongRunningTask() {
+  return await longRunningTask();
+}
+```
+
+</CodeGroup>
+
+In Trigger.dev it looks like this:
+
+<CodeGroup>
+
+```ts /trigger/someTasks.ts
+import { performLongRunningTask } from "@/utils/performLongRunningTask";
+import { task } from "@trigger.dev/sdk/v3";
+
+//named export
+export const longRunningTask = task({
+  //a unique and stable ID so you can refactor the function name
+  id: "long-running-task",
+  queue: {
+    concurrencyLimit: 2, // want maximum 2 executions of this function in parallel
+  },
+  retry: {
+    maxAttempts: 5, // adding retry to recover from potential network issues or rate limiting
+  },
+  run: async (payload: any) => {
+    // runs a fake task for 30s
+    await performLongRunningTask();
+  },
+});
+```
+
+```ts /app/actions/actions.ts
+"use server";
+
+import { longRunningTask } from "@/trigger/longRunningTask";
+
+export async function runLongRunningTask() {
+  return await longRunningTask.trigger({ payload: { foo: "bar" } });
+}
+```
+
+</CodeGroup>
+
+<Warning>
+  You need to set your `TRIGGER_SECRET_KEY` environment variable in your `.env` or `.env.local` file
+  to trigger tasks from your code. See the [API keys page](/v3/apikeys) for more information.
+</Warning>
+
+#### Example 2: A CRON task
+
+<Warning>
+  "Scheduled" tasks will be available by mid-April. This will allow you to replace Defer CRON tasks.
+</Warning>

docs/v3/quick-start.mdx

@@ -34,76 +34,10 @@ Once you've created an account, follow the steps in the app to:
 
 </Step>
 
-<Step title="Run the CLI `init` command">
-
-The easiest way to get started it to use the CLI. It will add Trigger.dev to your existing project, create a `/trigger` folder and give you an example task.
-
-Run this command in the root of your project to get started:
-
-<CodeGroup>
-
-```bash npm
-npx trigger.dev@beta init
-```
-
-```bash pnpm
-pnpm dlx trigger.dev@beta init
-```
-
-```bash yarn
-yarn dlx trigger.dev@beta init
-```
-
-</CodeGroup>
-
-It will do a few things:
-
-1. Log you into the CLI if you're not already logged in.
-2. Create a `trigger.config.ts` file in the root of your project.
-3. Ask where you'd like to create the `/trigger` directory.
-4. Create the `/trigger` directory with an example task, `/trigger/example.[ts/js]`.
-
-</Step>
-
-<Step title="Run the CLI `dev` command">
-
-The CLI `dev` command runs a server for your tasks. It will watches for changes in your `/trigger` directory and communicates with the Trigger.dev platform to register your tasks, perform runs, and send data back and forth.
-
-<CodeGroup>
-
-```bash npm
-npx trigger.dev@beta dev
-```
-
-```bash pnpm
-pnpm dlx trigger.dev@beta dev
-```
-
-```bash yarn
-yarn dlx trigger.dev@beta dev
-```
-
-</CodeGroup>
-
-</Step>
-
-<Step title="Perform a run test using the dashboard">
-
-The CLI `dev` command spits out various useful URLs. Right now we want to visit the Test page it provided.
-
-You should see our Example task in the list, select it. Most tasks have a "payload" which you enter in the JSON editor, but our example task doesn't need any input.
-
-Press the "Run test" button.
-
-</Step>
-
-<Step title="View your run">
-
-Congratulations, you should see the run page which will live reload showing you the current state of the run.
-
-If you go back to your terminal you'll see that the dev command also shows tasks that are running and links to the run log.
-
-</Step>
+<Snippet file="v3/step-cli-init.mdx" />
+<Snippet file="v3/step-cli-dev.mdx" />
+<Snippet file="v3/step-run-test.mdx" />
+<Snippet file="v3/step-view-run.mdx" />
 
 </Steps>
 

docs/v3/reference-trigger-config.mdx

@@ -1,7 +0,0 @@
----
-title: "The trigger.config.js file"
-sidebarTitle: "trigger.config file"
-description: "This file is used to configure some settings for your project."
----
-
-<Snippet file="incomplete-docs.mdx" />

docs/v3/tasks-overview.mdx

@@ -27,7 +27,7 @@ export const helloWorld = task({
 
 You can trigger this in two ways:
 
-1. From the dashboard [using the "Test" feature](/v3/develop-run-tests).
+1. From the dashboard [using the "Test" feature](/v3/run-tests).
 2. Trigger it from your backend code. See the [full triggering guide here](/v3/triggering).
 
 Here's how to trigger a single run from elsewhere in your code:
@@ -149,6 +149,9 @@ When a task attempt fails, the `onError` function is called. It's useful for sen
 ## Next steps
 
 <CardGroup>
+  <Card title="Triggering" icon="bolt" href="/v3/triggering">
+    Learn how to trigger your tasks from your code.
+  </Card>
   <Card title="Writing tasks" icon="wand-magic-sparkles" href="/v3/writing-tasks-introduction">
     Tasks are the core of Trigger.dev. Learn how to write them.
   </Card>

docs/v3/trigger-config.mdx

@@ -0,0 +1,127 @@
+---
+title: "The trigger.config.ts file"
+sidebarTitle: "trigger.config file"
+description: "This file is used to configure your project and how it's bundled."
+---
+
+Let's take a look at a basic `trigger.config.ts` file. This is generated for you when you follow [the quick start guide](/v3/quick-start). This file is used to configure your project and how it's bundled.
+
+```ts trigger.config.ts
+import type { TriggerConfig } from "@trigger.dev/sdk/v3";
+
+export const config: TriggerConfig = {
+  //Your project ref (you can see it on the Project settings page in the dashboard)
+  project: "proj_gtcwttqhhtlasxgfuhxs",
+  retries: {
+    //If you want to retry a task in dev mode (when using the CLI)
+    enabledInDev: false,
+    //the default retry settings. Used if you don't specify on a task.
+    default: {
+      maxAttempts: 3,
+      minTimeoutInMs: 1000,
+      maxTimeoutInMs: 10000,
+      factor: 2,
+      randomize: true,
+    },
+  },
+  //The paths for your trigger folders
+  triggerDirectories: ["./trigger"],
+};
+```
+
+Most of the time you don't need to change anything in this file, or if you do then we will tell you when you the run the CLI command.
+
+## Instrumentations
+
+We use OpenTelemetry (OTEL) for our run logs. This means you get a lot of information about your tasks with no effort. But you probably want to add more information to your logs. For example, here's all the Prisma calls automatically logged:
+
+![The run log
+  ](/images/v3/auto-instrumentation.png)
+
+Here we add Prisma and OpenAI instrumentations to your `trigger.config.ts` file.
+
+```ts trigger.config.ts
+import type { TriggerConfig } from "@trigger.dev/sdk/v3";
+import { PrismaInstrumentation } from "@prisma/instrumentation";
+import { OpenAIInstrumentation } from "@traceloop/instrumentation-openai";
+
+export const config: TriggerConfig = {
+  //..other stuff
+  instrumentations: [new PrismaInstrumentation(), new OpenAIInstrumentation()],
+};
+```
+
+## ESM-only packages
+
+We'll let you know when run the CLI dev command if this is a problem. Some packages are ESM-only so they don't work directly from CJS when using Node.js. In that case you need to add them to the `dependenciesToBundle` array in your `trigger.config.ts` file.
+
+```ts trigger.config.ts
+import type { TriggerConfig } from "@trigger.dev/sdk/v3";
+
+export const config: TriggerConfig = {
+  //..other stuff
+  //either regex or strings of package names
+  dependenciesToBundle: [/@sindresorhus/, "escape-string-regexp"],
+};
+```
+
+## Prisma (and other generators)
+
+<Accordion title="The Prisma error you might see">
+
+```bash
+✘ [ERROR] Error: @prisma/client did not initialize yet. Please run "prisma generate" and try to import it again.
+
+In case this error is unexpected for you, please report it in
+https://pris.ly/prisma-prisma-bug-report
+at new PrismaClient (/app/node_modules/.prisma/client/default.js:43:11)
+at Object.<anonymous> (/lib/prisma.ts:7:33)
+at Module.\_compile (node:internal/modules/cjs/loader:1356:14)
+at Object.Module.\_extensions..js (node:internal/modules/cjs/loader:1414:10)
+at Module.load (node:internal/modules/cjs/loader:1197:32)
+at Function.Module.\_load (node:internal/modules/cjs/loader:1013:12)
+at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:128:12)
+at node:internal/main/run_main_module:28:49
+```
+
+</Accordion>
+
+Prisma works by generating a client from your `prisma.schema` file. This means you need to do a couple of things to get it to work with Trigger:
+
+<Steps>
+
+  <Step title="package.json postinstall `prisma generate`">
+
+    ```json
+    {
+      "scripts": {
+        "postinstall": "prisma generate"
+      }
+    }
+    ```
+
+    Anything you put in `postinstall` will be run as part of the install step. This is how Next.js recommends you set up Prisma anyway.
+
+</Step>
+
+  <Step title="Add prisma and the schema to trigger.config.ts">
+
+    ```ts trigger.config.ts
+    import type { TriggerConfig } from "@trigger.dev/sdk/v3";
+
+    export const config: TriggerConfig = {
+      //..other stuff
+      additionalFiles: ["./prisma/schema.prisma"],
+      additionalPackages: ["prisma@5.11.0"],
+    };
+    ```
+
+    This tells Trigger to bundle the Prisma client and the schema file.
+
+</Step>
+
+</Steps>
+
+## Troubleshooting
+
+If you have an issue with bundling let us know on [Discord](https://trigger.dev/discord) or [via email](https://trigger.dev/contact).

docs/v3/trigger-folder.mdx

@@ -11,7 +11,7 @@ We automatically bundle everything for your tasks. This includes:
 - Imported npm packages.
 - Other imports from your code.
 
-This means you shouldn't need to think about what gets bundled. Just write your tasks and we'll take care of the rest.
+Mostly this means you shouldn't need to think about what gets bundled. Just write your tasks and we'll take care of the rest. If you need to alter the bundling you use the [trigger.config file](/v3/trigger-config).
 
 ## Multiple `/trigger` folders
 
@@ -22,4 +22,4 @@ You can have multiple `/trigger` folders in your repository.
 
 ### (Optional) configuration
 
-It is possible to manually set one or more folders as `/trigger` folders in your `trigger.config` file. View the [trigger.config documentation](/v3/referece-trigger-config) for more information.
+It is possible to manually set one or more folders as `/trigger` folders in your [trigger.config file](/v3/trigger-config).

docs/v3/triggering.mdx

@@ -8,7 +8,7 @@ There are currently four ways you can trigger any task from your own code:
 | Function                         | Where does this work? | What it does                                                                                                                       |
 | -------------------------------- | --------------------- | ---------------------------------------------------------------------------------------------------------------------------------- |
 | `yourTask.trigger()`             | Anywhere              | Triggers a task and gets a handle you can use to monitor and manage the run. It does not wait for the result.                      |
-| `yourTask.batchTriggerAndWait()` | Anywhere              | Triggers a task multiple times and gets a handle you can use to monitor and manage the runs. It does not wait for the results.     |
+| `yourTask.batchTrigger()`        | Anywhere              | Triggers a task multiple times and gets a handle you can use to monitor and manage the runs. It does not wait for the results.     |
 | `yourTask.triggerAndWait()`      | Inside a task         | Triggers a task and then waits until it's complete. You get the result data to continue with.                                      |
 | `yourTask.batchTriggerAndWait()` | Inside a task         | Triggers a task multiple times in parallel and then waits until they're all complete. You get the resulting data to continue with. |
 
@@ -23,6 +23,8 @@ You can trigger any task from your backend code, using either `trigger()` or `ba
   Trigger.dev API key to the world.
 </Note>
 
+You can use Next.js Server Actions but [you need to be careful with bundling](#next-js-server-actions).
+
 ### Authentication
 
 When you trigger a task from your backend code, you need to set the `TRIGGER_SECRET_KEY` environment variable. You can find the value on the API keys page in the Trigger.dev dashboard. [More info on API keys](/v3/apikeys).
@@ -188,3 +190,71 @@ export const batchParentTask = task({
   },
 });
 ```
+
+## Next.js Server Actions
+
+Server Actions allow you to call your backend code without creating API routes. This is very useful for triggering tasks but you need to be careful you don't accidentally bundle the Trigger.dev SDK into your frontend code.
+
+If you see an error like this then you've bundled `@trigger.dev/sdk` into your frontend code:
+
+```bash
+Module build failed: UnhandledSchemeError: Reading from "node:crypto" is not handled by plugins (Unhandled scheme).
+Module build failed: UnhandledSchemeError: Reading from "node:process" is not handled by plugins (Unhandled scheme).
+Webpack supports "data:" and "file:" URIs by default.
+You may need an additional plugin to handle "node:" URIs.
+```
+
+When you use server actions that use `@trigger.dev/sdk`:
+
+- The file can't have any React components in it.
+- The file should have `"use server"` on the first line.
+
+Here's an example of how to do it with a component that calls the server action and the actions file:
+
+<CodeGroup>
+
+```tsx app/page.tsx
+"use client";
+
+import { create } from "@/app/actions";
+
+export default function Home() {
+  return (
+    <main className="flex min-h-screen flex-col items-center justify-between p-24">
+      <button
+        onClick={async () => {
+          const handle = await create();
+          console.log(handle);
+        }}
+      >
+        Create a thing
+      </button>
+    </main>
+  );
+}
+```
+
+```tsx app/actions.ts
+"use server";
+
+import { createAvatar } from "@/trigger/create-avatar";
+
+export async function create() {
+  try {
+    const handle = await createAvatar.trigger({
+      payload: {
+        userImage: "http://...",
+      },
+    });
+
+    return { handle };
+  } catch (error) {
+    console.error(error);
+    return {
+      error: "something went wrong",
+    };
+  }
+}
+```
+
+</CodeGroup>

docs/v3/upgrading-from-v2.mdx

@@ -1,6 +1,200 @@
 ---
 title: "Upgrading from v2"
-description: "How to upgrade your v2 jobs to v3 tasks."
+description: "How to upgrade v2 jobs to v3 tasks, and how to use them together."
 ---
 
+## v3 feature matrix
+
+As v3 is in Developer Preview there are some features that are not yet available. Take a look at the [Feature matrix](/v3/feature-matrix) to check what is currently available.
+
+## Changes from v2 to v3
+
+The main difference is that things in v3 are far simpler. That's because in v3 your code is deployed to our servers (unless you self-host) which are long-running.
+
+1. No timeouts.
+2. No `io.runTask()` (and no `cacheKeys`).
+3. Just use official SDKs, not integrations.
+4. `task`s are the new primitive, not `job`s.
+
+## OpenAI example comparison
+
+This is a (very contrived) example that does a long OpenAI API call (>10s), stores the result in a database, waits for 5 mins, and then returns the result.
+
+### v2
+
+First, the old v2 code, which uses the OpenAI integration. Comments inline:
+
+```ts v2 OpenAI task
+import { client } from "~/trigger";
+import { eventTrigger } from "@trigger.dev/sdk";
+
+//1. A Trigger.dev integration for OpenAI
+import { OpenAI } from "@trigger.dev/openai";
+const openai = new OpenAI({
+  id: "openai",
+  apiKey: process.env["OPENAI_API_KEY"]!,
+});
+
+//2. Use the client to define a "Job"
+client.defineJob({
+  id: "openai-tasks",
+  name: "OpenAI Tasks",
+  version: "0.0.1",
+  trigger: eventTrigger({
+    name: "openai.tasks",
+    schema: z.object({
+      prompt: z.string(),
+    }),
+  }),
+  //3. integrations are added and come through to `io` in the run fn
+  integrations: {
+    openai,
+  },
+  run: async (payload, io, ctx) => {
+    //4. You use `io` to get the integration
+    //5. Also note that "backgroundCreate" was needed for OpenAI
+    //   to do work that lasted longer than your serverless timeout
+    const chatCompletion = await io.openai.chat.completions.backgroundCreate(
+      //6. You needed to add "cacheKeys" to any "task"
+      "background-chat-completion",
+      {
+        messages: [{ role: "user", content: payload.prompt }],
+        model: "gpt-3.5-turbo",
+      }
+    );
+
+    const result = chatCompletion.choices[0]?.message.content;
+    if (!result) {
+      //7. throwing an error at the top-level in v2 failed the task immediately
+      throw new Error("No result from OpenAI");
+    }
+
+    //8. io.runTask needed to be used to prevent work from happening twice
+    const dbRow = await io.runTask("store-in-db", async (task) => {
+      //9. Custom logic can be put here
+      //   Anything returned must be JSON-serializable, so no Date objects etc.
+      return saveToDb(result);
+    });
+
+    //10. Wait for 5 minutes.
+    //    You need a cacheKey and the 2nd param is a number
+    await io.wait("wait some time", 60 * 5);
+
+    //11. Anything returned must be JSON-serializable, so no Date objects etc.
+    return result;
+  },
+});
+```
+
+### v3
+
+In v3 we eliminate a lot of code mainly because we don't need tricks to try avoid timeouts. Here's the equivalent v3 code:
+
+```ts v3 OpenAI task
+import { logger, task, wait } from "@trigger.dev/sdk/v3";
+
+//1. Official OpenAI SDK
+import OpenAI from "openai";
+const openai = new OpenAI({
+  apiKey: process.env.OPENAI_API_KEY,
+});
+
+//2. Jobs don't exist now, use "task"
+export const openaiTask = task({
+  id: "openai-task",
+  //3. Retries happen if a task throws an error that isn't caught
+  //   The default settings are in your trigger.config.ts (used if not overriden here)
+  retry: {
+    maxAttempts: 3,
+  },
+  run: async (payload: { prompt: string }) => {
+    //4. Use the official SDK
+    //5. No timeouts, so this can take a long time
+    const chatCompletion = await openai.chat.completions.create({
+      messages: [{ role: "user", content: payload.prompt }],
+      model: "gpt-3.5-turbo",
+    });
+
+    const result = chatCompletion.choices[0]?.message.content;
+    if (!result) {
+      //6. throwing an error at the top-level will retry the task (if retries are enabled)
+      throw new Error("No result from OpenAI");
+    }
+
+    //7. No need to use runTask, just call the function
+    const dbRow = await saveToDb(result);
+
+    //8. You can provide seconds, minutes, hours etc.
+    //   You don't need cacheKeys in v3
+    await wait.for({ minutes: 5 });
+
+    //9. You can return anything that's serializable using SuperJSON
+    //   That includes undefined, Date, bigint, RegExp, Set, Map, Error and URL.
+    return result;
+  },
+});
+```
+
+## Triggering tasks comparison
+
+### v2
+
+In v2 there were different trigger types and triggering each type was slightly different.
+
+```ts v2 triggering
+async function yourBackendFunction() {
+  //1. for `eventTrigger` you use `client.sendEvent`
+  const event = await client.sendEvent({
+    name: "openai.tasks",
+    payload: { prompt: "Create a good programming joke about background jobs" },
+  });
+
+  //2. for `invokeTrigger` you'd call `invoke` on the job
+  const { id } = await invocableJob.invoke({
+    prompt: "What is the meaning of life?",
+  });
+}
+```
+
+### v3
+
+We've unified triggering in v3. You use `trigger()` or `batchTrigger()` which you can do on any type of task. Including scheduled, webhooks, etc if you want.
+
+```ts v3 triggering
+async function yourBackendFunction() {
+  //call `trigger()` on any task
+  const handle = await openaiTask.trigger({
+    payload: {
+      prompt: "Tell me a programming joke",
+    },
+  });
+}
+```
+
+## Upgrading your project
+
+<Steps>
+
+<Step title="Upgrade the v2 Trigger.dev packages">
+
+You can run this command to upgrade all the packages to the beta:
+
+```bash
+npx @trigger.dev/cli@beta update --to beta
+```
+
+</Step>
+
+<Step title="Follow the v3 quick start">
+
+Follow the [v3 quick start](/v3/quick-start) to get started with v3.
+
+</Step>
+
+</Steps>
+
+## Using v2 together with v3
+
+You can use v2 and v3 in the same codebase. This can be useful where you already have v2 jobs or where we don't support features you need (yet).
+
 <Snippet file="incomplete-docs.mdx" />

integrations/airtable/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @trigger.dev/airtable
 
+## 3.0.0-beta.7
+
+### Patch Changes
+
+- Updated dependencies [f854cb90e]
+- Updated dependencies [f854cb90e]
+  - @trigger.dev/sdk@3.0.0-beta.7
+  - @trigger.dev/integration-kit@3.0.0-beta.7
+
+## 3.0.0-beta.6
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.6
+- @trigger.dev/sdk@3.0.0-beta.6
+
 ## 3.0.0-beta.5
 
 ### Patch Changes

integrations/airtable/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/airtable",
-  "version": "3.0.0-beta.5",
+  "version": "3.0.0-beta.7",
   "description": "Trigger.dev integration for airtable",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -25,8 +25,8 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.5",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.5",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.7",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.7",
     "airtable": "^0.12.1",
     "zod": "3.22.3"
   },

integrations/github/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @trigger.dev/github
 
+## 3.0.0-beta.7
+
+### Patch Changes
+
+- Updated dependencies [f854cb90e]
+- Updated dependencies [f854cb90e]
+  - @trigger.dev/sdk@3.0.0-beta.7
+  - @trigger.dev/integration-kit@3.0.0-beta.7
+
+## 3.0.0-beta.6
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.6
+- @trigger.dev/sdk@3.0.0-beta.6
+
 ## 3.0.0-beta.5
 
 ### Patch Changes

integrations/github/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/github",
-  "version": "3.0.0-beta.5",
+  "version": "3.0.0-beta.7",
   "description": "The official GitHub integration for Trigger.dev",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -30,8 +30,8 @@
     "@octokit/request-error": "^5.0.1",
     "@octokit/webhooks": "^12.0.10",
     "octokit": "^3.1.2",
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.5",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.5",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.7",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.7",
     "zod": "3.22.3"
   },
   "engines": {

integrations/linear/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @trigger.dev/linear
 
+## 3.0.0-beta.7
+
+### Patch Changes
+
+- Updated dependencies [f854cb90e]
+- Updated dependencies [f854cb90e]
+  - @trigger.dev/sdk@3.0.0-beta.7
+  - @trigger.dev/integration-kit@3.0.0-beta.7
+
+## 3.0.0-beta.6
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.6
+- @trigger.dev/sdk@3.0.0-beta.6
+
 ## 3.0.0-beta.5
 
 ### Patch Changes

integrations/linear/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/linear",
-  "version": "3.0.0-beta.5",
+  "version": "3.0.0-beta.7",
   "description": "Trigger.dev integration for @linear/sdk",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -26,8 +26,8 @@
   },
   "dependencies": {
     "@linear/sdk": "^8.0.0",
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.5",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.5",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.7",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.7",
     "zod": "3.22.3"
   },
   "engines": {

integrations/openai/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @trigger.dev/slack
 
+## 3.0.0-beta.7
+
+### Patch Changes
+
+- Updated dependencies [f854cb90e]
+- Updated dependencies [f854cb90e]
+  - @trigger.dev/sdk@3.0.0-beta.7
+  - @trigger.dev/integration-kit@3.0.0-beta.7
+
+## 3.0.0-beta.6
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.6
+- @trigger.dev/sdk@3.0.0-beta.6
+
 ## 3.0.0-beta.5
 
 ### Patch Changes

integrations/openai/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/openai",
-  "version": "3.0.0-beta.5",
+  "version": "3.0.0-beta.7",
   "description": "The official OpenAI integration for Trigger.dev",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -42,8 +42,8 @@
   },
   "dependencies": {
     "openai": "^4.16.1",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.5",
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.5"
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.7",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.7"
   },
   "engines": {
     "node": ">=18.0.0"

integrations/plain/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @trigger.dev/plain
 
+## 3.0.0-beta.7
+
+### Patch Changes
+
+- Updated dependencies [f854cb90e]
+- Updated dependencies [f854cb90e]
+  - @trigger.dev/sdk@3.0.0-beta.7
+  - @trigger.dev/integration-kit@3.0.0-beta.7
+
+## 3.0.0-beta.6
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.6
+- @trigger.dev/sdk@3.0.0-beta.6
+
 ## 3.0.0-beta.5
 
 ### Patch Changes

integrations/plain/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/plain",
-  "version": "3.0.0-beta.5",
+  "version": "3.0.0-beta.7",
   "description": "The official Plain.com integration for Trigger.dev",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -24,8 +24,8 @@
     "build:tsup": "tsup"
   },
   "dependencies": {
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.5",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.5",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.7",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.7",
     "@team-plain/typescript-sdk": "^2.7.0"
   },
   "engines": {

integrations/replicate/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @trigger.dev/replicate
 
+## 3.0.0-beta.7
+
+### Patch Changes
+
+- Updated dependencies [f854cb90e]
+- Updated dependencies [f854cb90e]
+  - @trigger.dev/sdk@3.0.0-beta.7
+  - @trigger.dev/integration-kit@3.0.0-beta.7
+
+## 3.0.0-beta.6
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.6
+- @trigger.dev/sdk@3.0.0-beta.6
+
 ## 3.0.0-beta.5
 
 ### Patch Changes

integrations/replicate/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/replicate",
-  "version": "3.0.0-beta.5",
+  "version": "3.0.0-beta.7",
   "description": "Trigger.dev integration for replicate",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -25,8 +25,8 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.5",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.5",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.7",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.7",
     "replicate": "^0.18.1",
     "zod": "3.22.3"
   },

integrations/resend/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @trigger.dev/resend
 
+## 3.0.0-beta.7
+
+### Patch Changes
+
+- Updated dependencies [f854cb90e]
+- Updated dependencies [f854cb90e]
+  - @trigger.dev/sdk@3.0.0-beta.7
+  - @trigger.dev/integration-kit@3.0.0-beta.7
+
+## 3.0.0-beta.6
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.6
+- @trigger.dev/sdk@3.0.0-beta.6
+
 ## 3.0.0-beta.5
 
 ### Patch Changes

integrations/resend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/resend",
-  "version": "3.0.0-beta.5",
+  "version": "3.0.0-beta.7",
   "description": "The official Resend.com integration for Trigger.dev",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -24,8 +24,8 @@
     "build:tsup": "tsup"
   },
   "dependencies": {
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.5",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.5",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.7",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.7",
     "resend": "^2.1.0"
   },
   "engines": {

integrations/sendgrid/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @trigger.dev/sendgrid
 
+## 3.0.0-beta.7
+
+### Patch Changes
+
+- Updated dependencies [f854cb90e]
+- Updated dependencies [f854cb90e]
+  - @trigger.dev/sdk@3.0.0-beta.7
+  - @trigger.dev/integration-kit@3.0.0-beta.7
+
+## 3.0.0-beta.6
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.6
+- @trigger.dev/sdk@3.0.0-beta.6
+
 ## 3.0.0-beta.5
 
 ### Patch Changes

integrations/sendgrid/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/sendgrid",
-  "version": "3.0.0-beta.5",
+  "version": "3.0.0-beta.7",
   "description": "Trigger.dev integration for @sendgrid/mail",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -26,8 +26,8 @@
   },
   "dependencies": {
     "@sendgrid/mail": "^7.7.0",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.5",
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.5"
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.7",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.7"
   },
   "engines": {
     "node": ">=16.8.0"

integrations/shopify/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @trigger.dev/shopify
 
+## 3.0.0-beta.7
+
+### Patch Changes
+
+- Updated dependencies [f854cb90e]
+- Updated dependencies [f854cb90e]
+  - @trigger.dev/sdk@3.0.0-beta.7
+  - @trigger.dev/integration-kit@3.0.0-beta.7
+
+## 3.0.0-beta.6
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.6
+- @trigger.dev/sdk@3.0.0-beta.6
+
 ## 3.0.0-beta.5
 
 ### Patch Changes

integrations/shopify/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/shopify",
-  "version": "3.0.0-beta.5",
+  "version": "3.0.0-beta.7",
   "description": "Trigger.dev integration for @shopify/shopify-api",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -26,8 +26,8 @@
   },
   "dependencies": {
     "@shopify/shopify-api": "^8.0.2",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.5",
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.5",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.7",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.7",
     "zod": "3.22.3"
   },
   "engines": {

integrations/slack/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @trigger.dev/slack
 
+## 3.0.0-beta.7
+
+### Patch Changes
+
+- Updated dependencies [f854cb90e]
+- Updated dependencies [f854cb90e]
+  - @trigger.dev/sdk@3.0.0-beta.7
+
+## 3.0.0-beta.6
+
+### Patch Changes
+
+- @trigger.dev/sdk@3.0.0-beta.6
+
 ## 3.0.0-beta.5
 
 ### Patch Changes

integrations/slack/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/slack",
-  "version": "3.0.0-beta.5",
+  "version": "3.0.0-beta.7",
   "description": "The official Slack integration for Trigger.dev",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -25,7 +25,7 @@
   },
   "dependencies": {
     "@slack/web-api": "^6.8.1",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.5",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.7",
     "zod": "3.22.3"
   },
   "engines": {

integrations/stripe/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @trigger.dev/stripe
 
+## 3.0.0-beta.7
+
+### Patch Changes
+
+- Updated dependencies [f854cb90e]
+- Updated dependencies [f854cb90e]
+  - @trigger.dev/sdk@3.0.0-beta.7
+  - @trigger.dev/integration-kit@3.0.0-beta.7
+
+## 3.0.0-beta.6
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.6
+- @trigger.dev/sdk@3.0.0-beta.6
+
 ## 3.0.0-beta.5
 
 ### Patch Changes