Fix pnpm lock file

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.changeset/config.json

@@ -4,7 +4,8 @@
   "commit": false,
   "fixed": [
     [
-      "@trigger.dev/*"
+      "@trigger.dev/*",
+      "trigger.dev"
     ]
   ],
   "linked": [],

.changeset/cuddly-feet-approve.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Don’t swallow some error messages when deploying

.changeset/few-students-share.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix permissions inside node_modules

.changeset/late-icons-lie.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Display errors for runs and deployments

.changeset/late-steaks-behave.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+v3: fix digest extraction

.changeset/light-dragons-complain.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Correctly handle self-hosted deploy command errors

.changeset/loud-actors-remember.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Default to retrying enabled in dev when running init

.changeset/new-rivers-tell.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Added replayRun function to the SDK

.changeset/polite-ducks-switch.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix post start hooks

.changeset/poor-flowers-cross.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Use the dashboard url instead of the API url for the View logs link

.changeset/pre.json

@@ -48,20 +48,34 @@
     "chilled-hornets-move",
     "clean-pianos-listen",
     "cool-glasses-bake",
+    "cuddly-feet-approve",
     "eight-pumas-float",
+    "few-students-share",
     "green-bags-wink",
     "khaki-apricots-design",
+    "late-icons-lie",
+    "late-steaks-behave",
     "lemon-jobs-repair",
     "light-bulldogs-press",
+    "light-dragons-complain",
+    "loud-actors-remember",
     "many-ligers-pump",
     "mighty-camels-joke",
+    "new-rivers-tell",
     "odd-poets-own",
+    "polite-ducks-switch",
+    "poor-flowers-cross",
+    "rare-roses-float",
     "real-planets-stare",
     "sharp-zebras-serve",
     "smart-olives-eat",
+    "spicy-lamps-smoke",
     "strange-ghosts-matter",
     "stupid-bulldogs-applaud",
     "sweet-lizards-press",
-    "tall-bees-wave"
+    "tall-bees-wave",
+    "tame-guests-know",
+    "tender-oranges-rhyme",
+    "tiny-doors-type"
   ]
 }

.changeset/rare-roses-float.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Add openssl to prod worker image and allow passing auth token via env var for deploy

.changeset/spicy-lamps-smoke.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fixing an issue with bundling @trigger.dev/core/v3 in dev when using pnpm

.changeset/tame-guests-know.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Changed the binary name from trigger.dev to triggerdev to fix a Windows issue

.changeset/tender-oranges-rhyme.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Stop swallowing deployment errors and display them better

.changeset/tiny-doors-type.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Added cancelRun to the SDK

.vscode/launch.json

@@ -23,7 +23,7 @@
     {
       "type": "node-terminal",
       "request": "launch",
-      "name": "Debug BYO Auth",
+      "name": "Debug v2 job catalog",
       "command": "pnpm run byo-auth",
       "envFile": "${workspaceFolder}/references/job-catalog/.env",
       "cwd": "${workspaceFolder}/references/job-catalog",
@@ -33,7 +33,7 @@
       "type": "node-terminal",
       "request": "launch",
       "name": "Debug V3 Dev CLI",
-      "command": "pnpm exec trigger.dev dev --log-level debug",
+      "command": "pnpm exec triggerdev dev --log-level debug",
       "cwd": "${workspaceFolder}/references/v3-catalog",
       "sourceMaps": true
     },
@@ -41,7 +41,7 @@
       "type": "node-terminal",
       "request": "launch",
       "name": "Debug V3 Deploy CLI",
-      "command": "pnpm exec trigger.dev deploy",
+      "command": "pnpm exec triggerdev deploy",
       "cwd": "${workspaceFolder}/references/v3-catalog",
       "sourceMaps": true
     },

README.md

@@ -7,7 +7,7 @@
   
 ### The open source background jobs framework
 
-[Discord](https://discord.gg/JtBAxBr2m3) | [Website](https://trigger.dev) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Docs](https://trigger.dev/docs)
+[Discord](https://trigger.dev/discord) | [Website](https://trigger.dev) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Docs](https://trigger.dev/docs)
 
 [![Twitter](https://img.shields.io/twitter/url/https/twitter.com/triggerdotdev.svg?style=social&label=Follow%20%40trigger.dev)](https://twitter.com/triggerdotdev)
 [![GitHub Repo stars](https://img.shields.io/github/stars/triggerdotdev/trigger.dev?style=social)](https://github.com/triggerdotdev/trigger.dev)

apps/kubernetes-provider/package.json

@@ -19,6 +19,7 @@
     "@kubernetes/client-node": "^0.20.0",
     "@trigger.dev/core": "workspace:*",
     "@trigger.dev/core-apps": "workspace:*",
+    "p-queue": "^8.0.1",
     "socket.io-client": "^4.7.4"
   },
   "devDependencies": {

apps/kubernetes-provider/src/index.ts

@@ -9,6 +9,7 @@ import {
 } from "@trigger.dev/core-apps";
 import { Machine, PostStartCauses, PreStopCauses, EnvironmentType } from "@trigger.dev/core/v3";
 import { randomUUID } from "crypto";
+import { TaskMonitor } from "./taskMonitor";
 
 const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
 const NODE_NAME = process.env.NODE_NAME || "local";
@@ -24,6 +25,10 @@ type Namespace = {
   };
 };
 
+type ComputeResources = {
+  [K in "cpu" | "memory" | "ephemeral-storage"]?: string;
+};
+
 class KubernetesTaskOperations implements TaskOperations {
   #namespace: Namespace;
   #k8sApi: {
@@ -55,25 +60,15 @@ class KubernetesTaskOperations implements TaskOperations {
           template: {
             metadata: {
               labels: {
+                ...this.#getSharedLabels(opts),
                 app: "task-index",
                 "app.kubernetes.io/part-of": "trigger-worker",
                 "app.kubernetes.io/component": "index",
-                env: opts.envId,
-                envtype: this.#envTypeToLabelValue(opts.envType),
-                org: opts.orgId,
-                project: opts.projectId,
+                deployment: opts.deploymentId,
               },
             },
             spec: {
-              restartPolicy: "Never",
-              imagePullSecrets: [
-                {
-                  name: "registry-trigger",
-                },
-              ],
-              nodeSelector: {
-                nodetype: "worker",
-              },
+              ...this.#defaultPodSpec,
               containers: [
                 {
                   name: this.#getIndexContainerName(opts.shortCode),
@@ -83,12 +78,13 @@ class KubernetesTaskOperations implements TaskOperations {
                       containerPort: 8000,
                     },
                   ],
-                  // resources: {
-                  //   limits: {
-                  //     cpu: "100m",
-                  //     memory: "50Mi",
-                  //   },
-                  // },
+                  resources: {
+                    limits: {
+                      cpu: "250m",
+                      memory: "0.5G",
+                      "ephemeral-storage": "2Gi",
+                    },
+                  },
                   lifecycle: {
                     preStop: {
                       exec: {
@@ -97,10 +93,7 @@ class KubernetesTaskOperations implements TaskOperations {
                     },
                   },
                   env: [
-                    {
-                      name: "DEBUG",
-                      value: "true",
-                    },
+                    ...this.#getSharedEnv(opts.envId),
                     {
                       name: "INDEX_TASKS",
                       value: "true",
@@ -113,42 +106,6 @@ class KubernetesTaskOperations implements TaskOperations {
                       name: "TRIGGER_API_URL",
                       value: opts.apiUrl,
                     },
-                    {
-                      name: "TRIGGER_ENV_ID",
-                      value: opts.envId,
-                    },
-                    {
-                      name: "OTEL_EXPORTER_OTLP_ENDPOINT",
-                      value: OTEL_EXPORTER_OTLP_ENDPOINT,
-                    },
-                    {
-                      name: "HTTP_SERVER_PORT",
-                      value: "8000",
-                    },
-                    {
-                      name: "POD_NAME",
-                      valueFrom: {
-                        fieldRef: {
-                          fieldPath: "metadata.name",
-                        },
-                      },
-                    },
-                    {
-                      name: "COORDINATOR_HOST",
-                      valueFrom: {
-                        fieldRef: {
-                          fieldPath: "status.hostIP",
-                        },
-                      },
-                    },
-                    {
-                      name: "MACHINE_NAME",
-                      valueFrom: {
-                        fieldRef: {
-                          fieldPath: "spec.nodeName",
-                        },
-                      },
-                    },
                   ],
                 },
               ],
@@ -167,26 +124,15 @@ class KubernetesTaskOperations implements TaskOperations {
           name: this.#getRunContainerName(opts.runId),
           namespace: this.#namespace.metadata.name,
           labels: {
+            ...this.#getSharedLabels(opts),
             app: "task-run",
             "app.kubernetes.io/part-of": "trigger-worker",
             "app.kubernetes.io/component": "create",
-            env: opts.envId,
-            envtype: this.#envTypeToLabelValue(opts.envType),
-            org: opts.orgId,
-            project: opts.projectId,
             run: opts.runId,
           },
         },
         spec: {
-          restartPolicy: "Never",
-          imagePullSecrets: [
-            {
-              name: "registry-trigger",
-            },
-          ],
-          nodeSelector: {
-            nodetype: "worker",
-          },
+          ...this.#defaultPodSpec,
           containers: [
             {
               name: this.#getRunContainerName(opts.runId),
@@ -197,14 +143,15 @@ class KubernetesTaskOperations implements TaskOperations {
                 },
               ],
               resources: {
-                limits: this.#getResourcesFromMachineConfig(opts.machine),
+                requests: {
+                  ...this.#defaultResourceRequests,
+                },
+                limits: {
+                  ...this.#defaultResourceLimits,
+                  ...this.#getResourcesFromMachineConfig(opts.machine),
+                },
               },
               lifecycle: {
-                postStart: {
-                  exec: {
-                    command: this.#getLifecycleCommand("postStart", "create"),
-                  },
-                },
                 preStop: {
                   exec: {
                     command: this.#getLifecycleCommand("preStop", "terminate"),
@@ -212,54 +159,11 @@ class KubernetesTaskOperations implements TaskOperations {
                 },
               },
               env: [
-                {
-                  name: "DEBUG",
-                  value: "true",
-                },
-                {
-                  name: "HTTP_SERVER_PORT",
-                  value: "8000",
-                },
-                {
-                  name: "TRIGGER_ENV_ID",
-                  value: opts.envId,
-                },
+                ...this.#getSharedEnv(opts.envId),
                 {
                   name: "TRIGGER_RUN_ID",
                   value: opts.runId,
                 },
-                {
-                  name: "TRIGGER_WORKER_VERSION",
-                  value: opts.version,
-                },
-                {
-                  name: "OTEL_EXPORTER_OTLP_ENDPOINT",
-                  value: OTEL_EXPORTER_OTLP_ENDPOINT,
-                },
-                {
-                  name: "POD_NAME",
-                  valueFrom: {
-                    fieldRef: {
-                      fieldPath: "metadata.name",
-                    },
-                  },
-                },
-                {
-                  name: "COORDINATOR_HOST",
-                  valueFrom: {
-                    fieldRef: {
-                      fieldPath: "status.hostIP",
-                    },
-                  },
-                },
-                {
-                  name: "NODE_NAME",
-                  valueFrom: {
-                    fieldRef: {
-                      fieldPath: "spec.nodeName",
-                    },
-                  },
-                },
               ],
               volumeMounts: [
                 {
@@ -288,27 +192,16 @@ class KubernetesTaskOperations implements TaskOperations {
           name: `${this.#getRunContainerName(opts.runId)}-${randomUUID().slice(0, 8)}`,
           namespace: this.#namespace.metadata.name,
           labels: {
+            ...this.#getSharedLabels(opts),
             app: "task-run",
             "app.kubernetes.io/part-of": "trigger-worker",
             "app.kubernetes.io/component": "restore",
-            env: opts.envId,
-            envtype: this.#envTypeToLabelValue(opts.envType),
-            org: opts.orgId,
-            project: opts.projectId,
             run: opts.runId,
             checkpoint: opts.checkpointId,
           },
         },
         spec: {
-          restartPolicy: "Never",
-          imagePullSecrets: [
-            {
-              name: "registry-trigger",
-            },
-          ],
-          nodeSelector: {
-            nodetype: "worker",
-          },
+          ...this.#defaultPodSpec,
           initContainers: [
             {
               name: "pull-base-image",
@@ -348,7 +241,13 @@ class KubernetesTaskOperations implements TaskOperations {
                 },
               ],
               resources: {
-                limits: this.#getResourcesFromMachineConfig(opts.machine),
+                requests: {
+                  ...this.#defaultResourceRequests,
+                },
+                limits: {
+                  ...this.#defaultResourceLimits,
+                  ...this.#getResourcesFromMachineConfig(opts.machine),
+                },
               },
               lifecycle: {
                 postStart: {
@@ -406,7 +305,90 @@ class KubernetesTaskOperations implements TaskOperations {
     }
   }
 
-  #getResourcesFromMachineConfig(config: Machine) {
+  get #defaultPodSpec(): Omit<k8s.V1PodSpec, "containers"> {
+    return {
+      restartPolicy: "Never",
+      automountServiceAccountToken: false,
+      imagePullSecrets: [
+        {
+          name: "registry-trigger",
+        },
+      ],
+      nodeSelector: {
+        nodetype: "worker",
+      },
+    };
+  }
+
+  get #defaultResourceRequests(): ComputeResources {
+    return {
+      "ephemeral-storage": "2Gi",
+    };
+  }
+
+  get #defaultResourceLimits(): ComputeResources {
+    return {
+      "ephemeral-storage": "10Gi",
+    };
+  }
+
+  #getSharedEnv(envId: string): k8s.V1EnvVar[] {
+    return [
+      {
+        name: "TRIGGER_ENV_ID",
+        value: envId,
+      },
+      {
+        name: "DEBUG",
+        value: process.env.DEBUG ? "1" : "0",
+      },
+      {
+        name: "HTTP_SERVER_PORT",
+        value: "8000",
+      },
+      {
+        name: "OTEL_EXPORTER_OTLP_ENDPOINT",
+        value: OTEL_EXPORTER_OTLP_ENDPOINT,
+      },
+      {
+        name: "POD_NAME",
+        valueFrom: {
+          fieldRef: {
+            fieldPath: "metadata.name",
+          },
+        },
+      },
+      {
+        name: "COORDINATOR_HOST",
+        valueFrom: {
+          fieldRef: {
+            fieldPath: "status.hostIP",
+          },
+        },
+      },
+      {
+        name: "MACHINE_NAME",
+        valueFrom: {
+          fieldRef: {
+            fieldPath: "spec.nodeName",
+          },
+        },
+      },
+    ];
+  }
+
+  #getSharedLabels(
+    opts: TaskOperationsIndexOptions | TaskOperationsCreateOptions | TaskOperationsRestoreOptions
+  ): Record<string, string> {
+    return {
+      env: opts.envId,
+      envtype: this.#envTypeToLabelValue(opts.envType),
+      org: opts.orgId,
+      project: opts.projectId,
+    };
+  }
+
+  #getResourcesFromMachineConfig(config: Machine): ComputeResources {
     return {
       cpu: `${config.cpu}`,
       memory: `${config.memory}G`,
@@ -417,7 +399,19 @@ class KubernetesTaskOperations implements TaskOperations {
     type: THookType,
     cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses
   ) {
-    return ["/bin/sh", "-c", `sleep 1; wget -q -O- 127.0.0.1:8000/${type}?cause=${cause}`];
+    const retries = 5;
+
+    // This will retry sending the lifecycle hook up to `retries` times
+    // The sleep is required as this may start running before the HTTP server is up
+    const exec = [
+      "/bin/sh",
+      "-c",
+      `for i in $(seq ${retries}); do sleep 1; busybox wget -q -O- 127.0.0.1:8000/${type}?cause=${cause} && break; done`,
+    ];
+
+    logger.log("getLifecycleCommand()", { exec });
+
+    return exec;
   }
 
   #getIndexContainerName(suffix: string) {
@@ -516,3 +510,34 @@ const provider = new ProviderShell({
 });
 
 provider.listen();
+
+const taskMonitor = new TaskMonitor({
+  runtimeEnv: RUNTIME_ENV,
+  onIndexFailure: async (deploymentId, failureInfo) => {
+    logger.log("Indexing failed", { deploymentId, failureInfo });
+
+    try {
+      provider.platformSocket.send("INDEXING_FAILED", {
+        deploymentId,
+        error: {
+          name: `Crashed with exit code ${failureInfo.exitCode}`,
+          message: failureInfo.reason,
+          stack: failureInfo.logs,
+        },
+      });
+    } catch (error) {
+      logger.error(error);
+    }
+  },
+  onRunFailure: async (runId, failureInfo) => {
+    logger.log("Run failed:", { runId, failureInfo });
+
+    try {
+      provider.platformSocket.send("WORKER_CRASHED", { runId, ...failureInfo });
+    } catch (error) {
+      logger.error(error);
+    }
+  },
+});
+
+taskMonitor.start();

apps/kubernetes-provider/src/taskMonitor.ts

@@ -0,0 +1,442 @@
+import * as k8s from "@kubernetes/client-node";
+import { SimpleLogger } from "@trigger.dev/core-apps";
+import { setTimeout } from "timers/promises";
+import PQueue from "p-queue";
+
+type IndexFailureHandler = (
+  deploymentId: string,
+  failureInfo: {
+    exitCode: number;
+    reason: string;
+    logs: string;
+  }
+) => Promise<any>;
+
+type RunFailureHandler = (
+  runId: string,
+  failureInfo: {
+    exitCode: number;
+    reason: string;
+    logs: string;
+  }
+) => Promise<any>;
+
+type TaskMonitorOptions = {
+  runtimeEnv: "local" | "kubernetes";
+  onIndexFailure?: IndexFailureHandler;
+  onRunFailure?: RunFailureHandler;
+  namespace?: string;
+};
+
+export class TaskMonitor {
+  #enabled = false;
+  #logger = new SimpleLogger("[TaskMonitor]");
+  #taskInformer: ReturnType<typeof k8s.makeInformer<k8s.V1Pod>>;
+  #processedPods = new Map<string, number>();
+  #queue = new PQueue({ concurrency: 10 });
+  #k8sClient: {
+    core: k8s.CoreV1Api;
+    kubeConfig: k8s.KubeConfig;
+  };
+
+  private namespace = "default";
+  private fieldSelector = "status.phase=Failed";
+  private labelSelector = "app in (task-index, task-run)";
+
+  constructor(private opts: TaskMonitorOptions) {
+    this.#k8sClient = this.#createK8sClient();
+
+    this.#taskInformer = this.#createTaskInformer();
+    this.#taskInformer.on("connect", this.#onInformerConnected.bind(this));
+    this.#taskInformer.on("error", this.#onInformerError.bind(this));
+    this.#taskInformer.on("update", this.#enqueueOnPodUpdated.bind(this));
+  }
+
+  #createTaskInformer() {
+    const listTasks = () =>
+      this.#k8sClient.core.listNamespacedPod(
+        this.namespace,
+        undefined,
+        undefined,
+        undefined,
+        this.fieldSelector,
+        this.labelSelector
+      );
+
+    // Uses watch with local caching
+    // https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes
+    const informer = k8s.makeInformer(
+      this.#k8sClient.kubeConfig,
+      `/api/v1/namespaces/${this.namespace}/pods`,
+      listTasks,
+      this.labelSelector,
+      this.fieldSelector
+    );
+
+    return informer;
+  }
+
+  async #onInformerConnected() {
+    this.#logger.log("Connected");
+  }
+
+  async #onInformerError(error: any) {
+    this.#logger.error("Error:", error);
+
+    // Automatic reconnect
+    await setTimeout(2_000);
+    this.#taskInformer.start();
+  }
+
+  #enqueueOnPodUpdated(pod: k8s.V1Pod) {
+    this.#queue.add(async () => {
+      try {
+        // It would be better to only pass the cache key, but the pod may already be removed from the cache by the time we process it
+        await this.#onPodUpdated(pod);
+      } catch (error) {
+        this.#logger.error("Caught onPodUpdated() error:", error);
+      }
+    });
+  }
+
+  async #onPodUpdated(pod: k8s.V1Pod) {
+    this.#logger.debug(`Updated: ${pod.metadata?.name}`);
+    this.#logger.debug("Updated", JSON.stringify(pod, null, 2));
+
+    // We only care about failures
+    if (pod.status?.phase !== "Failed") {
+      return;
+    }
+
+    const podName = pod.metadata?.name;
+
+    if (!podName) {
+      this.#logger.error("Pod is nameless", { pod });
+      return;
+    }
+
+    const containerStatus = pod.status.containerStatuses?.[0];
+
+    if (!containerStatus?.state) {
+      this.#logger.error("Pod failed, but container status doesn't have state", {
+        status: pod.status,
+      });
+      return;
+    }
+
+    if (this.#processedPods.has(podName)) {
+      this.#logger.debug("Pod update already processed", {
+        podName,
+        timestamp: this.#processedPods.get(podName),
+      });
+      return;
+    }
+
+    this.#processedPods.set(podName, Date.now());
+
+    const podStatus = this.#getPodStatusSummary(pod.status);
+    const containerState = this.#getContainerStateSummary(containerStatus.state);
+    const exitCode = containerState.exitCode ?? -1;
+
+    // We use this special exit code to signal any errors were already handled elsewhere
+    if (exitCode === 111) {
+      return;
+    }
+
+    const rawLogs = await this.#getLogTail(podName);
+
+    this.#logger.log(`${podName} failed with:`, {
+      podStatus,
+      containerState,
+      rawLogs,
+    });
+
+    const rawReason = podStatus.reason ?? containerState.reason ?? "";
+    const message = podStatus.message ?? containerState.message ?? "";
+
+    let reason = rawReason || "Unknown error";
+    let logs = rawLogs || "";
+
+    switch (rawReason) {
+      case "Error":
+        reason = "Unknown error.";
+        break;
+      case "Evicted":
+        if (message.startsWith("Pod ephemeral local storage usage")) {
+          reason = "Storage limit exceeded.";
+        } else if (message) {
+          reason = `Evicted: ${message}`;
+        } else {
+          reason = "Evicted for unknown reason.";
+        }
+
+        if (logs.startsWith("failed to try resolving symlinks")) {
+          logs = "";
+        }
+        break;
+      case "OOMKilled":
+        reason = "Out of memory! Try increasing the memory on this task.";
+        break;
+      default:
+        break;
+    }
+
+    const failureInfo = {
+      exitCode,
+      reason,
+      logs,
+    };
+
+    const app = pod.metadata?.labels?.app;
+
+    switch (app) {
+      case "task-index":
+        const deploymentId = pod.metadata?.labels?.deployment;
+
+        if (!deploymentId) {
+          this.#logger.error("Index is missing ID", { pod });
+          return;
+        }
+
+        if (this.opts.onIndexFailure) {
+          await this.opts.onIndexFailure(deploymentId, failureInfo);
+        }
+        break;
+      case "task-run":
+        const runId = pod.metadata?.labels?.run;
+
+        if (!runId) {
+          this.#logger.error("Run is missing ID", { pod });
+          return;
+        }
+
+        if (this.opts.onRunFailure) {
+          await this.opts.onRunFailure(runId, failureInfo);
+        }
+        break;
+      default:
+        this.#logger.error("Pod has invalid app label", { pod });
+        return;
+    }
+
+    await this.#deletePod(podName);
+  }
+
+  async #getLogTail(podName: string) {
+    try {
+      const logs = await this.#k8sClient.core.readNamespacedPodLog(
+        podName,
+        this.namespace,
+        undefined,
+        undefined,
+        undefined,
+        1024, // limitBytes
+        undefined,
+        undefined,
+        undefined,
+        20 // tailLines
+      );
+
+      const responseBody = logs.body ?? "";
+
+      if (responseBody.startsWith("unable to retrieve container logs")) {
+        return "";
+      }
+
+      // Type is wrong, body may be undefined
+      return responseBody;
+    } catch (error) {
+      this.#logger.error("Log tail error:", error instanceof Error ? error.message : "unknown");
+      return "";
+    }
+  }
+
+  #getPodStatusSummary(status: k8s.V1PodStatus) {
+    return {
+      reason: status.reason,
+      message: status.message,
+    };
+  }
+
+  #getContainerStateSummary(state: k8s.V1ContainerState) {
+    return {
+      reason: state.terminated?.reason,
+      exitCode: state.terminated?.exitCode,
+      message: state.terminated?.message,
+    };
+  }
+
+  #createK8sClient() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (this.opts.runtimeEnv === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (this.opts.runtimeEnv === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      kubeConfig: kubeConfig,
+    };
+  }
+
+  #isRecord(candidate: unknown): candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  #logK8sError(err: unknown, debugOnly = false) {
+    if (debugOnly) {
+      this.#logger.debug("K8s API Error", err);
+    } else {
+      this.#logger.error("K8s API Error", err);
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
+      this.#logK8sError(err);
+      return;
+    }
+
+    this.#logK8sError(err, true);
+
+    if (typeof err.body.message === "string") {
+      this.#logK8sError({ message: err.body.message });
+      return;
+    }
+
+    this.#logK8sError({ body: err.body });
+  }
+
+  #printStats(includeMoreDetails = false) {
+    this.#logger.log("Stats:", {
+      cacheSize: this.#taskInformer.list().length,
+      totalProcessed: this.#processedPods.size,
+      ...(includeMoreDetails && {
+        processedPods: this.#processedPods,
+      }),
+    });
+  }
+
+  async #deletePod(name: string) {
+    this.#logger.debug("Deleting pod:", name);
+
+    await this.#k8sClient.core
+      .deleteNamespacedPod(name, this.namespace)
+      .catch(this.#handleK8sError.bind(this));
+  }
+
+  async start() {
+    this.#enabled = true;
+
+    const interval = setInterval(() => {
+      if (!this.#enabled) {
+        clearInterval(interval);
+        return;
+      }
+
+      this.#printStats();
+    }, 300_000);
+
+    await this.#taskInformer.start();
+
+    // this.#launchTests();
+  }
+
+  async stop() {
+    if (!this.#enabled) {
+      return;
+    }
+
+    this.#enabled = false;
+    this.#logger.log("Shutting down..");
+
+    await this.#taskInformer.stop();
+
+    this.#printStats(true);
+  }
+
+  async #launchTests() {
+    const createPod = async (
+      container: k8s.V1Container,
+      name: string,
+      labels?: Record<string, string>
+    ) => {
+      this.#logger.log("Creating pod:", name);
+
+      const pod = {
+        metadata: {
+          name,
+          labels,
+        },
+        spec: {
+          restartPolicy: "Never",
+          automountServiceAccountToken: false,
+          terminationGracePeriodSeconds: 1,
+          containers: [container],
+        },
+      } satisfies k8s.V1Pod;
+
+      await this.#k8sClient.core
+        .createNamespacedPod(this.namespace, pod)
+        .catch(this.#handleK8sError.bind(this));
+    };
+
+    const createOomPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "polinux/stress",
+        resources: {
+          limits: {
+            memory: "100Mi",
+          },
+        },
+        command: ["stress"],
+        args: ["--vm", "1", "--vm-bytes", "150M", "--vm-hang", "1"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createNonZeroExitPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "busybox",
+        command: ["sh"],
+        args: ["-c", "exit 1"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createOoDiskPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "busybox",
+        command: ["sh"],
+        args: [
+          "-c",
+          "echo creating huge-file..; head -c 1000m /dev/zero > huge-file; ls -lh huge-file; sleep infinity",
+        ],
+        resources: {
+          limits: {
+            "ephemeral-storage": "500Mi",
+          },
+        },
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    await createNonZeroExitPod("non-zero-exit-task", { app: "task-run", run: "123" });
+    await createOomPod("oom-task", { app: "task-index", deployment: "456" });
+    await createOoDiskPod("ood-task", { app: "task-run", run: "abc" });
+  }
+}

apps/webapp/app/components/navigation/SideMenu.tsx

@@ -57,6 +57,7 @@ import { UserProfilePhoto } from "../UserProfilePhoto";
 import { FreePlanUsage } from "../billing/FreePlanUsage";
 import { Badge } from "../primitives/Badge";
 import { Button } from "../primitives/Buttons";
+import { Callout } from "../primitives/Callout";
 import { ClipboardField } from "../primitives/ClipboardField";
 import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "../primitives/Dialog";
 import { Icon } from "../primitives/Icon";
@@ -187,7 +188,22 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
             />
           </div>
         </div>
+        <div className="m-2">
+          {project.version === "V2" ? (
+            <Callout variant={"info"}>This is a v2 project</Callout>
+          ) : (
+            <Callout variant={"idea"}>This is a v3 project in Developer Preview</Callout>
+          )}
+        </div>
         <div className="flex flex-col gap-1 border-t border-grid-bright p-1">
+          {project.version === "V2" && (
+            <SideMenuItem
+              to="https://trigger.dev/v3-early-access"
+              target="_blank"
+              name="Request access to v3"
+              icon={V3Icon}
+            />
+          )}
           {currentPlan?.subscription?.isPaying === true && (
             <Dialog>
               <DialogTrigger asChild>
@@ -586,3 +602,15 @@ function V3ProjectSideMenu({
     </>
   );
 }
+
+function V3Icon() {
+  return (
+    <svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <rect width="16" height="16" rx="8" fill="#A8FF53" />
+      <path
+        d="M7.7488 6.17L5.5818 12H3.6678L1.5008 6.17H3.2058L4.6248 10.339L6.0328 6.17H7.7488ZM11.0527 7.292C11.6357 7.303 12.2297 7.006 12.2297 6.28C12.2297 5.774 11.7787 5.433 11.0527 5.433C10.4147 5.433 9.98567 5.741 9.91967 6.214L8.22567 6.126C8.36867 4.861 9.51267 4.014 11.0857 4.014C12.8457 4.014 13.9567 4.806 13.9567 6.049C13.9567 6.951 13.3847 7.534 12.3067 7.776C13.5387 8.04 14.2207 8.777 14.2207 9.855C14.2207 11.274 13.0107 12.176 11.0857 12.176C9.32567 12.176 8.12667 11.197 8.04967 9.712L9.75467 9.646C9.83167 10.405 10.4917 10.757 11.0967 10.757C11.8007 10.757 12.4937 10.394 12.4937 9.591C12.4937 8.81 11.7897 8.425 11.0527 8.447L10.3817 8.458V7.281L11.0527 7.292Z"
+        fill="#15171A"
+      />
+    </svg>
+  );
+}

apps/webapp/app/components/primitives/Dialog.tsx

@@ -43,7 +43,7 @@ const DialogContent = React.forwardRef<
     <DialogPrimitive.Content
       ref={ref}
       className={cn(
-        "fixed z-50 grid w-full gap-4 rounded-b-lg border bg-background-dimmed px-4 pb-6 pt-3.5 shadow-lg animate-in data-[state=open]:fade-in-90 data-[state=open]:slide-in-from-bottom-10 sm:max-w-lg sm:rounded-lg sm:zoom-in-90 data-[state=open]:sm:slide-in-from-bottom-0",
+        "fixed z-50 grid w-full gap-4 rounded-b-lg border bg-background-dimmed px-4 pb-4 pt-3.5 shadow-lg animate-in data-[state=open]:fade-in-90 data-[state=open]:slide-in-from-bottom-10 sm:max-w-lg sm:rounded-lg sm:zoom-in-90 data-[state=open]:sm:slide-in-from-bottom-0",
         className
       )}
       {...props}

apps/webapp/app/components/primitives/PageHeader.tsx

@@ -15,6 +15,7 @@ import { Tabs, TabsProps } from "./Tabs";
 
 type WithChildren = {
   children: React.ReactNode;
+  className?: string;
 };
 
 export function NavBar({ children }: WithChildren) {
@@ -72,8 +73,8 @@ export function PageAccessories({ children }: WithChildren) {
   return <div className="flex items-center gap-3">{children}</div>;
 }
 
-export function PageInfoRow({ children }: WithChildren) {
-  return <div className="flex w-full items-center gap-2">{children}</div>;
+export function PageInfoRow({ children, className }: WithChildren) {
+  return <div className={cn("flex w-full items-center gap-2", className)}>{children}</div>;
 }
 
 export function PageInfoGroup({

apps/webapp/app/components/run/RunOverview.tsx

@@ -124,9 +124,9 @@ export function RunOverview({ run, trigger, showRerun, paths, currentUser }: Run
           {!run.isFinished && <CancelRun runId={run.id} />}
         </PageAccessories>
       </NavBar>
-      <PageBody scrollable={false}>
-        <div className="px-4 pt-4">
-          <PageInfoRow>
+      <PageBody scrollable={false} className="grid grid-rows-[auto_1fr] overflow-hidden">
+        <div className="border-b border-grid-dimmed px-4 py-4">
+          <PageInfoRow className="overflow-hidden">
             <PageInfoGroup>
               <PageInfoProperty
                 icon={<RunStatusIcon status={run.status} className="h-4 w-4" />}
@@ -166,7 +166,7 @@ export function RunOverview({ run, trigger, showRerun, paths, currentUser }: Run
             </PageInfoGroup>
           </PageInfoRow>
         </div>
-        <div className="grid h-full grid-cols-2 gap-2">
+        <div className="grid h-full grid-cols-2 gap-2 overflow-hidden">
           <div className="flex flex-col gap-6 overflow-y-auto py-4 pl-4 pr-2 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600">
             <div>
               {run.status === "SUCCESS" &&

apps/webapp/app/components/runs/v3/CancelRunDialog.tsx

@@ -1,5 +1,5 @@
 import { StopCircleIcon } from "@heroicons/react/20/solid";
-import { useFetcher } from "@remix-run/react";
+import { Form, useFetcher, useNavigation } from "@remix-run/react";
 import { Button } from "~/components/primitives/Buttons";
 import {
   DialogContent,
@@ -14,29 +14,32 @@ type CancelRunDialogProps = {
 };
 
 export function CancelRunDialog({ runFriendlyId, redirectPath }: CancelRunDialogProps) {
-  const cancelFetcher = useFetcher();
+  const navigation = useNavigation();
+
+  const formAction = `/resources/taskruns/${runFriendlyId}/cancel`;
+  const isLoading = navigation.formAction === formAction;
 
   return (
-    <DialogContent>
+    <DialogContent key="cancel">
       <DialogHeader>Cancel this run?</DialogHeader>
       <DialogDescription>
         Canceling a run will stop execution. If you want to run this later you will have to replay
         the entire run with the original payload.
       </DialogDescription>
       <DialogFooter>
-        <cancelFetcher.Form action={`/resources/taskruns/${runFriendlyId}/cancel`} method="post">
+        <Form action={`/resources/taskruns/${runFriendlyId}/cancel`} method="post">
           <Button
             type="submit"
             name="redirectUrl"
             value={redirectPath}
             variant="danger/small"
-            LeadingIcon={cancelFetcher.state === "idle" ? StopCircleIcon : "spinner-white"}
-            disabled={cancelFetcher.state !== "idle"}
+            LeadingIcon={isLoading ? "spinner-white" : StopCircleIcon}
+            disabled={isLoading}
             shortcut={{ modifiers: ["meta"], key: "enter" }}
           >
-            {cancelFetcher.state === "idle" ? "Cancel run" : "Canceling..."}
+            {isLoading ? "Canceling..." : "Cancel run"}
           </Button>
-        </cancelFetcher.Form>
+        </Form>
       </DialogFooter>
     </DialogContent>
   );

apps/webapp/app/components/runs/v3/DeploymentError.tsx

@@ -0,0 +1,39 @@
+import { CodeBlock } from "~/components/code/CodeBlock";
+import { Callout } from "~/components/primitives/Callout";
+import { Header2 } from "~/components/primitives/Headers";
+import type { ErrorData } from "~/presenters/v3/DeploymentPresenter.server";
+
+type DeploymentErrorProps = {
+  errorData: ErrorData;
+};
+
+export function DeploymentError({ errorData }: DeploymentErrorProps) {
+  return (
+    <div className="flex flex-col gap-2 rounded-sm border border-rose-500/50 p-3">
+      <DeploymentErrorHeader title={errorData.name ?? "Error"} titleClassName="text-rose-500" />
+      {errorData.message && <Callout variant="error">{errorData.message}</Callout>}
+      {errorData.stack && (
+        <CodeBlock
+          showCopyButton={false}
+          showLineNumbers={false}
+          code={errorData.stack}
+          maxLines={20}
+        />
+      )}
+    </div>
+  );
+}
+
+function DeploymentErrorHeader({
+  title,
+  titleClassName,
+}: {
+  title: string;
+  titleClassName?: string;
+}) {
+  return (
+    <div className="flex items-center justify-between">
+      <Header2 className={titleClassName}>{title}</Header2>
+    </div>
+  );
+}

apps/webapp/app/components/runs/v3/ReplayRunDialog.tsx

@@ -0,0 +1,44 @@
+import { ArrowPathIcon } from "@heroicons/react/20/solid";
+import { Form, useFetcher, useNavigation } from "@remix-run/react";
+import { Button } from "~/components/primitives/Buttons";
+import {
+  DialogContent,
+  DialogDescription,
+  DialogFooter,
+  DialogHeader,
+} from "~/components/primitives/Dialog";
+
+type ReplayRunDialogProps = {
+  runFriendlyId: string;
+  failedRedirect: string;
+};
+
+export function ReplayRunDialog({ runFriendlyId, failedRedirect }: ReplayRunDialogProps) {
+  const navigation = useNavigation();
+
+  const formAction = `/resources/taskruns/${runFriendlyId}/replay`;
+  const isLoading = navigation.formAction === formAction;
+
+  return (
+    <DialogContent key="replay">
+      <DialogHeader>Replay this run?</DialogHeader>
+      <DialogDescription>
+        Replaying a run will create a new run with the same payload and environment as the original.
+      </DialogDescription>
+      <DialogFooter>
+        <Form action={formAction} method="post">
+          <input type="hidden" name="failedRedirect" value={failedRedirect} />
+          <Button
+            type="submit"
+            variant="primary/small"
+            LeadingIcon={isLoading ? "spinner-white" : ArrowPathIcon}
+            disabled={isLoading}
+            shortcut={{ modifiers: ["meta"], key: "enter" }}
+          >
+            {isLoading ? "Replaying..." : "Replay run"}
+          </Button>
+        </Form>
+      </DialogFooter>
+    </DialogContent>
+  );
+}

apps/webapp/app/components/runs/v3/RunFilters.tsx

@@ -27,6 +27,7 @@ import { TaskRunStatusCombo, descriptionForTaskRunStatus } from "./TaskRunStatus
 
 export const allTaskRunStatuses = [
   "PENDING",
+  "WAITING_FOR_DEPLOY",
   "EXECUTING",
   "RETRYING_AFTER_FAILURE",
   "WAITING_TO_RESUME",
@@ -35,6 +36,7 @@ export const allTaskRunStatuses = [
   "COMPLETED_WITH_ERRORS",
   "INTERRUPTED",
   "SYSTEM_FAILURE",
+  "CRASHED",
 ] as TaskRunStatusType[];
 
 export const TaskAttemptStatus = z.nativeEnum(TaskRunStatus);
@@ -227,16 +229,18 @@ export function RunsFilters({ possibleEnvironments, possibleTasks }: RunFiltersP
                 All tasks
               </Paragraph>
             </SelectItem>
-            {possibleTasks.map((task) => (
-              <SelectItem key={task} value={task}>
-                <Paragraph
-                  variant="extra-small"
-                  className="pl-0.5 transition group-hover:text-text-bright"
-                >
-                  {task}
-                </Paragraph>
-              </SelectItem>
-            ))}
+            {possibleTasks
+              .sort((a, b) => a.localeCompare(b)) // 🔤
+              .map((task) => (
+                <SelectItem key={task} value={task}>
+                  <Paragraph
+                    variant="extra-small"
+                    className="pl-0.5 transition group-hover:text-text-bright"
+                  >
+                    {task}
+                  </Paragraph>
+                </SelectItem>
+              ))}
           </SelectContent>
         </Select>
       </SelectGroup>

apps/webapp/app/components/runs/v3/SpanEvents.tsx

@@ -66,7 +66,11 @@ function SpanEventError({
 }) {
   return (
     <div className="flex flex-col gap-2 rounded-sm border border-rose-500/50 p-3">
-      <SpanEventHeader title={"Error"} time={spanEvent.time} titleClassName="text-rose-500" />
+      <SpanEventHeader
+        title={exception.type ?? "Error"}
+        time={spanEvent.time}
+        titleClassName="text-rose-500"
+      />
       {exception.message && <Callout variant="error">{exception.message}</Callout>}
       {exception.stacktrace && (
         <CodeBlock

apps/webapp/app/components/runs/v3/TaskRunStatus.tsx

@@ -3,7 +3,7 @@ import {
   BoltSlashIcon,
   BugAntIcon,
   CheckCircleIcon,
-  ClockIcon,
+  FireIcon,
   NoSymbolIcon,
   PauseCircleIcon,
   RectangleStackIcon,
@@ -16,6 +16,7 @@ import { cn } from "~/utils/cn";
 
 const taskRunStatusDescriptions: Record<TaskRunStatus, string> = {
   PENDING: "Task is waiting to be executed",
+  WAITING_FOR_DEPLOY: "Task needs to be deployed first to start executing",
   EXECUTING: "Task is currently being executed",
   RETRYING_AFTER_FAILURE: "Task is being reattempted after a failure",
   WAITING_TO_RESUME: "Task has been frozen and is waiting to be resumed",
@@ -25,6 +26,7 @@ const taskRunStatusDescriptions: Record<TaskRunStatus, string> = {
   INTERRUPTED: "Task has failed because it was interrupted",
   SYSTEM_FAILURE: "Task has failed due to a system failure",
   PAUSED: "Task has been paused by the user",
+  CRASHED: "Task has crashed and won't be retried",
 };
 
 export function descriptionForTaskRunStatus(status: TaskRunStatus): string {
@@ -62,6 +64,8 @@ export function TaskRunStatusIcon({
   switch (status) {
     case "PENDING":
       return <RectangleStackIcon className={cn(runStatusClassNameColor(status), className)} />;
+    case "WAITING_FOR_DEPLOY":
+      return <RectangleStackIcon className={cn(runStatusClassNameColor(status), className)} />;
     case "EXECUTING":
       return <Spinner className={cn(runStatusClassNameColor(status), className)} />;
     case "WAITING_TO_RESUME":
@@ -80,6 +84,8 @@ export function TaskRunStatusIcon({
       return <XCircleIcon className={cn(runStatusClassNameColor(status), className)} />;
     case "SYSTEM_FAILURE":
       return <BugAntIcon className={cn(runStatusClassNameColor(status), className)} />;
+    case "CRASHED":
+      return <FireIcon className={cn(runStatusClassNameColor(status), className)} />;
 
     default: {
       const _exhaustiveCheck: never = status;
@@ -92,6 +98,8 @@ export function runStatusClassNameColor(status: TaskRunStatus): string {
   switch (status) {
     case "PENDING":
       return "text-charcoal-500";
+    case "WAITING_FOR_DEPLOY":
+      return "text-amber-500";
     case "EXECUTING":
     case "RETRYING_AFTER_FAILURE":
       return "text-pending";
@@ -109,6 +117,8 @@ export function runStatusClassNameColor(status: TaskRunStatus): string {
       return "text-error";
     case "SYSTEM_FAILURE":
       return "text-error";
+    case "CRASHED":
+      return "text-error";
     default: {
       const _exhaustiveCheck: never = status;
       throw new Error(`Non-exhaustive match for value: ${status}`);
@@ -120,6 +130,8 @@ export function runStatusTitle(status: TaskRunStatus): string {
   switch (status) {
     case "PENDING":
       return "Queued";
+    case "WAITING_FOR_DEPLOY":
+      return "Waiting for deploy";
     case "EXECUTING":
       return "Executing";
     case "WAITING_TO_RESUME":
@@ -138,6 +150,8 @@ export function runStatusTitle(status: TaskRunStatus): string {
       return "Failed";
     case "SYSTEM_FAILURE":
       return "System failure";
+    case "CRASHED":
+      return "Crashed";
     default: {
       const _exhaustiveCheck: never = status;
       throw new Error(`Non-exhaustive match for value: ${status}`);

apps/webapp/app/components/runs/v3/TaskRunsTable.tsx

@@ -24,10 +24,11 @@ import { formatDuration } from "@trigger.dev/core/v3";
 import { TaskRunStatusCombo } from "./TaskRunStatus";
 import { useEnvironments } from "~/hooks/useEnvironments";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
-import { StopCircleIcon } from "@heroicons/react/20/solid";
+import { ArrowPathIcon, StopCircleIcon } from "@heroicons/react/20/solid";
 import { Dialog, DialogTrigger } from "~/components/primitives/Dialog";
 import { CancelRunDialog } from "./CancelRunDialog";
 import { useLocation } from "@remix-run/react";
+import { ReplayRunDialog } from "./ReplayRunDialog";
 
 type RunsTableProps = {
   total: number;
@@ -49,7 +50,6 @@ export function TaskRunsTable({
 }: RunsTableProps) {
   const organization = useOrganization();
   const project = useProject();
-  const location = useLocation();
 
   return (
     <Table>
@@ -110,23 +110,7 @@ export function TaskRunsTable({
                 <TableCell to={path}>
                   {run.createdAt ? <DateTime date={run.createdAt} /> : "–"}
                 </TableCell>
-                {run.isCancellable ? (
-                  <TableCellMenu isSticky>
-                    <Dialog>
-                      <DialogTrigger asChild>
-                        <Button variant="small-menu-item" LeadingIcon={StopCircleIcon}>
-                          Cancel run
-                        </Button>
-                      </DialogTrigger>
-                      <CancelRunDialog
-                        runFriendlyId={run.friendlyId}
-                        redirectPath={`${location.pathname}${location.search}`}
-                      />
-                    </Dialog>
-                  </TableCellMenu>
-                ) : (
-                  <TableCell to={path}>{""}</TableCell>
-                )}
+                <RunActionsCell run={run} path={path} />
               </TableRow>
             );
           })
@@ -144,6 +128,43 @@ export function TaskRunsTable({
   );
 }
 
+function RunActionsCell({ run, path }: { run: RunListItem; path: string }) {
+  const location = useLocation();
+
+  if (!run.isCancellable && !run.isReplayable) return <TableCell to={path}>{""}</TableCell>;
+
+  return (
+    <TableCellMenu isSticky>
+      {run.isCancellable && (
+        <Dialog>
+          <DialogTrigger asChild>
+            <Button variant="small-menu-item" LeadingIcon={StopCircleIcon}>
+              Cancel run
+            </Button>
+          </DialogTrigger>
+          <CancelRunDialog
+            runFriendlyId={run.friendlyId}
+            redirectPath={`${location.pathname}${location.search}`}
+          />
+        </Dialog>
+      )}
+      {run.isReplayable && (
+        <Dialog>
+          <DialogTrigger asChild>
+            <Button variant="small-menu-item" LeadingIcon={ArrowPathIcon}>
+              Replay run
+            </Button>
+          </DialogTrigger>
+          <ReplayRunDialog
+            runFriendlyId={run.friendlyId}
+            failedRedirect={`${location.pathname}${location.search}`}
+          />
+        </Dialog>
+      )}
+    </TableCellMenu>
+  );
+}
+
 function NoRuns({ title }: { title: string }) {
   return (
     <div className="flex items-center justify-center">

apps/webapp/app/env.server.ts

@@ -137,8 +137,9 @@ const EnvironmentSchema = z.object({
   INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE: z.string().optional(),
   INTERNAL_OTEL_TRACE_LOGGING_ENABLED: z.string().default("1"),
   // this means 1/20 traces or 5% of traces will be sampled (sampled = recorded)
-  INTERNAL_OTEL_TRACE_SAMPING_RATE: z.string().default("20"),
+  INTERNAL_OTEL_TRACE_SAMPLING_RATE: z.string().default("20"),
   INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED: z.string().default("0"),
+  INTERNAL_OTEL_TRACE_DISABLED: z.string().default("0"),
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/metrics.server.ts

@@ -0,0 +1,19 @@
+import { OpenMetricsContentType, Registry, collectDefaultMetrics, register } from "prom-client";
+import { singleton } from "./utils/singleton";
+import { env } from "./env.server";
+
+export const metricsRegister = singleton("metricsRegister", initializeMetricsRegister);
+
+function initializeMetricsRegister() {
+  const registry = new Registry<OpenMetricsContentType>();
+
+  register.setDefaultLabels({
+    serviceName: env.SERVICE_NAME,
+  });
+
+  registry.setContentType("application/openmetrics-text; version=1.0.0; charset=utf-8");
+
+  collectDefaultMetrics({ register: registry });
+
+  return registry;
+}

apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts

@@ -12,6 +12,12 @@ import { User } from "~/models/user.server";
 import { safeJsonParse } from "~/utils/json";
 import { getUsername } from "~/utils/username";
 
+export type ErrorData = {
+  name: string;
+  message: string;
+  stack?: string;
+};
+
 export class DeploymentPresenter {
   #prismaClient: PrismaClient;
 
@@ -133,7 +139,7 @@ export class DeploymentPresenter {
     };
   }
 
-  #prepareErrorData(errorData: WorkerDeployment["errorData"]) {
+  #prepareErrorData(errorData: WorkerDeployment["errorData"]): ErrorData | undefined {
     if (!errorData) {
       return;
     }

apps/webapp/app/presenters/v3/RunListPresenter.server.ts

@@ -222,6 +222,7 @@ export class RunListPresenter {
           version: run.version,
           taskIdentifier: run.taskIdentifier,
           attempts: Number(run.attempts),
+          isReplayable: true,
           isCancellable: CANCELLABLE_STATUSES.includes(run.status),
           environment: {
             type: environment.type,

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.triggers._index/route.tsx

@@ -44,7 +44,7 @@ export default function Integrations() {
 
   return (
     <>
-      <Paragraph variant="small" spacing>
+      <Paragraph variant="small" spacing className="pt-2">
         External Triggers get registered with external APIs, for example a webhook.
       </Paragraph>
       <Table containerClassName="mt-4">

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.triggers.scheduled/route.tsx

@@ -44,7 +44,7 @@ export default function Integrations() {
 
   return (
     <>
-      <Paragraph variant="small" spacing>
+      <Paragraph variant="small" spacing className="pt-2">
         A Scheduled Trigger runs a Job on a repeated schedule. The schedule can use a CRON
         expression or an interval.
       </Paragraph>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.triggers.webhooks/route.tsx

@@ -44,7 +44,7 @@ export default function Integrations() {
 
   return (
     <>
-      <Paragraph variant="small" spacing>
+      <Paragraph variant="small" spacing className="pt-2">
         A Webhook Trigger runs a Job when it receives a matching payload at a registered HTTP
         Endpoint.
       </Paragraph>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.triggers/route.tsx

@@ -32,7 +32,7 @@ export default function Page() {
       </NavBar>
 
       <PageBody scrollable={false}>
-        <div className="grid grid-rows-[auto_1fr] gap-y-4 px-4">
+        <div className="grid max-h-full grid-rows-[auto_1fr] overflow-hidden px-4">
           <PageTabs
             layoutId="triggers"
             tabs={[

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.deployments.$deploymentParam/route.tsx

@@ -19,6 +19,7 @@ import {
   TableHeaderCell,
   TableRow,
 } from "~/components/primitives/Table";
+import { DeploymentError } from "~/components/runs/v3/DeploymentError";
 import { DeploymentStatus } from "~/components/runs/v3/DeploymentStatus";
 import { TaskFunctionName } from "~/components/runs/v3/TaskPath";
 import { useOrganization } from "~/hooks/useOrganizations";
@@ -158,25 +159,7 @@ export default function Page() {
               </Table>
             </div>
           ) : deployment.errorData ? (
-            <div className="flex flex-col">
-              {deployment.errorData.stack ? (
-                <CodeBlock
-                  language="markdown"
-                  rowTitle={deployment.errorData.message}
-                  code={deployment.errorData.stack}
-                  maxLines={20}
-                />
-              ) : (
-                <div className="flex flex-col">
-                  <Paragraph
-                    variant="base/bright"
-                    className="w-full border-b border-grid-dimmed py-2.5"
-                  >
-                    {deployment.errorData.message}
-                  </Paragraph>
-                </div>
-              )}
-            </div>
+            <DeploymentError errorData={deployment.errorData} />
           ) : null}
         </div>
       </div>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.environment-variables.new/route.tsx

@@ -16,6 +16,7 @@ import { FormError } from "~/components/primitives/FormError";
 import { Input } from "~/components/primitives/Input";
 import { InputGroup } from "~/components/primitives/InputGroup";
 import { Label } from "~/components/primitives/Label";
+import { Switch } from "~/components/primitives/Switch";
 import { prisma } from "~/db.server";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
@@ -134,6 +135,8 @@ export default function Page() {
     shouldRevalidate: "onSubmit",
   });
 
+  const [revealAll, setRevealAll] = useState(false);
+
   useEffect(() => {
     setIsOpen(true);
   }, []);
@@ -169,7 +172,15 @@ export default function Page() {
               />
             </InputGroup>
             <InputGroup fullWidth>
-              <Label>Values</Label>
+              <div className="flex items-center justify-between">
+                <Label>Values</Label>
+                <Switch
+                  variant="small"
+                  label="Reveal values"
+                  checked={revealAll}
+                  onCheckedChange={(e) => setRevealAll(e.valueOf())}
+                />
+              </div>
               <div className="grid grid-cols-[auto_1fr] gap-x-2 gap-y-2">
                 {environments.map((environment, index) => {
                   return (
@@ -186,7 +197,7 @@ export default function Page() {
                         <EnvironmentLabel environment={environment} className="h-5 px-2" />
                       </label>
                       <Input
-                        type="password"
+                        type={revealAll ? "text" : "password"}
                         name={`values[${index}].value`}
                         placeholder="Not set"
                       />

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx

@@ -1,4 +1,9 @@
-import { CloudArrowDownIcon, QueueListIcon, StopCircleIcon } from "@heroicons/react/20/solid";
+import {
+  ArrowPathIcon,
+  CloudArrowDownIcon,
+  QueueListIcon,
+  StopCircleIcon,
+} from "@heroicons/react/20/solid";
 import { useParams } from "@remix-run/react";
 import { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { formatDurationNanoseconds, nanosecondsToMilliseconds } from "@trigger.dev/core/v3";
@@ -14,6 +19,7 @@ import { Paragraph } from "~/components/primitives/Paragraph";
 import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
 import { CancelRunDialog } from "~/components/runs/v3/CancelRunDialog";
 import { LiveTimer } from "~/components/runs/v3/LiveTimer";
+import { ReplayRunDialog } from "~/components/runs/v3/ReplayRunDialog";
 import { RunIcon } from "~/components/runs/v3/RunIcon";
 import { SpanEvents } from "~/components/runs/v3/SpanEvents";
 import { SpanTitle } from "~/components/runs/v3/SpanTitle";
@@ -22,7 +28,7 @@ import { TaskRunAttemptStatusCombo } from "~/components/runs/v3/TaskRunAttemptSt
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
 import { redirectWithErrorMessage } from "~/models/message.server";
-import { SpanPresenter } from "~/presenters/v3/SpanPresenter.server";
+import { Span, SpanPresenter } from "~/presenters/v3/SpanPresenter.server";
 import { requireUserId } from "~/services/session.server";
 import { cn } from "~/utils/cn";
 import { v3RunPath, v3RunSpanPath, v3SpanParamsSchema, v3TraceSpanPath } from "~/utils/pathBuilder";
@@ -188,24 +194,7 @@ export default function Page() {
             )}
           </div>
           <div className="flex items-center gap-4">
-            {event.isPartial && runParam && (
-              <Dialog>
-                <DialogTrigger asChild>
-                  <Button variant="danger/small" LeadingIcon={StopCircleIcon}>
-                    Cancel run
-                  </Button>
-                </DialogTrigger>
-                <CancelRunDialog
-                  runFriendlyId={event.runId}
-                  redirectPath={v3RunSpanPath(
-                    organization,
-                    project,
-                    { friendlyId: runParam },
-                    { spanId: event.spanId }
-                  )}
-                />
-              </Dialog>
-            )}
+            <RunActionButtons span={event} />
           </div>
         </div>
       ) : null}
@@ -213,6 +202,54 @@ export default function Page() {
   );
 }
 
+function RunActionButtons({ span }: { span: Span }) {
+  const organization = useOrganization();
+  const project = useProject();
+  const { runParam } = useParams();
+
+  if (!runParam) return null;
+
+  if (span.isPartial) {
+    return (
+      <Dialog>
+        <DialogTrigger asChild>
+          <Button variant="danger/small" LeadingIcon={StopCircleIcon}>
+            Cancel run
+          </Button>
+        </DialogTrigger>
+        <CancelRunDialog
+          runFriendlyId={span.runId}
+          redirectPath={v3RunSpanPath(
+            organization,
+            project,
+            { friendlyId: runParam },
+            { spanId: span.spanId }
+          )}
+        />
+      </Dialog>
+    );
+  }
+
+  return (
+    <Dialog>
+      <DialogTrigger asChild>
+        <Button variant="tertiary/small" LeadingIcon={ArrowPathIcon}>
+          Replay run
+        </Button>
+      </DialogTrigger>
+      <ReplayRunDialog
+        runFriendlyId={span.runId}
+        failedRedirect={v3RunSpanPath(
+          organization,
+          project,
+          { friendlyId: runParam },
+          { spanId: span.spanId }
+        )}
+      />
+    </Dialog>
+  );
+}
+
 function PacketDisplay({
   data,
   dataType,

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx

@@ -12,16 +12,19 @@ import {
   millisecondsToNanoseconds,
   nanosecondsToMilliseconds,
 } from "@trigger.dev/core/v3";
+import { RuntimeEnvironmentType } from "@trigger.dev/database";
 import { motion } from "framer-motion";
 import { useEffect, useRef, useState } from "react";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { ShowParentIcon, ShowParentIconSelected } from "~/assets/icons/ShowParentIcon";
 import tileBgPath from "~/assets/images/error-banner-tile@2x.png";
 import { BlankstateInstructions } from "~/components/BlankstateInstructions";
+import { InlineCode } from "~/components/code/InlineCode";
 import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
 import { MainCenteredContainer, PageBody } from "~/components/layout/AppLayout";
 import { Badge } from "~/components/primitives/Badge";
 import { LinkButton } from "~/components/primitives/Buttons";
+import { Callout } from "~/components/primitives/Callout";
 import { Input } from "~/components/primitives/Input";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
@@ -181,6 +184,7 @@ export default function Page() {
               totalDuration={duration}
               rootSpanStatus={rootSpanStatus}
               rootStartedAt={rootStartedAt}
+              environmentType={run.environment.type}
             />
           ) : (
             <ResizablePanelGroup
@@ -209,6 +213,7 @@ export default function Page() {
                   totalDuration={duration}
                   rootSpanStatus={rootSpanStatus}
                   rootStartedAt={rootStartedAt}
+                  environmentType={run.environment.type}
                 />
               </ResizablePanel>
               <ResizableHandle withHandle />
@@ -231,6 +236,7 @@ type TasksTreeViewProps = {
   totalDuration: number;
   rootSpanStatus: "executing" | "completed" | "failed";
   rootStartedAt: Date | undefined;
+  environmentType: RuntimeEnvironmentType;
 };
 
 function TasksTreeView({
@@ -241,6 +247,7 @@ function TasksTreeView({
   totalDuration,
   rootSpanStatus,
   rootStartedAt,
+  environmentType,
 }: TasksTreeViewProps) {
   const [filterText, setFilterText] = useState("");
   const [errorsOnly, setErrorsOnly] = useState(false);
@@ -345,63 +352,68 @@ function TasksTreeView({
               getNodeProps={getNodeProps}
               getTreeProps={getTreeProps}
               renderNode={({ node, state }) => (
-                <div
-                  className={cn(
-                    "delay-[25ms] flex h-8 cursor-pointer items-center overflow-hidden rounded-l-sm pr-2 transition-colors",
-                    state.selected
-                      ? "bg-grid-dimmed hover:bg-grid-bright"
-                      : "bg-transparent hover:bg-grid-dimmed"
-                  )}
-                  onClick={() => {
-                    toggleNodeSelection(node.id);
-                  }}
-                >
-                  <div className="flex h-8 items-center">
-                    {Array.from({ length: node.level }).map((_, index) => (
-                      <TaskLine
-                        key={index}
-                        isError={node.data.isError}
-                        isSelected={state.selected}
-                      />
-                    ))}
-                    <div
-                      className={cn(
-                        "flex h-8 w-4 items-center",
-                        node.hasChildren && "hover:bg-charcoal-600"
-                      )}
-                      onClick={(e) => {
-                        e.stopPropagation();
-                        toggleExpandNode(node.id);
-                        scrollToNode(node.id);
-                      }}
-                    >
-                      {node.hasChildren ? (
-                        state.expanded ? (
-                          <ChevronDownIcon className="h-4 w-4 text-charcoal-400" />
+                <>
+                  <div
+                    className={cn(
+                      "delay-[25ms] flex h-8 cursor-pointer items-center overflow-hidden rounded-l-sm pr-2 transition-colors",
+                      state.selected
+                        ? "bg-grid-dimmed hover:bg-grid-bright"
+                        : "bg-transparent hover:bg-grid-dimmed"
+                    )}
+                    onClick={() => {
+                      toggleNodeSelection(node.id);
+                    }}
+                  >
+                    <div className="flex h-8 items-center">
+                      {Array.from({ length: node.level }).map((_, index) => (
+                        <TaskLine
+                          key={index}
+                          isError={node.data.isError}
+                          isSelected={state.selected}
+                        />
+                      ))}
+                      <div
+                        className={cn(
+                          "flex h-8 w-4 items-center",
+                          node.hasChildren && "hover:bg-charcoal-600"
+                        )}
+                        onClick={(e) => {
+                          e.stopPropagation();
+                          toggleExpandNode(node.id);
+                          scrollToNode(node.id);
+                        }}
+                      >
+                        {node.hasChildren ? (
+                          state.expanded ? (
+                            <ChevronDownIcon className="h-4 w-4 text-charcoal-400" />
+                          ) : (
+                            <ChevronRightIcon className="h-4 w-4 text-charcoal-400" />
+                          )
                         ) : (
-                          <ChevronRightIcon className="h-4 w-4 text-charcoal-400" />
-                        )
-                      ) : (
-                        <div className="h-8 w-4" />
-                      )}
+                          <div className="h-8 w-4" />
+                        )}
+                      </div>
                     </div>
-                  </div>
 
-                  <div className="flex w-full items-center justify-between gap-2 pl-1">
-                    <div className="flex items-center gap-2 overflow-x-hidden">
-                      <RunIcon
-                        name={node.data.style?.icon}
-                        spanName={node.data.message}
-                        className="h-4 min-h-4 w-4 min-w-4"
-                      />
-                      <NodeText node={node} />
-                      {node.data.isRoot && <Badge variant="outline-rounded">Root</Badge>}
-                    </div>
-                    <div className="flex items-center gap-1">
-                      <NodeStatusIcon node={node} />
+                    <div className="flex w-full items-center justify-between gap-2 pl-1">
+                      <div className="flex items-center gap-2 overflow-x-hidden">
+                        <RunIcon
+                          name={node.data.style?.icon}
+                          spanName={node.data.message}
+                          className="h-4 min-h-4 w-4 min-w-4"
+                        />
+                        <NodeText node={node} />
+                        {node.data.isRoot && <Badge variant="outline-rounded">Root</Badge>}
+                      </div>
+                      <div className="flex items-center gap-1">
+                        <NodeStatusIcon node={node} />
+                      </div>
                     </div>
                   </div>
-                </div>
+                  {events.length === 1 && environmentType === "DEVELOPMENT" && (
+                    <ConnectedDevWarning />
+                  )}
+                </>
               )}
               onScroll={(scrollTop) => {
                 //sync the scroll to the tree
@@ -841,3 +853,34 @@ function CurrentTimeIndicator({ totalDuration }: { totalDuration: number }) {
     </Timeline.FollowCursor>
   );
 }
+
+function ConnectedDevWarning() {
+  const [isVisible, setIsVisible] = useState(false);
+
+  useEffect(() => {
+    const timer = setTimeout(() => {
+      setIsVisible(true);
+    }, 3000);
+
+    return () => clearTimeout(timer);
+  }, []);
+
+  return (
+    <div
+      className={cn(
+        "flex items-center overflow-hidden pl-5 pr-2 transition-opacity duration-500",
+        isVisible ? "opacity-100" : "h-0 opacity-0"
+      )}
+    >
+      <Callout variant="info">
+        <div className="flex flex-col gap-1">
+          <Paragraph variant="small">
+            Runs usually start within 1 second in{" "}
+            <EnvironmentLabel environment={{ type: "DEVELOPMENT" }} />. Check you're running the
+            CLI: <InlineCode className="whitespace-nowrap">npx trigger.dev@beta dev</InlineCode>
+          </Paragraph>
+        </div>
+      </Callout>
+    </div>
+  );
+}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs._index/route.tsx

@@ -64,10 +64,12 @@ export default function Page() {
         <PageTitle title="Runs" />
       </NavBar>
       <PageBody>
-        {list.possibleTasks.length === 0 ? (
-          <CreateFirstTaskInstructions />
-        ) : list.runs.length === 0 && !list.hasFilters ? (
-          <RunTaskInstructions />
+        {list.runs.length === 0 && !list.hasFilters ? (
+          list.possibleTasks.length === 0 ? (
+            <CreateFirstTaskInstructions />
+          ) : (
+            <RunTaskInstructions />
+          )
         ) : (
           <div className={cn("grid h-fit grid-cols-1 gap-4")}>
             <div>

apps/webapp/app/routes/admin.api.v1.gc.ts

@@ -0,0 +1,52 @@
+import { type DataFunctionArgs } from "@remix-run/node";
+import { PerformanceObserver } from "node:perf_hooks";
+import { runInNewContext } from "node:vm";
+import v8 from "v8";
+import { requireUser } from "~/services/session.server";
+
+async function waitTillGcFinishes() {
+  let resolver: (value: PerformanceEntry) => void;
+  let rejector: (reason?: any) => void;
+
+  const promise = new Promise<PerformanceEntry>((resolve, reject) => {
+    resolver = resolve;
+    rejector = reject;
+  });
+
+  const obs = new PerformanceObserver((list) => {
+    const entry = list.getEntries()[0];
+
+    if (entry.name === "gc") {
+      resolver(entry);
+    }
+  });
+
+  obs.observe({ entryTypes: ["gc"] });
+
+  v8.setFlagsFromString("--expose-gc");
+  const gc = global.gc ?? runInNewContext("gc");
+
+  gc();
+
+  // disable expose-gc
+  v8.setFlagsFromString("--noexpose-gc");
+
+  return promise;
+}
+
+export async function loader({ request }: DataFunctionArgs) {
+  const user = await requireUser(request);
+
+  if (!user.admin) {
+    throw new Response("You must be an admin to perform this action", { status: 403 });
+  }
+
+  const entry = await waitTillGcFinishes();
+
+  return new Response(JSON.stringify(entry), {
+    status: 200,
+    headers: {
+      "Content-Type": "application/json",
+    },
+  });
+}

apps/webapp/app/routes/admin.api.v1.snapshot.ts

@@ -1,11 +1,9 @@
-import path from "path";
-import os from "os";
+import { type DataFunctionArgs } from "@remix-run/node";
 import fs from "fs";
-import v8 from "v8";
+import os from "os";
+import path from "path";
 import { PassThrough } from "stream";
-import { json, type DataFunctionArgs } from "@remix-run/node";
-import { prisma } from "~/db.server";
-import { authenticateApiRequest } from "~/services/apiAuth.server";
+import v8 from "v8";
 import { requireUser } from "~/services/session.server";
 
 // Format date as yyyy-MM-dd HH_mm_ss_SSS
@@ -32,10 +30,11 @@ export async function loader({ request }: DataFunctionArgs) {
     throw new Response("You must be an admin to perform this action", { status: 403 });
   }
 
-  const host = request.headers.get("X-Forwarded-Host") ?? request.headers.get("host");
-
   const tempDir = os.tmpdir();
-  const filepath = path.join(tempDir, `${host}-${formatDate(new Date())}.heapsnapshot`);
+  const filepath = path.join(
+    tempDir,
+    `${getTaskIdentifier()}-${formatDate(new Date())}.heapsnapshot`
+  );
 
   const snapshotPath = v8.writeHeapSnapshot(filepath);
   if (!snapshotPath) {
@@ -57,3 +56,13 @@ export async function loader({ request }: DataFunctionArgs) {
     },
   });
 }
+
+function getTaskIdentifier() {
+  if (!process.env.ECS_CONTAINER_METADATA_URI) {
+    return "local";
+  }
+
+  const url = new URL(process.env.ECS_CONTAINER_METADATA_URI);
+
+  return url.pathname.split("/")[2].split("-")[0];
+}

apps/webapp/app/routes/api.v1.runs.$runParam.replay.ts

@@ -0,0 +1,72 @@
+import type { ActionFunctionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import { PrismaErrorSchema, prisma } from "~/db.server";
+import { z } from "zod";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { CancelRunService } from "~/services/runs/cancelRun.server";
+import { ApiRunPresenter } from "~/presenters/ApiRunPresenter.server";
+import { ReplayTaskRunService } from "~/v3/services/replayTaskRun.server";
+import { logger } from "~/services/logger.server";
+
+const ParamsSchema = z.object({
+  /* This is the run friendly ID */
+  runParam: z.string(),
+});
+
+export async function action({ request, params }: ActionFunctionArgs) {
+  // Ensure this is a POST request
+  if (request.method.toUpperCase() !== "POST") {
+    return { status: 405, body: "Method Not Allowed" };
+  }
+
+  // Authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API Key" }, { status: 401 });
+  }
+
+  const parsed = ParamsSchema.safeParse(params);
+  if (!parsed.success) {
+    return json({ error: "Invalid or missing run ID" }, { status: 400 });
+  }
+
+  const { runParam } = parsed.data;
+
+  try {
+    const taskRun = await prisma.taskRun.findUnique({
+      where: {
+        friendlyId: runParam,
+      },
+    });
+
+    if (!taskRun) {
+      return json({ error: "Run not found" }, { status: 404 });
+    }
+
+    const service = new ReplayTaskRunService();
+    const newRun = await service.call(taskRun);
+
+    if (!newRun) {
+      return json({ error: "Failed to create new run" }, { status: 400 });
+    }
+
+    return json({
+      id: newRun?.friendlyId,
+    });
+  } catch (error) {
+    if (error instanceof Error) {
+      logger.error("Failed to replay run", {
+        error: {
+          name: error.name,
+          message: error.message,
+          stack: error.stack,
+        },
+        run: runParam,
+      });
+      return json({ error: error.message }, { status: 400 });
+    } else {
+      logger.error("Failed to replay run", { error: JSON.stringify(error), run: runParam });
+      return json({ error: JSON.stringify(error) }, { status: 400 });
+    }
+  }
+}

apps/webapp/app/routes/api.v1.token.ts

@@ -23,7 +23,7 @@ export async function action({ request }: ActionFunctionArgs) {
   const anyBody = await request.json();
   const body = GetPersonalAccessTokenRequestSchema.safeParse(anyBody);
   if (!body.success) {
-    return json({ message: generateErrorMessage(body.error.issues) }, { status: 422 });
+    return json({ error: generateErrorMessage(body.error.issues) }, { status: 422 });
   }
 
   try {
@@ -45,6 +45,6 @@ export async function action({ request }: ActionFunctionArgs) {
       return json({ error: error.message }, { status: 400 });
     }
 
-    return json({ error: "Something went wrong" }, { status: 500 });
+    return json({ error: "Something went wrong" }, { status: 400 });
   }
 }

apps/webapp/app/routes/api.v2.runs.$runParam.cancel.ts

@@ -25,7 +25,7 @@ export async function action({ request, params }: ActionFunctionArgs) {
   const parsed = ParamsSchema.safeParse(params);
 
   if (!parsed.success) {
-    return json({ error: "Invalid or Missing runId" }, { status: 400 });
+    return json({ error: "Invalid or Missing run id" }, { status: 400 });
   }
 
   const { runParam } = parsed.data;

apps/webapp/app/routes/api.v2.whoami.ts

@@ -8,29 +8,34 @@ import { authenticateApiRequestWithPersonalAccessToken } from "~/services/person
 
 export async function loader({ request }: LoaderFunctionArgs) {
   logger.info("whoami v2", { url: request.url });
+  try {
+    const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
+    if (!authenticationResult) {
+      return json({ error: "Invalid or Missing Access Token" }, { status: 401 });
+    }
 
-  const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
-  if (!authenticationResult) {
-    return json({ error: "Invalid or Missing Access Token" }, { status: 401 });
+    const user = await prisma.user.findUnique({
+      select: {
+        email: true,
+      },
+      where: {
+        id: authenticationResult.userId,
+      },
+    });
+
+    if (!user) {
+      return json({ error: "User not found" }, { status: 404 });
+    }
+
+    const result: WhoAmIResponse = {
+      userId: authenticationResult.userId,
+      email: user.email,
+      dashboardUrl: env.APP_ORIGIN,
+    };
+    return json(result);
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : "Something went wrong";
+    logger.error("Error in whoami v2", { error: errorMessage });
+    return json({ error: errorMessage }, { status: 400 });
   }
-
-  const user = await prisma.user.findUnique({
-    select: {
-      email: true,
-    },
-    where: {
-      id: authenticationResult.userId,
-    },
-  });
-
-  if (!user) {
-    return json({ error: "User not found" }, { status: 404 });
-  }
-
-  const result: WhoAmIResponse = {
-    userId: authenticationResult.userId,
-    email: user.email,
-    dashboardUrl: env.APP_ORIGIN,
-  };
-  return json(result);
 }

apps/webapp/app/routes/metrics.ts

@@ -0,0 +1,10 @@
+import { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { metricsRegister } from "~/metrics.server";
+
+export async function loader({ request }: LoaderFunctionArgs) {
+  return new Response(await metricsRegister.metrics(), {
+    headers: {
+      "Content-Type": metricsRegister.contentType,
+    },
+  });
+}

apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts

@@ -0,0 +1,85 @@
+import { parse } from "@conform-to/zod";
+import { ActionFunction, json } from "@remix-run/node";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
+import { logger } from "~/services/logger.server";
+import { v3RunPath } from "~/utils/pathBuilder";
+import { ReplayTaskRunService } from "~/v3/services/replayTaskRun.server";
+
+const FormSchema = z.object({
+  failedRedirect: z.string(),
+});
+
+const ParamSchema = z.object({
+  runParam: z.string(),
+});
+
+export const action: ActionFunction = async ({ request, params }) => {
+  const { runParam } = ParamSchema.parse(params);
+
+  const formData = await request.formData();
+  const submission = parse(formData, { schema: FormSchema });
+
+  if (!submission.value) {
+    return json(submission);
+  }
+
+  try {
+    const taskRun = await prisma.taskRun.findUnique({
+      where: {
+        friendlyId: runParam,
+      },
+      include: {
+        project: {
+          include: {
+            organization: true,
+          },
+        },
+      },
+    });
+
+    if (!taskRun) {
+      return redirectWithErrorMessage(submission.value.failedRedirect, request, "Run not found");
+    }
+
+    const replayRunService = new ReplayTaskRunService();
+    const newRun = await replayRunService.call(taskRun);
+
+    if (!newRun) {
+      return redirectWithErrorMessage(
+        submission.value.failedRedirect,
+        request,
+        "Failed to replay run"
+      );
+    }
+
+    const runPath = v3RunPath(
+      {
+        slug: taskRun.project.organization.slug,
+      },
+      { slug: taskRun.project.slug },
+      { friendlyId: newRun.friendlyId }
+    );
+
+    return redirectWithSuccessMessage(runPath, request, `Replaying run`);
+  } catch (error) {
+    if (error instanceof Error) {
+      logger.error("Failed to replay run", {
+        error: {
+          name: error.name,
+          message: error.message,
+          stack: error.stack,
+        },
+      });
+      return redirectWithErrorMessage(submission.value.failedRedirect, request, error.message);
+    } else {
+      logger.error("Failed to replay run", { error });
+      return redirectWithErrorMessage(
+        submission.value.failedRedirect,
+        request,
+        JSON.stringify(error)
+      );
+    }
+  }
+};

apps/webapp/app/services/apiRateLimit.server.ts

@@ -67,6 +67,11 @@ export function authorizationRateLimitMiddleware({
       logger.info(`RateLimiter (${keyPrefix}): request to ${req.path}`);
     }
 
+    // allow OPTIONS requests
+    if (req.method.toUpperCase() === "OPTIONS") {
+      return next();
+    }
+
     //first check if any of the pathMatchers match the request path
     const path = req.path;
     if (
@@ -109,6 +114,7 @@ export function authorizationRateLimitMiddleware({
             status: 401,
             type: "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/401",
             detail: "No authorization header provided",
+            error: "No authorization header provided",
           },
           null,
           2
@@ -160,6 +166,7 @@ export function authorizationRateLimitMiddleware({
           detail: `Rate limit exceeded ${remaining}/${limit} requests remaining. Retry after ${reset} seconds.`,
           reset: reset,
           limit: limit,
+          error: `Rate limit exceeded ${remaining}/${limit} requests remaining. Retry after ${reset} seconds.`,
         },
         null,
         2
@@ -182,7 +189,16 @@ export const apiRateLimiter = authorizationRateLimitMiddleware({
   },
   limiter: Ratelimit.slidingWindow(env.API_RATE_LIMIT_MAX, env.API_RATE_LIMIT_WINDOW as Duration),
   pathMatchers: [/^\/api/],
-  pathWhiteList: ["/api/v1/authorization-code", "/api/v1/token"],
+  // Allow /api/v1/tasks/:id/callback/:secret
+  pathWhiteList: [
+    "/api/v1/authorization-code",
+    "/api/v1/token",
+    /^\/api\/v1\/tasks\/[^\/]+\/callback\/[^\/]+$/, // /api/v1/tasks/$id/callback/$secret
+    /^\/api\/v1\/runs\/[^\/]+\/tasks\/[^\/]+\/callback\/[^\/]+$/, // /api/v1/runs/$runId/tasks/$id/callback/$secret
+    /^\/api\/v1\/http-endpoints\/[^\/]+\/env\/[^\/]+\/[^\/]+$/, // /api/v1/http-endpoints/$httpEndpointId/env/$envType/$shortcode
+    /^\/api\/v1\/sources\/http\/[^\/]+$/, // /api/v1/sources/http/$id
+    /^\/api\/v1\/endpoints\/[^\/]+\/[^\/]+\/index\/[^\/]+$/, // /api/v1/endpoints/$environmentId/$endpointSlug/index/$indexHookIdentifier
+  ],
   log: {
     rejections: env.API_RATE_LIMIT_REJECTION_LOGS_ENABLED === "1",
     requests: env.API_RATE_LIMIT_REQUEST_LOGS_ENABLED === "1",

apps/webapp/app/services/personalAccessToken.server.ts

@@ -1,10 +1,10 @@
+import { PersonalAccessToken } from "@trigger.dev/database";
 import { customAlphabet, nanoid } from "nanoid";
 import nodeCrypto from "node:crypto";
 import { z } from "zod";
 import { prisma } from "~/db.server";
 import { env } from "~/env.server";
 import { logger } from "./logger.server";
-import { PersonalAccessToken } from "@trigger.dev/database";
 
 const tokenValueLength = 40;
 //lowercase only, removed 0 and l to avoid confusion
@@ -54,7 +54,6 @@ export async function getPersonalAccessTokenFromAuthorizationCode(authorizationC
     },
     where: {
       code: authorizationCode,
-
       createdAt: {
         gte: tenMinutesAgo,
       },
@@ -199,13 +198,25 @@ export async function createPersonalAccessTokenFromAuthorizationCode(
     },
   });
 
-  //we only allow you to have one CLI PAT at a time
+  //we only allow you to have one CLI PAT at a time, so return this
   if (existingCliPersonalAccessToken) {
-    await prisma.personalAccessToken.delete({
+    //associate this authorization code with the existing personal access token
+    await prisma.authorizationCode.update({
       where: {
-        id: existingCliPersonalAccessToken.id,
+        code: authorizationCode,
+      },
+      data: {
+        personalAccessTokenId: existingCliPersonalAccessToken.id,
       },
     });
+
+    //we don't return the decrypted token
+    return {
+      id: existingCliPersonalAccessToken.id,
+      name: existingCliPersonalAccessToken.name,
+      userId: existingCliPersonalAccessToken.userId,
+      obfuscateToken: existingCliPersonalAccessToken.obfuscatedToken,
+    };
   }
 
   const token = await createPersonalAccessToken({

apps/webapp/app/services/worker.server.ts

@@ -35,6 +35,7 @@ import { ResumeBatchRunService } from "~/v3/services/resumeBatchRun.server";
 import { ResumeTaskDependencyService } from "~/v3/services/resumeTaskDependency.server";
 import { TimeoutDeploymentService } from "~/v3/services/timeoutDeployment.server";
 import { eventRepository } from "~/v3/eventRepository.server";
+import { ExecuteTasksWaitingForDeployService } from "~/v3/services/executeTasksWaitingForDeploy";
 
 const workerCatalog = {
   indexEndpoint: z.object({
@@ -128,6 +129,9 @@ const workerCatalog = {
     fromStatus: z.string(),
     errorMessage: z.string(),
   }),
+  "v3.executeTasksWaitingForDeploy": z.object({
+    backgroundWorkerId: z.string(),
+  }),
 };
 
 const executionWorkerCatalog = {
@@ -507,6 +511,15 @@ function getWorkerQueue() {
           return await service.call(payload.deploymentId, payload.fromStatus, payload.errorMessage);
         },
       },
+      "v3.executeTasksWaitingForDeploy": {
+        priority: 0,
+        maxAttempts: 5,
+        handler: async (payload, job) => {
+          const service = new ExecuteTasksWaitingForDeployService();
+
+          return await service.call(payload.backgroundWorkerId);
+        },
+      },
     },
   });
 }

apps/webapp/app/v3/authenticatedSocketConnection.server.ts

@@ -6,11 +6,10 @@ import {
 } from "@trigger.dev/core/v3";
 import { Evt } from "evt";
 import { randomUUID } from "node:crypto";
+import type { CloseEvent, ErrorEvent, MessageEvent, WebSocket } from "ws";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { DevQueueConsumer } from "./marqs/devQueueConsumer.server";
-import type { WebSocket, MessageEvent, CloseEvent, ErrorEvent } from "ws";
-import { env } from "~/env.server";
 
 export class AuthenticatedSocketConnection {
   public id: string;
@@ -93,7 +92,11 @@ export class AuthenticatedSocketConnection {
 
     await this._consumer.stop();
 
-    this.onClose.post(ev);
+    const result = this.onClose.post(ev);
+
+    logger.debug("[AuthenticatedSocketConnection] Called onClose", {
+      result,
+    });
   }
 
   async #handleError(ev: ErrorEvent) {

apps/webapp/app/v3/eventRepository.server.ts

@@ -3,6 +3,7 @@ import { RandomIdGenerator } from "@opentelemetry/sdk-trace-base";
 import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions";
 import {
   ExceptionEventProperties,
+  ExceptionSpanEvent,
   PRIMARY_VARIANT,
   SemanticInternalAttributes,
   SpanEvent,
@@ -25,6 +26,9 @@ import { env } from "~/env.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { DynamicFlushScheduler } from "./dynamicFlushScheduler.server";
+import { singleton } from "~/utils/singleton";
+import { Gauge } from "prom-client";
+import { metricsRegister } from "~/metrics.server";
 
 export type CreatableEvent = Omit<
   Prisma.TaskEventCreateInput,
@@ -147,6 +151,11 @@ export class EventRepository {
   private readonly _flushScheduler: DynamicFlushScheduler<CreatableEvent>;
   private _randomIdGenerator = new RandomIdGenerator();
   private _redisPublishClient: Redis;
+  private _subscriberCount = 0;
+
+  get subscriberCount() {
+    return this._subscriberCount;
+  }
 
   constructor(private db: PrismaClient = prisma, private readonly _config: EventRepoConfig) {
     this._flushScheduler = new DynamicFlushScheduler({
@@ -255,6 +264,47 @@ export class EventRepository {
     });
   }
 
+  async crashEvent({
+    event,
+    crashedAt,
+    exception,
+  }: {
+    event: TaskEventRecord;
+    crashedAt: Date;
+    exception: ExceptionEventProperties;
+  }) {
+    if (!event.isPartial) {
+      return;
+    }
+
+    await this.insertImmediate({
+      ...omit(event, "id"),
+      isPartial: false,
+      isError: true,
+      isCancelled: false,
+      status: "ERROR",
+      links: event.links ?? [],
+      events: [
+        {
+          name: "exception",
+          time: crashedAt,
+          properties: {
+            exception,
+          },
+        } satisfies ExceptionSpanEvent,
+        ...((event.events as any[]) ?? []),
+      ],
+      duration: calculateDurationFromStart(event.startTime, crashedAt),
+      properties: event.properties as Attributes,
+      metadata: event.metadata as Attributes,
+      style: event.style as Attributes,
+      output: event.output as Attributes,
+      outputType: event.outputType,
+      payload: event.payload as Attributes,
+      payloadType: event.payloadType,
+    });
+  }
+
   async queryEvents(queryOptions: QueryOptions): Promise<TaskEventRecord[]> {
     return await this.db.taskEvent.findMany({
       where: queryOptions,
@@ -679,6 +729,9 @@ export class EventRepository {
     // Subscribe to the channel.
     await redis.psubscribe(channel);
 
+    // Increment the subscriber count.
+    this._subscriberCount++;
+
     const eventEmitter = new EventEmitter();
 
     // Define the message handler.
@@ -691,6 +744,8 @@ export class EventRepository {
     // Return a function that can be used to unsubscribe.
     const unsubscribe = async () => {
       await redis.punsubscribe(channel);
+      redis.quit();
+      this._subscriberCount--;
     };
 
     return {
@@ -754,19 +809,34 @@ export class EventRepository {
   }
 }
 
-export const eventRepository = new EventRepository(prisma, {
-  batchSize: env.EVENTS_BATCH_SIZE,
-  batchInterval: env.EVENTS_BATCH_INTERVAL,
-  retentionInDays: env.EVENTS_DEFAULT_LOG_RETENTION,
-  redis: {
-    port: env.REDIS_PORT,
-    host: env.REDIS_HOST,
-    username: env.REDIS_USERNAME,
-    password: env.REDIS_PASSWORD,
-    enableAutoPipelining: true,
-    ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
-  },
-});
+export const eventRepository = singleton("eventRepo", initializeEventRepo);
+
+function initializeEventRepo() {
+  const repo = new EventRepository(prisma, {
+    batchSize: env.EVENTS_BATCH_SIZE,
+    batchInterval: env.EVENTS_BATCH_INTERVAL,
+    retentionInDays: env.EVENTS_DEFAULT_LOG_RETENTION,
+    redis: {
+      port: env.REDIS_PORT,
+      host: env.REDIS_HOST,
+      username: env.REDIS_USERNAME,
+      password: env.REDIS_PASSWORD,
+      enableAutoPipelining: true,
+      ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+    },
+  });
+
+  new Gauge({
+    name: "event_repository_subscriber_count",
+    help: "Number of event repository subscribers",
+    collect() {
+      this.set(repo.subscriberCount);
+    },
+    registers: [metricsRegister],
+  });
+
+  return repo;
+}
 
 export function stripAttributePrefix(attributes: Attributes, prefix: string) {
   const result: Attributes = {};

apps/webapp/app/v3/handleSocketIo.server.ts

@@ -21,6 +21,7 @@ import { ResumeAttemptService } from "./services/resumeAttempt.server";
 import { DeploymentIndexFailed } from "./services/deploymentIndexFailed.server";
 import { Redis } from "ioredis";
 import { createAdapter } from "@socket.io/redis-adapter";
+import { CrashTaskRunService } from "./services/crashTaskRun.server";
 
 export const socketIo = singleton("socketIo", initalizeIoServer);
 
@@ -135,7 +136,7 @@ function createCoordinatorNamespace(io: Server) {
 
           await service.call(message.deploymentId, message.error);
         } catch (e) {
-          logger.error("Error while indexing failed", { error: e });
+          logger.error("Error while indexing", { error: e });
         }
       },
     },
@@ -151,6 +152,28 @@ function createProviderNamespace(io: Server) {
     authToken: env.PROVIDER_SECRET,
     clientMessages: ProviderToPlatformMessages,
     serverMessages: PlatformToProviderMessages,
+    handlers: {
+      WORKER_CRASHED: async (message) => {
+        try {
+          const service = new CrashTaskRunService();
+
+          await service.call(message.runId, {
+            ...message,
+          });
+        } catch (error) {
+          logger.error("Error while handling crashed worker", { error });
+        }
+      },
+      INDEXING_FAILED: async (message) => {
+        try {
+          const service = new DeploymentIndexFailed();
+
+          await service.call(message.deploymentId, message.error);
+        } catch (e) {
+          logger.error("Error while indexing", { error: e });
+        }
+      },
+    },
   });
 
   return provider.namespace;

apps/webapp/app/v3/handleWebsockets.server.ts

@@ -4,6 +4,8 @@ import { authenticateApiKey } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { singleton } from "../utils/singleton";
 import { AuthenticatedSocketConnection } from "./authenticatedSocketConnection.server";
+import { Gauge } from "prom-client";
+import { metricsRegister } from "~/metrics.server";
 
 export const wss = singleton("wss", initalizeWebSocketServer);
 
@@ -16,6 +18,15 @@ function initalizeWebSocketServer() {
 
   authenticatedConnections = new Map();
 
+  new Gauge({
+    name: "dev_authenticated_connections",
+    help: "Number of authenticated dev connections",
+    collect() {
+      this.set(authenticatedConnections.size);
+    },
+    registers: [metricsRegister],
+  });
+
   return server;
 }
 
@@ -47,8 +58,11 @@ async function handleWebSocketConnection(ws: WebSocket, req: IncomingMessage) {
 
   authenticatedConnections.set(authenticatedConnection.id, authenticatedConnection);
 
-  authenticatedConnection.onClose.attach((closeEvent) => {
-    logger.debug("Websocket closed", { closeEvent });
+  authenticatedConnection.onClose.attachOnce((closeEvent) => {
+    logger.debug("Websocket closed", {
+      closeEvent,
+      authenticatedConnectionId: authenticatedConnection.id,
+    });
 
     authenticatedConnections.delete(authenticatedConnection.id);
   });

apps/webapp/app/v3/marqs/devPubSub.server.ts

@@ -2,6 +2,8 @@ import { z } from "zod";
 import { singleton } from "~/utils/singleton";
 import { ZodPubSub, ZodSubscriber } from "../utils/zodPubSub.server";
 import { env } from "~/env.server";
+import { Gauge } from "prom-client";
+import { metricsRegister } from "~/metrics.server";
 
 const messageCatalog = {
   CANCEL_ATTEMPT: z.object({
@@ -17,7 +19,7 @@ export type DevSubscriber = ZodSubscriber<typeof messageCatalog>;
 export const devPubSub = singleton("devPubSub", initializeDevPubSub);
 
 function initializeDevPubSub() {
-  return new ZodPubSub({
+  const pubSub = new ZodPubSub({
     redis: {
       port: env.REDIS_PORT,
       host: env.REDIS_HOST,
@@ -28,4 +30,15 @@ function initializeDevPubSub() {
     },
     schema: messageCatalog,
   });
+
+  new Gauge({
+    name: "dev_pub_sub_subscribers",
+    help: "Number of dev pub sub subscribers",
+    collect() {
+      this.set(pubSub.subscriberCount);
+    },
+    registers: [metricsRegister],
+  });
+
+  return pubSub;
 }

apps/webapp/app/v3/marqs/devQueueConsumer.server.ts

@@ -16,12 +16,14 @@ import { generateFriendlyId } from "../friendlyIdentifiers";
 import { marqs } from "~/v3/marqs/index.server";
 import { CancelAttemptService } from "../services/cancelAttempt.server";
 import { CompleteAttemptService } from "../services/completeAttempt.server";
-import { attributesFromAuthenticatedEnv } from "../tracer.server";
+import {
+  SEMINTATTRS_FORCE_RECORDING,
+  attributesFromAuthenticatedEnv,
+  tracer,
+} from "../tracer.server";
 import { DevSubscriber, devPubSub } from "./devPubSub.server";
 import { CancelTaskRunService } from "../services/cancelTaskRun.server";
 
-const tracer = trace.getTracer("devQueueConsumer");
-
 const MessageBody = z.discriminatedUnion("type", [
   z.object({
     type: z.literal("EXECUTE"),
@@ -165,6 +167,11 @@ export class DevQueueConsumer {
 
       logger.debug("Unsubscribed from background worker channel", { id });
     }
+
+    // We need to end the current span
+    if (this._currentSpan) {
+      this._currentSpan.end();
+    }
   }
 
   async #cancelInProgressRunsAndAttempts(reason: string) {
@@ -284,6 +291,10 @@ export class DevQueueConsumer {
         this._currentSpan.setAttribute("tasks.period.failures", this._taskFailures);
         this._currentSpan.setAttribute("tasks.period.successes", this._taskSuccesses);
 
+        logger.debug("Ending DevQueueConsumer.doWork() trace", {
+          isRecording: this._currentSpan.isRecording(),
+        });
+
         this._currentSpan.end();
       }
 
@@ -294,6 +305,7 @@ export class DevQueueConsumer {
           kind: SpanKind.CONSUMER,
           attributes: {
             ...attributesFromAuthenticatedEnv(this.env),
+            [SEMINTATTRS_FORCE_RECORDING]: true,
           },
         },
         ROOT_CONTEXT

apps/webapp/app/v3/marqs/index.server.ts

@@ -92,15 +92,14 @@ export class MarQS {
     queue: string,
     messageId: string,
     messageData: Record<string, unknown>,
-    concurrencyKey?: string
+    concurrencyKey?: string,
+    timestamp?: number
   ) {
     return await this.#trace(
       "enqueueMessage",
       async (span) => {
         const messageQueue = this.keys.queueKey(env, queue, concurrencyKey);
 
-        const timestamp = Date.now();
-
         const parentQueue = this.keys.envSharedQueueKey(env);
 
         propagation.inject(context.active(), messageData);
@@ -110,7 +109,7 @@ export class MarQS {
           data: messageData,
           queue: messageQueue,
           concurrencyKey,
-          timestamp,
+          timestamp: timestamp ?? Date.now(),
           messageId,
           parentQueue,
         };
@@ -139,7 +138,7 @@ export class MarQS {
   public async dequeueMessageInEnv(env: AuthenticatedEnvironment) {
     return this.#trace(
       "dequeueMessageInEnv",
-      async (span, abort) => {
+      async (span) => {
         const parentQueue = this.keys.envSharedQueueKey(env);
 
         // Read the parent queue for matching queues
@@ -150,7 +149,6 @@ export class MarQS {
         );
 
         if (!messageQueue) {
-          abort();
           return;
         }
 
@@ -167,7 +165,6 @@ export class MarQS {
         });
 
         if (!messageData) {
-          abort();
           return;
         }
 
@@ -181,8 +178,6 @@ export class MarQS {
             [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey,
             [SemanticAttributes.PARENT_QUEUE]: message.parentQueue,
           });
-        } else {
-          abort();
         }
 
         return message;
@@ -204,7 +199,7 @@ export class MarQS {
   public async dequeueMessageInSharedQueue() {
     return this.#trace(
       "dequeueMessageInSharedQueue",
-      async (span, abort) => {
+      async (span) => {
         const parentQueue = constants.SHARED_QUEUE;
 
         // Read the parent queue for matching queues
@@ -215,7 +210,6 @@ export class MarQS {
         );
 
         if (!messageQueue) {
-          abort();
           return;
         }
 
@@ -233,7 +227,6 @@ export class MarQS {
         });
 
         if (!messageData) {
-          abort();
           return;
         }
 
@@ -247,8 +240,6 @@ export class MarQS {
             [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey,
             [SemanticAttributes.PARENT_QUEUE]: message.parentQueue,
           });
-        } else {
-          abort();
         }
 
         return message;
@@ -355,17 +346,12 @@ export class MarQS {
 
   async #trace<T>(
     name: string,
-    fn: (span: Span, abort: () => void) => Promise<T>,
-    options?: SpanOptions
+    fn: (span: Span) => Promise<T>,
+    options?: SpanOptions & { sampleRate?: number }
   ): Promise<T> {
     return tracer.startActiveSpan(name, options ?? {}, async (span) => {
-      let _abort = false;
-      let aborter = () => {
-        _abort = true;
-      };
-
       try {
-        return await fn(span, aborter);
+        return await fn(span);
       } catch (e) {
         if (e instanceof Error) {
           span.recordException(e);
@@ -375,9 +361,7 @@ export class MarQS {
 
         throw e;
       } finally {
-        if (!_abort) {
-          span.end();
-        }
+        span.end();
       }
     });
   }
@@ -480,7 +464,7 @@ export class MarQS {
   ) {
     return this.#trace(
       "getRandomQueueFromParentQueue",
-      async (span, abort) => {
+      async (span) => {
         const { range, selectionId } = await queuePriorityStrategy.nextCandidateSelection(
           parentQueue
         );
@@ -497,7 +481,6 @@ export class MarQS {
         );
 
         if (typeof choice !== "string") {
-          abort();
           return;
         }
 

apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts

@@ -1,12 +1,4 @@
-import {
-  Context,
-  ROOT_CONTEXT,
-  Span,
-  SpanKind,
-  context,
-  propagation,
-  trace,
-} from "@opentelemetry/api";
+import { Context, ROOT_CONTEXT, Span, SpanKind, context, trace } from "@opentelemetry/api";
 import {
   Machine,
   ProdTaskRunExecution,
@@ -28,16 +20,15 @@ import {
 import { z } from "zod";
 import { prisma } from "~/db.server";
 import { logger } from "~/services/logger.server";
-import { generateFriendlyId } from "../friendlyIdentifiers";
+import { singleton } from "~/utils/singleton";
 import { marqs } from "~/v3/marqs/index.server";
 import { EnvironmentVariablesRepository } from "../environmentVariables/environmentVariablesRepository.server";
-import { CancelAttemptService } from "../services/cancelAttempt.server";
+import { generateFriendlyId } from "../friendlyIdentifiers";
 import { socketIo } from "../handleSocketIo.server";
-import { singleton } from "~/utils/singleton";
-import { RestoreCheckpointService } from "../services/restoreCheckpoint.server";
 import { findCurrentWorkerDeployment } from "../models/workerDeployment.server";
-
-const tracer = trace.getTracer("sharedQueueConsumer");
+import { CancelAttemptService } from "../services/cancelAttempt.server";
+import { RestoreCheckpointService } from "../services/restoreCheckpoint.server";
+import { tracer } from "../tracer.server";
 
 const WithTraceContext = z.object({
   traceparent: z.string().optional(),
@@ -86,7 +77,6 @@ export class SharedQueueConsumer {
   private _currentSpan: Span | undefined;
   private _endSpanInNextIteration = false;
   private _tasks = sharedQueueTasks;
-  private _inProgressAttempts: Map<string, string> = new Map(); // Keys are task attempt friendly IDs, values are TaskRun ids/queue message ids
 
   constructor(
     private _sender: ZodMessageSender<typeof serverWebsocketMessages>,
@@ -154,37 +144,9 @@ export class SharedQueueConsumer {
 
     logger.debug("Stopping shared queue consumer");
     this._enabled = false;
-  }
 
-  async #cancelInProgressAttempts(reason: string) {
-    const service = new CancelAttemptService();
-
-    const cancelledAt = new Date();
-
-    const inProgressAttempts = new Map(this._inProgressAttempts);
-
-    this._inProgressAttempts.clear();
-
-    for (const [attemptId, messageId] of inProgressAttempts) {
-      await this.#cancelInProgressAttempt(attemptId, messageId, service, cancelledAt, reason);
-    }
-  }
-
-  async #cancelInProgressAttempt(
-    attemptId: string,
-    messageId: string,
-    cancelAttemptService: CancelAttemptService,
-    cancelledAt: Date,
-    reason: string
-  ) {
-    try {
-      await cancelAttemptService.call(attemptId, messageId, cancelledAt, reason);
-    } catch (e) {
-      logger.error("Failed to cancel in progress attempt", {
-        attemptId,
-        messageId,
-        error: e,
-      });
+    if (this._currentSpan) {
+      this._currentSpan.end();
     }
   }
 
@@ -280,10 +242,12 @@ export class SharedQueueConsumer {
         error: messageBody.error,
       });
 
-      this.#ackAndDoMoreWork(message.messageId);
+      await this.#ackAndDoMoreWork(message.messageId);
       return;
     }
 
+    // TODO: For every ACK, decide what should be done with the existing run and attempts. Make sure to check the current statuses first.
+
     switch (messageBody.data.type) {
       case "EXECUTE": {
         const existingTaskRun = await prisma.taskRun.findUnique({
@@ -298,7 +262,10 @@ export class SharedQueueConsumer {
             messageId: message.messageId,
           });
 
-          this.#ackAndDoMoreWork(message.messageId);
+          // INFO: There used to be a race condition where tasks could be triggered, but execute messages could be dequeued before the run finished being created in the DB
+          //       This should not be happening anymore. In case it does, consider reqeueuing here with a brief delay while limiting total retries.
+
+          await this.#ackAndDoMoreWork(message.messageId);
           return;
         }
 
@@ -326,7 +293,7 @@ export class SharedQueueConsumer {
             retryingFromCheckpoint,
           });
 
-          this.#ackAndDoMoreWork(message.messageId);
+          await this.#ackAndDoMoreWork(message.messageId);
           return;
         }
 
@@ -338,7 +305,9 @@ export class SharedQueueConsumer {
             messageId: message.messageId,
           });
 
-          this.#ackAndDoMoreWork(message.messageId);
+          await this.#markRunAsWaitingForDeploy(existingTaskRun.id);
+
+          await this.#ackAndDoMoreWork(message.messageId);
           return;
         }
 
@@ -349,7 +318,9 @@ export class SharedQueueConsumer {
             deployment: deployment.id,
           });
 
-          this.#ackAndDoMoreWork(message.messageId);
+          await this.#markRunAsWaitingForDeploy(existingTaskRun.id);
+
+          await this.#ackAndDoMoreWork(message.messageId);
           return;
         }
 
@@ -358,15 +329,39 @@ export class SharedQueueConsumer {
         );
 
         if (!backgroundTask) {
-          logger.warn("No matching background task found for task run", {
-            taskRun: existingTaskRun.id,
-            taskIdentifier: existingTaskRun.taskIdentifier,
-            deployment: deployment.id,
-            backgroundWorker: deployment.worker.id,
-            taskSlugs: deployment.worker.tasks.map((task) => task.slug),
+          const nonCurrentTask = await prisma.backgroundWorkerTask.findFirst({
+            where: {
+              slug: existingTaskRun.taskIdentifier,
+              projectId: existingTaskRun.projectId,
+              runtimeEnvironmentId: existingTaskRun.runtimeEnvironmentId,
+            },
+            include: {
+              worker: {
+                include: {
+                  deployment: {
+                    include: {},
+                  },
+                },
+              },
+            },
           });
 
-          this.#ackAndDoMoreWork(message.messageId);
+          if (nonCurrentTask) {
+            logger.warn("Task for this run exists but is not part of the current deploy", {
+              taskRun: existingTaskRun.id,
+              taskIdentifier: existingTaskRun.taskIdentifier,
+            });
+          } else {
+            logger.warn("Task for this run has never been deployed", {
+              taskRun: existingTaskRun.id,
+              taskIdentifier: existingTaskRun.taskIdentifier,
+            });
+          }
+
+          await this.#markRunAsWaitingForDeploy(existingTaskRun.id);
+
+          // If this task is ever deployed, a new message will be enqueued after successful indexing
+          await this.#ackAndDoMoreWork(message.messageId);
           return;
         }
 
@@ -403,7 +398,7 @@ export class SharedQueueConsumer {
             messageId: message.messageId,
           });
 
-          this.#ackAndDoMoreWork(message.messageId);
+          await this.#ackAndDoMoreWork(message.messageId);
           return;
         }
 
@@ -500,8 +495,6 @@ export class SharedQueueConsumer {
               },
             });
           }
-
-          this._inProgressAttempts.set(taskRunAttempt.friendlyId, message.messageId);
         } catch (e) {
           if (e instanceof Error) {
             this._currentSpan?.recordException(e);
@@ -762,9 +755,23 @@ export class SharedQueueConsumer {
     this.#doMoreWork(intervalInMs);
   }
 
-  async #nackAndDoMoreWork(messageId: string, intervalInMs?: number) {
-    await marqs?.nackMessage(messageId);
-    this.#doMoreWork(intervalInMs);
+  async #nackAndDoMoreWork(messageId: string, queueIntervalInMs?: number, nackRetryInMs?: number) {
+    const retryAt = nackRetryInMs ? Date.now() + nackRetryInMs : undefined;
+    await marqs?.nackMessage(messageId, retryAt);
+    this.#doMoreWork(queueIntervalInMs);
+  }
+
+  async #markRunAsWaitingForDeploy(runId: string) {
+    logger.debug("Marking run as waiting for deploy", { runId });
+
+    return await prisma.taskRun.update({
+      where: {
+        id: runId,
+      },
+      data: {
+        status: "WAITING_FOR_DEPLOY",
+      },
+    });
   }
 }
 

apps/webapp/app/v3/services/cancelTaskRun.server.ts

@@ -1,4 +1,4 @@
-import { TaskRun, TaskRunAttemptStatus, TaskRunStatus } from "@trigger.dev/database";
+import { Prisma, TaskRun, TaskRunAttemptStatus, TaskRunStatus } from "@trigger.dev/database";
 import { eventRepository } from "../eventRepository.server";
 import { marqs } from "~/v3/marqs/index.server";
 import { devPubSub } from "../marqs/devPubSub.server";
@@ -10,6 +10,7 @@ import { logger } from "~/services/logger.server";
 
 export const CANCELLABLE_STATUSES: Array<TaskRunStatus> = [
   "PENDING",
+  "WAITING_FOR_DEPLOY",
   "EXECUTING",
   "PAUSED",
   "WAITING_TO_RESUME",
@@ -23,6 +24,13 @@ const CANCELLABLE_ATTEMPT_STATUSES: Array<TaskRunAttemptStatus> = [
   "PENDING",
 ];
 
+type ExtendedTaskRunAttempt = Prisma.TaskRunAttemptGetPayload<{
+  include: {
+    runtimeEnvironment: true;
+    backgroundWorker: true;
+  };
+}>;
+
 export type CancelTaskRunServiceOptions = {
   reason?: string;
   cancelAttempts?: boolean;
@@ -87,56 +95,60 @@ export class CancelTaskRunService extends BaseService {
 
     // Cancel any in progress attempts
     if (opts.cancelAttempts) {
-      for (const attempt of cancelledTaskRun.attempts) {
-        if (attempt.runtimeEnvironment.type === "DEVELOPMENT") {
-          // Signal the task run attempt to stop
-          await devPubSub.publish(
-            `backgroundWorker:${attempt.backgroundWorkerId}:${attempt.id}`,
-            "CANCEL_ATTEMPT",
-            {
-              attemptId: attempt.friendlyId,
-              backgroundWorkerId: attempt.backgroundWorker.friendlyId,
-              taskRunId: cancelledTaskRun.friendlyId,
-            }
-          );
-        } else {
-          switch (attempt.status) {
-            case "EXECUTING": {
-              // We need to send a cancel message to the coordinator
-              socketIo.coordinatorNamespace.emit("REQUEST_ATTEMPT_CANCELLATION", {
-                version: "v1",
-                attemptId: attempt.id,
-                attemptFriendlyId: attempt.friendlyId,
-              });
+      await this.#cancelPotentiallyRunningAttempts(cancelledTaskRun, cancelledTaskRun.attempts);
+    }
+  }
 
-              break;
-            }
-            case "PENDING":
-            case "PAUSED": {
-              logger.debug("Cancelling pending or paused attempt", {
-                attempt,
-              });
+  async #cancelPotentiallyRunningAttempts(run: TaskRun, attempts: ExtendedTaskRunAttempt[]) {
+    for (const attempt of attempts) {
+      if (attempt.runtimeEnvironment.type === "DEVELOPMENT") {
+        // Signal the task run attempt to stop
+        await devPubSub.publish(
+          `backgroundWorker:${attempt.backgroundWorkerId}:${attempt.id}`,
+          "CANCEL_ATTEMPT",
+          {
+            attemptId: attempt.friendlyId,
+            backgroundWorkerId: attempt.backgroundWorker.friendlyId,
+            taskRunId: run.friendlyId,
+          }
+        );
+      } else {
+        switch (attempt.status) {
+          case "EXECUTING": {
+            // We need to send a cancel message to the coordinator
+            socketIo.coordinatorNamespace.emit("REQUEST_ATTEMPT_CANCELLATION", {
+              version: "v1",
+              attemptId: attempt.id,
+              attemptFriendlyId: attempt.friendlyId,
+            });
 
-              const service = new CancelAttemptService();
+            break;
+          }
+          case "PENDING":
+          case "PAUSED": {
+            logger.debug("Cancelling pending or paused attempt", {
+              attempt,
+            });
 
-              await service.call(
-                attempt.friendlyId,
-                taskRun.id,
-                new Date(),
-                "Task run was cancelled by user"
-              );
+            const service = new CancelAttemptService();
 
-              break;
-            }
-            case "CANCELED":
-            case "COMPLETED":
-            case "FAILED": {
-              // Do nothing
-              break;
-            }
-            default: {
-              assertUnreachable(attempt.status);
-            }
+            await service.call(
+              attempt.friendlyId,
+              run.id,
+              new Date(),
+              "Task run was cancelled by user"
+            );
+
+            break;
+          }
+          case "CANCELED":
+          case "COMPLETED":
+          case "FAILED": {
+            // Do nothing
+            break;
+          }
+          default: {
+            assertUnreachable(attempt.status);
           }
         }
       }

apps/webapp/app/v3/services/crashTaskRun.server.ts

@@ -0,0 +1,158 @@
+import {
+  TaskRun,
+  TaskRunAttempt,
+  TaskRunAttemptStatus,
+  TaskRunStatus,
+} from "@trigger.dev/database";
+import { eventRepository } from "../eventRepository.server";
+import { marqs } from "~/v3/marqs/index.server";
+import { BaseService } from "./baseService.server";
+import { logger } from "~/services/logger.server";
+import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { ResumeTaskRunDependenciesService } from "./resumeTaskRunDependencies.server";
+
+export const CRASHABLE_RUN_STATUSES: Array<TaskRunStatus> = [
+  "PENDING",
+  "WAITING_FOR_DEPLOY",
+  "EXECUTING",
+  "PAUSED",
+  "WAITING_TO_RESUME",
+  "PAUSED",
+  "RETRYING_AFTER_FAILURE",
+];
+
+const CRASHABLE_ATTEMPT_STATUSES: Array<TaskRunAttemptStatus> = ["EXECUTING", "PAUSED", "PENDING"];
+
+export type CrashTaskRunServiceOptions = {
+  reason?: string;
+  exitCode?: number;
+  logs?: string;
+  crashAttempts?: boolean;
+  crashedAt?: Date;
+};
+
+export class CrashTaskRunService extends BaseService {
+  public async call(runId: string, options?: CrashTaskRunServiceOptions) {
+    const opts = {
+      reason: "Worker crashed",
+      crashAttempts: true,
+      crashedAt: new Date(),
+      ...options,
+    };
+
+    const taskRun = await this._prisma.taskRun.findFirst({
+      where: {
+        id: runId,
+      },
+    });
+
+    if (!taskRun) {
+      logger.error("Task run not found", { runId });
+      return;
+    }
+
+    // Make sure the task run is in a crashable state
+    if (!CRASHABLE_RUN_STATUSES.includes(taskRun.status)) {
+      return;
+    }
+
+    // Remove the task run from the queue if it's there for some reason
+    await marqs?.acknowledgeMessage(taskRun.id);
+
+    // Set the task run status to crashed
+    const crashedTaskRun = await this._prisma.taskRun.update({
+      where: {
+        id: taskRun.id,
+      },
+      data: {
+        status: "CRASHED",
+      },
+      include: {
+        attempts: {
+          where: {
+            status: {
+              in: CRASHABLE_ATTEMPT_STATUSES,
+            },
+          },
+          include: {
+            backgroundWorker: true,
+            runtimeEnvironment: true,
+          },
+        },
+        dependency: true,
+        runtimeEnvironment: {
+          include: {
+            organization: true,
+            project: true,
+          },
+        },
+      },
+    });
+
+    const inProgressEvents = await eventRepository.queryIncompleteEvents({
+      runId: taskRun.friendlyId,
+    });
+
+    logger.debug("Crashing in-progress events", {
+      inProgressEvents: inProgressEvents.map((event) => event.id),
+    });
+
+    await Promise.all(
+      inProgressEvents.map((event) => {
+        return eventRepository.crashEvent({
+          event: event,
+          crashedAt: opts.crashedAt,
+          exception: {
+            type: "Worker crashed",
+            message: opts.reason,
+            stacktrace: opts.logs,
+          },
+        });
+      })
+    );
+
+    if (!opts.crashAttempts) {
+      return;
+    }
+
+    // Cancel any in progress attempts
+    for (const attempt of crashedTaskRun.attempts) {
+      await this.#failAttempt(
+        attempt,
+        crashedTaskRun,
+        new Date(),
+        crashedTaskRun.runtimeEnvironment
+      );
+    }
+  }
+
+  async #failAttempt(
+    attempt: TaskRunAttempt,
+    run: TaskRun,
+    failedAt: Date,
+    environment: AuthenticatedEnvironment
+  ) {
+    return await this.traceWithEnv("failAttempt()", environment, async (span) => {
+      span.setAttribute("taskRunId", run.id);
+      span.setAttribute("attemptId", attempt.id);
+
+      await marqs?.acknowledgeMessage(run.id);
+
+      await this._prisma.taskRunAttempt.update({
+        where: {
+          id: attempt.id,
+        },
+        data: {
+          status: "FAILED",
+          completedAt: failedAt,
+        },
+      });
+
+      if (environment.type === "DEVELOPMENT") {
+        return;
+      }
+
+      await ResumeTaskRunDependenciesService.enqueue(attempt.id, this._prisma);
+    });
+  }
+}

apps/webapp/app/v3/services/createDeployedBackgroundWorker.server.ts

@@ -8,6 +8,7 @@ import { CURRENT_DEPLOYMENT_LABEL } from "~/consts";
 import { projectPubSub } from "./projectPubSub.server";
 import { marqs } from "~/v3/marqs/index.server";
 import { logger } from "~/services/logger.server";
+import { ExecuteTasksWaitingForDeployService } from "./executeTasksWaitingForDeploy";
 
 export class CreateDeployedBackgroundWorkerService extends BaseService {
   public async call(
@@ -96,6 +97,8 @@ export class CreateDeployedBackgroundWorkerService extends BaseService {
         logger.error("Failed to publish WORKER_CREATED event", { err });
       }
 
+      await ExecuteTasksWaitingForDeployService.enqueue(backgroundWorker.id, this._prisma);
+
       return backgroundWorker;
     });
   }

apps/webapp/app/v3/services/deploymentIndexFailed.server.ts

@@ -2,13 +2,19 @@ import { BaseService } from "./baseService.server";
 
 export class DeploymentIndexFailed extends BaseService {
   public async call(
-    deploymentId: string,
+    maybeFriendlyId: string,
     error: { name: string; message: string; stack?: string }
   ) {
+    const isFriendlyId = maybeFriendlyId.startsWith("deployment_");
+
     const deployment = await this._prisma.workerDeployment.update({
-      where: {
-        friendlyId: deploymentId,
-      },
+      where: isFriendlyId
+        ? {
+            friendlyId: maybeFriendlyId,
+          }
+        : {
+            id: maybeFriendlyId,
+          },
       data: {
         status: "FAILED",
         failedAt: new Date(),

apps/webapp/app/v3/services/executeTasksWaitingForDeploy.ts

@@ -0,0 +1,122 @@
+import { PrismaClientOrTransaction } from "~/db.server";
+import { workerQueue } from "~/services/worker.server";
+import { marqs } from "~/v3/marqs/index.server";
+import { BaseService } from "./baseService.server";
+import { logger } from "~/services/logger.server";
+
+export class ExecuteTasksWaitingForDeployService extends BaseService {
+  public async call(backgroundWorkerId: string) {
+    const backgroundWorker = await this._prisma.backgroundWorker.findFirst({
+      where: {
+        id: backgroundWorkerId,
+      },
+      include: {
+        runtimeEnvironment: {
+          include: {
+            project: true,
+            organization: true,
+          },
+        },
+        tasks: true,
+      },
+    });
+
+    if (!backgroundWorker) {
+      logger.error("Background worker not found", { id: backgroundWorkerId });
+      return;
+    }
+
+    const runsWaitingForDeploy = await this._prisma.taskRun.findMany({
+      where: {
+        runtimeEnvironmentId: backgroundWorker.runtimeEnvironmentId,
+        projectId: backgroundWorker.projectId,
+        status: "WAITING_FOR_DEPLOY",
+        taskIdentifier: {
+          in: backgroundWorker.tasks.map((task) => task.slug),
+        },
+      },
+      orderBy: {
+        number: "asc",
+      },
+    });
+
+    if (!runsWaitingForDeploy.length) {
+      return;
+    }
+
+    // Clear any runs awaiting deployment for execution
+    const pendingRuns = await this._prisma.taskRun.updateMany({
+      where: {
+        id: {
+          in: runsWaitingForDeploy.map((run) => run.id),
+        },
+      },
+      data: {
+        status: "PENDING",
+      },
+    });
+
+    if (pendingRuns.count) {
+      logger.debug("Task runs waiting for deploy are now ready for execution", {
+        tasks: runsWaitingForDeploy.map((run) => run.id),
+        total: pendingRuns.count,
+      });
+    }
+
+    if (!marqs) {
+      return;
+    }
+
+    const enqueues: Promise<any>[] = [];
+    let i = 0;
+
+    for (const run of runsWaitingForDeploy) {
+      enqueues.push(
+        marqs.enqueueMessage(
+          backgroundWorker.runtimeEnvironment,
+          run.queue,
+          run.id,
+          {
+            type: "EXECUTE",
+            taskIdentifier: run.taskIdentifier,
+          },
+          run.concurrencyKey ?? undefined,
+          Date.now() + i * 5 // slight delay to help preserve order
+        )
+      );
+
+      i++;
+    }
+
+    const settled = await Promise.allSettled(enqueues);
+
+    if (settled.some((s) => s.status === "rejected")) {
+      const rejectedRuns: { id: string; reason: any }[] = [];
+
+      runsWaitingForDeploy.forEach((run, i) => {
+        if (settled[i].status === "rejected") {
+          const rejected = settled[i] as PromiseRejectedResult;
+
+          rejectedRuns.push({ id: run.id, reason: rejected.reason });
+        }
+      });
+
+      logger.error("Failed to requeue task runs for immediate execution", {
+        rejectedRuns,
+      });
+    }
+  }
+
+  static async enqueue(backgroundWorkerId: string, tx: PrismaClientOrTransaction, runAt?: Date) {
+    return await workerQueue.enqueue(
+      "v3.executeTasksWaitingForDeploy",
+      {
+        backgroundWorkerId,
+      },
+      {
+        tx,
+        runAt,
+      }
+    );
+  }
+}

apps/webapp/app/v3/services/indexDeployment.server.ts

@@ -56,6 +56,7 @@ export class IndexDeploymentService extends BaseService {
         envType: deployment.environment.type,
         projectId: deployment.projectId,
         orgId: deployment.environment.organizationId,
+        deploymentId: deployment.id,
       });
 
       logger.debug("Index ACK received", { responses });

apps/webapp/app/v3/services/projectPubSub.server.ts

@@ -2,6 +2,8 @@ import { z } from "zod";
 import { singleton } from "~/utils/singleton";
 import { ZodPubSub, ZodSubscriber } from "../utils/zodPubSub.server";
 import { env } from "~/env.server";
+import { Gauge } from "prom-client";
+import { metricsRegister } from "~/metrics.server";
 
 const messageCatalog = {
   WORKER_CREATED: z.object({
@@ -18,7 +20,7 @@ export type ProjectSubscriber = ZodSubscriber<typeof messageCatalog>;
 export const projectPubSub = singleton("projectPubSub", initializeProjectPubSub);
 
 function initializeProjectPubSub() {
-  return new ZodPubSub({
+  const pubSub = new ZodPubSub({
     redis: {
       port: env.REDIS_PORT,
       host: env.REDIS_HOST,
@@ -29,4 +31,15 @@ function initializeProjectPubSub() {
     },
     schema: messageCatalog,
   });
+
+  new Gauge({
+    name: "project_pub_sub_subscribers",
+    help: "Number of project pub sub subscribers",
+    collect() {
+      this.set(pubSub.subscriberCount);
+    },
+    registers: [metricsRegister],
+  });
+
+  return pubSub;
 }

apps/webapp/app/v3/services/replayTaskRun.server.ts

@@ -0,0 +1,53 @@
+import { conditionallyImportPacket, parsePacket } from "@trigger.dev/core/v3";
+import { Prisma, TaskRun } from "@trigger.dev/database";
+import { findEnvironmentById } from "~/models/runtimeEnvironment.server";
+import { logger } from "~/services/logger.server";
+import { BaseService } from "./baseService.server";
+import { TriggerTaskService } from "./triggerTask.server";
+
+type ExtendedTaskRunAttempt = Prisma.TaskRunAttemptGetPayload<{
+  include: {
+    runtimeEnvironment: true;
+    backgroundWorker: true;
+  };
+}>;
+
+export class ReplayTaskRunService extends BaseService {
+  public async call(existingTaskRun: TaskRun) {
+    const authenticatedEnvironment = await findEnvironmentById(
+      existingTaskRun.runtimeEnvironmentId
+    );
+    if (!authenticatedEnvironment) {
+      return;
+    }
+
+    logger.info("Replaying task run", {
+      taskRunId: existingTaskRun.id,
+      taskRunFriendlyId: existingTaskRun.friendlyId,
+    });
+
+    const payloadPacket = await conditionallyImportPacket({
+      data: existingTaskRun.payload,
+      dataType: existingTaskRun.payloadType,
+    });
+    const parsedPayload = await parsePacket(payloadPacket);
+
+    logger.info("Replaying task run payload", {
+      taskRunId: existingTaskRun.id,
+      taskRunFriendlyId: existingTaskRun.friendlyId,
+      payloadPacketType: payloadPacket.dataType,
+    });
+
+    const triggerTaskService = new TriggerTaskService();
+    return await triggerTaskService.call(existingTaskRun.taskIdentifier, authenticatedEnvironment, {
+      payload: parsedPayload,
+      options: {
+        queue: {
+          name: existingTaskRun.queue,
+        },
+        concurrencyKey: existingTaskRun.concurrencyKey ?? undefined,
+        test: existingTaskRun.isTest,
+      },
+    });
+  }
+}

apps/webapp/app/v3/services/triggerTask.server.ts

@@ -71,7 +71,7 @@ export class TriggerTaskService extends BaseService {
         async (event, traceContext) => {
           const lockId = taskIdentifierToLockId(taskId);
 
-          return await $transaction(this._prisma, async (tx) => {
+          const run = await $transaction(this._prisma, async (tx) => {
             await tx.$executeRaw`SELECT pg_advisory_xact_lock(${lockId})`;
 
             const lockedToBackgroundWorker = body.options?.lockToVersion
@@ -169,17 +169,23 @@ export class TriggerTaskService extends BaseService {
               }
             }
 
-            // We need to enqueue the task run into the appropriate queue
-            await marqs?.enqueueMessage(
-              environment,
-              queueName,
-              taskRun.id,
-              { type: "EXECUTE", taskIdentifier: taskId },
-              body.options?.concurrencyKey
-            );
-
             return taskRun;
           });
+
+          if (!run) {
+            return;
+          }
+
+          // We need to enqueue the task run into the appropriate queue. This is done after the tx completes to prevent a race condition where the task run hasn't been created yet by the time we dequeue.
+          await marqs?.enqueueMessage(
+            environment,
+            run.queue,
+            run.id,
+            { type: "EXECUTE", taskIdentifier: taskId },
+            body.options?.concurrencyKey
+          );
+
+          return run;
         }
       );
     });

apps/webapp/app/v3/tracer.server.ts

@@ -30,6 +30,8 @@ import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { singleton } from "~/utils/singleton";
 import { LoggerSpanExporter } from "./telemetry/loggerExporter.server";
 
+export const SEMINTATTRS_FORCE_RECORDING = "forceRecording";
+
 class CustomWebappSampler implements Sampler {
   constructor(private readonly _baseSampler: Sampler) {}
 
@@ -49,8 +51,22 @@ class CustomWebappSampler implements Sampler {
       return { decision: SamplingDecision.NOT_RECORD };
     }
 
+    // If the span has the forceRecording attribute, always record it
+    if (attributes[SEMINTATTRS_FORCE_RECORDING]) {
+      return { decision: SamplingDecision.RECORD_AND_SAMPLED };
+    }
+
     // For all other spans, defer to the base sampler
-    return this._baseSampler.shouldSample(context, traceId, name, spanKind, attributes, links);
+    const result = this._baseSampler.shouldSample(
+      context,
+      traceId,
+      name,
+      spanKind,
+      attributes,
+      links
+    );
+
+    return result;
   }
 
   toString(): string {
@@ -61,9 +77,15 @@ class CustomWebappSampler implements Sampler {
 export const tracer = singleton("tracer", getTracer);
 
 function getTracer() {
+  if (env.INTERNAL_OTEL_TRACE_DISABLED === "1") {
+    console.log(`🔦 Tracer disabled, returning a noop tracer`);
+
+    return trace.getTracer("trigger.dev", "3.0.0.dp.1");
+  }
+
   diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.ERROR);
 
-  const samplingRate = 1.0 / Math.max(parseInt(env.INTERNAL_OTEL_TRACE_SAMPING_RATE, 10), 1);
+  const samplingRate = 1.0 / Math.max(parseInt(env.INTERNAL_OTEL_TRACE_SAMPLING_RATE, 10), 1);
 
   const provider = new NodeTracerProvider({
     forceFlushTimeoutMillis: 500,
@@ -71,8 +93,8 @@ function getTracer() {
       [SEMRESATTRS_SERVICE_NAME]: env.SERVICE_NAME,
     }),
     sampler: new ParentBasedSampler({
-      root: new CustomWebappSampler(new TraceIdRatioBasedSampler(samplingRate)), // 5% sampling
-    }), // 5% sampling
+      root: new CustomWebappSampler(new TraceIdRatioBasedSampler(samplingRate)),
+    }),
   });
 
   if (env.INTERNAL_OTEL_TRACE_EXPORTER_URL) {
@@ -92,13 +114,15 @@ function getTracer() {
     provider.addSpanProcessor(
       new BatchSpanProcessor(exporter, {
         maxExportBatchSize: 512,
-        scheduledDelayMillis: 200,
+        scheduledDelayMillis: 1000,
         exportTimeoutMillis: 30000,
         maxQueueSize: 2048,
       })
     );
 
-    console.log(`🔦 Tracer: OTLP exporter enabled to ${env.INTERNAL_OTEL_TRACE_EXPORTER_URL}`);
+    console.log(
+      `🔦 Tracer: OTLP exporter enabled to ${env.INTERNAL_OTEL_TRACE_EXPORTER_URL} (sampling = ${samplingRate})`
+    );
   } else {
     if (env.INTERNAL_OTEL_TRACE_LOGGING_ENABLED === "1") {
       console.log(`🔦 Tracer: Logger exporter enabled`);

apps/webapp/app/v3/utils/zodPubSub.server.ts

@@ -1,5 +1,6 @@
 import { Logger } from "@trigger.dev/core-backend";
 import { ZodMessageCatalogSchema, ZodMessageHandler, ZodMessageSender } from "@trigger.dev/core/v3";
+import { Evt } from "evt";
 import Redis, { RedisOptions } from "ioredis";
 import { z } from "zod";
 import { logger } from "~/services/logger.server";
@@ -26,6 +27,10 @@ class RedisZodSubscriber<TMessageCatalog extends ZodMessageCatalogSchema>
   private _listeners: Map<string, (payload: unknown) => Promise<void>> = new Map();
   private _messageHandler: ZodMessageHandler<TMessageCatalog>;
 
+  public onUnsubscribed: Evt<{
+    pattern: string;
+  }> = new Evt();
+
   constructor(
     private readonly _pattern: string,
     private readonly _options: ZodPubSubOptions<TMessageCatalog>,
@@ -51,7 +56,11 @@ class RedisZodSubscriber<TMessageCatalog extends ZodMessageCatalogSchema>
 
   public async stopListening(): Promise<void> {
     this._listeners.clear();
-    await this._subscriber.unsubscribe();
+    await this._subscriber.punsubscribe();
+
+    this.onUnsubscribed.post({ pattern: this._pattern });
+
+    this._subscriber.quit();
   }
 
   async #onMessage(pattern: string, channel: string, serializedMessage: string) {
@@ -90,6 +99,11 @@ class RedisZodSubscriber<TMessageCatalog extends ZodMessageCatalogSchema>
 export class ZodPubSub<TMessageCatalog extends ZodMessageCatalogSchema> {
   private _publisher: Redis;
   private _logger = logger.child({ module: "ZodPubSub" });
+  private _subscriberCount = 0;
+
+  get subscriberCount() {
+    return this._subscriberCount;
+  }
 
   constructor(private _options: ZodPubSubOptions<TMessageCatalog>) {
     this._publisher = new Redis(_options.redis);
@@ -112,6 +126,14 @@ export class ZodPubSub<TMessageCatalog extends ZodMessageCatalogSchema> {
 
     await subscriber.initialize();
 
+    this._subscriberCount++;
+
+    subscriber.onUnsubscribed.attachOnce(({ pattern }) => {
+      logger.debug("Subscriber unsubscribed", { pattern });
+
+      this._subscriberCount--;
+    });
+
     return subscriber;
   }
 }

apps/webapp/package.json

@@ -126,6 +126,7 @@
     "posthog-node": "^3.1.3",
     "prism-react-renderer": "^1.3.5",
     "prismjs": "^1.29.0",
+    "prom-client": "^15.1.0",
     "random-words": "^2.0.0",
     "react": "^18.2.0",
     "react-aria": "^3.31.1",

apps/webapp/server.ts

@@ -120,6 +120,10 @@ if (process.env.HTTP_SERVER_DISABLED !== "true") {
       )}`
     );
 
+    socket.on("error", (err) => {
+      console.error("Connection upgrade error:", err);
+    });
+
     const url = new URL(req.url ?? "", "http://localhost");
 
     // Upgrade socket.io connection
@@ -133,6 +137,7 @@ if (process.env.HTTP_SERVER_DISABLED !== "true") {
 
     // Only upgrade the connecting if the path is `/ws`
     if (url.pathname !== "/ws") {
+      // Setting the socket.destroy() error param causes an error event to be emitted which needs to be handled with socket.on("error") to prevent uncaught exceptions.
       socket.destroy(
         new Error(
           "Cannot connect because of invalid path: Please include `/ws` in the path of your upgrade request."

docs/_snippets/quickstart-whats-next.mdx

@@ -11,7 +11,7 @@
   >
     Learn more about how Trigger.dev works and how it can help you.
   </Card>
-  <Card title="Examples" icon="slot-machine" href="https:/trigger.dev/apis">
+  <Card title="Examples" icon="slot-machine" href="https://trigger.dev/apis">
     Find code examples for many popular APIs. These can be copied / modified for use in your own
     projects.
   </Card>

docs/_snippets/v3/step-cli-dev.mdx

@@ -0,0 +1,21 @@
+<Step title="Run the CLI `dev` command">
+
+The CLI `dev` command runs a server for your tasks. It will watches for changes in your `/trigger` directory and communicates with the Trigger.dev platform to register your tasks, perform runs, and send data back and forth.
+
+<CodeGroup>
+
+```bash npm
+npx trigger.dev@beta dev
+```
+
+```bash pnpm
+pnpm dlx trigger.dev@beta dev
+```
+
+```bash yarn
+yarn dlx trigger.dev@beta dev
+```
+
+</CodeGroup>
+
+</Step>

docs/_snippets/v3/step-cli-init.mdx

@@ -0,0 +1,30 @@
+<Step title="Run the CLI `init` command">
+
+The easiest way to get started it to use the CLI. It will add Trigger.dev to your existing project, create a `/trigger` folder and give you an example task.
+
+Run this command in the root of your project to get started:
+
+<CodeGroup>
+
+```bash npm
+npx trigger.dev@beta init
+```
+
+```bash pnpm
+pnpm dlx trigger.dev@beta init
+```
+
+```bash yarn
+yarn dlx trigger.dev@beta init
+```
+
+</CodeGroup>
+
+It will do a few things:
+
+1. Log you into the CLI if you're not already logged in.
+2. Create a `trigger.config.ts` file in the root of your project.
+3. Ask where you'd like to create the `/trigger` directory.
+4. Create the `/trigger` directory with an example task, `/trigger/example.[ts/js]`.
+
+</Step>

docs/_snippets/v3/step-run-test.mdx

@@ -0,0 +1,9 @@
+<Step title="Perform a test run using the dashboard">
+
+The CLI `dev` command spits out various useful URLs. Right now we want to visit the Test page it provided.
+
+You should see our Example task in the list, select it. Most tasks have a "payload" which you enter in the JSON editor, but our example task doesn't need any input.
+
+Press the "Run test" button.
+
+</Step>

docs/_snippets/v3/step-view-run.mdx

@@ -0,0 +1,7 @@
+<Step title="View your run">
+
+Congratulations, you should see the run page which will live reload showing you the current state of the run.
+
+If you go back to your terminal you'll see that the dev command also shows tasks that are running and links to the run log.
+
+</Step>

docs/documentation/introduction.mdx

@@ -1,9 +1,14 @@
 ---
-title: "Getting Started: Introduction"
+title: "v2: Introduction"
 sidebarTitle: "Introduction"
-description: "Welcome to the Trigger.dev documentation."
+description: "Welcome to the Trigger.dev (v2) documentation."
 ---
 
+<Warning>
+  These are the docs for Trigger.dev v2. You probably want [the v3
+  docs](https://trigger.dev/docs/v3).
+</Warning>
+
 Trigger.dev is an open source framework for creating long-running Jobs directly in your app with API Integrations, webhooks, scheduling and delays. You can reliably run Jobs that wouldn't normally work in serverless environments (like Vercel) because of timeouts.
 
 You can use [Trigger.dev Cloud](https://cloud.trigger.dev) or [Self-host Trigger.dev](/documentation/guides/self-hosting) on your own infrastructure.

docs/mint.json

@@ -1,7 +1,8 @@
 {
   "$schema": "https://mintlify.com/schema.json",
   "name": "Trigger.dev",
-  "openapi": ["/openapi.yml"],
+  "openapi": ["/openapi.yml", "/v3-openapi.json"],
+  "versions": ["v3 (Developer Preview)", "v2"],
   "logo": {
     "dark": "/logo/dark.png",
     "light": "/logo/light.png",
@@ -37,22 +38,34 @@
       "url": "https://discord.gg/kA47vcd8P6"
     }
   ],
+  "primaryTab": {
+    "name": "Home"
+  },
   "tabs": [
+    {
+      "name": "v3 Developer Preview",
+      "url": "https://trigger.dev/docs/v3",
+      "version": "v2"
+    },
+    {
+      "name": "v2",
+      "url": "https://trigger.dev/docs",
+      "version": "v3 (Developer Preview)"
+    },
     {
       "name": "Integrations",
-      "url": "integrations"
+      "url": "integrations",
+      "version": "v2"
     },
     {
       "name": "Reference",
-      "url": "sdk"
+      "url": "sdk",
+      "version": "v2"
     },
     {
       "name": "Examples",
-      "url": "https://trigger.dev/apis"
-    },
-    {
-      "name": "v3 Developer Preview",
-      "url": "v3"
+      "url": "https://trigger.dev/apis",
+      "version": "v2"
     }
   ],
   "redirects": [
@@ -74,8 +87,183 @@
     }
   ],
   "navigation": [
+    {
+      "group": "",
+      "version": "v3 (Developer Preview)",
+      "pages": ["v3/introduction"]
+    },
     {
       "group": "Getting Started",
+      "version": "v3 (Developer Preview)",
+      "pages": [
+        "v3/quick-start",
+        "v3/upgrading-from-v2",
+        "v3/changelog",
+        "v3/feature-matrix",
+        "v3/known-issues",
+        "v3/limits-performance"
+      ]
+    },
+    {
+      "group": "Fundamentals",
+      "version": "v3 (Developer Preview)",
+      "pages": [
+        "v3/trigger-folder",
+        "v3/tasks-overview",
+        "v3/triggering",
+        "v3/apikeys",
+        {
+          "group": "Task types",
+          "pages": ["v3/tasks-regular", "v3/tasks-scheduled", "v3/tasks-zod", "v3/tasks-webhooks"]
+        },
+        "v3/trigger-config"
+      ]
+    },
+    {
+      "group": "Development",
+      "version": "v3 (Developer Preview)",
+      "pages": ["v3/cli-dev", "v3/run-tests"]
+    },
+    {
+      "group": "Deployment",
+      "version": "v3 (Developer Preview)",
+      "pages": [
+        "v3/deploy-environment-variables",
+        "v3/cli-deploy",
+        "v3/github-actions",
+        {
+          "group": "Deployment integrations",
+          "pages": ["v3/vercel-integration"]
+        }
+      ]
+    },
+    {
+      "group": "Writing tasks",
+      "version": "v3 (Developer Preview)",
+      "pages": [
+        "v3/writing-tasks-introduction",
+        "v3/logging",
+        "v3/errors-retrying",
+        {
+          "group": "Wait",
+          "pages": [
+            "v3/wait",
+            "v3/wait-for",
+            "v3/wait-until",
+            "v3/wait-for-event",
+            "v3/wait-for-request"
+          ]
+        },
+        "v3/queue-concurrency",
+        "v3/versioning",
+        "v3/machines",
+        "v3/idempotency",
+        "v3/reattempting-replaying",
+        "v3/trigger-filters",
+        "v3/notifications",
+        "v3/rollbacks",
+        "v3/using-apis",
+        "v3/middleware",
+        "v3/automated-tests"
+      ]
+    },
+    {
+      "group": "Dashboard",
+      "version": "v3 (Developer Preview)",
+      "pages": [
+        "v3/dashboard-overview",
+        "v3/dashboard-runs",
+        "v3/dashboard-tests",
+        "v3/dashboard-environment-variables"
+      ]
+    },
+    {
+      "group": "API reference",
+      "version": "v3 (Developer Preview)",
+      "pages": [
+        {
+          "group": "Functions",
+          "pages": [
+            "v3/reference-task",
+            "v3/reference-cron-task",
+            "v3/reference-cron-dynamic",
+            "v3/reference-interval-task",
+            "v3/reference-interval-dynamic",
+            "v3/reference-zod-task",
+            "v3/reference-zod-catalog",
+            "v3/reference-task-trigger",
+            "v3/reference-task-trigger-and-wait",
+            "v3/reference-task-batch-trigger",
+            "v3/reference-task-batch-trigger-and-wait",
+            "v3/reference-wait-for",
+            "v3/reference-wait-until",
+            "v3/reference-wait-for-event",
+            "v3/reference-wait-for-request",
+            "v3/reference-retry-on-throw",
+            "v3/reference-retry-fetch",
+            "v3/reference-retry-intercept-fetch",
+            "v3/reference-notification-catalog",
+            "v3/reference-notify",
+            "v3/reference-queue"
+          ]
+        },
+        {
+          "group": "Objects",
+          "pages": ["v3/reference-context"]
+        },
+        {
+          "group": "CLI",
+          "pages": [
+            "v3/reference-cli-init",
+            "v3/reference-cli-dev",
+            "v3/reference-cli-deploy",
+            "v3/reference-cli-login",
+            "v3/reference-cli-logout",
+            "v3/reference-cli-update",
+            "v3/reference-cli-build",
+            "v3/reference-cli-who-am-i"
+          ]
+        },
+        {
+          "group": "Management API",
+          "pages": [
+            "v3/management-start-run",
+            "v3/management-get-run",
+            "v3/management-get-runs",
+            "v3/management-replay-run",
+            "v3/management-cancel-run"
+          ]
+        }
+      ]
+    },
+    {
+      "group": "Architecture",
+      "version": "v3 (Developer Preview)",
+      "pages": [
+        "v3/architecture-how-it-works",
+        "v3/architecture-multi-tenant-queue",
+        "v3/architecture-reliability"
+      ]
+    },
+    {
+      "group": "Open source",
+      "version": "v3 (Developer Preview)",
+      "pages": ["v3/github-repo", "v3/open-source-self-hosting", "v3/open-source-contributing"]
+    },
+    {
+      "group": "Help",
+      "version": "v3 (Developer Preview)",
+      "pages": [
+        "v3/help-faqs",
+        "v3/community",
+        "v3/help-email",
+        "v3/help-slack",
+        "v3/help-uptime-status"
+      ]
+    },
+    {
+      "group": "Getting Started",
+      "version": "v2",
       "pages": [
         "documentation/introduction",
         {
@@ -107,6 +295,7 @@
     },
     {
       "group": "Concepts",
+      "version": "v2",
       "pages": [
         "documentation/concepts/what-is-triggerdotdev",
         "documentation/concepts/limits",
@@ -142,6 +331,7 @@
     },
     {
       "group": "Guides",
+      "version": "v2",
       "pages": [
         {
           "group": "Frameworks",
@@ -226,6 +416,7 @@
     },
     {
       "group": "Community & Support",
+      "version": "v2",
       "pages": [
         "documentation/get-help",
         "documentation/faq",
@@ -430,169 +621,13 @@
     },
     {
       "group": "Overview",
+      "version": "v2",
       "pages": ["examples/introduction"]
-    },
-    {
-      "group": "",
-      "pages": ["v3/introduction"]
-    },
-    {
-      "group": "Getting Started",
-      "pages": [
-        "v3/quick-start",
-        "v3/upgrading-from-v2",
-        "v3/changelog",
-        "v3/feature-matrix",
-        "v3/limits"
-      ]
-    },
-    {
-      "group": "Fundamentals",
-      "pages": [
-        "v3/trigger-folder",
-        "v3/tasks-overview",
-        "v3/triggering",
-        "v3/apikeys",
-        {
-          "group": "Task types",
-          "pages": ["v3/tasks-regular", "v3/tasks-scheduled", "v3/tasks-zod", "v3/tasks-webhooks"]
-        }
-      ]
-    },
-    {
-      "group": "Development",
-      "pages": ["v3/cli-dev", "v3/run-tests"]
-    },
-    {
-      "group": "Deployment",
-      "pages": [
-        "v3/deploy-environment-variables",
-        "v3/cli-deploy",
-        "v3/github-actions",
-        {
-          "group": "Deployment integrations",
-          "pages": ["v3/vercel-integration"]
-        }
-      ]
-    },
-    {
-      "group": "Writing tasks",
-      "pages": [
-        "v3/writing-tasks-introduction",
-        "v3/logging",
-        "v3/errors-retrying",
-        {
-          "group": "Wait",
-          "pages": [
-            "v3/wait",
-            "v3/wait-for",
-            "v3/wait-until",
-            "v3/wait-for-event",
-            "v3/wait-for-request"
-          ]
-        },
-        "v3/queue-concurrency",
-        "v3/versioning",
-        "v3/machines",
-        "v3/idempotency",
-        "v3/reattempting-replaying",
-        "v3/trigger-filters",
-        "v3/notifications",
-        "v3/rollbacks",
-        "v3/using-apis",
-        "v3/middleware",
-        "v3/automated-tests"
-      ]
-    },
-    {
-      "group": "Dashboard",
-      "pages": [
-        "v3/dashboard-overview",
-        "v3/dashboard-runs",
-        "v3/dashboard-tests",
-        "v3/dashboard-environment-variables"
-      ]
-    },
-    {
-      "group": "API reference",
-      "pages": [
-        {
-          "group": "Functions",
-          "pages": [
-            "v3/reference-task",
-            "v3/reference-cron-task",
-            "v3/reference-cron-dynamic",
-            "v3/reference-interval-task",
-            "v3/reference-interval-dynamic",
-            "v3/reference-zod-task",
-            "v3/reference-zod-catalog",
-            "v3/reference-task-trigger",
-            "v3/reference-task-trigger-and-wait",
-            "v3/reference-task-batch-trigger",
-            "v3/reference-task-batch-trigger-and-wait",
-            "v3/reference-wait-for",
-            "v3/reference-wait-until",
-            "v3/reference-wait-for-event",
-            "v3/reference-wait-for-request",
-            "v3/reference-retry-on-throw",
-            "v3/reference-retry-fetch",
-            "v3/reference-retry-intercept-fetch",
-            "v3/reference-notification-catalog",
-            "v3/reference-notify",
-            "v3/reference-queue"
-          ]
-        },
-        {
-          "group": "Objects",
-          "pages": ["v3/reference-context"]
-        },
-        {
-          "group": "CLI",
-          "pages": [
-            "v3/reference-cli-init",
-            "v3/reference-cli-dev",
-            "v3/reference-cli-deploy",
-            "v3/reference-cli-login",
-            "v3/reference-cli-logout",
-            "v3/reference-cli-update",
-            "v3/reference-cli-build",
-            "v3/reference-cli-who-am-i"
-          ]
-        },
-        "v3/reference-trigger-config"
-      ]
-    },
-    {
-      "group": "Architecture",
-      "pages": [
-        "v3/architecture-how-it-works",
-        "v3/architecture-multi-tenant-queue",
-        "v3/architecture-reliability"
-      ]
-    },
-    {
-      "group": "Open source",
-      "pages": ["v3/github-repo", "v3/open-source-self-hosting", "v3/open-source-contributing"]
-    },
-    {
-      "group": "Help",
-      "pages": [
-        "v3/help-faqs",
-        "v3/community",
-        "v3/help-email",
-        "v3/help-slack",
-        "v3/help-uptime-status"
-      ]
     }
   ],
   "footerSocials": {
     "twitter": "https://twitter.com/triggerdotdev",
     "github": "https://github.com/triggerdotdev",
     "linkedin": "https://www.linkedin.com/company/triggerdotdev"
-  },
-  "analytics": {
-    "posthog": {
-      "apiKey": "phc_9aSDbJCaDUMdZdHxxMPTvcj7A9fsl3mCgM1RBPmPsl7"
-    }
   }
 }

docs/v3-openapi.json

@@ -0,0 +1,216 @@
+{
+  "openapi": "3.1.0",
+  "info": {
+    "title": "Trigger.dev v3 REST API",
+    "description": "The REST API lets you trigger and manage runs on Trigger.dev. You can trigger a run, get the status of a run, and get the results of a run. ",
+    "version": "2024-04"
+  },
+  "servers": [
+    {
+      "url": "https://api.trigger.dev",
+      "description": "Trigger.dev API"
+    }
+  ],
+  "paths": {
+    "/api/v1/runs/{run_id}/replay": {
+      "post": {
+        "description": "Creates a new run with the same payload and options as the original run.",
+        "parameters": [
+          {
+            "in": "path",
+            "name": "run_id",
+            "required": true,
+            "schema": {
+              "type": "string"
+            },
+            "description": "The ID of an existing run. When you trigger a run you will get an id in the response."
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Successful request",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "id": {
+                      "type": "string",
+                      "description": "The ID of the new run."
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "400": {
+            "description": "Invalid request",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "error": {
+                      "type": "string",
+                      "enum": ["Invalid or missing run ID", "Failed to create new run"]
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "401": {
+            "description": "Unauthorized request",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "error": {
+                      "type": "string",
+                      "enum": ["Invalid or Missing API key"]
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "404": {
+            "description": "Resource not found",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "error": {
+                      "type": "string",
+                      "enum": ["Run not found"]
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        "tags": ["run"],
+        "security": [{ "bearerAuth": [] }],
+        "operationId": "replay_run_v1",
+        "x-codeSamples": [
+          {
+            "lang": "typescript",
+            "source": "const handle = await runs.replay(existingRun.id);"
+          },
+          {
+            "lang": "sh",
+            "source": "curl --request POST \\\n\t--url https://api.trigger.dev/api/v1/runs/{run_id}/replay \\\n\t--header 'Authorization: Bearer <token>'"
+          }
+        ]
+      }
+    },
+    "/api/v1/runs/{run_id}/cancel": {
+      "post": {
+        "description": "Cancels a run.",
+        "parameters": [
+          {
+            "in": "path",
+            "name": "run_id",
+            "required": true,
+            "schema": {
+              "type": "string"
+            },
+            "description": "The ID of an existing run. When you trigger a run you will get an id in the response."
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Successful request",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "message": {
+                      "type": "string",
+                      "description": "Confirmation message that the run was canceled."
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "400": {
+            "description": "Invalid request",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "error": {
+                      "type": "string",
+                      "enum": ["Invalid or missing run ID", "Failed to create new run"]
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "401": {
+            "description": "Unauthorized request",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "error": {
+                      "type": "string",
+                      "enum": ["Invalid or Missing API key"]
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "404": {
+            "description": "Resource not found",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "error": {
+                      "type": "string",
+                      "enum": ["Run not found"]
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        "tags": ["run"],
+        "security": [{ "bearerAuth": [] }],
+        "operationId": "replay_run_v1",
+        "x-codeSamples": [
+          {
+            "lang": "typescript",
+            "source": "await runs.cancel(existingRun.id);"
+          },
+          {
+            "lang": "sh",
+            "source": "curl --request POST \\\n\t--url https://api.trigger.dev/api/v1/runs/{run_id}/cancel \\\n\t--header 'Authorization: Bearer <token>'"
+          }
+        ]
+      }
+    }
+  },
+  "components": {
+    "securitySchemes": {
+      "bearerAuth": {
+        "type": "http",
+        "scheme": "bearer",
+        "description": "Use your Secret API key in the form 'Bearer <SECRET KEY>' (without the quotation marks)"
+      }
+    }
+  },
+  "security": [{ "bearerAuth": [] }]
+}

docs/v3/errors-retrying.mdx

@@ -7,12 +7,12 @@ When an uncaught error is thrown inside your task, that task attempt will fail.
 
 You can configure retrying in two ways:
 
-1. In your [trigger.config file](/v3/reference-trigger-config) you can set the default retrying behavior for all tasks.
+1. In your [trigger.config file](/v3/trigger-config) you can set the default retrying behavior for all tasks.
 2. On each task you can set the retrying behavior.
 
 <Note>
   By default when you create your project using the CLI init command we disabled retrying in the DEV
-  environment. You can enable it in your [trigger.config file](/v3/reference-trigger-config).
+  environment. You can enable it in your [trigger.config file](/v3/trigger-config).
 </Note>
 
 ## A simple example with OpenAI

docs/v3/feature-matrix.mdx

@@ -3,8 +3,24 @@ title: "Feature matrix"
 description: "What features are currently available in the Developer Preview"
 ---
 
-| Feature                                | Description                                | Status |
-| -------------------------------------- | ------------------------------------------ | ------ |
-| [Regular tasks](/v3/tasks-regular)     | A task that can be triggered from anywhere | ✅     |
-| [Scheduled tasks](/v3/tasks-scheduled) | A task that can be triggered on a schedule | ⏳     |
-| [Webhook tasks](v3/tasks-webhooks)     | A task that can be triggered by a webhook  | ⏳     |
+| Feature                                                                             | Description                                        | Status |
+| ----------------------------------------------------------------------------------- | -------------------------------------------------- | ------ |
+| [Regular tasks](/v3/tasks-regular)                                                  | A task that can be triggered from anywhere         | ✅     |
+| [Triggering](/v3/triggering)                                                        | Triggering and batch triggering tasks              | ✅     |
+| [Testing from the dashboard](/v3/run-tests)                                         | Test your tasks from the dashboard                 | ✅     |
+| [Queues and concurrency controls](/v3/queue-concurrency)                            | Queues and concurrency controls                    | ✅     |
+| [Per-tenant queuing](/v3/queue-concurrency#concurrency-keys-and-per-tenant-queuing) | Separate queues for each of your users             | ✅     |
+| [Reattempts and retrying](/v3/errors-retrying)                                      | Write reliable tasks using retries                 | ✅     |
+| [Atomic versioning](/v3/versioning)                                                 | Each deploy creates a new version                  | ✅     |
+| [Deploy via CLI](/v3/cli-deploy)                                                    | Deploy from the command line                       | ✅     |
+| [Deploy via GitHub Actions](/v3/github-actions)                                     | Deploy using GitHub Actions                        | ✅     |
+| [Scheduled tasks](/v3/tasks-scheduled)                                              | A task that can be triggered on a schedule         | ⏳     |
+| [Zod tasks](/v3/tasks-zod)                                                          | Define tasks using Zod schemas                     | ⏳     |
+| [Webhook tasks](/v3/tasks-webhooks)                                                 | A task that can be triggered by a webhook          | ⏳     |
+| Full text search of runs                                                            | Find a run by searching the payload and output     | ⏳     |
+| Logs view with search                                                               | All logs view with filtering and full text search  | ⏳     |
+| Alerts                                                                              | Add alerts in the UI for errors and queue backlogs | ⏳     |
+| Notifications                                                                       | Send data to your web app from a run               | ⏳     |
+| Rollbacks                                                                           | Easily rollback changes when errors happen         | ⏳     |
+
+[Let us know](https://trigger.dev/discord) what we should prioritize and what we are missing.

docs/v3/known-issues.mdx

@@ -0,0 +1,12 @@
+---
+title: "Known issues"
+description: "Please let us know if you encounter an issue not on this list."
+---
+
+#### `pnpm` is not currently supported
+
+By default, pnpm has a more complicated package structure than npm. If you see this error when running the CLI then you're experiencing this bug:
+
+```bash
+✘ [ERROR] Could not resolve "@trigger.dev/core/v3"
+```

docs/v3/limits-performance.mdx

@@ -1,5 +1,5 @@
 ---
-title: "Limits"
+title: "Limits & Performance"
 ---
 
 <Snippet file="incomplete-docs.mdx" />

docs/v3/logging.mdx

@@ -38,18 +38,20 @@ Tracing is a way to follow the flow of your code. It's very useful for debugging
 
 Trigger.dev uses OpenTelemetry tracing under the hood. With automatic tracing for many things like task triggering, task attempts, HTTP requests, and more.
 
-### Automatic instrumentation
-
 | Name          | Description                      |
 | ------------- | -------------------------------- |
 | Task triggers | Task triggers.                   |
 | Task attempts | Task attempts.                   |
 | HTTP requests | HTTP requests made by your code. |
-| OpenAI        | OpenAI SDK calls.                |
 
-We want to provide automatic instrumentation for as many things as possible. Please do [request any automatic instrumentation](https://github.com/triggerdotdev/trigger.dev/issues/new?template=instrumentation_request.yml) you would like to see.
+### Adding instrumentations
 
-## Add custom traces
+![The run log
+  ](/images/v3/auto-instrumentation.png)
+
+You can [add instrumentations](/v3/trigger-config#instrumentations). The Prisma one above will automatically trace all Prisma queries.
+
+### Add custom traces
 
 If you want to add custom traces to your code, you can use the `logger.trace` function. It will create a new OTEL trace and you can set attributes on it.
 

docs/v3/management-cancel-run.mdx

@@ -0,0 +1,4 @@
+---
+title: "Cancel run"
+openapi: "v3-openapi POST /api/v1/runs/{run_id}/cancel"
+---

docs/v3/management-get-run.mdx

@@ -0,0 +1,6 @@
+---
+title: "Get run"
+description: "Get a run using the Task id."
+---
+
+<Snippet file="incomplete-docs.mdx" />

docs/v3/management-get-runs.mdx

@@ -0,0 +1,6 @@
+---
+title: "Get runs"
+description: "Get runs using a Task id."
+---
+
+<Snippet file="incomplete-docs.mdx" />