Release 3.0.0-beta.55

* tasks no longer inside a group in the side menu (and added “cron”)

* Delay using a timezone

* Added React Not Defined error to the troubleshooting page

* Improved the React common problem

* Link to v2 docs

* New Development section and entry in Common Problems

* Concurrently running the terminal

* Fixed the .env weirdness

* Added section on creating PATs for Github actions

* Improved the Machine spec and limits page

* Quick start steps now have nice images

* Added a diagram for the lifecycle functions

* Added note about onFailure

* CRON -> cron/Cron

* WIP adding more steps to the next.js guide

* WIP next.js

* WIP adding tabbed steps for pages/app router

* References to Infisical links to their homepage so it’s clearer

* WIP updating the nextjs guide

* WIP nextjs guide

* More nextjs guide steps

* More copy

* Added rate limit trouble shooting

* Removed old prisma error title

* Added secret key step

* Added a note for logging in using a specified domain if self hosting

* typo

* App router docs copy

* Deploy copy update

* Added a favicon.png to fix a docs build error

* Removed unused snippet

* Server actions now inside a tab

* Server actions + restructured the triggering section

* Added troubleshooting snippet for react event handlers

* Added a new troubleshooting snippet for ESM

* Updated old replaying image to reflect the new UI

* Removed references to reattempting

* Updated replaying from the run page

* Added a bulk replay section

* Updated the writing tasks intro page

* Removed edge runtime code for now

* Added edge runtime – it seems to just work!

* import type

.changeset/brown-boats-bathe.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/core-apps": patch
+"@trigger.dev/core": patch
+---
+
+Pre-pull deployment images for faster startups

.changeset/early-impalas-itch.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Fixes for continuing after waits

.changeset/fast-melons-listen.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Added version to ctx.run

.changeset/few-poems-vanish.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Fix trigger functions for custom queues

.changeset/green-pens-battle.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix automatic opening of login URL on linux-server systems with missing xdg-open

.changeset/hungry-sloths-promise.md

@@ -0,0 +1,12 @@
+---
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Fix issues that could result in unreezable state run crashes. Details:
+- Never checkpoint between attempts
+- Some messages and socket data now include attempt numbers
+- Remove attempt completion replays
+- Additional prod entry point logging
+- Fail runs that receive deprecated (pre-lazy attempt) execute messages

.changeset/long-feet-invent.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Rollback to try and fix some dependent attempt issues

.changeset/pre.json

@@ -52,6 +52,7 @@
     "big-tomatoes-deliver",
     "blue-pumas-whisper",
     "breezy-gorillas-mate",
+    "brown-boats-bathe",
     "brown-spies-burn",
     "chilled-hornets-move",
     "clean-pianos-listen",
@@ -64,10 +65,13 @@
     "curly-monkeys-tell",
     "dry-walls-check",
     "dull-mangos-press",
+    "early-impalas-itch",
     "eight-pumas-float",
     "eleven-paws-join",
     "famous-boats-tease",
     "fast-colts-relax",
+    "fast-melons-listen",
+    "few-poems-vanish",
     "few-students-share",
     "fifty-lions-think",
     "five-toes-destroy",
@@ -77,9 +81,11 @@
     "gorgeous-cycles-guess",
     "gorgeous-gorillas-compete",
     "green-bags-wink",
+    "green-pens-battle",
     "hot-buckets-behave",
     "hot-fishes-retire",
     "hot-wasps-sin",
+    "hungry-sloths-promise",
     "itchy-chairs-itch",
     "khaki-apricots-design",
     "khaki-poems-lay",
@@ -92,6 +98,7 @@
     "light-bulldogs-press",
     "light-dragons-complain",
     "little-crabs-cross",
+    "long-feet-invent",
     "long-fireants-search",
     "long-hounds-wave",
     "loud-actors-remember",

apps/coordinator/src/chaosMonkey.ts

@@ -12,7 +12,11 @@ export class ChaosMonkey {
   private chaosEventRate = 0.2;
   private delayInSeconds = 45;
 
-  constructor(private enabled = false) {
+  constructor(
+    private enabled = false,
+    private disableErrors = false,
+    private disableDelays = false
+  ) {
     if (this.enabled) {
       console.log("🍌 Chaos monkey enabled");
     }
@@ -32,8 +36,8 @@ export class ChaosMonkey {
 
   async call({
     $,
-    throwErrors = true,
-    addDelays = true,
+    throwErrors = !this.disableErrors,
+    addDelays = !this.disableDelays,
   }: {
     $?: Execa$<string>;
     throwErrors?: boolean;

apps/coordinator/src/checkpointer.ts

@@ -17,6 +17,7 @@ type CheckpointAndPushOptions = {
   projectRef: string;
   deploymentVersion: string;
   shouldHeartbeat?: boolean;
+  attemptNumber?: number;
 };
 
 type CheckpointAndPushResult =
@@ -258,6 +259,7 @@ export class Checkpointer {
     leaveRunning = true, // This mirrors kubernetes behaviour more accurately
     projectRef,
     deploymentVersion,
+    attemptNumber,
   }: CheckpointAndPushOptions): Promise<CheckpointAndPushResult> {
     this.#logger.log("Checkpointing with backoff", {
       runId,
@@ -297,6 +299,7 @@ export class Checkpointer {
           leaveRunning,
           projectRef,
           deploymentVersion,
+          attemptNumber,
         });
 
         if (result.success) {
@@ -359,6 +362,7 @@ export class Checkpointer {
     leaveRunning = true, // This mirrors kubernetes behaviour more accurately
     projectRef,
     deploymentVersion,
+    attemptNumber,
   }: CheckpointAndPushOptions): Promise<CheckpointAndPushResult> {
     await this.init();
 
@@ -367,6 +371,7 @@ export class Checkpointer {
       leaveRunning,
       projectRef,
       deploymentVersion,
+      attemptNumber,
     };
 
     if (!this.#dockerMode && !this.#canCheckpoint) {
@@ -418,13 +423,14 @@ export class Checkpointer {
       this.#logger.log("Checkpointing:", { options });
 
       const containterName = this.#getRunContainerName(runId);
+      const containterNameWithAttempt = this.#getRunContainerName(runId, attemptNumber);
 
       // Create checkpoint (docker)
       if (this.#dockerMode) {
         try {
           if (this.opts.forceSimulate || !this.#canCheckpoint) {
             this.#logger.log("Simulating checkpoint");
-            this.#logger.debug(await $$`docker pause ${containterName}`);
+            this.#logger.debug(await $$`docker pause ${containterNameWithAttempt}`);
           } else {
             if (this.simulateCheckpointFailure) {
               if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) {
@@ -435,11 +441,11 @@ export class Checkpointer {
 
             if (leaveRunning) {
               this.#logger.debug(
-                await $$`docker checkpoint create --leave-running ${containterName} ${exportLocation}`
+                await $$`docker checkpoint create --leave-running ${containterNameWithAttempt} ${exportLocation}`
               );
             } else {
               this.#logger.debug(
-                await $$`docker checkpoint create ${containterName} ${exportLocation}`
+                await $$`docker checkpoint create ${containterNameWithAttempt} ${exportLocation}`
               );
             }
           }
@@ -581,7 +587,7 @@ export class Checkpointer {
     return this.#failedCheckpoints.has(runId);
   }
 
-  #getRunContainerName(suffix: string) {
-    return `task-run-${suffix}`;
+  #getRunContainerName(suffix: string, attemptNumber?: number) {
+    return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
   }
 }

apps/coordinator/src/index.ts

@@ -49,7 +49,11 @@ const PLATFORM_SECRET = process.env.PLATFORM_SECRET || "coordinator-secret";
 const SECURE_CONNECTION = ["1", "true"].includes(process.env.SECURE_CONNECTION ?? "false");
 
 const logger = new SimpleLogger(`[${NODE_NAME}]`);
-const chaosMonkey = new ChaosMonkey(!!process.env.CHAOS_MONKEY_ENABLED);
+const chaosMonkey = new ChaosMonkey(
+  !!process.env.CHAOS_MONKEY_ENABLED,
+  !!process.env.CHAOS_MONKEY_DISABLE_ERRORS,
+  !!process.env.CHAOS_MONKEY_DISABLE_DELAYS
+);
 
 class TaskCoordinator {
   #httpServer: ReturnType<typeof createServer>;
@@ -290,6 +294,7 @@ class TaskCoordinator {
           setSocketDataFromHeader("projectRef", "x-trigger-project-ref");
           setSocketDataFromHeader("runId", "x-trigger-run-id");
           setSocketDataFromHeader("attemptFriendlyId", "x-trigger-attempt-friendly-id", false);
+          setSocketDataFromHeader("attemptNumber", "x-trigger-attempt-number", false);
           setSocketDataFromHeader("envId", "x-trigger-env-id");
           setSocketDataFromHeader("deploymentId", "x-trigger-deployment-id");
           setSocketDataFromHeader("deploymentVersion", "x-trigger-deployment-version");
@@ -306,6 +311,10 @@ class TaskCoordinator {
       onConnection: async (socket, handler, sender) => {
         const logger = new SimpleLogger(`[prod-worker][${socket.id}]`);
 
+        const getAttemptNumber = () => {
+          return socket.data.attemptNumber ? parseInt(socket.data.attemptNumber) : undefined;
+        };
+
         const crashRun = async (error: { name: string; message: string; stack?: string }) => {
           try {
             this.#platformSocket?.send("RUN_CRASHED", {
@@ -381,6 +390,10 @@ class TaskCoordinator {
           socket.data.attemptFriendlyId = attemptFriendlyId;
         };
 
+        const updateAttemptNumber = (attemptNumber: string | number) => {
+          socket.data.attemptNumber = String(attemptNumber);
+        };
+
         this.#platformSocket?.send("LOG", {
           metadata: socket.data,
           text: "connected",
@@ -430,6 +443,7 @@ class TaskCoordinator {
             });
 
             updateAttemptFriendlyId(executionAck.payload.execution.attempt.id);
+            updateAttemptNumber(executionAck.payload.execution.attempt.number);
           } catch (error) {
             logger.error("Error", { error });
 
@@ -505,11 +519,17 @@ class TaskCoordinator {
 
           updateAttemptFriendlyId(message.attemptFriendlyId);
 
-          this.#platformSocket?.send("READY_FOR_RESUME", message);
+          if (message.version === "v2") {
+            updateAttemptNumber(message.attemptNumber);
+          }
+
+          this.#platformSocket?.send("READY_FOR_RESUME", { ...message, version: "v1" });
         });
 
         // MARK: RUN COMPLETED
-        socket.on("TASK_RUN_COMPLETED", async ({ completion, execution }, callback) => {
+        socket.on("TASK_RUN_COMPLETED", async (message, callback) => {
+          const { completion, execution } = message;
+
           logger.log("completed task", { completionId: completion.id });
 
           // Cancel all in-progress checkpoints (if any)
@@ -518,8 +538,10 @@ class TaskCoordinator {
           await chaosMonkey.call({ throwErrors: false });
 
           const completeWithoutCheckpoint = (shouldExit: boolean) => {
+            const supportsRetryCheckpoints = message.version === "v1";
+
             this.#platformSocket?.send("TASK_RUN_COMPLETED", {
-              version: "v1",
+              version: supportsRetryCheckpoints ? "v1" : "v2",
               execution,
               completion,
             });
@@ -546,6 +568,15 @@ class TaskCoordinator {
 
           if (completion.retry.delay < this.#delayThresholdInMs) {
             completeWithoutCheckpoint(false);
+
+            // Prevents runs that fail fast from never sending a heartbeat
+            this.#sendRunHeartbeat(socket.data.runId);
+
+            return;
+          }
+
+          if (message.version === "v2") {
+            completeWithoutCheckpoint(true);
             return;
           }
 
@@ -681,6 +712,7 @@ class TaskCoordinator {
             runId: socket.data.runId,
             projectRef: socket.data.projectRef,
             deploymentVersion: socket.data.deploymentVersion,
+            attemptNumber: getAttemptNumber(),
           });
 
           if (!checkpoint) {
@@ -752,6 +784,7 @@ class TaskCoordinator {
             runId: socket.data.runId,
             projectRef: socket.data.projectRef,
             deploymentVersion: socket.data.deploymentVersion,
+            attemptNumber: getAttemptNumber(),
           });
 
           if (!checkpoint) {
@@ -821,6 +854,7 @@ class TaskCoordinator {
             runId: socket.data.runId,
             projectRef: socket.data.projectRef,
             deploymentVersion: socket.data.deploymentVersion,
+            attemptNumber: getAttemptNumber(),
           });
 
           if (!checkpoint) {
@@ -905,6 +939,7 @@ class TaskCoordinator {
           }
 
           updateAttemptFriendlyId(createAttempt.executionPayload.execution.attempt.id);
+          updateAttemptNumber(createAttempt.executionPayload.execution.attempt.number);
 
           callback({
             success: true,
@@ -924,6 +959,10 @@ class TaskCoordinator {
           if (message.attemptFriendlyId) {
             updateAttemptFriendlyId(message.attemptFriendlyId);
           }
+
+          if (message.attemptNumber) {
+            updateAttemptNumber(message.attemptNumber);
+          }
         });
       },
       onDisconnect: async (socket, handler, sender, logger) => {

apps/docker-provider/src/index.ts

@@ -109,7 +109,7 @@ class DockerTaskOperations implements TaskOperations {
   async create(opts: TaskOperationsCreateOptions) {
     await this.init();
 
-    const containerName = this.#getRunContainerName(opts.runId);
+    const containerName = this.#getRunContainerName(opts.runId, opts.nextAttemptNumber);
 
     const runArgs = [
       "run",
@@ -150,7 +150,7 @@ class DockerTaskOperations implements TaskOperations {
   async restore(opts: TaskOperationsRestoreOptions) {
     await this.init();
 
-    const containerName = this.#getRunContainerName(opts.runId);
+    const containerName = this.#getRunContainerName(opts.runId, opts.attemptNumber);
 
     if (!this.#canCheckpoint || this.opts.forceSimulate) {
       logger.log("Simulating restore");
@@ -195,8 +195,8 @@ class DockerTaskOperations implements TaskOperations {
     return `task-index-${suffix}`;
   }
 
-  #getRunContainerName(suffix: string) {
-    return `task-run-${suffix}`;
+  #getRunContainerName(suffix: string, attemptNumber?: number) {
+    return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
   }
 
   async #sendPostStart(containerName: string): Promise<void> {

apps/docker-provider/tsconfig.json

@@ -1,6 +1,6 @@
 {
   "compilerOptions": {
-    "target": "es2016",
+    "target": "es2018",
     "module": "commonjs",
     "esModuleInterop": true,
     "forceConsistentCasingInFileNames": true,

apps/kubernetes-provider/src/index.ts

@@ -4,6 +4,7 @@ import {
   TaskOperations,
   TaskOperationsCreateOptions,
   TaskOperationsIndexOptions,
+  TaskOperationsPrePullDeploymentOptions,
   TaskOperationsRestoreOptions,
 } from "@trigger.dev/core-apps/provider";
 import { SimpleLogger } from "@trigger.dev/core-apps/logger";
@@ -49,6 +50,7 @@ class KubernetesTaskOperations implements TaskOperations {
   #k8sApi: {
     core: k8s.CoreV1Api;
     batch: k8s.BatchV1Api;
+    apps: k8s.AppsV1Api;
   };
 
   constructor(namespace = "default") {
@@ -100,7 +102,7 @@ class KubernetesTaskOperations implements TaskOperations {
                   resources: {
                     limits: {
                       cpu: "1",
-                      memory: "1G",
+                      memory: "2G",
                       "ephemeral-storage": "2Gi",
                     },
                   },
@@ -137,10 +139,12 @@ class KubernetesTaskOperations implements TaskOperations {
   }
 
   async create(opts: TaskOperationsCreateOptions) {
+    const containerName = this.#getRunContainerName(opts.runId, opts.nextAttemptNumber);
+
     await this.#createPod(
       {
         metadata: {
-          name: this.#getRunContainerName(opts.runId),
+          name: containerName,
           namespace: this.#namespace.metadata.name,
           labels: {
             ...this.#getSharedLabels(opts),
@@ -155,7 +159,7 @@ class KubernetesTaskOperations implements TaskOperations {
           terminationGracePeriodSeconds: 60 * 60,
           containers: [
             {
-              name: this.#getRunContainerName(opts.runId),
+              name: containerName,
               image: opts.image,
               ports: [
                 {
@@ -209,7 +213,7 @@ class KubernetesTaskOperations implements TaskOperations {
     await this.#createPod(
       {
         metadata: {
-          name: `${this.#getRunContainerName(opts.runId)}-${randomUUID().slice(0, 8)}`,
+          name: `${this.#getRunContainerName(opts.runId)}-${opts.checkpointId.slice(-8)}`,
           namespace: this.#namespace.metadata.name,
           labels: {
             ...this.#getSharedLabels(opts),
@@ -313,6 +317,72 @@ class KubernetesTaskOperations implements TaskOperations {
     await this.#getPod(opts.runId, this.#namespace);
   }
 
+  async prePullDeployment(opts: TaskOperationsPrePullDeploymentOptions) {
+    const metaName = this.#getPrePullContainerName(opts.shortCode);
+
+    const metaLabels = {
+      ...this.#getSharedLabels(opts),
+      app: "task-prepull",
+      "app.kubernetes.io/part-of": "trigger-worker",
+      "app.kubernetes.io/component": "prepull",
+      deployment: opts.deploymentId,
+      name: metaName,
+    } satisfies k8s.V1ObjectMeta["labels"];
+
+    await this.#createDaemonSet(
+      {
+        metadata: {
+          name: metaName,
+          namespace: this.#namespace.metadata.name,
+          labels: metaLabels,
+        },
+        spec: {
+          selector: {
+            matchLabels: {
+              name: metaName,
+            },
+          },
+          template: {
+            metadata: {
+              labels: metaLabels,
+            },
+            spec: {
+              ...this.#defaultPodSpec,
+              restartPolicy: "Always",
+              initContainers: [
+                {
+                  name: "prepull",
+                  image: opts.imageRef,
+                  command: ["/usr/bin/true"],
+                  resources: {
+                    limits: {
+                      cpu: "0.25",
+                      memory: "100Mi",
+                      "ephemeral-storage": "1Gi",
+                    },
+                  },
+                },
+              ],
+              containers: [
+                {
+                  name: "pause",
+                  image: "registry.k8s.io/pause:3.9",
+                  resources: {
+                    limits: {
+                      cpu: "1m",
+                      memory: "12Mi",
+                    },
+                  },
+                },
+              ],
+            },
+          },
+        },
+      },
+      this.#namespace
+    );
+  }
+
   #envTypeToLabelValue(type: EnvironmentType) {
     switch (type) {
       case "PRODUCTION":
@@ -402,7 +472,11 @@ class KubernetesTaskOperations implements TaskOperations {
   }
 
   #getSharedLabels(
-    opts: TaskOperationsIndexOptions | TaskOperationsCreateOptions | TaskOperationsRestoreOptions
+    opts:
+      | TaskOperationsIndexOptions
+      | TaskOperationsCreateOptions
+      | TaskOperationsRestoreOptions
+      | TaskOperationsPrePullDeploymentOptions
   ): Record<string, string> {
     return {
       env: opts.envId,
@@ -442,8 +516,12 @@ class KubernetesTaskOperations implements TaskOperations {
     return `task-index-${suffix}`;
   }
 
-  #getRunContainerName(suffix: string) {
-    return `task-run-${suffix}`;
+  #getRunContainerName(suffix: string, attemptNumber?: number) {
+    return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
+  }
+
+  #getPrePullContainerName(suffix: string) {
+    return `task-prepull-${suffix}`;
   }
 
   #createK8sApi() {
@@ -460,6 +538,7 @@ class KubernetesTaskOperations implements TaskOperations {
     return {
       core: kubeConfig.makeApiClient(k8s.CoreV1Api),
       batch: kubeConfig.makeApiClient(k8s.BatchV1Api),
+      apps: kubeConfig.makeApiClient(k8s.AppsV1Api),
     };
   }
 
@@ -503,6 +582,18 @@ class KubernetesTaskOperations implements TaskOperations {
     }
   }
 
+  async #createDaemonSet(daemonSet: k8s.V1DaemonSet, namespace: Namespace) {
+    try {
+      const res = await this.#k8sApi.apps.createNamespacedDaemonSet(
+        namespace.metadata.name,
+        daemonSet
+      );
+      logger.debug(res.body);
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
+    }
+  }
+
   #throwUnlessRecord(candidate: unknown): asserts candidate is Record<string, unknown> {
     if (typeof candidate !== "object" || candidate === null) {
       throw candidate;

apps/kubernetes-provider/src/podCleaner.ts

@@ -15,6 +15,7 @@ export class PodCleaner {
   private logger = new SimpleLogger("[PodCleaner]");
   private k8sClient: {
     core: k8s.CoreV1Api;
+    apps: k8s.AppsV1Api;
     kubeConfig: k8s.KubeConfig;
   };
 
@@ -43,6 +44,7 @@ export class PodCleaner {
 
     return {
       core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      apps: kubeConfig.makeApiClient(k8s.AppsV1Api),
       kubeConfig: kubeConfig,
     };
   }
@@ -98,6 +100,25 @@ export class PodCleaner {
       .catch(this.#handleK8sError.bind(this));
   }
 
+  async #deleteDaemonSets(opts: {
+    namespace: string;
+    dryRun?: boolean;
+    fieldSelector?: string;
+    labelSelector?: string;
+  }) {
+    return await this.k8sClient.apps
+      .deleteCollectionNamespacedDaemonSet(
+        opts.namespace,
+        undefined, // pretty
+        undefined, // continue
+        opts.dryRun ? "All" : undefined,
+        opts.fieldSelector,
+        undefined, // gracePeriodSeconds
+        opts.labelSelector
+      )
+      .catch(this.#handleK8sError.bind(this));
+  }
+
   async #deleteCompletedRuns() {
     this.logger.log("Deleting completed runs");
 
@@ -152,6 +173,28 @@ export class PodCleaner {
     });
   }
 
+  async #deleteCompletedPrePulls() {
+    this.logger.log("Deleting completed pre-pulls");
+
+    const start = Date.now();
+
+    const result = await this.#deleteDaemonSets({
+      namespace: this.namespace,
+      labelSelector: "app=task-prepull",
+    });
+
+    const elapsedMs = Date.now() - start;
+
+    if (!result) {
+      this.logger.log("Deleting completed pre-pulls: No delete result", { elapsedMs });
+      return;
+    }
+
+    const total = (result.response as any)?.body?.items?.length ?? 0;
+
+    this.logger.log("Deleting completed pre-pulls: Done", { total, elapsedMs });
+  }
+
   async start() {
     this.enabled = true;
     this.logger.log("Starting");
@@ -186,6 +229,22 @@ export class PodCleaner {
       2 * this.intervalInSeconds * 1000
     );
 
+    const completedPrePullInterval = setInterval(
+      async () => {
+        if (!this.enabled) {
+          clearInterval(completedPrePullInterval);
+          return;
+        }
+
+        try {
+          await this.#deleteCompletedPrePulls();
+        } catch (error) {
+          this.logger.error("Error deleting completed pre-pulls", error);
+        }
+      },
+      2 * this.intervalInSeconds * 1000
+    );
+
     // this.#launchTests();
   }
 

apps/webapp/app/components/admin/debugTooltip.tsx

@@ -1,14 +1,16 @@
 import { ShieldCheckIcon } from "@heroicons/react/20/solid";
+import * as Property from "~/components/primitives/PropertyTable";
 import {
   Tooltip,
   TooltipContent,
   TooltipProvider,
   TooltipTrigger,
 } from "~/components/primitives/Tooltip";
-import { useIsImpersonating } from "~/hooks/useOrganizations";
-import { useHasAdminAccess } from "~/hooks/useUser";
+import { useIsImpersonating, useOptionalOrganization } from "~/hooks/useOrganizations";
+import { useOptionalProject } from "~/hooks/useProject";
+import { useHasAdminAccess, useUser } from "~/hooks/useUser";
 
-export function AdminDebugTooltip({ children }: { children: React.ReactNode }) {
+export function AdminDebugTooltip({ children }: { children?: React.ReactNode }) {
   const hasAdminAccess = useHasAdminAccess();
   const isImpersonating = useIsImpersonating();
 
@@ -22,10 +24,46 @@ export function AdminDebugTooltip({ children }: { children: React.ReactNode }) {
         <TooltipTrigger>
           <ShieldCheckIcon className="size-5" />
         </TooltipTrigger>
-        <TooltipContent className="flex max-h-[90vh] items-center gap-1 overflow-y-auto">
-          {children}
+        <TooltipContent className="max-h-[90vh] overflow-y-auto">
+          <Content>{children}</Content>
         </TooltipContent>
       </Tooltip>
     </TooltipProvider>
   );
 }
+
+function Content({ children }: { children: React.ReactNode }) {
+  const organization = useOptionalOrganization();
+  const project = useOptionalProject();
+  const user = useUser();
+
+  return (
+    <div className="flex flex-col gap-2 divide-y divide-slate-700">
+      <Property.Table>
+        <Property.Item>
+          <Property.Label>User ID</Property.Label>
+          <Property.Value>{user.id}</Property.Value>
+        </Property.Item>
+        {organization && (
+          <Property.Item>
+            <Property.Label>Org ID</Property.Label>
+            <Property.Value>{organization.id}</Property.Value>
+          </Property.Item>
+        )}
+        {project && (
+          <>
+            <Property.Item>
+              <Property.Label>Project ID</Property.Label>
+              <Property.Value>{project.id}</Property.Value>
+            </Property.Item>
+            <Property.Item>
+              <Property.Label>Project ref</Property.Label>
+              <Property.Value>{project.ref}</Property.Value>
+            </Property.Item>
+          </>
+        )}
+      </Property.Table>
+      <div className="pt-2">{children}</div>
+    </div>
+  );
+}

apps/webapp/app/components/code/CodeBlock.tsx

@@ -241,16 +241,12 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
                 onMouseEnter={() => setMouseOver(true)}
                 onMouseLeave={() => setMouseOver(false)}
                 className={cn(
-                  "absolute  right-3 z-50 transition-colors duration-100 hover:cursor-pointer",
-                  showChrome ? "top-10" : "top-3",
+                  "absolute right-3 z-50 transition-colors duration-100 hover:cursor-pointer",
+                  showChrome ? "top-10" : "top-2.5",
                   copied ? "text-emerald-500" : "text-charcoal-500 hover:text-charcoal-300"
                 )}
               >
-                {copied ? (
-                  <ClipboardCheck className="h-5 w-5" />
-                ) : (
-                  <Clipboard className="h-5 w-5" />
-                )}
+                {copied ? <ClipboardCheck className="size-4" /> : <Clipboard className="size-4" />}
               </TooltipTrigger>
               <TooltipContent side="left" className="text-xs">
                 {copied ? "Copied" : "Copy"}
@@ -391,8 +387,8 @@ function Chrome({ title }: { title?: string }) {
 
 export function TitleRow({ title }: { title: string }) {
   return (
-    <div className="flex items-center justify-between px-4">
-      <Paragraph variant="base/bright" className="w-full border-b border-grid-dimmed py-2.5">
+    <div className="flex items-center justify-between px-3">
+      <Paragraph variant="small/bright" className="w-full border-b border-grid-dimmed py-2">
         {title}
       </Paragraph>
     </div>

apps/webapp/app/components/code/JSONEditor.tsx

@@ -107,39 +107,51 @@ export function JSONEditor(opts: JSONEditorProps) {
     }, 1500);
   }, [view]);
 
+  const showButtons = showClearButton || showCopyButton;
+
   return (
-    <div className={cn(opts.className, "grid grid-rows-[2.5rem_1fr]")}>
-      <div className="mx-3 flex items-center justify-end gap-2 border-b border-grid-dimmed">
-        {showClearButton && (
-          <Button
-            type="button"
-            variant="minimal/small"
-            TrailingIcon={TrashIcon}
-            onClick={(event) => {
-              event.preventDefault();
-              event.stopPropagation();
-              clear();
-            }}
-          >
-            Clear
-          </Button>
-        )}
-        {showCopyButton && (
-          <Button
-            type="button"
-            variant="minimal/small"
-            TrailingIcon={copied ? CheckIcon : ClipboardIcon}
-            trailingIconClassName={copied ? "text-green-500 group-hover:text-green-500" : undefined}
-            onClick={(event) => {
-              event.preventDefault();
-              event.stopPropagation();
-              copy();
-            }}
-          >
-            Copy
-          </Button>
-        )}
-      </div>
+    <div
+      className={cn(
+        opts.className,
+        "grid",
+        showButtons ? "grid-rows-[2.5rem_1fr]" : "grid-rows-[1fr]"
+      )}
+    >
+      {showButtons && (
+        <div className="mx-3 flex items-center justify-end gap-2 border-b border-grid-dimmed">
+          {showClearButton && (
+            <Button
+              type="button"
+              variant="minimal/small"
+              TrailingIcon={TrashIcon}
+              onClick={(event) => {
+                event.preventDefault();
+                event.stopPropagation();
+                clear();
+              }}
+            >
+              Clear
+            </Button>
+          )}
+          {showCopyButton && (
+            <Button
+              type="button"
+              variant="minimal/small"
+              TrailingIcon={copied ? CheckIcon : ClipboardIcon}
+              trailingIconClassName={
+                copied ? "text-green-500 group-hover:text-green-500" : undefined
+              }
+              onClick={(event) => {
+                event.preventDefault();
+                event.stopPropagation();
+                copy();
+              }}
+            >
+              Copy
+            </Button>
+          )}
+        </div>
+      )}
       <div
         className="w-full overflow-auto"
         ref={editor}

apps/webapp/app/components/navigation/SideMenu.tsx

@@ -9,6 +9,7 @@ import {
   CursorArrowRaysIcon,
   IdentificationIcon,
   KeyIcon,
+  RectangleStackIcon,
   ServerStackIcon,
   ShieldCheckIcon,
   SignalIcon,
@@ -19,9 +20,9 @@ import { DiscordIcon, SlackIcon } from "@trigger.dev/companyicons";
 import { Fragment, useEffect, useRef, useState } from "react";
 import { TaskIcon } from "~/assets/icons/TaskIcon";
 import { useFeatures } from "~/hooks/useFeatures";
-import { MatchedOrganization } from "~/hooks/useOrganizations";
-import { MatchedProject } from "~/hooks/useProject";
-import { User } from "~/models/user.server";
+import { type MatchedOrganization } from "~/hooks/useOrganizations";
+import { type MatchedProject } from "~/hooks/useProject";
+import { type User } from "~/models/user.server";
 import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route";
 import { cn } from "~/utils/cn";
 import {
@@ -46,6 +47,7 @@ import {
   projectTriggersPath,
   v3ApiKeysPath,
   v3BillingPath,
+  v3ConcurrencyPath,
   v3DeploymentsPath,
   v3EnvironmentVariablesPath,
   v3ProjectAlertsPath,
@@ -63,8 +65,7 @@ import { StepContentContainer } from "../StepContentContainer";
 import { UserProfilePhoto } from "../UserProfilePhoto";
 import { FreePlanUsage } from "../billing/v2/FreePlanUsage";
 import { Badge } from "../primitives/Badge";
-import { Button } from "../primitives/Buttons";
-import { Callout } from "../primitives/Callout";
+import { Button, LinkButton } from "../primitives/Buttons";
 import { ClipboardField } from "../primitives/ClipboardField";
 import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "../primitives/Dialog";
 import { Icon } from "../primitives/Icon";
@@ -80,7 +81,7 @@ import {
 import { StepNumber } from "../primitives/StepNumber";
 import { TextLink } from "../primitives/TextLink";
 import { SideMenuHeader } from "./SideMenuHeader";
-import { MenuCount, SideMenuItem } from "./SideMenuItem";
+import { SideMenuItem } from "./SideMenuItem";
 
 type SideMenuUser = Pick<User, "email" | "admin"> & { isImpersonating: boolean };
 type SideMenuProject = Pick<
@@ -107,6 +108,9 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
   const currentPlan = useCurrentPlan();
   const { isManagedCloud } = useFeatures();
 
+  const isV3Project = project.version === "V3";
+  const isFreeV3User = currentPlan?.v3Subscription?.isPaying === false;
+
   useEffect(() => {
     const handleScroll = () => {
       if (borderRef.current) {
@@ -195,6 +199,11 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
                   to={v3BillingPath(organization)}
                   iconColor="text-blue-600"
                   data-action="billing"
+                  badge={
+                    currentPlan?.v3Subscription?.isPaying
+                      ? currentPlan?.v3Subscription?.plan?.title
+                      : undefined
+                  }
                 />
               </>
             )}
@@ -218,14 +227,25 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
         </div>
         <div className="m-2">
           {project.version === "V2" && (
-            <Callout variant={"info"}>
-              <Paragraph variant="small">
-                This is a v2 project.{" "}
-                <TextLink href="https://trigger.dev/docs/v3/upgrading-from-v2">
-                  Upgrade to v3
+            <div className="flex flex-col gap-3 rounded border border-success/50 bg-success/10 p-3">
+              <Paragraph variant="small/bright">
+                This is a v2 project. V2 will be deprecated on January 31, 2025.{" "}
+                <TextLink
+                  className="text-text-bright underline decoration-text-dimmed underline-offset-2 transition hover:text-text-bright hover:decoration-text-bright"
+                  to="https://trigger.dev/blog/v2-end-of-life-announcement"
+                >
+                  Learn more
                 </TextLink>
+                .
               </Paragraph>
-            </Callout>
+              <LinkButton
+                variant="primary/medium"
+                to="https://trigger.dev/docs/v3/upgrading-from-v2"
+                fullWidth
+              >
+                Upgrade to v3
+              </LinkButton>
+            </div>
           )}
         </div>
         <div className="flex flex-col gap-1 border-t border-grid-bright p-1">
@@ -319,7 +339,7 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
               </Button>
             }
           />
-          {currentPlan?.v3Subscription?.isPaying === false && (
+          {isV3Project && isFreeV3User && (
             <FreePlanUsage
               to={v3BillingPath(organization)}
               percentage={currentPlan.v3Usage.usagePercentage}
@@ -374,10 +394,10 @@ function ProjectSelector({
                       title={
                         <div className="flex w-full items-center justify-between text-text-bright">
                           <span className="grow truncate text-left">{p.name}</span>
-                          {p.version === "V2" ? (
-                            <MenuCount count={p.jobCount} />
-                          ) : (
-                            <Badge variant="v3">v3</Badge>
+                          {p.version === "V2" && (
+                            <Badge variant="small" className="normal-case">
+                              v2
+                            </Badge>
                           )}
                         </div>
                       }
@@ -396,7 +416,7 @@ function ProjectSelector({
             </div>
           </Fragment>
         ))}
-        <div className="border-t border-charcoal-800 p-1">
+        <div className="border-t border-charcoal-700 p-1">
           <PopoverMenuItem to={newOrganizationPath()} title="New Organization" icon="plus" />
         </div>
       </PopoverContent>
@@ -486,7 +506,6 @@ function V2ProjectSideMenu({
         name="Jobs"
         icon="job"
         iconColor="text-indigo-500"
-        count={project.jobCount}
         to={projectPath(organization, project)}
         data-action="jobs"
       />
@@ -514,7 +533,6 @@ function V2ProjectSideMenu({
         name="HTTP endpoints"
         icon="http-endpoint"
         iconColor="text-pink-500"
-        count={project.httpEndpointCount}
         to={projectHttpEndpointsPath(organization, project)}
         data-action="httpendpoints"
       />
@@ -589,6 +607,7 @@ function V3ProjectSideMenu({
         to={v3EnvironmentVariablesPath(organization, project)}
         data-action="environment variables"
       />
+
       <SideMenuItem
         name="Deployments"
         icon={ServerStackIcon}
@@ -605,6 +624,13 @@ function V3ProjectSideMenu({
           data-action="alerts"
         />
       )}
+      <SideMenuItem
+        name="Concurrency limits"
+        icon={RectangleStackIcon}
+        iconColor="text-indigo-500"
+        to={v3ConcurrencyPath(organization, project)}
+        data-action="concurrency"
+      />
       <SideMenuItem
         name="Project settings"
         icon="settings"

apps/webapp/app/components/navigation/SideMenuItem.tsx

@@ -1,8 +1,8 @@
-import { AnchorHTMLAttributes } from "react";
+import { type AnchorHTMLAttributes } from "react";
 import { usePathName } from "~/hooks/usePathName";
 import { cn } from "~/utils/cn";
 import { LinkButton } from "../primitives/Buttons";
-import { IconNames } from "../primitives/NamedIcon";
+import { type IconNames } from "../primitives/NamedIcon";
 import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "../primitives/Tooltip";
 import { Icon } from "../primitives/Icon";
 import { IconExclamationCircle } from "@tabler/icons-react";
@@ -13,7 +13,7 @@ export function SideMenuItem({
   name,
   to,
   hasWarning,
-  count,
+  badge,
   target,
   subItem = false,
 }: {
@@ -22,7 +22,7 @@ export function SideMenuItem({
   name: string;
   to: string;
   hasWarning?: string | boolean;
-  count?: number;
+  badge?: string;
   target?: AnchorHTMLAttributes<HTMLAnchorElement>["target"];
   subItem?: boolean;
 }) {
@@ -47,20 +47,20 @@ export function SideMenuItem({
       <div className="flex w-full items-center justify-between">
         {name}
         <div className="flex items-center gap-1">
-          {count !== undefined && count > 0 && <MenuCount count={count} />}
+          {badge !== undefined && <MenuCount count={badge} />}
           {typeof hasWarning === "string" ? (
             <TooltipProvider>
               <Tooltip>
                 <TooltipTrigger>
-                  <Icon icon={IconExclamationCircle} className="h-5 w-5 text-rose-500" />
+                  <Icon icon={IconExclamationCircle} className="h-5 w-5 text-error" />
                 </TooltipTrigger>
-                <TooltipContent className="flex items-center gap-1 border border-rose-500 bg-rose-500/20 backdrop-blur-xl">
+                <TooltipContent className="flex items-center gap-1 border border-error bg-error/20 backdrop-blur-xl">
                   {hasWarning}
                 </TooltipContent>
               </Tooltip>
             </TooltipProvider>
           ) : (
-            hasWarning && <Icon icon={IconExclamationCircle} className="h-5 w-5 text-rose-500" />
+            hasWarning && <Icon icon={IconExclamationCircle} className="h-5 w-5 text-error" />
           )}
         </div>
       </div>
@@ -68,8 +68,10 @@ export function SideMenuItem({
   );
 }
 
-export function MenuCount({ count }: { count: number | string }) {
+function MenuCount({ count }: { count: number | string }) {
   return (
-    <div className="rounded-full bg-charcoal-900 px-2 py-1 text-xxs text-text-dimmed">{count}</div>
+    <div className="rounded-full bg-charcoal-900 px-2 py-1 text-xxs uppercase tracking-wider text-text-dimmed">
+      {count}
+    </div>
   );
 }

apps/webapp/app/components/primitives/Badge.tsx

@@ -8,7 +8,6 @@ const variants = {
     "grid place-items-center rounded-full px-[0.4rem] h-4 tracking-wider text-xxs bg-background-dimmed text-text-dimmed uppercase whitespace-nowrap",
   outline:
     "grid place-items-center rounded-sm px-1.5 h-5 tracking-wider text-xxs border border-dimmed text-text-dimmed uppercase whitespace-nowrap",
-  v3: "grid place-items-center rounded-full px-[0.4rem] h-5 tracking-wider text-xxs bg-charcoal-750 text-primary whitespace-nowrap",
   "outline-rounded":
     "grid place-items-center rounded-full px-1 h-4 tracking-wider text-xxs border border-blue-500 text-blue-500 uppercase whitespace-nowrap",
 };

apps/webapp/app/components/primitives/Buttons.tsx

@@ -245,12 +245,12 @@ export function ButtonContent(props: ButtonContentPropsType) {
 
 type ButtonPropsType = Pick<
   JSX.IntrinsicElements["button"],
-  "type" | "disabled" | "onClick" | "name" | "value" | "form"
+  "type" | "disabled" | "onClick" | "name" | "value" | "form" | "autoFocus"
 > &
   React.ComponentProps<typeof ButtonContent>;
 
 export const Button = forwardRef<HTMLButtonElement, ButtonPropsType>(
-  ({ type, disabled, onClick, ...props }, ref) => {
+  ({ type, disabled, autoFocus, onClick, ...props }, ref) => {
     const innerRef = useRef<HTMLButtonElement>(null);
     useImperativeHandle(ref, () => innerRef.current as HTMLButtonElement);
 
@@ -276,6 +276,7 @@ export const Button = forwardRef<HTMLButtonElement, ButtonPropsType>(
         value={props.value}
         ref={innerRef}
         form={props.form}
+        autoFocus={autoFocus}
       >
         <ButtonContent {...props} />
       </button>

apps/webapp/app/components/primitives/PageHeader.tsx

@@ -10,6 +10,7 @@ import { LoadingBarDivider } from "./LoadingBarDivider";
 import { NamedIcon } from "./NamedIcon";
 import { Paragraph } from "./Paragraph";
 import { Tabs, TabsProps } from "./Tabs";
+import { ReactNode } from "react";
 
 type WithChildren = {
   children: React.ReactNode;
@@ -34,7 +35,7 @@ export function NavBar({ children }: WithChildren) {
 }
 
 type PageTitleProps = {
-  title: string;
+  title: ReactNode;
   backButton?: {
     to: string;
     text: string;

apps/webapp/app/components/primitives/Popover.tsx

@@ -4,8 +4,8 @@ import { ChevronDownIcon, EllipsisVerticalIcon } from "@heroicons/react/24/solid
 import * as PopoverPrimitive from "@radix-ui/react-popover";
 import * as React from "react";
 import { cn } from "~/utils/cn";
-import { ButtonContentPropsType, LinkButton } from "./Buttons";
-import { Paragraph, ParagraphVariant } from "./Paragraph";
+import { type ButtonContentPropsType, LinkButton } from "./Buttons";
+import { Paragraph, type ParagraphVariant } from "./Paragraph";
 
 const Popover = PopoverPrimitive.Root;
 const PopoverTrigger = PopoverPrimitive.Trigger;
@@ -41,7 +41,7 @@ function PopoverSectionHeader({
   variant?: ParagraphVariant;
 }) {
   return (
-    <Paragraph variant={variant} className="bg-charcoal-900 px-2.5 py-1.5">
+    <Paragraph variant={variant} className="bg-charcoal-750 px-2.5 py-1.5">
       {title}
     </Paragraph>
   );
@@ -71,7 +71,10 @@ function PopoverMenuItem({
       fullWidth
       textAlignLeft
       TrailingIcon={isSelected ? "check" : undefined}
-      className={isSelected ? "bg-charcoal-750 group-hover:bg-charcoal-800" : undefined}
+      className={cn(
+        "group-hover:bg-charcoal-700",
+        isSelected ? "bg-charcoal-750 group-hover:bg-charcoal-600/50" : undefined
+      )}
     >
       {title}
     </LinkButton>

apps/webapp/app/components/primitives/PropertyTable.tsx

@@ -1,40 +1,31 @@
-import { ReactNode } from "react";
+import { type ReactNode } from "react";
 import { Paragraph } from "./Paragraph";
 import { cn } from "~/utils/cn";
 
-export function PropertyTable({
-  children,
-  className,
-}: {
+type ChildrenClassName = {
   children: ReactNode;
   className?: string;
-}) {
-  return (
-    <div className={cn("grid grid-cols-[auto,1fr] items-center gap-x-4 gap-y-2", className)}>
-      {children}
-    </div>
-  );
-}
-
-export type PropertyProps = {
-  label: ReactNode;
-  labelClassName?: string;
-  children: ReactNode;
 };
 
-export function Property({ label, labelClassName, children }: PropertyProps) {
-  return (
-    <>
-      <div className={labelClassName}>
-        {typeof label === "string" ? <Paragraph variant="small">{label}</Paragraph> : label}
-      </div>
-      <div>
-        {typeof children === "string" ? (
-          <Paragraph variant="small/bright">{children}</Paragraph>
-        ) : (
-          children
-        )}
-      </div>
-    </>
-  );
+function PropertyTable({ children, className }: { children: ReactNode; className?: string }) {
+  return <div className={cn("flex flex-col gap-y-3", className)}>{children}</div>;
 }
+
+function PropertyItem({ children, className }: ChildrenClassName) {
+  return <div className={cn("flex flex-col gap-0 text-sm", className)}>{children}</div>;
+}
+
+function PropertyLabel({ children, className }: ChildrenClassName) {
+  return <div className={cn("font-medium text-text-bright", className)}>{children}</div>;
+}
+
+function PropertyValue({ children, className }: ChildrenClassName) {
+  return <div className={cn("text-text-dimmed", className)}>{children}</div>;
+}
+
+export {
+  PropertyTable as Table,
+  PropertyItem as Item,
+  PropertyLabel as Label,
+  PropertyValue as Value,
+};

apps/webapp/app/components/primitives/Tabs.tsx

@@ -1,6 +1,11 @@
-import { NavLink, useLocation } from "@remix-run/react";
+import { Link, NavLink, useLocation } from "@remix-run/react";
 import { motion } from "framer-motion";
+import { ReactNode, useRef } from "react";
+import { useOptimisticLocation } from "~/hooks/useOptimisticLocation";
+import { ShortcutDefinition, useShortcutKeys } from "~/hooks/useShortcutKeys";
 import { cn } from "~/utils/cn";
+import { projectPubSub } from "~/v3/services/projectPubSub.server";
+import { ShortcutKey } from "./ShortcutKey";
 
 export type TabsProps = {
   tabs: {
@@ -13,28 +18,106 @@ export type TabsProps = {
 
 export function Tabs({ tabs, className, layoutId }: TabsProps) {
   return (
-    <div className={cn(`flex flex-row gap-x-6 border-b border-grid-bright`, className)}>
+    <TabContainer className={className}>
       {tabs.map((tab, index) => (
-        <NavLink key={index} to={tab.to} className="group flex flex-col items-center pt-1" end>
-          {({ isActive, isPending }) => (
-            <>
-              <span
-                className={cn(
-                  "text-sm transition duration-200",
-                  isActive || isPending ? "text-indigo-500" : "text-charcoal-200"
-                )}
-              >
-                {tab.label}
-              </span>
-              {isActive || isPending ? (
-                <motion.div layoutId={layoutId} className="mt-1 h-0.5 w-full bg-indigo-500" />
-              ) : (
-                <div className="mt-1 h-0.5 w-full bg-charcoal-500 opacity-0 transition duration-200 group-hover:opacity-100" />
-              )}
-            </>
-          )}
-        </NavLink>
+        <TabLink key={index} to={tab.to} layoutId={layoutId}>
+          {tab.label}
+        </TabLink>
       ))}
+    </TabContainer>
+  );
+}
+
+export function TabContainer({ children, className }: { children: ReactNode; className?: string }) {
+  return (
+    <div className={cn(`flex flex-row gap-x-6 border-b border-grid-bright`, className)}>
+      {children}
     </div>
   );
 }
+
+export function TabLink({
+  to,
+  children,
+  layoutId,
+}: {
+  to: string;
+  children: ReactNode;
+  layoutId: string;
+}) {
+  return (
+    <NavLink to={to} className="group flex flex-col items-center pt-1" end>
+      {({ isActive, isPending }) => {
+        return (
+          <>
+            <span
+              className={cn(
+                "text-sm transition duration-200",
+                isActive || isPending ? "text-text-bright" : "text-text-bright"
+              )}
+            >
+              {children}
+            </span>
+            {isActive || isPending ? (
+              <motion.div layoutId={layoutId} className="mt-1 h-0.5 w-full bg-indigo-500" />
+            ) : (
+              <div className="mt-1 h-0.5 w-full bg-charcoal-500 opacity-0 transition duration-200 group-hover:opacity-100" />
+            )}
+          </>
+        );
+      }}
+    </NavLink>
+  );
+}
+
+export function TabButton({
+  isActive,
+  layoutId,
+  shortcut,
+  ...props
+}: {
+  isActive: boolean;
+  shortcut?: ShortcutDefinition;
+  layoutId: string;
+} & React.ButtonHTMLAttributes<HTMLButtonElement>) {
+  const ref = useRef<HTMLButtonElement>(null);
+
+  if (shortcut) {
+    useShortcutKeys({
+      shortcut: shortcut,
+      action: () => {
+        if (ref.current) {
+          ref.current.click();
+        }
+      },
+      disabled: props.disabled,
+    });
+  }
+
+  return (
+    <button
+      className={cn("group flex flex-col items-center pt-1", props.className)}
+      ref={ref}
+      {...props}
+    >
+      <>
+        <div className="flex items-center gap-1">
+          <span
+            className={cn(
+              "text-sm transition duration-200",
+              isActive ? "text-text-bright" : "text-text-bright"
+            )}
+          >
+            {props.children}
+          </span>
+          {shortcut && <ShortcutKey className={cn("")} shortcut={shortcut} variant={"small"} />}
+        </div>
+        {isActive ? (
+          <motion.div layoutId={layoutId} className="mt-1 h-0.5 w-full bg-indigo-500" />
+        ) : (
+          <div className="mt-1 h-0.5 w-full bg-charcoal-500 opacity-0 transition duration-200 group-hover:opacity-100" />
+        )}
+      </>
+    </button>
+  );
+}

apps/webapp/app/components/runs/v3/LiveTimer.tsx

@@ -23,11 +23,11 @@ export function LiveTimer({
     }, updateInterval);
 
     return () => clearInterval(interval);
-  }, [startTime]);
+  }, [startTime, endTime]);
 
   return (
     <>
-      {formatDuration(startTime, now, {
+      {formatDuration(startTime, endTime ?? now, {
         style: "short",
         maxDecimalPoints: 0,
         units: ["d", "h", "m", "s"],

apps/webapp/app/components/runs/v3/ReplayRunDialog.tsx

@@ -1,12 +1,16 @@
-import { ArrowPathIcon } from "@heroicons/react/20/solid";
-import { Form, useFetcher, useNavigation } from "@remix-run/react";
+import { Form, useFetcher, useNavigation, useSubmit } from "@remix-run/react";
+import { useCallback, useEffect, useRef } from "react";
+import { UseDataFunctionReturn, useTypedFetcher } from "remix-typedjson";
+import { JSONEditor } from "~/components/code/JSONEditor";
+import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
 import { Button } from "~/components/primitives/Buttons";
-import {
-  DialogContent,
-  DialogDescription,
-  DialogFooter,
-  DialogHeader,
-} from "~/components/primitives/Dialog";
+import { DialogContent, DialogDescription, DialogHeader } from "~/components/primitives/Dialog";
+import { Header3 } from "~/components/primitives/Headers";
+import { InputGroup } from "~/components/primitives/InputGroup";
+import { Label } from "~/components/primitives/Label";
+import { Select, SelectItem } from "~/components/primitives/Select";
+import { ButtonSpinner, Spinner } from "~/components/primitives/Spinner";
+import { type loader } from "~/routes/resources.taskruns.$runParam.replay";
 
 type ReplayRunDialogProps = {
   runFriendlyId: string;
@@ -14,31 +18,142 @@ type ReplayRunDialogProps = {
 };
 
 export function ReplayRunDialog({ runFriendlyId, failedRedirect }: ReplayRunDialogProps) {
-  const navigation = useNavigation();
-
-  const formAction = `/resources/taskruns/${runFriendlyId}/replay`;
-  const isLoading = navigation.formAction === formAction;
-
   return (
-    <DialogContent key="replay">
-      <DialogHeader>Replay this run?</DialogHeader>
-      <DialogDescription>
-        Replaying a run will create a new run with the same payload and environment as the original.
-      </DialogDescription>
-      <DialogFooter>
-        <Form action={formAction} method="post">
-          <input type="hidden" name="failedRedirect" value={failedRedirect} />
-          <Button
-            type="submit"
-            variant="primary/small"
-            LeadingIcon={isLoading ? "spinner-white" : ArrowPathIcon}
-            disabled={isLoading}
-            shortcut={{ modifiers: ["meta"], key: "enter" }}
-          >
-            {isLoading ? "Replaying..." : "Replay run"}
-          </Button>
-        </Form>
-      </DialogFooter>
+    <DialogContent key={`replay`} className="md:max-w-3xl">
+      <ReplayContent runFriendlyId={runFriendlyId} failedRedirect={failedRedirect} />
     </DialogContent>
   );
 }
+
+function ReplayContent({ runFriendlyId, failedRedirect }: ReplayRunDialogProps) {
+  const fetcher = useTypedFetcher<typeof loader>();
+  const isLoading = fetcher.state !== "idle";
+
+  useEffect(() => {
+    fetcher.load(`/resources/taskruns/${runFriendlyId}/replay`);
+  }, [runFriendlyId]);
+
+  return (
+    <>
+      <DialogHeader>Replay this run</DialogHeader>
+      {isLoading ? (
+        <div className="grid place-items-center p-6">
+          <Spinner />
+        </div>
+      ) : fetcher.data ? (
+        <ReplayForm
+          {...fetcher.data}
+          failedRedirect={failedRedirect}
+          runFriendlyId={runFriendlyId}
+        />
+      ) : (
+        <>Failed to get run data</>
+      )}
+    </>
+  );
+}
+
+function ReplayForm({
+  payload,
+  payloadType,
+  environment,
+  environments,
+  failedRedirect,
+  runFriendlyId,
+}: UseDataFunctionReturn<typeof loader> & { failedRedirect: string; runFriendlyId: string }) {
+  const navigation = useNavigation();
+  const submit = useSubmit();
+  const currentJson = useRef<string>(payload);
+  const formAction = `/resources/taskruns/${runFriendlyId}/replay`;
+  const isSubmitting = navigation.formAction === formAction;
+
+  const editablePayload =
+    payloadType === "application/json" || payloadType === "application/super+json";
+
+  const submitForm = useCallback(
+    (e: React.FormEvent<HTMLFormElement>) => {
+      const formData = new FormData(e.currentTarget);
+      const data: Record<string, string> = {
+        environment: formData.get("environment") as string,
+        failedRedirect: formData.get("failedRedirect") as string,
+      };
+
+      if (editablePayload) {
+        data.payload = currentJson.current;
+      }
+
+      submit(data, {
+        action: formAction,
+        method: "post",
+      });
+      e.preventDefault();
+    },
+    [currentJson]
+  );
+
+  return (
+    <Form action={formAction} method="post" onSubmit={(e) => submitForm(e)} className="pt-2">
+      {editablePayload ? (
+        <>
+          <Header3 spacing>Payload</Header3>
+          <div className="mb-3 max-h-[70vh] overflow-y-auto rounded-sm border border-grid-dimmed bg-charcoal-900 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600">
+            <JSONEditor
+              autoFocus
+              defaultValue={currentJson.current}
+              readOnly={false}
+              basicSetup
+              onChange={(v) => {
+                currentJson.current = v;
+              }}
+              showClearButton={false}
+              showCopyButton={false}
+              height="100%"
+              min-height="100%"
+              max-height="100%"
+            />
+          </div>
+        </>
+      ) : null}
+      <InputGroup>
+        <Label>Environment</Label>
+        <Select
+          id="environment"
+          name="environment"
+          placeholder="Select an environment"
+          defaultValue={environment.id}
+          items={environments}
+          dropdownIcon
+          variant="tertiary/medium"
+          className="w-fit pl-2"
+          text={(value) => {
+            const env = environments.find((env) => env.id === value)!;
+            return (
+              <div className="flex items-center pr-2">
+                <EnvironmentLabel environment={env} userName={env.userName} />
+              </div>
+            );
+          }}
+        >
+          {(matches) =>
+            matches.map((env) => (
+              <SelectItem key={env.id} value={env.id}>
+                <EnvironmentLabel environment={env} userName={env.userName} />
+              </SelectItem>
+            ))
+          }
+        </Select>
+      </InputGroup>
+      <input type="hidden" name="failedRedirect" value={failedRedirect} />
+      <Button
+        type="submit"
+        variant="primary/medium"
+        LeadingIcon={isSubmitting ? ButtonSpinner : undefined}
+        disabled={isSubmitting}
+        shortcut={{ modifiers: ["meta"], key: "enter", enabledOnInputElements: true }}
+        className="mt-5"
+      >
+        {isSubmitting ? "Replaying..." : "Replay run"}
+      </Button>
+    </Form>
+  );
+}

apps/webapp/app/components/runs/v3/SpanEvents.tsx

@@ -6,7 +6,7 @@ import {
 import { CodeBlock } from "~/components/code/CodeBlock";
 import { Callout } from "~/components/primitives/Callout";
 import { DateTimeAccurate } from "~/components/primitives/DateTime";
-import { Header2 } from "~/components/primitives/Headers";
+import { Header2, Header3 } from "~/components/primitives/Headers";
 import { Paragraph } from "~/components/primitives/Paragraph";
 
 type SpanEventsProps = {
@@ -34,8 +34,8 @@ function SpanEventHeader({
 }) {
   return (
     <div className="flex items-center justify-between">
-      <Header2 className={titleClassName}>{title}</Header2>
-      <Paragraph variant="small">
+      <Header3 className={titleClassName}>{title}</Header3>
+      <Paragraph variant="extra-small">
         <DateTimeAccurate date={time} />
       </Paragraph>
     </div>
@@ -57,7 +57,7 @@ function SpanEvent({ spanEvent }: { spanEvent: OtelSpanEvent }) {
   );
 }
 
-function SpanEventError({
+export function SpanEventError({
   spanEvent,
   exception,
 }: {
@@ -65,7 +65,7 @@ function SpanEventError({
   exception: ExceptionEventProperties;
 }) {
   return (
-    <div className="flex flex-col gap-2 rounded-sm border border-rose-500/50 p-3">
+    <div className="flex flex-col gap-2 rounded-sm border border-rose-500/50 px-3 pb-3 pt-2">
       <SpanEventHeader
         title={exception.type ?? "Error"}
         time={spanEvent.time}

apps/webapp/app/components/runs/v3/TaskRunStatus.tsx

@@ -67,15 +67,19 @@ const taskRunStatusDescriptions: Record<TaskRunStatus, string> = {
   EXPIRED: "Task has surpassed its ttl and won't be executed",
 };
 
-export const QUEUED_STATUSES: TaskRunStatus[] = ["PENDING", "WAITING_FOR_DEPLOY", "DELAYED"];
+export const QUEUED_STATUSES = [
+  "PENDING",
+  "WAITING_FOR_DEPLOY",
+  "DELAYED",
+] satisfies TaskRunStatus[];
 
-export const RUNNING_STATUSES: TaskRunStatus[] = [
+export const RUNNING_STATUSES = [
   "EXECUTING",
   "RETRYING_AFTER_FAILURE",
   "WAITING_TO_RESUME",
-];
+] satisfies TaskRunStatus[];
 
-export const FINISHED_STATUSES: TaskRunStatus[] = [
+export const FINISHED_STATUSES = [
   "COMPLETED_SUCCESSFULLY",
   "CANCELED",
   "COMPLETED_WITH_ERRORS",
@@ -83,7 +87,9 @@ export const FINISHED_STATUSES: TaskRunStatus[] = [
   "SYSTEM_FAILURE",
   "CRASHED",
   "EXPIRED",
-];
+] satisfies TaskRunStatus[];
+
+export type FINISHED_STATUSES = (typeof FINISHED_STATUSES)[number];
 
 export function descriptionForTaskRunStatus(status: TaskRunStatus): string {
   return taskRunStatusDescriptions[status];

apps/webapp/app/lib.es5.d.ts

@@ -13,3 +13,7 @@ interface Array<T> {
 export type Prettify<T> = {
   [K in keyof T]: T[K];
 } & {};
+
+export type EnsureProperty<T, K extends keyof T> = T & {
+  [P in K]-?: NonNullable<T[P]>;
+};

apps/webapp/app/models/runtimeEnvironment.server.ts

@@ -116,7 +116,7 @@ export async function disconnectSession(environmentId: string) {
   return session;
 }
 
-type DisplayableInputEnvironment = Prisma.RuntimeEnvironmentGetPayload<{
+export type DisplayableInputEnvironment = Prisma.RuntimeEnvironmentGetPayload<{
   select: {
     id: true;
     type: true;

apps/webapp/app/models/task.server.ts

@@ -1,5 +1,6 @@
-import type { JobRun, Task, TaskAttempt } from "@trigger.dev/database";
+import type { JobRun, Task, TaskAttempt, TaskTriggerSource } from "@trigger.dev/database";
 import { CachedTask, ServerTask } from "@trigger.dev/core";
+import { PrismaClientOrTransaction, sqlDatabaseSchema } from "~/db.server";
 
 export type TaskWithAttempts = Task & {
   attempts: TaskAttempt[];
@@ -116,3 +117,22 @@ function prepareTaskForCaching(task: TaskForCaching): CachedTask {
 function calculateCachedTaskSize(task: CachedTask): number {
   return JSON.stringify(task).length;
 }
+
+/**
+ *
+ * @param prisma An efficient query to get all task identifiers for a project.
+ * It has indexes for fast performance.
+ * It does NOT care about versions, so includes all tasks ever created.
+ */
+export function getAllTaskIdentifiers(prisma: PrismaClientOrTransaction, projectId: string) {
+  return prisma.$queryRaw<
+    {
+      slug: string;
+      triggerSource: TaskTriggerSource;
+    }[]
+  >`
+    SELECT DISTINCT(slug), "triggerSource"
+    FROM ${sqlDatabaseSchema}."BackgroundWorkerTask"
+    WHERE "projectId" = ${projectId}
+    ORDER BY slug ASC;`;
+}

apps/webapp/app/models/taskRunTag.server.ts

@@ -1,6 +1,8 @@
 import { prisma } from "~/db.server";
 import { generateFriendlyId } from "~/v3/friendlyIdentifiers";
 
+export const MAX_TAGS_PER_RUN = 5;
+
 export async function createTag({ tag, projectId }: { tag: string; projectId: string }) {
   if (tag.trim().length === 0) return;
   return prisma.taskRunTag.upsert({

apps/webapp/app/presenters/RunPresenter.server.ts

@@ -4,7 +4,7 @@ import {
   EventSpecificationSchema,
   StyleSchema,
 } from "@trigger.dev/core";
-import { PrismaClient, prisma } from "~/db.server";
+import { $replica, PrismaClient, prisma } from "~/db.server";
 import { isRunCompleted, runBasicStatus } from "~/models/jobRun.server";
 import { mergeProperties } from "~/utils/mergeProperties.server";
 import { taskListToTree } from "~/utils/taskListToTree";
@@ -110,7 +110,7 @@ export class RunPresenter {
   }
 
   query({ id, userId }: RunOptions) {
-    return this.#prismaClient.jobRun.findFirst({
+    return $replica.jobRun.findFirst({
       select: {
         id: true,
         number: true,
@@ -198,6 +198,7 @@ export class RunPresenter {
           orderBy: {
             createdAt: "asc",
           },
+          take: 1000,
         },
         runConnections: {
           select: {

apps/webapp/app/presenters/TaskDetailsPresenter.server.ts

@@ -1,5 +1,5 @@
 import { RedactSchema, StyleSchema } from "@trigger.dev/core";
-import { PrismaClient, prisma } from "~/db.server";
+import { $replica, PrismaClient, prisma } from "~/db.server";
 import { mergeProperties } from "~/utils/mergeProperties.server";
 import { Redactor } from "~/utils/redactor";
 
@@ -16,7 +16,7 @@ export class TaskDetailsPresenter {
   }
 
   public async call({ id, userId }: DetailsProps) {
-    const task = await this.#prismaClient.task.findFirst({
+    const task = await $replica.task.findFirst({
       select: {
         id: true,
         displayKey: true,

apps/webapp/app/presenters/v3/ConcurrencyPresenter.server.ts

@@ -0,0 +1,102 @@
+import { QUEUED_STATUSES } from "~/components/runs/v3/TaskRunStatus";
+import { Prisma, sqlDatabaseSchema } from "~/db.server";
+import { type Project } from "~/models/project.server";
+import {
+  displayableEnvironment,
+  type DisplayableInputEnvironment,
+} from "~/models/runtimeEnvironment.server";
+import { type User } from "~/models/user.server";
+import { getLimit } from "~/services/platform.v3.server";
+import { filterOrphanedEnvironments, sortEnvironments } from "~/utils/environmentSort";
+import { concurrencyTracker } from "~/v3/services/taskRunConcurrencyTracker.server";
+import { BasePresenter } from "./basePresenter.server";
+
+export type Environment = Awaited<
+  ReturnType<ConcurrencyPresenter["environmentConcurrency"]>
+>[number];
+
+export class ConcurrencyPresenter extends BasePresenter {
+  public async call({ userId, projectSlug }: { userId: User["id"]; projectSlug: Project["slug"] }) {
+    const project = await this._replica.project.findFirst({
+      select: {
+        id: true,
+        organizationId: true,
+        environments: {
+          select: {
+            id: true,
+            apiKey: true,
+            pkApiKey: true,
+            type: true,
+            slug: true,
+            updatedAt: true,
+            orgMember: {
+              select: {
+                user: { select: { id: true, name: true, displayName: true } },
+              },
+            },
+            maximumConcurrencyLimit: true,
+          },
+        },
+      },
+      where: {
+        slug: projectSlug,
+        organization: {
+          members: {
+            some: {
+              userId,
+            },
+          },
+        },
+      },
+    });
+
+    if (!project) {
+      throw new Error(`Project not found: ${projectSlug}`);
+    }
+
+    return {
+      environments: this.environmentConcurrency(
+        project.id,
+        userId,
+        filterOrphanedEnvironments(project.environments)
+      ),
+    };
+  }
+
+  async environmentConcurrency(
+    projectId: string,
+    userId: string,
+    environments: (DisplayableInputEnvironment & { maximumConcurrencyLimit: number })[]
+  ) {
+    const environmentConcurrency = await concurrencyTracker.environmentConcurrentRunCounts(
+      projectId,
+      environments.map((env) => env.id)
+    );
+
+    const queued = await this._replica.$queryRaw<
+      {
+        runtimeEnvironmentId: string;
+        count: BigInt;
+      }[]
+    >`
+SELECT
+    "runtimeEnvironmentId",
+    COUNT(*)
+FROM
+    ${sqlDatabaseSchema}."TaskRun" as tr
+WHERE
+    tr."projectId" = ${projectId}
+    AND tr."status" = ANY(ARRAY[${Prisma.join(QUEUED_STATUSES)}]::\"TaskRunStatus\"[])
+GROUP BY
+    tr."runtimeEnvironmentId";`;
+
+    const sortedEnvironments = sortEnvironments(environments).map((environment) => ({
+      ...displayableEnvironment(environment, userId),
+      concurrencyLimit: environment.maximumConcurrencyLimit,
+      concurrency: environmentConcurrency[environment.id] ?? 0,
+      queued: Number(queued.find((q) => q.runtimeEnvironmentId === environment.id)?.count ?? 0),
+    }));
+
+    return sortedEnvironments;
+  }
+}

apps/webapp/app/presenters/v3/EditSchedulePresenter.server.ts

@@ -1,6 +1,8 @@
 import { RuntimeEnvironmentType } from "@trigger.dev/database";
 import { PrismaClient, prisma } from "~/db.server";
 import { displayableEnvironment } from "~/models/runtimeEnvironment.server";
+import { logger } from "~/services/logger.server";
+import { filterOrphanedEnvironments } from "~/utils/environmentSort";
 import { getTimezones } from "~/utils/timezones.server";
 
 type EditScheduleOptions = {
@@ -68,9 +70,11 @@ export class EditSchedulePresenter {
       },
     });
 
-    const possibleEnvironments = project.environments.map((environment) => {
-      return displayableEnvironment(environment, userId);
-    });
+    const possibleEnvironments = filterOrphanedEnvironments(project.environments).map(
+      (environment) => {
+        return displayableEnvironment(environment, userId);
+      }
+    );
 
     return {
       possibleTasks: possibleTasks.map((task) => task.slug),
@@ -115,13 +119,16 @@ export class EditSchedulePresenter {
     return {
       ...schedule,
       cron: schedule.generatorExpression,
-      environments: schedule.instances.map((instance) => {
+      environments: schedule.instances.flatMap((instance) => {
         const environment = possibleEnvironments.find((env) => env.id === instance.environmentId);
         if (!environment) {
-          throw new Error(`Environment with id ${instance.environmentId} not found`);
+          logger.error(
+            `EditSchedulePresenter: environment with id ${instance.environmentId} not found`
+          );
+          return [];
         }
 
-        return environment;
+        return [environment];
       }),
     };
   }

apps/webapp/app/presenters/v3/EnvironmentVariablesPresenter.server.ts

@@ -1,7 +1,7 @@
 import { PrismaClient, prisma } from "~/db.server";
 import { Project } from "~/models/project.server";
 import { User } from "~/models/user.server";
-import { sortEnvironments } from "~/utils/environmentSort";
+import { filterOrphanedEnvironments, sortEnvironments } from "~/utils/environmentSort";
 import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server";
 
 type Result = Awaited<ReturnType<EnvironmentVariablesPresenter["call"]>>;
@@ -78,11 +78,11 @@ export class EnvironmentVariablesPresenter {
       where: {
         project: {
           slug: projectSlug,
-        }
+        },
       },
     });
 
-    const sortedEnvironments = sortEnvironments(environments);
+    const sortedEnvironments = sortEnvironments(filterOrphanedEnvironments(environments));
 
     const repository = new EnvironmentVariablesRepository(this.#prismaClient);
     const variables = await repository.getProject(project.id);
@@ -104,12 +104,12 @@ export class EnvironmentVariablesPresenter {
           }, {} as Record<string, { value: string | undefined; environment: { type: string; id: string } }>),
         };
       }),
-      environments: sortedEnvironments.filter(
-        (e) => e.orgMember?.userId === userId || e.orgMember === null
-      ).map((environment) => ({
-        id: environment.id,
-        type: environment.type,
-      })),
+      environments: sortedEnvironments
+        .filter((e) => e.orgMember?.userId === userId || e.orgMember === null)
+        .map((environment) => ({
+          id: environment.id,
+          type: environment.type,
+        })),
       hasStaging: environments.some((environment) => environment.type === "STAGING"),
     };
   }

apps/webapp/app/presenters/v3/RunListPresenter.server.ts

@@ -6,6 +6,7 @@ import { sqlDatabaseSchema } from "~/db.server";
 import { displayableEnvironment } from "~/models/runtimeEnvironment.server";
 import { isCancellableRunStatus } from "~/v3/taskStatus";
 import { BasePresenter } from "./basePresenter.server";
+import { getAllTaskIdentifiers } from "~/models/task.server";
 
 export type RunListOptions = {
   userId?: string;
@@ -97,16 +98,7 @@ export class RunListPresenter extends BasePresenter {
     });
 
     //get all possible tasks
-    const possibleTasksAsync = this._replica.$queryRaw<
-      {
-        slug: string;
-        triggerSource: TaskTriggerSource;
-      }[]
-    >`
-    SELECT DISTINCT(slug), "triggerSource"
-    FROM ${sqlDatabaseSchema}."BackgroundWorkerTask"
-    WHERE "projectId" = ${project.id}
-    ORDER BY slug ASC;`;
+    const possibleTasksAsync = getAllTaskIdentifiers(this._replica, project.id);
 
     //get possible bulk actions
     const bulkActionsAsync = this._replica.bulkActionGroup.findMany({

apps/webapp/app/presenters/v3/RunPresenter.server.ts

@@ -1,5 +1,6 @@
 import { millisecondsToNanoseconds } from "@trigger.dev/core/v3";
 import { createTreeFromFlatItems, flattenTree } from "~/components/primitives/TreeView/TreeView";
+import { FINISHED_STATUSES } from "~/components/runs/v3/TaskRunStatus";
 import { PrismaClient, prisma } from "~/db.server";
 import { getUsername } from "~/utils/username";
 import { eventRepository } from "~/v3/eventRepository.server";
@@ -33,6 +34,9 @@ export class RunPresenter {
         traceId: true,
         spanId: true,
         friendlyId: true,
+        status: true,
+        completedAt: true,
+        logsDeletedAt: true,
         runtimeEnvironment: {
           select: {
             id: true,
@@ -71,6 +75,11 @@ export class RunPresenter {
           number: run.number,
           friendlyId: run.friendlyId,
           traceId: run.traceId,
+          spanId: run.spanId,
+          status: run.status,
+          isFinished: FINISHED_STATUSES.includes(run.status),
+          completedAt: run.completedAt,
+          logsDeletedAt: run.logsDeletedAt,
           environment: {
             id: run.runtimeEnvironment.id,
             organizationId: run.runtimeEnvironment.organizationId,
@@ -127,6 +136,11 @@ export class RunPresenter {
         number: run.number,
         friendlyId: run.friendlyId,
         traceId: run.traceId,
+        spanId: run.spanId,
+        status: run.status,
+        isFinished: FINISHED_STATUSES.includes(run.status),
+        completedAt: run.completedAt,
+        logsDeletedAt: run.logsDeletedAt,
         environment: {
           id: run.runtimeEnvironment.id,
           organizationId: run.runtimeEnvironment.organizationId,

apps/webapp/app/presenters/v3/SpanPresenter.server.ts

@@ -1,10 +1,13 @@
-import { prettyPrintPacket } from "@trigger.dev/core/v3";
-import { PrismaClient, prisma } from "~/db.server";
+import { MachinePresetName, prettyPrintPacket, TaskRunError } from "@trigger.dev/core/v3";
+import { FINISHED_STATUSES, RUNNING_STATUSES } from "~/components/runs/v3/TaskRunStatus";
 import { eventRepository } from "~/v3/eventRepository.server";
+import { machinePresetFromName } from "~/v3/machinePresets.server";
+import { FINAL_ATTEMPT_STATUSES } from "~/v3/taskStatus";
 import { BasePresenter } from "./basePresenter.server";
 
 type Result = Awaited<ReturnType<SpanPresenter["call"]>>;
-export type Span = NonNullable<Result>["event"];
+export type Span = NonNullable<NonNullable<Result>["span"]>;
+export type SpanRun = NonNullable<NonNullable<Result>["run"]>;
 
 export class SpanPresenter extends BasePresenter {
   public async call({
@@ -30,6 +33,244 @@ export class SpanPresenter extends BasePresenter {
       throw new Error("Project not found");
     }
 
+    const run = await this.getRun(spanId);
+    if (run) {
+      return {
+        type: "run" as const,
+        run,
+      };
+    }
+
+    //get the run
+    const span = await this.getSpan(runFriendlyId, spanId);
+
+    if (!span) {
+      throw new Error("Span not found");
+    }
+
+    return {
+      type: "span" as const,
+      span,
+    };
+  }
+
+  async getRun(spanId: string) {
+    const run = await this._replica.taskRun.findFirst({
+      select: {
+        id: true,
+        traceId: true,
+        //metadata
+        number: true,
+        taskIdentifier: true,
+        friendlyId: true,
+        isTest: true,
+        tags: {
+          select: {
+            name: true,
+          },
+        },
+        machinePreset: true,
+        lockedToVersion: {
+          select: {
+            version: true,
+            sdkVersion: true,
+          },
+        },
+        //status + duration
+        status: true,
+        startedAt: true,
+        createdAt: true,
+        updatedAt: true,
+        queuedAt: true,
+        completedAt: true,
+        logsDeletedAt: true,
+        //idempotency
+        idempotencyKey: true,
+        //delayed
+        delayUntil: true,
+        //ttl
+        ttl: true,
+        expiredAt: true,
+        //queue
+        queue: true,
+        concurrencyKey: true,
+        //schedule
+        schedule: {
+          select: {
+            friendlyId: true,
+            generatorExpression: true,
+            timezone: true,
+            generatorDescription: true,
+          },
+        },
+        //usage
+        baseCostInCents: true,
+        costInCents: true,
+        usageDurationMs: true,
+        //env
+        runtimeEnvironment: {
+          select: { id: true, slug: true, type: true },
+        },
+        payload: true,
+        payloadType: true,
+        maxAttempts: true,
+        project: {
+          include: {
+            organization: true,
+          },
+        },
+        lockedBy: {
+          select: {
+            filePath: true,
+            exportName: true,
+          },
+        },
+      },
+      where: {
+        spanId,
+      },
+    });
+
+    if (!run) {
+      return;
+    }
+
+    const finishedAttempt = await this._replica.taskRunAttempt.findFirst({
+      select: {
+        output: true,
+        outputType: true,
+        error: true,
+      },
+      where: {
+        status: { in: FINAL_ATTEMPT_STATUSES },
+        taskRunId: run.id,
+      },
+      orderBy: {
+        createdAt: "desc",
+      },
+    });
+
+    const output =
+      finishedAttempt === null
+        ? undefined
+        : finishedAttempt.outputType === "application/store"
+        ? `/resources/packets/${run.runtimeEnvironment.id}/${finishedAttempt.output}`
+        : typeof finishedAttempt.output !== "undefined" && finishedAttempt.output !== null
+        ? await prettyPrintPacket(finishedAttempt.output, finishedAttempt.outputType ?? undefined)
+        : undefined;
+
+    const payload =
+      run.payloadType === "application/store"
+        ? `/resources/packets/${run.runtimeEnvironment.id}/${run.payload}`
+        : typeof run.payload !== "undefined" && run.payload !== null
+        ? await prettyPrintPacket(run.payload, run.payloadType ?? undefined)
+        : undefined;
+
+    let error: TaskRunError | undefined = undefined;
+    if (finishedAttempt?.error) {
+      const result = TaskRunError.safeParse(finishedAttempt.error);
+      if (result.success) {
+        error = result.data;
+      } else {
+        error = {
+          type: "CUSTOM_ERROR",
+          raw: JSON.stringify(finishedAttempt.error),
+        };
+      }
+    }
+
+    const span = await eventRepository.getSpan(spanId, run.traceId);
+
+    const context = {
+      task: {
+        id: run.taskIdentifier,
+        filePath: run.lockedBy?.filePath,
+        exportName: run.lockedBy?.exportName,
+      },
+      run: {
+        id: run.friendlyId,
+        createdAt: run.createdAt,
+        tags: run.tags.map((tag) => tag.name),
+        isTest: run.isTest,
+        idempotencyKey: run.idempotencyKey ?? undefined,
+        startedAt: run.startedAt ?? run.createdAt,
+        durationMs: run.usageDurationMs,
+        costInCents: run.costInCents,
+        baseCostInCents: run.baseCostInCents,
+        maxAttempts: run.maxAttempts ?? undefined,
+        version: run.lockedToVersion?.version,
+      },
+      queue: {
+        name: run.queue,
+      },
+      environment: {
+        id: run.runtimeEnvironment.id,
+        slug: run.runtimeEnvironment.slug,
+        type: run.runtimeEnvironment.type,
+      },
+      organization: {
+        id: run.project.organization.id,
+        slug: run.project.organization.slug,
+        name: run.project.organization.title,
+      },
+      project: {
+        id: run.project.id,
+        ref: run.project.externalRef,
+        slug: run.project.slug,
+        name: run.project.name,
+      },
+      machine: run.machinePreset
+        ? machinePresetFromName(run.machinePreset as MachinePresetName)
+        : undefined,
+    };
+
+    return {
+      friendlyId: run.friendlyId,
+      status: run.status,
+      createdAt: run.createdAt,
+      startedAt: run.startedAt,
+      updatedAt: run.updatedAt,
+      delayUntil: run.delayUntil,
+      expiredAt: run.expiredAt,
+      completedAt: run.completedAt,
+      logsDeletedAt: run.logsDeletedAt,
+      ttl: run.ttl,
+      taskIdentifier: run.taskIdentifier,
+      version: run.lockedToVersion?.version,
+      sdkVersion: run.lockedToVersion?.sdkVersion,
+      isTest: run.isTest,
+      environmentId: run.runtimeEnvironment.id,
+      schedule: run.schedule
+        ? {
+            friendlyId: run.schedule.friendlyId,
+            generatorExpression: run.schedule.generatorExpression,
+            description: run.schedule.generatorDescription,
+            timezone: run.schedule.timezone,
+          }
+        : undefined,
+      queue: {
+        name: run.queue,
+        isCustomQueue: !run.queue.startsWith("task/"),
+        concurrencyKey: run.concurrencyKey,
+      },
+      tags: run.tags.map((tag) => tag.name),
+      baseCostInCents: run.baseCostInCents,
+      costInCents: run.costInCents,
+      totalCostInCents: run.costInCents + run.baseCostInCents,
+      usageDurationMs: run.usageDurationMs,
+      isFinished: FINISHED_STATUSES.includes(run.status),
+      isRunning: RUNNING_STATUSES.includes(run.status),
+      payload,
+      payloadType: run.payloadType,
+      output,
+      outputType: finishedAttempt?.outputType ?? "application/json",
+      error,
+      links: span?.links,
+      context: JSON.stringify(context, null, 2),
+    };
+  }
+
+  async getSpan(runFriendlyId: string, spanId: string) {
     const run = await this._prisma.taskRun.findFirst({
       select: {
         traceId: true,
@@ -49,31 +290,11 @@ export class SpanPresenter extends BasePresenter {
       return;
     }
 
-    const output =
-      span.outputType === "application/store"
-        ? `/resources/packets/${span.environmentId}/${span.output}`
-        : typeof span.output !== "undefined"
-        ? await prettyPrintPacket(span.output, span.outputType ?? undefined)
-        : undefined;
-
-    const payload =
-      span.payloadType === "application/store"
-        ? `/resources/packets/${span.environmentId}/${span.payload}`
-        : typeof span.payload !== "undefined" && span.payload !== null
-        ? await prettyPrintPacket(span.payload, span.payloadType ?? undefined)
-        : undefined;
-
     return {
-      event: {
-        ...span,
-        events: span.events,
-        output,
-        outputType: span.outputType ?? "application/json",
-        payload,
-        payloadType: span.payloadType ?? "application/json",
-        properties: span.properties ? JSON.stringify(span.properties, null, 2) : undefined,
-        showActionBar: span.show?.actions === true,
-      },
+      ...span,
+      events: span.events,
+      properties: span.properties ? JSON.stringify(span.properties, null, 2) : undefined,
+      showActionBar: span.show?.actions === true,
     };
   }
 }

apps/webapp/app/presenters/v3/TaskListPresenter.server.ts

@@ -1,9 +1,9 @@
+import { Prisma } from "@trigger.dev/database";
 import type {
   RuntimeEnvironmentType,
   TaskTriggerSource,
   TaskRunStatus as TaskRunStatusType,
 } from "@trigger.dev/database";
-import { Prisma } from "@trigger.dev/database";
 import { QUEUED_STATUSES, RUNNING_STATUSES } from "~/components/runs/v3/TaskRunStatus";
 import { sqlDatabaseSchema } from "~/db.server";
 import type { Organization } from "~/models/organization.server";
@@ -20,6 +20,7 @@ import { logger } from "~/services/logger.server";
 import { BasePresenter } from "./basePresenter.server";
 import { TaskRunStatus } from "~/database-types";
 import { CURRENT_DEPLOYMENT_LABEL } from "~/consts";
+import { concurrencyTracker } from "~/v3/services/taskRunConcurrencyTracker.server";
 
 export type Task = {
   slug: string;
@@ -114,7 +115,7 @@ export class TaskListPresenter extends BasePresenter {
     JOIN ${sqlDatabaseSchema}."BackgroundWorkerTask" tasks ON tasks."workerId" = workers.id
     ORDER BY slug ASC;`;
 
-    //group by the task identifier (task.slug). Add the latestRun and add all the environments.
+    //group by the task identifier (task.slug).
     const outputTasks = tasks.reduce((acc, task) => {
       const environment = project.environments.find((env) => env.id === task.runtimeEnvironmentId);
       if (!environment) {
@@ -251,51 +252,40 @@ export class TaskListPresenter extends BasePresenter {
       return {};
     }
 
-    const statuses = await this._replica.$queryRaw<
+    const concurrencies = await concurrencyTracker.taskConcurrentRunCounts(projectId, tasks);
+
+    const queued = await this._replica.$queryRaw<
       {
         taskIdentifier: string;
-        status: TaskRunStatusType;
         count: BigInt;
       }[]
     >`
     SELECT 
-    tr."taskIdentifier", 
-    tr."status",
+    tr."taskIdentifier",
     COUNT(*) 
   FROM 
     ${sqlDatabaseSchema}."TaskRun" as tr
   WHERE 
     tr."taskIdentifier" IN (${Prisma.join(tasks)})
     AND tr."projectId" = ${projectId}
-    AND tr."status" IN ('PENDING', 'WAITING_FOR_DEPLOY', 'EXECUTING', 'RETRYING_AFTER_FAILURE', 'WAITING_TO_RESUME')
+    AND tr."status" = ANY(ARRAY[${Prisma.join(QUEUED_STATUSES)}]::\"TaskRunStatus\"[])
   GROUP BY 
-    tr."taskIdentifier", 
-    tr."status"
+    tr."taskIdentifier"
   ORDER BY 
-    tr."taskIdentifier" ASC,
-    tr."status" ASC;`;
+    tr."taskIdentifier" ASC`;
 
-    return statuses.reduce((acc, a) => {
-      let existingTask = acc[a.taskIdentifier];
+    //create an object combining the queued and concurrency counts
+    const result: Record<string, { queued: number; running: number }> = {};
+    for (const task of tasks) {
+      const concurrency = concurrencies[task] ?? 0;
+      const queuedCount = queued.find((q) => q.taskIdentifier === task)?.count ?? 0;
 
-      if (!existingTask) {
-        existingTask = {
-          queued: 0,
-          running: 0,
-        };
-
-        acc[a.taskIdentifier] = existingTask;
-      }
-
-      if (QUEUED_STATUSES.includes(a.status)) {
-        existingTask.queued += Number(a.count);
-      }
-      if (RUNNING_STATUSES.includes(a.status)) {
-        existingTask.running += Number(a.count);
-      }
-
-      return acc;
-    }, {} as Record<string, { queued: number; running: number }>);
+      result[task] = {
+        queued: Number(queuedCount),
+        running: concurrency,
+      };
+    }
+    return result;
   }
 
   async #getAverageDurations(tasks: string[], projectId: string) {

apps/webapp/app/presenters/v3/UsagePresenter.server.ts

@@ -27,6 +27,10 @@ export type UsageSeriesData = {
 
 export class UsagePresenter extends BasePresenter {
   public async call({ organizationId, startDate }: Options) {
+    if (isNaN(startDate.getTime())) {
+      throw new Error("Invalid start date");
+    }
+
     //month period
     const startOfMonth = new Date(startDate);
     startOfMonth.setUTCDate(1);

apps/webapp/app/routes/_app.orgs.$organizationSlug._index/route.tsx

@@ -49,13 +49,7 @@ export default function Page() {
                     <NamedIcon name="folder" className="h-10 w-10 flex-none" />
                     <div className="flex flex-col">
                       <Header3>{project.name}</Header3>
-                      {project.version === "V2" ? (
-                        <Paragraph variant="small">{simplur`${project.jobCount} Job[|s]`}</Paragraph>
-                      ) : (
-                        <Badge className="max-w-max" variant="v3">
-                          v3
-                        </Badge>
-                      )}
+                      <Badge className="max-w-max">{project.version}</Badge>
                     </div>
                   </Link>
                 </li>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam.runs.$runParam/route.tsx

@@ -49,7 +49,7 @@ export default function Page() {
   const revalidator = useRevalidator();
   const events = useEventSource(runStreamingPath(organization, project, job, run), {
     event: "message",
-    disabled: !!run.completedAt,
+    disabled: !!run.completedAt || run.tasks.length > 100,
   });
   useEffect(() => {
     if (events !== null) {

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam._index/route.tsx

@@ -20,7 +20,7 @@ import { Header1, Header2, Header3 } from "~/components/primitives/Headers";
 import { Input } from "~/components/primitives/Input";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
-import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
+import * as Property from "~/components/primitives/PropertyTable";
 import { Spinner } from "~/components/primitives/Spinner";
 import { StepNumber } from "~/components/primitives/StepNumber";
 import {
@@ -129,21 +129,20 @@ export default function Page() {
         <PageTitle title="Tasks" />
         <PageAccessories>
           <AdminDebugTooltip>
-            <PropertyTable>
+            <Property.Table>
               {tasks.map((task) => (
-                <Property label={task.exportName} key={task.slug}>
-                  <div className="flex items-center gap-2">
-                    <Paragraph variant="extra-small/bright/mono">
-                      {task.environments
-                        .map((e) =>
-                          e.userName ? `${e.userName}/${e.id}` : `${e.type.slice(0, 3)}/${e.id}`
-                        )
-                        .join(", ")}
-                    </Paragraph>
-                  </div>
-                </Property>
+                <Property.Item key={task.slug}>
+                  <Property.Label>{task.exportName}</Property.Label>
+                  <Property.Value>
+                    {task.environments
+                      .map((e) =>
+                        e.userName ? `${e.userName}/${e.id}` : `${e.type.slice(0, 3)}/${e.id}`
+                      )
+                      .join(", ")}
+                  </Property.Value>
+                </Property.Item>
               ))}
-            </PropertyTable>
+            </Property.Table>
           </AdminDebugTooltip>
         </PageAccessories>
       </NavBar>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.apikeys/route.tsx

@@ -11,7 +11,7 @@ import { DateTime } from "~/components/primitives/DateTime";
 import { InfoPanel } from "~/components/primitives/InfoPanel";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
-import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
+import * as Property from "~/components/primitives/PropertyTable";
 import {
   Table,
   TableBody,
@@ -61,15 +61,14 @@ export default function Page() {
         <PageTitle title="API keys" />
         <PageAccessories>
           <AdminDebugTooltip>
-            <PropertyTable>
+            <Property.Table>
               {environments.map((environment) => (
-                <Property label={environment.slug} key={environment.id}>
-                  <div className="flex items-center gap-2">
-                    <Paragraph variant="extra-small/bright/mono">{environment.id}</Paragraph>
-                  </div>
-                </Property>
+                <Property.Item key={environment.id}>
+                  <Property.Label>{environment.slug}</Property.Label>
+                  <Property.Value>{environment.id}</Property.Value>
+                </Property.Item>
               ))}
-            </PropertyTable>
+            </Property.Table>
           </AdminDebugTooltip>
 
           <LinkButton

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.concurrency/route.tsx

@@ -0,0 +1,138 @@
+import { ArrowUpCircleIcon, BookOpenIcon } from "@heroicons/react/20/solid";
+import { Await } from "@remix-run/react";
+import { type LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { Suspense } from "react";
+import { typeddefer, useTypedLoaderData } from "remix-typedjson";
+import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
+import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
+import { Feedback } from "~/components/Feedback";
+import { PageBody, PageContainer } from "~/components/layout/AppLayout";
+import { Button, LinkButton } from "~/components/primitives/Buttons";
+import { Header2 } from "~/components/primitives/Headers";
+import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
+import { Spinner } from "~/components/primitives/Spinner";
+import {
+  Table,
+  TableBody,
+  TableCell,
+  TableHeader,
+  TableHeaderCell,
+  TableRow,
+} from "~/components/primitives/Table";
+import { useOrganization } from "~/hooks/useOrganizations";
+import {
+  ConcurrencyPresenter,
+  type Environment,
+} from "~/presenters/v3/ConcurrencyPresenter.server";
+import { requireUserId } from "~/services/session.server";
+import { docsPath, ProjectParamSchema, v3BillingPath } from "~/utils/pathBuilder";
+import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route";
+
+export const loader = async ({ request, params }: LoaderFunctionArgs) => {
+  const userId = await requireUserId(request);
+  const { projectParam } = ProjectParamSchema.parse(params);
+
+  try {
+    const presenter = new ConcurrencyPresenter();
+    const result = await presenter.call({
+      userId,
+      projectSlug: projectParam,
+    });
+
+    return typeddefer(result);
+  } catch (error) {
+    console.error(error);
+    throw new Response(undefined, {
+      status: 400,
+      statusText: "Something went wrong, if this problem persists please contact support.",
+    });
+  }
+};
+
+export default function Page() {
+  const { environments } = useTypedLoaderData<typeof loader>();
+
+  const organization = useOrganization();
+  const plan = useCurrentPlan();
+
+  return (
+    <PageContainer>
+      <NavBar>
+        <PageTitle title="Concurrency limits" />
+        <PageAccessories>
+          <AdminDebugTooltip />
+          <LinkButton
+            variant={"minimal/small"}
+            LeadingIcon={BookOpenIcon}
+            to={docsPath("/queue-concurrency")}
+          >
+            Concurrency docs
+          </LinkButton>
+        </PageAccessories>
+      </NavBar>
+      <PageBody>
+        <div className="flex flex-col gap-4">
+          <div>
+            <div className="mb-2 flex items-center justify-between">
+              <Header2>Environments</Header2>
+              {plan ? (
+                plan?.v3Subscription?.plan?.limits.concurrentRuns.canExceed ? (
+                  <Feedback
+                    button={
+                      <Button LeadingIcon={ArrowUpCircleIcon} variant="tertiary/small">
+                        Request more concurrency
+                      </Button>
+                    }
+                    defaultValue="help"
+                  />
+                ) : (
+                  <LinkButton
+                    LeadingIcon={ArrowUpCircleIcon}
+                    to={v3BillingPath(organization)}
+                    variant="tertiary/small"
+                  >
+                    Upgrade for more concurrency
+                  </LinkButton>
+                )
+              ) : null}
+            </div>
+            <Table>
+              <TableHeader>
+                <TableRow>
+                  <TableHeaderCell>Environment</TableHeaderCell>
+                  <TableHeaderCell alignment="right">Queued</TableHeaderCell>
+                  <TableHeaderCell alignment="right">Running</TableHeaderCell>
+                  <TableHeaderCell alignment="right">Concurrency limit</TableHeaderCell>
+                </TableRow>
+              </TableHeader>
+              <TableBody>
+                <Suspense fallback={<Spinner />}>
+                  <Await resolve={environments} errorElement={<p>Error loading environments</p>}>
+                    {(environments) => <EnvironmentsTable environments={environments} />}
+                  </Await>
+                </Suspense>
+              </TableBody>
+            </Table>
+          </div>
+        </div>
+      </PageBody>
+    </PageContainer>
+  );
+}
+
+function EnvironmentsTable({ environments }: { environments: Environment[] }) {
+  return (
+    <>
+      {environments.map((environment) => (
+        <TableRow key={environment.id}>
+          <TableCell>
+            <EnvironmentLabel environment={environment} userName={environment.userName} />
+          </TableCell>
+          <TableCell alignment="right">{environment.queued}</TableCell>
+          <TableCell alignment="right">{environment.concurrency}</TableCell>
+          <TableCell alignment="right">{environment.concurrencyLimit}</TableCell>
+        </TableRow>
+      ))}
+    </>
+  );
+}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.deployments.$deploymentParam/route.tsx

@@ -11,7 +11,7 @@ import { LinkButton } from "~/components/primitives/Buttons";
 import { DateTimeAccurate } from "~/components/primitives/DateTime";
 import { Header2 } from "~/components/primitives/Headers";
 import { Paragraph } from "~/components/primitives/Paragraph";
-import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
+import * as Property from "~/components/primitives/PropertyTable";
 import {
   Table,
   TableBody,
@@ -71,44 +71,39 @@ export default function Page() {
         <Header2 className={cn("whitespace-nowrap")}>Deploy: {deployment.shortCode}</Header2>
 
         <AdminDebugTooltip>
-          <PropertyTable>
-            <Property label="ID">
-              <div className="flex items-center gap-2">
-                <Paragraph variant="extra-small/bright/mono">{deployment.id}</Paragraph>
-              </div>
-            </Property>
-            <Property label="Project ID">
-              <div className="flex items-center gap-2">
-                <Paragraph variant="extra-small/bright/mono">{deployment.projectId}</Paragraph>
-              </div>
-            </Property>
-            <Property label="Org ID">
-              <div className="flex items-center gap-2">
-                <Paragraph variant="extra-small/bright/mono">{deployment.organizationId}</Paragraph>
-              </div>
-            </Property>
+          <Property.Table>
+            <Property.Item>
+              <Property.Label>ID</Property.Label>
+              <Property.Value>{deployment.id}</Property.Value>
+            </Property.Item>
+            <Property.Item>
+              <Property.Label>Project ID</Property.Label>
+              <Property.Value>{deployment.projectId}</Property.Value>
+            </Property.Item>
+            <Property.Item>
+              <Property.Label>Org ID</Property.Label>
+              <Property.Value>{deployment.organizationId}</Property.Value>
+            </Property.Item>
             {deployment.imageReference && (
-              <Property label="Image">
-                <div className="flex items-center gap-2">
-                  <Paragraph variant="extra-small/bright/mono">
-                    {deployment.imageReference}
-                  </Paragraph>
-                </div>
-              </Property>
+              <Property.Item>
+                <Property.Label>Image</Property.Label>
+                <Property.Value>{deployment.imageReference}</Property.Value>
+              </Property.Item>
             )}
             {deployment.externalBuildData && (
-              <Property label="Build Server">
-                <div className="flex items-center gap-2">
+              <Property.Item>
+                <Property.Label>Build Server</Property.Label>
+                <Property.Value>
                   <Link
                     to={`/resources/${deployment.projectId}/deployments/${deployment.id}/logs`}
                     className="extra-small/bright/mono underline"
                   >
                     {deployment.externalBuildData.buildId}
                   </Link>
-                </div>
-              </Property>
+                </Property.Value>
+              </Property.Item>
             )}
-          </PropertyTable>
+          </Property.Table>
         </AdminDebugTooltip>
 
         <LinkButton
@@ -120,35 +115,51 @@ export default function Page() {
       </div>
       <div className="overflow-y-auto px-3 pt-4 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600">
         <div className="flex flex-col gap-4">
-          <PropertyTable>
-            <Property label="Deploy">
-              <div className="flex items-center gap-2">
-                <Paragraph variant="small/bright">{deployment.shortCode}</Paragraph>
+          <Property.Table>
+            <Property.Item>
+              <Property.Label>Deploy</Property.Label>
+              <Property.Value className="flex items-center gap-2">
+                <span>{deployment.shortCode}</span>
                 {deployment.label && <Badge variant="outline-rounded">{deployment.label}</Badge>}
-              </div>
-            </Property>
-            <Property label="Environment">
-              <EnvironmentLabel environment={deployment.environment} userName={usernameForEnv} />
-            </Property>
-            <Property label="Version">{deployment.version}</Property>
-            <Property label="Status">
-              <DeploymentStatus
-                status={deployment.status}
-                isBuilt={deployment.isBuilt}
-                className="text-sm"
-              />
-            </Property>
-            <Property label="Tasks">{deployment.tasks ? deployment.tasks.length : "–"}</Property>
-            <Property label="SDK Version">
-              {deployment.sdkVersion ? deployment.sdkVersion : "–"}
-            </Property>
-            <Property label="Started at">
-              <Paragraph variant="small/bright">
+              </Property.Value>
+            </Property.Item>
+            <Property.Item>
+              <Property.Label>Environment</Property.Label>
+              <Property.Value>
+                <EnvironmentLabel environment={deployment.environment} userName={usernameForEnv} />
+              </Property.Value>
+            </Property.Item>
+            <Property.Item>
+              <Property.Label>Version</Property.Label>
+              <Property.Value>{deployment.version}</Property.Value>
+            </Property.Item>
+            <Property.Item>
+              <Property.Label>Status</Property.Label>
+              <Property.Value>
+                <DeploymentStatus
+                  status={deployment.status}
+                  isBuilt={deployment.isBuilt}
+                  className="text-sm"
+                />
+              </Property.Value>
+            </Property.Item>
+            <Property.Item>
+              <Property.Label>Tasks</Property.Label>
+              <Property.Value>{deployment.tasks ? deployment.tasks.length : "–"}</Property.Value>
+            </Property.Item>
+            <Property.Item>
+              <Property.Label>SDK Version</Property.Label>
+              <Property.Value>{deployment.sdkVersion ? deployment.sdkVersion : "–"}</Property.Value>
+            </Property.Item>
+            <Property.Item>
+              <Property.Label>Started at</Property.Label>
+              <Property.Value>
                 <DateTimeAccurate date={deployment.createdAt} /> UTC
-              </Paragraph>
-            </Property>
-            <Property label="Built at">
-              <Paragraph variant="small/bright">
+              </Property.Value>
+            </Property.Item>
+            <Property.Item>
+              <Property.Label>Built at</Property.Label>
+              <Property.Value>
                 {deployment.builtAt ? (
                   <>
                     <DateTimeAccurate date={deployment.builtAt} /> UTC
@@ -156,10 +167,11 @@ export default function Page() {
                 ) : (
                   "–"
                 )}
-              </Paragraph>
-            </Property>
-            <Property label="Deployed at">
-              <Paragraph variant="small/bright">
+              </Property.Value>
+            </Property.Item>
+            <Property.Item>
+              <Property.Label>Deployed at</Property.Label>
+              <Property.Value>
                 {deployment.deployedAt ? (
                   <>
                     <DateTimeAccurate date={deployment.deployedAt} /> UTC
@@ -167,25 +179,28 @@ export default function Page() {
                 ) : (
                   "–"
                 )}
-              </Paragraph>
-            </Property>
-            <Property label="Deployed by">
-              {deployment.deployedBy ? (
-                <div className="flex items-center gap-1">
-                  <UserAvatar
-                    avatarUrl={deployment.deployedBy.avatarUrl}
-                    name={deployment.deployedBy.name ?? deployment.deployedBy.displayName}
-                    className="h-4 w-4"
-                  />
-                  <Paragraph variant="small">
-                    {deployment.deployedBy.name ?? deployment.deployedBy.displayName}
-                  </Paragraph>
-                </div>
-              ) : (
-                "–"
-              )}
-            </Property>
-          </PropertyTable>
+              </Property.Value>
+            </Property.Item>
+            <Property.Item>
+              <Property.Label>Deployed by</Property.Label>
+              <Property.Value>
+                {deployment.deployedBy ? (
+                  <div className="flex items-center gap-1">
+                    <UserAvatar
+                      avatarUrl={deployment.deployedBy.avatarUrl}
+                      name={deployment.deployedBy.name ?? deployment.deployedBy.displayName}
+                      className="h-4 w-4"
+                    />
+                    <Paragraph variant="small">
+                      {deployment.deployedBy.name ?? deployment.deployedBy.displayName}
+                    </Paragraph>
+                  </div>
+                ) : (
+                  "–"
+                )}
+              </Property.Value>
+            </Property.Item>
+          </Property.Table>
 
           {deployment.tasks ? (
             <div className="divide-y divide-charcoal-800 overflow-y-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600">

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.environment-variables/route.tsx

@@ -248,6 +248,7 @@ export default function Page() {
                       <EditEnvironmentVariablePanel
                         environments={environments}
                         variable={variable}
+                        revealAll={revealAll}
                       />
                       <DeleteEnvironmentVariableButton variable={variable} />
                     </TableCellMenu>
@@ -293,15 +294,21 @@ export default function Page() {
 function EditEnvironmentVariablePanel({
   variable,
   environments,
+  revealAll,
 }: {
   variable: EnvironmentVariableWithSetValues;
   environments: Pick<RuntimeEnvironment, "id" | "type">[];
+  revealAll: boolean;
 }) {
+  const [reveal, setReveal] = useState(revealAll);
+
   const [isOpen, setIsOpen] = useState(false);
   const lastSubmission = useActionData();
   const navigation = useNavigation();
 
-  const hiddenValues = Object.values(variable.values).filter((value) => !environments.map(e => e.id).includes(value.environment.id));
+  const hiddenValues = Object.values(variable.values).filter(
+    (value) => !environments.map((e) => e.id).includes(value.environment.id)
+  );
 
   const isLoading =
     navigation.state !== "idle" &&
@@ -340,7 +347,11 @@ function EditEnvironmentVariablePanel({
           <input type="hidden" name="key" value={variable.key} />
           {hiddenValues.map((value, index) => (
             <Fragment key={index}>
-              <input type="hidden" name={`values[${index}].environmentId`} value={value.environment.id} />
+              <input
+                type="hidden"
+                name={`values[${index}].environmentId`}
+                value={value.environment.id}
+              />
               <input type="hidden" name={`values[${index}].value`} value={value.value} />
             </Fragment>
           ))}
@@ -355,7 +366,15 @@ function EditEnvironmentVariablePanel({
           </Fieldset>
           <Fieldset>
             <InputGroup fullWidth>
-              <Label>Values</Label>
+              <div className="flex justify-between gap-1">
+                <Label>Values</Label>
+                <Switch
+                  variant="small"
+                  label="Reveal"
+                  checked={reveal}
+                  onCheckedChange={(e) => setReveal(e.valueOf())}
+                />
+              </div>
               <div className="grid grid-cols-[auto_1fr] gap-x-2 gap-y-2">
                 {environments.map((environment, index) => {
                   const value = variable.values[environment.id]?.value;
@@ -377,7 +396,7 @@ function EditEnvironmentVariablePanel({
                         name={`values[${index}].value`}
                         placeholder="Not set"
                         defaultValue={value}
-                        type="password"
+                        type={reveal ? "text" : "password"}
                       />
                     </Fragment>
                   );

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx

@@ -1,13 +1,17 @@
 import {
+  ArrowUturnLeftIcon,
   BoltSlashIcon,
   ChevronDownIcon,
   ChevronRightIcon,
+  InformationCircleIcon,
+  LockOpenIcon,
   MagnifyingGlassMinusIcon,
   MagnifyingGlassPlusIcon,
+  StopCircleIcon,
 } from "@heroicons/react/20/solid";
 import type { Location } from "@remix-run/react";
 import { useLoaderData, useParams, useRevalidator } from "@remix-run/react";
-import { LoaderFunctionArgs, SerializeFrom } from "@remix-run/server-runtime";
+import { LoaderFunctionArgs, SerializeFrom, json } from "@remix-run/server-runtime";
 import { Virtualizer } from "@tanstack/react-virtual";
 import {
   formatDurationMilliseconds,
@@ -20,20 +24,21 @@ import { useCallback, useEffect, useRef, useState } from "react";
 import { useHotkeys } from "react-hotkeys-hook";
 import { ShowParentIcon, ShowParentIconSelected } from "~/assets/icons/ShowParentIcon";
 import tileBgPath from "~/assets/images/error-banner-tile@2x.png";
-import { BlankstateInstructions } from "~/components/BlankstateInstructions";
 import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
 import { InlineCode } from "~/components/code/InlineCode";
 import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
-import { MainCenteredContainer, PageBody } from "~/components/layout/AppLayout";
+import { PageBody } from "~/components/layout/AppLayout";
 import { Badge } from "~/components/primitives/Badge";
-import { LinkButton } from "~/components/primitives/Buttons";
+import { Button, LinkButton } from "~/components/primitives/Buttons";
 import { Callout } from "~/components/primitives/Callout";
+import { Dialog, DialogTrigger } from "~/components/primitives/Dialog";
 import { Header3 } from "~/components/primitives/Headers";
+import { InfoPanel } from "~/components/primitives/InfoPanel";
 import { Input } from "~/components/primitives/Input";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
 import { Popover, PopoverArrowTrigger, PopoverContent } from "~/components/primitives/Popover";
-import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
+import * as Property from "~/components/primitives/PropertyTable";
 import {
   ResizableHandle,
   ResizablePanel,
@@ -43,11 +48,15 @@ import { ShortcutKey, variants } from "~/components/primitives/ShortcutKey";
 import { Slider } from "~/components/primitives/Slider";
 import { Switch } from "~/components/primitives/Switch";
 import * as Timeline from "~/components/primitives/Timeline";
+import { SimpleTooltip } from "~/components/primitives/Tooltip";
 import { TreeView, UseTreeStateOutput, useTree } from "~/components/primitives/TreeView/TreeView";
 import { NodesState } from "~/components/primitives/TreeView/reducer";
+import { CancelRunDialog } from "~/components/runs/v3/CancelRunDialog";
+import { ReplayRunDialog } from "~/components/runs/v3/ReplayRunDialog";
 import { RunIcon } from "~/components/runs/v3/RunIcon";
 import { SpanTitle, eventBackgroundClassName } from "~/components/runs/v3/SpanTitle";
 import { TaskRunStatusIcon, runStatusClassNameColor } from "~/components/runs/v3/TaskRunStatus";
+import { env } from "~/env.server";
 import { useDebounce } from "~/hooks/useDebounce";
 import { useEventSource } from "~/hooks/useEventSource";
 import { useInitialDimensions } from "~/hooks/useInitialDimensions";
@@ -62,6 +71,7 @@ import { requireUserId } from "~/services/session.server";
 import { cn } from "~/utils/cn";
 import { lerp } from "~/utils/lerp";
 import {
+  v3BillingPath,
   v3RunParamsSchema,
   v3RunPath,
   v3RunSpanPath,
@@ -69,8 +79,7 @@ import {
   v3RunsPath,
 } from "~/utils/pathBuilder";
 import { SpanView } from "../resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route";
-import { SimpleTooltip } from "~/components/primitives/Tooltip";
-import { env } from "~/env.server";
+import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route";
 
 type TraceEvent = NonNullable<SerializeFrom<typeof loader>["trace"]>["events"][0];
 
@@ -89,14 +98,16 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   //resizable settings
   const resizeSettings = await getResizableRunSettings(request);
 
-  return {
+  return json({
     run: result.run,
     trace: result.trace,
     maximumLiveReloadingSetting: env.MAXIMUM_LIVE_RELOADING_EVENTS,
     resizeSettings,
-  };
+  });
 };
 
+type LoaderData = SerializeFrom<typeof loader>;
+
 function getSpanId(location: Location<any>): string | undefined {
   const search = new URLSearchParams(location.search);
   return search.get("span") ?? undefined;
@@ -105,72 +116,121 @@ function getSpanId(location: Location<any>): string | undefined {
 export default function Page() {
   const { run, trace, resizeSettings, maximumLiveReloadingSetting } =
     useLoaderData<typeof loader>();
+  const user = useUser();
   const organization = useOrganization();
   const project = useProject();
-  const user = useUser();
-  const { location, replaceSearchParam } = useReplaceLocation();
-  const selectedSpanId = getSpanId(location);
 
   const usernameForEnv = user.id !== run.environment.userId ? run.environment.userName : undefined;
 
-  if (!trace) {
-    return (
-      <>
-        <NavBar>
-          <PageTitle
-            backButton={{
-              to: v3RunsPath(organization, project),
-              text: "Runs",
-            }}
-            title={`Run #${run.number}`}
-          />
-          <PageAccessories>
-            <AdminDebugTooltip>
-              <PropertyTable>
-                <Property label="ID">
-                  <div className="flex items-center gap-2">
-                    <Paragraph variant="extra-small/bright/mono">{run.id}</Paragraph>
-                  </div>
-                </Property>
-                <Property label="Trace ID">
-                  <div className="flex items-center gap-2">
-                    <Paragraph variant="extra-small/bright/mono">{run.traceId}</Paragraph>
-                  </div>
-                </Property>
-                <Property label="Env ID">
-                  <div className="flex items-center gap-2">
-                    <Paragraph variant="extra-small/bright/mono">{run.environment.id}</Paragraph>
-                  </div>
-                </Property>
-                <Property label="Org ID">
-                  <div className="flex items-center gap-2">
-                    <Paragraph variant="extra-small/bright/mono">
-                      {run.environment.organizationId}
-                    </Paragraph>
-                  </div>
-                </Property>
-              </PropertyTable>
-            </AdminDebugTooltip>
-
-            <EnvironmentLabel
-              size="large"
-              environment={run.environment}
-              userName={usernameForEnv}
+  return (
+    <>
+      <NavBar>
+        <PageTitle
+          backButton={{
+            to: v3RunsPath(organization, project),
+            text: "Runs",
+          }}
+          title={
+            <div className="flex items-center gap-3">
+              <span>Run #{run.number}</span>
+              <EnvironmentLabel
+                size="large"
+                environment={run.environment}
+                userName={usernameForEnv}
+              />
+            </div>
+          }
+        />
+        <PageAccessories>
+          <AdminDebugTooltip>
+            <Property.Table>
+              <Property.Item>
+                <Property.Label>ID</Property.Label>
+                <Property.Value>{run.id}</Property.Value>
+              </Property.Item>
+              <Property.Item>
+                <Property.Label>Trace ID</Property.Label>
+                <Property.Value>{run.traceId}</Property.Value>
+              </Property.Item>
+              <Property.Item>
+                <Property.Label>Env ID</Property.Label>
+                <Property.Value>{run.environment.id}</Property.Value>
+              </Property.Item>
+              <Property.Item>
+                <Property.Label>Org ID</Property.Label>
+                <Property.Value>{run.environment.organizationId}</Property.Value>
+              </Property.Item>
+            </Property.Table>
+          </AdminDebugTooltip>
+          <Dialog key={`replay-${run.friendlyId}`}>
+            <DialogTrigger asChild>
+              <Button
+                variant="tertiary/small"
+                LeadingIcon={ArrowUturnLeftIcon}
+                shortcut={{ key: "R" }}
+              >
+                Replay run
+              </Button>
+            </DialogTrigger>
+            <ReplayRunDialog
+              runFriendlyId={run.friendlyId}
+              failedRedirect={v3RunSpanPath(
+                organization,
+                project,
+                { friendlyId: run.friendlyId },
+                { spanId: run.spanId }
+              )}
             />
-          </PageAccessories>
-        </NavBar>
-        <PageBody>
-          <MainCenteredContainer className="max-w-prose">
-            <BlankstateInstructions title="These logs have taken a walk">
-              <Paragraph spacing>
-                Looks like the logs from this run have wandered off after their 7-day stay. We tidy
-                up older logs to keep things running smoothly.
-              </Paragraph>
-            </BlankstateInstructions>
-          </MainCenteredContainer>
-        </PageBody>
-      </>
-    );
+          </Dialog>
+          {run.isFinished ? null : (
+            <Dialog key={`cancel-${run.friendlyId}`}>
+              <DialogTrigger asChild>
+                <Button variant="danger/small" LeadingIcon={StopCircleIcon}>
+                  Cancel run
+                </Button>
+              </DialogTrigger>
+              <CancelRunDialog
+                runFriendlyId={run.friendlyId}
+                redirectPath={v3RunSpanPath(
+                  organization,
+                  project,
+                  { friendlyId: run.friendlyId },
+                  { spanId: run.spanId }
+                )}
+              />
+            </Dialog>
+          )}
+        </PageAccessories>
+      </NavBar>
+      <PageBody scrollable={false}>
+        {trace ? (
+          <TraceView
+            run={run}
+            trace={trace}
+            maximumLiveReloadingSetting={maximumLiveReloadingSetting}
+            resizeSettings={resizeSettings}
+          />
+        ) : (
+          <NoLogsView
+            run={run}
+            trace={trace}
+            maximumLiveReloadingSetting={maximumLiveReloadingSetting}
+            resizeSettings={resizeSettings}
+          />
+        )}
+      </PageBody>
+    </>
+  );
+}
+
+function TraceView({ run, trace, maximumLiveReloadingSetting, resizeSettings }: LoaderData) {
+  const organization = useOrganization();
+  const project = useProject();
+  const { location, replaceSearchParam } = useReplaceLocation();
+  const selectedSpanId = getSpanId(location);
+
+  if (!trace) {
+    return <></>;
   }
 
   const { events, parentRunFriendlyId, duration, rootSpanStatus, rootStartedAt } = trace;
@@ -193,94 +253,134 @@ export default function Page() {
   }, [streamedEvents]); // eslint-disable-line react-hooks/exhaustive-deps
 
   return (
-    <>
-      <NavBar>
-        <PageTitle
-          backButton={{
-            to: v3RunsPath(organization, project),
-            text: "Runs",
-          }}
-          title={`Run #${run.number}`}
-        />
-        <PageAccessories>
-          <AdminDebugTooltip>
-            <PropertyTable>
-              <Property label="ID">
-                <div className="flex items-center gap-2">
-                  <Paragraph variant="extra-small/bright/mono">{run.id}</Paragraph>
-                </div>
-              </Property>
-              <Property label="Trace ID">
-                <div className="flex items-center gap-2">
-                  <Paragraph variant="extra-small/bright/mono">{run.traceId}</Paragraph>
-                </div>
-              </Property>
-              <Property label="Env ID">
-                <div className="flex items-center gap-2">
-                  <Paragraph variant="extra-small/bright/mono">{run.environment.id}</Paragraph>
-                </div>
-              </Property>
-              <Property label="Org ID">
-                <div className="flex items-center gap-2">
-                  <Paragraph variant="extra-small/bright/mono">
-                    {run.environment.organizationId}
-                  </Paragraph>
-                </div>
-              </Property>
-            </PropertyTable>
-          </AdminDebugTooltip>
+    <div className={cn("grid h-full max-h-full grid-cols-1 overflow-hidden")}>
+      <ResizablePanelGroup
+        direction="horizontal"
+        className="h-full max-h-full"
+        onLayout={(layout) => {
+          if (layout.length !== 2) return;
+          if (!selectedSpanId) return;
+          setResizableRunSettings(document, layout);
+        }}
+      >
+        <ResizablePanel order={1} minSize={30} defaultSize={resizeSettings.layout?.[0] ?? 70}>
+          <TasksTreeView
+            selectedId={selectedSpanId}
+            key={events[0]?.id ?? "-"}
+            events={events}
+            parentRunFriendlyId={parentRunFriendlyId}
+            onSelectedIdChanged={(selectedSpan) => {
+              //instantly close the panel if no span is selected
+              if (!selectedSpan) {
+                replaceSearchParam("span");
+                return;
+              }
 
-          <EnvironmentLabel size="large" environment={run.environment} userName={usernameForEnv} />
-        </PageAccessories>
-      </NavBar>
-      <PageBody scrollable={false}>
-        <div className={cn("grid h-full max-h-full grid-cols-1 overflow-hidden")}>
-          <ResizablePanelGroup
-            direction="horizontal"
-            className="h-full max-h-full"
-            onLayout={(layout) => {
-              if (layout.length !== 2) return;
-              if (!selectedSpanId) return;
-              setResizableRunSettings(document, layout);
+              changeToSpan(selectedSpan);
             }}
-          >
-            <ResizablePanel order={1} minSize={30} defaultSize={resizeSettings.layout?.[0]}>
-              <TasksTreeView
-                selectedId={selectedSpanId}
-                key={events[0]?.id ?? "-"}
-                events={events}
-                parentRunFriendlyId={parentRunFriendlyId}
-                onSelectedIdChanged={(selectedSpan) => {
-                  //instantly close the panel if no span is selected
-                  if (!selectedSpan) {
-                    replaceSearchParam("span");
-                    return;
-                  }
+            totalDuration={duration}
+            rootSpanStatus={rootSpanStatus}
+            rootStartedAt={rootStartedAt ? new Date(rootStartedAt) : undefined}
+            environmentType={run.environment.type}
+            shouldLiveReload={shouldLiveReload}
+            maximumLiveReloadingSetting={maximumLiveReloadingSetting}
+          />
+        </ResizablePanel>
+        <ResizableHandle withHandle />
+        {selectedSpanId && (
+          <ResizablePanel order={2} minSize={25} defaultSize={resizeSettings.layout?.[1] ?? 30}>
+            <SpanView
+              runParam={run.friendlyId}
+              spanId={selectedSpanId}
+              closePanel={() => replaceSearchParam("span")}
+            />
+          </ResizablePanel>
+        )}
+      </ResizablePanelGroup>
+    </div>
+  );
+}
 
-                  changeToSpan(selectedSpan);
-                }}
-                totalDuration={duration}
-                rootSpanStatus={rootSpanStatus}
-                rootStartedAt={rootStartedAt ? new Date(rootStartedAt) : undefined}
-                environmentType={run.environment.type}
-                shouldLiveReload={shouldLiveReload}
-                maximumLiveReloadingSetting={maximumLiveReloadingSetting}
-              />
-            </ResizablePanel>
-            <ResizableHandle withHandle />
-            {selectedSpanId && (
-              <ResizablePanel order={2} minSize={30} defaultSize={resizeSettings.layout?.[1]}>
-                <SpanView
-                  runParam={run.friendlyId}
-                  spanId={selectedSpanId}
-                  closePanel={() => replaceSearchParam("span")}
-                />
-              </ResizablePanel>
+function NoLogsView({ run, resizeSettings }: LoaderData) {
+  const plan = useCurrentPlan();
+  const organization = useOrganization();
+
+  const logRetention = plan?.v3Subscription?.plan?.limits.logRetentionDays.number ?? 30;
+
+  const completedAt = run.completedAt ? new Date(run.completedAt) : undefined;
+  const now = new Date();
+
+  const daysSinceCompleted = completedAt
+    ? Math.floor((now.getTime() - completedAt.getTime()) / (1000 * 60 * 60 * 24))
+    : undefined;
+
+  const isWithinLogRetention =
+    daysSinceCompleted !== undefined && daysSinceCompleted <= logRetention;
+
+  return (
+    <div className={cn("grid h-full max-h-full grid-cols-1 overflow-hidden")}>
+      <ResizablePanelGroup
+        direction="horizontal"
+        className="h-full max-h-full"
+        onLayout={(layout) => {
+          if (layout.length !== 2) return;
+          setResizableRunSettings(document, layout);
+        }}
+      >
+        <ResizablePanel order={1} minSize={30} defaultSize={resizeSettings.layout?.[0] ?? 70}>
+          <div className="grid h-full place-items-center">
+            {daysSinceCompleted === undefined ? (
+              <InfoPanel variant="info" icon={InformationCircleIcon} title="We delete old logs">
+                <Paragraph variant="small">
+                  We tidy up older logs to keep things running smoothly.
+                </Paragraph>
+              </InfoPanel>
+            ) : isWithinLogRetention ? (
+              <InfoPanel
+                variant="info"
+                icon={InformationCircleIcon}
+                title="These logs have been deleted"
+              >
+                <Paragraph variant="small">
+                  Your log retention is {logRetention} days but these logs had already been deleted.
+                  From now on only logs from runs that completed {logRetention} days ago will be
+                  deleted.
+                </Paragraph>
+              </InfoPanel>
+            ) : daysSinceCompleted <= 30 ? (
+              <InfoPanel
+                variant="upgrade"
+                icon={LockOpenIcon}
+                iconClassName="text-indigo-500"
+                title="Unlock longer log retention"
+                to={v3BillingPath(organization)}
+                buttonLabel="Upgrade"
+              >
+                <Paragraph variant="small">
+                  The logs for this run have been deleted because the run completed{" "}
+                  {daysSinceCompleted} days ago.
+                </Paragraph>
+                <Paragraph variant="small">Upgrade your plan to keep logs for longer.</Paragraph>
+              </InfoPanel>
+            ) : (
+              <InfoPanel
+                variant="info"
+                icon={InformationCircleIcon}
+                title="These logs are more than 30 days old"
+              >
+                <Paragraph variant="small">
+                  We tidy up older logs to keep things running smoothly.
+                </Paragraph>
+              </InfoPanel>
             )}
-          </ResizablePanelGroup>
-        </div>
-      </PageBody>
-    </>
+          </div>
+        </ResizablePanel>
+        <ResizableHandle withHandle />
+        <ResizablePanel order={2} minSize={25} defaultSize={resizeSettings.layout?.[1] ?? 30}>
+          <SpanView runParam={run.friendlyId} spanId={run.spanId} />
+        </ResizablePanel>
+      </ResizablePanelGroup>
+    </div>
   );
 }
 
@@ -311,7 +411,7 @@ function TasksTreeView({
 }: TasksTreeViewProps) {
   const [filterText, setFilterText] = useState("");
   const [errorsOnly, setErrorsOnly] = useState(false);
-  const [showDurations, setShowDurations] = useState(false);
+  const [showDurations, setShowDurations] = useState(true);
   const [scale, setScale] = useState(0);
   const parentRef = useRef<HTMLDivElement>(null);
   const treeScrollRef = useRef<HTMLDivElement>(null);
@@ -927,7 +1027,7 @@ function SpanWithDuration({
             !showDuration && "opacity-0"
           )}
         >
-          <div className="rounded-sm px-1 py-0.5 text-xxs text-text-bright text-shadow-custom">
+          <div className="whitespace-nowrap rounded-sm px-1 py-0.5 text-xxs text-text-bright text-shadow-custom">
             {formatDurationMilliseconds(props.durationMs, {
               style: "short",
               maxDecimalPoints: props.durationMs < 1000 ? 0 : 1,
@@ -1028,16 +1128,11 @@ function KeyboardShortcuts({
         title="Expand all"
       />
       <ShortcutWithAction
-        shortcut={{ key: "c" }}
+        shortcut={{ key: "w" }}
         action={() => collapseAllBelowDepth(1)}
         title="Collapse all"
       />
       <NumberShortcuts toggleLevel={(number) => toggleExpandLevel(number)} />
-      <ShortcutWithAction
-        shortcut={{ key: "d" }}
-        action={() => setShowDurations((d) => !d)}
-        title="Toggle durations"
-      />
     </>
   );
 }

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.schedules.$scheduleParam/route.tsx

@@ -15,7 +15,6 @@ import { ExitIcon } from "~/assets/icons/ExitIcon";
 import { InlineCode } from "~/components/code/InlineCode";
 import { EnvironmentLabels } from "~/components/environments/EnvironmentLabel";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
-import { Callout, variantClasses } from "~/components/primitives/Callout";
 import { DateTime } from "~/components/primitives/DateTime";
 import {
   Dialog,
@@ -27,7 +26,7 @@ import {
 import { Header2, Header3 } from "~/components/primitives/Headers";
 import { InfoPanel } from "~/components/primitives/InfoPanel";
 import { Paragraph } from "~/components/primitives/Paragraph";
-import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
+import * as Property from "~/components/primitives/PropertyTable";
 import {
   Table,
   TableBlankRow,
@@ -225,36 +224,63 @@ export default function Page() {
       <div className="overflow-y-scroll scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600">
         <div className="p-3">
           <div className="space-y-3">
-            <PropertyTable>
-              <Property label="Schedule ID">{schedule.friendlyId}</Property>
-              <Property label="Task ID">{schedule.taskIdentifier}</Property>
-              <Property label="Type">
-                <ScheduleTypeCombo type={schedule.type} className="text-sm" />
-              </Property>
-              <Property label="CRON (UTC)" labelClassName="self-start">
-                <div className="space-y-2">
-                  <InlineCode variant="extra-small">{schedule.cron}</InlineCode>
-                  <Paragraph variant="small">{schedule.cronDescription}</Paragraph>
-                </div>
-              </Property>
-              <Property label="Timezone">{schedule.timezone}</Property>
-              <Property label="Environments">
-                <EnvironmentLabels size="small" environments={schedule.environments} />
-              </Property>
+            <Property.Table>
+              <Property.Item>
+                <Property.Label>Schedule ID</Property.Label>
+                <Property.Value>{schedule.friendlyId}</Property.Value>
+              </Property.Item>
+              <Property.Item>
+                <Property.Label>Task ID</Property.Label>
+                <Property.Value>{schedule.taskIdentifier}</Property.Value>
+              </Property.Item>
+              <Property.Item>
+                <Property.Label>Type</Property.Label>
+                <Property.Value>
+                  <ScheduleTypeCombo type={schedule.type} className="text-sm" />
+                </Property.Value>
+              </Property.Item>
+              <Property.Item>
+                <Property.Label>CRON</Property.Label>
+                <Property.Value>
+                  <div className="space-y-2">
+                    <InlineCode variant="extra-small">{schedule.cron}</InlineCode>
+                    <Paragraph variant="small">{schedule.cronDescription}</Paragraph>
+                  </div>
+                </Property.Value>
+              </Property.Item>
+              <Property.Item>
+                <Property.Label>Timezone</Property.Label>
+                <Property.Value>{schedule.timezone}</Property.Value>
+              </Property.Item>
+              <Property.Item>
+                <Property.Label>Environments</Property.Label>
+                <Property.Value>
+                  <EnvironmentLabels size="small" environments={schedule.environments} />
+                </Property.Value>
+              </Property.Item>
               {isImperative && (
                 <>
-                  <Property label="External ID">
-                    {schedule.externalId ? schedule.externalId : "–"}
-                  </Property>
-                  <Property label="Deduplication key">
-                    {schedule.userProvidedDeduplicationKey ? schedule.deduplicationKey : "–"}
-                  </Property>
-                  <Property label="Status">
-                    <EnabledStatus enabled={schedule.active} />
-                  </Property>
+                  <Property.Item>
+                    <Property.Label>External ID</Property.Label>
+                    <Property.Value>
+                      {schedule.externalId ? schedule.externalId : "–"}
+                    </Property.Value>
+                  </Property.Item>
+                  <Property.Item>
+                    <Property.Label>Deduplication key</Property.Label>
+                    <Property.Value>
+                      {schedule.userProvidedDeduplicationKey ? schedule.deduplicationKey : "–"}
+                    </Property.Value>
+                  </Property.Item>
+                  <Property.Item>
+                    <Property.Label>Status</Property.Label>
+                    <Property.Value>
+                      <EnabledStatus enabled={schedule.active} />
+                    </Property.Value>
+                  </Property.Item>
                 </>
               )}
-            </PropertyTable>
+            </Property.Table>
             <div className="flex flex-col gap-1">
               <Header3>Last 5 runs</Header3>
               <TaskRunsTable
@@ -297,12 +323,12 @@ export default function Page() {
                         </TableRow>
                       ))
                     ) : (
-                      <TableBlankRow colSpan={1}>
+                      <TableBlankRow colSpan={isUtc ? 1 : 2}>
                         <PlaceholderText title="You found a bug" />
                       </TableBlankRow>
                     )
                   ) : (
-                    <TableBlankRow colSpan={1}>
+                    <TableBlankRow colSpan={isUtc ? 1 : 2}>
                       <PlaceholderText title="Schedule disabled" />
                     </TableBlankRow>
                   )}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.schedules/route.tsx

@@ -28,7 +28,7 @@ import { InfoPanel } from "~/components/primitives/InfoPanel";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { PaginationControls } from "~/components/primitives/Pagination";
 import { Paragraph } from "~/components/primitives/Paragraph";
-import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
+import * as Property from "~/components/primitives/PropertyTable";
 import {
   ResizableHandle,
   ResizablePanel,
@@ -51,7 +51,7 @@ import { useProject } from "~/hooks/useProject";
 import { redirectWithErrorMessage } from "~/models/message.server";
 import { findProjectBySlug } from "~/models/project.server";
 import {
-  ScheduleListItem,
+  type ScheduleListItem,
   ScheduleListPresenter,
 } from "~/presenters/v3/ScheduleListPresenter.server";
 import { requireUserId } from "~/services/session.server";
@@ -122,15 +122,14 @@ export default function Page() {
         <PageTitle title="Schedules" />
         <PageAccessories>
           <AdminDebugTooltip>
-            <PropertyTable>
+            <Property.Table>
               {schedules.map((schedule) => (
-                <Property label={schedule.friendlyId} key={schedule.id}>
-                  <div className="flex items-center gap-2">
-                    <Paragraph variant="extra-small/bright/mono">{schedule.id}</Paragraph>
-                  </div>
-                </Property>
+                <Property.Item key={schedule.id}>
+                  <Property.Label>{schedule.friendlyId}</Property.Label>
+                  <Property.Value>{schedule.id}</Property.Value>
+                </Property.Item>
               ))}
-            </PropertyTable>
+            </Property.Table>
           </AdminDebugTooltip>
 
           {limits.used >= limits.limit ? (
@@ -180,72 +179,74 @@ export default function Page() {
       <PageBody scrollable={false}>
         <ResizablePanelGroup direction="horizontal" className="h-full max-h-full">
           <ResizablePanel order={1} minSize={20} defaultSize={60}>
-            {possibleTasks.length === 0 ? (
-              <CreateScheduledTaskInstructions />
-            ) : schedules.length === 0 && !hasFilters ? (
-              <AttachYourFirstScheduleInstructions />
-            ) : (
-              <div className="p-3">
-                <div className="mb-2 flex items-center justify-between gap-x-2">
-                  <ScheduleFilters
-                    possibleEnvironments={possibleEnvironments}
-                    possibleTasks={possibleTasks}
-                  />
-                  <div className="flex items-center justify-end gap-x-2">
-                    <PaginationControls
-                      currentPage={currentPage}
-                      totalPages={totalPages}
-                      showPageNumbers={false}
+            <div className="max-h-full overflow-y-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600">
+              {possibleTasks.length === 0 ? (
+                <CreateScheduledTaskInstructions />
+              ) : schedules.length === 0 && !hasFilters ? (
+                <AttachYourFirstScheduleInstructions />
+              ) : (
+                <div className="p-3">
+                  <div className="mb-2 flex items-center justify-between gap-x-2">
+                    <ScheduleFilters
+                      possibleEnvironments={possibleEnvironments}
+                      possibleTasks={possibleTasks}
                     />
+                    <div className="flex items-center justify-end gap-x-2">
+                      <PaginationControls
+                        currentPage={currentPage}
+                        totalPages={totalPages}
+                        showPageNumbers={false}
+                      />
+                    </div>
+                  </div>
+
+                  <SchedulesTable schedules={schedules} hasFilters={hasFilters} />
+                  <div className="mt-3 flex w-full items-start justify-between">
+                    {requiresUpgrade ? (
+                      <InfoPanel
+                        variant="upgrade"
+                        icon={LockOpenIcon}
+                        iconClassName="text-indigo-500"
+                        title="Unlock more schedules"
+                        to={v3BillingPath(organization)}
+                        buttonLabel="Upgrade"
+                      >
+                        <Paragraph variant="small">
+                          You've used all {limits.limit} of your available schedules. Upgrade your
+                          plan to enable more.
+                        </Paragraph>
+                      </InfoPanel>
+                    ) : (
+                      <div className="flex h-fit flex-col items-start gap-4 rounded-md border border-grid-bright bg-background-bright p-4">
+                        <div className="flex items-center justify-between gap-6">
+                          <Header3>
+                            You've used {limits.used}/{limits.limit} of your schedules.
+                          </Header3>
+
+                          {canUpgrade ? (
+                            <LinkButton to={v3BillingPath(organization)} variant="secondary/small">
+                              Upgrade
+                            </LinkButton>
+                          ) : (
+                            <Feedback
+                              button={<Button variant="secondary/small">Request more</Button>}
+                              defaultValue="help"
+                            />
+                          )}
+                        </div>
+                        <div className="h-2 w-full overflow-hidden rounded-full border border-grid-bright">
+                          <div
+                            className="h-full bg-grid-bright"
+                            style={{ width: `${(limits.used / limits.limit) * 100}%` }}
+                          />
+                        </div>
+                      </div>
+                    )}
+                    <PaginationControls currentPage={currentPage} totalPages={totalPages} />
                   </div>
                 </div>
-
-                <SchedulesTable schedules={schedules} hasFilters={hasFilters} />
-                <div className="mt-3 flex w-full items-start justify-between">
-                  {requiresUpgrade ? (
-                    <InfoPanel
-                      variant="upgrade"
-                      icon={LockOpenIcon}
-                      iconClassName="text-indigo-500"
-                      title="Unlock more schedules"
-                      to={v3BillingPath(organization)}
-                      buttonLabel="Upgrade"
-                    >
-                      <Paragraph variant="small">
-                        You've used all {limits.limit} of your available schedules. Upgrade your
-                        plan to enable more.
-                      </Paragraph>
-                    </InfoPanel>
-                  ) : (
-                    <div className="flex h-fit flex-col items-start gap-4 rounded-md border border-grid-bright bg-background-bright p-4">
-                      <div className="flex items-center justify-between gap-6">
-                        <Header3>
-                          You've used {limits.used}/{limits.limit} of your schedules.
-                        </Header3>
-
-                        {canUpgrade ? (
-                          <LinkButton to={v3BillingPath(organization)} variant="secondary/small">
-                            Upgrade
-                          </LinkButton>
-                        ) : (
-                          <Feedback
-                            button={<Button variant="secondary/small">Request more</Button>}
-                            defaultValue="help"
-                          />
-                        )}
-                      </div>
-                      <div className="h-2 w-full overflow-hidden rounded-full border border-grid-bright">
-                        <div
-                          className="h-full bg-grid-bright"
-                          style={{ width: `${(limits.used / limits.limit) * 100}%` }}
-                        />
-                      </div>
-                    </div>
-                  )}
-                  <PaginationControls currentPage={currentPage} totalPages={totalPages} />
-                </div>
-              </div>
-            )}
+              )}
+            </div>
           </ResizablePanel>
           {(isShowingNewPane || isShowingSchedule) && (
             <>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.settings/route.tsx

@@ -17,7 +17,7 @@ import { InputGroup } from "~/components/primitives/InputGroup";
 import { Label } from "~/components/primitives/Label";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
-import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
+import * as Property from "~/components/primitives/PropertyTable";
 import { prisma } from "~/db.server";
 import { useProject } from "~/hooks/useProject";
 import { redirectWithSuccessMessage } from "~/models/message.server";
@@ -116,18 +116,19 @@ export default function Page() {
 
         <PageAccessories>
           <AdminDebugTooltip>
-            <PropertyTable>
-              <Property label="ID">
+            <Property.Table>
+              <Property.Item>
+                <Property.Label>ID</Property.Label>
+                <Property.Value>{project.id}</Property.Value>
                 <div className="flex items-center gap-2">
                   <Paragraph variant="extra-small/bright/mono">{project.id}</Paragraph>
                 </div>
-              </Property>
-              <Property label="Org ID">
-                <div className="flex items-center gap-2">
-                  <Paragraph variant="extra-small/bright/mono">{project.organizationId}</Paragraph>
-                </div>
-              </Property>
-            </PropertyTable>
+              </Property.Item>
+              <Property.Item>
+                <Property.Label>Org ID</Property.Label>
+                <Property.Value>{project.organizationId}</Property.Value>
+              </Property.Item>
+            </Property.Table>
           </AdminDebugTooltip>
         </PageAccessories>
       </NavBar>

apps/webapp/app/routes/_app.orgs.$organizationSlug.team/route.tsx

@@ -27,7 +27,7 @@ import { InfoPanel } from "~/components/primitives/InfoPanel";
 import { NamedIcon } from "~/components/primitives/NamedIcon";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
-import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
+import * as Property from "~/components/primitives/PropertyTable";
 import { SimpleTooltip } from "~/components/primitives/Tooltip";
 import { $replica } from "~/db.server";
 import { useOrganization } from "~/hooks/useOrganizations";
@@ -128,23 +128,25 @@ export default function Page() {
 
         <PageAccessories>
           <AdminDebugTooltip>
-            <PropertyTable>
-              <Property label="Org ID">
-                <div className="flex items-center gap-2">
-                  <Paragraph variant="extra-small/bright/mono">{organization.id}</Paragraph>
-                </div>
-              </Property>
+            <Property.Table>
+              <Property.Item>
+                <Property.Label>Org ID</Property.Label>
+                <Property.Value>{organization.id}</Property.Value>
+              </Property.Item>
 
               {members.map((member) => (
-                <Property label={member.user.name} key={member.id}>
-                  <div className="flex items-center gap-2">
-                    <Paragraph variant="extra-small/bright/mono">
-                      {member.user.email} - {member.user.id}
-                    </Paragraph>
-                  </div>
-                </Property>
+                <Property.Item key={member.id}>
+                  <Property.Label>{member.user.name}</Property.Label>
+                  <Property.Value>
+                    <div className="flex items-center gap-2">
+                      <Paragraph variant="extra-small/bright/mono">
+                        {member.user.email} - {member.user.id}
+                      </Paragraph>
+                    </div>
+                  </Property.Value>
+                </Property.Item>
               ))}
-            </PropertyTable>
+            </Property.Table>
           </AdminDebugTooltip>
         </PageAccessories>
       </NavBar>

apps/webapp/app/routes/_app.orgs.$organizationSlug.v3.usage/route.tsx

@@ -66,7 +66,7 @@ export async function loader({ params, request }: LoaderFunctionArgs) {
 
   const search = new URL(request.url).searchParams;
   const searchMonth = search.get("month");
-  const startDate = searchMonth ? new Date(searchMonth) : months[0];
+  const startDate = searchMonth ? new Date(decodeURIComponent(searchMonth)) : months[0];
   startDate.setUTCDate(1);
   startDate.setUTCHours(0, 0, 0, 0);
 

apps/webapp/app/routes/admin.concurrency.tsx

@@ -0,0 +1,46 @@
+import { InformationCircleIcon } from "@heroicons/react/20/solid";
+import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { redirect, typedjson, useTypedLoaderData } from "remix-typedjson";
+import { Header1 } from "~/components/primitives/Headers";
+import { InfoPanel } from "~/components/primitives/InfoPanel";
+import { Paragraph } from "~/components/primitives/Paragraph";
+import { requireUser } from "~/services/session.server";
+import { concurrencyTracker } from "~/v3/services/taskRunConcurrencyTracker.server";
+
+export const loader = async ({ request, params }: LoaderFunctionArgs) => {
+  const user = await requireUser(request);
+  if (!user.admin) {
+    return redirect("/");
+  }
+
+  const deployedConcurrency = await concurrencyTracker.globalConcurrentRunCount(true);
+  const devConcurrency = await concurrencyTracker.globalConcurrentRunCount(false);
+
+  return typedjson({ deployedConcurrency, devConcurrency });
+};
+
+export default function AdminDashboardRoute() {
+  const { deployedConcurrency, devConcurrency } = useTypedLoaderData<typeof loader>();
+
+  return (
+    <main
+      aria-labelledby="primary-heading"
+      className="flex h-full w-fit min-w-0 flex-1 flex-col gap-4 overflow-y-auto px-4 pb-4 lg:order-last"
+    >
+      <div className="flex items-center divide-x divide-grid-bright rounded border border-grid-bright">
+        <div className="w-1/2 p-3">
+          <Paragraph spacing>Dev</Paragraph>
+          <Header1>{devConcurrency}</Header1>
+        </div>
+        <div className="w-1/2 p-3">
+          <Paragraph spacing>Deployed</Paragraph>
+          <Header1>{deployedConcurrency}</Header1>
+        </div>
+      </div>
+      <InfoPanel icon={InformationCircleIcon}>
+        This refers to the number of 'Dequeued' runs, which are either currently executing or about
+        to begin execution.
+      </InfoPanel>
+    </main>
+  );
+}

apps/webapp/app/routes/admin.orgs.tsx

@@ -1,7 +1,7 @@
 import { MagnifyingGlassIcon } from "@heroicons/react/20/solid";
 import { Form } from "@remix-run/react";
 import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
-import { typedjson, useTypedLoaderData } from "remix-typedjson";
+import { redirect, typedjson, useTypedLoaderData } from "remix-typedjson";
 import { z } from "zod";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
 import { Input } from "~/components/primitives/Input";
@@ -17,7 +17,7 @@ import {
   TableRow,
 } from "~/components/primitives/Table";
 import { adminGetOrganizations } from "~/models/admin.server";
-import { requireUserId } from "~/services/session.server";
+import { requireUser, requireUserId } from "~/services/session.server";
 import { createSearchParams } from "~/utils/searchParams";
 
 export const SearchParams = z.object({
@@ -28,13 +28,16 @@ export const SearchParams = z.object({
 export type SearchParams = z.infer<typeof SearchParams>;
 
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
-  const userId = await requireUserId(request);
+  const user = await requireUser(request);
+  if (!user.admin) {
+    return redirect("/");
+  }
 
   const searchParams = createSearchParams(request.url, SearchParams);
   if (!searchParams.success) {
     throw new Error(searchParams.error);
   }
-  const result = await adminGetOrganizations(userId, searchParams.params.getAll());
+  const result = await adminGetOrganizations(user.id, searchParams.params.getAll());
 
   return typedjson(result);
 };

apps/webapp/app/routes/admin.tsx

@@ -3,15 +3,10 @@ import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { redirect, typedjson } from "remix-typedjson";
 import { LinkButton } from "~/components/primitives/Buttons";
 import { Tabs } from "~/components/primitives/Tabs";
-import { getUser, requireUserId } from "~/services/session.server";
+import { requireUser } from "~/services/session.server";
 
 export async function loader({ request }: LoaderFunctionArgs) {
-  await requireUserId(request);
-  const user = await getUser(request);
-  if (user == null) {
-    return redirect("/");
-  }
-
+  const user = await requireUser(request);
   if (!user.admin) {
     return redirect("/");
   }
@@ -33,6 +28,10 @@ export default function Page() {
               label: "Organizations",
               to: "/admin/orgs",
             },
+            {
+              label: "Concurrency",
+              to: "/admin/concurrency",
+            },
           ]}
           layoutId={"admin"}
         />

apps/webapp/app/routes/api.v1.runs.$runId.tags.ts

@@ -2,7 +2,7 @@ import { type ActionFunctionArgs, json } from "@remix-run/server-runtime";
 import { AddTagsRequestBody } from "@trigger.dev/core/v3";
 import { z } from "zod";
 import { prisma } from "~/db.server";
-import { createTag, getTagsForRunId } from "~/models/taskRunTag.server";
+import { createTag, getTagsForRunId, MAX_TAGS_PER_RUN } from "~/models/taskRunTag.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
 import { generateFriendlyId } from "~/v3/friendlyIdentifiers";
 
@@ -51,12 +51,12 @@ export async function action({ request, params }: ActionFunctionArgs) {
       return !existingTags.map((t) => t.name).includes(tag);
     });
 
-    if (existingTags.length + newTags.length > 3) {
+    if (existingTags.length + newTags.length > MAX_TAGS_PER_RUN) {
       return json(
         {
-          error: `Runs can only have 3 tags, you're trying to set ${
+          error: `Runs can only have ${MAX_TAGS_PER_RUN} tags, you're trying to set ${
             existingTags.length + newTags.length
-          }.`,
+          }. These tags have not been set: ${newTags.map((t) => `'${t}'`).join(", ")}.`,
         },
         { status: 422 }
       );

apps/webapp/app/routes/resources.orgs.$organizationSlug.select-plan.tsx

@@ -119,7 +119,7 @@ const pricingDefinitions = {
   },
   schedules: {
     title: "Schedules",
-    content: "You can attach recurring schedules to tasks using CRON syntax.",
+    content: "You can attach recurring schedules to tasks using cron syntax.",
   },
   alerts: {
     title: "Alert destination",

apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts

@@ -1,21 +1,93 @@
 import { parse } from "@conform-to/zod";
-import { ActionFunction, json } from "@remix-run/node";
+import { ActionFunction, json, LoaderFunctionArgs } from "@remix-run/node";
+import { prettyPrintPacket } from "@trigger.dev/core/v3";
+import { typedjson } from "remix-typedjson";
 import { z } from "zod";
-import { prisma } from "~/db.server";
+import { $replica, prisma } from "~/db.server";
 import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
+import { displayableEnvironment } from "~/models/runtimeEnvironment.server";
 import { logger } from "~/services/logger.server";
+import { requireUserId } from "~/services/session.server";
+import { sortEnvironments } from "~/utils/environmentSort";
 import { v3RunSpanPath } from "~/utils/pathBuilder";
 import { ReplayTaskRunService } from "~/v3/services/replayTaskRun.server";
 
-const FormSchema = z.object({
-  failedRedirect: z.string(),
-});
-
 const ParamSchema = z.object({
   runParam: z.string(),
 });
 
+export async function loader({ request, params }: LoaderFunctionArgs) {
+  const userId = await requireUserId(request);
+  const { runParam } = ParamSchema.parse(params);
+
+  const run = await $replica.taskRun.findFirst({
+    select: {
+      payload: true,
+      payloadType: true,
+      runtimeEnvironmentId: true,
+      project: {
+        select: {
+          environments: {
+            select: {
+              id: true,
+              type: true,
+              slug: true,
+              orgMember: {
+                select: {
+                  user: true,
+                },
+              },
+            },
+            where: {
+              OR: [
+                {
+                  type: {
+                    in: ["PREVIEW", "STAGING", "PRODUCTION"],
+                  },
+                },
+                {
+                  type: "DEVELOPMENT",
+                  orgMember: {
+                    userId,
+                  },
+                },
+              ],
+            },
+          },
+        },
+      },
+    },
+    where: { friendlyId: runParam, project: { organization: { members: { some: { userId } } } } },
+  });
+
+  if (!run) {
+    throw new Response("Not Found", { status: 404 });
+  }
+
+  const environment = run.project.environments.find((env) => env.id === run.runtimeEnvironmentId);
+  if (!environment) {
+    throw new Response("Environment not found", { status: 404 });
+  }
+
+  return typedjson({
+    payload: await prettyPrintPacket(run.payload, run.payloadType),
+    payloadType: run.payloadType,
+    environment: displayableEnvironment(environment, userId),
+    environments: sortEnvironments(
+      run.project.environments.map((environment) => displayableEnvironment(environment, userId))
+    ),
+  });
+}
+
+const FormSchema = z.object({
+  environment: z.string().optional(),
+  payload: z.string().optional(),
+  failedRedirect: z.string(),
+});
+
 export const action: ActionFunction = async ({ request, params }) => {
+  const userId = await requireUserId(request);
+
   const { runParam } = ParamSchema.parse(params);
 
   const formData = await request.formData();
@@ -44,7 +116,10 @@ export const action: ActionFunction = async ({ request, params }) => {
     }
 
     const replayRunService = new ReplayTaskRunService();
-    const newRun = await replayRunService.call(taskRun);
+    const newRun = await replayRunService.call(taskRun, {
+      environmentId: submission.value.environment,
+      payload: submission.value.payload,
+    });
 
     if (!newRun) {
       return redirectWithErrorMessage(

apps/webapp/app/routes/storybook.badges/route.tsx

@@ -8,7 +8,6 @@ export default function Story() {
         <Badge variant="small">Small</Badge>
       </div>
       <Badge variant="outline">Outline</Badge>
-      <Badge variant="v3">v3</Badge>
       <Badge variant="outline-rounded">Outline rounded</Badge>
     </div>
   );

apps/webapp/app/services/autoIncrementCounter.server.ts

@@ -31,27 +31,21 @@ export class AutoIncrementCounter {
     let performedBackfill = false;
 
     try {
-      return await $transaction(
-        client,
-        async (tx) => {
-          let newNumber = await this.#increment(key);
+      let newNumber = await this.#increment(key);
 
-          performedIncrement = true;
+      performedIncrement = true;
 
-          if (newNumber === 1 && backfiller) {
-            const backfilledNumber = await backfiller(key, tx);
+      if (newNumber === 1 && backfiller) {
+        const backfilledNumber = await backfiller(key, client);
 
-            if (backfilledNumber && backfilledNumber > 1) {
-              newNumber = backfilledNumber + 1;
-              await this._redis.set(key, newNumber);
-              performedBackfill = true;
-            }
-          }
+        if (backfilledNumber && backfilledNumber > 1) {
+          newNumber = backfilledNumber + 1;
+          await this._redis.set(key, newNumber);
+          performedBackfill = true;
+        }
+      }
 
-          return await callback(newNumber, tx);
-        },
-        transactionOptions
-      );
+      return await callback(newNumber, client);
     } catch (e) {
       if (
         e instanceof Prisma.PrismaClientKnownRequestError ||

apps/webapp/app/services/worker.server.ts

@@ -4,7 +4,6 @@ import { z } from "zod";
 import { prisma } from "~/db.server";
 import { env } from "~/env.server";
 import { ZodWorker } from "~/platform/zodWorker.server";
-import { eventRepository } from "~/v3/eventRepository.server";
 import { MarqsConcurrencyMonitor } from "~/v3/marqs/concurrencyMonitor.server";
 import { RequeueV2Message } from "~/v3/marqs/requeueV2Message.server";
 import { RequeueTaskRunService } from "~/v3/requeueTaskRun.server";
@@ -132,7 +131,6 @@ const workerCatalog = {
   }),
   "v3.resumeBatchRun": z.object({
     batchRunId: z.string(),
-    sourceTaskAttemptId: z.string(),
   }),
   "v3.resumeTaskDependency": z.object({
     dependencyId: z.string(),
@@ -312,13 +310,6 @@ function getWorkerQueue() {
           });
         },
       },
-      // Run this every hour at the 13 minute mark
-      purgeOldTaskEvents: {
-        match: "47 * * * *",
-        handler: async (payload, job) => {
-          await eventRepository.truncateEvents();
-        },
-      },
       "marqs.v3.queueConcurrencyMonitor": {
         // run every 5 minutes
         match: "*/5 * * * *",
@@ -558,7 +549,7 @@ function getWorkerQueue() {
         handler: async (payload, job) => {
           const service = new ResumeBatchRunService();
 
-          return await service.call(payload.batchRunId, payload.sourceTaskAttemptId);
+          return await service.call(payload.batchRunId);
         },
       },
       "v3.resumeTaskDependency": {

apps/webapp/app/utils/pathBuilder.ts

@@ -323,6 +323,10 @@ export function v3EnvironmentVariablesPath(organization: OrgForPath, project: Pr
   return `${v3ProjectPath(organization, project)}/environment-variables`;
 }
 
+export function v3ConcurrencyPath(organization: OrgForPath, project: ProjectForPath) {
+  return `${v3ProjectPath(organization, project)}/concurrency`;
+}
+
 export function v3NewEnvironmentVariablesPath(organization: OrgForPath, project: ProjectForPath) {
   return `${v3EnvironmentVariablesPath(organization, project)}/new`;
 }

apps/webapp/app/v3/eventRepository.server.ts

@@ -1102,16 +1102,6 @@ export class EventRepository {
     return this._randomIdGenerator.generateSpanId();
   }
 
-  public async truncateEvents() {
-    await this.db.taskEvent.deleteMany({
-      where: {
-        createdAt: {
-          lt: new Date(Date.now() - this._config.retentionInDays * 24 * 60 * 60 * 1000),
-        },
-      },
-    });
-  }
-
   /**
    * Returns a deterministically random 8-byte span ID formatted/encoded as a 16 lowercase hex
    * characters corresponding to 64 bits, based on the trace ID and seed.

apps/webapp/app/v3/failedTaskRun.server.ts

@@ -1,12 +1,9 @@
 import { TaskRunFailedExecutionResult } from "@trigger.dev/core/v3";
 import { logger } from "~/services/logger.server";
-import { marqs } from "~/v3/marqs/index.server";
-
-import { TaskRunStatus } from "@trigger.dev/database";
 import { createExceptionPropertiesFromError, eventRepository } from "./eventRepository.server";
 import { BaseService } from "./services/baseService.server";
-
-const FAILABLE_TASK_RUN_STATUSES: TaskRunStatus[] = ["EXECUTING", "PENDING", "WAITING_FOR_DEPLOY"];
+import { FinalizeTaskRunService } from "./services/finalizeTaskRun.server";
+import { FAILABLE_RUN_STATUSES } from "./taskStatus";
 
 export class FailedTaskRunService extends BaseService {
   public async call(anyRunId: string, completion: TaskRunFailedExecutionResult) {
@@ -28,7 +25,7 @@ export class FailedTaskRunService extends BaseService {
       return;
     }
 
-    if (!FAILABLE_TASK_RUN_STATUSES.includes(taskRun.status)) {
+    if (!FAILABLE_RUN_STATUSES.includes(taskRun.status)) {
       logger.error("[FailedTaskRunService] Task run is not in a failable state", {
         taskRun,
         completion,
@@ -40,7 +37,12 @@ export class FailedTaskRunService extends BaseService {
     // No more retries, we need to fail the task run
     logger.debug("[FailedTaskRunService] Failing task run", { taskRun, completion });
 
-    await marqs?.acknowledgeMessage(taskRun.id);
+    const finalizeService = new FinalizeTaskRunService();
+    await finalizeService.call({
+      id: taskRun.id,
+      status: "SYSTEM_FAILURE",
+      completedAt: new Date(),
+    });
 
     // Now we need to "complete" the task run event/span
     await eventRepository.completeEvent(taskRun.spanId, {
@@ -58,14 +60,5 @@ export class FailedTaskRunService extends BaseService {
         },
       ],
     });
-
-    await this._prisma.taskRun.update({
-      where: {
-        id: taskRun.id,
-      },
-      data: {
-        status: "SYSTEM_FAILURE",
-      },
-    });
   }
 }

apps/webapp/app/v3/handleSocketIo.server.ts

@@ -128,6 +128,7 @@ function createCoordinatorNamespace(io: Server) {
           completion: message.completion,
           execution: message.execution,
           checkpoint: message.checkpoint,
+          supportsRetryCheckpoints: message.version === "v1",
         });
       },
       TASK_RUN_FAILED_TO_RUN: async (message) => {

apps/webapp/app/v3/machinePresets.server.ts

@@ -24,7 +24,7 @@ export function machinePresetFromConfig(config: unknown): MachinePreset {
   return machinePresetFromName("small-1x");
 }
 
-function machinePresetFromName(name: MachinePresetName): MachinePreset {
+export function machinePresetFromName(name: MachinePresetName): MachinePreset {
   return {
     name,
     ...machines[name],

apps/webapp/app/v3/marqs/index.server.ts

@@ -26,11 +26,13 @@ import {
   MarQSKeyProducer,
   MarQSQueuePriorityStrategy,
   MessagePayload,
+  MessageQueueSubscriber,
   QueueCapacities,
   QueueRange,
   VisibilityTimeoutStrategy,
 } from "./types";
 import { V3VisibilityTimeout } from "./v3VisibilityTimeout.server";
+import { concurrencyTracker } from "../services/taskRunConcurrencyTracker.server";
 
 const KEY_PREFIX = "marqs:";
 
@@ -60,6 +62,7 @@ export type MarQSOptions = {
   visibilityTimeoutStrategy: VisibilityTimeoutStrategy;
   enableRebalancing?: boolean;
   verbose?: boolean;
+  subscriber?: MessageQueueSubscriber;
 };
 
 /**
@@ -207,6 +210,8 @@ export class MarQS {
         });
 
         await this.#callEnqueueMessage(messagePayload);
+
+        await this.options.subscriber?.messageEnqueued(messagePayload);
       },
       {
         kind: SpanKind.PRODUCER,
@@ -264,6 +269,8 @@ export class MarQS {
             [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey,
             [SemanticAttributes.PARENT_QUEUE]: message.parentQueue,
           });
+
+          await this.options.subscriber?.messageDequeued(message);
         } else {
           logger.error(`Failed to read message, undoing the dequeueing of the message`, {
             messageData,
@@ -379,6 +386,8 @@ export class MarQS {
             [SemanticAttributes.CONCURRENCY_KEY]: message.concurrencyKey,
             [SemanticAttributes.PARENT_QUEUE]: message.parentQueue,
           });
+
+          await this.options.subscriber?.messageDequeued(message);
         }
 
         await this.options.visibilityTimeoutStrategy.heartbeat(
@@ -427,6 +436,8 @@ export class MarQS {
           orgConcurrencyKey: this.keys.orgCurrentConcurrencyKeyFromQueue(message.queue),
           messageId,
         });
+
+        await this.options.subscriber?.messageAcked(message);
       },
       {
         kind: SpanKind.CONSUMER,
@@ -469,7 +480,7 @@ export class MarQS {
         const newMessage: MessagePayload = {
           version: "1",
           // preserve original trace context
-          data: { ...messageData, ...traceContext },
+          data: { ...oldMessage.data, ...messageData, ...traceContext },
           queue: oldMessage.queue,
           concurrencyKey: oldMessage.concurrencyKey,
           timestamp: timestamp ?? Date.now(),
@@ -496,6 +507,8 @@ export class MarQS {
         });
 
         await this.#callEnqueueMessage(newMessage);
+
+        await this.options.subscriber?.messageReplaced(newMessage);
       },
       {
         kind: SpanKind.CONSUMER,
@@ -580,6 +593,8 @@ export class MarQS {
           messageId,
           messageScore: retryAt,
         });
+
+        await this.options.subscriber?.messageNacked(message);
       },
       {
         kind: SpanKind.CONSUMER,
@@ -645,17 +660,21 @@ export class MarQS {
     return this.#trace(
       "getRandomQueueFromParentQueue",
       async (span) => {
+        span.setAttribute("consumerId", consumerId);
+
         const { range } = await queuePriorityStrategy.nextCandidateSelection(
           parentQueue,
           consumerId
         );
 
-        const queues = await this.#getChildQueuesWithScores(parentQueue, range);
+        const queues = await this.#getChildQueuesWithScores(parentQueue, range, span);
+        span.setAttribute("queueCount", queues.length);
 
         const queuesWithScores = await this.#calculateQueueScores(queues, calculateCapacities);
+        span.setAttribute("queuesWithScoresCount", queuesWithScores.length);
 
         // We need to priority shuffle here to ensure all workers aren't just working on the highest priority queue
-        const choice = this.queuePriorityStrategy.chooseQueue(
+        const { choice, nextRange } = this.queuePriorityStrategy.chooseQueue(
           queuesWithScores,
           parentQueue,
           consumerId,
@@ -668,17 +687,20 @@ export class MarQS {
         span.setAttributes({
           ...flattenAttributes(queuesWithScores, "marqs.queuesWithScores"),
         });
-        span.setAttribute("nextRange.offset", range.offset);
-        span.setAttribute("nextRange.count", range.count);
-        span.setAttribute("queueCount", queues.length);
+        span.setAttribute("range.offset", range.offset);
+        span.setAttribute("range.count", range.count);
+        span.setAttribute("nextRange.offset", nextRange.offset);
+        span.setAttribute("nextRange.count", nextRange.count);
 
-        if (this.options.verbose) {
+        if (this.options.verbose || nextRange.offset > 0) {
           if (typeof choice === "string") {
             logger.debug(`[${this.name}] getRandomQueueFromParentQueue`, {
               queues,
               queuesWithScores,
-              nextRange: range,
+              range,
+              nextRange,
               queueCount: queues.length,
+              queuesWithScoresCount: queuesWithScores.length,
               queueChoice: choice,
               consumerId,
             });
@@ -686,8 +708,10 @@ export class MarQS {
             logger.debug(`[${this.name}] getRandomQueueFromParentQueue`, {
               queues,
               queuesWithScores,
-              nextRange: range,
+              range,
+              nextRange,
               queueCount: queues.length,
+              queuesWithScoresCount: queuesWithScores.length,
               noQueueChoice: true,
               consumerId,
             });
@@ -752,7 +776,8 @@ export class MarQS {
 
   async #getChildQueuesWithScores(
     key: string,
-    range: QueueRange
+    range: QueueRange,
+    span?: Span
   ): Promise<Array<{ value: string; score: number }>> {
     const valuesWithScores = await this.redis.zrangebyscore(
       key,
@@ -763,6 +788,12 @@ export class MarQS {
       range.offset,
       range.count
     );
+
+    span?.setAttribute("zrangebyscore.valuesWithScores.rawLength", valuesWithScores.length);
+    span?.setAttributes({
+      ...flattenAttributes(valuesWithScores, "zrangebyscore.valuesWithScores.rawValues"),
+    });
+
     const result: Array<{ value: string; score: number }> = [];
 
     for (let i = 0; i < valuesWithScores.length; i += 2) {
@@ -1705,6 +1736,7 @@ function getMarQSClient() {
         defaultOrgConcurrency: env.DEFAULT_ORG_EXECUTION_CONCURRENCY_LIMIT,
         visibilityTimeoutInMs: 120 * 1000, // 2 minutes,
         enableRebalancing: !env.MARQS_DISABLE_REBALANCING,
+        subscriber: concurrencyTracker,
       });
     } else {
       console.warn(

apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts

@@ -37,7 +37,7 @@ import {
 import { CrashTaskRunService } from "../services/crashTaskRun.server";
 import { CreateTaskRunAttemptService } from "../services/createTaskRunAttempt.server";
 import { RestoreCheckpointService } from "../services/restoreCheckpoint.server";
-import { tracer } from "../tracer.server";
+import { SEMINTATTRS_FORCE_RECORDING, tracer } from "../tracer.server";
 import { generateJWTTokenForEnvironment } from "~/services/apiAuth.server";
 import { EnvironmentVariable } from "../environmentVariables/repository";
 import { machinePresetFromConfig } from "../machinePresets.server";
@@ -54,6 +54,7 @@ export const SharedQueueMessageBody = z.discriminatedUnion("type", [
     type: z.literal("EXECUTE"),
     taskIdentifier: z.string(),
     checkpointEventId: z.string().optional(),
+    retryCheckpointsDisabled: z.boolean().optional(),
   }),
   WithTraceContext.extend({
     type: z.literal("RESUME"),
@@ -103,8 +104,8 @@ export class SharedQueueConsumer {
     options: SharedQueueConsumerOptions = {}
   ) {
     this._options = {
-      maximumItemsPerTrace: options.maximumItemsPerTrace ?? 1_000, // 1k items per trace
-      traceTimeoutSeconds: options.traceTimeoutSeconds ?? 60, // 60 seconds
+      maximumItemsPerTrace: options.maximumItemsPerTrace ?? 500,
+      traceTimeoutSeconds: options.traceTimeoutSeconds ?? 10,
       nextTickInterval: options.nextTickInterval ?? 1000, // 1 second
       interval: options.interval ?? 100, // 100ms
     };
@@ -213,6 +214,9 @@ export class SharedQueueConsumer {
         "SharedQueueConsumer.doWork()",
         {
           kind: SpanKind.CONSUMER,
+          attributes: {
+            [SEMINTATTRS_FORCE_RECORDING]: true,
+          },
         },
         ROOT_CONTEXT
       );
@@ -274,14 +278,6 @@ export class SharedQueueConsumer {
           where: {
             id: message.messageId,
           },
-          include: {
-            lockedToVersion: {
-              include: {
-                deployment: true,
-                tasks: true,
-              },
-            },
-          },
         });
 
         if (!existingTaskRun) {
@@ -299,12 +295,9 @@ export class SharedQueueConsumer {
 
         const retryingFromCheckpoint = !!messageBody.data.checkpointEventId;
 
-        const EXECUTABLE_RUN_STATUSES: {
-          fromCheckpoint: TaskRunStatus[];
-          withoutCheckpoint: TaskRunStatus[];
-        } = {
-          fromCheckpoint: ["WAITING_TO_RESUME"],
-          withoutCheckpoint: ["PENDING", "RETRYING_AFTER_FAILURE"],
+        const EXECUTABLE_RUN_STATUSES = {
+          fromCheckpoint: ["WAITING_TO_RESUME"] satisfies TaskRunStatus[],
+          withoutCheckpoint: ["PENDING", "RETRYING_AFTER_FAILURE"] satisfies TaskRunStatus[],
         };
 
         if (
@@ -479,7 +472,10 @@ export class SharedQueueConsumer {
           ? lockedTaskRun.attempts[0].number + 1
           : 1;
 
-        const isRetry = lockedTaskRun.status === "WAITING_TO_RESUME" && nextAttemptNumber > 1;
+        const isRetry =
+          nextAttemptNumber > 1 &&
+          (lockedTaskRun.status === "WAITING_TO_RESUME" ||
+            lockedTaskRun.status === "RETRYING_AFTER_FAILURE");
 
         try {
           if (messageBody.data.checkpointEventId) {
@@ -520,11 +516,13 @@ export class SharedQueueConsumer {
             }
           }
 
-          if (isRetry) {
+          if (isRetry && !messageBody.data.retryCheckpointsDisabled) {
             socketIo.coordinatorNamespace.emit("READY_FOR_RETRY", {
               version: "v1",
               runId: lockedTaskRun.id,
             });
+
+            // Retries for workers with disabled retry checkpoints will be handled just like normal attempts
           } else {
             const machineConfig = lockedTaskRun.lockedBy?.machineConfig;
             const machine = machinePresetFromConfig(machineConfig ?? {});
@@ -536,6 +534,7 @@ export class SharedQueueConsumer {
                 image: deployment.imageReference,
                 version: deployment.version,
                 machine,
+                nextAttemptNumber,
                 // identifiers
                 id: "placeholder", // TODO: Remove this completely in a future release
                 envId: lockedTaskRun.runtimeEnvironment.id,
@@ -618,16 +617,6 @@ export class SharedQueueConsumer {
           return;
         }
 
-        if (messageBody.data.completedAttemptIds.length < 1) {
-          logger.error("No attempt IDs provided", {
-            queueMessage: message.data,
-            messageId: message.messageId,
-          });
-
-          await this.#ackAndDoMoreWork(message.messageId);
-          return;
-        }
-
         const resumableRun = await prisma.taskRun.findUnique({
           where: {
             id: message.messageId,

apps/webapp/app/v3/marqs/simpleWeightedPriorityStrategy.server.ts

@@ -1,4 +1,3 @@
-import { RedisOptions } from "ioredis";
 import { nanoid } from "nanoid";
 import {
   MarQSQueuePriorityStrategy,
@@ -32,7 +31,7 @@ export class SimpleWeightedChoiceStrategy implements MarQSQueuePriorityStrategy
     parentQueue: string,
     consumerId: string,
     previousRange: QueueRange
-  ): PriorityStrategyChoice {
+  ): { choice: PriorityStrategyChoice; nextRange: QueueRange } {
     const filteredQueues = filterQueuesAtCapacity(queues);
 
     if (queues.length === this.options.queueSelectionCount) {
@@ -40,6 +39,7 @@ export class SimpleWeightedChoiceStrategy implements MarQSQueuePriorityStrategy
         offset: previousRange.offset + this.options.queueSelectionCount,
         count: this.options.queueSelectionCount,
       };
+
       // If all queues are at capacity, and we were passed the max number of queues, then we will slide the window "to the right"
       this._nextRangesByParentQueue.set(`${consumerId}:${parentQueue}`, nextRange);
     } else {
@@ -47,12 +47,20 @@ export class SimpleWeightedChoiceStrategy implements MarQSQueuePriorityStrategy
     }
 
     if (filteredQueues.length === 0) {
-      return { abort: true };
+      return {
+        choice: { abort: true },
+        nextRange: this.nextRangeForParentQueue(parentQueue, consumerId),
+      };
     }
 
     const queueWeights = this.#calculateQueueWeights(filteredQueues);
 
-    return weightedRandomChoice(queueWeights);
+    const choice = weightedRandomChoice(queueWeights);
+
+    return {
+      choice,
+      nextRange: this.nextRangeForParentQueue(parentQueue, consumerId),
+    };
   }
 
   async nextCandidateSelection(
@@ -117,8 +125,8 @@ export class NoopWeightedChoiceStrategy implements MarQSQueuePriorityStrategy {
     queues: QueueWithScores[],
     parentQueue: string,
     selectionId: string
-  ): PriorityStrategyChoice {
-    return { abort: true };
+  ): { choice: PriorityStrategyChoice; nextRange: QueueRange } {
+    return { choice: { abort: true }, nextRange: { offset: 0, count: 0 } };
   }
 
   nextCandidateSelection(parentQueue: string): Promise<{ range: QueueRange; selectionId: string }> {

apps/webapp/app/v3/marqs/types.ts

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { type AuthenticatedEnvironment } from "~/services/apiAuth.server";
 
 export type QueueCapacity = {
   current: number;
@@ -65,7 +65,7 @@ export interface MarQSQueuePriorityStrategy {
     parentQueue: string,
     consumerId: string,
     previousRange: QueueRange
-  ): PriorityStrategyChoice;
+  ): { choice: PriorityStrategyChoice; nextRange: QueueRange };
 
   /**
    * This function is called to get the next candidate selection for the queue
@@ -92,6 +92,14 @@ export const MessagePayload = z.object({
 
 export type MessagePayload = z.infer<typeof MessagePayload>;
 
+export interface MessageQueueSubscriber {
+  messageEnqueued(message: MessagePayload): Promise<void>;
+  messageDequeued(message: MessagePayload): Promise<void>;
+  messageAcked(message: MessagePayload): Promise<void>;
+  messageNacked(message: MessagePayload): Promise<void>;
+  messageReplaced(message: MessagePayload): Promise<void>;
+}
+
 export interface VisibilityTimeoutStrategy {
   heartbeat(messageId: string, timeoutInMs: number): Promise<void>;
   cancelHeartbeat(messageId: string): Promise<void>;

apps/webapp/app/v3/models/workerDeployment.server.ts

@@ -1,6 +1,8 @@
 import type { Prettify } from "@trigger.dev/core";
+import { BackgroundWorker } from "@trigger.dev/database";
 import { CURRENT_DEPLOYMENT_LABEL } from "~/consts";
 import { Prisma, prisma } from "~/db.server";
+import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 
 export type CurrentWorkerDeployment = Prettify<
   NonNullable<Awaited<ReturnType<typeof findCurrentWorkerDeployment>>>
@@ -42,6 +44,25 @@ export async function findCurrentWorkerDeployment(
   return promotion?.deployment;
 }
 
+export async function findCurrentWorkerFromEnvironment(
+  environment: Pick<AuthenticatedEnvironment, "id" | "type">
+): Promise<BackgroundWorker | null> {
+  if (environment.type === "DEVELOPMENT") {
+    const latestDevWorker = await prisma.backgroundWorker.findFirst({
+      where: {
+        runtimeEnvironmentId: environment.id,
+      },
+      orderBy: {
+        createdAt: "desc",
+      },
+    });
+    return latestDevWorker;
+  } else {
+    const deployment = await findCurrentWorkerDeployment(environment.id);
+    return deployment?.worker ?? null;
+  }
+}
+
 export async function getWorkerDeploymentFromWorker(
   workerId: string
 ): Promise<WorkerDeploymentWithWorkerTasks | undefined> {

apps/webapp/app/v3/registryProxy.server.ts

@@ -281,6 +281,11 @@ export class RegistryProxy {
 
       proxyReq.on("error", (error) => {
         logger.error("Error proxying request", { error: error.message });
+
+        if (response.headersSent) {
+          return;
+        }
+
         response.writeHead(500, { "Content-Type": "text/plain" });
         response.end(`Internal Server Error: ${error.message}`);
       });

apps/webapp/app/v3/services/cancelAttempt.server.ts

@@ -1,11 +1,10 @@
-import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { $transaction, type PrismaClientOrTransaction, prisma } from "~/db.server";
+import { type AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
-import { marqs } from "~/v3/marqs/index.server";
 import { eventRepository } from "../eventRepository.server";
-import { BaseService } from "./baseService.server";
-
-import { PrismaClientOrTransaction, prisma } from "~/db.server";
 import { isCancellableRunStatus } from "../taskStatus";
+import { BaseService } from "./baseService.server";
+import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
 import { ResumeTaskRunDependenciesService } from "./resumeTaskRunDependencies.server";
 
 export class CancelAttemptService extends BaseService {
@@ -51,25 +50,25 @@ export class CancelAttemptService extends BaseService {
         return;
       }
 
-      await marqs?.acknowledgeMessage(taskRunId);
-
-      await this._prisma.taskRunAttempt.update({
-        where: {
-          friendlyId: attemptId,
-        },
-        data: {
-          status: "CANCELED",
-          completedAt: cancelledAt,
-          taskRun: {
-            update: {
-              data: {
-                status: isCancellableRunStatus(taskRunAttempt.taskRun.status)
-                  ? "INTERRUPTED"
-                  : undefined,
-              },
-            },
+      await $transaction(this._prisma, async (tx) => {
+        await tx.taskRunAttempt.update({
+          where: {
+            friendlyId: attemptId,
           },
-        },
+          data: {
+            status: "CANCELED",
+            completedAt: cancelledAt,
+          },
+        });
+
+        const finalizeService = new FinalizeTaskRunService(tx);
+        await finalizeService.call({
+          id: taskRunId,
+          status: isCancellableRunStatus(taskRunAttempt.taskRun.status) ? "INTERRUPTED" : undefined,
+          completedAt: isCancellableRunStatus(taskRunAttempt.taskRun.status)
+            ? cancelledAt
+            : undefined,
+        });
       });
 
       const inProgressEvents = await eventRepository.queryIncompleteEvents({

apps/webapp/app/v3/services/cancelTaskRun.server.ts

@@ -1,14 +1,14 @@
-import { Prisma, TaskRun } from "@trigger.dev/database";
+import { type Prisma, type TaskRun } from "@trigger.dev/database";
 import assertNever from "assert-never";
 import { logger } from "~/services/logger.server";
-import { marqs } from "~/v3/marqs/index.server";
 import { eventRepository } from "../eventRepository.server";
 import { socketIo } from "../handleSocketIo.server";
 import { devPubSub } from "../marqs/devPubSub.server";
+import { CANCELLABLE_ATTEMPT_STATUSES, isCancellableRunStatus } from "../taskStatus";
 import { BaseService } from "./baseService.server";
 import { CancelAttemptService } from "./cancelAttempt.server";
-import { CANCELLABLE_ATTEMPT_STATUSES, isCancellableRunStatus } from "../taskStatus";
 import { CancelTaskAttemptDependenciesService } from "./cancelTaskAttemptDependencies.server";
+import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
 
 type ExtendedTaskRun = Prisma.TaskRunGetPayload<{
   include: {
@@ -47,17 +47,11 @@ export class CancelTaskRunService extends BaseService {
       return;
     }
 
-    // Remove the task run from the queue if it's there for some reason
-    await marqs?.acknowledgeMessage(taskRun.id);
-
-    // Set the task run status to cancelled
-    const cancelledTaskRun = await this._prisma.taskRun.update({
-      where: {
-        id: taskRun.id,
-      },
-      data: {
-        status: "CANCELED",
-      },
+    const finalizeService = new FinalizeTaskRunService();
+    const cancelledTaskRun = await finalizeService.call({
+      id: taskRun.id,
+      status: "CANCELED",
+      completedAt: opts.cancelledAt,
       include: {
         attempts: {
           where: {

apps/webapp/app/v3/services/completeAttempt.server.ts

@@ -3,12 +3,13 @@ import {
   TaskRunContext,
   TaskRunExecution,
   TaskRunExecutionResult,
+  TaskRunExecutionRetry,
   TaskRunFailedExecutionResult,
   TaskRunSuccessfulExecutionResult,
   flattenAttributes,
   sanitizeError,
 } from "@trigger.dev/core/v3";
-import { PrismaClientOrTransaction } from "~/db.server";
+import { $transaction, PrismaClientOrTransaction } from "~/db.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { safeJsonParse } from "~/utils/json";
@@ -23,6 +24,8 @@ import { TaskRun } from "@trigger.dev/database";
 import { PerformTaskAttemptAlertsService } from "./alerts/performTaskAttemptAlerts.server";
 import { RetryAttemptService } from "./retryAttempt.server";
 import { isFinalAttemptStatus, isFinalRunStatus } from "../taskStatus";
+import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
+import { env } from "~/env.server";
 
 type FoundAttempt = Awaited<ReturnType<typeof findAttempt>>;
 
@@ -37,11 +40,13 @@ export class CompleteAttemptService extends BaseService {
     execution,
     env,
     checkpoint,
+    supportsRetryCheckpoints,
   }: {
     completion: TaskRunExecutionResult;
     execution: TaskRunExecution;
     env?: AuthenticatedEnvironment;
     checkpoint?: CheckpointData;
+    supportsRetryCheckpoints?: boolean;
   }): Promise<"COMPLETED" | "RETRIED"> {
     const taskRunAttempt = await findAttempt(this._prisma, execution.attempt.id);
 
@@ -50,16 +55,30 @@ export class CompleteAttemptService extends BaseService {
         id: execution.attempt.id,
       });
 
-      // Update the task run to be failed
-      await this._prisma.taskRun.update({
+      const run = await this._prisma.taskRun.findFirst({
         where: {
           friendlyId: execution.run.id,
         },
-        data: {
-          status: "SYSTEM_FAILURE",
+        select: {
+          id: true,
         },
       });
 
+      if (!run) {
+        logger.error("[CompleteAttemptService] Task run not found", {
+          friendlyId: execution.run.id,
+        });
+
+        return "COMPLETED";
+      }
+
+      const finalizeService = new FinalizeTaskRunService();
+      await finalizeService.call({
+        id: run.id,
+        status: "SYSTEM_FAILURE",
+        completedAt: new Date(),
+      });
+
       // No attempt, so there's no message to ACK
       return "COMPLETED";
     }
@@ -80,13 +99,14 @@ export class CompleteAttemptService extends BaseService {
     if (completion.ok) {
       return await this.#completeAttemptSuccessfully(completion, taskRunAttempt, env);
     } else {
-      return await this.#completeAttemptFailed(
+      return await this.#completeAttemptFailed({
         completion,
         execution,
         taskRunAttempt,
         env,
-        checkpoint
-      );
+        checkpoint,
+        supportsRetryCheckpoints,
+      });
     }
   }
 
@@ -95,28 +115,26 @@ export class CompleteAttemptService extends BaseService {
     taskRunAttempt: NonNullable<FoundAttempt>,
     env?: AuthenticatedEnvironment
   ): Promise<"COMPLETED"> {
-    await this._prisma.taskRunAttempt.update({
-      where: { id: taskRunAttempt.id },
-      data: {
-        status: "COMPLETED",
-        completedAt: new Date(),
-        output: completion.output,
-        outputType: completion.outputType,
-        usageDurationMs: completion.usage?.durationMs,
-        taskRun: {
-          update: {
-            data: {
-              status: "COMPLETED_SUCCESSFULLY",
-            },
-          },
+    await $transaction(this._prisma, async (tx) => {
+      await tx.taskRunAttempt.update({
+        where: { id: taskRunAttempt.id },
+        data: {
+          status: "COMPLETED",
+          completedAt: new Date(),
+          output: completion.output,
+          outputType: completion.outputType,
+          usageDurationMs: completion.usage?.durationMs,
         },
-      },
+      });
+
+      const finalizeService = new FinalizeTaskRunService(tx);
+      await finalizeService.call({
+        id: taskRunAttempt.taskRunId,
+        status: "COMPLETED_SUCCESSFULLY",
+        completedAt: new Date(),
+      });
     });
 
-    logger.debug("Completed attempt successfully, ACKing message");
-
-    await marqs?.acknowledgeMessage(taskRunAttempt.taskRunId);
-
     // Now we need to "complete" the task run event/span
     await eventRepository.completeEvent(taskRunAttempt.taskRun.spanId, {
       endTime: new Date(),
@@ -139,13 +157,21 @@ export class CompleteAttemptService extends BaseService {
     return "COMPLETED";
   }
 
-  async #completeAttemptFailed(
-    completion: TaskRunFailedExecutionResult,
-    execution: TaskRunExecution,
-    taskRunAttempt: NonNullable<FoundAttempt>,
-    env?: AuthenticatedEnvironment,
-    checkpoint?: CheckpointData
-  ): Promise<"COMPLETED" | "RETRIED"> {
+  async #completeAttemptFailed({
+    completion,
+    execution,
+    taskRunAttempt,
+    env,
+    checkpoint,
+    supportsRetryCheckpoints,
+  }: {
+    completion: TaskRunFailedExecutionResult;
+    execution: TaskRunExecution;
+    taskRunAttempt: NonNullable<FoundAttempt>;
+    env?: AuthenticatedEnvironment;
+    checkpoint?: CheckpointData;
+    supportsRetryCheckpoints?: boolean;
+  }): Promise<"COMPLETED" | "RETRIED"> {
     if (
       completion.error.type === "INTERNAL_ERROR" &&
       completion.error.code === "TASK_RUN_CANCELLED"
@@ -230,12 +256,13 @@ export class CompleteAttemptService extends BaseService {
       }
 
       if (!checkpoint) {
-        await this.#retryAttempt(
-          taskRunAttempt.taskRun,
-          completion.retry.timestamp,
-          undefined,
-          taskRunAttempt.backgroundWorker.supportsLazyAttempts
-        );
+        await this.#retryAttempt({
+          run: taskRunAttempt.taskRun,
+          retry: completion.retry,
+          supportsLazyAttempts: taskRunAttempt.backgroundWorker.supportsLazyAttempts,
+          supportsRetryCheckpoints,
+        });
+
         return "RETRIED";
       }
 
@@ -250,37 +277,29 @@ export class CompleteAttemptService extends BaseService {
         },
       });
 
-      if (!checkpointCreateResult) {
+      if (!checkpointCreateResult.success) {
         logger.error("Failed to create checkpoint", { checkpoint, execution: execution.run.id });
 
-        // Update the task run to be failed
-        await this._prisma.taskRun.update({
-          where: {
-            friendlyId: execution.run.id,
-          },
-          data: {
-            status: "SYSTEM_FAILURE",
-          },
+        const finalizeService = new FinalizeTaskRunService();
+        await finalizeService.call({
+          id: taskRunAttempt.taskRunId,
+          status: "SYSTEM_FAILURE",
+          completedAt: new Date(),
         });
 
-        await marqs?.acknowledgeMessage(taskRunAttempt.taskRunId);
-
         return "COMPLETED";
       }
 
-      await this.#retryAttempt(
-        taskRunAttempt.taskRun,
-        completion.retry.timestamp,
-        checkpointCreateResult.event.id
-      );
+      await this.#retryAttempt({
+        run: taskRunAttempt.taskRun,
+        retry: completion.retry,
+        checkpointEventId: checkpointCreateResult.event.id,
+        supportsLazyAttempts: taskRunAttempt.backgroundWorker.supportsLazyAttempts,
+        supportsRetryCheckpoints,
+      });
 
       return "RETRIED";
     } else {
-      // No more retries, we need to fail the task run
-      logger.debug("Completed attempt, ACKing message", taskRunAttempt);
-
-      await marqs?.acknowledgeMessage(taskRunAttempt.taskRunId);
-
       // Now we need to "complete" the task run event/span
       await eventRepository.completeEvent(taskRunAttempt.taskRun.spanId, {
         endTime: new Date(),
@@ -302,6 +321,13 @@ export class CompleteAttemptService extends BaseService {
         sanitizedError.type === "INTERNAL_ERROR" &&
         sanitizedError.code === "GRACEFUL_EXIT_TIMEOUT"
       ) {
+        const finalizeService = new FinalizeTaskRunService();
+        await finalizeService.call({
+          id: taskRunAttempt.taskRunId,
+          status: "SYSTEM_FAILURE",
+          completedAt: new Date(),
+        });
+
         // We need to fail all incomplete spans
         const inProgressEvents = await eventRepository.queryIncompleteEvents({
           attemptId: execution.attempt.id,
@@ -325,23 +351,12 @@ export class CompleteAttemptService extends BaseService {
             });
           })
         );
-
-        await this._prisma.taskRun.update({
-          where: {
-            id: taskRunAttempt.taskRunId,
-          },
-          data: {
-            status: "SYSTEM_FAILURE",
-          },
-        });
       } else {
-        await this._prisma.taskRun.update({
-          where: {
-            id: taskRunAttempt.taskRunId,
-          },
-          data: {
-            status: "COMPLETED_WITH_ERRORS",
-          },
+        const finalizeService = new FinalizeTaskRunService();
+        await finalizeService.call({
+          id: taskRunAttempt.taskRunId,
+          status: "COMPLETED_WITH_ERRORS",
+          completedAt: new Date(),
         });
       }
 
@@ -353,30 +368,64 @@ export class CompleteAttemptService extends BaseService {
     }
   }
 
-  async #retryAttempt(
-    run: TaskRun,
-    retryTimestamp: number,
-    checkpointEventId?: string,
-    supportsLazyAttempts?: boolean
-  ) {
-    if (checkpointEventId || !supportsLazyAttempts) {
-      // Workers without lazy attempt support always need to go through the queue, which is where the attempt is created
+  async #retryAttempt({
+    run,
+    retry,
+    checkpointEventId,
+    supportsLazyAttempts,
+    supportsRetryCheckpoints,
+  }: {
+    run: TaskRun;
+    retry: TaskRunExecutionRetry;
+    checkpointEventId?: string;
+    supportsLazyAttempts: boolean;
+    supportsRetryCheckpoints?: boolean;
+  }) {
+    const retryViaQueue = () => {
       // We have to replace a potential RESUME with EXECUTE to correctly retry the attempt
-      return await marqs?.replaceMessage(
+      return marqs?.replaceMessage(
         run.id,
         {
           type: "EXECUTE",
           taskIdentifier: run.taskIdentifier,
-          checkpointEventId: checkpointEventId,
+          checkpointEventId: supportsRetryCheckpoints ? checkpointEventId : undefined,
+          retryCheckpointsDisabled: !supportsRetryCheckpoints,
         },
-        retryTimestamp
+        retry.timestamp
       );
-    } else {
-      // There's no checkpoint and the worker supports lazy attempts
-      // This means the worker is still running and waiting for a retry message
-      // It supports lazy attempts so we can bypass the queue and send the message directly to it
-      RetryAttemptService.enqueue(run.id, this._prisma, new Date(retryTimestamp));
+    };
+
+    const retryDirectly = () => {
+      return RetryAttemptService.enqueue(run.id, this._prisma, new Date(retry.timestamp));
+    };
+
+    // There's a checkpoint, so we need to go through the queue
+    if (checkpointEventId) {
+      if (!supportsRetryCheckpoints) {
+        logger.error("Worker does not support retry checkpoints, but a checkpoint was created", {
+          runId: run.id,
+          checkpointEventId,
+        });
+      }
+
+      await retryViaQueue();
+      return;
     }
+
+    // Workers without lazy attempt support always need to go through the queue, which is where the attempt is created
+    if (!supportsLazyAttempts) {
+      await retryViaQueue();
+      return;
+    }
+
+    // Workers that never checkpoint between attempts will exit after completing their current attempt if the retry delay exceeds the threshold
+    if (!supportsRetryCheckpoints && retry.delay >= env.CHECKPOINT_THRESHOLD_IN_MS) {
+      await retryViaQueue();
+      return;
+    }
+
+    // The worker is still running and waiting for a retry message
+    await retryDirectly();
   }
 
   #generateMetadataAttributesForNextAttempt(execution: TaskRunExecution) {

apps/webapp/app/v3/services/crashTaskRun.server.ts

@@ -6,6 +6,8 @@ import { logger } from "~/services/logger.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { ResumeTaskRunDependenciesService } from "./resumeTaskRunDependencies.server";
 import { CRASHABLE_ATTEMPT_STATUSES, isCrashableRunStatus } from "../taskStatus";
+import { sanitizeError } from "@trigger.dev/core/v3";
+import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
 
 export type CrashTaskRunServiceOptions = {
   reason?: string;
@@ -42,17 +44,11 @@ export class CrashTaskRunService extends BaseService {
       return;
     }
 
-    // Remove the task run from the queue if it's there for some reason
-    await marqs?.acknowledgeMessage(taskRun.id);
-
-    // Set the task run status to crashed
-    const crashedTaskRun = await this._prisma.taskRun.update({
-      where: {
-        id: taskRun.id,
-      },
-      data: {
-        status: "CRASHED",
-      },
+    const finalizeService = new FinalizeTaskRunService();
+    const crashedTaskRun = await finalizeService.call({
+      id: taskRun.id,
+      status: "CRASHED",
+      completedAt: new Date(),
       include: {
         attempts: {
           where: {
@@ -110,7 +106,11 @@ export class CrashTaskRunService extends BaseService {
         attempt,
         crashedTaskRun,
         new Date(),
-        crashedTaskRun.runtimeEnvironment
+        crashedTaskRun.runtimeEnvironment,
+        {
+          reason: opts.reason,
+          logs: opts.logs,
+        }
       );
     }
   }
@@ -119,7 +119,11 @@ export class CrashTaskRunService extends BaseService {
     attempt: TaskRunAttempt,
     run: TaskRun,
     failedAt: Date,
-    environment: AuthenticatedEnvironment
+    environment: AuthenticatedEnvironment,
+    error: {
+      reason: string;
+      logs?: string;
+    }
   ) {
     return await this.traceWithEnv("failAttempt()", environment, async (span) => {
       span.setAttribute("taskRunId", run.id);
@@ -134,6 +138,12 @@ export class CrashTaskRunService extends BaseService {
         data: {
           status: "FAILED",
           completedAt: failedAt,
+          error: sanitizeError({
+            type: "INTERNAL_ERROR",
+            code: "TASK_RUN_CRASHED",
+            message: error.reason,
+            stackTrace: error.logs,
+          }),
         },
       });
 

apps/webapp/app/v3/services/createCheckpoint.server.ts

@@ -2,12 +2,19 @@ import { CoordinatorToPlatformMessages } from "@trigger.dev/core/v3";
 import type { InferSocketMessageSchema } from "@trigger.dev/core/v3/zodSocket";
 import type { Checkpoint, CheckpointRestoreEvent } from "@trigger.dev/database";
 import { logger } from "~/services/logger.server";
-import { generateFriendlyId } from "../friendlyIdentifiers";
 import { marqs } from "~/v3/marqs/index.server";
-import { CreateCheckpointRestoreEventService } from "./createCheckpointRestoreEvent.server";
+import { generateFriendlyId } from "../friendlyIdentifiers";
+import {
+  FINAL_ATTEMPT_STATUSES,
+  isFinalAttemptStatus,
+  isFinalRunStatus,
+  isFreezableAttemptStatus,
+  isFreezableRunStatus,
+} from "../taskStatus";
 import { BaseService } from "./baseService.server";
-import { CrashTaskRunService } from "./crashTaskRun.server";
-import { isFinalRunStatus, isFreezableAttemptStatus, isFreezableRunStatus } from "../taskStatus";
+import { CreateCheckpointRestoreEventService } from "./createCheckpointRestoreEvent.server";
+import { ResumeBatchRunService } from "./resumeBatchRun.server";
+import { ResumeTaskDependencyService } from "./resumeTaskDependency.server";
 
 export class CreateCheckpointService extends BaseService {
   public async call(
@@ -17,11 +24,15 @@ export class CreateCheckpointService extends BaseService {
     >
   ): Promise<
     | {
+        success: true;
         checkpoint: Checkpoint;
         event: CheckpointRestoreEvent;
         keepRunAlive: boolean;
       }
-    | undefined
+    | {
+        success: false;
+        keepRunAlive?: boolean;
+      }
   > {
     logger.debug(`Creating checkpoint`, params);
 
@@ -46,7 +57,10 @@ export class CreateCheckpointService extends BaseService {
 
     if (!attempt) {
       logger.error("Attempt not found", { attemptFriendlyId: params.attemptFriendlyId });
-      return;
+
+      return {
+        success: false,
+      };
     }
 
     if (
@@ -64,14 +78,10 @@ export class CreateCheckpointService extends BaseService {
         },
       });
 
-      // This should only affect CLIs < beta.24, in very limited scenarios
-      const service = new CrashTaskRunService(this._prisma);
-      await service.call(attempt.taskRunId, {
-        crashAttempts: true,
-        reason: "Unfreezable state: Please upgrade your CLI",
-      });
-
-      return;
+      return {
+        success: false,
+        keepRunAlive: true,
+      };
     }
 
     const imageRef = attempt.backgroundWorker.deployment?.imageReference;
@@ -81,15 +91,22 @@ export class CreateCheckpointService extends BaseService {
         attemptId: attempt.id,
         workerId: attempt.backgroundWorker.id,
       });
-      return;
+
+      return {
+        success: false,
+      };
     }
 
+    //sleep to test slow checkpoints
+    // await new Promise((resolve) => setTimeout(resolve, 60_000));
+
     const checkpoint = await this._prisma.checkpoint.create({
       data: {
         friendlyId: generateFriendlyId("checkpoint"),
         runtimeEnvironmentId: attempt.taskRun.runtimeEnvironmentId,
         projectId: attempt.taskRun.projectId,
         attemptId: attempt.id,
+        attemptNumber: attempt.number,
         runId: attempt.taskRunId,
         location: params.location,
         type: params.docker ? "DOCKER" : "KUBERNETES",
@@ -134,10 +151,133 @@ export class CreateCheckpointService extends BaseService {
           dependencyFriendlyRunId: reason.friendlyId,
         });
 
-        keepRunAlive = await this.#isRunCompleted(reason.friendlyId);
+        if (checkpointEvent) {
+          const dependency = await this._prisma.taskRunDependency.findFirst({
+            select: {
+              id: true,
+              taskRunId: true,
+            },
+            where: {
+              taskRun: {
+                friendlyId: reason.friendlyId,
+              },
+            },
+          });
 
-        if (!keepRunAlive) {
-          await marqs?.acknowledgeMessage(attempt.taskRunId);
+          logger.log("CreateCheckpointService: Created checkpoint WAIT_FOR_TASK", {
+            checkpointId: checkpoint.id,
+            runFriendlyId: reason.friendlyId,
+            dependencyId: dependency?.id,
+          });
+
+          if (!dependency) {
+            logger.error("CreateCheckpointService: Dependency not found", {
+              friendlyId: reason.friendlyId,
+            });
+
+            return {
+              success: true,
+              checkpoint,
+              event: checkpointEvent,
+              keepRunAlive: false,
+            };
+          }
+
+          const childRun = await this._prisma.taskRun.findFirst({
+            select: {
+              id: true,
+              status: true,
+            },
+            where: {
+              id: dependency.taskRunId,
+            },
+          });
+
+          if (!childRun) {
+            logger.error("CreateCheckpointService: Dependency child run not found", {
+              taskRunId: dependency.taskRunId,
+              runFriendlyId: reason.friendlyId,
+              dependencyId: dependency.id,
+            });
+
+            return {
+              success: true,
+              checkpoint,
+              event: checkpointEvent,
+              keepRunAlive: false,
+            };
+          }
+
+          const isFinished = isFinalRunStatus(childRun.status);
+          if (!isFinished) {
+            logger.debug("CreateCheckpointService: Dependency child run not finished", {
+              taskRunId: dependency.taskRunId,
+              runFriendlyId: reason.friendlyId,
+              dependencyId: dependency.id,
+              childRunStatus: childRun.status,
+              childRunId: childRun.id,
+            });
+
+            return {
+              success: true,
+              checkpoint,
+              event: checkpointEvent,
+              keepRunAlive: false,
+            };
+          }
+
+          const lastAttempt = await this._prisma.taskRunAttempt.findFirst({
+            select: {
+              id: true,
+              status: true,
+            },
+            where: {
+              taskRunId: dependency.taskRunId,
+            },
+            orderBy: {
+              createdAt: "desc",
+            },
+          });
+
+          if (!lastAttempt) {
+            logger.debug("CreateCheckpointService: Dependency child attempt not found", {
+              taskRunId: dependency.taskRunId,
+              runFriendlyId: reason.friendlyId,
+              dependencyId: dependency?.id,
+            });
+            return {
+              success: true,
+              checkpoint,
+              event: checkpointEvent,
+              keepRunAlive: false,
+            };
+          }
+
+          if (!isFinalAttemptStatus(lastAttempt.status)) {
+            logger.debug("CreateCheckpointService: Dependency child attempt not final", {
+              taskRunId: dependency.taskRunId,
+              runFriendlyId: reason.friendlyId,
+              dependencyId: dependency.id,
+              lastAttemptId: lastAttempt.id,
+              lastAttemptStatus: lastAttempt.status,
+            });
+
+            return {
+              success: true,
+              checkpoint,
+              event: checkpointEvent,
+              keepRunAlive: false,
+            };
+          }
+
+          await ResumeTaskDependencyService.enqueue(dependency.id, lastAttempt.id, this._prisma);
+
+          return {
+            success: true,
+            checkpoint,
+            event: checkpointEvent,
+            keepRunAlive: false,
+          };
         }
 
         break;
@@ -148,10 +288,37 @@ export class CreateCheckpointService extends BaseService {
           batchDependencyFriendlyId: reason.batchFriendlyId,
         });
 
-        keepRunAlive = await this.#isBatchCompleted(reason.batchFriendlyId);
+        if (checkpointEvent) {
+          const batchRun = await this._prisma.batchTaskRun.findFirst({
+            select: {
+              id: true,
+            },
+            where: {
+              friendlyId: reason.batchFriendlyId,
+            },
+          });
 
-        if (!keepRunAlive) {
-          await marqs?.acknowledgeMessage(attempt.taskRunId);
+          if (!batchRun) {
+            logger.error("CreateCheckpointService: Batch not found", {
+              friendlyId: reason.batchFriendlyId,
+            });
+
+            return {
+              success: true,
+              checkpoint,
+              event: checkpointEvent,
+              keepRunAlive: false,
+            };
+          }
+
+          await ResumeBatchRunService.enqueue(batchRun.id, this._prisma);
+
+          return {
+            success: true,
+            checkpoint,
+            event: checkpointEvent,
+            keepRunAlive: false,
+          };
         }
 
         break;
@@ -175,7 +342,10 @@ export class CreateCheckpointService extends BaseService {
         checkpointId: checkpoint.id,
       });
       await marqs?.acknowledgeMessage(attempt.taskRunId);
-      return;
+
+      return {
+        success: false,
+      };
     }
 
     if (reason.type === "WAIT_FOR_DURATION") {
@@ -191,39 +361,10 @@ export class CreateCheckpointService extends BaseService {
     }
 
     return {
+      success: true,
       checkpoint,
       event: checkpointEvent,
       keepRunAlive,
     };
   }
-
-  async #isBatchCompleted(friendlyId: string): Promise<boolean> {
-    const batch = await this._prisma.batchTaskRun.findUnique({
-      where: {
-        friendlyId,
-      },
-    });
-
-    if (!batch) {
-      logger.error("Batch not found", { friendlyId });
-      return false;
-    }
-
-    return batch.status === "COMPLETED";
-  }
-
-  async #isRunCompleted(friendlyId: string): Promise<boolean> {
-    const run = await this._prisma.taskRun.findUnique({
-      where: {
-        friendlyId,
-      },
-    });
-
-    if (!run) {
-      logger.error("Run not found", { friendlyId });
-      return false;
-    }
-
-    return isFinalRunStatus(run.status);
-  }
 }

apps/webapp/app/v3/services/createDeployedBackgroundWorker.server.ts

@@ -11,6 +11,7 @@ import { logger } from "~/services/logger.server";
 import { ExecuteTasksWaitingForDeployService } from "./executeTasksWaitingForDeploy";
 import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server";
 import { TimeoutDeploymentService } from "./timeoutDeployment.server";
+import { socketIo } from "../handleSocketIo.server";
 
 export class CreateDeployedBackgroundWorkerService extends BaseService {
   public async call(
@@ -132,6 +133,20 @@ export class CreateDeployedBackgroundWorkerService extends BaseService {
         logger.error("Failed to publish WORKER_CREATED event", { err });
       }
 
+      if (deployment.imageReference) {
+        socketIo.providerNamespace.emit("PRE_PULL_DEPLOYMENT", {
+          version: "v1",
+          imageRef: deployment.imageReference,
+          shortCode: deployment.shortCode,
+          // identifiers
+          deploymentId: deployment.id,
+          envId: environment.id,
+          envType: environment.type,
+          orgId: environment.organizationId,
+          projectId: deployment.projectId,
+        });
+      }
+
       await ExecuteTasksWaitingForDeployService.enqueue(backgroundWorker.id, this._prisma);
       await PerformDeploymentAlertsService.enqueue(deployment.id, this._prisma);
       await TimeoutDeploymentService.dequeue(deployment.id, this._prisma);

apps/webapp/app/v3/services/createTaskRunAttempt.server.ts

@@ -9,6 +9,7 @@ import { machinePresetFromConfig } from "../machinePresets.server";
 import { workerQueue } from "~/services/worker.server";
 import { MAX_TASK_RUN_ATTEMPTS } from "~/consts";
 import { CrashTaskRunService } from "./crashTaskRun.server";
+import { reportInvocationUsage } from "~/services/platform.v3.server";
 
 export class CreateTaskRunAttemptService extends BaseService {
   public async call(
@@ -149,14 +150,8 @@ export class CreateTaskRunAttemptService extends BaseService {
       }
 
       if (taskRunAttempt.number === 1 && taskRun.baseCostInCents > 0) {
-        await workerQueue.enqueue("v3.reportUsage", {
-          orgId: environment.organizationId,
-          data: {
-            costInCents: String(taskRun.baseCostInCents),
-          },
-          additionalData: {
-            runId: taskRun.id,
-          },
+        await reportInvocationUsage(environment.organizationId, taskRun.baseCostInCents, {
+          runId: taskRun.id,
         });
       }
 
@@ -190,6 +185,7 @@ export class CreateTaskRunAttemptService extends BaseService {
           costInCents: taskRun.costInCents,
           baseCostInCents: taskRun.baseCostInCents,
           maxAttempts: taskRun.maxAttempts ?? undefined,
+          version: taskRun.lockedBy.worker.version,
         },
         queue: {
           id: queue.friendlyId,

apps/webapp/app/v3/services/enqueueDelayedRun.server.ts

@@ -65,7 +65,13 @@ export class EnqueueDelayedRunService extends BaseService {
       run.runtimeEnvironment,
       run.queue,
       run.id,
-      { type: "EXECUTE", taskIdentifier: run.taskIdentifier },
+      {
+        type: "EXECUTE",
+        taskIdentifier: run.taskIdentifier,
+        projectId: run.runtimeEnvironment.projectId,
+        environmentId: run.runtimeEnvironment.id,
+        environmentType: run.runtimeEnvironment.type,
+      },
       run.concurrencyKey ?? undefined
     );
   }

apps/webapp/app/v3/services/executeTasksWaitingForDeploy.ts

@@ -79,6 +79,9 @@ export class ExecuteTasksWaitingForDeployService extends BaseService {
           {
             type: "EXECUTE",
             taskIdentifier: run.taskIdentifier,
+            projectId: backgroundWorker.runtimeEnvironment.projectId,
+            environmentId: backgroundWorker.runtimeEnvironment.id,
+            environmentType: backgroundWorker.runtimeEnvironment.type,
           },
           run.concurrencyKey ?? undefined,
           Date.now() + i * 5 // slight delay to help preserve order

apps/webapp/app/v3/services/expireEnqueuedRun.server.ts

@@ -1,6 +1,7 @@
 import { logger } from "~/services/logger.server";
-import { marqs } from "~/v3/marqs/index.server";
 import { BaseService } from "./baseService.server";
+import { eventRepository } from "../eventRepository.server";
+import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
 
 export class ExpireEnqueuedRunService extends BaseService {
   public async call(runId: string) {
@@ -38,16 +39,30 @@ export class ExpireEnqueuedRunService extends BaseService {
       run,
     });
 
-    await this._prisma.taskRun.update({
-      where: {
-        id: run.id,
-      },
-      data: {
-        status: "EXPIRED",
-        expiredAt: new Date(),
-      },
+    const finalizeService = new FinalizeTaskRunService();
+    await finalizeService.call({
+      id: run.id,
+      status: "EXPIRED",
+      expiredAt: new Date(),
+      completedAt: new Date(),
     });
 
-    await marqs?.acknowledgeMessage(run.id);
+    await eventRepository.completeEvent(run.spanId, {
+      endTime: new Date(),
+      attributes: {
+        isError: true,
+      },
+      events: [
+        {
+          name: "exception",
+          time: new Date(),
+          properties: {
+            exception: {
+              message: `Run expired because the TTL (${run.ttl}) was reached`,
+            },
+          },
+        },
+      ],
+    });
   }
 }

apps/webapp/app/v3/services/finalizeTaskRun.server.ts

@@ -0,0 +1,59 @@
+import { type Prisma, type TaskRun } from "@trigger.dev/database";
+import { type FINISHED_STATUSES } from "~/components/runs/v3/TaskRunStatus";
+import { logger } from "~/services/logger.server";
+import { marqs } from "~/v3/marqs/index.server";
+import { BaseService } from "./baseService.server";
+
+type BaseInput = {
+  id: string;
+  status?: FINISHED_STATUSES;
+  expiredAt?: Date;
+  completedAt?: Date;
+};
+
+type InputWithInclude<T extends Prisma.TaskRunInclude> = BaseInput & {
+  include: T;
+};
+
+type InputWithoutInclude = BaseInput & {
+  include?: undefined;
+};
+
+type Output<T extends Prisma.TaskRunInclude | undefined> = T extends Prisma.TaskRunInclude
+  ? Prisma.TaskRunGetPayload<{ include: T }>
+  : TaskRun;
+
+export class FinalizeTaskRunService extends BaseService {
+  public async call<T extends Prisma.TaskRunInclude | undefined>({
+    id,
+    status,
+    expiredAt,
+    completedAt,
+    include,
+  }: T extends Prisma.TaskRunInclude ? InputWithInclude<T> : InputWithoutInclude): Promise<
+    Output<T>
+  > {
+    logger.debug("Finalizing run marqs ack", {
+      id,
+      status,
+      expiredAt,
+      completedAt,
+    });
+    await marqs?.acknowledgeMessage(id);
+
+    logger.debug("Finalizing run updating run status", {
+      id,
+      status,
+      expiredAt,
+      completedAt,
+    });
+
+    const run = await this._prisma.taskRun.update({
+      where: { id },
+      data: { status, expiredAt, completedAt },
+      ...(include ? { include } : {}),
+    });
+
+    return run as Output<T>;
+  }
+}

apps/webapp/app/v3/services/replayTaskRun.server.ts

@@ -1,15 +1,27 @@
-import { conditionallyImportPacket, parsePacket, RunTags } from "@trigger.dev/core/v3";
+import {
+  conditionallyImportPacket,
+  IOPacket,
+  parsePacket,
+  RunTags,
+  stringifyIO,
+} from "@trigger.dev/core/v3";
+import { replaceSuperJsonPayload } from "@trigger.dev/core/v3/utils/ioSerialization";
 import { TaskRun } from "@trigger.dev/database";
 import { findEnvironmentById } from "~/models/runtimeEnvironment.server";
+import { getTagsForRunId } from "~/models/taskRunTag.server";
 import { logger } from "~/services/logger.server";
 import { BaseService } from "./baseService.server";
 import { OutOfEntitlementError, TriggerTaskService } from "./triggerTask.server";
-import { getTagsForRunId } from "~/models/taskRunTag.server";
+
+type OverrideOptions = {
+  environmentId?: string;
+  payload?: string;
+};
 
 export class ReplayTaskRunService extends BaseService {
-  public async call(existingTaskRun: TaskRun) {
+  public async call(existingTaskRun: TaskRun, overrideOptions?: OverrideOptions) {
     const authenticatedEnvironment = await findEnvironmentById(
-      existingTaskRun.runtimeEnvironmentId
+      overrideOptions?.environmentId ?? existingTaskRun.runtimeEnvironmentId
     );
     if (!authenticatedEnvironment) {
       return;
@@ -20,10 +32,27 @@ export class ReplayTaskRunService extends BaseService {
       taskRunFriendlyId: existingTaskRun.friendlyId,
     });
 
-    const payloadPacket = await conditionallyImportPacket({
-      data: existingTaskRun.payload,
-      dataType: existingTaskRun.payloadType,
-    });
+    let payloadPacket: IOPacket;
+
+    if (overrideOptions?.payload) {
+      if (existingTaskRun.payloadType === "application/super+json") {
+        const newPayload = await replaceSuperJsonPayload(
+          existingTaskRun.payload,
+          overrideOptions.payload
+        );
+        payloadPacket = await stringifyIO(newPayload);
+      } else {
+        payloadPacket = await conditionallyImportPacket({
+          data: overrideOptions.payload,
+          dataType: existingTaskRun.payloadType,
+        });
+      }
+    } else {
+      payloadPacket = await conditionallyImportPacket({
+        data: existingTaskRun.payload,
+        dataType: existingTaskRun.payloadType,
+      });
+    }
 
     const parsedPayload =
       payloadPacket.dataType === "application/json"
@@ -38,7 +67,7 @@ export class ReplayTaskRunService extends BaseService {
 
     try {
       const tags = await getTagsForRunId({
-        friendlyId: existingTaskRun.id,
+        friendlyId: existingTaskRun.friendlyId,
         environmentId: authenticatedEnvironment.id,
       });
 

apps/webapp/app/v3/services/restoreCheckpoint.server.ts

@@ -100,6 +100,7 @@ export class RestoreCheckpointService extends BaseService {
       reason: checkpoint.reason ?? undefined,
       imageRef: checkpoint.imageRef,
       machine,
+      attemptNumber: checkpoint.attemptNumber ?? undefined,
       // identifiers
       checkpointId: checkpoint.id,
       envId: checkpoint.runtimeEnvironment.id,

apps/webapp/app/v3/services/resumeAttempt.server.ts

@@ -152,9 +152,6 @@ export class ResumeAttemptService extends BaseService {
           break;
         }
       }
-
-      // Prevent infinite restores by failing runs that don't heartbeat after post-restore resume requests
-      await this.#replaceResumeWithFailMessage(attempt.taskRunId, params.type);
     });
   }
 
@@ -255,50 +252,4 @@ export class ResumeAttemptService extends BaseService {
       },
     });
   }
-
-  async #replaceResumeWithFailMessage(messageId: string, waitReason: WaitReason) {
-    const currentMessage = await marqs?.readMessage(messageId);
-
-    if (!currentMessage) {
-      logger.debug("No message to replace", { messageId, waitReason });
-      return;
-    }
-
-    const currentBody = SharedQueueMessageBody.safeParse(currentMessage.data);
-
-    if (!currentBody.success) {
-      logger.debug("Invalid message body", { messageId, waitReason, currentBody });
-      return;
-    }
-
-    const currentType = currentBody.data.type;
-
-    if (currentType !== "RESUME" && currentType !== "RESUME_AFTER_DURATION") {
-      logger.debug("Not a resume message", { messageId, waitReason, currentBody });
-      return;
-    }
-
-    let reason = "Worker unresponsive after restore";
-
-    switch (waitReason) {
-      case "WAIT_FOR_DURATION":
-        reason = "Worker unresponsive after waiting for duration";
-        break;
-      case "WAIT_FOR_TASK":
-        reason = "Worker unresponsive after waiting for task";
-        break;
-      case "WAIT_FOR_BATCH":
-        reason = "Worker unresponsive after waiting for batch task";
-        break;
-      default:
-        break;
-    }
-
-    const failMessage: SharedQueueMessageBody = {
-      type: "FAIL",
-      reason,
-    };
-
-    return await marqs?.replaceMessage(messageId, failMessage, undefined, true);
-  }
 }

apps/webapp/app/v3/services/resumeBatchRun.server.ts

@@ -5,7 +5,7 @@ import { BaseService } from "./baseService.server";
 import { logger } from "~/services/logger.server";
 
 export class ResumeBatchRunService extends BaseService {
-  public async call(batchRunId: string, sourceTaskAttemptId: string) {
+  public async call(batchRunId: string) {
     const batchRun = await this._prisma.batchTaskRun.findFirst({
       where: {
         id: batchRunId,
@@ -41,35 +41,35 @@ export class ResumeBatchRunService extends BaseService {
       return;
     }
 
-    // We need to update the batchRun status so we don't resume it again
-    await this._prisma.batchTaskRun.update({
-      where: {
-        id: batchRun.id,
-      },
-      data: {
-        status: "COMPLETED",
-      },
-    });
-
     // This batch has a dependent attempt and just finalized, we should resume that attempt
     const environment = batchRun.dependentTaskAttempt.runtimeEnvironment;
 
     // If we are in development, we don't need to resume the dependent task (that will happen automatically)
     if (environment.type === "DEVELOPMENT") {
+      // We need to update the batchRun status so we don't resume it again
+      await this._prisma.batchTaskRun.update({
+        where: {
+          id: batchRun.id,
+        },
+        data: {
+          status: "COMPLETED",
+        },
+      });
       return;
     }
 
     const dependentRun = batchRun.dependentTaskAttempt.taskRun;
 
-    if (batchRun.dependentTaskAttempt.status === "PAUSED") {
-      if (!batchRun.checkpointEventId) {
-        logger.error("Can't resume paused attempt without checkpoint event", {
-          batchRunId: batchRun.id,
-        });
-
-        await marqs?.acknowledgeMessage(dependentRun.id);
-        return;
-      }
+    if (batchRun.dependentTaskAttempt.status === "PAUSED" && batchRun.checkpointEventId) {
+      // We need to update the batchRun status so we don't resume it again
+      await this._prisma.batchTaskRun.update({
+        where: {
+          id: batchRun.id,
+        },
+        data: {
+          status: "COMPLETED",
+        },
+      });
 
       await marqs?.enqueueMessage(
         environment,
@@ -77,13 +77,33 @@ export class ResumeBatchRunService extends BaseService {
         dependentRun.id,
         {
           type: "RESUME",
-          completedAttemptIds: [sourceTaskAttemptId],
+          completedAttemptIds: [],
           resumableAttemptId: batchRun.dependentTaskAttempt.id,
           checkpointEventId: batchRun.checkpointEventId,
+          projectId: batchRun.dependentTaskAttempt.runtimeEnvironment.projectId,
+          environmentId: batchRun.dependentTaskAttempt.runtimeEnvironment.id,
+          environmentType: batchRun.dependentTaskAttempt.runtimeEnvironment.type,
         },
         dependentRun.concurrencyKey ?? undefined
       );
     } else {
+      logger.debug("Batch run resume: Attempt is not paused or there's no checkpoint event", {
+        batchRunId: batchRun.id,
+        dependentTaskAttempt: batchRun.dependentTaskAttempt,
+        checkpointEventId: batchRun.checkpointEventId,
+        hasCheckpointEvent: !!batchRun.checkpointEventId,
+      });
+
+      if (batchRun.dependentTaskAttempt.status === "PAUSED" && !batchRun.checkpointEventId) {
+        // In case of race conditions the status can be PAUSED without a checkpoint event
+        // When the checkpoint is created, it will continue the run
+        logger.error("Batch run resume: Attempt is paused but there's no checkpoint event", {
+          batchRunId: batchRun.id,
+          dependentTaskAttemptId: batchRun.dependentTaskAttempt.id,
+        });
+        return;
+      }
+
       await marqs?.replaceMessage(dependentRun.id, {
         type: "RESUME",
         completedAttemptIds: batchRun.items.map((item) => item.taskRunAttemptId).filter(Boolean),
@@ -92,17 +112,11 @@ export class ResumeBatchRunService extends BaseService {
     }
   }
 
-  static async enqueue(
-    batchRunId: string,
-    sourceTaskAttemptId: string,
-    tx: PrismaClientOrTransaction,
-    runAt?: Date
-  ) {
+  static async enqueue(batchRunId: string, tx: PrismaClientOrTransaction, runAt?: Date) {
     return await workerQueue.enqueue(
       "v3.resumeBatchRun",
       {
         batchRunId,
-        sourceTaskAttemptId,
       },
       {
         tx,

apps/webapp/app/v3/services/resumeTaskDependency.server.ts

@@ -38,16 +38,7 @@ export class ResumeTaskDependencyService extends BaseService {
 
     const dependentRun = dependency.dependentAttempt.taskRun;
 
-    if (dependency.dependentAttempt.status === "PAUSED") {
-      if (!dependency.checkpointEventId) {
-        logger.error("Can't resume paused attempt without checkpoint event", {
-          attemptId: dependency.id,
-        });
-
-        await marqs?.acknowledgeMessage(dependentRun.id);
-        return;
-      }
-
+    if (dependency.dependentAttempt.status === "PAUSED" && dependency.checkpointEventId) {
       await marqs?.enqueueMessage(
         dependency.taskRun.runtimeEnvironment,
         dependentRun.queue,
@@ -57,10 +48,30 @@ export class ResumeTaskDependencyService extends BaseService {
           completedAttemptIds: [sourceTaskAttemptId],
           resumableAttemptId: dependency.dependentAttempt.id,
           checkpointEventId: dependency.checkpointEventId,
+          projectId: dependency.taskRun.runtimeEnvironment.projectId,
+          environmentId: dependency.taskRun.runtimeEnvironment.id,
+          environmentType: dependency.taskRun.runtimeEnvironment.type,
         },
         dependentRun.concurrencyKey ?? undefined
       );
     } else {
+      logger.debug("Task dependency resume: Attempt is not paused or there's no checkpoint event", {
+        attemptId: dependency.id,
+        dependentAttempt: dependency.dependentAttempt,
+        checkpointEventId: dependency.checkpointEventId,
+        hasCheckpointEvent: !!dependency.checkpointEventId,
+      });
+
+      if (dependency.dependentAttempt.status === "PAUSED" && !dependency.checkpointEventId) {
+        // In case of race conditions the status can be PAUSED without a checkpoint event
+        // When the checkpoint is created, it will continue the run
+        logger.error("Task dependency resume: Attempt is paused but there's no checkpoint event", {
+          attemptId: dependency.id,
+          dependentAttemptId: dependency.dependentAttempt.id,
+        });
+        return;
+      }
+
       await marqs?.replaceMessage(dependentRun.id, {
         type: "RESUME",
         completedAttemptIds: [sourceTaskAttemptId],

apps/webapp/app/v3/services/resumeTaskRunDependencies.server.ts

@@ -65,7 +65,7 @@ export class ResumeTaskRunDependenciesService extends BaseService {
         },
       });
 
-      await ResumeBatchRunService.enqueue(batchItem.batchTaskRunId, taskAttempt.id, tx);
+      await ResumeBatchRunService.enqueue(batchItem.batchTaskRunId, tx);
     });
   }
 

apps/webapp/app/v3/services/taskRunConcurrencyTracker.server.ts

@@ -0,0 +1,307 @@
+import { env } from "~/env.server";
+import Redis, { type RedisOptions } from "ioredis";
+import { singleton } from "~/utils/singleton";
+import { type MessagePayload, type MessageQueueSubscriber } from "../marqs/types";
+import { z } from "zod";
+import { logger } from "~/services/logger.server";
+
+type Options = {
+  redis: RedisOptions;
+};
+
+const ConcurrentMessageData = z.object({
+  taskIdentifier: z.string(),
+  projectId: z.string(),
+  environmentId: z.string(),
+  environmentType: z.string(),
+});
+
+class TaskRunConcurrencyTracker implements MessageQueueSubscriber {
+  private redis: Redis;
+
+  constructor(config: Options) {
+    this.redis = new Redis(config.redis);
+  }
+
+  async messageEnqueued(message: MessagePayload): Promise<void> {}
+
+  async messageDequeued(message: MessagePayload): Promise<void> {
+    logger.debug("TaskRunConcurrencyTracker.messageDequeued()", {
+      data: message.data,
+      messageId: message.messageId,
+    });
+
+    const data = this.getMessageData(message);
+    if (!data) {
+      logger.info(
+        `TaskRunConcurrencyTracker.messageDequeued(): could not parse message data`,
+        message
+      );
+      return;
+    }
+
+    await this.executionStarted({
+      projectId: data.projectId,
+      taskId: data.taskIdentifier,
+      runId: message.messageId,
+      environmentId: data.environmentId,
+      deployed: data.environmentType !== "DEVELOPMENT",
+    });
+  }
+
+  async messageAcked(message: MessagePayload): Promise<void> {
+    logger.debug("TaskRunConcurrencyTracker.messageAcked()", {
+      data: message.data,
+      messageId: message.messageId,
+    });
+
+    const data = this.getMessageData(message);
+    if (!data) {
+      logger.info(
+        `TaskRunConcurrencyTracker.messageAcked(): could not parse message data`,
+        message
+      );
+      return;
+    }
+
+    await this.executionFinished({
+      projectId: data.projectId,
+      taskId: data.taskIdentifier,
+      runId: message.messageId,
+      environmentId: data.environmentId,
+      deployed: data.environmentType !== "DEVELOPMENT",
+    });
+  }
+
+  async messageNacked(message: MessagePayload): Promise<void> {
+    logger.debug("TaskRunConcurrencyTracker.messageNacked()", {
+      data: message.data,
+      messageId: message.messageId,
+    });
+
+    const data = this.getMessageData(message);
+    if (!data) {
+      logger.info(
+        `TaskRunConcurrencyTracker.messageNacked(): could not parse message data`,
+        message
+      );
+      return;
+    }
+
+    await this.executionFinished({
+      projectId: data.projectId,
+      taskId: data.taskIdentifier,
+      runId: message.messageId,
+      environmentId: data.environmentId,
+      deployed: data.environmentType !== "DEVELOPMENT",
+    });
+  }
+
+  async messageReplaced(message: MessagePayload): Promise<void> {
+    logger.debug("TaskRunConcurrencyTracker.messageReplaced()", {
+      data: message.data,
+      messageId: message.messageId,
+    });
+
+    const data = this.getMessageData(message);
+    if (!data) {
+      logger.info(
+        `TaskRunConcurrencyTracker.messageReplaced(): could not parse message data`,
+        message
+      );
+      return;
+    }
+
+    await this.executionFinished({
+      projectId: data.projectId,
+      taskId: data.taskIdentifier,
+      runId: message.messageId,
+      environmentId: data.environmentId,
+      deployed: data.environmentType !== "DEVELOPMENT",
+    });
+  }
+
+  private getMessageData(message: MessagePayload) {
+    const result = ConcurrentMessageData.safeParse(message.data);
+    if (result.success) {
+      return result.data;
+    }
+    return;
+  }
+
+  private async executionStarted({
+    projectId,
+    taskId,
+    runId,
+    environmentId,
+    deployed,
+  }: {
+    projectId: string;
+    taskId: string;
+    runId: string;
+    environmentId: string;
+    deployed: boolean;
+  }): Promise<void> {
+    try {
+      const pipeline = this.redis.pipeline();
+
+      pipeline.sadd(this.getTaskKey(projectId, taskId), runId);
+      pipeline.sadd(this.getTaskEnvironmentKey(projectId, taskId, environmentId), runId);
+      pipeline.sadd(this.getEnvironmentKey(projectId, environmentId), runId);
+      pipeline.sadd(this.getGlobalKey(deployed), runId);
+
+      await pipeline.exec();
+    } catch (error) {
+      logger.error("TaskRunConcurrencyTracker.executionStarted() error", { error });
+    }
+  }
+
+  private async executionFinished({
+    projectId,
+    taskId,
+    runId,
+    environmentId,
+    deployed,
+  }: {
+    projectId: string;
+    taskId: string;
+    runId: string;
+    environmentId: string;
+    deployed: boolean;
+  }): Promise<void> {
+    try {
+      const pipeline = this.redis.pipeline();
+
+      pipeline.srem(this.getTaskKey(projectId, taskId), runId);
+      pipeline.srem(this.getTaskEnvironmentKey(projectId, taskId, environmentId), runId);
+      pipeline.srem(this.getEnvironmentKey(projectId, environmentId), runId);
+      pipeline.srem(this.getGlobalKey(deployed), runId);
+
+      await pipeline.exec();
+    } catch (error) {
+      logger.error("TaskRunConcurrencyTracker.executionFinished() error", { error });
+    }
+  }
+
+  async taskConcurrentRunCount(projectId: string, taskId: string): Promise<number> {
+    return await this.redis.scard(this.getTaskKey(projectId, taskId));
+  }
+
+  async globalConcurrentRunCount(deployed: boolean): Promise<number> {
+    return await this.redis.scard(this.getGlobalKey(deployed));
+  }
+
+  async currentlyExecutingRuns(projectId: string, taskId: string): Promise<string[]> {
+    return await this.redis.smembers(this.getTaskKey(projectId, taskId));
+  }
+
+  private async getTaskCounts(projectId: string, taskIds: string[]): Promise<number[]> {
+    try {
+      const pipeline = this.redis.pipeline();
+      taskIds.forEach((taskId) => {
+        pipeline.scard(this.getTaskKey(projectId, taskId));
+      });
+      const results = await pipeline.exec();
+      if (!results) {
+        return [];
+      }
+      return results.map(([err, count]) => {
+        if (err) {
+          console.error("Error in getTaskCounts:", err);
+          return 0;
+        }
+        return count as number;
+      });
+    } catch (error) {
+      logger.error("TaskRunConcurrencyTracker.getTaskCounts() error", { error });
+      return [];
+    }
+  }
+
+  async projectTotalConcurrentRunCount(projectId: string, taskIds: string[]): Promise<number> {
+    const counts = await this.getTaskCounts(projectId, taskIds);
+    return counts.reduce((total, count) => total + count, 0);
+  }
+
+  async taskConcurrentRunCounts(
+    projectId: string,
+    taskIds: string[]
+  ): Promise<Record<string, number>> {
+    const counts = await this.getTaskCounts(projectId, taskIds);
+    return taskIds.reduce((acc, taskId, index) => {
+      acc[taskId] = counts[index] ?? 0;
+      return acc;
+    }, {} as Record<string, number>);
+  }
+
+  async environmentConcurrentRunCounts(
+    projectId: string,
+    environmentIds: string[]
+  ): Promise<Record<string, number>> {
+    try {
+      const pipeline = this.redis.pipeline();
+      environmentIds.forEach((environmentId) => {
+        pipeline.scard(this.getEnvironmentKey(projectId, environmentId));
+      });
+      const results = await pipeline.exec();
+      if (!results) {
+        return Object.fromEntries(environmentIds.map((id) => [id, 0]));
+      }
+
+      return results.reduce((acc, [err, count], index) => {
+        if (err) {
+          console.error("Error in environmentConcurrentRunCounts:", err);
+          return acc;
+        }
+        acc[environmentIds[index]] = count as number;
+        return acc;
+      }, {} as Record<string, number>);
+    } catch (error) {
+      logger.error("TaskRunConcurrencyTracker.environmentConcurrentRunCounts() error", { error });
+      return Object.fromEntries(environmentIds.map((id) => [id, 0]));
+    }
+  }
+
+  private getTaskKey(projectId: string, taskId: string): string {
+    return `project:${projectId}:task:${taskId}`;
+  }
+
+  private getTaskEnvironmentKey(projectId: string, taskId: string, environmentId: string): string {
+    return `project:${projectId}:task:${taskId}:env:${environmentId}`;
+  }
+
+  private getGlobalKey(deployed: boolean): string {
+    return `global:${deployed ? "deployed" : "dev"}`;
+  }
+
+  private getEnvironmentKey(projectId: string, environmentId: string): string {
+    return `project:${projectId}:env:${environmentId}`;
+  }
+}
+
+export const concurrencyTracker = singleton("concurrency-tracker", getTracker);
+
+function getTracker() {
+  if (!env.REDIS_HOST || !env.REDIS_PORT) {
+    throw new Error(
+      "Could not initialize TaskRunConcurrencyTracker because process.env.REDIS_HOST and process.env.REDIS_PORT are required to be set. "
+    );
+  }
+
+  logger.debug("Initializing TaskRunConcurrencyTracker", {
+    redisHost: env.REDIS_HOST,
+    redisPort: env.REDIS_PORT,
+  });
+
+  return new TaskRunConcurrencyTracker({
+    redis: {
+      keyPrefix: "concurrencytracker:",
+      port: env.REDIS_PORT,
+      host: env.REDIS_HOST,
+      username: env.REDIS_USERNAME,
+      password: env.REDIS_PASSWORD,
+      enableAutoPipelining: true,
+      ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+    },
+  });
+}

apps/webapp/app/v3/services/triggerTask.server.ts

@@ -1,5 +1,6 @@
 import {
   IOPacket,
+  QueueOptions,
   SemanticInternalAttributes,
   TriggerTaskRequestBody,
   packetRequiresOffloading,
@@ -17,7 +18,8 @@ import { getEntitlement } from "~/services/platform.v3.server";
 import { BaseService, ServiceValidationError } from "./baseService.server";
 import { logger } from "~/services/logger.server";
 import { isFinalAttemptStatus, isFinalRunStatus } from "../taskStatus";
-import { createTag } from "~/models/taskRunTag.server";
+import { createTag, MAX_TAGS_PER_RUN } from "~/models/taskRunTag.server";
+import { findCurrentWorkerFromEnvironment } from "../models/workerDeployment.server";
 
 export type TriggerTaskServiceOptions = {
   idempotencyKey?: string;
@@ -78,6 +80,16 @@ export class TriggerTaskService extends BaseService {
         }
       }
 
+      if (
+        body.options?.tags &&
+        typeof body.options.tags !== "string" &&
+        body.options.tags.length > MAX_TAGS_PER_RUN
+      ) {
+        throw new ServiceValidationError(
+          `Runs can only have ${MAX_TAGS_PER_RUN} tags, you're trying to set ${body.options.tags.length}.`
+        );
+      }
+
       const runFriendlyId = generateFriendlyId("run");
 
       const payloadPacket = await this.#handlePayloadPacket(
@@ -201,7 +213,9 @@ export class TriggerTaskService extends BaseService {
                   })
                 : undefined;
 
-              let queueName = sanitizeQueueName(body.options?.queue?.name ?? `task/${taskId}`);
+              let queueName = sanitizeQueueName(
+                await this.#getQueueName(taskId, environment, body.options?.queue?.name)
+              );
 
               // Check that the queuename is not an empty string
               if (!queueName) {
@@ -297,28 +311,37 @@ export class TriggerTaskService extends BaseService {
                     ? Math.max(0, body.options.queue.concurrencyLimit)
                     : undefined;
 
-                const taskQueue = await tx.taskQueue.upsert({
+                let taskQueue = await tx.taskQueue.findFirst({
                   where: {
-                    runtimeEnvironmentId_name: {
-                      runtimeEnvironmentId: environment.id,
-                      name: queueName,
-                    },
-                  },
-                  update: {
-                    concurrencyLimit,
-                    rateLimit: body.options.queue.rateLimit,
-                  },
-                  create: {
-                    friendlyId: generateFriendlyId("queue"),
-                    name: queueName,
-                    concurrencyLimit,
                     runtimeEnvironmentId: environment.id,
-                    projectId: environment.projectId,
-                    rateLimit: body.options.queue.rateLimit,
-                    type: "NAMED",
+                    name: queueName,
                   },
                 });
 
+                if (taskQueue) {
+                  taskQueue = await tx.taskQueue.update({
+                    where: {
+                      id: taskQueue.id,
+                    },
+                    data: {
+                      concurrencyLimit,
+                      rateLimit: body.options.queue.rateLimit,
+                    },
+                  });
+                } else {
+                  taskQueue = await tx.taskQueue.create({
+                    data: {
+                      friendlyId: generateFriendlyId("queue"),
+                      name: queueName,
+                      concurrencyLimit,
+                      runtimeEnvironmentId: environment.id,
+                      projectId: environment.projectId,
+                      rateLimit: body.options.queue.rateLimit,
+                      type: "NAMED",
+                    },
+                  });
+                }
+
                 if (typeof taskQueue.concurrencyLimit === "number") {
                   await marqs?.updateQueueConcurrencyLimits(
                     environment,
@@ -378,7 +401,13 @@ export class TriggerTaskService extends BaseService {
               environment,
               run.queue,
               run.id,
-              { type: "EXECUTE", taskIdentifier: taskId },
+              {
+                type: "EXECUTE",
+                taskIdentifier: taskId,
+                projectId: environment.projectId,
+                environmentId: environment.id,
+                environmentType: environment.type,
+              },
               body.options?.concurrencyKey
             );
           }
@@ -389,6 +418,57 @@ export class TriggerTaskService extends BaseService {
     });
   }
 
+  async #getQueueName(taskId: string, environment: AuthenticatedEnvironment, queueName?: string) {
+    if (queueName) {
+      return queueName;
+    }
+
+    const defaultQueueName = `task/${taskId}`;
+
+    const worker = await findCurrentWorkerFromEnvironment(environment);
+
+    if (!worker) {
+      logger.debug("Failed to get queue name: No worker found", {
+        taskId,
+        environmentId: environment.id,
+      });
+
+      return defaultQueueName;
+    }
+
+    const task = await this._prisma.backgroundWorkerTask.findUnique({
+      where: {
+        workerId_slug: {
+          workerId: worker.id,
+          slug: taskId,
+        },
+      },
+    });
+
+    if (!task) {
+      console.log("Failed to get queue name: No task found", {
+        taskId,
+        environmentId: environment.id,
+      });
+
+      return defaultQueueName;
+    }
+
+    const queueConfig = QueueOptions.optional().nullable().safeParse(task.queueConfig);
+
+    if (!queueConfig.success) {
+      console.log("Failed to get queue name: Invalid queue config", {
+        taskId,
+        environmentId: environment.id,
+        queueConfig: task.queueConfig,
+      });
+
+      return defaultQueueName;
+    }
+
+    return queueConfig.data?.name ?? defaultQueueName;
+  }
+
   async #handlePayloadPacket(
     payload: any,
     payloadType: string,

apps/webapp/app/v3/taskStatus.ts

@@ -69,3 +69,10 @@ export function isRestorableRunStatus(status: TaskRunStatus): boolean {
 export function isRestorableAttemptStatus(status: TaskRunAttemptStatus): boolean {
   return RESTORABLE_ATTEMPT_STATUSES.includes(status);
 }
+
+export const FAILABLE_RUN_STATUSES = [
+  "EXECUTING",
+  "PENDING",
+  "WAITING_FOR_DEPLOY",
+  "RETRYING_AFTER_FAILURE",
+] satisfies TaskRunStatus[];

apps/webapp/app/v3/tracer.server.ts

@@ -124,6 +124,9 @@ function getTracer() {
     sampler: new ParentBasedSampler({
       root: new CustomWebappSampler(new TraceIdRatioBasedSampler(samplingRate)),
     }),
+    spanLimits: {
+      attributeCountLimit: 1024,
+    },
   });
 
   if (env.INTERNAL_OTEL_TRACE_EXPORTER_URL) {

apps/webapp/prisma/populate.ts

@@ -56,6 +56,7 @@ async function populate() {
 
         return {
           status: "CANCELED",
+          completedAt: new Date(),
           number: index + 1,
           friendlyId,
           runtimeEnvironmentId: project.environments[randomIndex(project.environments)].id,