Release 3.1.1

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.github/workflows/release.yml

@@ -17,7 +17,7 @@ concurrency:
 jobs:
   release:
     name: 🦋 Changesets Release
-    runs-on: buildjet-8vcpu-ubuntu-2204
+    runs-on: ubuntu-latest
     if: github.repository == 'triggerdotdev/trigger.dev'
     outputs:
       published: ${{ steps.changesets.outputs.published }}

.github/workflows/unit-tests.yml

@@ -6,7 +6,7 @@ on:
 jobs:
   unitTests:
     name: "🧪 Unit Tests"
-    runs-on: buildjet-8vcpu-ubuntu-2204
+    runs-on: buildjet-16vcpu-ubuntu-2204
     steps:
       - name: ⬇️ Checkout repo
         uses: actions/checkout@v4
@@ -30,5 +30,17 @@ jobs:
       - name: 📀 Generate Prisma Client
         run: pnpm run generate
 
-      - name: 🧪 Run Unit Tests
-        run: pnpm run test
+      - name: 🧪 Run Webapp Unit Tests
+        run: pnpm run test --filter webapp
+        env:
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
+          DIRECT_URL: postgresql://postgres:postgres@localhost:5432/postgres
+          SESSION_SECRET: "secret"
+          MAGIC_LINK_SECRET: "secret"
+          ENCRYPTION_KEY: "secret"
+      
+      - name: 🧪 Run Package Unit Tests
+        run: pnpm run test --filter "@trigger.dev/*"
+      
+      - name: 🧪 Run Internal Unit Tests
+        run: pnpm run test --filter "@internal/*"

.npmrc

@@ -1,2 +1,3 @@
 link-workspace-packages=false
-public-hoist-pattern[]=*prisma*
+public-hoist-pattern[]=*prisma*
+prefer-workspace-packages=true

.vscode/extensions.json

@@ -1,8 +1,4 @@
 {
-	"recommendations": [
-        "denoland.vscode-deno"
-    ],
-	"unwantedRecommendations": [
-		
-	]
+  "recommendations": ["bierner.comment-tagged-templates"],
+  "unwantedRecommendations": []
 }

README.md

@@ -15,13 +15,20 @@
 
 ## About Trigger.dev
 
-Trigger.dev is an open source platform and SDK which allows you to create long-running background jobs with no timeouts. Write normal async code, deploy, and never hit a timeout.
+Trigger.dev is an open source platform and SDK which allows you to create long-running background jobs. Write normal async code, deploy, and never hit a timeout.
 
-#### Features:
+### Key features:
 
 - JavaScript and TypeScript SDK
 - Write reliable code by default
+- No timeouts
+- Retries (with exponential backoff)
+- Queues and concurrency controls
+- Schedules and crons
+- Full Observability; logs, live trace views, advanced filtering
+- Custom alerts, get notified by email, Slack or webhooks
 - No infrastructure to manage
+- Elastic (scaling)
 - Works with your existing tech stack
 
 ## In your codebase
@@ -45,7 +52,7 @@ export const helloWorld = task({
 
 ## Deployment
 
-Use our SDK to write tasks in your codebase. There's no infrastructure to manage, your tasks automatically scale and connect to our cloud. Or you can always [self-host](https://trigger.dev/docs/v3/open-source-self-hosting#overview).
+Use our SDK to write tasks in your codebase. There's no infrastructure to manage, your tasks automatically scale and connect to our cloud. Or you can always self-host.
 
 ## Environments
 
@@ -59,11 +66,19 @@ View every task in every run so you can tell exactly what happened. We provide a
 
 # Getting started
 
-Visit our docs [here](https://trigger.dev/docs/v3/introduction) for a full guide on how to get started with Trigger.dev.
+The quickest way to get started is to create an account and project in our [web app](https://cloud.trigger.dev), and follow the instructions in the onboarding. Build and deploy your first task in minutes.
 
-## Self-host
+### Useful links:
 
-If you prefer to self-host, you can follow our [self-hosting guide](https://trigger.dev/docs/v3/open-source-self-hosting#overview).
+- [Quick start](https://trigger.dev/docs/quick-start) - get up and running in minutes
+- [How it works](https://trigger.dev/docs/v3/how-it-works) - understand how Trigger.dev works under the hood
+- [Guides and examples](https://trigger.dev/docs/guides/introduction) - walk-through guides and code examples for popular frameworks and use cases
+
+## Self-hosting
+
+If you prefer to self-host Trigger.dev, you can follow our [self-hosting guide](https://trigger.dev/docs/v3/open-source-self-hosting#overview).
+
+We also have a dedicated self-hosting channel in our [Discord server](https://trigger.dev/discord) for support.
 
 ## Development
 

apps/coordinator/src/checkpointer.ts

@@ -436,7 +436,10 @@ export class Checkpointer {
         this.#logger.error("Error during cleanup", { ...metadata, error });
       }
 
-      this.#abortControllers.delete(runId);
+      // Ensure only the current controller is removed
+      if (this.#abortControllers.get(runId) === controller) {
+        this.#abortControllers.delete(runId);
+      }
       controller.signal.removeEventListener("abort", onAbort);
     };
 

apps/coordinator/src/exec.ts

@@ -64,7 +64,18 @@ export class Exec {
       command,
       argsRaw: args,
       argsTrimmed,
-      ...output,
+      globalOpts: {
+        trimArgs: this.trimArgs,
+        neverThrow: this.neverThrow,
+        hasAbortSignal: !!this.abortSignal,
+      },
+      localOpts: opts,
+      stdout: output.stdout,
+      stderr: output.stderr,
+      pid: result.pid,
+      exitCode: result.exitCode,
+      aborted: result.aborted,
+      killed: result.killed,
     };
 
     if (this.logOutput) {

apps/coordinator/src/index.ts

@@ -536,7 +536,11 @@ class TaskCoordinator {
         socket.on("TEST", (message, callback) => {
           logger.log("Handling TEST", { eventName: "TEST", ...getSocketMetadata(), ...message });
 
-          callback();
+          try {
+            callback();
+          } catch (error) {
+            logger.error("TEST error", { error });
+          }
         });
 
         // Deprecated: Only workers without support for lazy attempts use this
@@ -669,13 +673,25 @@ class TaskCoordinator {
 
           log.log("Handling READY_FOR_RESUME");
 
-          updateAttemptFriendlyId(message.attemptFriendlyId);
+          try {
+            updateAttemptFriendlyId(message.attemptFriendlyId);
 
-          if (message.version === "v2") {
-            updateAttemptNumber(message.attemptNumber);
+            if (message.version === "v2") {
+              updateAttemptNumber(message.attemptNumber);
+            }
+
+            this.#platformSocket?.send("READY_FOR_RESUME", { ...message, version: "v1" });
+          } catch (error) {
+            log.error("READY_FOR_RESUME error", { error });
+
+            await crashRun({
+              name: "ReadyForResumeError",
+              message:
+                error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error",
+            });
+
+            return;
           }
-
-          this.#platformSocket?.send("READY_FOR_RESUME", { ...message, version: "v1" });
         });
 
         // MARK: RUN COMPLETED
@@ -692,101 +708,113 @@ class TaskCoordinator {
 
           log.log("Handling TASK_RUN_COMPLETED");
 
-          const { completion, execution } = message;
+          try {
+            const { completion, execution } = message;
 
-          // Cancel all in-progress checkpoints (if any)
-          this.#cancelCheckpoint(socket.data.runId);
+            // Cancel all in-progress checkpoints (if any)
+            this.#cancelCheckpoint(socket.data.runId);
 
-          await chaosMonkey.call({ throwErrors: false });
+            await chaosMonkey.call({ throwErrors: false });
 
-          const completeWithoutCheckpoint = (shouldExit: boolean) => {
-            const supportsRetryCheckpoints = message.version === "v1";
+            const completeWithoutCheckpoint = (shouldExit: boolean) => {
+              const supportsRetryCheckpoints = message.version === "v1";
+
+              this.#platformSocket?.send("TASK_RUN_COMPLETED", {
+                version: supportsRetryCheckpoints ? "v1" : "v2",
+                execution,
+                completion,
+              });
+              callback({ willCheckpointAndRestore: false, shouldExit });
+            };
+
+            if (completion.ok) {
+              completeWithoutCheckpoint(true);
+              return;
+            }
+
+            if (
+              completion.error.type === "INTERNAL_ERROR" &&
+              completion.error.code === "TASK_RUN_CANCELLED"
+            ) {
+              completeWithoutCheckpoint(true);
+              return;
+            }
+
+            if (completion.retry === undefined) {
+              completeWithoutCheckpoint(true);
+              return;
+            }
+
+            if (completion.retry.delay < this.#delayThresholdInMs) {
+              completeWithoutCheckpoint(false);
+
+              // Prevents runs that fail fast from never sending a heartbeat
+              this.#sendRunHeartbeat(socket.data.runId);
+
+              return;
+            }
+
+            if (message.version === "v2") {
+              completeWithoutCheckpoint(true);
+              return;
+            }
+
+            const { canCheckpoint, willSimulate } = await this.#checkpointer.init();
+
+            const willCheckpointAndRestore = canCheckpoint || willSimulate;
+
+            if (!willCheckpointAndRestore) {
+              completeWithoutCheckpoint(false);
+              return;
+            }
+
+            // The worker will then put itself in a checkpointable state
+            callback({ willCheckpointAndRestore: true, shouldExit: false });
+
+            const ready = await readyToCheckpoint("RETRY");
+
+            if (!ready.success) {
+              log.error("Failed to become checkpointable", { reason: ready.reason });
+
+              return;
+            }
+
+            const checkpoint = await this.#checkpointer.checkpointAndPush({
+              runId: socket.data.runId,
+              projectRef: socket.data.projectRef,
+              deploymentVersion: socket.data.deploymentVersion,
+              shouldHeartbeat: true,
+            });
+
+            if (!checkpoint) {
+              log.error("Failed to checkpoint");
+              completeWithoutCheckpoint(false);
+              return;
+            }
+
+            log.addFields({ checkpoint });
 
             this.#platformSocket?.send("TASK_RUN_COMPLETED", {
-              version: supportsRetryCheckpoints ? "v1" : "v2",
+              version: "v1",
               execution,
               completion,
+              checkpoint,
             });
-            callback({ willCheckpointAndRestore: false, shouldExit });
-          };
 
-          if (completion.ok) {
-            completeWithoutCheckpoint(true);
-            return;
-          }
+            if (!checkpoint.docker || !willSimulate) {
+              exitRun();
+            }
+          } catch (error) {
+            log.error("TASK_RUN_COMPLETED error", { error });
 
-          if (
-            completion.error.type === "INTERNAL_ERROR" &&
-            completion.error.code === "TASK_RUN_CANCELLED"
-          ) {
-            completeWithoutCheckpoint(true);
-            return;
-          }
-
-          if (completion.retry === undefined) {
-            completeWithoutCheckpoint(true);
-            return;
-          }
-
-          if (completion.retry.delay < this.#delayThresholdInMs) {
-            completeWithoutCheckpoint(false);
-
-            // Prevents runs that fail fast from never sending a heartbeat
-            this.#sendRunHeartbeat(socket.data.runId);
+            await crashRun({
+              name: "TaskRunCompletedError",
+              message:
+                error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error",
+            });
 
             return;
           }
-
-          if (message.version === "v2") {
-            completeWithoutCheckpoint(true);
-            return;
-          }
-
-          const { canCheckpoint, willSimulate } = await this.#checkpointer.init();
-
-          const willCheckpointAndRestore = canCheckpoint || willSimulate;
-
-          if (!willCheckpointAndRestore) {
-            completeWithoutCheckpoint(false);
-            return;
-          }
-
-          // The worker will then put itself in a checkpointable state
-          callback({ willCheckpointAndRestore: true, shouldExit: false });
-
-          const ready = await readyToCheckpoint("RETRY");
-
-          if (!ready.success) {
-            log.error("Failed to become checkpointable", { reason: ready.reason });
-
-            return;
-          }
-
-          const checkpoint = await this.#checkpointer.checkpointAndPush({
-            runId: socket.data.runId,
-            projectRef: socket.data.projectRef,
-            deploymentVersion: socket.data.deploymentVersion,
-            shouldHeartbeat: true,
-          });
-
-          if (!checkpoint) {
-            log.error("Failed to checkpoint");
-            completeWithoutCheckpoint(false);
-            return;
-          }
-
-          log.addFields({ checkpoint });
-
-          this.#platformSocket?.send("TASK_RUN_COMPLETED", {
-            version: "v1",
-            execution,
-            completion,
-            checkpoint,
-          });
-
-          if (!checkpoint.docker || !willSimulate) {
-            exitRun();
-          }
         });
 
         // MARK: TASK FAILED
@@ -802,15 +830,21 @@ class TaskCoordinator {
 
           log.log("Handling TASK_RUN_FAILED_TO_RUN");
 
-          // Cancel all in-progress checkpoints (if any)
-          this.#cancelCheckpoint(socket.data.runId);
+          try {
+            // Cancel all in-progress checkpoints (if any)
+            this.#cancelCheckpoint(socket.data.runId);
 
-          this.#platformSocket?.send("TASK_RUN_FAILED_TO_RUN", {
-            version: "v1",
-            completion,
-          });
+            this.#platformSocket?.send("TASK_RUN_FAILED_TO_RUN", {
+              version: "v1",
+              completion,
+            });
 
-          exitRun();
+            exitRun();
+          } catch (error) {
+            log.error("TASK_RUN_FAILED_TO_RUN error", { error });
+
+            return;
+          }
         });
 
         // MARK: CHECKPOINT
@@ -823,14 +857,20 @@ class TaskCoordinator {
 
           log.log("Handling READY_FOR_CHECKPOINT");
 
-          const checkpointable = this.#checkpointableTasks.get(socket.data.runId);
+          try {
+            const checkpointable = this.#checkpointableTasks.get(socket.data.runId);
+
+            if (!checkpointable) {
+              log.error("No checkpoint scheduled");
+              return;
+            }
+
+            checkpointable.resolve();
+          } catch (error) {
+            log.error("READY_FOR_CHECKPOINT error", { error });
 
-          if (!checkpointable) {
-            log.error("No checkpoint scheduled");
             return;
           }
-
-          checkpointable.resolve();
         });
 
         // MARK: CXX CHECKPOINT
@@ -843,15 +883,19 @@ class TaskCoordinator {
 
           log.log("Handling CANCEL_CHECKPOINT");
 
-          if (message.version === "v1") {
-            this.#cancelCheckpoint(socket.data.runId);
-            // v1 has no callback
-            return;
+          try {
+            if (message.version === "v1") {
+              this.#cancelCheckpoint(socket.data.runId);
+              // v1 has no callback
+              return;
+            }
+
+            const checkpointCanceled = this.#cancelCheckpoint(socket.data.runId);
+
+            callback({ version: "v2", checkpointCanceled });
+          } catch (error) {
+            log.error("CANCEL_CHECKPOINT error", { error });
           }
-
-          const checkpointCanceled = this.#cancelCheckpoint(socket.data.runId);
-
-          callback({ version: "v2", checkpointCanceled });
         });
 
         // MARK: DURATION WAIT
@@ -864,66 +908,79 @@ class TaskCoordinator {
 
           log.log("Handling WAIT_FOR_DURATION");
 
-          await chaosMonkey.call({ throwErrors: false });
+          try {
+            await chaosMonkey.call({ throwErrors: false });
+
+            if (checkpointInProgress()) {
+              log.error("Checkpoint already in progress");
+              callback({ willCheckpointAndRestore: false });
+              return;
+            }
+
+            const { canCheckpoint, willSimulate } = await this.#checkpointer.init();
+
+            const willCheckpointAndRestore = canCheckpoint || willSimulate;
+
+            callback({ willCheckpointAndRestore });
+
+            if (!willCheckpointAndRestore) {
+              return;
+            }
+
+            const ready = await readyToCheckpoint("WAIT_FOR_DURATION");
+
+            if (!ready.success) {
+              log.error("Failed to become checkpointable", { reason: ready.reason });
+              return;
+            }
+
+            const checkpoint = await this.#checkpointer.checkpointAndPush({
+              runId: socket.data.runId,
+              projectRef: socket.data.projectRef,
+              deploymentVersion: socket.data.deploymentVersion,
+              attemptNumber: getAttemptNumber(),
+            });
+
+            if (!checkpoint) {
+              // The task container will keep running until the wait duration has elapsed
+              log.error("Failed to checkpoint");
+              return;
+            }
+
+            log.addFields({ checkpoint });
+
+            const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", {
+              version: "v1",
+              runId: socket.data.runId,
+              attemptFriendlyId: message.attemptFriendlyId,
+              docker: checkpoint.docker,
+              location: checkpoint.location,
+              reason: {
+                type: "WAIT_FOR_DURATION",
+                ms: message.ms,
+                now: message.now,
+              },
+            });
+
+            if (ack?.keepRunAlive) {
+              log.log("keeping run alive after duration checkpoint");
+              return;
+            }
+
+            if (!checkpoint.docker || !willSimulate) {
+              exitRun();
+            }
+          } catch (error) {
+            log.error("WAIT_FOR_DURATION error", { error });
+
+            await crashRun({
+              name: "WaitForDurationError",
+              message:
+                error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error",
+            });
 
-          if (checkpointInProgress()) {
-            log.error("Checkpoint already in progress");
-            callback({ willCheckpointAndRestore: false });
             return;
           }
-
-          const { canCheckpoint, willSimulate } = await this.#checkpointer.init();
-
-          const willCheckpointAndRestore = canCheckpoint || willSimulate;
-
-          callback({ willCheckpointAndRestore });
-
-          if (!willCheckpointAndRestore) {
-            return;
-          }
-
-          const ready = await readyToCheckpoint("WAIT_FOR_DURATION");
-
-          if (!ready.success) {
-            log.error("Failed to become checkpointable", { reason: ready.reason });
-            return;
-          }
-
-          const checkpoint = await this.#checkpointer.checkpointAndPush({
-            runId: socket.data.runId,
-            projectRef: socket.data.projectRef,
-            deploymentVersion: socket.data.deploymentVersion,
-            attemptNumber: getAttemptNumber(),
-          });
-
-          if (!checkpoint) {
-            // The task container will keep running until the wait duration has elapsed
-            log.error("Failed to checkpoint");
-            return;
-          }
-
-          log.addFields({ checkpoint });
-
-          const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", {
-            version: "v1",
-            attemptFriendlyId: message.attemptFriendlyId,
-            docker: checkpoint.docker,
-            location: checkpoint.location,
-            reason: {
-              type: "WAIT_FOR_DURATION",
-              ms: message.ms,
-              now: message.now,
-            },
-          });
-
-          if (ack?.keepRunAlive) {
-            log.log("keeping run alive after duration checkpoint");
-            return;
-          }
-
-          if (!checkpoint.docker || !willSimulate) {
-            exitRun();
-          }
         });
 
         // MARK: TASK WAIT
@@ -936,74 +993,87 @@ class TaskCoordinator {
 
           log.log("Handling WAIT_FOR_TASK");
 
-          await chaosMonkey.call({ throwErrors: false });
+          try {
+            await chaosMonkey.call({ throwErrors: false });
 
-          if (checkpointInProgress()) {
-            log.error("Checkpoint already in progress");
-            callback({ willCheckpointAndRestore: false });
-            return;
-          }
-
-          const { canCheckpoint, willSimulate } = await this.#checkpointer.init();
-
-          const willCheckpointAndRestore = canCheckpoint || willSimulate;
-
-          callback({ willCheckpointAndRestore });
-
-          if (!willCheckpointAndRestore) {
-            return;
-          }
-
-          // Workers with v1 schemas don't signal when they're ready to checkpoint for dependency waits
-          if (message.version === "v2") {
-            const ready = await readyToCheckpoint("WAIT_FOR_TASK");
-
-            if (!ready.success) {
-              log.error("Failed to become checkpointable", { reason: ready.reason });
+            if (checkpointInProgress()) {
+              log.error("Checkpoint already in progress");
+              callback({ willCheckpointAndRestore: false });
               return;
             }
-          }
 
-          const checkpoint = await this.#checkpointer.checkpointAndPush({
-            runId: socket.data.runId,
-            projectRef: socket.data.projectRef,
-            deploymentVersion: socket.data.deploymentVersion,
-            attemptNumber: getAttemptNumber(),
-          });
+            const { canCheckpoint, willSimulate } = await this.#checkpointer.init();
+
+            const willCheckpointAndRestore = canCheckpoint || willSimulate;
+
+            callback({ willCheckpointAndRestore });
+
+            if (!willCheckpointAndRestore) {
+              return;
+            }
+
+            // Workers with v1 schemas don't signal when they're ready to checkpoint for dependency waits
+            if (message.version === "v2") {
+              const ready = await readyToCheckpoint("WAIT_FOR_TASK");
+
+              if (!ready.success) {
+                log.error("Failed to become checkpointable", { reason: ready.reason });
+                return;
+              }
+            }
+
+            const checkpoint = await this.#checkpointer.checkpointAndPush({
+              runId: socket.data.runId,
+              projectRef: socket.data.projectRef,
+              deploymentVersion: socket.data.deploymentVersion,
+              attemptNumber: getAttemptNumber(),
+            });
+
+            if (!checkpoint) {
+              log.error("Failed to checkpoint");
+              return;
+            }
+
+            log.addFields({ checkpoint });
+
+            log.log("WAIT_FOR_TASK checkpoint created");
+
+            //setting this means we can only resume from a checkpoint
+            socket.data.requiresCheckpointResumeWithMessage = `location:${checkpoint.location}-docker:${checkpoint.docker}`;
+            log.log("WAIT_FOR_TASK set requiresCheckpointResumeWithMessage");
+
+            const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", {
+              version: "v1",
+              runId: socket.data.runId,
+              attemptFriendlyId: message.attemptFriendlyId,
+              docker: checkpoint.docker,
+              location: checkpoint.location,
+              reason: {
+                type: "WAIT_FOR_TASK",
+                friendlyId: message.friendlyId,
+              },
+            });
+
+            if (ack?.keepRunAlive) {
+              socket.data.requiresCheckpointResumeWithMessage = undefined;
+              log.log("keeping run alive after task checkpoint");
+              return;
+            }
+
+            if (!checkpoint.docker || !willSimulate) {
+              exitRun();
+            }
+          } catch (error) {
+            log.error("WAIT_FOR_TASK error", { error });
+
+            await crashRun({
+              name: "WaitForTaskError",
+              message:
+                error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error",
+            });
 
-          if (!checkpoint) {
-            log.error("Failed to checkpoint");
             return;
           }
-
-          log.addFields({ checkpoint });
-
-          log.log("WAIT_FOR_TASK checkpoint created");
-
-          //setting this means we can only resume from a checkpoint
-          socket.data.requiresCheckpointResumeWithMessage = `location:${checkpoint.location}-docker:${checkpoint.docker}`;
-          log.log("WAIT_FOR_TASK set requiresCheckpointResumeWithMessage");
-
-          const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", {
-            version: "v1",
-            attemptFriendlyId: message.attemptFriendlyId,
-            docker: checkpoint.docker,
-            location: checkpoint.location,
-            reason: {
-              type: "WAIT_FOR_TASK",
-              friendlyId: message.friendlyId,
-            },
-          });
-
-          if (ack?.keepRunAlive) {
-            socket.data.requiresCheckpointResumeWithMessage = undefined;
-            log.log("keeping run alive after task checkpoint");
-            return;
-          }
-
-          if (!checkpoint.docker || !willSimulate) {
-            exitRun();
-          }
         });
 
         // MARK: BATCH WAIT
@@ -1016,75 +1086,88 @@ class TaskCoordinator {
 
           log.log("Handling WAIT_FOR_BATCH", message);
 
-          await chaosMonkey.call({ throwErrors: false });
+          try {
+            await chaosMonkey.call({ throwErrors: false });
 
-          if (checkpointInProgress()) {
-            log.error("Checkpoint already in progress");
-            callback({ willCheckpointAndRestore: false });
-            return;
-          }
-
-          const { canCheckpoint, willSimulate } = await this.#checkpointer.init();
-
-          const willCheckpointAndRestore = canCheckpoint || willSimulate;
-
-          callback({ willCheckpointAndRestore });
-
-          if (!willCheckpointAndRestore) {
-            return;
-          }
-
-          // Workers with v1 schemas don't signal when they're ready to checkpoint for dependency waits
-          if (message.version === "v2") {
-            const ready = await readyToCheckpoint("WAIT_FOR_BATCH");
-
-            if (!ready.success) {
-              log.error("Failed to become checkpointable", { reason: ready.reason });
+            if (checkpointInProgress()) {
+              log.error("Checkpoint already in progress");
+              callback({ willCheckpointAndRestore: false });
               return;
             }
-          }
 
-          const checkpoint = await this.#checkpointer.checkpointAndPush({
-            runId: socket.data.runId,
-            projectRef: socket.data.projectRef,
-            deploymentVersion: socket.data.deploymentVersion,
-            attemptNumber: getAttemptNumber(),
-          });
+            const { canCheckpoint, willSimulate } = await this.#checkpointer.init();
+
+            const willCheckpointAndRestore = canCheckpoint || willSimulate;
+
+            callback({ willCheckpointAndRestore });
+
+            if (!willCheckpointAndRestore) {
+              return;
+            }
+
+            // Workers with v1 schemas don't signal when they're ready to checkpoint for dependency waits
+            if (message.version === "v2") {
+              const ready = await readyToCheckpoint("WAIT_FOR_BATCH");
+
+              if (!ready.success) {
+                log.error("Failed to become checkpointable", { reason: ready.reason });
+                return;
+              }
+            }
+
+            const checkpoint = await this.#checkpointer.checkpointAndPush({
+              runId: socket.data.runId,
+              projectRef: socket.data.projectRef,
+              deploymentVersion: socket.data.deploymentVersion,
+              attemptNumber: getAttemptNumber(),
+            });
+
+            if (!checkpoint) {
+              log.error("Failed to checkpoint");
+              return;
+            }
+
+            log.addFields({ checkpoint });
+
+            log.log("WAIT_FOR_BATCH checkpoint created");
+
+            //setting this means we can only resume from a checkpoint
+            socket.data.requiresCheckpointResumeWithMessage = `location:${checkpoint.location}-docker:${checkpoint.docker}`;
+            log.log("WAIT_FOR_BATCH set checkpoint");
+
+            const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", {
+              version: "v1",
+              runId: socket.data.runId,
+              attemptFriendlyId: message.attemptFriendlyId,
+              docker: checkpoint.docker,
+              location: checkpoint.location,
+              reason: {
+                type: "WAIT_FOR_BATCH",
+                batchFriendlyId: message.batchFriendlyId,
+                runFriendlyIds: message.runFriendlyIds,
+              },
+            });
+
+            if (ack?.keepRunAlive) {
+              socket.data.requiresCheckpointResumeWithMessage = undefined;
+              log.log("keeping run alive after batch checkpoint");
+              return;
+            }
+
+            if (!checkpoint.docker || !willSimulate) {
+              exitRun();
+            }
+          } catch (error) {
+            log.error("WAIT_FOR_BATCH error", { error });
+
+            await crashRun({
+              name: "WaitForBatchError",
+              message:
+                error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error",
+            });
 
-          if (!checkpoint) {
-            log.error("Failed to checkpoint");
             return;
           }
-
-          log.addFields({ checkpoint });
-
-          log.log("WAIT_FOR_BATCH checkpoint created");
-
-          //setting this means we can only resume from a checkpoint
-          socket.data.requiresCheckpointResumeWithMessage = `location:${checkpoint.location}-docker:${checkpoint.docker}`;
-          log.log("WAIT_FOR_BATCH set checkpoint");
-
-          const ack = await this.#platformSocket?.sendWithAck("CHECKPOINT_CREATED", {
-            version: "v1",
-            attemptFriendlyId: message.attemptFriendlyId,
-            docker: checkpoint.docker,
-            location: checkpoint.location,
-            reason: {
-              type: "WAIT_FOR_BATCH",
-              batchFriendlyId: message.batchFriendlyId,
-              runFriendlyIds: message.runFriendlyIds,
-            },
-          });
-
-          if (ack?.keepRunAlive) {
-            socket.data.requiresCheckpointResumeWithMessage = undefined;
-            log.log("keeping run alive after batch checkpoint");
-            return;
-          }
-
-          if (!checkpoint.docker || !willSimulate) {
-            exitRun();
-          }
         });
 
         // MARK: INDEX
@@ -1097,24 +1180,29 @@ class TaskCoordinator {
 
           log.log("Handling INDEX_TASKS");
 
-          const workerAck = await this.#platformSocket?.sendWithAck("CREATE_WORKER", {
-            version: "v2",
-            projectRef: socket.data.projectRef,
-            envId: socket.data.envId,
-            deploymentId: message.deploymentId,
-            metadata: {
-              contentHash: socket.data.contentHash,
-              packageVersion: message.packageVersion,
-              tasks: message.tasks,
-            },
-            supportsLazyAttempts: message.version !== "v1" && message.supportsLazyAttempts,
-          });
+          try {
+            const workerAck = await this.#platformSocket?.sendWithAck("CREATE_WORKER", {
+              version: "v2",
+              projectRef: socket.data.projectRef,
+              envId: socket.data.envId,
+              deploymentId: message.deploymentId,
+              metadata: {
+                contentHash: socket.data.contentHash,
+                packageVersion: message.packageVersion,
+                tasks: message.tasks,
+              },
+              supportsLazyAttempts: message.version !== "v1" && message.supportsLazyAttempts,
+            });
 
-          if (!workerAck) {
-            log.debug("no worker ack while indexing");
+            if (!workerAck) {
+              log.debug("no worker ack while indexing");
+            }
+
+            callback({ success: !!workerAck?.success });
+          } catch (error) {
+            log.error("INDEX_TASKS error", { error });
+            callback({ success: false });
           }
-
-          callback({ success: !!workerAck?.success });
         });
 
         // MARK: INDEX FAILED
@@ -1127,11 +1215,15 @@ class TaskCoordinator {
 
           log.log("Handling INDEXING_FAILED");
 
-          this.#platformSocket?.send("INDEXING_FAILED", {
-            version: "v1",
-            deploymentId: message.deploymentId,
-            error: message.error,
-          });
+          try {
+            this.#platformSocket?.send("INDEXING_FAILED", {
+              version: "v1",
+              deploymentId: message.deploymentId,
+              error: message.error,
+            });
+          } catch (error) {
+            log.error("INDEXING_FAILED error", { error });
+          }
         });
 
         // MARK: CREATE ATTEMPT
@@ -1144,26 +1236,38 @@ class TaskCoordinator {
 
           log.log("Handling CREATE_TASK_RUN_ATTEMPT");
 
-          await chaosMonkey.call({ throwErrors: false });
+          try {
+            await chaosMonkey.call({ throwErrors: false });
 
-          const createAttempt = await this.#platformSocket?.sendWithAck("CREATE_TASK_RUN_ATTEMPT", {
-            runId: message.runId,
-            envId: socket.data.envId,
-          });
+            const createAttempt = await this.#platformSocket?.sendWithAck(
+              "CREATE_TASK_RUN_ATTEMPT",
+              {
+                runId: message.runId,
+                envId: socket.data.envId,
+              }
+            );
 
-          if (!createAttempt?.success) {
-            log.debug("no ack while creating attempt", { reason: createAttempt?.reason });
-            callback({ success: false, reason: createAttempt?.reason });
-            return;
+            if (!createAttempt?.success) {
+              log.debug("no ack while creating attempt", { reason: createAttempt?.reason });
+              callback({ success: false, reason: createAttempt?.reason });
+              return;
+            }
+
+            updateAttemptFriendlyId(createAttempt.executionPayload.execution.attempt.id);
+            updateAttemptNumber(createAttempt.executionPayload.execution.attempt.number);
+
+            callback({
+              success: true,
+              executionPayload: createAttempt.executionPayload,
+            });
+          } catch (error) {
+            log.error("CREATE_TASK_RUN_ATTEMPT error", { error });
+            callback({
+              success: false,
+              reason:
+                error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error",
+            });
           }
-
-          updateAttemptFriendlyId(createAttempt.executionPayload.execution.attempt.id);
-          updateAttemptNumber(createAttempt.executionPayload.execution.attempt.number);
-
-          callback({
-            success: true,
-            executionPayload: createAttempt.executionPayload,
-          });
         });
 
         socket.on("UNRECOVERABLE_ERROR", async (message) => {
@@ -1175,7 +1279,11 @@ class TaskCoordinator {
 
           log.log("Handling UNRECOVERABLE_ERROR");
 
-          await crashRun(message.error);
+          try {
+            await crashRun(message.error);
+          } catch (error) {
+            log.error("UNRECOVERABLE_ERROR error", { error });
+          }
         });
 
         socket.on("SET_STATE", async (message) => {
@@ -1187,20 +1295,28 @@ class TaskCoordinator {
 
           log.log("Handling SET_STATE");
 
-          if (message.attemptFriendlyId) {
-            updateAttemptFriendlyId(message.attemptFriendlyId);
-          }
+          try {
+            if (message.attemptFriendlyId) {
+              updateAttemptFriendlyId(message.attemptFriendlyId);
+            }
 
-          if (message.attemptNumber) {
-            updateAttemptNumber(message.attemptNumber);
+            if (message.attemptNumber) {
+              updateAttemptNumber(message.attemptNumber);
+            }
+          } catch (error) {
+            log.error("SET_STATE error", { error });
           }
         });
       },
       onDisconnect: async (socket, handler, sender, logger) => {
-        this.#platformSocket?.send("LOG", {
-          metadata: socket.data,
-          text: "disconnect",
-        });
+        try {
+          this.#platformSocket?.send("LOG", {
+            metadata: socket.data,
+            text: "disconnect",
+          });
+        } catch (error) {
+          logger.error("onDisconnect error", { error });
+        }
       },
       handlers: {
         TASK_HEARTBEAT: async (message) => {

apps/kubernetes-provider/src/index.ts

@@ -34,6 +34,9 @@ const UPTIME_MAX_PENDING_RUNS = Number(process.env.UPTIME_MAX_PENDING_RUNS || "2
 const UPTIME_MAX_PENDING_INDECES = Number(process.env.UPTIME_MAX_PENDING_INDECES || "10");
 const UPTIME_MAX_PENDING_ERRORS = Number(process.env.UPTIME_MAX_PENDING_ERRORS || "10");
 
+const POD_EPHEMERAL_STORAGE_SIZE_LIMIT = process.env.POD_EPHEMERAL_STORAGE_SIZE_LIMIT || "10Gi";
+const POD_EPHEMERAL_STORAGE_SIZE_REQUEST = process.env.POD_EPHEMERAL_STORAGE_SIZE_REQUEST || "2Gi";
+
 const logger = new SimpleLogger(`[${NODE_NAME}]`);
 logger.log(`running in ${RUNTIME_ENV} mode`);
 
@@ -396,13 +399,13 @@ class KubernetesTaskOperations implements TaskOperations {
 
   get #defaultResourceRequests(): ResourceQuantities {
     return {
-      "ephemeral-storage": "2Gi",
+      "ephemeral-storage": POD_EPHEMERAL_STORAGE_SIZE_REQUEST,
     };
   }
 
   get #defaultResourceLimits(): ResourceQuantities {
     return {
-      "ephemeral-storage": "10Gi",
+      "ephemeral-storage": POD_EPHEMERAL_STORAGE_SIZE_LIMIT,
     };
   }
 
@@ -658,6 +661,7 @@ provider.listen();
 
 const taskMonitor = new TaskMonitor({
   runtimeEnv: RUNTIME_ENV,
+  namespace: KUBERNETES_NAMESPACE,
   onIndexFailure: async (deploymentId, details) => {
     logger.log("Indexing failed", { deploymentId, details });
 

apps/kubernetes-provider/src/taskMonitor.ts

@@ -160,7 +160,10 @@ export class TaskMonitor {
 
     let reason = rawReason || "Unknown error";
     let logs = rawLogs || "";
-    let overrideCompletion = false;
+
+    /** This will only override existing task errors. It will not crash the run.  */
+    let onlyOverrideExistingError = exitCode === EXIT_CODE_CHILD_NONZERO;
+
     let errorCode: TaskRunInternalError["code"] = TaskRunErrorCodes.POD_UNKNOWN_ERROR;
 
     switch (rawReason) {
@@ -185,10 +188,8 @@ export class TaskMonitor {
         }
         break;
       case "OOMKilled":
-        overrideCompletion = true;
-        reason = `${
-          exitCode === EXIT_CODE_CHILD_NONZERO ? "Child process" : "Parent process"
-        } ran out of memory! Try choosing a machine preset with more memory for this task.`;
+        reason =
+          "[TaskMonitor] Your task ran out of memory. Try increasing the machine specs. If this doesn't fix it there might be a memory leak.";
         errorCode = TaskRunErrorCodes.TASK_PROCESS_OOM_KILLED;
         break;
       default:
@@ -199,7 +200,7 @@ export class TaskMonitor {
       exitCode,
       reason,
       logs,
-      overrideCompletion,
+      overrideCompletion: onlyOverrideExistingError,
       errorCode,
     } satisfies FailureDetails;
 

apps/proxy/package.json

@@ -4,7 +4,8 @@
   "private": true,
   "scripts": {
     "deploy": "wrangler deploy",
-    "dev": "wrangler dev"
+    "dev": "wrangler dev",
+    "dry-run:staging": "wrangler deploy --dry-run --outdir=dist --env staging"
   },
   "devDependencies": {
     "@cloudflare/workers-types": "^4.20240512.0",

apps/proxy/src/index.ts

@@ -17,12 +17,9 @@ export interface Env {
 
 export default {
   async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
-    if (!env.REWRITE_HOSTNAME) throw new Error("Missing REWRITE_HOSTNAME");
-    console.log("url", request.url);
-
     if (!queueingIsEnabled(env)) {
       console.log("Missing AWS credentials. Passing through to the origin.");
-      return redirectToOrigin(request, env);
+      return fetch(request);
     }
 
     const url = new URL(request.url);
@@ -42,25 +39,10 @@ export default {
     }
 
     //the same request but with the hostname (and port) changed
-    return redirectToOrigin(request, env);
+    return fetch(request);
   },
 };
 
-function redirectToOrigin(request: Request, env: Env) {
-  const newUrl = new URL(request.url);
-  newUrl.hostname = env.REWRITE_HOSTNAME;
-  newUrl.port = env.REWRITE_PORT || newUrl.port;
-
-  const requestInit: RequestInit = {
-    method: request.method,
-    headers: request.headers,
-    body: request.body,
-  };
-
-  console.log("rewritten url", newUrl.toString());
-  return fetch(newUrl.toString(), requestInit);
-}
-
 function queueingIsEnabled(env: Env) {
   return (
     env.AWS_SQS_ACCESS_KEY_ID &&

apps/webapp/app/components/ErrorDisplay.tsx

@@ -1,7 +1,10 @@
+import { HomeIcon } from "@heroicons/react/20/solid";
 import { isRouteErrorResponse, useRouteError } from "@remix-run/react";
-import { LinkButton } from "./primitives/Buttons";
-import { Header1, Header3 } from "./primitives/Headers";
+import { motion } from "framer-motion";
 import { friendlyErrorDisplay } from "~/utils/httpErrors";
+import { LinkButton } from "./primitives/Buttons";
+import { Header1 } from "./primitives/Headers";
+import { Paragraph } from "./primitives/Paragraph";
 
 type ErrorDisplayOptions = {
   button?: {
@@ -39,12 +42,32 @@ type DisplayOptionsProps = {
 
 export function ErrorDisplay({ title, message, button }: DisplayOptionsProps) {
   return (
-    <div className="p-4">
-      <Header1 className="mb-4 border-b border-charcoal-800 pb-4">{title}</Header1>
-      {message && <Header3>{message}</Header3>}
-      <LinkButton to={button ? button.to : "/"} variant="primary/medium" className="mt-8">
-        {button ? button.title : "Home"}
-      </LinkButton>
+    <div className="relative flex min-h-screen flex-col items-center justify-center bg-[#16181C]">
+      <div className="z-10 mt-[30vh] flex flex-col items-center gap-8">
+        <Header1>{title}</Header1>
+        {message && <Paragraph>{message}</Paragraph>}
+        <LinkButton
+          to={button ? button.to : "/"}
+          shortcut={{ modifiers: ["meta"], key: "g" }}
+          variant="primary/medium"
+          LeadingIcon={HomeIcon}
+        >
+          {button ? button.title : "Go to homepage"}
+        </LinkButton>
+      </div>
+      <div className="pointer-events-none absolute bottom-4 right-4 z-10 h-[70px] w-[200px] bg-[rgb(24,26,30)]" />
+      <motion.div
+        className="pointer-events-none absolute inset-0 overflow-hidden"
+        initial={{ opacity: 0 }}
+        animate={{ opacity: 1 }}
+        transition={{ delay: 0.5, duration: 2, ease: "easeOut" }}
+      >
+        <iframe
+          src="https://my.spline.design/untitled-a6f70b5ebc46bdb2dcc0f21d5397e8ac/"
+          className="pointer-events-none absolute inset-0 h-full w-full object-cover"
+          style={{ border: "none" }}
+        />
+      </motion.div>
     </div>
   );
 }

apps/webapp/app/components/Feedback.tsx

@@ -1,31 +1,23 @@
 import { conform, useForm } from "@conform-to/react";
 import { parse } from "@conform-to/zod";
-import { BookOpenIcon } from "@heroicons/react/20/solid";
-import {
-  CalendarDaysIcon,
-  ChevronRightIcon,
-  EnvelopeIcon,
-  LifebuoyIcon,
-  LightBulbIcon,
-} from "@heroicons/react/24/solid";
+import { InformationCircleIcon } from "@heroicons/react/20/solid";
+import { EnvelopeIcon } from "@heroicons/react/24/solid";
 import { Form, useActionData, useLocation, useNavigation } from "@remix-run/react";
-import { DiscordIcon } from "@trigger.dev/companyicons";
-import { ActivityIcon } from "lucide-react";
-import { type ReactNode, useState } from "react";
+import { type ReactNode, useEffect, useState } from "react";
 import { type FeedbackType, feedbackTypeLabel, schema } from "~/routes/resources.feedback";
-import { cn } from "~/utils/cn";
-import { docsTroubleshootingPath } from "~/utils/pathBuilder";
-import { Button, LinkButton } from "./primitives/Buttons";
+import { Button } from "./primitives/Buttons";
+import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "./primitives/Dialog";
 import { Fieldset } from "./primitives/Fieldset";
 import { FormButtons } from "./primitives/FormButtons";
 import { FormError } from "./primitives/FormError";
-import { Header1 } from "./primitives/Headers";
+import { Icon } from "./primitives/Icon";
+import { InfoPanel } from "./primitives/InfoPanel";
 import { InputGroup } from "./primitives/InputGroup";
 import { Label } from "./primitives/Label";
 import { Paragraph } from "./primitives/Paragraph";
 import { Select, SelectItem } from "./primitives/Select";
-import { Sheet, SheetBody, SheetContent, SheetTrigger } from "./primitives/Sheet";
 import { TextArea } from "./primitives/TextArea";
+import { TextLink } from "./primitives/TextLink";
 
 type FeedbackProps = {
   button: ReactNode;
@@ -37,161 +29,111 @@ export function Feedback({ button, defaultValue = "bug" }: FeedbackProps) {
   const location = useLocation();
   const lastSubmission = useActionData();
   const navigation = useNavigation();
+  const [type, setType] = useState<FeedbackType>(defaultValue);
 
   const [form, { path, feedbackType, message }] = useForm({
     id: "accept-invite",
-    // TODO: type this
     lastSubmission: lastSubmission as any,
     onValidate({ formData }) {
       return parse(formData, { schema });
     },
+    shouldRevalidate: "onInput",
   });
 
-  if (
-    open &&
-    navigation.formAction === "/resources/feedback" &&
-    form.error === undefined &&
-    form.errors.length === 0
-  ) {
-    setOpen(false);
-  }
+  useEffect(() => {
+    if (
+      navigation.formAction === "/resources/feedback" &&
+      navigation.state === "loading" &&
+      form.error === undefined &&
+      form.errors.length === 0
+    ) {
+      setOpen(false);
+    }
+  }, [navigation, form]);
 
   return (
-    <Sheet open={open} onOpenChange={setOpen}>
-      <SheetTrigger asChild={true}>{button}</SheetTrigger>
-      <SheetContent className="@container">
-        <SheetBody className="flex h-full flex-col justify-between">
-          <LinkBanner
-            title="Join our Discord community"
-            icon={<DiscordIcon className="size-9" />}
-            to="https://trigger.dev/discord"
-            className="hover:border-text-link"
-          >
-            <Paragraph>The quickest way to get answers from the Trigger.dev community.</Paragraph>
-          </LinkBanner>
-          <LinkBanner
-            title="Book a 15 min chat with the founders"
-            icon={<CalendarDaysIcon className="size-9 text-green-500" />}
-            to="https://cal.com/team/triggerdotdev/founders-call"
-            className="hover:border-green-500"
-          >
-            <Paragraph>Have a question or want to chat? Book a time to talk with us.</Paragraph>
-          </LinkBanner>
-          <LinkBanner
-            title="Suggest a feature"
-            icon={<LightBulbIcon className="size-9 text-sun-500" />}
-            to="https://feedback.trigger.dev/"
-            className="hover:border-sun-400"
-          >
-            <Paragraph>Have an idea for a new feature or improvement? Let us know!</Paragraph>
-          </LinkBanner>
-          <LinkBanner
-            title="Troubleshooting"
-            icon={<LifebuoyIcon className="size-9 text-rose-500" />}
-          >
-            <Paragraph>
-              If you're having trouble, check out our troubleshooting guide or the Trigger.dev
-              Status page.
+    <Dialog open={open} onOpenChange={setOpen}>
+      <DialogTrigger asChild>{button}</DialogTrigger>
+      <DialogContent>
+        <DialogHeader>Contact us</DialogHeader>
+        <div className="mt-2 flex flex-col gap-4">
+          <div className="flex items-center gap-4">
+            <Icon icon={EnvelopeIcon} className="size-10 min-w-[2.5rem] text-blue-500" />
+            <Paragraph variant="base/bright">
+              How can we help? We read every message and will respond as quickly as we can.
             </Paragraph>
-            <div className="flex flex-wrap gap-2">
-              <LinkButton
-                to={docsTroubleshootingPath("")}
-                variant="tertiary/medium"
-                LeadingIcon={BookOpenIcon}
-              >
-                Troubleshooting Docs
-              </LinkButton>
-              <LinkButton
-                to={"https://status.trigger.dev/"}
-                variant="tertiary/medium"
-                LeadingIcon={ActivityIcon}
-              >
-                Trigger.dev Status
-              </LinkButton>
-            </div>
-          </LinkBanner>
-          <LinkBanner
-            title="Send us an email"
-            icon={<EnvelopeIcon className="size-9 text-blue-500" />}
-          >
-            <Paragraph>We read every message and respond quickly.</Paragraph>
-            <Form method="post" action="/resources/feedback" {...form.props} className="w-full">
-              <Fieldset className="max-w-full gap-y-3">
-                <input value={location.pathname} {...conform.input(path, { type: "hidden" })} />
-                <InputGroup className="max-w-full">
-                  <Select
-                    {...conform.select(feedbackType)}
-                    variant="tertiary/medium"
-                    defaultValue={defaultValue}
-                    placeholder="Select type"
-                    text={(value) => feedbackTypeLabel[value]}
-                    dropdownIcon
+          </div>
+          <hr className="border-charcoal-800" />
+          <Form method="post" action="/resources/feedback" {...form.props} className="w-full">
+            <Fieldset className="max-w-full gap-y-3">
+              <input value={location.pathname} {...conform.input(path, { type: "hidden" })} />
+              <InputGroup className="max-w-full">
+                {type === "feature" && (
+                  <InfoPanel
+                    icon={InformationCircleIcon}
+                    iconClassName="text-blue-500"
+                    panelClassName="w-full mb-2"
                   >
-                    {Object.entries(feedbackTypeLabel).map(([name, title]) => (
-                      <SelectItem key={name} value={name}>
-                        {title}
-                      </SelectItem>
-                    ))}
-                  </Select>
-                  <FormError id={feedbackType.errorId}>{feedbackType.error}</FormError>
-                </InputGroup>
-                <InputGroup className="max-w-full">
-                  <Label>Message</Label>
-                  <TextArea {...conform.textarea(message)} />
-                  <FormError id={message.errorId}>{message.error}</FormError>
-                </InputGroup>
-                <FormError>{form.error}</FormError>
-                <div className="flex w-full justify-end">
-                  <FormButtons
-                    className="m-0 w-max"
-                    confirmButton={
-                      <Button type="submit" variant="tertiary/medium">
-                        Send message
-                      </Button>
-                    }
-                  />
-                </div>
-              </Fieldset>
-            </Form>
-          </LinkBanner>
-        </SheetBody>
-      </SheetContent>
-    </Sheet>
-  );
-}
-
-function LinkBanner({
-  className,
-  icon,
-  title,
-  children,
-  to,
-}: {
-  className?: string;
-  icon?: ReactNode;
-  title?: string;
-  children?: ReactNode;
-  to?: string;
-}) {
-  return (
-    <a
-      href={to}
-      target="_blank"
-      className={cn(
-        "group/banner mb-4 flex w-full items-center justify-between rounded-md border border-grid-bright bg-charcoal-750 p-4 transition",
-        className
-      )}
-    >
-      <div className="flex w-full items-start gap-4">
-        <span>{icon}</span>
-        <div className="flex w-full flex-col gap-2">
-          <Header1 className="text-2xl font-semibold text-text-bright">{title}</Header1>
-          {children}
+                    <Paragraph variant="small">
+                      All our feature requests are public and voted on by the community. The best
+                      way to submit your feature request is to{" "}
+                      <TextLink to="https://feedback.trigger.dev">
+                        post it to our feedback forum
+                      </TextLink>
+                      .
+                    </Paragraph>
+                  </InfoPanel>
+                )}
+                {type === "help" && (
+                  <InfoPanel
+                    icon={InformationCircleIcon}
+                    iconClassName="text-blue-500"
+                    panelClassName="w-full mb-2"
+                  >
+                    <Paragraph variant="small">
+                      The quickest way to get answers from the Trigger.dev team and community is to{" "}
+                      <TextLink to="https://trigger.dev/discord">ask in our Discord</TextLink>.
+                    </Paragraph>
+                  </InfoPanel>
+                )}
+                <Select
+                  {...conform.select(feedbackType)}
+                  variant="tertiary/medium"
+                  value={type}
+                  defaultValue={type}
+                  setValue={(v) => setType(v as FeedbackType)}
+                  placeholder="Select type"
+                  text={(value) => feedbackTypeLabel[value as FeedbackType]}
+                  dropdownIcon
+                >
+                  {Object.entries(feedbackTypeLabel).map(([name, title]) => (
+                    <SelectItem key={name} value={name}>
+                      {title}
+                    </SelectItem>
+                  ))}
+                </Select>
+                <FormError id={feedbackType.errorId}>{feedbackType.error}</FormError>
+              </InputGroup>
+              <InputGroup className="max-w-full">
+                <Label>Message</Label>
+                <TextArea {...conform.textarea(message)} />
+                <FormError id={message.errorId}>{message.error}</FormError>
+              </InputGroup>
+              <FormError>{form.error}</FormError>
+              <div className="flex w-full justify-end">
+                <FormButtons
+                  className="m-0 w-max"
+                  confirmButton={
+                    <Button type="submit" variant="tertiary/medium">
+                      Send message
+                    </Button>
+                  }
+                />
+              </div>
+            </Fieldset>
+          </Form>
         </div>
-      </div>
-      {to && (
-        <ChevronRightIcon className="size-5 text-charcoal-500 transition group-hover:translate-x-1 group-hover/banner:text-text-bright" />
-      )}
-    </a>
+      </DialogContent>
+    </Dialog>
   );
 }

apps/webapp/app/components/billing/v2/FreePlanUsage.tsx

@@ -27,7 +27,7 @@ export function FreePlanUsage({ to, percentage }: { to: string; percentage: numb
           <ArrowUpCircleIcon className="h-5 w-5 text-text-dimmed" />
           <Paragraph className="text-2sm text-text-bright">Free Plan</Paragraph>
         </div>
-        <Link to={to} className="text-2sm text-text-link">
+        <Link to={to} className="text-2sm text-text-link focus-custom">
           Upgrade
         </Link>
       </div>

apps/webapp/app/components/code/CodeBlock.tsx

@@ -241,7 +241,7 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
                 onMouseEnter={() => setMouseOver(true)}
                 onMouseLeave={() => setMouseOver(false)}
                 className={cn(
-                  "absolute right-3 z-50 transition-colors duration-100 hover:cursor-pointer",
+                  "absolute right-3 z-50 transition-colors duration-100 focus-custom hover:cursor-pointer",
                   showChrome ? "top-10" : "top-2.5",
                   copied ? "text-emerald-500" : "text-charcoal-500 hover:text-charcoal-300"
                 )}

apps/webapp/app/components/navigation/AccountSideMenu.tsx

@@ -38,7 +38,7 @@ export function AccountSideMenu({ user }: { user: User }) {
             <SideMenuItem
               name="Your profile"
               icon="account"
-              iconColor="text-indigo-500"
+              activeIconColor="text-indigo-500"
               to={accountPath()}
               data-action="account"
             />
@@ -49,7 +49,7 @@ export function AccountSideMenu({ user }: { user: User }) {
               <SideMenuItem
                 name="Personal Access Tokens"
                 icon={ShieldCheckIcon}
-                iconColor="text-emerald-500"
+                activeIconColor="text-emerald-500"
                 to={personalAccessTokensPath()}
                 data-action="tokens"
               />

apps/webapp/app/components/navigation/SideMenu.tsx

@@ -1,14 +1,19 @@
 import {
   AcademicCapIcon,
   ArrowRightOnRectangleIcon,
+  ArrowUpRightIcon,
   BeakerIcon,
   BellAlertIcon,
+  CalendarDaysIcon,
   ChartBarIcon,
+  ChatBubbleLeftEllipsisIcon,
   ClockIcon,
   CreditCardIcon,
   CursorArrowRaysIcon,
+  EnvelopeIcon,
   IdentificationIcon,
   KeyIcon,
+  LightBulbIcon,
   RectangleStackIcon,
   ServerStackIcon,
   ShieldCheckIcon,
@@ -17,13 +22,14 @@ import {
 import { UserGroupIcon, UserPlusIcon } from "@heroicons/react/24/solid";
 import { useNavigation } from "@remix-run/react";
 import { DiscordIcon, SlackIcon } from "@trigger.dev/companyicons";
-import { Fragment, useEffect, useRef, useState } from "react";
+import { Fragment, ReactNode, useEffect, useRef, useState } from "react";
 import { TaskIcon } from "~/assets/icons/TaskIcon";
 import { useFeatures } from "~/hooks/useFeatures";
 import { type MatchedOrganization } from "~/hooks/useOrganizations";
 import { type MatchedProject } from "~/hooks/useProject";
 import { type User } from "~/models/user.server";
 import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route";
+import { FeedbackType } from "~/routes/resources.feedback";
 import { cn } from "~/utils/cn";
 import {
   accountPath,
@@ -77,11 +83,12 @@ import {
   PopoverCustomTrigger,
   PopoverMenuItem,
   PopoverSectionHeader,
+  PopoverSideMenuTrigger,
 } from "../primitives/Popover";
 import { StepNumber } from "../primitives/StepNumber";
 import { TextLink } from "../primitives/TextLink";
 import { SideMenuHeader } from "./SideMenuHeader";
-import { SideMenuItem } from "./SideMenuItem";
+import { MenuCount, SideMenuItem } from "./SideMenuItem";
 
 type SideMenuUser = Pick<User, "email" | "admin"> & { isImpersonating: boolean };
 type SideMenuProject = Pick<MatchedProject, "id" | "name" | "slug" | "version">;
@@ -91,6 +98,8 @@ type SideMenuProps = {
   project: SideMenuProject;
   organization: MatchedOrganization;
   organizations: MatchedOrganization[];
+  button?: ReactNode;
+  defaultValue?: FeedbackType;
 };
 
 export function SideMenu({ user, project, organization, organizations }: SideMenuProps) {
@@ -171,7 +180,7 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
               name="Team"
               icon={UserGroupIcon}
               to={organizationTeamPath(organization)}
-              iconColor="text-amber-500"
+              activeIconColor="text-amber-500"
               data-action="team"
             />
             {organization.projects.some((proj) => proj.version === "V3") && isManagedCloud && (
@@ -180,14 +189,14 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
                   name="Usage"
                   icon={ChartBarIcon}
                   to={v3UsagePath(organization)}
-                  iconColor="text-green-600"
+                  activeIconColor="text-green-600"
                   data-action="usage"
                 />
                 <SideMenuItem
                   name="Billing"
                   icon={CreditCardIcon}
                   to={v3BillingPath(organization)}
-                  iconColor="text-blue-600"
+                  activeIconColor="text-blue-600"
                   data-action="billing"
                   badge={
                     currentPlan?.v3Subscription?.isPaying
@@ -202,14 +211,14 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
                 name="Usage (v2)"
                 icon={ChartBarIcon}
                 to={organizationBillingPath(organization)}
-                iconColor="text-green-600"
+                activeIconColor="text-green-600"
                 data-action="usage & billing"
               />
             )}
             <SideMenuItem
               name="Organization settings"
               icon="settings"
-              iconColor="text-teal-500"
+              activeIconColor="text-teal-500"
               to={organizationSettingsPath(organization)}
               data-action="organization-settings"
             />
@@ -239,96 +248,7 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
           )}
         </div>
         <div className="flex flex-col gap-1 border-t border-grid-bright p-1">
-          {currentPlan?.v3Subscription?.plan?.limits.support === "slack" && (
-            <Dialog>
-              <DialogTrigger asChild>
-                <Button
-                  variant="small-menu-item"
-                  LeadingIcon={SlackIcon}
-                  data-action="join our slack"
-                  fullWidth
-                  textAlignLeft
-                >
-                  Join our Slack
-                </Button>
-              </DialogTrigger>
-              <DialogContent>
-                <DialogHeader>Join our Slack</DialogHeader>
-                <div className="mt-2 flex flex-col gap-4">
-                  <div className="flex items-center gap-4">
-                    <Icon icon={SlackIcon} className="h-10 w-10 min-w-[2.5rem]" />
-                    <Paragraph variant="base/bright">
-                      As a subscriber, you have access to a dedicated Slack channel for 1-to-1
-                      support with the Trigger.dev team.
-                    </Paragraph>
-                  </div>
-                  <hr className="border-charcoal-800" />
-                  <div>
-                    <StepNumber stepNumber="1" title="Email us" />
-                    <StepContentContainer>
-                      <Paragraph>
-                        Send us an email to this address from your Trigger.dev account email
-                        address:
-                        <ClipboardField
-                          variant="primary/medium"
-                          value="priority-support@trigger.dev"
-                          className="my-2"
-                        />
-                      </Paragraph>
-                    </StepContentContainer>
-                    <StepNumber stepNumber="2" title="Look out for an invite from Slack" />
-                    <StepContentContainer>
-                      <Paragraph>
-                        As soon as we can, we'll setup a Slack Connect channel and say hello!
-                      </Paragraph>
-                    </StepContentContainer>
-                  </div>
-                </div>
-              </DialogContent>
-            </Dialog>
-          )}
-          <SideMenuItem
-            name="Join our Discord"
-            icon={DiscordIcon}
-            to="https://trigger.dev/discord"
-            data-action="join our discord"
-            target="_blank"
-          />
-          <SideMenuItem
-            name="Documentation"
-            icon="docs"
-            to="https://trigger.dev/docs"
-            data-action="documentation"
-            target="_blank"
-          />
-          <SideMenuItem
-            name="Changelog"
-            icon="star"
-            to="https://trigger.dev/changelog"
-            data-action="changelog"
-            target="_blank"
-          />
-          <SideMenuItem
-            name="Status"
-            icon={SignalIcon}
-            to="https://status.trigger.dev/"
-            data-action="status"
-            target="_blank"
-          />
-          <Feedback
-            button={
-              <Button
-                variant="small-menu-item"
-                LeadingIcon="log"
-                leadingIconClassName="text-primary"
-                data-action="help & feedback"
-                fullWidth
-                textAlignLeft
-              >
-                <span className="text-primary">Help & Feedback</span>
-              </Button>
-            }
-          />
+          <HelpAndFeedback />
           {isV3Project && isFreeV3User && (
             <FreePlanUsage
               to={v3BillingPath(organization)}
@@ -341,6 +261,167 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
   );
 }
 
+function HelpAndFeedback() {
+  const [isHelpMenuOpen, setHelpMenuOpen] = useState(false);
+  const currentPlan = useCurrentPlan();
+
+  return (
+    <Popover onOpenChange={(open) => setHelpMenuOpen(open)}>
+      <PopoverSideMenuTrigger isOpen={isHelpMenuOpen} shortcut={{ key: "h" }}>
+        <div className="flex items-center gap-1.5">
+          <ChatBubbleLeftEllipsisIcon className="size-4 text-success" />
+          Help & Feedback
+        </div>
+      </PopoverSideMenuTrigger>
+      <PopoverContent
+        className="min-w-[14rem] divide-y divide-grid-bright overflow-y-auto p-0 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600"
+        align="start"
+      >
+        <Fragment>
+          <div className="flex flex-col gap-1 p-1">
+            <SideMenuItem
+              name="Documentation"
+              icon="docs"
+              trailingIcon={ArrowUpRightIcon}
+              trailingIconClassName="text-text-dimmed"
+              inactiveIconColor="text-green-500"
+              activeIconColor="text-green-500"
+              to="https://trigger.dev/docs"
+              data-action="documentation"
+              target="_blank"
+            />
+          </div>
+          <div className="flex flex-col gap-1 p-1">
+            <SideMenuItem
+              name="Status"
+              icon={SignalIcon}
+              trailingIcon={ArrowUpRightIcon}
+              trailingIconClassName="text-text-dimmed"
+              inactiveIconColor="text-green-500"
+              activeIconColor="text-green-500"
+              to="https://status.trigger.dev/"
+              data-action="status"
+              target="_blank"
+            />
+            <SideMenuItem
+              name="Suggest a feature"
+              icon={LightBulbIcon}
+              trailingIcon={ArrowUpRightIcon}
+              trailingIconClassName="text-text-dimmed"
+              inactiveIconColor="text-sun-500"
+              activeIconColor="text-sun-500"
+              to="https://feedback.trigger.dev/"
+              data-action="suggest-a-feature"
+              target="_blank"
+            />
+            <SideMenuItem
+              name="Changelog"
+              icon="star"
+              trailingIcon={ArrowUpRightIcon}
+              trailingIconClassName="text-text-dimmed"
+              inactiveIconColor="text-sun-500"
+              activeIconColor="text-sun-500"
+              to="https://trigger.dev/changelog"
+              data-action="changelog"
+              target="_blank"
+            />
+          </div>
+          <div className="flex flex-col gap-1 p-1">
+            <Paragraph className="pb-1 pl-1.5 pt-1.5 text-xs">Need help?</Paragraph>
+            {currentPlan?.v3Subscription?.plan?.limits.support === "slack" && (
+              <div>
+                <Dialog>
+                  <DialogTrigger asChild>
+                    <Button
+                      variant="small-menu-item"
+                      LeadingIcon={SlackIcon}
+                      data-action="join-our-slack"
+                      fullWidth
+                      textAlignLeft
+                    >
+                      <div className="flex w-full items-center justify-between">
+                        <span className="text-text-bright">Join our Slack…</span>
+                        <MenuCount count="PRO" />
+                      </div>
+                    </Button>
+                  </DialogTrigger>
+                  <DialogContent>
+                    <DialogHeader>Join our Slack</DialogHeader>
+                    <div className="mt-2 flex flex-col gap-4">
+                      <div className="flex items-center gap-4">
+                        <Icon icon={SlackIcon} className="h-10 w-10 min-w-[2.5rem]" />
+                        <Paragraph variant="base/bright">
+                          As a subscriber, you have access to a dedicated Slack channel for 1-to-1
+                          support with the Trigger.dev team.
+                        </Paragraph>
+                      </div>
+                      <hr className="border-charcoal-800" />
+                      <div>
+                        <StepNumber stepNumber="1" title="Email us" />
+                        <StepContentContainer>
+                          <Paragraph>
+                            Send us an email to this address from your Trigger.dev account email
+                            address:
+                            <ClipboardField
+                              variant="secondary/medium"
+                              value="priority-support@trigger.dev"
+                              className="my-2"
+                            />
+                          </Paragraph>
+                        </StepContentContainer>
+                        <StepNumber stepNumber="2" title="Look out for an invite from Slack" />
+                        <StepContentContainer>
+                          <Paragraph>
+                            As soon as we can, we'll setup a Slack Connect channel and say hello!
+                          </Paragraph>
+                        </StepContentContainer>
+                      </div>
+                    </div>
+                  </DialogContent>
+                </Dialog>
+              </div>
+            )}
+            <SideMenuItem
+              name="Ask in our Discord"
+              icon={DiscordIcon}
+              trailingIcon={ArrowUpRightIcon}
+              trailingIconClassName="text-text-dimmed"
+              to="https://trigger.dev/discord"
+              data-action="join our discord"
+              target="_blank"
+            />
+            <SideMenuItem
+              name="Book a 15 min call"
+              icon={CalendarDaysIcon}
+              trailingIcon={ArrowUpRightIcon}
+              trailingIconClassName="text-text-dimmed"
+              inactiveIconColor="text-rose-500"
+              activeIconColor="text-rose-500"
+              to="https://cal.com/team/triggerdotdev/founders-call"
+              data-action="book-a-call"
+              target="_blank"
+            />
+            <Feedback
+              button={
+                <Button
+                  variant="small-menu-item"
+                  LeadingIcon={EnvelopeIcon}
+                  leadingIconClassName="text-blue-500"
+                  data-action="contact-us"
+                  fullWidth
+                  textAlignLeft
+                >
+                  Contact us…
+                </Button>
+              }
+            />
+          </div>
+        </Fragment>
+      </PopoverContent>
+    </Popover>
+  );
+}
+
 function ProjectSelector({
   project,
   organizations,
@@ -495,47 +576,47 @@ function V2ProjectSideMenu({
       <SideMenuItem
         name="Jobs"
         icon="job"
-        iconColor="text-indigo-500"
+        activeIconColor="text-indigo-500"
         to={projectPath(organization, project)}
         data-action="jobs"
       />
       <SideMenuItem
         name="Runs"
         icon="runs"
-        iconColor="text-teal-500"
+        activeIconColor="text-teal-500"
         to={projectRunsPath(organization, project)}
       />
       <SideMenuItem
         name="Triggers"
         icon="trigger"
-        iconColor="text-amber-500"
+        activeIconColor="text-amber-500"
         to={projectTriggersPath(organization, project)}
         data-action="triggers"
       />
       <SideMenuItem
         name="Events"
         icon={CursorArrowRaysIcon}
-        iconColor="text-sky-500"
+        activeIconColor="text-sky-500"
         to={projectEventsPath(organization, project)}
       />
       <SideMenuItem
         name="HTTP endpoints"
         icon="http-endpoint"
-        iconColor="text-pink-500"
+        activeIconColor="text-pink-500"
         to={projectHttpEndpointsPath(organization, project)}
         data-action="httpendpoints"
       />
       <SideMenuItem
         name="Environments & API Keys"
         icon="environment"
-        iconColor="text-rose-500"
+        activeIconColor="text-rose-500"
         to={projectEnvironmentsPath(organization, project)}
         data-action="environments & api keys"
       />
       <SideMenuItem
         name="Project settings"
         icon="settings"
-        iconColor="text-teal-500"
+        activeIconColor="text-teal-500"
         to={projectSettingsPath(organization, project)}
         data-action="project-settings"
       />
@@ -558,41 +639,41 @@ function V3ProjectSideMenu({
       <SideMenuItem
         name="Tasks"
         icon={TaskIcon}
-        iconColor="text-blue-500"
+        activeIconColor="text-blue-500"
         to={v3ProjectPath(organization, project)}
         data-action="tasks"
       />
       <SideMenuItem
         name="Runs"
         icon="runs"
-        iconColor="text-teal-500"
+        activeIconColor="text-teal-500"
         to={v3RunsPath(organization, project)}
       />
       <SideMenuItem
         name="Test"
         icon={BeakerIcon}
-        iconColor="text-lime-500"
+        activeIconColor="text-lime-500"
         to={v3TestPath(organization, project)}
         data-action="test"
       />
       <SideMenuItem
         name="Schedules"
         icon={ClockIcon}
-        iconColor="text-sun-500"
+        activeIconColor="text-sun-500"
         to={v3SchedulesPath(organization, project)}
         data-action="schedules"
       />
       <SideMenuItem
         name="API keys"
         icon={KeyIcon}
-        iconColor="text-amber-500"
+        activeIconColor="text-amber-500"
         to={v3ApiKeysPath(organization, project)}
         data-action="api keys"
       />
       <SideMenuItem
         name="Environment variables"
         icon={IdentificationIcon}
-        iconColor="text-pink-500"
+        activeIconColor="text-pink-500"
         to={v3EnvironmentVariablesPath(organization, project)}
         data-action="environment variables"
       />
@@ -600,7 +681,7 @@ function V3ProjectSideMenu({
       <SideMenuItem
         name="Deployments"
         icon={ServerStackIcon}
-        iconColor="text-blue-500"
+        activeIconColor="text-blue-500"
         to={v3DeploymentsPath(organization, project)}
         data-action="deployments"
       />
@@ -608,7 +689,7 @@ function V3ProjectSideMenu({
         <SideMenuItem
           name="Alerts"
           icon={BellAlertIcon}
-          iconColor="text-red-500"
+          activeIconColor="text-red-500"
           to={v3ProjectAlertsPath(organization, project)}
           data-action="alerts"
         />
@@ -616,14 +697,14 @@ function V3ProjectSideMenu({
       <SideMenuItem
         name="Concurrency limits"
         icon={RectangleStackIcon}
-        iconColor="text-indigo-500"
+        activeIconColor="text-indigo-500"
         to={v3ConcurrencyPath(organization, project)}
         data-action="concurrency"
       />
       <SideMenuItem
         name="Project settings"
         icon="settings"
-        iconColor="text-teal-500"
+        activeIconColor="text-teal-500"
         to={v3ProjectSettingsPath(organization, project)}
         data-action="project-settings"
       />

apps/webapp/app/components/navigation/SideMenuItem.tsx

@@ -3,25 +3,26 @@ import { usePathName } from "~/hooks/usePathName";
 import { cn } from "~/utils/cn";
 import { LinkButton } from "../primitives/Buttons";
 import { type IconNames } from "../primitives/NamedIcon";
-import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "../primitives/Tooltip";
-import { Icon } from "../primitives/Icon";
-import { IconExclamationCircle } from "@tabler/icons-react";
 
 export function SideMenuItem({
   icon,
-  iconColor,
+  activeIconColor,
+  inactiveIconColor,
+  trailingIcon,
+  trailingIconClassName,
   name,
   to,
-  hasWarning,
   badge,
   target,
   subItem = false,
 }: {
   icon?: IconNames | React.ComponentType<any>;
-  iconColor?: string;
+  activeIconColor?: string;
+  inactiveIconColor?: string;
+  trailingIcon?: IconNames | React.ComponentType<any>;
+  trailingIconClassName?: string;
   name: string;
   to: string;
-  hasWarning?: string | boolean;
   badge?: string;
   target?: AnchorHTMLAttributes<HTMLAnchorElement>["target"];
   subItem?: boolean;
@@ -35,11 +36,13 @@ export function SideMenuItem({
       fullWidth
       textAlignLeft
       LeadingIcon={icon}
-      leadingIconClassName={isActive ? iconColor : "text-text-dimmed"}
+      leadingIconClassName={isActive ? activeIconColor : inactiveIconColor ?? "text-text-dimmed"}
+      TrailingIcon={trailingIcon}
+      trailingIconClassName={trailingIconClassName}
       to={to}
       target={target}
       className={cn(
-        "text-text-bright group-hover:bg-charcoal-750",
+        "text-text-bright group-hover:bg-charcoal-750 focus-visible:outline-none focus-visible:ring-0 focus-visible:ring-offset-0",
         subItem ? "text-text-dimmed" : "",
         isActive ? "bg-tertiary text-text-bright" : "group-hover:text-text-bright"
       )}
@@ -48,27 +51,13 @@ export function SideMenuItem({
         {name}
         <div className="flex items-center gap-1">
           {badge !== undefined && <MenuCount count={badge} />}
-          {typeof hasWarning === "string" ? (
-            <TooltipProvider>
-              <Tooltip>
-                <TooltipTrigger>
-                  <Icon icon={IconExclamationCircle} className="h-5 w-5 text-error" />
-                </TooltipTrigger>
-                <TooltipContent className="flex items-center gap-1 border border-error bg-error/20 backdrop-blur-xl">
-                  {hasWarning}
-                </TooltipContent>
-              </Tooltip>
-            </TooltipProvider>
-          ) : (
-            hasWarning && <Icon icon={IconExclamationCircle} className="h-5 w-5 text-error" />
-          )}
         </div>
       </div>
     </LinkButton>
   );
 }
 
-function MenuCount({ count }: { count: number | string }) {
+export function MenuCount({ count }: { count: number | string }) {
   return (
     <div className="rounded-full bg-charcoal-900 px-2 py-1 text-xxs uppercase tracking-wider text-text-dimmed">
       {count}

apps/webapp/app/components/primitives/AppliedFilter.tsx

@@ -45,7 +45,10 @@ export function AppliedFilter({
       </div>
       {removable && (
         <button
-          className={cn("group flex size-6 items-center justify-center", variantClassName.clear)}
+          className={cn(
+            "group flex size-6 items-center justify-center focus-custom",
+            variantClassName.clear
+          )}
           onClick={onRemove}
         >
           <XMarkIcon className="size-3.5" />

apps/webapp/app/components/primitives/Buttons.tsx

@@ -48,11 +48,11 @@ const theme = {
       "border-black/40 text-charcoal-900 group-hover:border-black/60 group-hover:text-charcoal-900",
   },
   secondary: {
-    textColor: "text-secondary group-hover:text-secondary transition group-disabled:text-secondary",
+    textColor: "text-text-bright transition group-disabled:text-text-dimmed/80",
     button:
-      "bg-transparent border border-secondary group-hover:border-secondary group-hover:bg-secondary/10 group-disabled:opacity-30 group-disabled:border-secondary group-disabled:bg-transparent group-disabled:pointer-events-none",
+      "bg-secondary group-hover:bg-charcoal-600 group-hover:border-charcoal-650 border border-charcoal-600 group-disabled:bg-secondary group-disabled:opacity-60 group-disabled:pointer-events-none",
     shortcut:
-      "border-secondary/30 text-secondary group-hover:text-text-bright/80 group-hover:border-dimmed/60",
+      "border-text-dimmed/40 text-text-dimmed group-hover:text-text-bright group-hover:border-text-dimmed",
   },
   tertiary: {
     textColor: "text-text-bright transition group-disabled:text-text-dimmed/80",
@@ -114,8 +114,7 @@ const variant = {
   "danger/extra-large": createVariant("extra-large", "danger"),
   "menu-item": {
     textColor: "text-text-bright px-1",
-    button:
-      "h-9 px-[0.475rem] text-sm rounded-sm bg-transparent group-hover:bg-charcoal-800 transition",
+    button: "h-9 px-[0.475rem] text-sm rounded-sm bg-transparent group-hover:bg-charcoal-750",
     icon: "h-5",
     iconSpacing: "gap-x-0.5",
     shortcutVariant: undefined,
@@ -124,7 +123,7 @@ const variant = {
   "small-menu-item": {
     textColor: "text-text-bright",
     button:
-      "h-[1.8rem] px-[0.4rem] text-2sm rounded-sm text-text-dimmed bg-transparent group-hover:bg-charcoal-850 transition",
+      "h-[1.8rem] px-[0.4rem] text-2sm rounded-sm text-text-dimmed bg-transparent group-hover:bg-charcoal-750",
     icon: "h-4",
     iconSpacing: "gap-x-1.5",
     shortcutVariant: undefined,
@@ -133,7 +132,7 @@ const variant = {
   "small-menu-sub-item": {
     textColor: "text-text-dimmed",
     button:
-      "h-[1.8rem] px-[0.5rem] ml-5 text-2sm rounded-sm text-text-dimmed bg-transparent group-hover:bg-charcoal-850 transition",
+      "h-[1.8rem] px-[0.5rem] ml-5 text-2sm rounded-sm text-text-dimmed bg-transparent group-hover:bg-charcoal-750 focus-custom",
     icon: undefined,
     iconSpacing: undefined,
     shortcutVariant: undefined,
@@ -142,7 +141,7 @@ const variant = {
 };
 
 const allVariants = {
-  $all: "font-normal text-center font-sans justify-center items-center shrink-0 transition duration-150 rounded-[3px] select-none group-focus:outline-none group-disabled:opacity-75 group-disabled:pointer-events-none",
+  $all: "font-normal text-center font-sans justify-center items-center shrink-0 transition duration-150 rounded-[3px] select-none group-focus:outline-none group-disabled:opacity-75 group-disabled:pointer-events-none focus-custom",
   variant: variant,
 };
 
@@ -268,7 +267,7 @@ export const Button = forwardRef<HTMLButtonElement, ButtonPropsType>(
 
     return (
       <button
-        className={cn("group outline-none", props.fullWidth ? "w-full" : "")}
+        className={cn("group outline-none focus-custom", props.fullWidth ? "w-full" : "")}
         type={type}
         disabled={disabled}
         onClick={onClick}
@@ -328,7 +327,7 @@ export const LinkButton = ({
       <ExtLink
         href={to.toString()}
         ref={innerRef}
-        className={cn("group outline-none", props.fullWidth ? "w-full" : "")}
+        className={cn("group focus-custom", props.fullWidth ? "w-full" : "")}
         onClick={onClick}
         onMouseDown={onMouseDown}
         onMouseEnter={onMouseEnter}
@@ -343,7 +342,7 @@ export const LinkButton = ({
       <Link
         to={to}
         ref={innerRef}
-        className={cn("group outline-none", props.fullWidth ? "w-full" : "")}
+        className={cn("group focus-custom", props.fullWidth ? "w-full" : "")}
         onClick={onClick}
         onMouseDown={onMouseDown}
         onMouseEnter={onMouseEnter}

apps/webapp/app/components/primitives/Checkbox.tsx

@@ -61,6 +61,7 @@ export type CheckboxProps = Omit<
   description?: string;
   badges?: string[];
   className?: string;
+  labelClassName?: string;
   onChange?: (isChecked: boolean) => void;
 };
 
@@ -78,6 +79,7 @@ export const CheckboxWithLabel = React.forwardRef<HTMLInputElement, CheckboxProp
       badges,
       disabled,
       className,
+      labelClassName: externalLabelClassName,
       ...props
     },
     ref
@@ -148,7 +150,8 @@ export const CheckboxWithLabel = React.forwardRef<HTMLInputElement, CheckboxProp
               htmlFor={id}
               className={cn(
                 props.readOnly || disabled ? "cursor-default" : "cursor-pointer",
-                labelClassName
+                labelClassName,
+                externalLabelClassName
               )}
               onClick={(e) => e.preventDefault()}
             >

apps/webapp/app/components/primitives/Dialog.tsx

@@ -50,7 +50,7 @@ const DialogContent = React.forwardRef<
     >
       <hr className="absolute left-0 top-11 w-full" />
       {children}
-      <DialogPrimitive.Close className="ring-offset-background data-[state=open]:bg-accent data-[state=open]:text-muted-foreground focus-visible:ring-ring absolute right-3 top-3 rounded-sm opacity-70 transition-opacity hover:opacity-100 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 disabled:pointer-events-none">
+      <DialogPrimitive.Close className="data-[state=open]:bg-accent data-[state=open]:text-muted-foreground absolute right-3 top-2 rounded-sm py-1 pr-1 opacity-70 transition-opacity focus-custom hover:opacity-100 disabled:pointer-events-none">
         <div className="flex gap-x-2">
           <ShortcutKey
             shortcut={{
@@ -82,7 +82,7 @@ DialogHeader.displayName = "DialogHeader";
 
 const DialogFooter = ({ className, ...props }: React.HTMLAttributes<HTMLDivElement>) => (
   <div
-    className={cn("flex flex-col-reverse sm:flex-row sm:justify-end sm:space-x-2", className)}
+    className={cn("flex flex-col-reverse sm:flex-row sm:justify-between sm:space-x-2", className)}
     {...props}
   />
 );

apps/webapp/app/components/primitives/InfoPanel.tsx

@@ -1,7 +1,7 @@
-import { LinkButton } from "./Buttons";
-import { Header3 } from "./Headers";
-import { Paragraph } from "./Paragraph";
 import { cn } from "~/utils/cn";
+import { LinkButton } from "./Buttons";
+import { Header2 } from "./Headers";
+import { Paragraph } from "./Paragraph";
 
 const variants = {
   info: {
@@ -57,7 +57,7 @@ export function InfoPanel({
         )}
       </div>
       <div className="flex flex-col gap-1">
-        {title && <Header3 className="text-text-bright">{title}</Header3>}
+        {title && <Header2 className="text-text-bright">{title}</Header2>}
         {typeof children === "string" ? (
           <Paragraph variant={"small"} className="text-text-dimmed">
             {children}

apps/webapp/app/components/primitives/Input.tsx

@@ -4,7 +4,7 @@ import { cn } from "~/utils/cn";
 import { Icon, RenderIcon } from "./Icon";
 
 const containerBase =
-  "has-[:focus-visible]:outline-none has-[:focus-visible]:ring-1 has-[:focus-visible]:ring-ring has-[:focus-visible]:ring-offset-0 has-[:focus]:border-ring has-[:focus]:outline-none has-[:focus]:ring-2 has-[:focus]:ring-ring has-[:disabled]:cursor-not-allowed has-[:disabled]:opacity-50 ring-offset-background transition cursor-text";
+  "has-[:focus-visible]:outline-none has-[:focus-visible]:ring-1 has-[:focus-visible]:ring-text-link has-[:focus-visible]:ring-offset-0 has-[:focus]:border-ring has-[:focus]:outline-none has-[:focus]:ring-2 has-[:focus]:ring-ring has-[:disabled]:cursor-not-allowed has-[:disabled]:opacity-50 ring-offset-background transition cursor-text";
 
 const inputBase =
   "h-full w-full text-text-bright bg-transparent file:border-0 file:bg-transparent file:text-base file:font-medium placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-0 disabled:cursor-not-allowed outline-none ring-0 border-none";

apps/webapp/app/components/primitives/PageHeader.tsx

@@ -49,7 +49,7 @@ export function PageTitle({ title, backButton }: PageTitleProps) {
         <div className="group -ml-1.5 flex items-center gap-0">
           <Link
             to={backButton.to}
-            className="rounded px-1.5 py-1 text-xs text-text-dimmed transition group-hover:bg-charcoal-700 group-hover:text-text-bright"
+            className="rounded px-1.5 py-1 text-xs text-text-dimmed transition focus-custom group-hover:bg-charcoal-700 group-hover:text-text-bright"
           >
             {backButton.text}
           </Link>

apps/webapp/app/components/primitives/Popover.tsx

@@ -6,6 +6,8 @@ import * as React from "react";
 import { cn } from "~/utils/cn";
 import { type ButtonContentPropsType, LinkButton } from "./Buttons";
 import { Paragraph, type ParagraphVariant } from "./Paragraph";
+import { ShortcutKey } from "./ShortcutKey";
+import { ShortcutDefinition, useShortcutKeys } from "~/hooks/useShortcutKeys";
 
 const Popover = PopoverPrimitive.Root;
 const PopoverTrigger = PopoverPrimitive.Trigger;
@@ -91,7 +93,7 @@ function PopoverCustomTrigger({
     <PopoverTrigger
       {...props}
       className={cn(
-        "group flex items-center justify-end gap-1 rounded text-text-dimmed transition hover:bg-charcoal-850 hover:text-text-bright",
+        "group flex items-center justify-end gap-1 rounded text-text-dimmed transition focus-custom hover:bg-charcoal-850 hover:text-text-bright",
         className
       )}
     >
@@ -100,6 +102,45 @@ function PopoverCustomTrigger({
   );
 }
 
+function PopoverSideMenuTrigger({
+  isOpen,
+  children,
+  className,
+  shortcut,
+  ...props
+}: { isOpen?: boolean; shortcut?: ShortcutDefinition } & React.ComponentPropsWithoutRef<
+  typeof PopoverTrigger
+>) {
+  const ref = React.useRef<HTMLButtonElement>(null);
+  useShortcutKeys({
+    shortcut: shortcut,
+    action: (e) => {
+      e.preventDefault();
+      e.stopPropagation();
+      if (ref.current) {
+        ref.current.click();
+      }
+    },
+  });
+
+  return (
+    <PopoverTrigger
+      {...props}
+      ref={ref}
+      className={cn(
+        "flex h-[1.8rem] shrink-0 select-none items-center gap-x-1.5 rounded-sm bg-transparent px-[0.4rem] text-center font-sans text-2sm font-normal text-text-bright transition duration-150 focus-custom hover:bg-charcoal-750",
+        shortcut ? "justify-between" : "",
+        className
+      )}
+    >
+      {children}
+      {shortcut && (
+        <ShortcutKey className={cn("size-4 flex-none")} shortcut={shortcut} variant={"small"} />
+      )}
+    </PopoverTrigger>
+  );
+}
+
 function PopoverArrowTrigger({
   isOpen,
   children,
@@ -116,7 +157,7 @@ function PopoverArrowTrigger({
     <PopoverTrigger
       {...props}
       className={cn(
-        "group flex h-6 items-center gap-1 rounded px-2 text-text-dimmed transition hover:bg-charcoal-700 hover:text-text-bright",
+        "group flex h-6 items-center gap-1 rounded px-2 text-text-dimmed transition focus-custom hover:bg-charcoal-700 hover:text-text-bright",
         fullWidth && "w-full justify-between",
         className
       )}
@@ -149,7 +190,7 @@ function PopoverVerticalEllipseTrigger({
     <PopoverTrigger
       {...props}
       className={cn(
-        "group flex items-center justify-end gap-1 rounded px-1.5 py-1.5 text-text-dimmed transition hover:bg-charcoal-750 hover:text-text-bright",
+        "group flex items-center justify-end gap-1 rounded px-1.5 py-1.5 text-text-dimmed transition focus-custom hover:bg-charcoal-750 hover:text-text-bright",
         className
       )}
     >
@@ -165,6 +206,7 @@ export {
   PopoverCustomTrigger,
   PopoverMenuItem,
   PopoverSectionHeader,
+  PopoverSideMenuTrigger,
   PopoverTrigger,
   PopoverVerticalEllipseTrigger,
 };

apps/webapp/app/components/primitives/RadioButton.tsx

@@ -70,7 +70,7 @@ export function RadioButtonCircle({
   return (
     <div
       className={cn(
-        "ring-offset-background focus-visible:ring-ring aspect-square h-4 w-4 shrink-0 overflow-hidden rounded-full border border-charcoal-600 focus:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50",
+        "ring-offset-background aspect-square h-4 w-4 shrink-0 overflow-hidden rounded-full border border-charcoal-600 focus-custom disabled:cursor-not-allowed disabled:opacity-50",
         boxClassName
       )}
     >
@@ -81,7 +81,9 @@ export function RadioButtonCircle({
             outerCircleClassName
           )}
         >
-          <Circle className={cn("h-1.5 w-1.5 fill-white text-white", innerCircleClassName)} />
+          <Circle
+            className={cn("size-1.5 fill-text-bright text-text-bright", innerCircleClassName)}
+          />
         </div>
       )}
     </div>
@@ -121,7 +123,7 @@ export const RadioGroupItem = React.forwardRef<
       <RadioGroupPrimitive.Item
         ref={ref}
         className={cn(
-          "group flex cursor-pointer items-start gap-x-2 transition",
+          "group flex cursor-pointer items-start gap-x-2 transition focus-custom",
           variation.button,
           className
         )}
@@ -129,7 +131,7 @@ export const RadioGroupItem = React.forwardRef<
       >
         <div
           className={cn(
-            "ring-offset-background focus-visible:ring-ring aspect-square h-4 w-4 shrink-0 overflow-hidden rounded-full border border-charcoal-600 focus:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50",
+            "ring-offset-background focus-visible:ring-ring aspect-square h-4 w-4 shrink-0 overflow-hidden rounded-full border border-charcoal-600 focus-custom disabled:cursor-not-allowed disabled:opacity-50",
             variation.inputPosition
           )}
         >

apps/webapp/app/components/primitives/Resizable.tsx

@@ -26,7 +26,7 @@ const ResizableHandle = ({
 }) => (
   <PanelResizer
     className={cn(
-      "focus-visible:ring-ring group relative flex w-0.75 items-center justify-center after:absolute after:inset-y-0 after:left-1/2 after:w-1 after:-translate-x-1/2 data-[panel-group-direction=vertical]:h-px data-[panel-group-direction=vertical]:w-full data-[panel-group-direction=vertical]:after:left-0 data-[panel-group-direction=vertical]:after:h-1 data-[panel-group-direction=vertical]:after:w-full data-[panel-group-direction=vertical]:after:-translate-y-1/2 data-[panel-group-direction=vertical]:after:translate-x-0 hover:w-0.75 focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-secondary/50 focus-visible:ring-offset-0 [&[data-panel-group-direction=vertical]>div]:rotate-90",
+      "group relative flex w-0.75 items-center justify-center focus-custom after:absolute after:inset-y-0 after:left-1/2 after:w-1 after:-translate-x-1/2 data-[panel-group-direction=vertical]:h-px data-[panel-group-direction=vertical]:w-full data-[panel-group-direction=vertical]:after:left-0 data-[panel-group-direction=vertical]:after:h-1 data-[panel-group-direction=vertical]:after:w-full data-[panel-group-direction=vertical]:after:-translate-y-1/2 data-[panel-group-direction=vertical]:after:translate-x-0 hover:w-0.75 [&[data-panel-group-direction=vertical]>div]:rotate-90",
       className
     )}
     size="3px"

apps/webapp/app/components/primitives/SegmentedControl.tsx

@@ -65,10 +65,7 @@ export default function SegmentedControl({
               value={option.value}
               className={({ active, checked }) =>
                 cn(
-                  "relative flex h-full grow cursor-pointer text-center font-normal focus:outline-none",
-                  active
-                    ? "ring-offset-2 focus-visible:ring focus-visible:ring-secondary focus-visible:ring-opacity-60"
-                    : "",
+                  "relative flex h-full grow cursor-pointer text-center font-normal focus-custom",
                   checked
                     ? variants[variant].active
                     : "text-text-dimmed transition hover:text-text-bright"

apps/webapp/app/components/primitives/Select.tsx

@@ -15,18 +15,18 @@ const sizes = {
     button: "h-6 rounded text-xs px-2 ",
   },
   medium: {
-    button: "h-8 rounded text-xs px-3 text-sm",
+    button: "h-8 rounded px-3 text-sm",
   },
 };
 
 const style = {
   tertiary: {
     button:
-      "bg-tertiary focus-within:ring-charcoal-500 border border-tertiary hover:text-text-bright hover:border-charcoal-600",
+      "bg-tertiary focus-custom border border-tertiary hover:text-text-bright hover:border-charcoal-600",
   },
   minimal: {
     button:
-      "bg-transparent focus-within:ring-charcoal-500 hover:bg-tertiary disabled:bg-transparent disabled:pointer-events-none",
+      "bg-transparent focus-custom hover:bg-tertiary disabled:bg-transparent disabled:pointer-events-none",
   },
 };
 
@@ -322,7 +322,7 @@ export function SelectTrigger({
         render={
           <Ariakit.Select
             className={cn(
-              "group flex items-center gap-1 outline-offset-0 focus-within:outline-none focus-within:ring-1 disabled:cursor-not-allowed disabled:opacity-50",
+              "group flex items-center gap-1 focus-custom disabled:cursor-not-allowed disabled:opacity-50",
               variantClasses.button,
               className
             )}
@@ -426,7 +426,7 @@ export function SelectList(props: SelectListProps) {
     <Component
       {...props}
       className={cn(
-        "overflow-y-auto overscroll-contain outline-none scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600",
+        "overflow-y-auto overscroll-contain scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600 focus-custom",
         props.className
       )}
     />
@@ -440,11 +440,11 @@ export interface SelectItemProps extends Ariakit.SelectItemProps {
 }
 
 const selectItemClasses =
-  "group cursor-pointer px-1 pt-1 text-xs text-text-dimmed outline-none last:pb-1";
+  "group cursor-pointer px-1 pt-1 text-sm text-text-dimmed focus-custom last:pb-1";
 
 export function SelectItem({
   icon,
-  checkIcon = <Ariakit.SelectItemCheck className="size-8 flex-none text-white" />,
+  checkIcon = <Ariakit.SelectItemCheck className="size-8 flex-none text-text-bright" />,
   shortcut,
   ...props
 }: SelectItemProps) {
@@ -477,7 +477,7 @@ export function SelectItem({
       )}
       ref={ref}
     >
-      <div className="flex h-7 w-full items-center gap-1 rounded-sm px-2 group-data-[active-item=true]:bg-tertiary">
+      <div className="flex h-8 w-full items-center gap-1 rounded-sm px-2 group-data-[active-item=true]:bg-tertiary">
         {icon}
         <div className="grow truncate">{props.children || props.value}</div>
         {checkIcon}

apps/webapp/app/components/primitives/Switch.tsx

@@ -6,14 +6,14 @@ import { cn } from "~/utils/cn";
 
 const variations = {
   large: {
-    container: "flex items-center gap-x-2 rounded-md hover:bg-tertiary p-2 transition",
+    container: "flex items-center gap-x-2 rounded-md hover:bg-tertiary p-2 transition focus-custom",
     root: "h-6 w-11",
     thumb: "h-5 w-5 data-[state=checked]:translate-x-5 data-[state=unchecked]:translate-x-0",
     text: "text-sm text-charcoal-400 group-hover:text-charcoal-200 transition",
   },
   small: {
     container:
-      "flex items-center gap-x-1.5 rounded hover:bg-tertiary pr-1 py-[0.1rem] pl-1.5 transition",
+      "flex items-center h-[1.5rem] gap-x-1.5 rounded hover:bg-tertiary pr-1 py-[0.1rem] pl-1.5 transition focus-custom",
     root: "h-3 w-6",
     thumb: "h-2.5 w-2.5 data-[state=checked]:translate-x-2.5 data-[state=unchecked]:translate-x-0",
     text: "text-xs text-charcoal-400 group-hover:text-charcoal-200 hover:cursor-pointer transition",
@@ -38,14 +38,14 @@ export const Switch = React.forwardRef<React.ElementRef<typeof SwitchPrimitives.
         ) : null}
         <div
           className={cn(
-            "group-focus-visible:ring-ring group-focus-visible:ring-offset-background peer inline-flex shrink-0 cursor-pointer items-center rounded-full border-2 border-transparent transition-colors group-focus-visible:ring-2 group-focus-visible:ring-offset-2 group-disabled:cursor-not-allowed group-disabled:opacity-50 group-data-[state=checked]:bg-secondary group-data-[state=unchecked]:bg-charcoal-700 focus-visible:outline-none",
+            "inline-flex shrink-0 cursor-pointer items-center rounded-full border-2 border-transparent transition-colors group-disabled:cursor-not-allowed group-disabled:opacity-50 group-data-[state=checked]:bg-text-link group-data-[state=unchecked]:bg-charcoal-700",
             root
           )}
         >
           <SwitchPrimitives.Thumb
             className={cn(
               thumb,
-              "pointer-events-none block rounded-full bg-charcoal-200 shadow-lg ring-0 transition group-data-[state=checked]:bg-charcoal-700"
+              "pointer-events-none block rounded-full bg-charcoal-200 transition group-data-[state=checked]:bg-text-bright"
             )}
           />
         </div>

apps/webapp/app/components/primitives/Table.tsx

@@ -151,7 +151,7 @@ const rowHoverStyles = {
 };
 
 const stickyStyles =
-  "sticky right-0 z-10 w-[2.8rem] min-w-[2.8rem] bg-background-dimmed before:absolute before:pointer-events-none before:-left-8 before:top-0 before:h-full before:min-w-[2rem]";
+  "sticky right-0 w-[2.8rem] min-w-[2.8rem] bg-background-dimmed before:absolute before:pointer-events-none before:-left-8 before:top-0 before:h-full before:min-w-[2rem]";
 
 export const TableCell = forwardRef<HTMLTableCellElement, TableCellProps>(
   (
@@ -202,11 +202,11 @@ export const TableCell = forwardRef<HTMLTableCellElement, TableCellProps>(
         colSpan={colSpan}
       >
         {to ? (
-          <Link to={to} className={cn(flexClasses, actionClassName)}>
+          <Link to={to} className={cn("focus-custom", flexClasses, actionClassName)}>
             {children}
           </Link>
         ) : onClick ? (
-          <button onClick={onClick} className={cn(flexClasses, actionClassName)}>
+          <button onClick={onClick} className={cn("focus-custom", flexClasses, actionClassName)}>
             {children}
           </button>
         ) : (

apps/webapp/app/components/primitives/Tabs.tsx

@@ -46,7 +46,7 @@ export function TabLink({
   layoutId: string;
 }) {
   return (
-    <NavLink to={to} className="group flex flex-col items-center pt-1" end>
+    <NavLink to={to} className="group flex flex-col items-center pt-1 focus-custom" end>
       {({ isActive, isPending }) => {
         return (
           <>
@@ -96,7 +96,7 @@ export function TabButton({
 
   return (
     <button
-      className={cn("group flex flex-col items-center pt-1", props.className)}
+      className={cn("group flex flex-col items-center pt-1 focus-custom", props.className)}
       type="button"
       ref={ref}
       {...props}

apps/webapp/app/components/primitives/TextArea.tsx

@@ -8,7 +8,7 @@ export function TextArea({ className, rows, ...props }: TextAreaProps) {
       {...props}
       rows={rows ?? 6}
       className={cn(
-        "ring-offset-background placeholder:text-muted-foreground focus:border-ring focus:ring-ring focus-visible:ring-ring w-full rounded-md border border-tertiary bg-tertiary px-3 text-sm text-text-bright transition file:border-0 file:bg-transparent file:text-base file:font-medium hover:border-charcoal-600 focus:outline-none focus:ring-2 focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-offset-0 disabled:cursor-not-allowed disabled:opacity-50",
+        "placeholder:text-muted-foreground w-full rounded border border-charcoal-800 bg-charcoal-750 px-3 text-sm text-text-bright transition focus-custom focus-custom file:border-0 file:bg-transparent file:text-base file:font-medium hover:border-charcoal-600 hover:bg-charcoal-650 disabled:cursor-not-allowed disabled:opacity-50",
         className
       )}
     />

apps/webapp/app/components/primitives/Tooltip.tsx

@@ -41,7 +41,7 @@ const TooltipContent = React.forwardRef<
     ref={ref}
     sideOffset={sideOffset}
     className={cn(
-      "z-50 overflow-hidden animate-in data-[side=bottom]:slide-in-from-top-1 data-[side=left]:slide-in-from-right-1 data-[side=right]:slide-in-from-left-1 data-[side=top]:slide-in-from-bottom-1",
+      "z-50 overflow-hidden animate-in data-[side=bottom]:slide-in-from-top-1 data-[side=left]:slide-in-from-right-1 data-[side=right]:slide-in-from-left-1 data-[side=top]:slide-in-from-bottom-1 focus-visible:outline-none",
       variantClasses[variant],
       className
     )}
@@ -72,7 +72,9 @@ function SimpleTooltip({
   return (
     <TooltipProvider disableHoverableContent={disableHoverableContent}>
       <Tooltip>
-        <TooltipTrigger className={cn("h-fit", buttonClassName)}>{button}</TooltipTrigger>
+        <TooltipTrigger tabIndex={-1} className={cn("h-fit", buttonClassName)}>
+          {button}
+        </TooltipTrigger>
         <TooltipContent
           side={side}
           hidden={hidden}

apps/webapp/app/components/runs/v3/RunFilters.tsx

@@ -368,7 +368,7 @@ function AppliedStatusFilter() {
       {(search, setSearch) => (
         <StatusDropdown
           trigger={
-            <Ariakit.Select render={<div className="group cursor-pointer" />}>
+            <Ariakit.Select render={<div className="group cursor-pointer focus-custom" />}>
               <AppliedFilter
                 label="Status"
                 value={appliedSummary(statuses.map((v) => runStatusTitle(v as TaskRunStatus)))}
@@ -456,7 +456,7 @@ function AppliedEnvironmentFilter({
       {(search, setSearch) => (
         <EnvironmentsDropdown
           trigger={
-            <Ariakit.Select render={<div className="group cursor-pointer" />}>
+            <Ariakit.Select render={<div className="group cursor-pointer focus-custom" />}>
               <AppliedFilter
                 label="Environment"
                 value={appliedSummary(
@@ -547,7 +547,7 @@ function AppliedTaskFilter({ possibleTasks }: Pick<RunFiltersProps, "possibleTas
       {(search, setSearch) => (
         <TasksDropdown
           trigger={
-            <Ariakit.Select render={<div className="group cursor-pointer" />}>
+            <Ariakit.Select render={<div className="group cursor-pointer focus-custom" />}>
               <AppliedFilter
                 label="Task"
                 value={appliedSummary(
@@ -645,7 +645,7 @@ function AppliedBulkActionsFilter({ bulkActions }: Pick<RunFiltersProps, "bulkAc
       {(search, setSearch) => (
         <BulkActionsDropdown
           trigger={
-            <Ariakit.Select render={<div className="group cursor-pointer" />}>
+            <Ariakit.Select render={<div className="group cursor-pointer focus-custom" />}>
               <AppliedFilter
                 label="Bulk action"
                 value={bulkId}
@@ -764,7 +764,7 @@ function AppliedTagsFilter() {
       {(search, setSearch) => (
         <TagsDropdown
           trigger={
-            <Ariakit.Select render={<div className="group cursor-pointer" />}>
+            <Ariakit.Select render={<div className="group cursor-pointer focus-custom" />}>
               <AppliedFilter
                 label="Tags"
                 value={appliedSummary(values("tags"))}
@@ -902,7 +902,7 @@ function AppliedPeriodFilter() {
       {(search, setSearch) => (
         <CreatedDropdown
           trigger={
-            <Ariakit.Select render={<div className="group cursor-pointer" />}>
+            <Ariakit.Select render={<div className="group cursor-pointer focus-custom" />}>
               <AppliedFilter
                 label="Created"
                 value={

apps/webapp/app/components/runs/v3/RunInspector.tsx

@@ -40,7 +40,7 @@ import {
 } from "~/utils/pathBuilder";
 import { TraceSpan } from "~/utils/taskEvent";
 import { SpanLink } from "~/v3/eventRepository.server";
-import { isFinalRunStatus } from "~/v3/taskStatus";
+import { isFailedRunStatus, isFinalRunStatus } from "~/v3/taskStatus";
 import { RunTimelineEvent, RunTimelineLine } from "./InspectorTimeline";
 import { RunTag } from "./RunTag";
 import { TaskRunStatusCombo } from "./TaskRunStatus";
@@ -75,7 +75,7 @@ export function RunInspector({
   if (!run) {
     return (
       <div className="grid h-full max-h-full grid-rows-[2.5rem_1fr] overflow-hidden bg-background-bright">
-        <div className="mx-3 flex items-center justify-between gap-2 overflow-x-hidden">
+        <div className="flex items-center justify-between gap-2 overflow-x-hidden px-3">
           <div className="flex items-center gap-1 overflow-x-hidden">
             <RunIcon name={"task"} spanName="" className="h-4 min-h-4 w-4 min-w-4" />
             <Header2 className={cn("overflow-x-hidden text-blue-500")}>
@@ -101,7 +101,7 @@ export function RunInspector({
 
   return (
     <div className="grid h-full max-h-full grid-rows-[2.5rem_2rem_1fr_3.25rem] overflow-hidden bg-background-bright">
-      <div className="mx-3 flex items-center justify-between gap-2 overflow-x-hidden">
+      <div className="flex items-center justify-between gap-2 overflow-x-hidden px-3">
         <div className="flex items-center gap-1 overflow-x-hidden">
           <RunIcon
             name={"task"}
@@ -479,6 +479,7 @@ function RunTimeline({ run }: { run: RawRun }) {
   const updatedAt = new Date(run.updatedAt);
 
   const isFinished = isFinalRunStatus(run.status);
+  const isError = isFailedRunStatus(run.status);
 
   return (
     <div className="min-w-fit max-w-80">
@@ -535,7 +536,7 @@ function RunTimeline({ run }: { run: RawRun }) {
               <RunTimelineEvent
                 title="Finished"
                 subtitle={<DateTimeAccurate date={updatedAt} />}
-                state="complete"
+                state={isError ? "error" : "complete"}
               />
             </>
           ) : (

apps/webapp/app/components/runs/v3/SpanEvents.tsx

@@ -1,3 +1,4 @@
+import { EnvelopeIcon } from "@heroicons/react/20/solid";
 import {
   exceptionEventEnhancer,
   isExceptionSpanEvent,
@@ -5,6 +6,8 @@ import {
   type SpanEvent as OtelSpanEvent,
 } from "@trigger.dev/core/v3";
 import { CodeBlock } from "~/components/code/CodeBlock";
+import { Feedback } from "~/components/Feedback";
+import { Button } from "~/components/primitives/Buttons";
 import { Callout } from "~/components/primitives/Callout";
 import { DateTimeAccurate } from "~/components/primitives/DateTime";
 import { Header2, Header3 } from "~/components/primitives/Headers";
@@ -75,11 +78,26 @@ export function SpanEventError({
         titleClassName="text-rose-500"
       />
       {enhancedException.message && <Callout variant="error">{enhancedException.message}</Callout>}
-      {enhancedException.link && (
-        <Callout variant="docs" to={enhancedException.link.href}>
-          {enhancedException.link.name}
-        </Callout>
-      )}
+      {enhancedException.link &&
+        (enhancedException.link.magic === "CONTACT_FORM" ? (
+          <Feedback
+            button={
+              <Button
+                variant="tertiary/medium"
+                LeadingIcon={EnvelopeIcon}
+                leadingIconClassName="text-blue-400"
+                fullWidth
+                textAlignLeft
+              >
+                {enhancedException.link.name}
+              </Button>
+            }
+          />
+        ) : (
+          <Callout variant="docs" to={enhancedException.link.href}>
+            {enhancedException.link.name}
+          </Callout>
+        ))}
       {enhancedException.stacktrace && (
         <CodeBlock
           showCopyButton={false}

apps/webapp/app/components/runs/v3/SpanInspector.tsx

@@ -47,7 +47,7 @@ export function SpanInspector({
 
   return (
     <div className="grid h-full max-h-full grid-rows-[2.5rem_2rem_1fr] overflow-hidden bg-background-bright">
-      <div className="mx-3 flex items-center justify-between gap-2 overflow-x-hidden">
+      <div className="flex items-center justify-between gap-2 overflow-x-hidden px-3">
         <div className="flex items-center gap-1 overflow-x-hidden">
           <RunIcon
             name={span.style?.icon}

apps/webapp/app/env.server.ts

@@ -31,7 +31,8 @@ const EnvironmentSchema = z.object({
   REMIX_APP_PORT: z.string().optional(),
   LOGIN_ORIGIN: z.string().default("http://localhost:3030"),
   APP_ORIGIN: z.string().default("http://localhost:3030"),
-  ELECTRIC_ORIGIN: z.string(),
+  API_ORIGIN: z.string().optional(),
+  ELECTRIC_ORIGIN: z.string().default("http://localhost:3060"),
   APP_ENV: z.string().default(process.env.NODE_ENV),
   SERVICE_NAME: z.string().default("trigger.dev webapp"),
   SECRET_STORE: SecretStoreOptionsSchema.default("DATABASE"),
@@ -103,6 +104,25 @@ const EnvironmentSchema = z.object({
   API_RATE_LIMIT_REFILL_RATE: z.coerce.number().int().default(250), // refix 250 tokens every 10 seconds
   API_RATE_LIMIT_REQUEST_LOGS_ENABLED: z.string().default("0"),
   API_RATE_LIMIT_REJECTION_LOGS_ENABLED: z.string().default("1"),
+  API_RATE_LIMIT_LIMITER_LOGS_ENABLED: z.string().default("0"),
+
+  API_RATE_LIMIT_JWT_WINDOW: z.string().default("1m"),
+  API_RATE_LIMIT_JWT_TOKENS: z.coerce.number().int().default(60),
+
+  //Realtime rate limiting
+  /**
+   * @example "60s"
+   * @example "1m"
+   * @example "1h"
+   * @example "1d"
+   * @example "1000ms"
+   * @example "1000s"
+   */
+  REALTIME_RATE_LIMIT_WINDOW: z.string().default("1m"),
+  REALTIME_RATE_LIMIT_TOKENS: z.coerce.number().int().default(100),
+  REALTIME_RATE_LIMIT_REQUEST_LOGS_ENABLED: z.string().default("0"),
+  REALTIME_RATE_LIMIT_REJECTION_LOGS_ENABLED: z.string().default("1"),
+  REALTIME_RATE_LIMIT_LIMITER_LOGS_ENABLED: z.string().default("0"),
 
   //Ingesting event rate limit
   INGEST_EVENT_RATE_LIMIT_WINDOW: z.string().default("60s"),

apps/webapp/app/models/taskRun.server.ts

@@ -3,7 +3,7 @@ import type {
   TaskRunFailedExecutionResult,
   TaskRunSuccessfulExecutionResult,
 } from "@trigger.dev/core/v3";
-import { TaskRunError } from "@trigger.dev/core/v3";
+import { TaskRunError, TaskRunErrorCodes } from "@trigger.dev/core/v3";
 
 import type {
   TaskRun,
@@ -62,7 +62,7 @@ export function executionResultForTaskRun(
         id: taskRun.friendlyId,
         error: {
           type: "INTERNAL_ERROR",
-          code: "TASK_RUN_CANCELLED",
+          code: TaskRunErrorCodes.TASK_RUN_CANCELLED,
         },
       } satisfies TaskRunFailedExecutionResult;
     }
@@ -94,7 +94,7 @@ export function executionResultForTaskRun(
         id: taskRun.friendlyId,
         error: {
           type: "INTERNAL_ERROR",
-          code: "CONFIGURED_INCORRECTLY",
+          code: TaskRunErrorCodes.CONFIGURED_INCORRECTLY,
         },
       } satisfies TaskRunFailedExecutionResult;
     }

apps/webapp/app/models/taskRunTag.server.ts

@@ -1,7 +1,7 @@
 import { prisma } from "~/db.server";
 import { generateFriendlyId } from "~/v3/friendlyIdentifiers";
 
-export const MAX_TAGS_PER_RUN = 5;
+export const MAX_TAGS_PER_RUN = 10;
 
 export async function createTag({ tag, projectId }: { tag: string; projectId: string }) {
   if (tag.trim().length === 0) return;

apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts

@@ -62,8 +62,7 @@ type CommonRelatedRun = Prisma.Result<
 export class ApiRetrieveRunPresenter extends BasePresenter {
   public async call(
     friendlyId: string,
-    env: AuthenticatedEnvironment,
-    showSecretDetails: boolean
+    env: AuthenticatedEnvironment
   ): Promise<RetrieveRunResponse | undefined> {
     return this.traceWithEnv("call", env, async (span) => {
       const taskRun = await this._replica.taskRun.findFirst({
@@ -72,11 +71,7 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
           runtimeEnvironmentId: env.id,
         },
         include: {
-          attempts: {
-            orderBy: {
-              createdAt: "desc",
-            },
-          },
+          attempts: true,
           lockedToVersion: true,
           schedule: true,
           tags: true,
@@ -111,50 +106,48 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
       let $output: any;
       let $outputPresignedUrl: string | undefined;
 
-      if (showSecretDetails) {
-        const payloadPacket = await conditionallyImportPacket({
-          data: taskRun.payload,
-          dataType: taskRun.payloadType,
-        });
+      const payloadPacket = await conditionallyImportPacket({
+        data: taskRun.payload,
+        dataType: taskRun.payloadType,
+      });
 
-        if (
-          payloadPacket.dataType === "application/store" &&
-          typeof payloadPacket.data === "string"
-        ) {
-          $payloadPresignedUrl = await generatePresignedUrl(
-            env.project.externalRef,
-            env.slug,
-            payloadPacket.data,
-            "GET"
-          );
-        } else {
-          $payload = await parsePacket(payloadPacket);
-        }
+      if (
+        payloadPacket.dataType === "application/store" &&
+        typeof payloadPacket.data === "string"
+      ) {
+        $payloadPresignedUrl = await generatePresignedUrl(
+          env.project.externalRef,
+          env.slug,
+          payloadPacket.data,
+          "GET"
+        );
+      } else {
+        $payload = await parsePacket(payloadPacket);
+      }
 
-        if (taskRun.status === "COMPLETED_SUCCESSFULLY") {
-          const completedAttempt = taskRun.attempts.find(
-            (a) => a.status === "COMPLETED" && typeof a.output !== null
-          );
+      if (taskRun.status === "COMPLETED_SUCCESSFULLY") {
+        const completedAttempt = taskRun.attempts.find(
+          (a) => a.status === "COMPLETED" && typeof a.output !== null
+        );
 
-          if (completedAttempt && completedAttempt.output) {
-            const outputPacket = await conditionallyImportPacket({
-              data: completedAttempt.output,
-              dataType: completedAttempt.outputType,
-            });
+        if (completedAttempt && completedAttempt.output) {
+          const outputPacket = await conditionallyImportPacket({
+            data: completedAttempt.output,
+            dataType: completedAttempt.outputType,
+          });
 
-            if (
-              outputPacket.dataType === "application/store" &&
-              typeof outputPacket.data === "string"
-            ) {
-              $outputPresignedUrl = await generatePresignedUrl(
-                env.project.externalRef,
-                env.slug,
-                outputPacket.data,
-                "GET"
-              );
-            } else {
-              $output = await parsePacket(outputPacket);
-            }
+          if (
+            outputPacket.dataType === "application/store" &&
+            typeof outputPacket.data === "string"
+          ) {
+            $outputPresignedUrl = await generatePresignedUrl(
+              env.project.externalRef,
+              env.slug,
+              outputPacket.data,
+              "GET"
+            );
+          } else {
+            $output = await parsePacket(outputPacket);
           }
         }
       }
@@ -165,6 +158,7 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
         payloadPresignedUrl: $payloadPresignedUrl,
         output: $output,
         outputPresignedUrl: $outputPresignedUrl,
+        error: ApiRetrieveRunPresenter.apiErrorFromError(taskRun.error),
         schedule: taskRun.schedule
           ? {
               id: taskRun.schedule.friendlyId,
@@ -179,17 +173,9 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
               },
             }
           : undefined,
-        attempts: !showSecretDetails
-          ? []
-          : taskRun.attempts.map((a) => ({
-              id: a.friendlyId,
-              status: ApiRetrieveRunPresenter.apiStatusFromAttemptStatus(a.status),
-              createdAt: a.createdAt ?? undefined,
-              updatedAt: a.updatedAt ?? undefined,
-              startedAt: a.startedAt ?? undefined,
-              completedAt: a.completedAt ?? undefined,
-              error: ApiRetrieveRunPresenter.apiErrorFromError(a.error),
-            })),
+        // We're removing attempts from the API
+        attemptCount: taskRun.attempts.length,
+        attempts: [],
         relatedRuns: {
           root: taskRun.rootTaskRun
             ? await createCommonRunStructure(taskRun.rootTaskRun)

apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts

@@ -29,7 +29,7 @@ const CoercedDate = z.preprocess((arg) => {
   return arg;
 }, z.date().optional());
 
-const SearchParamsSchema = z.object({
+export const ApiRunListSearchParams = z.object({
   "page[size]": z.coerce.number().int().positive().min(1).max(100).optional(),
   "page[after]": z.string().optional(),
   "page[before]": z.string().optional(),
@@ -121,45 +121,31 @@ const SearchParamsSchema = z.object({
   "filter[createdAt][period]": z.string().optional(),
 });
 
-type SearchParamsSchema = z.infer<typeof SearchParamsSchema>;
+type ApiRunListSearchParams = z.infer<typeof ApiRunListSearchParams>;
 
 export class ApiRunListPresenter extends BasePresenter {
   public async call(
     project: Project,
-    searchParams: URLSearchParams,
+    searchParams: ApiRunListSearchParams,
     environment?: RuntimeEnvironment
   ): Promise<ListRunResponse> {
     return this.trace("call", async (span) => {
-      const rawSearchParams = Object.fromEntries(searchParams.entries());
-      const $searchParams = SearchParamsSchema.safeParse(rawSearchParams);
-
-      if (!$searchParams.success) {
-        logger.error("Invalid search params", {
-          searchParams: rawSearchParams,
-          errors: $searchParams.error.errors,
-        });
-
-        throw fromZodError($searchParams.error);
-      }
-
-      logger.debug("Valid search params", { searchParams: $searchParams.data });
-
       const options: RunListOptions = {
         projectId: project.id,
       };
 
       // pagination
-      if ($searchParams.data["page[size]"]) {
-        options.pageSize = $searchParams.data["page[size]"];
+      if (searchParams["page[size]"]) {
+        options.pageSize = searchParams["page[size]"];
       }
 
-      if ($searchParams.data["page[after]"]) {
-        options.cursor = $searchParams.data["page[after]"];
+      if (searchParams["page[after]"]) {
+        options.cursor = searchParams["page[after]"];
         options.direction = "forward";
       }
 
-      if ($searchParams.data["page[before]"]) {
-        options.cursor = $searchParams.data["page[before]"];
+      if (searchParams["page[before]"]) {
+        options.cursor = searchParams["page[before]"];
         options.direction = "backward";
       }
 
@@ -167,12 +153,12 @@ export class ApiRunListPresenter extends BasePresenter {
       if (environment) {
         options.environments = [environment.id];
       } else {
-        if ($searchParams.data["filter[env]"]) {
+        if (searchParams["filter[env]"]) {
           const environments = await this._prisma.runtimeEnvironment.findMany({
             where: {
               projectId: project.id,
               slug: {
-                in: $searchParams.data["filter[env]"],
+                in: searchParams["filter[env]"],
               },
             },
           });
@@ -181,46 +167,46 @@ export class ApiRunListPresenter extends BasePresenter {
         }
       }
 
-      if ($searchParams.data["filter[status]"]) {
-        options.statuses = $searchParams.data["filter[status]"].flatMap((status) =>
+      if (searchParams["filter[status]"]) {
+        options.statuses = searchParams["filter[status]"].flatMap((status) =>
           ApiRunListPresenter.apiStatusToRunStatuses(status)
         );
       }
 
-      if ($searchParams.data["filter[taskIdentifier]"]) {
-        options.tasks = $searchParams.data["filter[taskIdentifier]"];
+      if (searchParams["filter[taskIdentifier]"]) {
+        options.tasks = searchParams["filter[taskIdentifier]"];
       }
 
-      if ($searchParams.data["filter[version]"]) {
-        options.versions = $searchParams.data["filter[version]"];
+      if (searchParams["filter[version]"]) {
+        options.versions = searchParams["filter[version]"];
       }
 
-      if ($searchParams.data["filter[tag]"]) {
-        options.tags = $searchParams.data["filter[tag]"];
+      if (searchParams["filter[tag]"]) {
+        options.tags = searchParams["filter[tag]"];
       }
 
-      if ($searchParams.data["filter[bulkAction]"]) {
-        options.bulkId = $searchParams.data["filter[bulkAction]"];
+      if (searchParams["filter[bulkAction]"]) {
+        options.bulkId = searchParams["filter[bulkAction]"];
       }
 
-      if ($searchParams.data["filter[schedule]"]) {
-        options.scheduleId = $searchParams.data["filter[schedule]"];
+      if (searchParams["filter[schedule]"]) {
+        options.scheduleId = searchParams["filter[schedule]"];
       }
 
-      if ($searchParams.data["filter[createdAt][from]"]) {
-        options.from = $searchParams.data["filter[createdAt][from]"].getTime();
+      if (searchParams["filter[createdAt][from]"]) {
+        options.from = searchParams["filter[createdAt][from]"].getTime();
       }
 
-      if ($searchParams.data["filter[createdAt][to]"]) {
-        options.to = $searchParams.data["filter[createdAt][to]"].getTime();
+      if (searchParams["filter[createdAt][to]"]) {
+        options.to = searchParams["filter[createdAt][to]"].getTime();
       }
 
-      if ($searchParams.data["filter[createdAt][period]"]) {
-        options.period = $searchParams.data["filter[createdAt][period]"];
+      if (searchParams["filter[createdAt][period]"]) {
+        options.period = searchParams["filter[createdAt][period]"];
       }
 
-      if (typeof $searchParams.data["filter[isTest]"] === "boolean") {
-        options.isTest = $searchParams.data["filter[isTest]"];
+      if (typeof searchParams["filter[isTest]"] === "boolean") {
+        options.isTest = searchParams["filter[isTest]"];
       }
 
       const presenter = new RunListPresenter();

apps/webapp/app/presenters/v3/SpanPresenter.server.ts

@@ -7,7 +7,7 @@ import {
 import { RUNNING_STATUSES } from "~/components/runs/v3/TaskRunStatus";
 import { eventRepository } from "~/v3/eventRepository.server";
 import { machinePresetFromName } from "~/v3/machinePresets.server";
-import { FINAL_ATTEMPT_STATUSES, isFinalRunStatus } from "~/v3/taskStatus";
+import { FINAL_ATTEMPT_STATUSES, isFailedRunStatus, isFinalRunStatus } from "~/v3/taskStatus";
 import { BasePresenter } from "./basePresenter.server";
 import { getMaxDuration } from "~/v3/utils/maxDuration";
 
@@ -294,6 +294,7 @@ export class SpanPresenter extends BasePresenter {
       usageDurationMs: run.usageDurationMs,
       isFinished,
       isRunning: RUNNING_STATUSES.includes(run.status),
+      isError: isFailedRunStatus(run.status),
       payload,
       payloadType: run.payloadType,
       output,

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx

@@ -501,16 +501,14 @@ function TasksTreeView({
 
   return (
     <div className="grid h-full grid-rows-[2.5rem_1fr_3.25rem] overflow-hidden">
-      <div className="mx-3 flex items-center justify-between gap-2 border-b border-grid-dimmed">
+      <div className="flex items-center justify-between gap-2 border-b border-grid-dimmed px-2">
         <SearchField onChange={setFilterText} />
-        <div className="flex items-center gap-2">
-          <Switch
-            variant="small"
-            label="Errors only"
-            checked={errorsOnly}
-            onCheckedChange={(e) => setErrorsOnly(e.valueOf())}
-          />
-        </div>
+        <Switch
+          variant="small"
+          label="Errors only"
+          checked={errorsOnly}
+          onCheckedChange={(e) => setErrorsOnly(e.valueOf())}
+        />
       </div>
       <ResizablePanelGroup autosaveId={resizableSettings.tree.autosaveId}>
         {/* Tree list */}
@@ -1008,7 +1006,7 @@ function ShowParentLink({
       {mouseOver ? (
         <ShowParentIconSelected className="h-4 w-4 text-indigo-500" />
       ) : (
-        <ShowParentIcon className="text-charcoal-650 h-4 w-4" />
+        <ShowParentIcon className="h-4 w-4 text-charcoal-650" />
       )}
       <Paragraph
         variant="small"

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.electric.$runParam/route.tsx

@@ -528,16 +528,14 @@ function TasksTreeView({
 
   return (
     <div className="grid h-full grid-rows-[2.5rem_1fr_3.25rem] overflow-hidden">
-      <div className="mx-3 flex items-center justify-between gap-2 border-b border-grid-dimmed">
+      <div className="flex items-center justify-between gap-2 border-b border-grid-dimmed px-3">
         <SearchField onChange={setFilterText} />
-        <div className="flex items-center gap-2">
-          <Switch
-            variant="small"
-            label="Errors only"
-            checked={errorsOnly}
-            onCheckedChange={(e) => setErrorsOnly(e.valueOf())}
-          />
-        </div>
+        <Switch
+          variant="small"
+          label="Errors only"
+          checked={errorsOnly}
+          onCheckedChange={(e) => setErrorsOnly(e.valueOf())}
+        />
       </div>
       <ResizablePanelGroup autosaveId={resizableSettings.tree.autosaveId}>
         {/* Tree list */}
@@ -1015,7 +1013,7 @@ function ShowParentLink({ runFriendlyId }: { runFriendlyId: string }) {
       {mouseOver ? (
         <ShowParentIconSelected className="h-4 w-4 text-indigo-500" />
       ) : (
-        <ShowParentIcon className="text-charcoal-650 h-4 w-4" />
+        <ShowParentIcon className="h-4 w-4 text-charcoal-650" />
       )}
       <Paragraph
         variant="small"

apps/webapp/app/routes/_app.orgs.$organizationSlug.v3.billing/route.tsx

@@ -117,6 +117,7 @@ export default function ChoosePlanPage() {
               subscription={v3Subscription}
               organizationSlug={organizationSlug}
               hasPromotedPlan={false}
+              periodEnd={periodEnd}
             />
           </div>
         </div>

apps/webapp/app/routes/_app.orgs.$organizationSlug/route.tsx

@@ -5,13 +5,12 @@ import { z } from "zod";
 import { RouteErrorDisplay } from "~/components/ErrorDisplay";
 import { MainBody } from "~/components/layout/AppLayout";
 import { SideMenu } from "~/components/navigation/SideMenu";
-import { featuresForRequest } from "~/features.server";
 import { useOptionalOrganization } from "~/hooks/useOrganizations";
 import { useTypedMatchesData } from "~/hooks/useTypedMatchData";
 import { useUser } from "~/hooks/useUser";
 import { OrganizationsPresenter } from "~/presenters/OrganizationsPresenter.server";
 import { getImpersonationId } from "~/services/impersonation.server";
-import { getCurrentPlan, getUsage } from "~/services/platform.v3.server";
+import { getCachedUsage, getCurrentPlan, getUsage } from "~/services/platform.v3.server";
 import { requireUserId } from "~/services/session.server";
 import { telemetry } from "~/services/telemetry.server";
 import { organizationPath } from "~/utils/pathBuilder";
@@ -30,6 +29,27 @@ export function useCurrentPlan(matches?: UIMatch[]) {
   return data?.currentPlan;
 }
 
+export const shouldRevalidate: ShouldRevalidateFunction = (params) => {
+  const { currentParams, nextParams } = params;
+
+  const current = ParamsSchema.safeParse(currentParams);
+  const next = ParamsSchema.safeParse(nextParams);
+
+  if (current.success && next.success) {
+    if (current.data.organizationSlug !== next.data.organizationSlug) {
+      return true;
+    }
+    if (current.data.projectParam !== next.data.projectParam) {
+      return true;
+    }
+  }
+
+  // This prevents revalidation when there are search params changes
+  // IMPORTANT: If the loader function depends on search params, this should be updated
+  return params.currentUrl.pathname !== params.nextUrl.pathname;
+};
+
+// IMPORTANT: Make sure to update shouldRevalidate if this loader depends on search params
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const userId = await requireUserId(request);
   const impersonationId = await getImpersonationId(request);
@@ -51,11 +71,17 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const firstDayOfMonth = new Date();
   firstDayOfMonth.setUTCDate(1);
   firstDayOfMonth.setUTCHours(0, 0, 0, 0);
-  const tomorrow = new Date();
-  tomorrow.setUTCDate(tomorrow.getDate() + 1);
+
+  // Using the 1st day of next month means we get the usage for the current month
+  // and the cache key for getCachedUsage is stable over the month
+  const firstDayOfNextMonth = new Date();
+  firstDayOfNextMonth.setUTCMonth(firstDayOfNextMonth.getUTCMonth() + 1);
+  firstDayOfNextMonth.setUTCDate(1);
+  firstDayOfNextMonth.setUTCHours(0, 0, 0, 0);
+
   const [plan, usage] = await Promise.all([
     getCurrentPlan(organization.id),
-    getUsage(organization.id, { from: firstDayOfMonth, to: tomorrow }),
+    getCachedUsage(organization.id, { from: firstDayOfMonth, to: firstDayOfNextMonth }),
   ]);
 
   let hasExceededFreeTier = false;
@@ -101,26 +127,6 @@ export function ErrorBoundary() {
   return org ? (
     <RouteErrorDisplay button={{ title: org.title, to: organizationPath(org) }} />
   ) : (
-    <RouteErrorDisplay button={{ title: "Home", to: "/" }} />
+    <RouteErrorDisplay button={{ title: "Go to homepage", to: "/" }} />
   );
 }
-
-export const shouldRevalidate: ShouldRevalidateFunction = ({
-  defaultShouldRevalidate,
-  currentParams,
-  nextParams,
-}) => {
-  const current = ParamsSchema.safeParse(currentParams);
-  const next = ParamsSchema.safeParse(nextParams);
-
-  if (current.success && next.success) {
-    if (current.data.organizationSlug !== next.data.organizationSlug) {
-      return true;
-    }
-    if (current.data.projectParam !== next.data.projectParam) {
-      return true;
-    }
-  }
-
-  return defaultShouldRevalidate;
-};

apps/webapp/app/routes/_app.orgs.$organizationSlug_.select-plan/route.tsx

@@ -37,11 +37,15 @@ export async function loader({ params, request }: LoaderFunctionArgs) {
 
   const currentPlan = await getCurrentPlan(organization.id);
 
-  return typedjson({ ...plans, ...currentPlan, organizationSlug });
+  const periodEnd = new Date();
+  periodEnd.setMonth(periodEnd.getMonth() + 1);
+
+  return typedjson({ ...plans, ...currentPlan, organizationSlug, periodEnd });
 }
 
 export default function ChoosePlanPage() {
-  const { plans, v3Subscription, organizationSlug } = useTypedLoaderData<typeof loader>();
+  const { plans, v3Subscription, organizationSlug, periodEnd } =
+    useTypedLoaderData<typeof loader>();
 
   return (
     <MainCenteredContainer className="flex max-w-[80rem] flex-col items-center gap-8 p-3">
@@ -52,6 +56,7 @@ export default function ChoosePlanPage() {
         organizationSlug={organizationSlug}
         hasPromotedPlan
         showGithubVerificationBadge
+        periodEnd={periodEnd}
       />
     </MainCenteredContainer>
   );

apps/webapp/app/routes/_app.orgs.new/route.tsx

@@ -4,6 +4,7 @@ import { RadioGroup } from "@radix-ui/react-radio-group";
 import type { ActionFunction, LoaderFunctionArgs } from "@remix-run/node";
 import { json, redirect } from "@remix-run/node";
 import { Form, useActionData, useNavigation } from "@remix-run/react";
+import { uiComponent } from "@team-plain/typescript-sdk";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { z } from "zod";
 import { MainCenteredContainer } from "~/components/layout/AppLayout";
@@ -17,21 +18,25 @@ import { Input } from "~/components/primitives/Input";
 import { InputGroup } from "~/components/primitives/InputGroup";
 import { Label } from "~/components/primitives/Label";
 import { RadioGroupItem } from "~/components/primitives/RadioButton";
+import { TextArea } from "~/components/primitives/TextArea";
 import { useFeatures } from "~/hooks/useFeatures";
 import { createOrganization } from "~/models/organization.server";
 import { NewOrganizationPresenter } from "~/presenters/NewOrganizationPresenter.server";
-import { requireUserId } from "~/services/session.server";
+import { logger } from "~/services/logger.server";
+import { requireUser, requireUserId } from "~/services/session.server";
 import { organizationPath, rootPath } from "~/utils/pathBuilder";
+import { sendToPlain } from "~/utils/plain.server";
 
 const schema = z.object({
   orgName: z.string().min(3).max(50),
   companySize: z.string().optional(),
+  whyUseUs: z.string().optional(),
 });
 
 export const loader = async ({ request }: LoaderFunctionArgs) => {
   const userId = await requireUserId(request);
   const presenter = new NewOrganizationPresenter();
-  const { hasOrganizations } = await presenter.call({ userId });
+  const { hasOrganizations } = await presenter.call({ userId: userId });
 
   return typedjson({
     hasOrganizations,
@@ -39,8 +44,7 @@ export const loader = async ({ request }: LoaderFunctionArgs) => {
 };
 
 export const action: ActionFunction = async ({ request }) => {
-  const userId = await requireUserId(request);
-
+  const user = await requireUser(request);
   const formData = await request.formData();
   const submission = parse(formData, { schema });
 
@@ -51,10 +55,41 @@ export const action: ActionFunction = async ({ request }) => {
   try {
     const organization = await createOrganization({
       title: submission.value.orgName,
-      userId,
+      userId: user.id,
       companySize: submission.value.companySize ?? null,
     });
 
+    const whyUseUs = formData.get("whyUseUs");
+
+    if (whyUseUs) {
+      try {
+        await sendToPlain({
+          userId: user.id,
+          email: user.email,
+          name: user.name ?? user.displayName ?? user.email,
+          title: "New org feedback",
+          components: [
+            uiComponent.text({
+              text: `${submission.value.orgName} just created a new organization.`,
+            }),
+            uiComponent.divider({ spacingSize: "M" }),
+            uiComponent.text({
+              size: "L",
+              color: "NORMAL",
+              text: "What problem are you trying to solve?",
+            }),
+            uiComponent.text({
+              size: "L",
+              color: "NORMAL",
+              text: whyUseUs.toString(),
+            }),
+          ],
+        });
+      } catch (error) {
+        logger.error("Error sending data to Plain when creating an org:", { error });
+      }
+    }
+
     return redirect(organizationPath(organization));
   } catch (error: any) {
     return json({ errors: { body: error.message } }, { status: 400 });
@@ -97,39 +132,48 @@ export default function NewOrganizationPage() {
             <FormError id={orgName.errorId}>{orgName.error}</FormError>
           </InputGroup>
           {isManagedCloud && (
-            <InputGroup>
-              <Label htmlFor={"companySize"}>Number of employees</Label>
-              <RadioGroup name="companySize" className="flex items-center justify-between gap-2">
-                <RadioGroupItem
-                  id="employees-1-5"
-                  label="1-5"
-                  value={"1-5"}
-                  variant="button/small"
-                  className="grow"
-                />
-                <RadioGroupItem
-                  id="employees-6-49"
-                  label="6-49"
-                  value={"6-49"}
-                  variant="button/small"
-                  className="grow"
-                />
-                <RadioGroupItem
-                  id="employees-50-99"
-                  label="50-99"
-                  value={"50-99"}
-                  variant="button/small"
-                  className="grow"
-                />
-                <RadioGroupItem
-                  id="employees-100+"
-                  label="100+"
-                  value={"100+"}
-                  variant="button/small"
-                  className="grow"
-                />
-              </RadioGroup>
-            </InputGroup>
+            <>
+              <InputGroup>
+                <Label htmlFor={"companySize"}>Number of employees</Label>
+                <RadioGroup name="companySize" className="flex items-center justify-between gap-2">
+                  <RadioGroupItem
+                    id="employees-1-5"
+                    label="1-5"
+                    value={"1-5"}
+                    variant="button/small"
+                    className="grow"
+                  />
+                  <RadioGroupItem
+                    id="employees-6-49"
+                    label="6-49"
+                    value={"6-49"}
+                    variant="button/small"
+                    className="grow"
+                  />
+                  <RadioGroupItem
+                    id="employees-50-99"
+                    label="50-99"
+                    value={"50-99"}
+                    variant="button/small"
+                    className="grow"
+                  />
+                  <RadioGroupItem
+                    id="employees-100+"
+                    label="100+"
+                    value={"100+"}
+                    variant="button/small"
+                    className="grow"
+                  />
+                </RadioGroup>
+              </InputGroup>
+              <InputGroup>
+                <Label htmlFor={"whyUseUs"}>What problem are you trying to solve?</Label>
+                <TextArea name="whyUseUs" rows={4} spellCheck={false} />
+                <Hint>
+                  Your answer will help us understand your use case and provide better support.
+                </Hint>
+              </InputGroup>
+            </>
           )}
 
           <FormButtons

apps/webapp/app/routes/api.v1.auth.jwt.claims.ts

@@ -0,0 +1,19 @@
+import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+
+export async function action({ request }: LoaderFunctionArgs) {
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const claims = {
+    sub: authenticationResult.environment.id,
+    pub: true,
+  };
+
+  return json(claims);
+}

apps/webapp/app/routes/api.v1.auth.jwt.ts

@@ -0,0 +1,46 @@
+import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { z } from "zod";
+import { generateJWT as internal_generateJWT } from "@trigger.dev/core/v3";
+
+const RequestBodySchema = z.object({
+  claims: z
+    .object({
+      scopes: z.array(z.string()).default([]),
+    })
+    .optional(),
+  expirationTime: z.union([z.number(), z.string()]).optional(),
+});
+
+export async function action({ request }: LoaderFunctionArgs) {
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const parsedBody = RequestBodySchema.safeParse(await request.json());
+
+  if (!parsedBody.success) {
+    return json(
+      { error: "Invalid request body", issues: parsedBody.error.issues },
+      { status: 400 }
+    );
+  }
+
+  const claims = {
+    sub: authenticationResult.environment.id,
+    pub: true,
+    ...parsedBody.data.claims,
+  };
+
+  const jwt = await internal_generateJWT({
+    secretKey: authenticationResult.apiKey,
+    payload: claims,
+    expirationTime: parsedBody.data.expirationTime ?? "1h",
+  });
+
+  return json({ token: jwt });
+}

apps/webapp/app/routes/api.v1.integrations.$integrationSlug.connections.$connectionId.ts

@@ -33,7 +33,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
       id: parsedParams.data.connectionId,
       integration: {
         slug: parsedParams.data.integrationSlug,
-        organization: authenticatedEnv.organization,
+        organizationId: authenticatedEnv.organization.id,
       },
     },
     include: {

apps/webapp/app/routes/api.v1.projects.$projectRef.runs.ts

@@ -1,62 +1,36 @@
-import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
 import { z } from "zod";
-import { ValidationError } from "zod-validation-error";
 import { findProjectByRef } from "~/models/project.server";
-import { ApiRunListPresenter } from "~/presenters/v3/ApiRunListPresenter.server";
-import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
-import { apiCors } from "~/utils/apiCors";
+import {
+  ApiRunListPresenter,
+  ApiRunListSearchParams,
+} from "~/presenters/v3/ApiRunListPresenter.server";
+import { createLoaderPATApiRoute } from "~/services/routeBuiilders/apiBuilder.server";
 
 const ParamsSchema = z.object({
   projectRef: z.string(),
 });
 
-export async function loader({ request, params }: LoaderFunctionArgs) {
-  if (request.method.toUpperCase() === "OPTIONS") {
-    return apiCors(request, json({}));
-  }
+export const loader = createLoaderPATApiRoute(
+  {
+    params: ParamsSchema,
+    searchParams: ApiRunListSearchParams,
+    corsStrategy: "all",
+  },
+  async ({ searchParams, params, authentication }) => {
+    const project = await findProjectByRef(params.projectRef, authentication.userId);
 
-  const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
+    if (!project) {
+      return json({ error: "Project not found" }, { status: 404 });
+    }
 
-  if (!authenticationResult) {
-    return apiCors(request, json({ error: "Invalid or Missing API key" }, { status: 401 }));
-  }
-
-  const $params = ParamsSchema.safeParse(params);
-
-  if (!$params.success) {
-    return json({ error: "Invalid params" }, { status: 400 });
-  }
-
-  const project = await findProjectByRef($params.data.projectRef, authenticationResult.userId);
-
-  if (!project) {
-    return json({ error: "Project not found" }, { status: 404 });
-  }
-
-  const url = new URL(request.url);
-
-  const presenter = new ApiRunListPresenter();
-
-  try {
-    const result = await presenter.call(project, url.searchParams);
+    const presenter = new ApiRunListPresenter();
+    const result = await presenter.call(project, searchParams);
 
     if (!result) {
-      return apiCors(request, json({ data: [] }));
+      return json({ data: [] });
     }
 
-    return apiCors(request, json(result));
-  } catch (error) {
-    if (error instanceof ValidationError) {
-      return apiCors(
-        request,
-        json({ error: "Query Error", details: error.details }, { status: 400 })
-      );
-    } else {
-      return apiCors(
-        request,
-        json({ error: error instanceof Error ? error.message : String(error) }, { status: 400 })
-      );
-    }
+    return json(result);
   }
-}
+);

apps/webapp/app/routes/api.v1.runs.$runId.tags.ts

@@ -89,6 +89,9 @@ export async function action({ request, params }: ActionFunctionArgs) {
         tags: {
           connect: tagIds.map((id) => ({ id })),
         },
+        runTags: {
+          push: newTags,
+        },
       },
     });
 

apps/webapp/app/routes/api.v1.runs.$runParam.attempts.ts

@@ -29,7 +29,10 @@ export async function action({ request, params }: ActionFunctionArgs) {
   const service = new CreateTaskRunAttemptService();
 
   try {
-    const { execution } = await service.call(runParam, authenticationResult.environment);
+    const { execution } = await service.call({
+      runId: runParam,
+      authenticatedEnv: authenticationResult.environment,
+    });
 
     return json(execution, { status: 200 });
   } catch (error) {

apps/webapp/app/routes/api.v1.runs.$runParam.reschedule.ts

@@ -62,11 +62,7 @@ export async function action({ request, params }: ActionFunctionArgs) {
     }
 
     const presenter = new ApiRetrieveRunPresenter();
-    const result = await presenter.call(
-      updatedRun.friendlyId,
-      authenticationResult.environment,
-      true
-    );
+    const result = await presenter.call(updatedRun.friendlyId, authenticationResult.environment);
 
     if (!result) {
       return json({ error: "Run not found" }, { status: 404 });

apps/webapp/app/routes/api.v1.runs.ts

@@ -1,52 +1,29 @@
-import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
-import { ValidationError } from "zod-validation-error";
-import { ApiRunListPresenter } from "~/presenters/v3/ApiRunListPresenter.server";
-import { authenticateApiRequest } from "~/services/apiAuth.server";
-import { apiCors } from "~/utils/apiCors";
+import {
+  ApiRunListPresenter,
+  ApiRunListSearchParams,
+} from "~/presenters/v3/ApiRunListPresenter.server";
+import { createLoaderApiRoute } from "~/services/routeBuiilders/apiBuilder.server";
 
-export async function loader({ request, params }: LoaderFunctionArgs) {
-  if (request.method.toUpperCase() === "OPTIONS") {
-    return apiCors(request, json({}));
-  }
-
-  const authenticationResult = await authenticateApiRequest(request, {
-    allowPublicKey: false,
-  });
-
-  if (!authenticationResult) {
-    return apiCors(request, json({ error: "Invalid or Missing API key" }, { status: 401 }));
-  }
-
-  const authenticatedEnv = authenticationResult.environment;
-
-  const url = new URL(request.url);
-
-  const presenter = new ApiRunListPresenter();
-
-  try {
+export const loader = createLoaderApiRoute(
+  {
+    searchParams: ApiRunListSearchParams,
+    allowJWT: true,
+    corsStrategy: "all",
+    authorization: {
+      action: "read",
+      resource: (_, searchParams) => ({ tasks: searchParams["filter[taskIdentifier]"] }),
+      superScopes: ["read:runs", "read:all", "admin"],
+    },
+  },
+  async ({ searchParams, authentication }) => {
+    const presenter = new ApiRunListPresenter();
     const result = await presenter.call(
-      authenticatedEnv.project,
-      url.searchParams,
-      authenticatedEnv
+      authentication.environment.project,
+      searchParams,
+      authentication.environment
     );
 
-    if (!result) {
-      return apiCors(request, json({ data: [] }));
-    }
-
-    return apiCors(request, json(result));
-  } catch (error) {
-    if (error instanceof ValidationError) {
-      return apiCors(
-        request,
-        json({ error: "Query Error", details: error.details }, { status: 400 })
-      );
-    } else {
-      return apiCors(
-        request,
-        json({ error: error instanceof Error ? error.message : String(error) }, { status: 400 })
-      );
-    }
+    return json(result);
   }
-}
+);

apps/webapp/app/routes/api.v1.tasks.$taskId.batch.ts

@@ -104,10 +104,20 @@ export async function action({ request, params }: ActionFunctionArgs) {
       return json({ error: "Task not found" }, { status: 404 });
     }
 
-    return json({
-      batchId: result.batch.friendlyId,
-      runs: result.runs,
-    });
+    return json(
+      {
+        batchId: result.batch.friendlyId,
+        runs: result.runs,
+      },
+      {
+        headers: {
+          "x-trigger-jwt-claims": JSON.stringify({
+            sub: authenticationResult.environment.id,
+            pub: true,
+          }),
+        },
+      }
+    );
   } catch (error) {
     if (error instanceof Error) {
       return json({ error: error.message }, { status: 400 });

apps/webapp/app/routes/api.v1.tasks.$taskId.trigger.ts

@@ -30,6 +30,8 @@ export async function action({ request, params }: ActionFunctionArgs) {
     return { status: 405, body: "Method Not Allowed" };
   }
 
+  logger.debug("TriggerTask action", { headers: Object.fromEntries(request.headers) });
+
   // Next authenticate the request
   const authenticationResult = await authenticateApiRequest(request);
 
@@ -105,9 +107,19 @@ export async function action({ request, params }: ActionFunctionArgs) {
       return json({ error: "Task not found" }, { status: 404 });
     }
 
-    return json({
-      id: run.friendlyId,
-    });
+    return json(
+      {
+        id: run.friendlyId,
+      },
+      {
+        headers: {
+          "x-trigger-jwt-claims": JSON.stringify({
+            sub: authenticationResult.environment.id,
+            pub: true,
+          }),
+        },
+      }
+    );
   } catch (error) {
     if (error instanceof ServiceValidationError) {
       return json({ error: error.message }, { status: 422 });

apps/webapp/app/routes/api.v3.runs.$runId.ts

@@ -1,44 +1,31 @@
-import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
 import { z } from "zod";
 import { ApiRetrieveRunPresenter } from "~/presenters/v3/ApiRetrieveRunPresenter.server";
-import { authenticateApiRequest } from "~/services/apiAuth.server";
-import { apiCors } from "~/utils/apiCors";
+import { createLoaderApiRoute } from "~/services/routeBuiilders/apiBuilder.server";
 
 const ParamsSchema = z.object({
   runId: z.string(),
 });
 
-export async function loader({ request, params }: LoaderFunctionArgs) {
-  if (request.method.toUpperCase() === "OPTIONS") {
-    return apiCors(request, json({}));
+export const loader = createLoaderApiRoute(
+  {
+    params: ParamsSchema,
+    allowJWT: true,
+    corsStrategy: "all",
+    authorization: {
+      action: "read",
+      resource: (params) => ({ runs: params.runId }),
+      superScopes: ["read:runs", "read:all", "admin"],
+    },
+  },
+  async ({ params, authentication }) => {
+    const presenter = new ApiRetrieveRunPresenter();
+    const result = await presenter.call(params.runId, authentication.environment);
+
+    if (!result) {
+      return json({ error: "Run not found" }, { status: 404 });
+    }
+
+    return json(result);
   }
-
-  const authenticationResult = await authenticateApiRequest(request, {
-    allowPublicKey: true,
-  });
-  if (!authenticationResult) {
-    return apiCors(request, json({ error: "Invalid or Missing API key" }, { status: 401 }));
-  }
-
-  const authenticatedEnv = authenticationResult.environment;
-
-  const parsed = ParamsSchema.safeParse(params);
-
-  if (!parsed.success) {
-    return apiCors(request, json({ error: "Invalid or missing runId" }, { status: 400 }));
-  }
-
-  const { runId } = parsed.data;
-
-  const showSecretDetails = authenticationResult.type === "PRIVATE";
-
-  const presenter = new ApiRetrieveRunPresenter();
-  const result = await presenter.call(runId, authenticatedEnv, showSecretDetails);
-
-  if (!result) {
-    return apiCors(request, json({ error: "Run not found" }, { status: 404 }));
-  }
-
-  return apiCors(request, json(result));
-}
+);

apps/webapp/app/routes/confirm-basic-details.tsx

@@ -222,7 +222,7 @@ export default function Page() {
                 <Label htmlFor={confirmEmail.id}>How did you hear about us?</Label>
                 <Input
                   {...conform.input(referralSource, { type: "text" })}
-                  placeholder="Google, Twitter…?"
+                  placeholder="Google, X (Twitter)…?"
                   icon="heart"
                   spellCheck={false}
                 />

apps/webapp/app/routes/realtime.v1.batches.$batchId.ts

@@ -0,0 +1,36 @@
+import { json } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { $replica } from "~/db.server";
+import { realtimeClient } from "~/services/realtimeClientGlobal.server";
+import { createLoaderApiRoute } from "~/services/routeBuiilders/apiBuilder.server";
+
+const ParamsSchema = z.object({
+  batchId: z.string(),
+});
+
+export const loader = createLoaderApiRoute(
+  {
+    params: ParamsSchema,
+    allowJWT: true,
+    corsStrategy: "all",
+    authorization: {
+      action: "read",
+      resource: (params) => ({ batch: params.batchId }),
+      superScopes: ["read:runs", "read:all", "admin"],
+    },
+  },
+  async ({ params, authentication, request }) => {
+    const batchRun = await $replica.batchTaskRun.findFirst({
+      where: {
+        friendlyId: params.batchId,
+        runtimeEnvironmentId: authentication.environment.id,
+      },
+    });
+
+    if (!batchRun) {
+      return json({ error: "Batch not found" }, { status: 404 });
+    }
+
+    return realtimeClient.streamBatch(request.url, authentication.environment, batchRun.id);
+  }
+);

apps/webapp/app/routes/realtime.v1.runs.$runId.ts

@@ -0,0 +1,36 @@
+import { json } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { $replica } from "~/db.server";
+import { realtimeClient } from "~/services/realtimeClientGlobal.server";
+import { createLoaderApiRoute } from "~/services/routeBuiilders/apiBuilder.server";
+
+const ParamsSchema = z.object({
+  runId: z.string(),
+});
+
+export const loader = createLoaderApiRoute(
+  {
+    params: ParamsSchema,
+    allowJWT: true,
+    corsStrategy: "all",
+    authorization: {
+      action: "read",
+      resource: (params) => ({ runs: params.runId }),
+      superScopes: ["read:runs", "read:all", "admin"],
+    },
+  },
+  async ({ params, authentication, request }) => {
+    const run = await $replica.taskRun.findFirst({
+      where: {
+        friendlyId: params.runId,
+        runtimeEnvironmentId: authentication.environment.id,
+      },
+    });
+
+    if (!run) {
+      return json({ error: "Run not found" }, { status: 404 });
+    }
+
+    return realtimeClient.streamRun(request.url, authentication.environment, run.id);
+  }
+);

apps/webapp/app/routes/realtime.v1.runs.ts

@@ -0,0 +1,28 @@
+import { z } from "zod";
+import { realtimeClient } from "~/services/realtimeClientGlobal.server";
+import { createLoaderApiRoute } from "~/services/routeBuiilders/apiBuilder.server";
+
+const SearchParamsSchema = z.object({
+  tags: z
+    .string()
+    .optional()
+    .transform((value) => {
+      return value ? value.split(",") : undefined;
+    }),
+});
+
+export const loader = createLoaderApiRoute(
+  {
+    searchParams: SearchParamsSchema,
+    allowJWT: true,
+    corsStrategy: "all",
+    authorization: {
+      action: "read",
+      resource: (_, searchParams) => searchParams,
+      superScopes: ["read:runs", "read:all", "admin"],
+    },
+  },
+  async ({ searchParams, authentication, request }) => {
+    return realtimeClient.streamRuns(request.url, authentication.environment, searchParams);
+  }
+);

apps/webapp/app/routes/resources.feedback.ts

@@ -6,6 +6,7 @@ import { z } from "zod";
 import { env } from "~/env.server";
 import { redirectWithSuccessMessage } from "~/models/message.server";
 import { requireUser } from "~/services/session.server";
+import { sendToPlain } from "~/utils/plain.server";
 
 let client: PlainClient | undefined;
 
@@ -14,7 +15,7 @@ export const feedbackTypeLabel = {
   feature: "Feature request",
   help: "Help me out",
   enterprise: "Enterprise enquiry",
-  "developer preview": "Developer preview feedback",
+  feedback: "General feedback",
 };
 
 export type FeedbackType = keyof typeof feedbackTypeLabel;
@@ -32,7 +33,7 @@ const feedbackType = z.union(
 export const schema = z.object({
   path: z.string(),
   feedbackType,
-  message: z.string().min(1, "Must be at least 1 character"),
+  message: z.string().min(10, "Must be at least 10 characters"),
 });
 
 export async function action({ request }: ActionFunctionArgs) {
@@ -45,60 +46,12 @@ export async function action({ request }: ActionFunctionArgs) {
     return json(submission);
   }
 
+  const title = feedbackTypeLabel[submission.value.feedbackType as FeedbackType];
   try {
-    if (!env.PLAIN_API_KEY) {
-      console.error("PLAIN_API_KEY is not set");
-      submission.error.message = "PLAIN_API_KEY is not set";
-      return json(submission);
-    }
-
-    client = new PlainClient({
-      apiKey: env.PLAIN_API_KEY,
-    });
-
-    const upsertCustomerRes = await client.upsertCustomer({
-      identifier: {
-        emailAddress: user.email,
-      },
-      onCreate: {
-        externalId: user.id,
-        fullName: user.name ?? "",
-        // TODO - Optional: set 'first name' on user
-        // shortName: ''
-        email: {
-          email: user.email,
-          isVerified: true,
-        },
-      },
-      onUpdate: {
-        externalId: { value: user.id },
-        fullName: { value: user.name ?? "" },
-        // TODO - see above
-        // shortName: { value: "" },
-        email: {
-          email: user.email,
-          isVerified: true,
-        },
-      },
-    });
-
-    if (upsertCustomerRes.error) {
-      console.error(
-        inspect(upsertCustomerRes.error, {
-          showHidden: false,
-          depth: null,
-          colors: true,
-        })
-      );
-      submission.error.message = upsertCustomerRes.error.message;
-      return json(submission);
-    }
-
-    const title = feedbackTypeLabel[submission.value.feedbackType as FeedbackType];
-    const createThreadRes = await client.createThread({
-      customerIdentifier: {
-        customerId: upsertCustomerRes.data.customer.id,
-      },
+    await sendToPlain({
+      userId: user.id,
+      email: user.email,
+      name: user.name ?? user.displayName ?? user.email,
       title,
       components: [
         uiComponent.text({
@@ -123,31 +76,15 @@ export async function action({ request }: ActionFunctionArgs) {
           text: submission.value.message,
         }),
       ],
-      // TODO: Optional: set labels on threads here on creation
-      // labelTypeIds: [],
-
-      // TODO: Optional: set the priority (0 is urgent, 3 is low)
-      // priority: 0,
     });
 
-    if (createThreadRes.error) {
-      console.error(
-        inspect(createThreadRes.error, {
-          showHidden: false,
-          depth: null,
-          colors: true,
-        })
-      );
-      submission.error.message = createThreadRes.error.message;
-      return json(submission);
-    }
-
     return redirectWithSuccessMessage(
       submission.value.path,
       request,
       "Thanks for your feedback! We'll get back to you soon."
     );
   } catch (e) {
-    return json(e, { status: 400 });
+    submission.error.message = e instanceof Error ? e.message : "Unknown error";
+    return json(submission);
   }
 }

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.schedules.new/route.tsx

@@ -44,6 +44,7 @@ import { CronPattern, UpsertSchedule } from "~/v3/schedules";
 import { UpsertTaskScheduleService } from "~/v3/services/upsertTaskSchedule.server";
 import { AIGeneratedCronField } from "../resources.orgs.$organizationSlug.projects.$projectParam.schedules.new.natural-language";
 import { TimezoneList } from "~/components/scheduled/timezones";
+import { logger } from "~/services/logger.server";
 
 const cronFormat = `*    *    *    *    *
 ┬    ┬    ┬    ┬    ┬
@@ -94,9 +95,9 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
       submission.value?.friendlyId === result.id ? "Schedule updated" : "Schedule created"
     );
   } catch (error: any) {
-    const errorMessage = `Failed: ${
-      error instanceof Error ? error.message : JSON.stringify(error)
-    }`;
+    logger.error("Failed to create schedule", error);
+
+    const errorMessage = `Something went wrong. Please try again.`;
     return redirectWithErrorMessage(
       v3SchedulesPath({ slug: organizationSlug }, { slug: projectParam }),
       request,

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx

@@ -1,4 +1,10 @@
-import { CheckIcon, ClockIcon, CloudArrowDownIcon, QueueListIcon } from "@heroicons/react/20/solid";
+import {
+  CheckIcon,
+  ClockIcon,
+  CloudArrowDownIcon,
+  EnvelopeIcon,
+  QueueListIcon,
+} from "@heroicons/react/20/solid";
 import { Link } from "@remix-run/react";
 import { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import {
@@ -13,6 +19,7 @@ import { typedjson, useTypedFetcher } from "remix-typedjson";
 import { ExitIcon } from "~/assets/icons/ExitIcon";
 import { CodeBlock } from "~/components/code/CodeBlock";
 import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
+import { Feedback } from "~/components/Feedback";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
 import { Callout } from "~/components/primitives/Callout";
 import { DateTime, DateTimeAccurate } from "~/components/primitives/DateTime";
@@ -172,7 +179,7 @@ function SpanBody({
 
   return (
     <div className="grid h-full max-h-full grid-rows-[2.5rem_2rem_1fr] overflow-hidden bg-background-bright">
-      <div className="mx-3 flex items-center justify-between gap-2 overflow-x-hidden">
+      <div className="flex items-center justify-between gap-2 overflow-x-hidden px-3">
         <div className="flex items-center gap-1 overflow-x-hidden">
           <RunIcon
             name={span.style?.icon}
@@ -314,7 +321,7 @@ function SpanBody({
               <Property.Table>
                 <Property.Item>
                   <Property.Label>Message</Property.Label>
-                  <Property.Value>{span.message}</Property.Value>
+                  <Property.Value className="whitespace-pre-wrap">{span.message}</Property.Value>
                 </Property.Item>
                 {span.triggeredRuns.length > 0 && (
                   <Property.Item>
@@ -413,7 +420,7 @@ function RunBody({
 
   return (
     <div className="grid h-full max-h-full grid-rows-[2.5rem_2rem_1fr_3.25rem] overflow-hidden bg-background-bright">
-      <div className="mx-3 flex items-center justify-between gap-2 overflow-x-hidden">
+      <div className="flex items-center justify-between gap-2 overflow-x-hidden px-3">
         <div className="flex items-center gap-1 overflow-x-hidden">
           <RunIcon
             name={"task"}
@@ -850,7 +857,7 @@ function RunTimeline({ run }: { run: SpanRun }) {
               <RunTimelineEvent
                 title="Finished"
                 subtitle={<DateTimeAccurate date={run.updatedAt} />}
-                state="complete"
+                state={run.isError ? "error" : "complete"}
               />
             </>
           ) : (
@@ -963,11 +970,26 @@ function RunError({ error }: { error: TaskRunError }) {
         <div className="flex flex-col gap-2 rounded-sm border border-rose-500/50 px-3 pb-3 pt-2">
           <Header3 className="text-rose-500">{name}</Header3>
           {enhancedError.message && <Callout variant="error">{enhancedError.message}</Callout>}
-          {enhancedError.link && (
-            <Callout variant="docs" to={enhancedError.link.href}>
-              {enhancedError.link.name}
-            </Callout>
-          )}
+          {enhancedError.link &&
+            (enhancedError.link.magic === "CONTACT_FORM" ? (
+              <Feedback
+                button={
+                  <Button
+                    variant="tertiary/medium"
+                    LeadingIcon={EnvelopeIcon}
+                    leadingIconClassName="text-blue-400"
+                    fullWidth
+                    textAlignLeft
+                  >
+                    {enhancedError.link.name}
+                  </Button>
+                }
+              />
+            ) : (
+              <Callout variant="docs" to={enhancedError.link.href}>
+                {enhancedError.link.name}
+              </Callout>
+            ))}
           {enhancedError.stackTrace && (
             <CodeBlock
               showCopyButton={false}

apps/webapp/app/routes/resources.orgs.$organizationSlug.select-plan.tsx

@@ -1,11 +1,15 @@
 import {
+  ArrowUpRightIcon,
   CheckIcon,
   ExclamationTriangleIcon,
   ShieldCheckIcon,
   XMarkIcon,
 } from "@heroicons/react/20/solid";
+import { ArrowDownCircleIcon } from "@heroicons/react/24/outline";
 import { Form, useLocation, useNavigation } from "@remix-run/react";
 import { ActionFunctionArgs } from "@remix-run/server-runtime";
+import { uiComponent } from "@team-plain/typescript-sdk";
+import { GitHubLightIcon } from "@trigger.dev/companyicons";
 import {
   FreePlanDefinition,
   Limits,
@@ -14,11 +18,13 @@ import {
   SetPlanBody,
   SubscriptionResult,
 } from "@trigger.dev/platform/v3";
-import { GitHubLightIcon } from "@trigger.dev/companyicons";
+import React, { useEffect, useState } from "react";
 import { z } from "zod";
 import { DefinitionTip } from "~/components/DefinitionTooltip";
 import { Feedback } from "~/components/Feedback";
-import { Button, LinkButton } from "~/components/primitives/Buttons";
+import { Button } from "~/components/primitives/Buttons";
+import { CheckboxWithLabel } from "~/components/primitives/Checkbox";
+import { DateTime } from "~/components/primitives/DateTime";
 import {
   Dialog,
   DialogContent,
@@ -26,14 +32,18 @@ import {
   DialogHeader,
   DialogTrigger,
 } from "~/components/primitives/Dialog";
+import { Header2 } from "~/components/primitives/Headers";
 import { Paragraph } from "~/components/primitives/Paragraph";
 import { Spinner } from "~/components/primitives/Spinner";
+import { TextArea } from "~/components/primitives/TextArea";
 import { SimpleTooltip } from "~/components/primitives/Tooltip";
 import { prisma } from "~/db.server";
 import { redirectWithErrorMessage } from "~/models/message.server";
+import { logger } from "~/services/logger.server";
 import { setPlan } from "~/services/platform.v3.server";
-import { requireUserId } from "~/services/session.server";
+import { requireUser } from "~/services/session.server";
 import { cn } from "~/utils/cn";
+import { sendToPlain } from "~/utils/plain.server";
 
 const Params = z.object({
   organizationSlug: z.string(),
@@ -43,6 +53,8 @@ const schema = z.object({
   type: z.enum(["free", "paid"]),
   planCode: z.string().optional(),
   callerPath: z.string(),
+  reasons: z.union([z.string(), z.array(z.string())]).optional(),
+  message: z.string().optional(),
 });
 
 export async function action({ request, params }: ActionFunctionArgs) {
@@ -51,11 +63,16 @@ export async function action({ request, params }: ActionFunctionArgs) {
   }
 
   const { organizationSlug } = Params.parse(params);
+  const user = await requireUser(request);
+  const formData = await request.formData();
+  const reasons = formData.getAll("reasons");
+  const message = formData.get("message");
 
-  const userId = await requireUserId(request);
-
-  const formData = Object.fromEntries(await request.formData());
-  const form = schema.parse(formData);
+  const form = schema.parse({
+    ...Object.fromEntries(formData),
+    reasons,
+    message: message || undefined,
+  });
 
   const organization = await prisma.organization.findUnique({
     where: { slug: organizationSlug },
@@ -69,9 +86,53 @@ export async function action({ request, params }: ActionFunctionArgs) {
 
   switch (form.type) {
     case "free": {
+      try {
+        if (reasons.length > 0 || (message && message.toString().trim() !== "")) {
+          await sendToPlain({
+            userId: user.id,
+            email: user.email,
+            name: user.name ?? "",
+            title: "Plan cancelation feedback",
+            components: [
+              uiComponent.text({
+                text: `${user.name} (${user.email}) just canceled their plan.`,
+              }),
+              uiComponent.divider({ spacingSize: "M" }),
+              ...(reasons.length > 0
+                ? [
+                    uiComponent.spacer({ size: "L" }),
+                    uiComponent.text({
+                      size: "L",
+                      color: "NORMAL",
+                      text: "Reasons:",
+                    }),
+                    uiComponent.text({
+                      text: reasons.join(", "),
+                    }),
+                  ]
+                : []),
+              ...(message
+                ? [
+                    uiComponent.spacer({ size: "L" }),
+                    uiComponent.text({
+                      size: "L",
+                      color: "NORMAL",
+                      text: "Comment:",
+                    }),
+                    uiComponent.text({
+                      text: message.toString(),
+                    }),
+                  ]
+                : []),
+            ],
+          });
+        }
+      } catch (e) {
+        logger.error("Failed to submit to Plain the unsubscribe reason", { error: e });
+      }
       payload = {
         type: "free" as const,
-        userId,
+        userId: user.id,
       };
       break;
     }
@@ -82,10 +143,13 @@ export async function action({ request, params }: ActionFunctionArgs) {
       payload = {
         type: "paid" as const,
         planCode: form.planCode,
-        userId,
+        userId: user.id,
       };
       break;
     }
+    default: {
+      throw new Error("Invalid form type");
+    }
   }
 
   return setPlan(organization, request, form.callerPath, payload);
@@ -134,6 +198,7 @@ type PricingPlansProps = {
   organizationSlug: string;
   hasPromotedPlan: boolean;
   showGithubVerificationBadge?: boolean;
+  periodEnd: Date;
 };
 
 export function PricingPlans({
@@ -142,6 +207,7 @@ export function PricingPlans({
   organizationSlug,
   hasPromotedPlan,
   showGithubVerificationBadge,
+  periodEnd,
 }: PricingPlansProps) {
   return (
     <div className="flex w-full flex-col">
@@ -151,6 +217,7 @@ export function PricingPlans({
           subscription={subscription}
           organizationSlug={organizationSlug}
           showGithubVerificationBadge={showGithubVerificationBadge}
+          periodEnd={periodEnd}
         />
         <TierHobby
           plan={plans.hobby}
@@ -172,23 +239,33 @@ export function TierFree({
   subscription,
   organizationSlug,
   showGithubVerificationBadge,
+  periodEnd,
 }: {
   plan: FreePlanDefinition;
   subscription?: SubscriptionResult;
   organizationSlug: string;
   showGithubVerificationBadge?: boolean;
+  periodEnd: Date;
 }) {
   const location = useLocation();
   const navigation = useNavigation();
   const formAction = `/resources/orgs/${organizationSlug}/select-plan`;
   const isLoading = navigation.formAction === formAction;
-
+  const [isDialogOpen, setIsDialogOpen] = useState(false);
+  const [isLackingFeaturesChecked, setIsLackingFeaturesChecked] = useState(false);
   const status = subscription?.freeTierStatus ?? "requires_connect";
 
+  useEffect(() => {
+    setIsDialogOpen(false);
+  }, [subscription]);
+
   return (
     <TierContainer>
       <div className="relative">
         <PricingHeader title={plan.title} cost={0} />
+        <TierLimit href="https://trigger.dev/pricing#computePricing">
+          ${plan.limits.includedUsage / 100} free monthly usage
+        </TierLimit>
         {showGithubVerificationBadge && status === "approved" && (
           <SimpleTooltip
             buttonClassName="absolute right-1 top-1"
@@ -242,16 +319,11 @@ export function TierFree({
           </div>
         </div>
       ) : (
-        <Form action={formAction} method="post" id="subscribe">
-          <input type="hidden" name="type" value="free" />
-          <input type="hidden" name="callerPath" value={location.pathname} />
-          <TierLimit href="https://trigger.dev/pricing#computePricing">
-            ${plan.limits.includedUsage / 100} free usage
-          </TierLimit>
-          <div className="py-6">
-            {status === "requires_connect" ? (
-              <Dialog>
-                <DialogTrigger asChild>
+        <>
+          {status === "requires_connect" ? (
+            <Dialog>
+              <DialogTrigger asChild>
+                <div className="my-6">
                   <Button
                     type="button"
                     variant="tertiary/large"
@@ -260,12 +332,16 @@ export function TierFree({
                     disabled={isLoading}
                     LeadingIcon={isLoading ? Spinner : undefined}
                   >
-                    Unlock free plan
+                    Unlock Free plan
                   </Button>
-                </DialogTrigger>
-                <DialogContent className="sm:max-w-md">
+                </div>
+              </DialogTrigger>
+              <DialogContent className="sm:max-w-md">
+                <Form action={formAction} method="post" id="subscribe-free">
+                  <input type="hidden" name="type" value="free" />
+                  <input type="hidden" name="callerPath" value={location.pathname} />
                   <DialogHeader>Unlock the Free plan</DialogHeader>
-                  <div className="mb-3 mt-4 flex flex-col items-center gap-4 px-6">
+                  <div className="mb-5 mt-7 flex flex-col items-center gap-4 px-6">
                     <GitHubLightIcon className="size-16" />
                     <Paragraph variant="base/bright" className="text-center">
                       To unlock the Free plan, we need to verify that you have an active GitHub
@@ -282,16 +358,104 @@ export function TierFree({
                       fullWidth
                       disabled={isLoading}
                       LeadingIcon={isLoading ? Spinner : undefined}
-                      form="subscribe"
+                      form="subscribe-free"
                     >
                       Connect to GitHub
                     </Button>
                   </DialogFooter>
-                </DialogContent>
-              </Dialog>
-            ) : (
+                </Form>
+              </DialogContent>
+            </Dialog>
+          ) : subscription?.plan !== undefined &&
+            subscription.plan.type !== "free" &&
+            subscription.canceledAt === undefined ? (
+            <Dialog open={isDialogOpen} onOpenChange={setIsDialogOpen} key="cancel">
+              <DialogTrigger asChild>
+                <div className="my-6">
+                  <Button variant="tertiary/large" fullWidth className="text-md font-medium">
+                    {`Downgrade to ${plan.title}`}
+                  </Button>
+                </div>
+              </DialogTrigger>
+              <DialogContent className="sm:max-w-md">
+                <Form action={formAction} method="post" id="subscribe">
+                  <input type="hidden" name="type" value="free" />
+                  <input type="hidden" name="callerPath" value={location.pathname} />
+                  <DialogHeader>Downgrade plan?</DialogHeader>
+                  <div className="flex items-start gap-3 pb-6 pr-2 pt-8">
+                    <ArrowDownCircleIcon className="size-12 min-w-12 text-error" />
+                    <Paragraph variant="base/bright" className="text-text-bright">
+                      Are you sure you want to downgrade? You will lose access to your current
+                      plan's features on{" "}
+                      <DateTime
+                        includeTime={false}
+                        date={new Date(periodEnd.getTime() + 86400000)}
+                      />
+                      .
+                    </Paragraph>
+                  </div>
+                  <div>
+                    <div className="mb-4">
+                      <Header2 className="mb-1">Why are you thinking of downgrading?</Header2>
+                      <ul className="space-y-1">
+                        {[
+                          "Subscription or usage costs too expensive",
+                          "Bugs or technical issues",
+                          "No longer need the service",
+                          "Found a better alternative",
+                          "Lacking features I need",
+                        ].map((label, index) => (
+                          <li key={index}>
+                            <CheckboxWithLabel
+                              id={`reason-${index + 1}`}
+                              name="reasons"
+                              value={label}
+                              variant="simple"
+                              label={label}
+                              labelClassName="text-text-dimmed"
+                              onChange={(isChecked: boolean) => {
+                                if (label === "Lacking features I need") {
+                                  setIsLackingFeaturesChecked(isChecked);
+                                }
+                              }}
+                            />
+                          </li>
+                        ))}
+                      </ul>
+                    </div>
+                    <div>
+                      <Header2 className="mb-1">
+                        {isLackingFeaturesChecked
+                          ? "What features do you need? Or how can we improve?"
+                          : "What can we do to improve?"}
+                      </Header2>
+                      <TextArea id="improvement-suggestions" name="message" />
+                    </div>
+                  </div>
+                  <DialogFooter className="mt-2">
+                    <Button variant="tertiary/medium" onClick={() => setIsDialogOpen(false)}>
+                      Dismiss
+                    </Button>
+                    <Button
+                      variant="danger/medium"
+                      disabled={isLoading}
+                      LeadingIcon={isLoading ? () => <Spinner color="white" /> : undefined}
+                      type="submit"
+                    >
+                      Downgrade plan
+                    </Button>
+                  </DialogFooter>
+                </Form>
+              </DialogContent>
+            </Dialog>
+          ) : (
+            <Form action={formAction} method="post" id="subscribe-verified" className="my-6">
+              <input type="hidden" name="type" value="free" />
+              <input type="hidden" name="callerPath" value={location.pathname} />
               <Button
                 variant="tertiary/large"
+                type="submit"
+                form="subscribe-verified"
                 fullWidth
                 className="text-md font-medium"
                 disabled={
@@ -305,12 +469,14 @@ export function TierFree({
               >
                 {subscription?.plan === undefined
                   ? "Select plan"
-                  : subscription.plan.type === "free" || subscription.canceledAt !== undefined
+                  : subscription.plan.type === "free"
                   ? "Current plan"
-                  : `Downgrade to ${plan.title}`}
+                  : subscription.canceledAt !== undefined
+                  ? "Current plan"
+                  : "Select plan"}
               </Button>
-            )}
-          </div>
+            </Form>
+          )}
           <ul className="flex flex-col gap-2.5">
             <ConcurrentRuns limits={plan.limits} />
             <FeatureItem checked>
@@ -329,7 +495,7 @@ export function TierFree({
             <SupportLevel limits={plan.limits} />
             <Alerts limits={plan.limits} />
           </ul>
-        </Form>
+        </>
       )}
     </TierContainer>
   );
@@ -350,6 +516,11 @@ export function TierHobby({
   const navigation = useNavigation();
   const formAction = `/resources/orgs/${organizationSlug}/select-plan`;
   const isLoading = navigation.formAction === formAction;
+  const [isDialogOpen, setIsDialogOpen] = useState(false);
+
+  useEffect(() => {
+    setIsDialogOpen(false);
+  }, [subscription]);
 
   return (
     <TierContainer isHighlighted={isHighlighted}>
@@ -357,15 +528,52 @@ export function TierHobby({
       <TierLimit href="https://trigger.dev/pricing#computePricing">
         ${plan.limits.includedUsage / 100} usage included
       </TierLimit>
-      <Form action={formAction} method="post" id="subscribe">
-        <div className="py-6">
-          <input type="hidden" name="type" value="paid" />
-          <input type="hidden" name="planCode" value={plan.code} />
-          <input type="hidden" name="callerPath" value={location.pathname} />
+      <Form action={formAction} method="post" id="subscribe-hobby" className="py-6">
+        <input type="hidden" name="type" value="paid" />
+        <input type="hidden" name="planCode" value={plan.code} />
+        <input type="hidden" name="callerPath" value={location.pathname} />
+        {subscription?.plan !== undefined &&
+        subscription.plan.type !== "free" &&
+        subscription.canceledAt === undefined &&
+        subscription.plan.code !== plan.code ? (
+          <Dialog open={isDialogOpen} onOpenChange={setIsDialogOpen} key="downgrade">
+            <DialogTrigger asChild>
+              <Button variant="tertiary/large" fullWidth className="text-md font-medium">
+                {`Downgrade to ${plan.title}`}
+              </Button>
+            </DialogTrigger>
+            <DialogContent className="sm:max-w-md">
+              <DialogHeader>Downgrade plan?</DialogHeader>
+              <div className="mb-2 mt-4 flex items-start gap-3">
+                <span>
+                  <ArrowDownCircleIcon className="size-12 text-blue-500" />
+                </span>
+                <Paragraph variant="base/bright" className="text-text-bright">
+                  By downgrading you will lose access to your current plan's features and your
+                  included credits will be reduced.
+                </Paragraph>
+              </div>
+              <DialogFooter>
+                <Button variant="tertiary/medium" onClick={() => setIsDialogOpen(false)}>
+                  Dismiss
+                </Button>
+                <Button
+                  variant="tertiary/medium"
+                  disabled={isLoading}
+                  LeadingIcon={isLoading ? () => <Spinner color="white" /> : undefined}
+                  form="subscribe-hobby"
+                >
+                  {`Downgrade to ${plan.title}`}
+                </Button>
+              </DialogFooter>
+            </DialogContent>
+          </Dialog>
+        ) : (
           <Button
             variant={isHighlighted ? "primary/large" : "tertiary/large"}
             fullWidth
             className="text-md font-medium"
+            form="subscribe-hobby"
             disabled={
               isLoading ||
               (subscription?.plan?.code === plan.code && subscription.canceledAt === undefined)
@@ -380,9 +588,9 @@ export function TierHobby({
               ? `Upgrade to ${plan.title}`
               : subscription.plan.code === plan.code
               ? "Current plan"
-              : `Downgrade to ${plan.title}`}
+              : `Upgrade to ${plan.title}`}
           </Button>
-        </div>
+        )}
       </Form>
       <ul className="flex flex-col gap-2.5">
         <ConcurrentRuns limits={plan.limits} />
@@ -425,7 +633,7 @@ export function TierPro({
       <TierLimit href="https://trigger.dev/pricing#computePricing">
         ${plan.limits.includedUsage / 100} usage included
       </TierLimit>
-      <Form action={formAction} method="post" id="subscribe">
+      <Form action={formAction} method="post" id="subscribe-pro">
         <div className="py-6">
           <input type="hidden" name="type" value="paid" />
           <input type="hidden" name="planCode" value={plan.code} />
@@ -433,6 +641,7 @@ export function TierPro({
           <Button
             variant="tertiary/large"
             fullWidth
+            form="subscribe-pro"
             className="text-md font-medium"
             disabled={
               isLoading ||
@@ -589,21 +798,28 @@ function PricingHeader({
 function TierLimit({ children, href }: { children: React.ReactNode; href?: string }) {
   return (
     <>
+      <hr className="my-6 border-grid-bright" />
       {href ? (
-        <div>
-          <hr className="my-6 border-grid-bright" />
-          <a
-            href={href}
-            className="hover:decoration-bright font-sans text-lg font-normal text-text-bright underline decoration-charcoal-500 underline-offset-4 transition"
-          >
-            {children}
-          </a>
-        </div>
+        <SimpleTooltip
+          buttonClassName="text-left w-fit"
+          disableHoverableContent
+          button={
+            <a
+              href={href}
+              className="text-left font-sans text-lg font-normal text-text-bright underline decoration-charcoal-500 underline-offset-4 transition hover:decoration-text-bright"
+            >
+              {children}
+            </a>
+          }
+          content={
+            <div className="flex items-center gap-1">
+              <Paragraph variant="small">View compute pricing information</Paragraph>
+              <ArrowUpRightIcon className="size-4 text-text-dimmed" />
+            </div>
+          }
+        />
       ) : (
-        <div>
-          <hr className="my-6 border-grid-bright" />
-          <div className="font-sans text-lg font-normal text-text-bright">{children}</div>
-        </div>
+        <div className="font-sans text-lg font-normal text-text-bright">{children}</div>
       )}
     </>
   );

apps/webapp/app/routes/storybook.textarea/route.tsx

@@ -0,0 +1,17 @@
+import { Input } from "~/components/primitives/Input";
+import { TextArea } from "~/components/primitives/TextArea";
+
+export default function Story() {
+  return (
+    <div className="flex gap-16">
+      <div>
+        <div className="m-8 flex w-64 flex-col gap-4">
+          <TextArea placeholder="6 rows (default)" autoFocus />
+          <Input placeholder="Input" />
+          <TextArea placeholder="3 rows" rows={3} />
+          <TextArea disabled placeholder="Disabled" />
+        </div>
+      </div>
+    </div>
+  );
+}

apps/webapp/app/routes/storybook/route.tsx

@@ -126,16 +126,20 @@ const stories: Story[] = [
     slug: "date-fields",
   },
   {
-    name: "Simple form",
-    slug: "simple-form",
+    name: "Input fields",
+    slug: "input-fields",
   },
   {
     name: "Search fields",
     slug: "search-fields",
   },
   {
-    name: "Input fields",
-    slug: "input-fields",
+    name: "Simple form",
+    slug: "simple-form",
+  },
+  {
+    name: "Textarea",
+    slug: "textarea",
   },
   {
     sectionTitle: "Menus",

apps/webapp/app/services/apiAuth.server.ts

@@ -1,52 +1,56 @@
+import { json } from "@remix-run/server-runtime";
 import { Prettify } from "@trigger.dev/core";
+import { SignJWT, errors, jwtVerify } from "jose";
 import { z } from "zod";
+import { prisma } from "~/db.server";
+import { env } from "~/env.server";
+import { findProjectByRef } from "~/models/project.server";
 import {
   RuntimeEnvironment,
   findEnvironmentByApiKey,
   findEnvironmentByPublicApiKey,
 } from "~/models/runtimeEnvironment.server";
+import { logger } from "./logger.server";
 import {
   PersonalAccessTokenAuthenticationResult,
   authenticateApiRequestWithPersonalAccessToken,
   isPersonalAccessToken,
 } from "./personalAccessToken.server";
-import { prisma } from "~/db.server";
-import { json } from "@remix-run/server-runtime";
-import { findProjectByRef } from "~/models/project.server";
-import { SignJWT, jwtVerify, errors } from "jose";
-import { env } from "~/env.server";
-import { logger } from "./logger.server";
+import { isPublicJWT, validatePublicJwtKey } from "./realtime/jwtAuth.server";
+
+const ClaimsSchema = z.object({
+  scopes: z.array(z.string()).optional(),
+});
 
 type Optional<T, K extends keyof T> = Prettify<Omit<T, K> & Partial<Pick<T, K>>>;
 
-const AuthorizationHeaderSchema = z.string().regex(/^Bearer .+$/);
-
 export type AuthenticatedEnvironment = Optional<
   NonNullable<Awaited<ReturnType<typeof findEnvironmentByApiKey>>>,
   "orgMember"
 >;
 
-type ApiAuthenticationResult = {
+export type ApiAuthenticationResult = {
   apiKey: string;
-  type: "PUBLIC" | "PRIVATE";
+  type: "PUBLIC" | "PRIVATE" | "PUBLIC_JWT";
   environment: AuthenticatedEnvironment;
+  scopes?: string[];
 };
 
 export async function authenticateApiRequest(
   request: Request,
-  { allowPublicKey = false }: { allowPublicKey?: boolean } = {}
+  options: { allowPublicKey?: boolean; allowJWT?: boolean } = {}
 ): Promise<ApiAuthenticationResult | undefined> {
   const apiKey = getApiKeyFromRequest(request);
   if (!apiKey) {
     return;
   }
 
-  return authenticateApiKey(apiKey, { allowPublicKey });
+  return authenticateApiKey(apiKey, options);
 }
 
 export async function authenticateApiKey(
   apiKey: string,
-  { allowPublicKey = false }: { allowPublicKey?: boolean } = {}
+  options: { allowPublicKey?: boolean; allowJWT?: boolean } = {}
 ): Promise<ApiAuthenticationResult | undefined> {
   const result = getApiKeyResult(apiKey);
 
@@ -54,14 +58,12 @@ export async function authenticateApiKey(
     return;
   }
 
-  //if it's a public API key and we don't allow public keys, return
-  if (!allowPublicKey) {
-    const environment = await findEnvironmentByApiKey(result.apiKey);
-    if (!environment) return;
-    return {
-      ...result,
-      environment,
-    };
+  if (!options.allowPublicKey && result.type === "PUBLIC") {
+    return;
+  }
+
+  if (!options.allowJWT && result.type === "PUBLIC_JWT") {
+    return;
   }
 
   switch (result.type) {
@@ -81,27 +83,72 @@ export async function authenticateApiKey(
         environment,
       };
     }
+    case "PUBLIC_JWT": {
+      const validationResults = await validatePublicJwtKey(result.apiKey);
+
+      if (!validationResults) {
+        return;
+      }
+
+      const parsedClaims = ClaimsSchema.safeParse(validationResults.claims);
+
+      return {
+        ...result,
+        environment: validationResults.environment,
+        scopes: parsedClaims.success ? parsedClaims.data.scopes : [],
+      };
+    }
   }
 }
 
+export async function authenticateAuthorizationHeader(
+  authorization: string,
+  {
+    allowPublicKey = false,
+    allowJWT = false,
+  }: { allowPublicKey?: boolean; allowJWT?: boolean } = {}
+): Promise<ApiAuthenticationResult | undefined> {
+  const apiKey = getApiKeyFromHeader(authorization);
+
+  if (!apiKey) {
+    return;
+  }
+
+  return authenticateApiKey(apiKey, { allowPublicKey, allowJWT });
+}
+
 export function isPublicApiKey(key: string) {
   return key.startsWith("pk_");
 }
 
-export function getApiKeyFromRequest(request: Request) {
-  const rawAuthorization = request.headers.get("Authorization");
+export function isSecretApiKey(key: string) {
+  return key.startsWith("tr_");
+}
 
-  const authorization = AuthorizationHeaderSchema.safeParse(rawAuthorization);
-  if (!authorization.success) {
+export function getApiKeyFromRequest(request: Request) {
+  return getApiKeyFromHeader(request.headers.get("Authorization"));
+}
+
+export function getApiKeyFromHeader(authorization?: string | null) {
+  if (typeof authorization !== "string" || !authorization) {
     return;
   }
 
-  const apiKey = authorization.data.replace(/^Bearer /, "");
+  const apiKey = authorization.replace(/^Bearer /, "");
   return apiKey;
 }
 
-export function getApiKeyResult(apiKey: string) {
-  const type = isPublicApiKey(apiKey) ? ("PUBLIC" as const) : ("PRIVATE" as const);
+export function getApiKeyResult(apiKey: string): {
+  apiKey: string;
+  type: "PUBLIC" | "PRIVATE" | "PUBLIC_JWT";
+} {
+  const type = isPublicApiKey(apiKey)
+    ? "PUBLIC"
+    : isSecretApiKey(apiKey)
+    ? "PRIVATE"
+    : isPublicJWT(apiKey)
+    ? "PUBLIC_JWT"
+    : "PRIVATE"; // Fallback to private key
   return { apiKey, type };
 }
 

apps/webapp/app/services/apiRateLimit.server.ts

@@ -1,150 +1,48 @@
-import { Ratelimit } from "@upstash/ratelimit";
-import { Request as ExpressRequest, Response as ExpressResponse, NextFunction } from "express";
-import { RedisOptions } from "ioredis";
-import { createHash } from "node:crypto";
 import { env } from "~/env.server";
-import { logger } from "./logger.server";
-import { Duration, Limiter, RateLimiter, createRedisRateLimitClient } from "./rateLimiter.server";
-
-type Options = {
-  redis?: RedisOptions;
-  keyPrefix: string;
-  pathMatchers: (RegExp | string)[];
-  pathWhiteList?: (RegExp | string)[];
-  limiter: Limiter;
-  log?: {
-    requests?: boolean;
-    rejections?: boolean;
-  };
-};
-
-//returns an Express middleware that rate limits using the Bearer token in the Authorization header
-export function authorizationRateLimitMiddleware({
-  redis,
-  keyPrefix,
-  limiter,
-  pathMatchers,
-  pathWhiteList = [],
-  log = {
-    rejections: true,
-    requests: true,
-  },
-}: Options) {
-  const rateLimiter = new RateLimiter({
-    redis,
-    keyPrefix,
-    limiter,
-    logSuccess: log.requests,
-    logFailure: log.rejections,
-  });
-
-  return async (req: ExpressRequest, res: ExpressResponse, next: NextFunction) => {
-    if (log.requests) {
-      logger.info(`RateLimiter (${keyPrefix}): request to ${req.path}`);
-    }
-
-    // allow OPTIONS requests
-    if (req.method.toUpperCase() === "OPTIONS") {
-      return next();
-    }
-
-    //first check if any of the pathMatchers match the request path
-    const path = req.path;
-    if (
-      !pathMatchers.some((matcher) =>
-        matcher instanceof RegExp ? matcher.test(path) : path === matcher
-      )
-    ) {
-      if (log.requests) {
-        logger.info(`RateLimiter (${keyPrefix}): didn't match ${req.path}`);
-      }
-      return next();
-    }
-
-    // Check if the path matches any of the whitelisted paths
-    if (
-      pathWhiteList.some((matcher) =>
-        matcher instanceof RegExp ? matcher.test(path) : path === matcher
-      )
-    ) {
-      if (log.requests) {
-        logger.info(`RateLimiter (${keyPrefix}): whitelisted ${req.path}`);
-      }
-      return next();
-    }
-
-    if (log.requests) {
-      logger.info(`RateLimiter (${keyPrefix}): matched ${req.path}`);
-    }
-
-    const authorizationValue = req.headers.authorization;
-    if (!authorizationValue) {
-      if (log.requests) {
-        logger.info(`RateLimiter (${keyPrefix}): no key`, { headers: req.headers, url: req.url });
-      }
-      res.setHeader("Content-Type", "application/problem+json");
-      return res.status(401).send(
-        JSON.stringify(
-          {
-            title: "Unauthorized",
-            status: 401,
-            type: "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/401",
-            detail: "No authorization header provided",
-            error: "No authorization header provided",
-          },
-          null,
-          2
-        )
-      );
-    }
-
-    const hash = createHash("sha256");
-    hash.update(authorizationValue);
-    const hashedAuthorizationValue = hash.digest("hex");
-
-    const { success, pending, limit, reset, remaining } = await rateLimiter.limit(
-      hashedAuthorizationValue
-    );
-
-    const $remaining = Math.max(0, remaining); // remaining can be negative if the user has exceeded the limit, so clamp it to 0
-
-    res.set("x-ratelimit-limit", limit.toString());
-    res.set("x-ratelimit-remaining", $remaining.toString());
-    res.set("x-ratelimit-reset", reset.toString());
-
-    if (success) {
-      return next();
-    }
-
-    res.setHeader("Content-Type", "application/problem+json");
-    const secondsUntilReset = Math.max(0, (reset - new Date().getTime()) / 1000);
-    return res.status(429).send(
-      JSON.stringify(
-        {
-          title: "Rate Limit Exceeded",
-          status: 429,
-          type: "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429",
-          detail: `Rate limit exceeded ${$remaining}/${limit} requests remaining. Retry in ${secondsUntilReset} seconds.`,
-          reset,
-          limit,
-          remaining,
-          secondsUntilReset,
-          error: `Rate limit exceeded ${$remaining}/${limit} requests remaining. Retry in ${secondsUntilReset} seconds.`,
-        },
-        null,
-        2
-      )
-    );
-  };
-}
+import { authenticateAuthorizationHeader } from "./apiAuth.server";
+import { authorizationRateLimitMiddleware } from "./authorizationRateLimitMiddleware.server";
+import { Duration } from "./rateLimiter.server";
 
 export const apiRateLimiter = authorizationRateLimitMiddleware({
+  redis: {
+    port: env.REDIS_PORT,
+    host: env.REDIS_HOST,
+    username: env.REDIS_USERNAME,
+    password: env.REDIS_PASSWORD,
+    enableAutoPipelining: true,
+    ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+  },
   keyPrefix: "api",
-  limiter: Ratelimit.tokenBucket(
-    env.API_RATE_LIMIT_REFILL_RATE,
-    env.API_RATE_LIMIT_REFILL_INTERVAL as Duration,
-    env.API_RATE_LIMIT_MAX
-  ),
+  defaultLimiter: {
+    type: "tokenBucket",
+    refillRate: env.API_RATE_LIMIT_REFILL_RATE,
+    interval: env.API_RATE_LIMIT_REFILL_INTERVAL as Duration,
+    maxTokens: env.API_RATE_LIMIT_MAX,
+  },
+  limiterCache: {
+    fresh: 60_000 * 10, // Data is fresh for 10 minutes
+    stale: 60_000 * 20, // Date is stale after 20 minutes
+  },
+  limiterConfigOverride: async (authorizationValue) => {
+    const authenticatedEnv = await authenticateAuthorizationHeader(authorizationValue, {
+      allowPublicKey: true,
+      allowJWT: true,
+    });
+
+    if (!authenticatedEnv) {
+      return;
+    }
+
+    if (authenticatedEnv.type === "PUBLIC_JWT") {
+      return {
+        type: "fixedWindow",
+        window: env.API_RATE_LIMIT_JWT_WINDOW,
+        tokens: env.API_RATE_LIMIT_JWT_TOKENS,
+      };
+    } else {
+      return authenticatedEnv.environment.organization.apiRateLimiterConfig;
+    }
+  },
   pathMatchers: [/^\/api/],
   // Allow /api/v1/tasks/:id/callback/:secret
   pathWhiteList: [
@@ -159,11 +57,13 @@ export const apiRateLimiter = authorizationRateLimitMiddleware({
     /^\/api\/v1\/endpoints\/[^\/]+\/[^\/]+\/index\/[^\/]+$/, // /api/v1/endpoints/$environmentId/$endpointSlug/index/$indexHookIdentifier
     "/api/v1/timezones",
     "/api/v1/usage/ingest",
+    "/api/v1/auth/jwt/claims",
     /^\/api\/v1\/runs\/[^\/]+\/attempts$/, // /api/v1/runs/$runFriendlyId/attempts
   ],
   log: {
     rejections: env.API_RATE_LIMIT_REJECTION_LOGS_ENABLED === "1",
     requests: env.API_RATE_LIMIT_REQUEST_LOGS_ENABLED === "1",
+    limiter: env.API_RATE_LIMIT_LIMITER_LOGS_ENABLED === "1",
   },
 });
 

apps/webapp/app/services/authorization.server.ts

@@ -0,0 +1,103 @@
+export type AuthorizationAction = "read"; // Add more actions as needed
+
+const ResourceTypes = ["tasks", "tags", "runs", "batch"] as const;
+
+export type AuthorizationResources = {
+  [key in (typeof ResourceTypes)[number]]?: string | string[];
+};
+
+export type AuthorizationEntity = {
+  type: "PUBLIC" | "PRIVATE" | "PUBLIC_JWT";
+  scopes?: string[];
+};
+
+/**
+ * Checks if the given entity is authorized to perform a specific action on a resource.
+ *
+ * @param entity - The entity requesting authorization.
+ * @param action - The action the entity wants to perform.
+ * @param resource - The resource on which the action is to be performed.
+ * @param superScopes - An array of super scopes that can bypass the normal authorization checks.
+ *
+ * @example
+ *
+ * ```typescript
+ * import { checkAuthorization } from "./authorization.server";
+ *
+ * const entity = {
+ *  type: "PUBLIC",
+ *  scope: ["read:runs:run_1234", "read:tasks"]
+ * };
+ *
+ * checkAuthorization(entity, "read", { runs: "run_1234" }); // Returns true
+ * checkAuthorization(entity, "read", { runs: "run_5678" }); // Returns false
+ * checkAuthorization(entity, "read", { tasks: "task_1234" }); // Returns true
+ * checkAuthorization(entity, "read", { tasks: ["task_5678"] }); // Returns true
+ * ```
+ */
+export function checkAuthorization(
+  entity: AuthorizationEntity,
+  action: AuthorizationAction,
+  resource: AuthorizationResources,
+  superScopes?: string[]
+) {
+  // "PRIVATE" is a secret key and has access to everything
+  if (entity.type === "PRIVATE") {
+    return true;
+  }
+
+  // "PUBLIC" is a deprecated key and has no access
+  if (entity.type === "PUBLIC") {
+    return false;
+  }
+
+  // If the entity has no permissions, deny access
+  if (!entity.scopes || entity.scopes.length === 0) {
+    return false;
+  }
+
+  // If the resource object is empty, deny access
+  if (Object.keys(resource).length === 0) {
+    return false;
+  }
+
+  // Check for any of the super scopes
+  if (superScopes && superScopes.length > 0) {
+    if (superScopes.some((permission) => entity.scopes?.includes(permission))) {
+      return true;
+    }
+  }
+
+  const filteredResource = Object.keys(resource).reduce((acc, key) => {
+    if (ResourceTypes.includes(key)) {
+      acc[key as keyof AuthorizationResources] = resource[key as keyof AuthorizationResources];
+    }
+    return acc;
+  }, {} as AuthorizationResources);
+
+  // Check each resource type
+  for (const [resourceType, resourceValue] of Object.entries(filteredResource)) {
+    const resourceValues = Array.isArray(resourceValue) ? resourceValue : [resourceValue];
+
+    let resourceAuthorized = false;
+    for (const value of resourceValues) {
+      // Check for specific resource permission
+      const specificPermission = `${action}:${resourceType}:${value}`;
+      // Check for general resource type permission
+      const generalPermission = `${action}:${resourceType}`;
+
+      if (entity.scopes.includes(specificPermission) || entity.scopes.includes(generalPermission)) {
+        resourceAuthorized = true;
+        break;
+      }
+    }
+
+    // If any resource is not authorized, return false
+    if (!resourceAuthorized) {
+      return false;
+    }
+  }
+
+  // All resources are authorized
+  return true;
+}

apps/webapp/app/services/authorizationRateLimitMiddleware.server.ts

@@ -0,0 +1,301 @@
+import { createCache, DefaultStatefulContext, Namespace, Cache as UnkeyCache } from "@unkey/cache";
+import { MemoryStore } from "@unkey/cache/stores";
+import { Ratelimit } from "@upstash/ratelimit";
+import { Request as ExpressRequest, Response as ExpressResponse, NextFunction } from "express";
+import { RedisOptions } from "ioredis";
+import { createHash } from "node:crypto";
+import { z } from "zod";
+import { env } from "~/env.server";
+import { logger } from "./logger.server";
+import { createRedisRateLimitClient, Duration, RateLimiter } from "./rateLimiter.server";
+import { RedisCacheStore } from "./unkey/redisCacheStore.server";
+
+const DurationSchema = z.custom<Duration>((value) => {
+  if (typeof value !== "string") {
+    throw new Error("Duration must be a string");
+  }
+
+  return value as Duration;
+});
+
+export const RateLimitFixedWindowConfig = z.object({
+  type: z.literal("fixedWindow"),
+  window: DurationSchema,
+  tokens: z.number(),
+});
+
+export type RateLimitFixedWindowConfig = z.infer<typeof RateLimitFixedWindowConfig>;
+
+export const RateLimitSlidingWindowConfig = z.object({
+  type: z.literal("slidingWindow"),
+  window: DurationSchema,
+  tokens: z.number(),
+});
+
+export type RateLimitSlidingWindowConfig = z.infer<typeof RateLimitSlidingWindowConfig>;
+
+export const RateLimitTokenBucketConfig = z.object({
+  type: z.literal("tokenBucket"),
+  refillRate: z.number(),
+  interval: DurationSchema,
+  maxTokens: z.number(),
+});
+
+export type RateLimitTokenBucketConfig = z.infer<typeof RateLimitTokenBucketConfig>;
+
+export const RateLimiterConfig = z.discriminatedUnion("type", [
+  RateLimitFixedWindowConfig,
+  RateLimitSlidingWindowConfig,
+  RateLimitTokenBucketConfig,
+]);
+
+export type RateLimiterConfig = z.infer<typeof RateLimiterConfig>;
+
+type LimitConfigOverrideFunction = (authorizationValue: string) => Promise<unknown>;
+
+type Options = {
+  redis?: RedisOptions;
+  keyPrefix: string;
+  pathMatchers: (RegExp | string)[];
+  pathWhiteList?: (RegExp | string)[];
+  defaultLimiter: RateLimiterConfig;
+  limiterConfigOverride?: LimitConfigOverrideFunction;
+  limiterCache?: {
+    fresh: number;
+    stale: number;
+  };
+  log?: {
+    requests?: boolean;
+    rejections?: boolean;
+    limiter?: boolean;
+  };
+};
+
+async function resolveLimitConfig(
+  authorizationValue: string,
+  hashedAuthorizationValue: string,
+  defaultLimiter: RateLimiterConfig,
+  cache: UnkeyCache<{ limiter: RateLimiterConfig }>,
+  logsEnabled: boolean,
+  limiterConfigOverride?: LimitConfigOverrideFunction
+): Promise<RateLimiterConfig> {
+  if (!limiterConfigOverride) {
+    return defaultLimiter;
+  }
+
+  if (logsEnabled) {
+    logger.info("RateLimiter: checking for override", {
+      authorizationValue: hashedAuthorizationValue,
+      defaultLimiter,
+    });
+  }
+
+  const cacheResult = await cache.limiter.swr(hashedAuthorizationValue, async (key) => {
+    const override = await limiterConfigOverride(authorizationValue);
+
+    if (!override) {
+      if (logsEnabled) {
+        logger.info("RateLimiter: no override found", {
+          authorizationValue,
+          defaultLimiter,
+        });
+      }
+
+      return defaultLimiter;
+    }
+
+    const parsedOverride = RateLimiterConfig.safeParse(override);
+
+    if (!parsedOverride.success) {
+      logger.error("Error parsing rate limiter override", {
+        override,
+        errors: parsedOverride.error.errors,
+      });
+
+      return defaultLimiter;
+    }
+
+    if (logsEnabled && parsedOverride.data) {
+      logger.info("RateLimiter: override found", {
+        authorizationValue,
+        defaultLimiter,
+        override: parsedOverride.data,
+      });
+    }
+
+    return parsedOverride.data;
+  });
+
+  return cacheResult.val ?? defaultLimiter;
+}
+
+//returns an Express middleware that rate limits using the Bearer token in the Authorization header
+export function authorizationRateLimitMiddleware({
+  redis,
+  keyPrefix,
+  defaultLimiter,
+  pathMatchers,
+  pathWhiteList = [],
+  log = {
+    rejections: true,
+    requests: true,
+  },
+  limiterCache,
+  limiterConfigOverride,
+}: Options) {
+  const ctx = new DefaultStatefulContext();
+  const memory = new MemoryStore({ persistentMap: new Map() });
+  const redisCacheStore = new RedisCacheStore({
+    connection: {
+      keyPrefix: `cache:${keyPrefix}:rate-limit-cache:`,
+      ...redis,
+    },
+  });
+
+  // This cache holds the rate limit configuration for each org, so we don't have to fetch it every request
+  const cache = createCache({
+    limiter: new Namespace<RateLimiterConfig>(ctx, {
+      stores: [memory, redisCacheStore],
+      fresh: limiterCache?.fresh ?? 30_000,
+      stale: limiterCache?.stale ?? 60_000,
+    }),
+  });
+
+  const redisClient = createRedisRateLimitClient(
+    redis ?? {
+      port: env.REDIS_PORT,
+      host: env.REDIS_HOST,
+      username: env.REDIS_USERNAME,
+      password: env.REDIS_PASSWORD,
+      enableAutoPipelining: true,
+      ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+    }
+  );
+
+  return async (req: ExpressRequest, res: ExpressResponse, next: NextFunction) => {
+    if (log.requests) {
+      logger.info(`RateLimiter (${keyPrefix}): request to ${req.path}`);
+    }
+
+    // allow OPTIONS requests
+    if (req.method.toUpperCase() === "OPTIONS") {
+      return next();
+    }
+
+    //first check if any of the pathMatchers match the request path
+    const path = req.path;
+    if (
+      !pathMatchers.some((matcher) =>
+        matcher instanceof RegExp ? matcher.test(path) : path === matcher
+      )
+    ) {
+      if (log.requests) {
+        logger.info(`RateLimiter (${keyPrefix}): didn't match ${req.path}`);
+      }
+      return next();
+    }
+
+    // Check if the path matches any of the whitelisted paths
+    if (
+      pathWhiteList.some((matcher) =>
+        matcher instanceof RegExp ? matcher.test(path) : path === matcher
+      )
+    ) {
+      if (log.requests) {
+        logger.info(`RateLimiter (${keyPrefix}): whitelisted ${req.path}`);
+      }
+      return next();
+    }
+
+    if (log.requests) {
+      logger.info(`RateLimiter (${keyPrefix}): matched ${req.path}`);
+    }
+
+    const authorizationValue = req.headers.authorization;
+    if (!authorizationValue) {
+      if (log.requests) {
+        logger.info(`RateLimiter (${keyPrefix}): no key`, { headers: req.headers, url: req.url });
+      }
+      res.setHeader("Content-Type", "application/problem+json");
+      return res.status(401).send(
+        JSON.stringify(
+          {
+            title: "Unauthorized",
+            status: 401,
+            type: "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/401",
+            detail: "No authorization header provided",
+            error: "No authorization header provided",
+          },
+          null,
+          2
+        )
+      );
+    }
+
+    const hash = createHash("sha256");
+    hash.update(authorizationValue);
+    const hashedAuthorizationValue = hash.digest("hex");
+
+    const limiterConfig = await resolveLimitConfig(
+      authorizationValue,
+      hashedAuthorizationValue,
+      defaultLimiter,
+      cache,
+      typeof log.limiter === "boolean" ? log.limiter : false,
+      limiterConfigOverride
+    );
+
+    const limiter =
+      limiterConfig.type === "fixedWindow"
+        ? Ratelimit.fixedWindow(limiterConfig.tokens, limiterConfig.window)
+        : limiterConfig.type === "tokenBucket"
+        ? Ratelimit.tokenBucket(
+            limiterConfig.refillRate,
+            limiterConfig.interval,
+            limiterConfig.maxTokens
+          )
+        : Ratelimit.slidingWindow(limiterConfig.tokens, limiterConfig.window);
+
+    const rateLimiter = new RateLimiter({
+      redisClient,
+      keyPrefix,
+      limiter,
+      logSuccess: log.requests,
+      logFailure: log.rejections,
+    });
+
+    const { success, limit, reset, remaining } = await rateLimiter.limit(hashedAuthorizationValue);
+
+    const $remaining = Math.max(0, remaining); // remaining can be negative if the user has exceeded the limit, so clamp it to 0
+
+    res.set("x-ratelimit-limit", limit.toString());
+    res.set("x-ratelimit-remaining", $remaining.toString());
+    res.set("x-ratelimit-reset", reset.toString());
+
+    if (success) {
+      return next();
+    }
+
+    res.setHeader("Content-Type", "application/problem+json");
+    const secondsUntilReset = Math.max(0, (reset - new Date().getTime()) / 1000);
+    return res.status(429).send(
+      JSON.stringify(
+        {
+          title: "Rate Limit Exceeded",
+          status: 429,
+          type: "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429",
+          detail: `Rate limit exceeded ${$remaining}/${limit} requests remaining. Retry in ${secondsUntilReset} seconds.`,
+          reset,
+          limit,
+          remaining,
+          secondsUntilReset,
+          error: `Rate limit exceeded ${$remaining}/${limit} requests remaining. Retry in ${secondsUntilReset} seconds.`,
+        },
+        null,
+        2
+      )
+    );
+  };
+}
+
+export type RateLimitMiddleware = ReturnType<typeof authorizationRateLimitMiddleware>;

apps/webapp/app/services/platform.v3.server.ts

@@ -1,5 +1,13 @@
-import { BillingClient, Limits, SetPlanBody, UsageSeriesParams } from "@trigger.dev/platform/v3";
 import { Organization, Project } from "@trigger.dev/database";
+import {
+  BillingClient,
+  Limits,
+  SetPlanBody,
+  UsageSeriesParams,
+  UsageResult,
+} from "@trigger.dev/platform/v3";
+import { createCache, DefaultStatefulContext, Namespace } from "@unkey/cache";
+import { MemoryStore } from "@unkey/cache/stores";
 import { redirect } from "remix-typedjson";
 import { $replica } from "~/db.server";
 import { env } from "~/env.server";
@@ -7,10 +15,61 @@ import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/m
 import { createEnvironment } from "~/models/organization.server";
 import { logger } from "~/services/logger.server";
 import { newProjectPath, organizationBillingPath } from "~/utils/pathBuilder";
+import { singleton } from "~/utils/singleton";
+import { RedisCacheStore } from "./unkey/redisCacheStore.server";
+
+function initializeClient() {
+  if (isCloud() && process.env.BILLING_API_URL && process.env.BILLING_API_KEY) {
+    const client = new BillingClient({
+      url: process.env.BILLING_API_URL,
+      apiKey: process.env.BILLING_API_KEY,
+    });
+    console.log(`🤑 Billing client initialized: ${process.env.BILLING_API_URL}`);
+    return client;
+  } else {
+    console.log(`🤑 Billing client not initialized`);
+  }
+}
+
+const client = singleton("billingClient", initializeClient);
+
+function initializePlatformCache() {
+  const ctx = new DefaultStatefulContext();
+  const memory = new MemoryStore({ persistentMap: new Map() });
+  const redisCacheStore = new RedisCacheStore({
+    connection: {
+      keyPrefix: "tr:cache:platform:v3",
+      port: env.REDIS_PORT,
+      host: env.REDIS_HOST,
+      username: env.REDIS_USERNAME,
+      password: env.REDIS_PASSWORD,
+      enableAutoPipelining: true,
+      ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+    },
+  });
+
+  // This cache holds the limits fetched from the platform service
+  const cache = createCache({
+    limits: new Namespace<number>(ctx, {
+      stores: [memory, redisCacheStore],
+      fresh: 60_000 * 5, // 5 minutes
+      stale: 60_000 * 10, // 10 minutes
+    }),
+    usage: new Namespace<UsageResult>(ctx, {
+      stores: [memory, redisCacheStore],
+      fresh: 60_000 * 5, // 5 minutes
+      stale: 60_000 * 10, // 10 minutes
+    }),
+  });
+
+  return cache;
+}
+
+const platformCache = singleton("platformCache", initializePlatformCache);
 
 export async function getCurrentPlan(orgId: string) {
-  const client = getClient();
   if (!client) return undefined;
+
   try {
     const result = await client.currentPlan(orgId);
 
@@ -60,8 +119,8 @@ export async function getCurrentPlan(orgId: string) {
 }
 
 export async function getLimits(orgId: string) {
-  const client = getClient();
   if (!client) return undefined;
+
   try {
     const result = await client.currentPlan(orgId);
     if (!result.success) {
@@ -87,9 +146,15 @@ export async function getLimit(orgId: string, limit: keyof Limits, fallback: num
   return fallback;
 }
 
+export async function getCachedLimit(orgId: string, limit: keyof Limits, fallback: number) {
+  return platformCache.limits.swr(`${orgId}:${limit}`, async () => {
+    return getLimit(orgId, limit, fallback);
+  });
+}
+
 export async function customerPortalUrl(orgId: string, orgSlug: string) {
-  const client = getClient();
   if (!client) return undefined;
+
   try {
     return client.createPortalSession(orgId, {
       returnUrl: `${env.APP_ORIGIN}${organizationBillingPath({ slug: orgSlug })}`,
@@ -101,8 +166,8 @@ export async function customerPortalUrl(orgId: string, orgSlug: string) {
 }
 
 export async function getPlans() {
-  const client = getClient();
   if (!client) return undefined;
+
   try {
     const result = await client.plans();
     if (!result.success) {
@@ -122,7 +187,6 @@ export async function setPlan(
   callerPath: string,
   plan: SetPlanBody
 ) {
-  const client = getClient();
   if (!client) {
     throw redirectWithErrorMessage(callerPath, request, "Error setting plan");
   }
@@ -178,8 +242,8 @@ export async function setPlan(
 }
 
 export async function getUsage(organizationId: string, { from, to }: { from: Date; to: Date }) {
-  const client = getClient();
   if (!client) return undefined;
+
   try {
     const result = await client.usage(organizationId, { from, to });
     if (!result.success) {
@@ -193,9 +257,27 @@ export async function getUsage(organizationId: string, { from, to }: { from: Dat
   }
 }
 
-export async function getUsageSeries(organizationId: string, params: UsageSeriesParams) {
-  const client = getClient();
+export async function getCachedUsage(
+  organizationId: string,
+  { from, to }: { from: Date; to: Date }
+) {
   if (!client) return undefined;
+
+  const result = await platformCache.usage.swr(
+    `${organizationId}:${from.toISOString()}:${to.toISOString()}`,
+    async () => {
+      const usageResponse = await getUsage(organizationId, { from, to });
+
+      return usageResponse;
+    }
+  );
+
+  return result.val;
+}
+
+export async function getUsageSeries(organizationId: string, params: UsageSeriesParams) {
+  if (!client) return undefined;
+
   try {
     const result = await client.usageSeries(organizationId, params);
     if (!result.success) {
@@ -214,8 +296,8 @@ export async function reportInvocationUsage(
   costInCents: number,
   additionalData?: Record<string, any>
 ) {
-  const client = getClient();
   if (!client) return undefined;
+
   try {
     const result = await client.reportInvocationUsage({
       organizationId,
@@ -234,8 +316,8 @@ export async function reportInvocationUsage(
 }
 
 export async function reportComputeUsage(request: Request) {
-  const client = getClient();
   if (!client) return undefined;
+
   return fetch(`${process.env.BILLING_API_URL}/api/v1/usage/ingest/compute`, {
     method: "POST",
     headers: request.headers,
@@ -244,8 +326,8 @@ export async function reportComputeUsage(request: Request) {
 }
 
 export async function getEntitlement(organizationId: string) {
-  const client = getClient();
   if (!client) return undefined;
+
   try {
     const result = await client.getEntitlement(organizationId);
     if (!result.success) {
@@ -275,19 +357,6 @@ export async function projectCreated(organization: Organization, project: Projec
   }
 }
 
-function getClient() {
-  if (isCloud() && process.env.BILLING_API_URL && process.env.BILLING_API_KEY) {
-    const client = new BillingClient({
-      url: process.env.BILLING_API_URL,
-      apiKey: process.env.BILLING_API_KEY,
-    });
-    console.log(`Billing client initialized: ${process.env.BILLING_API_URL}`);
-    return client;
-  } else {
-    console.log(`Billing client not initialized`);
-  }
-}
-
 function isCloud(): boolean {
   const acceptableHosts = [
     "https://cloud.trigger.dev",

apps/webapp/app/services/rateLimiter.server.ts

@@ -5,6 +5,7 @@ import { logger } from "./logger.server";
 
 type Options = {
   redis?: RedisOptions;
+  redisClient?: RateLimiterRedisClient;
   keyPrefix: string;
   limiter: Limiter;
   logSuccess?: boolean;
@@ -14,34 +15,32 @@ type Options = {
 export type Limiter = ConstructorParameters<typeof Ratelimit>[0]["limiter"];
 export type Duration = Parameters<typeof Ratelimit.slidingWindow>[1];
 export type RateLimitResponse = Awaited<ReturnType<Ratelimit["limit"]>>;
+export type RateLimiterRedisClient = ConstructorParameters<typeof Ratelimit>[0]["redis"];
 
 export class RateLimiter {
   #ratelimit: Ratelimit;
 
   constructor(private readonly options: Options) {
-    const { redis, keyPrefix, limiter } = options;
+    const { redis, redisClient, keyPrefix, limiter } = options;
     const prefix = `ratelimit:${keyPrefix}`;
     this.#ratelimit = new Ratelimit({
-      redis: createRedisRateLimitClient(
-        redis ?? {
-          port: env.REDIS_PORT,
-          host: env.REDIS_HOST,
-          username: env.REDIS_USERNAME,
-          password: env.REDIS_PASSWORD,
-          enableAutoPipelining: true,
-          ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
-        }
-      ),
+      redis:
+        redisClient ??
+        createRedisRateLimitClient(
+          redis ?? {
+            port: env.REDIS_PORT,
+            host: env.REDIS_HOST,
+            username: env.REDIS_USERNAME,
+            password: env.REDIS_PASSWORD,
+            enableAutoPipelining: true,
+            ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+          }
+        ),
       limiter,
       ephemeralCache: new Map(),
       analytics: false,
       prefix,
     });
-
-    logger.info(`RateLimiter (${keyPrefix}): initialized`, {
-      keyPrefix,
-      redisKeyspace: prefix,
-    });
   }
 
   async limit(identifier: string, rate = 1): Promise<RateLimitResponse> {
@@ -71,9 +70,7 @@ export class RateLimiter {
   }
 }
 
-export function createRedisRateLimitClient(
-  redisOptions: RedisOptions
-): ConstructorParameters<typeof Ratelimit>[0]["redis"] {
+export function createRedisRateLimitClient(redisOptions: RedisOptions): RateLimiterRedisClient {
   const redis = new Redis(redisOptions);
 
   return {

apps/webapp/app/services/realtime/jwtAuth.server.ts

@@ -0,0 +1,85 @@
+import { validateJWT } from "@trigger.dev/core/v3/jwt";
+import { findEnvironmentById } from "~/models/runtimeEnvironment.server";
+
+export async function validatePublicJwtKey(token: string) {
+  // Get the sub claim from the token
+  // Use the sub claim to find the environment
+  // Validate the token against the environment.apiKey
+  // Once that's done, return the environment and the claims
+  const sub = extractJWTSub(token);
+
+  if (!sub) {
+    return;
+  }
+
+  const environment = await findEnvironmentById(sub);
+
+  if (!environment) {
+    return;
+  }
+
+  const claims = await validateJWT(token, environment.apiKey);
+
+  if (!claims) {
+    return;
+  }
+
+  return {
+    environment,
+    claims,
+  };
+}
+
+export function isPublicJWT(token: string): boolean {
+  // Split the token
+  const parts = token.split(".");
+  if (parts.length !== 3) return false;
+
+  try {
+    // Decode the payload (second part)
+    const payload = JSON.parse(decodeBase64Url(parts[1]));
+
+    if (payload === null || typeof payload !== "object") return false;
+
+    // Check for the pub: true claim
+    return "pub" in payload && payload.pub === true;
+  } catch (error) {
+    // If there's any error in decoding or parsing, it's not a valid JWT
+    return false;
+  }
+}
+
+function extractJWTSub(token: string): string | undefined {
+  // Split the token
+  const parts = token.split(".");
+  if (parts.length !== 3) return;
+
+  try {
+    // Decode the payload (second part)
+    const payload = JSON.parse(decodeBase64Url(parts[1]));
+
+    if (payload === null || typeof payload !== "object") return;
+
+    // Check for the pub: true claim
+    return "sub" in payload && typeof payload.sub === "string" ? payload.sub : undefined;
+  } catch (error) {
+    // If there's any error in decoding or parsing, it's not a valid JWT
+    return;
+  }
+}
+
+function decodeBase64Url(str: string): string {
+  // Replace URL-safe characters and add padding
+  str = str.replace(/-/g, "+").replace(/_/g, "/");
+  switch (str.length % 4) {
+    case 2:
+      str += "==";
+      break;
+    case 3:
+      str += "=";
+      break;
+  }
+
+  // Decode using Node.js Buffer
+  return Buffer.from(str, "base64").toString("utf8");
+}

apps/webapp/app/services/realtimeClient.server.ts

@@ -0,0 +1,253 @@
+import { json } from "@remix-run/server-runtime";
+import Redis, { Callback, Result, type RedisOptions } from "ioredis";
+import { randomUUID } from "node:crypto";
+import { longPollingFetch } from "~/utils/longPollingFetch";
+import { logger } from "./logger.server";
+
+export interface CachedLimitProvider {
+  getCachedLimit: (organizationId: string, defaultValue: number) => Promise<number | undefined>;
+}
+
+export type RealtimeClientOptions = {
+  electricOrigin: string;
+  redis: RedisOptions;
+  cachedLimitProvider: CachedLimitProvider;
+  keyPrefix: string;
+  expiryTimeInSeconds?: number;
+};
+
+export type RealtimeEnvironment = {
+  id: string;
+  organizationId: string;
+};
+
+export type RealtimeRunsParams = {
+  tags?: string[];
+};
+
+export class RealtimeClient {
+  private redis: Redis;
+  private expiryTimeInSeconds: number;
+  private cachedLimitProvider: CachedLimitProvider;
+
+  constructor(private options: RealtimeClientOptions) {
+    this.redis = new Redis(options.redis);
+    this.expiryTimeInSeconds = options.expiryTimeInSeconds ?? 60 * 5; // default to 5 minutes
+    this.cachedLimitProvider = options.cachedLimitProvider;
+    this.#registerCommands();
+  }
+
+  async streamRun(url: URL | string, environment: RealtimeEnvironment, runId: string) {
+    return this.#streamRunsWhere(url, environment, `id='${runId}'`);
+  }
+
+  async streamBatch(url: URL | string, environment: RealtimeEnvironment, batchId: string) {
+    return this.#streamRunsWhere(url, environment, `"batchId"='${batchId}'`);
+  }
+
+  async streamRuns(
+    url: URL | string,
+    environment: RealtimeEnvironment,
+    params: RealtimeRunsParams
+  ) {
+    const whereClauses: string[] = [`"runtimeEnvironmentId"='${environment.id}'`];
+
+    if (params.tags) {
+      whereClauses.push(`"runTags" @> ARRAY[${params.tags.map((t) => `'${t}'`).join(",")}]`);
+    }
+
+    const whereClause = whereClauses.join(" AND ");
+
+    return this.#streamRunsWhere(url, environment, whereClause);
+  }
+
+  async #streamRunsWhere(url: URL | string, environment: RealtimeEnvironment, whereClause: string) {
+    const electricUrl = this.#constructElectricUrl(url, whereClause);
+
+    return this.#performElectricRequest(electricUrl, environment);
+  }
+
+  #constructElectricUrl(url: URL | string, whereClause: string): URL {
+    const $url = new URL(url.toString());
+
+    const electricUrl = new URL(`${this.options.electricOrigin}/v1/shape/public."TaskRun"`);
+
+    // Copy over all the url search params to the electric url
+    $url.searchParams.forEach((value, key) => {
+      electricUrl.searchParams.set(key, value);
+    });
+
+    // const electricParams = ["shape_id", "live", "offset", "columns", "cursor"];
+
+    // electricParams.forEach((param) => {
+    //   if ($url.searchParams.has(param) && $url.searchParams.get(param)) {
+    //     electricUrl.searchParams.set(param, $url.searchParams.get(param)!);
+    //   }
+    // });
+
+    electricUrl.searchParams.set("where", whereClause);
+
+    return electricUrl;
+  }
+
+  async #performElectricRequest(url: URL, environment: RealtimeEnvironment) {
+    const shapeId = extractShapeId(url);
+
+    logger.debug("[realtimeClient] request", {
+      url: url.toString(),
+    });
+
+    if (!shapeId) {
+      // If the shapeId is not present, we're just getting the initial value
+      return longPollingFetch(url.toString());
+    }
+
+    const isLive = isLiveRequestUrl(url);
+
+    if (!isLive) {
+      return longPollingFetch(url.toString());
+    }
+
+    const requestId = randomUUID();
+
+    // We now need to wrap the longPollingFetch in a concurrency tracker
+    const concurrencyLimit = await this.cachedLimitProvider.getCachedLimit(
+      environment.organizationId,
+      100_000
+    );
+
+    if (!concurrencyLimit) {
+      logger.error("Failed to get concurrency limit", {
+        organizationId: environment.organizationId,
+      });
+
+      return json({ error: "Failed to get concurrency limit" }, { status: 500 });
+    }
+
+    logger.debug("[realtimeClient] increment and check", {
+      concurrencyLimit,
+      shapeId,
+      requestId,
+      environment: {
+        id: environment.id,
+        organizationId: environment.organizationId,
+      },
+    });
+
+    const canProceed = await this.#incrementAndCheck(environment.id, requestId, concurrencyLimit);
+
+    if (!canProceed) {
+      logger.debug("[realtimeClient] too many concurrent requests", {
+        requestId,
+        environmentId: environment.id,
+      });
+
+      return json({ error: "Too many concurrent requests" }, { status: 429 });
+    }
+
+    try {
+      // ... (rest of your existing code for the long polling request)
+      const response = await longPollingFetch(url.toString());
+
+      // Decrement the counter after the long polling request is complete
+      await this.#decrementConcurrency(environment.id, requestId);
+
+      return response;
+    } catch (error) {
+      // Decrement the counter if the request fails
+      await this.#decrementConcurrency(environment.id, requestId);
+
+      throw error;
+    }
+  }
+
+  async #incrementAndCheck(environmentId: string, requestId: string, limit: number) {
+    const key = this.#getKey(environmentId);
+    const now = Date.now();
+
+    const result = await this.redis.incrementAndCheckConcurrency(
+      key,
+      now.toString(),
+      requestId,
+      this.expiryTimeInSeconds.toString(), // expiry time
+      (now - this.expiryTimeInSeconds * 1000).toString(), // cutoff time
+      limit.toString()
+    );
+
+    return result === 1;
+  }
+
+  async #decrementConcurrency(environmentId: string, requestId: string) {
+    logger.debug("[realtimeClient] decrement", {
+      requestId,
+      environmentId,
+    });
+
+    const key = this.#getKey(environmentId);
+
+    await this.redis.zrem(key, requestId);
+  }
+
+  #getKey(environmentId: string): string {
+    return `${this.options.keyPrefix}:${environmentId}`;
+  }
+
+  #registerCommands() {
+    this.redis.defineCommand("incrementAndCheckConcurrency", {
+      numberOfKeys: 1,
+      lua: /* lua */ `
+        local concurrencyKey = KEYS[1]
+
+        local timestamp = tonumber(ARGV[1])
+        local requestId = ARGV[2]
+        local expiryTime = tonumber(ARGV[3])
+        local cutoffTime = tonumber(ARGV[4])
+        local limit = tonumber(ARGV[5])
+
+        -- Remove expired entries
+        redis.call('ZREMRANGEBYSCORE', concurrencyKey, '-inf', cutoffTime)
+
+        -- Add the new request to the sorted set
+        redis.call('ZADD', concurrencyKey, timestamp, requestId)
+
+        -- Set the expiry time on the key
+        redis.call('EXPIRE', concurrencyKey, expiryTime)
+
+        -- Get the total number of concurrent requests
+        local totalRequests = redis.call('ZCARD', concurrencyKey)
+
+        -- Check if the limit has been exceeded
+        if totalRequests > limit then
+            -- Remove the request we just added
+            redis.call('ZREM', concurrencyKey, requestId)
+            return 0
+        end
+
+        -- Return 1 to indicate success
+        return 1
+      `,
+    });
+  }
+}
+
+function extractShapeId(url: URL) {
+  return url.searchParams.get("shape_id");
+}
+
+function isLiveRequestUrl(url: URL) {
+  return url.searchParams.has("live") && url.searchParams.get("live") === "true";
+}
+
+declare module "ioredis" {
+  interface RedisCommander<Context> {
+    incrementAndCheckConcurrency(
+      key: string,
+      timestamp: string,
+      requestId: string,
+      expiryTime: string,
+      cutoffTime: string,
+      limit: string,
+      callback?: Callback<number>
+    ): Result<number, Context>;
+  }
+}

apps/webapp/app/services/realtimeClientGlobal.server.ts

@@ -0,0 +1,32 @@
+import { env } from "~/env.server";
+import { singleton } from "~/utils/singleton";
+import { RealtimeClient } from "./realtimeClient.server";
+import { getCachedLimit } from "./platform.v3.server";
+
+function initializeRealtimeClient() {
+  return new RealtimeClient({
+    electricOrigin: env.ELECTRIC_ORIGIN,
+    keyPrefix: "tr:realtime:concurrency",
+    redis: {
+      port: env.REDIS_PORT,
+      host: env.REDIS_HOST,
+      username: env.REDIS_USERNAME,
+      password: env.REDIS_PASSWORD,
+      enableAutoPipelining: true,
+      ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+    },
+    cachedLimitProvider: {
+      async getCachedLimit(organizationId, defaultValue) {
+        const result = await getCachedLimit(
+          organizationId,
+          "realtimeConcurrentConnections",
+          defaultValue
+        );
+
+        return result.val;
+      },
+    },
+  });
+}
+
+export const realtimeClient = singleton("realtimeClient", initializeRealtimeClient);

apps/webapp/app/services/routeBuiilders/apiBuilder.server.ts

@@ -0,0 +1,260 @@
+import { z } from "zod";
+import { ApiAuthenticationResult, authenticateApiRequest } from "../apiAuth.server";
+import { json, LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { fromZodError } from "zod-validation-error";
+import { apiCors } from "~/utils/apiCors";
+import {
+  AuthorizationAction,
+  AuthorizationResources,
+  checkAuthorization,
+} from "../authorization.server";
+import { logger } from "../logger.server";
+import {
+  authenticateApiRequestWithPersonalAccessToken,
+  PersonalAccessTokenAuthenticationResult,
+} from "../personalAccessToken.server";
+
+type ApiKeyRouteBuilderOptions<
+  TParamsSchema extends z.AnyZodObject | undefined = undefined,
+  TSearchParamsSchema extends z.AnyZodObject | undefined = undefined
+> = {
+  params?: TParamsSchema;
+  searchParams?: TSearchParamsSchema;
+  allowJWT?: boolean;
+  corsStrategy?: "all" | "none";
+  authorization?: {
+    action: AuthorizationAction;
+    resource: (
+      params: TParamsSchema extends z.AnyZodObject ? z.infer<TParamsSchema> : undefined,
+      searchParams: TSearchParamsSchema extends z.AnyZodObject
+        ? z.infer<TSearchParamsSchema>
+        : undefined
+    ) => AuthorizationResources;
+    superScopes?: string[];
+  };
+};
+
+type ApiKeyHandlerFunction<
+  TParamsSchema extends z.AnyZodObject | undefined,
+  TSearchParamsSchema extends z.AnyZodObject | undefined
+> = (args: {
+  params: TParamsSchema extends z.AnyZodObject ? z.infer<TParamsSchema> : undefined;
+  searchParams: TSearchParamsSchema extends z.AnyZodObject
+    ? z.infer<TSearchParamsSchema>
+    : undefined;
+  authentication: ApiAuthenticationResult;
+  request: Request;
+}) => Promise<Response>;
+
+export function createLoaderApiRoute<
+  TParamsSchema extends z.AnyZodObject | undefined = undefined,
+  TSearchParamsSchema extends z.AnyZodObject | undefined = undefined
+>(
+  options: ApiKeyRouteBuilderOptions<TParamsSchema, TSearchParamsSchema>,
+  handler: ApiKeyHandlerFunction<TParamsSchema, TSearchParamsSchema>
+) {
+  return async function loader({ request, params }: LoaderFunctionArgs) {
+    const {
+      params: paramsSchema,
+      searchParams: searchParamsSchema,
+      allowJWT = false,
+      corsStrategy = "none",
+      authorization,
+    } = options;
+
+    if (corsStrategy !== "none" && request.method.toUpperCase() === "OPTIONS") {
+      return apiCors(request, json({}));
+    }
+
+    const authenticationResult = await authenticateApiRequest(request, { allowJWT });
+
+    if (!authenticationResult) {
+      return wrapResponse(
+        request,
+        json({ error: "Invalid or Missing API key" }, { status: 401 }),
+        corsStrategy !== "none"
+      );
+    }
+
+    let parsedParams: any = undefined;
+    if (paramsSchema) {
+      const parsed = paramsSchema.safeParse(params);
+      if (!parsed.success) {
+        return wrapResponse(
+          request,
+          json(
+            { error: "Params Error", details: fromZodError(parsed.error).details },
+            { status: 400 }
+          ),
+          corsStrategy !== "none"
+        );
+      }
+      parsedParams = parsed.data;
+    }
+
+    let parsedSearchParams: any = undefined;
+    if (searchParamsSchema) {
+      const searchParams = Object.fromEntries(new URL(request.url).searchParams);
+      const parsed = searchParamsSchema.safeParse(searchParams);
+      if (!parsed.success) {
+        return wrapResponse(
+          request,
+          json(
+            { error: "Query Error", details: fromZodError(parsed.error).details },
+            { status: 400 }
+          ),
+          corsStrategy !== "none"
+        );
+      }
+      parsedSearchParams = parsed.data;
+    }
+
+    if (authorization) {
+      const { action, resource, superScopes } = authorization;
+      const $resource = resource(parsedParams, parsedSearchParams);
+
+      logger.debug("Checking authorization", {
+        action,
+        resource: $resource,
+        superScopes,
+        scopes: authenticationResult.scopes,
+      });
+
+      if (!checkAuthorization(authenticationResult, action, $resource, superScopes)) {
+        return wrapResponse(
+          request,
+          json({ error: "Unauthorized" }, { status: 403 }),
+          corsStrategy !== "none"
+        );
+      }
+    }
+
+    try {
+      const result = await handler({
+        params: parsedParams,
+        searchParams: parsedSearchParams,
+        authentication: authenticationResult,
+        request,
+      });
+      return wrapResponse(request, result, corsStrategy !== "none");
+    } catch (error) {
+      console.error("Error in API route:", error);
+      if (error instanceof Response) {
+        return wrapResponse(request, error, corsStrategy !== "none");
+      }
+      return wrapResponse(
+        request,
+        json({ error: "Internal Server Error" }, { status: 500 }),
+        corsStrategy !== "none"
+      );
+    }
+  };
+}
+
+type PATRouteBuilderOptions<
+  TParamsSchema extends z.AnyZodObject | undefined = undefined,
+  TSearchParamsSchema extends z.AnyZodObject | undefined = undefined
+> = {
+  params?: TParamsSchema;
+  searchParams?: TSearchParamsSchema;
+  corsStrategy?: "all" | "none";
+};
+
+type PATHandlerFunction<
+  TParamsSchema extends z.AnyZodObject | undefined,
+  TSearchParamsSchema extends z.AnyZodObject | undefined
+> = (args: {
+  params: TParamsSchema extends z.AnyZodObject ? z.infer<TParamsSchema> : undefined;
+  searchParams: TSearchParamsSchema extends z.AnyZodObject
+    ? z.infer<TSearchParamsSchema>
+    : undefined;
+  authentication: PersonalAccessTokenAuthenticationResult;
+  request: Request;
+}) => Promise<Response>;
+
+export function createLoaderPATApiRoute<
+  TParamsSchema extends z.AnyZodObject | undefined = undefined,
+  TSearchParamsSchema extends z.AnyZodObject | undefined = undefined
+>(
+  options: PATRouteBuilderOptions<TParamsSchema, TSearchParamsSchema>,
+  handler: PATHandlerFunction<TParamsSchema, TSearchParamsSchema>
+) {
+  return async function loader({ request, params }: LoaderFunctionArgs) {
+    const {
+      params: paramsSchema,
+      searchParams: searchParamsSchema,
+      corsStrategy = "none",
+    } = options;
+
+    if (corsStrategy !== "none" && request.method.toUpperCase() === "OPTIONS") {
+      return apiCors(request, json({}));
+    }
+
+    const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
+
+    if (!authenticationResult) {
+      return wrapResponse(
+        request,
+        json({ error: "Invalid or Missing API key" }, { status: 401 }),
+        corsStrategy !== "none"
+      );
+    }
+
+    let parsedParams: any = undefined;
+    if (paramsSchema) {
+      const parsed = paramsSchema.safeParse(params);
+      if (!parsed.success) {
+        return wrapResponse(
+          request,
+          json(
+            { error: "Params Error", details: fromZodError(parsed.error).details },
+            { status: 400 }
+          ),
+          corsStrategy !== "none"
+        );
+      }
+      parsedParams = parsed.data;
+    }
+
+    let parsedSearchParams: any = undefined;
+    if (searchParamsSchema) {
+      const searchParams = Object.fromEntries(new URL(request.url).searchParams);
+      const parsed = searchParamsSchema.safeParse(searchParams);
+      if (!parsed.success) {
+        return wrapResponse(
+          request,
+          json(
+            { error: "Query Error", details: fromZodError(parsed.error).details },
+            { status: 400 }
+          ),
+          corsStrategy !== "none"
+        );
+      }
+      parsedSearchParams = parsed.data;
+    }
+
+    try {
+      const result = await handler({
+        params: parsedParams,
+        searchParams: parsedSearchParams,
+        authentication: authenticationResult,
+        request,
+      });
+      return wrapResponse(request, result, corsStrategy !== "none");
+    } catch (error) {
+      console.error("Error in API route:", error);
+      if (error instanceof Response) {
+        return wrapResponse(request, error, corsStrategy !== "none");
+      }
+      return wrapResponse(
+        request,
+        json({ error: "Internal Server Error" }, { status: 500 }),
+        corsStrategy !== "none"
+      );
+    }
+  };
+}
+
+function wrapResponse(request: Request, response: Response, useCors: boolean) {
+  return useCors ? apiCors(request, response) : response;
+}

apps/webapp/app/services/unkey/redisCacheStore.server.ts

@@ -0,0 +1,97 @@
+import { Err, Ok, type Result } from "@unkey/error";
+import type { Entry, Store } from "@unkey/cache/stores";
+import type { RedisOptions } from "ioredis";
+import { Redis } from "ioredis";
+import { CacheError } from "@unkey/cache";
+
+export type RedisCacheStoreConfig = {
+  connection: RedisOptions;
+};
+
+export class RedisCacheStore<TNamespace extends string, TValue = any>
+  implements Store<TNamespace, TValue>
+{
+  public readonly name = "redis";
+  private readonly redis: Redis;
+
+  constructor(config: RedisCacheStoreConfig) {
+    this.redis = new Redis(config.connection);
+  }
+
+  private buildCacheKey(namespace: TNamespace, key: string): string {
+    return [namespace, key].join("::");
+  }
+
+  public async get(
+    namespace: TNamespace,
+    key: string
+  ): Promise<Result<Entry<TValue> | undefined, CacheError>> {
+    let raw: string | null;
+    try {
+      raw = await this.redis.get(this.buildCacheKey(namespace, key));
+    } catch (err) {
+      return Err(
+        new CacheError({
+          tier: this.name,
+          key,
+          message: (err as Error).message,
+        })
+      );
+    }
+
+    if (!raw) {
+      return Promise.resolve(Ok(undefined));
+    }
+
+    try {
+      const superjson = await import("superjson");
+      const entry = superjson.parse(raw) as Entry<TValue>;
+      return Ok(entry);
+    } catch (err) {
+      return Err(
+        new CacheError({
+          tier: this.name,
+          key,
+          message: (err as Error).message,
+        })
+      );
+    }
+  }
+
+  public async set(
+    namespace: TNamespace,
+    key: string,
+    entry: Entry<TValue>
+  ): Promise<Result<void, CacheError>> {
+    const cacheKey = this.buildCacheKey(namespace, key);
+    try {
+      const superjson = await import("superjson");
+      await this.redis.set(cacheKey, superjson.stringify(entry), "PXAT", entry.staleUntil);
+      return Ok();
+    } catch (err) {
+      return Err(
+        new CacheError({
+          tier: this.name,
+          key,
+          message: (err as Error).message,
+        })
+      );
+    }
+  }
+
+  public async remove(namespace: TNamespace, key: string): Promise<Result<void, CacheError>> {
+    try {
+      const cacheKey = this.buildCacheKey(namespace, key);
+      await this.redis.del(cacheKey);
+      return Promise.resolve(Ok());
+    } catch (err) {
+      return Err(
+        new CacheError({
+          tier: this.name,
+          key,
+          message: (err as Error).message,
+        })
+      );
+    }
+  }
+}

apps/webapp/app/utils/apiCors.ts

@@ -10,10 +10,25 @@ type CorsOptions = {
   credentials?: boolean;
 };
 
-export function apiCors(
+export async function apiCors(
   request: Request,
   response: Response,
   options: CorsOptions = { maxAge: 5 * 60 }
 ): Promise<Response> {
+  if (hasCorsHeaders(response)) {
+    return response;
+  }
+
   return cors(request, response, options);
 }
+
+export function makeApiCors(
+  request: Request,
+  options: CorsOptions = { maxAge: 5 * 60 }
+): (response: Response) => Promise<Response> {
+  return (response: Response) => apiCors(request, response, options);
+}
+
+function hasCorsHeaders(response: Response) {
+  return response.headers.has("access-control-allow-origin");
+}

apps/webapp/app/utils/longPollingFetch.ts

@@ -10,23 +10,16 @@ export async function longPollingFetch(url: string, options?: RequestInit) {
   try {
     let response = await fetch(url, options);
 
-    // Check if the response is ok (status in the range 200-299)
-    if (!response.ok) {
-      const body = await response.text();
-      throw new Error(`HTTP error! status: ${response.status}. ${body}`);
-    }
-
-    if (response.headers.get(`content-encoding`)) {
+    if (response.headers.get("content-encoding")) {
       const headers = new Headers(response.headers);
-      headers.delete(`content-encoding`);
-      headers.delete(`content-length`);
+      headers.delete("content-encoding");
+      headers.delete("content-length");
       response = new Response(response.body, {
+        headers,
         status: response.status,
         statusText: response.statusText,
-        headers,
       });
     }
-
     return response;
   } catch (error) {
     if (error instanceof TypeError) {

apps/webapp/app/utils/plain.server.ts

@@ -0,0 +1,59 @@
+import { PlainClient, uiComponent } from "@team-plain/typescript-sdk";
+import { env } from "~/env.server";
+
+type Input = {
+  userId: string;
+  email: string;
+  name: string;
+  title: string;
+  components: ReturnType<typeof uiComponent.text>[];
+};
+
+export async function sendToPlain({ userId, email, name, title, components }: Input) {
+  if (!env.PLAIN_API_KEY) {
+    return;
+  }
+
+  const client = new PlainClient({
+    apiKey: env.PLAIN_API_KEY,
+  });
+
+  const upsertCustomerRes = await client.upsertCustomer({
+    identifier: {
+      emailAddress: email,
+    },
+    onCreate: {
+      externalId: userId,
+      fullName: name,
+      email: {
+        email: email,
+        isVerified: true,
+      },
+    },
+    onUpdate: {
+      externalId: { value: userId },
+      fullName: { value: name },
+      email: {
+        email: email,
+        isVerified: true,
+      },
+    },
+  });
+
+  if (upsertCustomerRes.error) {
+    console.error("Failed to upsert customer in Plain", upsertCustomerRes.error);
+    return;
+  }
+
+  const createThreadRes = await client.createThread({
+    customerIdentifier: {
+      customerId: upsertCustomerRes.data.customer.id,
+    },
+    title: title,
+    components: components,
+  });
+
+  if (createThreadRes.error) {
+    console.error("Failed to create thread in Plain", createThreadRes.error);
+  }
+}

apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts

@@ -135,8 +135,8 @@ export class EnvironmentVariablesRepository implements Repository {
     }
 
     try {
-      const result = await $transaction(this.prismaClient, async (tx) => {
-        for (const variable of values) {
+      for (const variable of values) {
+        const result = await $transaction(this.prismaClient, async (tx) => {
           const environmentVariable = await tx.environmentVariable.upsert({
             where: {
               projectId_key: {
@@ -195,8 +195,8 @@ export class EnvironmentVariablesRepository implements Repository {
               secret: variable.value,
             });
           }
-        }
-      });
+        });
+      }
 
       return {
         success: true as const,
@@ -732,7 +732,7 @@ async function resolveBuiltInProdVariables(runtimeEnvironment: RuntimeEnvironmen
     },
     {
       key: "TRIGGER_API_URL",
-      value: env.APP_ORIGIN,
+      value: env.API_ORIGIN ?? env.APP_ORIGIN,
     },
     {
       key: "TRIGGER_RUNTIME_WAIT_THRESHOLD_IN_MS",

apps/webapp/app/v3/failedTaskRun.server.ts

@@ -1,15 +1,41 @@
-import { sanitizeError, TaskRunFailedExecutionResult } from "@trigger.dev/core/v3";
+import {
+  calculateNextRetryDelay,
+  RetryOptions,
+  TaskRunExecution,
+  TaskRunExecutionRetry,
+  TaskRunFailedExecutionResult,
+} from "@trigger.dev/core/v3";
 import { logger } from "~/services/logger.server";
-import { createExceptionPropertiesFromError, eventRepository } from "./eventRepository.server";
 import { BaseService } from "./services/baseService.server";
-import { FinalizeTaskRunService } from "./services/finalizeTaskRun.server";
-import { FAILABLE_RUN_STATUSES } from "./taskStatus";
+import { isFailableRunStatus, isFinalAttemptStatus } from "./taskStatus";
+import type { Prisma, TaskRun } from "@trigger.dev/database";
+import { CompleteAttemptService } from "./services/completeAttempt.server";
+import { CreateTaskRunAttemptService } from "./services/createTaskRunAttempt.server";
+import { sharedQueueTasks } from "./marqs/sharedQueueConsumer.server";
+import * as semver from "semver";
+
+const includeAttempts = {
+  attempts: {
+    orderBy: {
+      createdAt: "desc",
+    },
+    take: 1,
+  },
+  lockedBy: true, // task
+  lockedToVersion: true, // worker
+} satisfies Prisma.TaskRunInclude;
+
+type TaskRunWithAttempts = Prisma.TaskRunGetPayload<{
+  include: typeof includeAttempts;
+}>;
 
 export class FailedTaskRunService extends BaseService {
   public async call(anyRunId: string, completion: TaskRunFailedExecutionResult) {
+    logger.debug("[FailedTaskRunService] Handling failed task run", { anyRunId, completion });
+
     const isFriendlyId = anyRunId.startsWith("run_");
 
-    const taskRun = await this._prisma.taskRun.findUnique({
+    const taskRun = await this._prisma.taskRun.findFirst({
       where: {
         friendlyId: isFriendlyId ? anyRunId : undefined,
         id: !isFriendlyId ? anyRunId : undefined,
@@ -25,7 +51,7 @@ export class FailedTaskRunService extends BaseService {
       return;
     }
 
-    if (!FAILABLE_RUN_STATUSES.includes(taskRun.status)) {
+    if (!isFailableRunStatus(taskRun.status)) {
       logger.error("[FailedTaskRunService] Task run is not in a failable state", {
         taskRun,
         completion,
@@ -34,33 +60,217 @@ export class FailedTaskRunService extends BaseService {
       return;
     }
 
-    // No more retries, we need to fail the task run
-    logger.debug("[FailedTaskRunService] Failing task run", { taskRun, completion });
-
-    const finalizeService = new FinalizeTaskRunService();
-    await finalizeService.call({
-      id: taskRun.id,
-      status: "SYSTEM_FAILURE",
-      completedAt: new Date(),
-      attemptStatus: "FAILED",
-      error: sanitizeError(completion.error),
+    const retryHelper = new FailedTaskRunRetryHelper(this._prisma);
+    const retryResult = await retryHelper.call({
+      runId: taskRun.id,
+      completion,
     });
 
-    // Now we need to "complete" the task run event/span
-    await eventRepository.completeEvent(taskRun.spanId, {
-      endTime: new Date(),
-      attributes: {
-        isError: true,
-      },
-      events: [
-        {
-          name: "exception",
-          time: new Date(),
-          properties: {
-            exception: createExceptionPropertiesFromError(completion.error),
-          },
-        },
-      ],
+    logger.debug("[FailedTaskRunService] Completion result", {
+      runId: taskRun.id,
+      result: retryResult,
     });
   }
 }
+
+interface TaskRunWithWorker extends TaskRun {
+  lockedBy: { retryConfig: Prisma.JsonValue } | null;
+  lockedToVersion: { sdkVersion: string } | null;
+}
+
+export class FailedTaskRunRetryHelper extends BaseService {
+  async call({
+    runId,
+    completion,
+    isCrash,
+  }: {
+    runId: string;
+    completion: TaskRunFailedExecutionResult;
+    isCrash?: boolean;
+  }) {
+    const taskRun = await this._prisma.taskRun.findFirst({
+      where: {
+        id: runId,
+      },
+      include: includeAttempts,
+    });
+
+    if (!taskRun) {
+      logger.error("[FailedTaskRunRetryHelper] Task run not found", {
+        runId,
+        completion,
+      });
+
+      return "NO_TASK_RUN";
+    }
+
+    const retriableExecution = await this.#getRetriableAttemptExecution(taskRun, completion);
+
+    if (!retriableExecution) {
+      return "NO_EXECUTION";
+    }
+
+    logger.debug("[FailedTaskRunRetryHelper] Completing attempt", { taskRun, completion });
+
+    const completeAttempt = new CompleteAttemptService({
+      prisma: this._prisma,
+      isSystemFailure: !isCrash,
+      isCrash,
+    });
+    const completeResult = await completeAttempt.call({
+      completion,
+      execution: retriableExecution,
+    });
+
+    return completeResult;
+  }
+
+  async #getRetriableAttemptExecution(
+    run: TaskRunWithAttempts,
+    completion: TaskRunFailedExecutionResult
+  ): Promise<TaskRunExecution | undefined> {
+    let attempt = run.attempts[0];
+
+    // We need to create an attempt if:
+    // - None exists yet
+    // - The last attempt has a final status, e.g. we failed between attempts
+    if (!attempt || isFinalAttemptStatus(attempt.status)) {
+      logger.debug("[FailedTaskRunRetryHelper] No attempts found", {
+        run,
+        completion,
+      });
+
+      const createAttempt = new CreateTaskRunAttemptService(this._prisma);
+
+      try {
+        const { execution } = await createAttempt.call({
+          runId: run.id,
+          // This ensures we correctly respect `maxAttempts = 1` when failing before the first attempt was created
+          startAtZero: true,
+        });
+        return execution;
+      } catch (error) {
+        logger.error("[FailedTaskRunRetryHelper] Failed to create attempt", {
+          run,
+          completion,
+          error,
+        });
+
+        return;
+      }
+    }
+
+    // We already have an attempt with non-final status, let's use it
+    try {
+      const executionPayload = await sharedQueueTasks.getExecutionPayloadFromAttempt({
+        id: attempt.id,
+        skipStatusChecks: true,
+      });
+
+      return executionPayload?.execution;
+    } catch (error) {
+      logger.error("[FailedTaskRunRetryHelper] Failed to get execution payload", {
+        run,
+        completion,
+        error,
+      });
+
+      return;
+    }
+  }
+
+  static async getExecutionRetry({
+    run,
+    execution,
+  }: {
+    run: TaskRunWithWorker;
+    execution: TaskRunExecution;
+  }): Promise<TaskRunExecutionRetry | undefined> {
+    try {
+      const retryConfig = run.lockedBy?.retryConfig;
+
+      if (!retryConfig) {
+        if (!run.lockedToVersion) {
+          logger.error("[FailedTaskRunRetryHelper] Run not locked to version", {
+            run,
+            execution,
+          });
+
+          return;
+        }
+
+        const sdkVersion = run.lockedToVersion.sdkVersion ?? "0.0.0";
+        const isValid = semver.valid(sdkVersion);
+
+        if (!isValid) {
+          logger.error("[FailedTaskRunRetryHelper] Invalid SDK version", {
+            run,
+            execution,
+          });
+
+          return;
+        }
+
+        // With older SDK versions, tasks only have a retry config stored in the DB if it's explicitly defined on the task itself
+        // It won't get populated with retry.default in trigger.config.ts
+        if (semver.lt(sdkVersion, FailedTaskRunRetryHelper.DEFAULT_RETRY_CONFIG_SINCE_VERSION)) {
+          logger.warn(
+            "[FailedTaskRunRetryHelper] SDK version not recent enough to determine retry config",
+            {
+              run,
+              execution,
+            }
+          );
+
+          return;
+        }
+      }
+
+      const parsedRetryConfig = RetryOptions.nullable().safeParse(retryConfig);
+
+      if (!parsedRetryConfig.success) {
+        logger.error("[FailedTaskRunRetryHelper] Invalid retry config", {
+          run,
+          execution,
+        });
+
+        return;
+      }
+
+      if (!parsedRetryConfig.data) {
+        logger.debug("[FailedTaskRunRetryHelper] No retry config", {
+          run,
+          execution,
+        });
+
+        return;
+      }
+
+      const delay = calculateNextRetryDelay(parsedRetryConfig.data, execution.attempt.number);
+
+      if (!delay) {
+        logger.debug("[FailedTaskRunRetryHelper] No more retries", {
+          run,
+          execution,
+        });
+
+        return;
+      }
+
+      return {
+        timestamp: Date.now() + delay,
+        delay,
+      };
+    } catch (error) {
+      logger.error("[FailedTaskRunRetryHelper] Failed to get execution retry", {
+        run,
+        execution,
+        error,
+      });
+
+      return;
+    }
+  }
+
+  static DEFAULT_RETRY_CONFIG_SINCE_VERSION = "3.1.0";
+}

apps/webapp/app/v3/handleSocketIo.server.ts

@@ -24,6 +24,7 @@ import { Redis } from "ioredis";
 import { createAdapter } from "@socket.io/redis-adapter";
 import { CrashTaskRunService } from "./services/crashTaskRun.server";
 import { CreateTaskRunAttemptService } from "./services/createTaskRunAttempt.server";
+import { UpdateFatalRunErrorService } from "./services/updateFatalRunError.server";
 
 export const socketIo = singleton("socketIo", initalizeIoServer);
 
@@ -123,12 +124,13 @@ function createCoordinatorNamespace(io: Server) {
         await resumeAttempt.call(message);
       },
       TASK_RUN_COMPLETED: async (message) => {
-        const completeAttempt = new CompleteAttemptService();
+        const completeAttempt = new CompleteAttemptService({
+          supportsRetryCheckpoints: message.version === "v1",
+        });
         await completeAttempt.call({
           completion: message.completion,
           execution: message.execution,
           checkpoint: message.checkpoint,
-          supportsRetryCheckpoints: message.version === "v1",
         });
       },
       TASK_RUN_FAILED_TO_RUN: async (message) => {
@@ -193,9 +195,16 @@ function createCoordinatorNamespace(io: Server) {
           }
 
           const service = new CreateTaskRunAttemptService();
-          const { attempt } = await service.call(message.runId, environment, false);
+          const { attempt } = await service.call({
+            runId: message.runId,
+            authenticatedEnv: environment,
+            setToExecuting: false,
+          });
 
-          const payload = await sharedQueueTasks.getExecutionPayloadFromAttempt(attempt.id, true);
+          const payload = await sharedQueueTasks.getExecutionPayloadFromAttempt({
+            id: attempt.id,
+            setToExecuting: true,
+          });
 
           if (!payload) {
             logger.error("Failed to retrieve payload after attempt creation", message);
@@ -294,11 +303,13 @@ function createProviderNamespace(io: Server) {
     handlers: {
       WORKER_CRASHED: async (message) => {
         try {
-          const service = new CrashTaskRunService();
-
-          await service.call(message.runId, {
-            ...message,
-          });
+          if (message.overrideCompletion) {
+            const updateErrorService = new UpdateFatalRunErrorService();
+            await updateErrorService.call(message.runId, { ...message });
+          } else {
+            const crashRunService = new CrashTaskRunService();
+            await crashRunService.call(message.runId, { ...message });
+          }
         } catch (error) {
           logger.error("Error while handling crashed worker", { error });
         }

apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts

@@ -4,6 +4,7 @@ import {
   ProdTaskRunExecution,
   ProdTaskRunExecutionPayload,
   TaskRunError,
+  TaskRunErrorCodes,
   TaskRunExecution,
   TaskRunExecutionLazyAttemptPayload,
   TaskRunExecutionResult,
@@ -21,7 +22,7 @@ import {
   TaskRunStatus,
 } from "@trigger.dev/database";
 import { z } from "zod";
-import { prisma } from "~/db.server";
+import { $replica, prisma } from "~/db.server";
 import { findEnvironmentById } from "~/models/runtimeEnvironment.server";
 import { logger } from "~/services/logger.server";
 import { singleton } from "~/utils/singleton";
@@ -43,7 +44,12 @@ import { generateJWTTokenForEnvironment } from "~/services/apiAuth.server";
 import { EnvironmentVariable } from "../environmentVariables/repository";
 import { machinePresetFromConfig } from "../machinePresets.server";
 import { env } from "~/env.server";
-import { isFinalAttemptStatus, isFinalRunStatus } from "../taskStatus";
+import {
+  FINAL_ATTEMPT_STATUSES,
+  FINAL_RUN_STATUSES,
+  isFinalAttemptStatus,
+  isFinalRunStatus,
+} from "../taskStatus";
 import { getMaxDuration } from "../utils/maxDuration";
 
 const WithTraceContext = z.object({
@@ -504,13 +510,21 @@ export class SharedQueueConsumer {
           if (!deployment.worker.supportsLazyAttempts) {
             try {
               const service = new CreateTaskRunAttemptService();
-              await service.call(lockedTaskRun.friendlyId, undefined, false);
+              await service.call({
+                runId: lockedTaskRun.id,
+                setToExecuting: false,
+              });
             } catch (error) {
               logger.error("Failed to create task run attempt for outdate worker", {
                 error,
                 taskRun: lockedTaskRun.id,
               });
 
+              const service = new CrashTaskRunService();
+              await service.call(lockedTaskRun.id, {
+                errorCode: TaskRunErrorCodes.OUTDATED_SDK_VERSION,
+              });
+
               await this.#ackAndDoMoreWork(message.messageId);
               return;
             }
@@ -620,6 +634,9 @@ export class SharedQueueConsumer {
         const resumableRun = await prisma.taskRun.findUnique({
           where: {
             id: message.messageId,
+            status: {
+              notIn: FINAL_RUN_STATUSES,
+            },
           },
         });
 
@@ -633,6 +650,14 @@ export class SharedQueueConsumer {
           return;
         }
 
+        if (resumableRun.status !== "EXECUTING") {
+          logger.warn("Run is not executing, will try to resume anyway", {
+            queueMessage: message.data,
+            messageId: message.messageId,
+            runStatus: resumableRun.status,
+          });
+        }
+
         const resumableAttempt = await prisma.taskRunAttempt.findUnique({
           where: {
             id: messageBody.data.resumableAttemptId,
@@ -718,9 +743,9 @@ export class SharedQueueConsumer {
 
           completions.push(completion);
 
-          const executionPayload = await this._tasks.getExecutionPayloadFromAttempt(
-            completedAttempt.id
-          );
+          const executionPayload = await this._tasks.getExecutionPayloadFromAttempt({
+            id: completedAttempt.id,
+          });
 
           if (!executionPayload) {
             await this.#ackAndDoMoreWork(message.messageId);
@@ -740,7 +765,11 @@ export class SharedQueueConsumer {
             executions,
           };
 
-          logger.debug("Broadcasting RESUME_AFTER_DEPENDENCY_WITH_ACK", { resumeMessage, message });
+          logger.debug("Broadcasting RESUME_AFTER_DEPENDENCY_WITH_ACK", {
+            resumeMessage,
+            message,
+            resumableRun,
+          });
 
           // The attempt should still be running so we can broadcast to all coordinators to resume immediately
           const responses = await socketIo.coordinatorNamespace
@@ -763,15 +792,91 @@ export class SharedQueueConsumer {
           }
 
           const hasSuccess = responses.some((response) => response.success);
-          if (!hasSuccess) {
-            logger.warn("RESUME_AFTER_DEPENDENCY_WITH_ACK failed", {
-              resumeMessage,
-              responses,
-              message,
-            });
-            await this.#nackAndDoMoreWork(message.messageId, this._options.nextTickInterval, 5_000);
+
+          if (hasSuccess) {
+            this.#doMoreWork();
             return;
           }
+
+          // No coordinator was able to resume the run
+          logger.warn("RESUME_AFTER_DEPENDENCY_WITH_ACK failed", {
+            resumeMessage,
+            responses,
+            message,
+          });
+
+          // Let's check if the run is frozen
+          if (resumableRun.status === "WAITING_TO_RESUME") {
+            logger.debug("RESUME_AFTER_DEPENDENCY_WITH_ACK run is waiting to be restored", {
+              queueMessage: message.data,
+              messageId: message.messageId,
+            });
+
+            try {
+              const restoreService = new RestoreCheckpointService();
+
+              const checkpointEvent = await restoreService.getLastCheckpointEventIfUnrestored(
+                resumableRun.id
+              );
+
+              if (checkpointEvent) {
+                // The last checkpoint hasn't been restored yet, so restore it
+                const checkpoint = await restoreService.call({
+                  eventId: checkpointEvent.id,
+                });
+
+                if (!checkpoint) {
+                  logger.debug("RESUME_AFTER_DEPENDENCY_WITH_ACK failed to restore checkpoint", {
+                    queueMessage: message.data,
+                    messageId: message.messageId,
+                  });
+
+                  await this.#ackAndDoMoreWork(message.messageId);
+                  return;
+                }
+
+                logger.debug("RESUME_AFTER_DEPENDENCY_WITH_ACK restored checkpoint", {
+                  queueMessage: message.data,
+                  messageId: message.messageId,
+                  checkpoint,
+                });
+
+                this.#doMoreWork();
+                return;
+              } else {
+                logger.debug(
+                  "RESUME_AFTER_DEPENDENCY_WITH_ACK run is frozen without last checkpoint event",
+                  {
+                    queueMessage: message.data,
+                    messageId: message.messageId,
+                  }
+                );
+              }
+            } catch (e) {
+              if (e instanceof Error) {
+                this._currentSpan?.recordException(e);
+              } else {
+                this._currentSpan?.recordException(new Error(String(e)));
+              }
+
+              this._endSpanInNextIteration = true;
+
+              await this.#nackAndDoMoreWork(
+                message.messageId,
+                this._options.nextTickInterval,
+                5_000
+              );
+              return;
+            }
+          }
+
+          logger.debug("RESUME_AFTER_DEPENDENCY_WITH_ACK retrying", {
+            queueMessage: message.data,
+            messageId: message.messageId,
+          });
+
+          await this.#nackAndDoMoreWork(message.messageId, this._options.nextTickInterval, 5_000);
+          return;
         } catch (e) {
           if (e instanceof Error) {
             this._currentSpan?.recordException(e);
@@ -896,7 +1001,7 @@ class SharedQueueTasks {
       where: {
         id,
         status: {
-          in: ["COMPLETED", "FAILED"],
+          in: FINAL_ATTEMPT_STATUSES,
         },
       },
       include: {
@@ -942,11 +1047,17 @@ class SharedQueueTasks {
     }
   }
 
-  async getExecutionPayloadFromAttempt(
-    id: string,
-    setToExecuting?: boolean,
-    isRetrying?: boolean
-  ): Promise<ProdTaskRunExecutionPayload | undefined> {
+  async getExecutionPayloadFromAttempt({
+    id,
+    setToExecuting,
+    isRetrying,
+    skipStatusChecks,
+  }: {
+    id: string;
+    setToExecuting?: boolean;
+    isRetrying?: boolean;
+    skipStatusChecks?: boolean;
+  }): Promise<ProdTaskRunExecutionPayload | undefined> {
     const attempt = await prisma.taskRunAttempt.findUnique({
       where: {
         id,
@@ -979,27 +1090,29 @@ class SharedQueueTasks {
       return;
     }
 
-    switch (attempt.status) {
-      case "CANCELED":
-      case "EXECUTING": {
-        logger.error("Invalid attempt status for execution payload retrieval", {
-          attemptId: id,
-          status: attempt.status,
-        });
-        return;
+    if (!skipStatusChecks) {
+      switch (attempt.status) {
+        case "CANCELED":
+        case "EXECUTING": {
+          logger.error("Invalid attempt status for execution payload retrieval", {
+            attemptId: id,
+            status: attempt.status,
+          });
+          return;
+        }
       }
-    }
 
-    switch (attempt.taskRun.status) {
-      case "CANCELED":
-      case "EXECUTING":
-      case "INTERRUPTED": {
-        logger.error("Invalid run status for execution payload retrieval", {
-          attemptId: id,
-          runId: attempt.taskRunId,
-          status: attempt.taskRun.status,
-        });
-        return;
+      switch (attempt.taskRun.status) {
+        case "CANCELED":
+        case "EXECUTING":
+        case "INTERRUPTED": {
+          logger.error("Invalid run status for execution payload retrieval", {
+            attemptId: id,
+            runId: attempt.taskRunId,
+            status: attempt.taskRun.status,
+          });
+          return;
+        }
       }
     }
 
@@ -1150,7 +1263,11 @@ class SharedQueueTasks {
       return;
     }
 
-    return this.getExecutionPayloadFromAttempt(latestAttempt.id, setToExecuting, isRetrying);
+    return this.getExecutionPayloadFromAttempt({
+      id: latestAttempt.id,
+      setToExecuting,
+      isRetrying,
+    });
   }
 
   async getLazyAttemptPayload(
@@ -1225,13 +1342,13 @@ class SharedQueueTasks {
       return;
     }
 
-    await marqs?.heartbeatMessage(taskRunAttempt.taskRunId);
+    await this.#heartbeat(taskRunAttempt.taskRunId);
   }
 
   async taskRunHeartbeat(runId: string) {
     logger.debug("[SharedQueueConsumer] taskRunHeartbeat()", { runId });
 
-    await marqs?.heartbeatMessage(runId);
+    await this.#heartbeat(runId);
   }
 
   public async taskRunFailed(completion: TaskRunFailedExecutionResult) {
@@ -1242,6 +1359,66 @@ class SharedQueueTasks {
     await service.call(completion.id, completion);
   }
 
+  async #heartbeat(runId: string) {
+    await marqs?.heartbeatMessage(runId);
+
+    try {
+      // There can be a lot of calls per minute and the data doesn't have to be accurate, so use the read replica
+      const taskRun = await $replica.taskRun.findFirst({
+        where: {
+          id: runId,
+        },
+        select: {
+          id: true,
+          status: true,
+          runtimeEnvironment: {
+            select: {
+              type: true,
+            },
+          },
+          lockedToVersion: {
+            select: {
+              supportsLazyAttempts: true,
+            },
+          },
+        },
+      });
+
+      if (!taskRun) {
+        logger.error("SharedQueueTasks.#heartbeat: Task run not found", {
+          runId,
+        });
+
+        return;
+      }
+
+      if (taskRun.runtimeEnvironment.type === "DEVELOPMENT") {
+        return;
+      }
+
+      if (isFinalRunStatus(taskRun.status)) {
+        logger.debug("SharedQueueTasks.#heartbeat: Task run is in final status", {
+          runId,
+          status: taskRun.status,
+        });
+
+        // Signal to exit any leftover containers
+        socketIo.coordinatorNamespace.emit("REQUEST_RUN_CANCELLATION", {
+          version: "v1",
+          runId: taskRun.id,
+          // Give the run a few seconds to exit to complete any flushing etc
+          delayInMs: taskRun.lockedToVersion?.supportsLazyAttempts ? 5_000 : undefined,
+        });
+        return;
+      }
+    } catch (error) {
+      logger.error("SharedQueueTasks.#heartbeat: Error signaling run cancellation", {
+        runId,
+        error: error instanceof Error ? error.message : error,
+      });
+    }
+  }
+
   async #buildEnvironmentVariables(
     environment: RuntimeEnvironment,
     runId: string,

apps/webapp/app/v3/requeueTaskRun.server.ts

@@ -6,11 +6,30 @@ import { FailedTaskRunService } from "./failedTaskRun.server";
 import { BaseService } from "./services/baseService.server";
 import { PrismaClientOrTransaction } from "~/db.server";
 import { workerQueue } from "~/services/worker.server";
+import { socketIo } from "./handleSocketIo.server";
+import { TaskRunErrorCodes } from "@trigger.dev/core/v3";
 
 export class RequeueTaskRunService extends BaseService {
   public async call(runId: string) {
     const taskRun = await this._prisma.taskRun.findUnique({
-      where: { id: runId },
+      where: {
+        id: runId,
+      },
+      select: {
+        id: true,
+        friendlyId: true,
+        status: true,
+        runtimeEnvironment: {
+          select: {
+            type: true,
+          },
+        },
+        lockedToVersion: {
+          select: {
+            supportsLazyAttempts: true,
+          },
+        },
+      },
     });
 
     if (!taskRun) {
@@ -41,7 +60,7 @@ export class RequeueTaskRunService extends BaseService {
           retry: undefined,
           error: {
             type: "INTERNAL_ERROR",
-            code: "TASK_RUN_HEARTBEAT_TIMEOUT",
+            code: TaskRunErrorCodes.TASK_RUN_HEARTBEAT_TIMEOUT,
             message: "Did not receive a heartbeat from the worker in time",
           },
         });
@@ -76,6 +95,25 @@ export class RequeueTaskRunService extends BaseService {
 
         await marqs?.acknowledgeMessage(taskRun.id);
 
+        try {
+          if (taskRun.runtimeEnvironment.type === "DEVELOPMENT") {
+            return;
+          }
+
+          // Signal to exit any leftover containers
+          socketIo.coordinatorNamespace.emit("REQUEST_RUN_CANCELLATION", {
+            version: "v1",
+            runId: taskRun.id,
+            // Give the run a few seconds to exit to complete any flushing etc
+            delayInMs: taskRun.lockedToVersion?.supportsLazyAttempts ? 5_000 : undefined,
+          });
+        } catch (error) {
+          logger.error("[RequeueTaskRunService] Error signaling run cancellation", {
+            runId: taskRun.id,
+            error: error instanceof Error ? error.message : error,
+          });
+        }
+
         break;
       }
       default: {

apps/webapp/app/v3/services/completeAttempt.server.ts

@@ -1,6 +1,7 @@
 import { Attributes } from "@opentelemetry/api";
 import {
   TaskRunContext,
+  TaskRunErrorCodes,
   TaskRunExecution,
   TaskRunExecutionResult,
   TaskRunExecutionRetry,
@@ -8,6 +9,8 @@ import {
   TaskRunSuccessfulExecutionResult,
   flattenAttributes,
   sanitizeError,
+  shouldRetryError,
+  taskRunErrorEnhancer,
 } from "@trigger.dev/core/v3";
 import { $transaction, PrismaClientOrTransaction } from "~/db.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
@@ -21,9 +24,10 @@ import { MAX_TASK_RUN_ATTEMPTS } from "~/consts";
 import { CreateCheckpointService } from "./createCheckpoint.server";
 import { TaskRun } from "@trigger.dev/database";
 import { RetryAttemptService } from "./retryAttempt.server";
-import { isFinalAttemptStatus, isFinalRunStatus } from "../taskStatus";
+import { FAILED_RUN_STATUSES, isFinalAttemptStatus, isFinalRunStatus } from "../taskStatus";
 import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
 import { env } from "~/env.server";
+import { FailedTaskRunRetryHelper } from "../failedTaskRun.server";
 
 type FoundAttempt = Awaited<ReturnType<typeof findAttempt>>;
 
@@ -32,19 +36,28 @@ type CheckpointData = {
   location: string;
 };
 
+type CompleteAttemptServiceOptions = {
+  prisma?: PrismaClientOrTransaction;
+  supportsRetryCheckpoints?: boolean;
+  isSystemFailure?: boolean;
+  isCrash?: boolean;
+};
+
 export class CompleteAttemptService extends BaseService {
+  constructor(private opts: CompleteAttemptServiceOptions = {}) {
+    super(opts.prisma);
+  }
+
   public async call({
     completion,
     execution,
     env,
     checkpoint,
-    supportsRetryCheckpoints,
   }: {
     completion: TaskRunExecutionResult;
     execution: TaskRunExecution;
     env?: AuthenticatedEnvironment;
     checkpoint?: CheckpointData;
-    supportsRetryCheckpoints?: boolean;
   }): Promise<"COMPLETED" | "RETRIED"> {
     const taskRunAttempt = await findAttempt(this._prisma, execution.attempt.id);
 
@@ -78,7 +91,7 @@ export class CompleteAttemptService extends BaseService {
         attemptStatus: "FAILED",
         error: {
           type: "INTERNAL_ERROR",
-          code: "TASK_EXECUTION_FAILED",
+          code: TaskRunErrorCodes.TASK_EXECUTION_FAILED,
           message: "Tried to complete attempt but it doesn't exist",
         },
       });
@@ -109,7 +122,6 @@ export class CompleteAttemptService extends BaseService {
         taskRunAttempt,
         env,
         checkpoint,
-        supportsRetryCheckpoints,
       });
     }
   }
@@ -169,14 +181,12 @@ export class CompleteAttemptService extends BaseService {
     taskRunAttempt,
     env,
     checkpoint,
-    supportsRetryCheckpoints,
   }: {
     completion: TaskRunFailedExecutionResult;
     execution: TaskRunExecution;
     taskRunAttempt: NonNullable<FoundAttempt>;
     env?: AuthenticatedEnvironment;
     checkpoint?: CheckpointData;
-    supportsRetryCheckpoints?: boolean;
   }): Promise<"COMPLETED" | "RETRIED"> {
     if (
       completion.error.type === "INTERNAL_ERROR" &&
@@ -194,18 +204,17 @@ export class CompleteAttemptService extends BaseService {
         env
       );
 
-      // The cancel service handles ACK
-
       return "COMPLETED";
     }
 
+    const failedAt = new Date();
     const sanitizedError = sanitizeError(completion.error);
 
     await this._prisma.taskRunAttempt.update({
       where: { id: taskRunAttempt.id },
       data: {
         status: "FAILED",
-        completedAt: new Date(),
+        completedAt: failedAt,
         error: sanitizedError,
         usageDurationMs: completion.usage?.durationMs,
       },
@@ -213,226 +222,230 @@ export class CompleteAttemptService extends BaseService {
 
     const environment = env ?? (await this.#getEnvironment(execution.environment.id));
 
-    if (completion.retry !== undefined && taskRunAttempt.number < MAX_TASK_RUN_ATTEMPTS) {
-      const retryAt = new Date(completion.retry.timestamp);
+    // This means that tasks won't know they are being retried
+    let executionRetryInferred = false;
+    let executionRetry = completion.retry;
 
-      // Retry the task run
-      await eventRepository.recordEvent(`Retry #${execution.attempt.number} delay`, {
-        taskSlug: taskRunAttempt.taskRun.taskIdentifier,
+    const shouldInfer = this.opts.isCrash || this.opts.isSystemFailure;
+
+    if (!executionRetry && shouldInfer) {
+      executionRetryInferred = true;
+      executionRetry = await FailedTaskRunRetryHelper.getExecutionRetry({
+        run: {
+          ...taskRunAttempt.taskRun,
+          lockedBy: taskRunAttempt.backgroundWorkerTask,
+          lockedToVersion: taskRunAttempt.backgroundWorker,
+        },
+        execution,
+      });
+    }
+
+    const retriableError = shouldRetryError(taskRunErrorEnhancer(completion.error));
+
+    if (
+      retriableError &&
+      executionRetry !== undefined &&
+      taskRunAttempt.number < MAX_TASK_RUN_ATTEMPTS
+    ) {
+      return await this.#retryAttempt({
+        execution,
+        executionRetry,
+        executionRetryInferred,
+        taskRunAttempt,
         environment,
-        attributes: {
-          metadata: this.#generateMetadataAttributesForNextAttempt(execution),
+        checkpoint,
+      });
+    }
+
+    // The attempt has failed and we won't retry
+
+    // Now we need to "complete" the task run event/span
+    await eventRepository.completeEvent(taskRunAttempt.taskRun.spanId, {
+      endTime: failedAt,
+      attributes: {
+        isError: true,
+      },
+      events: [
+        {
+          name: "exception",
+          time: failedAt,
           properties: {
-            retryAt: retryAt.toISOString(),
+            exception: createExceptionPropertiesFromError(sanitizedError),
           },
-          runId: taskRunAttempt.taskRun.friendlyId,
-          style: {
-            icon: "schedule-attempt",
-          },
-          queueId: taskRunAttempt.queueId,
-          queueName: taskRunAttempt.taskRun.queue,
         },
-        context: taskRunAttempt.taskRun.traceContext as Record<string, string | undefined>,
-        spanIdSeed: `retry-${taskRunAttempt.number + 1}`,
-        endTime: retryAt,
-      });
+      ],
+    });
 
-      logger.debug("Retrying", {
-        taskRun: taskRunAttempt.taskRun.friendlyId,
-        retry: completion.retry,
-      });
+    await this._prisma.taskRun.update({
+      where: {
+        id: taskRunAttempt.taskRunId,
+      },
+      data: {
+        error: sanitizedError,
+      },
+    });
 
-      await this._prisma.taskRun.update({
-        where: {
-          id: taskRunAttempt.taskRunId,
-        },
-        data: {
-          status: "RETRYING_AFTER_FAILURE",
-        },
-      });
+    let status: FAILED_RUN_STATUSES;
 
-      if (environment.type === "DEVELOPMENT") {
-        // This is already an EXECUTE message so we can just NACK
-        await marqs?.nackMessage(taskRunAttempt.taskRunId, completion.retry.timestamp);
-        return "RETRIED";
-      }
-
-      if (!checkpoint) {
-        await this.#retryAttempt({
-          run: taskRunAttempt.taskRun,
-          retry: completion.retry,
-          supportsLazyAttempts: taskRunAttempt.backgroundWorker.supportsLazyAttempts,
-          supportsRetryCheckpoints,
-        });
-
-        return "RETRIED";
-      }
-
-      const createCheckpoint = new CreateCheckpointService(this._prisma);
-      const checkpointCreateResult = await createCheckpoint.call({
-        attemptFriendlyId: execution.attempt.id,
-        docker: checkpoint.docker,
-        location: checkpoint.location,
-        reason: {
-          type: "RETRYING_AFTER_FAILURE",
-          attemptNumber: execution.attempt.number,
-        },
-      });
-
-      if (!checkpointCreateResult.success) {
-        logger.error("Failed to create checkpoint", { checkpoint, execution: execution.run.id });
-
-        const finalizeService = new FinalizeTaskRunService();
-        await finalizeService.call({
-          id: taskRunAttempt.taskRunId,
-          status: "SYSTEM_FAILURE",
-          completedAt: new Date(),
-        });
-
-        return "COMPLETED";
-      }
-
-      await this.#retryAttempt({
-        run: taskRunAttempt.taskRun,
-        retry: completion.retry,
-        checkpointEventId: checkpointCreateResult.event.id,
-        supportsLazyAttempts: taskRunAttempt.backgroundWorker.supportsLazyAttempts,
-        supportsRetryCheckpoints,
-      });
-
-      return "RETRIED";
+    // Set the correct task run status
+    if (this.opts.isSystemFailure) {
+      status = "SYSTEM_FAILURE";
+    } else if (this.opts.isCrash) {
+      status = "CRASHED";
+    } else if (
+      sanitizedError.type === "INTERNAL_ERROR" &&
+      sanitizedError.code === "MAX_DURATION_EXCEEDED"
+    ) {
+      status = "TIMED_OUT";
+    } else if (sanitizedError.type === "INTERNAL_ERROR") {
+      status = "CRASHED";
     } else {
-      // Now we need to "complete" the task run event/span
-      await eventRepository.completeEvent(taskRunAttempt.taskRun.spanId, {
-        endTime: new Date(),
-        attributes: {
-          isError: true,
-        },
-        events: [
-          {
-            name: "exception",
-            time: new Date(),
-            properties: {
-              exception: createExceptionPropertiesFromError(sanitizedError),
-            },
-          },
-        ],
-      });
+      status = "COMPLETED_WITH_ERRORS";
+    }
 
-      if (
-        sanitizedError.type === "INTERNAL_ERROR" &&
-        sanitizedError.code === "GRACEFUL_EXIT_TIMEOUT"
-      ) {
-        const finalizeService = new FinalizeTaskRunService();
-        await finalizeService.call({
-          id: taskRunAttempt.taskRunId,
-          status: "SYSTEM_FAILURE",
-          completedAt: new Date(),
-        });
+    const finalizeService = new FinalizeTaskRunService();
+    await finalizeService.call({
+      id: taskRunAttempt.taskRunId,
+      status,
+      completedAt: failedAt,
+    });
 
-        // We need to fail all incomplete spans
-        const inProgressEvents = await eventRepository.queryIncompleteEvents({
-          attemptId: execution.attempt.id,
-        });
+    if (status !== "CRASHED" && status !== "SYSTEM_FAILURE") {
+      return "COMPLETED";
+    }
 
-        logger.debug("Failing in-progress events", {
+    const inProgressEvents = await eventRepository.queryIncompleteEvents({
+      runId: taskRunAttempt.taskRun.friendlyId,
+    });
+
+    // Handle in-progress events
+    switch (status) {
+      case "CRASHED": {
+        logger.debug("[CompleteAttemptService] Crashing in-progress events", {
           inProgressEvents: inProgressEvents.map((event) => event.id),
         });
 
-        const exception = {
-          type: "Graceful exit timeout",
-          message: sanitizedError.message,
-        };
-
         await Promise.all(
           inProgressEvents.map((event) => {
             return eventRepository.crashEvent({
-              event: event,
-              crashedAt: new Date(),
-              exception,
+              event,
+              crashedAt: failedAt,
+              exception: createExceptionPropertiesFromError(sanitizedError),
             });
           })
         );
-      } else {
-        await this._prisma.taskRun.update({
-          where: {
-            id: taskRunAttempt.taskRunId,
-          },
-          data: {
-            error: sanitizedError,
-          },
-        });
 
-        const status =
-          sanitizedError.type === "INTERNAL_ERROR" &&
-          sanitizedError.code === "MAX_DURATION_EXCEEDED"
-            ? "TIMED_OUT"
-            : "COMPLETED_WITH_ERRORS";
-
-        const finalizeService = new FinalizeTaskRunService();
-        await finalizeService.call({
-          id: taskRunAttempt.taskRunId,
-          status,
-          completedAt: new Date(),
-        });
+        break;
       }
+      case "SYSTEM_FAILURE": {
+        logger.debug("[CompleteAttemptService] Failing in-progress events", {
+          inProgressEvents: inProgressEvents.map((event) => event.id),
+        });
 
-      return "COMPLETED";
+        await Promise.all(
+          inProgressEvents.map((event) => {
+            return eventRepository.completeEvent(event.spanId, {
+              endTime: failedAt,
+              attributes: {
+                isError: true,
+              },
+              events: [
+                {
+                  name: "exception",
+                  time: failedAt,
+                  properties: {
+                    exception: createExceptionPropertiesFromError(sanitizedError),
+                  },
+                },
+              ],
+            });
+          })
+        );
+      }
     }
+
+    return "COMPLETED";
   }
 
-  async #retryAttempt({
+  async #enqueueReattempt({
     run,
-    retry,
+    executionRetry,
+    executionRetryInferred,
     checkpointEventId,
     supportsLazyAttempts,
-    supportsRetryCheckpoints,
   }: {
     run: TaskRun;
-    retry: TaskRunExecutionRetry;
+    executionRetry: TaskRunExecutionRetry;
+    executionRetryInferred: boolean;
     checkpointEventId?: string;
     supportsLazyAttempts: boolean;
-    supportsRetryCheckpoints?: boolean;
   }) {
     const retryViaQueue = () => {
+      logger.debug("[CompleteAttemptService] Enqueuing retry attempt", { runId: run.id });
+
       // We have to replace a potential RESUME with EXECUTE to correctly retry the attempt
       return marqs?.replaceMessage(
         run.id,
         {
           type: "EXECUTE",
           taskIdentifier: run.taskIdentifier,
-          checkpointEventId: supportsRetryCheckpoints ? checkpointEventId : undefined,
-          retryCheckpointsDisabled: !supportsRetryCheckpoints,
+          checkpointEventId: this.opts.supportsRetryCheckpoints ? checkpointEventId : undefined,
+          retryCheckpointsDisabled: !this.opts.supportsRetryCheckpoints,
         },
-        retry.timestamp
+        executionRetry.timestamp
       );
     };
 
     const retryDirectly = () => {
-      return RetryAttemptService.enqueue(run.id, this._prisma, new Date(retry.timestamp));
+      logger.debug("[CompleteAttemptService] Retrying attempt directly", { runId: run.id });
+      return RetryAttemptService.enqueue(run.id, this._prisma, new Date(executionRetry.timestamp));
     };
 
     // There's a checkpoint, so we need to go through the queue
     if (checkpointEventId) {
-      if (!supportsRetryCheckpoints) {
-        logger.error("Worker does not support retry checkpoints, but a checkpoint was created", {
-          runId: run.id,
-          checkpointEventId,
-        });
+      if (!this.opts.supportsRetryCheckpoints) {
+        logger.error(
+          "[CompleteAttemptService] Worker does not support retry checkpoints, but a checkpoint was created",
+          {
+            runId: run.id,
+            checkpointEventId,
+          }
+        );
       }
 
+      logger.debug("[CompleteAttemptService] Enqueuing retry attempt with checkpoint", {
+        runId: run.id,
+      });
       await retryViaQueue();
       return;
     }
 
     // Workers without lazy attempt support always need to go through the queue, which is where the attempt is created
     if (!supportsLazyAttempts) {
+      logger.debug("[CompleteAttemptService] Worker does not support lazy attempts", {
+        runId: run.id,
+      });
       await retryViaQueue();
       return;
     }
 
     // Workers that never checkpoint between attempts will exit after completing their current attempt if the retry delay exceeds the threshold
-    if (!supportsRetryCheckpoints && retry.delay >= env.CHECKPOINT_THRESHOLD_IN_MS) {
+    if (
+      !this.opts.supportsRetryCheckpoints &&
+      executionRetry.delay >= env.CHECKPOINT_THRESHOLD_IN_MS
+    ) {
+      logger.debug(
+        "[CompleteAttemptService] Worker does not support retry checkpoints and the delay exceeds the threshold",
+        { runId: run.id }
+      );
+      await retryViaQueue();
+      return;
+    }
+
+    if (executionRetryInferred) {
+      logger.debug("[CompleteAttemptService] Execution retry inferred, forcing retry via queue", {
+        runId: run.id,
+      });
       await retryViaQueue();
       return;
     }
@@ -441,6 +454,141 @@ export class CompleteAttemptService extends BaseService {
     await retryDirectly();
   }
 
+  async #retryAttempt({
+    execution,
+    executionRetry,
+    executionRetryInferred,
+    taskRunAttempt,
+    environment,
+    checkpoint,
+  }: {
+    execution: TaskRunExecution;
+    executionRetry: TaskRunExecutionRetry;
+    executionRetryInferred: boolean;
+    taskRunAttempt: NonNullable<FoundAttempt>;
+    environment: AuthenticatedEnvironment;
+    checkpoint?: CheckpointData;
+  }) {
+    const retryAt = new Date(executionRetry.timestamp);
+
+    // Retry the task run
+    await eventRepository.recordEvent(`Retry #${execution.attempt.number} delay`, {
+      taskSlug: taskRunAttempt.taskRun.taskIdentifier,
+      environment,
+      attributes: {
+        metadata: this.#generateMetadataAttributesForNextAttempt(execution),
+        properties: {
+          retryAt: retryAt.toISOString(),
+        },
+        runId: taskRunAttempt.taskRun.friendlyId,
+        style: {
+          icon: "schedule-attempt",
+        },
+        queueId: taskRunAttempt.queueId,
+        queueName: taskRunAttempt.taskRun.queue,
+      },
+      context: taskRunAttempt.taskRun.traceContext as Record<string, string | undefined>,
+      spanIdSeed: `retry-${taskRunAttempt.number + 1}`,
+      endTime: retryAt,
+    });
+
+    logger.debug("[CompleteAttemptService] Retrying", {
+      taskRun: taskRunAttempt.taskRun.friendlyId,
+      retry: executionRetry,
+    });
+
+    await this._prisma.taskRun.update({
+      where: {
+        id: taskRunAttempt.taskRunId,
+      },
+      data: {
+        status: "RETRYING_AFTER_FAILURE",
+      },
+    });
+
+    if (environment.type === "DEVELOPMENT") {
+      // This is already an EXECUTE message so we can just NACK
+      await marqs?.nackMessage(taskRunAttempt.taskRunId, executionRetry.timestamp);
+      return "RETRIED";
+    }
+
+    if (checkpoint) {
+      // This is only here for backwards compat - we don't checkpoint between attempts anymore
+      return await this.#retryAttemptWithCheckpoint({
+        execution,
+        taskRunAttempt,
+        executionRetry,
+        executionRetryInferred,
+        checkpoint,
+      });
+    }
+
+    await this.#enqueueReattempt({
+      run: taskRunAttempt.taskRun,
+      executionRetry,
+      supportsLazyAttempts: taskRunAttempt.backgroundWorker.supportsLazyAttempts,
+      executionRetryInferred,
+    });
+
+    return "RETRIED";
+  }
+
+  async #retryAttemptWithCheckpoint({
+    execution,
+    taskRunAttempt,
+    executionRetry,
+    executionRetryInferred,
+    checkpoint,
+  }: {
+    execution: TaskRunExecution;
+    taskRunAttempt: NonNullable<FoundAttempt>;
+    executionRetry: TaskRunExecutionRetry;
+    executionRetryInferred: boolean;
+    checkpoint: CheckpointData;
+  }) {
+    const createCheckpoint = new CreateCheckpointService(this._prisma);
+    const checkpointCreateResult = await createCheckpoint.call({
+      attemptFriendlyId: execution.attempt.id,
+      docker: checkpoint.docker,
+      location: checkpoint.location,
+      reason: {
+        type: "RETRYING_AFTER_FAILURE",
+        attemptNumber: execution.attempt.number,
+      },
+    });
+
+    if (!checkpointCreateResult.success) {
+      logger.error("[CompleteAttemptService] Failed to create reattempt checkpoint", {
+        checkpoint,
+        runId: execution.run.id,
+        attemptId: execution.attempt.id,
+      });
+
+      const finalizeService = new FinalizeTaskRunService();
+      await finalizeService.call({
+        id: taskRunAttempt.taskRunId,
+        status: "SYSTEM_FAILURE",
+        completedAt: new Date(),
+        error: {
+          type: "STRING_ERROR",
+          raw: "Failed to create reattempt checkpoint",
+        },
+      });
+
+      return "COMPLETED" as const;
+    }
+
+    await this.#enqueueReattempt({
+      run: taskRunAttempt.taskRun,
+      executionRetry,
+      checkpointEventId: checkpointCreateResult.event.id,
+      supportsLazyAttempts: taskRunAttempt.backgroundWorker.supportsLazyAttempts,
+      executionRetryInferred,
+    });
+
+    return "RETRIED" as const;
+  }
+
   #generateMetadataAttributesForNextAttempt(execution: TaskRunExecution) {
     const context = TaskRunContext.parse(execution);
 
@@ -475,6 +623,7 @@ async function findAttempt(prismaClient: PrismaClientOrTransaction, friendlyId:
         select: {
           id: true,
           supportsLazyAttempts: true,
+          sdkVersion: true,
         },
       },
     },

apps/webapp/app/v3/services/crashTaskRun.server.ts

@@ -1,12 +1,12 @@
 import { TaskRun, TaskRunAttempt } from "@trigger.dev/database";
 import { eventRepository } from "../eventRepository.server";
-import { marqs } from "~/v3/marqs/index.server";
 import { BaseService } from "./baseService.server";
 import { logger } from "~/services/logger.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { CRASHABLE_ATTEMPT_STATUSES, isCrashableRunStatus } from "../taskStatus";
-import { sanitizeError, TaskRunInternalError } from "@trigger.dev/core/v3";
+import { sanitizeError, TaskRunErrorCodes, TaskRunInternalError } from "@trigger.dev/core/v3";
 import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
+import { FailedTaskRunRetryHelper } from "../failedTaskRun.server";
 
 export type CrashTaskRunServiceOptions = {
   reason?: string;
@@ -29,6 +29,11 @@ export class CrashTaskRunService extends BaseService {
 
     logger.debug("CrashTaskRunService.call", { runId, opts });
 
+    if (options?.overrideCompletion) {
+      logger.error("CrashTaskRunService.call: overrideCompletion is deprecated", { runId });
+      return;
+    }
+
     const taskRun = await this._prisma.taskRun.findFirst({
       where: {
         id: runId,
@@ -36,16 +41,50 @@ export class CrashTaskRunService extends BaseService {
     });
 
     if (!taskRun) {
-      logger.error("Task run not found", { runId });
+      logger.error("[CrashTaskRunService] Task run not found", { runId });
       return;
     }
 
     // Make sure the task run is in a crashable state
     if (!opts.overrideCompletion && !isCrashableRunStatus(taskRun.status)) {
-      logger.error("Task run is not in a crashable state", { runId, status: taskRun.status });
+      logger.error("[CrashTaskRunService] Task run is not in a crashable state", {
+        runId,
+        status: taskRun.status,
+      });
       return;
     }
 
+    logger.debug("[CrashTaskRunService] Completing attempt", { runId, options });
+
+    const retryHelper = new FailedTaskRunRetryHelper(this._prisma);
+    const retryResult = await retryHelper.call({
+      runId,
+      completion: {
+        ok: false,
+        id: runId,
+        error: {
+          type: "INTERNAL_ERROR",
+          code: opts.errorCode ?? TaskRunErrorCodes.TASK_RUN_CRASHED,
+          message: opts.reason,
+          stackTrace: opts.logs,
+        },
+      },
+      isCrash: true,
+    });
+
+    logger.debug("[CrashTaskRunService] Completion result", { runId, retryResult });
+
+    if (retryResult === "RETRIED") {
+      logger.debug("[CrashTaskRunService] Retried task run", { runId });
+      return;
+    }
+
+    if (!opts.overrideCompletion) {
+      return;
+    }
+
+    logger.debug("[CrashTaskRunService] Overriding completion", { runId, options });
+
     const finalizeService = new FinalizeTaskRunService();
     const crashedTaskRun = await finalizeService.call({
       id: taskRun.id,
@@ -74,7 +113,7 @@ export class CrashTaskRunService extends BaseService {
       attemptStatus: "FAILED",
       error: {
         type: "INTERNAL_ERROR",
-        code: opts.errorCode ?? "TASK_RUN_CRASHED",
+        code: opts.errorCode ?? TaskRunErrorCodes.TASK_RUN_CRASHED,
         message: opts.reason,
         stackTrace: opts.logs,
       },
@@ -87,7 +126,7 @@ export class CrashTaskRunService extends BaseService {
       options?.overrideCompletion
     );
 
-    logger.debug("Crashing in-progress events", {
+    logger.debug("[CrashTaskRunService] Crashing in-progress events", {
       inProgressEvents: inProgressEvents.map((event) => event.id),
     });
 
@@ -97,7 +136,7 @@ export class CrashTaskRunService extends BaseService {
           event: event,
           crashedAt: opts.crashedAt,
           exception: {
-            type: opts.errorCode ?? "TASK_RUN_CRASHED",
+            type: opts.errorCode ?? TaskRunErrorCodes.TASK_RUN_CRASHED,
             message: opts.reason,
             stacktrace: opts.logs,
           },
@@ -136,27 +175,29 @@ export class CrashTaskRunService extends BaseService {
       code?: TaskRunInternalError["code"];
     }
   ) {
-    return await this.traceWithEnv("failAttempt()", environment, async (span) => {
-      span.setAttribute("taskRunId", run.id);
-      span.setAttribute("attemptId", attempt.id);
+    return await this.traceWithEnv(
+      "[CrashTaskRunService] failAttempt()",
+      environment,
+      async (span) => {
+        span.setAttribute("taskRunId", run.id);
+        span.setAttribute("attemptId", attempt.id);
 
-      await marqs?.acknowledgeMessage(run.id);
-
-      await this._prisma.taskRunAttempt.update({
-        where: {
-          id: attempt.id,
-        },
-        data: {
-          status: "FAILED",
-          completedAt: failedAt,
-          error: sanitizeError({
-            type: "INTERNAL_ERROR",
-            code: error.code ?? "TASK_RUN_CRASHED",
-            message: error.reason,
-            stackTrace: error.logs,
-          }),
-        },
-      });
-    });
+        await this._prisma.taskRunAttempt.update({
+          where: {
+            id: attempt.id,
+          },
+          data: {
+            status: "FAILED",
+            completedAt: failedAt,
+            error: sanitizeError({
+              type: "INTERNAL_ERROR",
+              code: error.code ?? TaskRunErrorCodes.TASK_RUN_CRASHED,
+              message: error.reason,
+              stackTrace: error.logs,
+            }),
+          },
+        });
+      }
+    );
   }
 }

apps/webapp/app/v3/services/createCheckpoint.server.ts

@@ -50,7 +50,7 @@ export class CreateCheckpointService extends BaseService {
     });
 
     if (!attempt) {
-      logger.error("Attempt not found", { attemptFriendlyId: params.attemptFriendlyId });
+      logger.error("Attempt not found", params);
 
       return {
         success: false,
@@ -70,6 +70,7 @@ export class CreateCheckpointService extends BaseService {
           id: attempt.taskRunId,
           status: attempt.taskRun.status,
         },
+        params,
       });
 
       return {
@@ -84,6 +85,7 @@ export class CreateCheckpointService extends BaseService {
       logger.error("Missing deployment or image ref", {
         attemptId: attempt.id,
         workerId: attempt.backgroundWorker.id,
+        params,
       });
 
       return {
@@ -170,6 +172,7 @@ export class CreateCheckpointService extends BaseService {
             taskRunId: attempt.taskRunId,
             type: "WAIT_FOR_TASK",
             reason,
+            params,
           });
           await marqs?.cancelHeartbeat(attempt.taskRunId);
 
@@ -182,6 +185,7 @@ export class CreateCheckpointService extends BaseService {
           if (!childRun) {
             logger.error("CreateCheckpointService: WAIT_FOR_TASK child run not found", {
               friendlyId: reason.friendlyId,
+              params,
             });
 
             return {
@@ -201,6 +205,7 @@ export class CreateCheckpointService extends BaseService {
               childRun,
               attempt,
               checkpointEvent,
+              params,
             });
           } else {
             logger.error("CreateCheckpointService: Failed to resume dependent parents", {
@@ -208,6 +213,7 @@ export class CreateCheckpointService extends BaseService {
               childRun,
               attempt,
               checkpointEvent,
+              params,
             });
           }
 
@@ -233,6 +239,7 @@ export class CreateCheckpointService extends BaseService {
             attemptId: attempt.id,
             taskRunId: attempt.taskRunId,
             type: "WAIT_FOR_BATCH",
+            params,
           });
           await marqs?.cancelHeartbeat(attempt.taskRunId);
 
@@ -248,6 +255,7 @@ export class CreateCheckpointService extends BaseService {
           if (!batchRun) {
             logger.error("CreateCheckpointService: Batch not found", {
               friendlyId: reason.batchFriendlyId,
+              params,
             });
 
             return {
@@ -297,6 +305,7 @@ export class CreateCheckpointService extends BaseService {
       logger.error("No checkpoint event", {
         attemptId: attempt.id,
         checkpointId: checkpoint.id,
+        params,
       });
       await marqs?.acknowledgeMessage(attempt.taskRunId);
 

apps/webapp/app/v3/services/createTaskRunAttempt.server.ts

@@ -12,11 +12,17 @@ import { CrashTaskRunService } from "./crashTaskRun.server";
 import { ExpireEnqueuedRunService } from "./expireEnqueuedRun.server";
 
 export class CreateTaskRunAttemptService extends BaseService {
-  public async call(
-    runId: string,
-    authenticatedEnv?: AuthenticatedEnvironment,
-    setToExecuting = true
-  ): Promise<{
+  public async call({
+    runId,
+    authenticatedEnv,
+    setToExecuting = true,
+    startAtZero = false,
+  }: {
+    runId: string;
+    authenticatedEnv?: AuthenticatedEnvironment;
+    setToExecuting?: boolean;
+    startAtZero?: boolean;
+  }): Promise<{
     execution: TaskRunExecution;
     run: TaskRun;
     attempt: TaskRunAttempt;
@@ -102,7 +108,11 @@ export class CreateTaskRunAttemptService extends BaseService {
         throw new ServiceValidationError("Queue not found", 404);
       }
 
-      const nextAttemptNumber = taskRun.attempts[0] ? taskRun.attempts[0].number + 1 : 1;
+      const nextAttemptNumber = taskRun.attempts[0]
+        ? taskRun.attempts[0].number + 1
+        : startAtZero
+        ? 0
+        : 1;
 
       if (nextAttemptNumber > MAX_TASK_RUN_ATTEMPTS) {
         const service = new CrashTaskRunService(this._prisma);

apps/webapp/app/v3/services/finalizeTaskRun.server.ts

@@ -3,11 +3,17 @@ import { type Prisma, type TaskRun } from "@trigger.dev/database";
 import { logger } from "~/services/logger.server";
 import { marqs, sanitizeQueueName } from "~/v3/marqs/index.server";
 import { generateFriendlyId } from "../friendlyIdentifiers";
-import { FINAL_ATTEMPT_STATUSES, isFailedRunStatus, type FINAL_RUN_STATUSES } from "../taskStatus";
+import {
+  FINAL_ATTEMPT_STATUSES,
+  isFailedRunStatus,
+  isFatalRunStatus,
+  type FINAL_RUN_STATUSES,
+} from "../taskStatus";
 import { PerformTaskRunAlertsService } from "./alerts/performTaskRunAlerts.server";
 import { BaseService } from "./baseService.server";
 import { ResumeDependentParentsService } from "./resumeDependentParents.server";
 import { ExpireEnqueuedRunService } from "./expireEnqueuedRun.server";
+import { socketIo } from "../handleSocketIo.server";
 
 type BaseInput = {
   id: string;
@@ -90,6 +96,42 @@ export class FinalizeTaskRunService extends BaseService {
       await PerformTaskRunAlertsService.enqueue(run.id, this._prisma);
     }
 
+    if (isFatalRunStatus(run.status)) {
+      logger.error("FinalizeTaskRunService: Fatal status", { runId: run.id, status: run.status });
+
+      const extendedRun = await this._prisma.taskRun.findFirst({
+        where: { id: run.id },
+        select: {
+          id: true,
+          lockedToVersion: {
+            select: {
+              supportsLazyAttempts: true,
+            },
+          },
+          runtimeEnvironment: {
+            select: {
+              type: true,
+            },
+          },
+        },
+      });
+
+      if (extendedRun && extendedRun.runtimeEnvironment.type !== "DEVELOPMENT") {
+        logger.error("FinalizeTaskRunService: Fatal status, requesting worker exit", {
+          runId: run.id,
+          status: run.status,
+        });
+
+        // Signal to exit any leftover containers
+        socketIo.coordinatorNamespace.emit("REQUEST_RUN_CANCELLATION", {
+          version: "v1",
+          runId: run.id,
+          // Give the run a few seconds to exit to complete any flushing etc
+          delayInMs: extendedRun.lockedToVersion?.supportsLazyAttempts ? 5_000 : undefined,
+        });
+      }
+    }
+
     return run as Output<T>;
   }
 
@@ -111,83 +153,90 @@ export class FinalizeTaskRunService extends BaseService {
     error?: TaskRunError;
     run: TaskRun;
   }) {
-    if (attemptStatus || error) {
-      const latestAttempt = await this._prisma.taskRunAttempt.findFirst({
-        where: { taskRunId: run.id },
-        orderBy: { id: "desc" },
-        take: 1,
+    if (!attemptStatus && !error) {
+      logger.error("FinalizeTaskRunService: No attemptStatus or error provided", { runId: run.id });
+      return;
+    }
+
+    const latestAttempt = await this._prisma.taskRunAttempt.findFirst({
+      where: { taskRunId: run.id },
+      orderBy: { id: "desc" },
+      take: 1,
+    });
+
+    if (latestAttempt) {
+      logger.debug("Finalizing run attempt", {
+        id: latestAttempt.id,
+        status: attemptStatus,
+        error,
       });
 
-      if (latestAttempt) {
-        logger.debug("Finalizing run attempt", {
-          id: latestAttempt.id,
-          status: attemptStatus,
-          error,
-        });
+      await this._prisma.taskRunAttempt.update({
+        where: { id: latestAttempt.id },
+        data: { status: attemptStatus, error: error ? sanitizeError(error) : undefined },
+      });
 
-        await this._prisma.taskRunAttempt.update({
-          where: { id: latestAttempt.id },
-          data: { status: attemptStatus, error: error ? sanitizeError(error) : undefined },
-        });
-      } else {
-        logger.debug("Finalizing run no attempt found", {
-          runId: run.id,
-          attemptStatus,
-          error,
-        });
-
-        if (!run.lockedById) {
-          logger.error(
-            "FinalizeTaskRunService: No lockedById, so can't get the BackgroundWorkerTask. Not creating an attempt.",
-            { runId: run.id }
-          );
-          return;
-        }
-
-        const workerTask = await this._prisma.backgroundWorkerTask.findFirst({
-          select: {
-            id: true,
-            workerId: true,
-            runtimeEnvironmentId: true,
-          },
-          where: {
-            id: run.lockedById,
-          },
-        });
-
-        if (!workerTask) {
-          logger.error("FinalizeTaskRunService: No worker task found", { runId: run.id });
-          return;
-        }
-
-        const queue = await this._prisma.taskQueue.findUnique({
-          where: {
-            runtimeEnvironmentId_name: {
-              runtimeEnvironmentId: workerTask.runtimeEnvironmentId,
-              name: sanitizeQueueName(run.queue),
-            },
-          },
-        });
-
-        if (!queue) {
-          logger.error("FinalizeTaskRunService: No queue found", { runId: run.id });
-          return;
-        }
-
-        await this._prisma.taskRunAttempt.create({
-          data: {
-            number: 1,
-            friendlyId: generateFriendlyId("attempt"),
-            taskRunId: run.id,
-            backgroundWorkerId: workerTask?.workerId,
-            backgroundWorkerTaskId: workerTask?.id,
-            queueId: queue.id,
-            runtimeEnvironmentId: workerTask.runtimeEnvironmentId,
-            status: attemptStatus,
-            error: error ? sanitizeError(error) : undefined,
-          },
-        });
-      }
+      return;
     }
+
+    // There's no attempt, so create one
+
+    logger.debug("Finalizing run no attempt found", {
+      runId: run.id,
+      attemptStatus,
+      error,
+    });
+
+    if (!run.lockedById) {
+      logger.error(
+        "FinalizeTaskRunService: No lockedById, so can't get the BackgroundWorkerTask. Not creating an attempt.",
+        { runId: run.id }
+      );
+      return;
+    }
+
+    const workerTask = await this._prisma.backgroundWorkerTask.findFirst({
+      select: {
+        id: true,
+        workerId: true,
+        runtimeEnvironmentId: true,
+      },
+      where: {
+        id: run.lockedById,
+      },
+    });
+
+    if (!workerTask) {
+      logger.error("FinalizeTaskRunService: No worker task found", { runId: run.id });
+      return;
+    }
+
+    const queue = await this._prisma.taskQueue.findUnique({
+      where: {
+        runtimeEnvironmentId_name: {
+          runtimeEnvironmentId: workerTask.runtimeEnvironmentId,
+          name: sanitizeQueueName(run.queue),
+        },
+      },
+    });
+
+    if (!queue) {
+      logger.error("FinalizeTaskRunService: No queue found", { runId: run.id });
+      return;
+    }
+
+    await this._prisma.taskRunAttempt.create({
+      data: {
+        number: 1,
+        friendlyId: generateFriendlyId("attempt"),
+        taskRunId: run.id,
+        backgroundWorkerId: workerTask?.workerId,
+        backgroundWorkerTaskId: workerTask?.id,
+        queueId: queue.id,
+        runtimeEnvironmentId: workerTask.runtimeEnvironmentId,
+        status: attemptStatus,
+        error: error ? sanitizeError(error) : undefined,
+      },
+    });
   }
 }

apps/webapp/app/v3/services/restoreCheckpoint.server.ts

@@ -112,4 +112,24 @@ export class RestoreCheckpointService extends BaseService {
 
     return checkpoint;
   }
+
+  async getLastCheckpointEventIfUnrestored(runId: string) {
+    const event = await this._prisma.checkpointRestoreEvent.findFirst({
+      where: {
+        runId,
+      },
+      take: 1,
+      orderBy: {
+        createdAt: "desc",
+      },
+    });
+
+    if (!event) {
+      return;
+    }
+
+    if (event.type === "CHECKPOINT") {
+      return event;
+    }
+  }
 }

apps/webapp/app/v3/services/resumeAttempt.server.ts

@@ -2,23 +2,24 @@ import {
   CoordinatorToPlatformMessages,
   TaskRunExecution,
   TaskRunExecutionResult,
-  WaitReason,
 } from "@trigger.dev/core/v3";
 import type { InferSocketMessageSchema } from "@trigger.dev/core/v3/zodSocket";
 import { $transaction, PrismaClientOrTransaction } from "~/db.server";
 import { logger } from "~/services/logger.server";
 import { marqs } from "~/v3/marqs/index.server";
 import { socketIo } from "../handleSocketIo.server";
-import { SharedQueueMessageBody, sharedQueueTasks } from "../marqs/sharedQueueConsumer.server";
+import { sharedQueueTasks } from "../marqs/sharedQueueConsumer.server";
 import { BaseService } from "./baseService.server";
 import { TaskRunAttempt } from "@trigger.dev/database";
 import { isFinalRunStatus } from "../taskStatus";
 
 export class ResumeAttemptService extends BaseService {
+  private _logger = logger;
+
   public async call(
     params: InferSocketMessageSchema<typeof CoordinatorToPlatformMessages, "READY_FOR_RESUME">
   ): Promise<void> {
-    logger.debug(`ResumeAttemptService.call()`, params);
+    this._logger.debug(`ResumeAttemptService.call()`, params);
 
     await $transaction(this._prisma, async (tx) => {
       const attempt = await tx.taskRunAttempt.findUnique({
@@ -77,16 +78,18 @@ export class ResumeAttemptService extends BaseService {
       });
 
       if (!attempt) {
-        logger.error("Could not find attempt", { attemptFriendlyId: params.attemptFriendlyId });
+        this._logger.error("Could not find attempt", params);
         return;
       }
 
+      this._logger = logger.child({
+        attemptId: attempt.id,
+        attemptFriendlyId: attempt.friendlyId,
+        taskRun: attempt.taskRun,
+      });
+
       if (isFinalRunStatus(attempt.taskRun.status)) {
-        logger.error("Run is not resumable", {
-          attemptId: attempt.id,
-          runId: attempt.taskRunId,
-          status: attempt.taskRun.status,
-        });
+        this._logger.error("Run is not resumable");
         return;
       }
 
@@ -94,10 +97,7 @@ export class ResumeAttemptService extends BaseService {
 
       switch (params.type) {
         case "WAIT_FOR_DURATION": {
-          logger.debug("Sending duration wait resume message", {
-            attemptId: attempt.id,
-            attemptFriendlyId: params.attemptFriendlyId,
-          });
+          this._logger.debug("Sending duration wait resume message");
 
           await this.#setPostResumeStatuses(attempt, tx);
 
@@ -114,13 +114,13 @@ export class ResumeAttemptService extends BaseService {
             const dependentAttempt = attempt.dependencies[0].taskRun.attempts[0];
 
             if (!dependentAttempt) {
-              logger.error("No dependent attempt", { attemptId: attempt.id });
+              this._logger.error("No dependent attempt");
               return;
             }
 
             completedAttemptIds = [dependentAttempt.id];
           } else {
-            logger.error("No task dependency", { attemptId: attempt.id });
+            this._logger.error("No task dependency");
             return;
           }
 
@@ -134,13 +134,13 @@ export class ResumeAttemptService extends BaseService {
             const dependentBatchItems = attempt.batchDependencies[0].items;
 
             if (!dependentBatchItems) {
-              logger.error("No dependent batch items", { attemptId: attempt.id });
+              this._logger.error("No dependent batch items");
               return;
             }
 
             completedAttemptIds = dependentBatchItems.map((item) => item.taskRun.attempts[0]?.id);
           } else {
-            logger.error("No batch dependency", { attemptId: attempt.id });
+            this._logger.error("No batch dependency");
             return;
           }
 
@@ -161,7 +161,7 @@ export class ResumeAttemptService extends BaseService {
     tx: PrismaClientOrTransaction
   ) {
     if (completedAttemptIds.length === 0) {
-      logger.error("No completed attempt IDs", { attemptId: attempt.id });
+      this._logger.error("No completed attempt IDs");
       return;
     }
 
@@ -184,38 +184,36 @@ export class ResumeAttemptService extends BaseService {
       });
 
       if (!completedAttempt) {
-        logger.error("Completed attempt not found", {
-          attemptId: attempt.id,
-          completedAttemptId,
-        });
+        this._logger.error("Completed attempt not found", { completedAttemptId });
         await marqs?.acknowledgeMessage(attempt.taskRunId);
         return;
       }
 
+      const logger = this._logger.child({
+        completedAttemptId: completedAttempt.id,
+        completedAttemptFriendlyId: completedAttempt.friendlyId,
+        completedRunId: completedAttempt.taskRunId,
+      });
+
       const completion = await sharedQueueTasks.getCompletionPayloadFromAttempt(
         completedAttempt.id
       );
 
       if (!completion) {
-        logger.error("Failed to get completion payload", {
-          attemptId: attempt.id,
-          completedAttemptId,
-        });
+        logger.error("Failed to get completion payload");
         await marqs?.acknowledgeMessage(attempt.taskRunId);
         return;
       }
 
       completions.push(completion);
 
-      const executionPayload = await sharedQueueTasks.getExecutionPayloadFromAttempt(
-        completedAttempt.id
-      );
+      const executionPayload = await sharedQueueTasks.getExecutionPayloadFromAttempt({
+        id: completedAttempt.id,
+        skipStatusChecks: true, // already checked when getting the completion
+      });
 
       if (!executionPayload) {
-        logger.error("Failed to get execution payload", {
-          attemptId: attempt.id,
-          completedAttemptId,
-        });
+        logger.error("Failed to get execution payload");
         await marqs?.acknowledgeMessage(attempt.taskRunId);
         return;
       }

apps/webapp/app/v3/services/triggerTask.server.ts

@@ -378,6 +378,7 @@ export class TriggerTaskService extends BaseService {
                   maxDurationInSeconds: body.options?.maxDuration
                     ? clampMaxDuration(body.options.maxDuration)
                     : undefined,
+                  runTags: bodyTags,
                 },
               });