Latest lockfile

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.changeset/angry-eagles-trade.md

@@ -0,0 +1,9 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+- Fix additionalFiles that aren't decendants
+- Stop swallowing uncaught exceptions in prod
+- Improve warnings and errors, fail early on critical warnings
+- New arg to --save-logs even for successful builds

.changeset/beige-pens-dance.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+v3 CLI update command and package manager detection fix

.changeset/clever-apes-collect.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Fix error stack traces

.changeset/clever-carrots-travel.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Implement task.onSuccess/onFailure and config.onSuccess/onFailure

.changeset/config.json

@@ -4,7 +4,8 @@
   "commit": false,
   "fixed": [
     [
-      "@trigger.dev/*"
+      "@trigger.dev/*",
+      "trigger.dev"
     ]
   ],
   "linked": [],

.changeset/cuddly-feet-approve.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Don’t swallow some error messages when deploying

.changeset/dry-walls-check.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Add option to print console logs in the dev CLI locally (issue #1014)

.changeset/eight-pumas-float.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Fixed batch otel flushing

.changeset/eleven-paws-join.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Remove the env var check during deploy (too many false negatives)

.changeset/few-students-share.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix permissions inside node_modules

.changeset/funny-swans-destroy.md

@@ -0,0 +1,9 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Make msw a normal dependency (for now) to fix Module Not Found error in Next.js.
+
+It turns out that webpack will "hoist" dynamically imported modules and attempt to resolve them at build time, even though it's an optional peer dep:
+
+https://x.com/maverickdotdev/status/1782465214308319404

.changeset/green-bags-wink.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/react": patch
+---
+
+Fix for shared queryKey between useRunDetails and useRunStatuses

.changeset/khaki-poems-lay.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Export queue from the SDK

.changeset/late-icons-lie.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Display errors for runs and deployments

.changeset/late-steaks-behave.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+v3: fix digest extraction

.changeset/light-dragons-complain.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Correctly handle self-hosted deploy command errors

.changeset/loud-actors-remember.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Default to retrying enabled in dev when running init

.changeset/mighty-flowers-train.md

@@ -0,0 +1,83 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Adds support for `emitDecoratorMetadata: true` and `experimentalDecorators: true` in your tsconfig using the [`@anatine/esbuild-decorators`](https://github.com/anatine/esbuildnx/tree/main/packages/esbuild-decorators) package. This allows you to use libraries like TypeORM: 
+
+```ts orm/index.ts
+import "reflect-metadata";
+import { DataSource } from "typeorm";
+import { Entity, Column, PrimaryColumn } from "typeorm";
+
+@Entity()
+export class Photo {
+  @PrimaryColumn()
+  id!: number;
+
+  @Column()
+  name!: string;
+
+  @Column()
+  description!: string;
+
+  @Column()
+  filename!: string;
+
+  @Column()
+  views!: number;
+
+  @Column()
+  isPublished!: boolean;
+}
+
+export const AppDataSource = new DataSource({
+  type: "postgres",
+  host: "localhost",
+  port: 5432,
+  username: "postgres",
+  password: "postgres",
+  database: "v3-catalog",
+  entities: [Photo],
+  synchronize: true,
+  logging: false,
+});
+```
+
+And then in your trigger.config.ts file you can initialize the datasource using the new `init` option:
+
+```ts trigger.config.ts
+import type { TriggerConfig } from "@trigger.dev/sdk/v3";
+import { AppDataSource } from "@/trigger/orm";
+
+export const config: TriggerConfig = {
+  // ... other options here
+  init: async (payload, { ctx }) => {
+    await AppDataSource.initialize();
+  },
+};
+```
+
+Now you are ready to use this in your tasks:
+
+```ts
+import { task } from "@trigger.dev/sdk/v3";
+import { AppDataSource, Photo } from "./orm";
+
+export const taskThatUsesDecorators = task({
+  id: "taskThatUsesDecorators",
+  run: async (payload: { message: string }) => {
+    console.log("Creating a photo...");
+
+    const photo = new Photo();
+    photo.id = 2;
+    photo.name = "Me and Bears";
+    photo.description = "I am near polar bears";
+    photo.filename = "photo-with-bears.jpg";
+    photo.views = 1;
+    photo.isPublished = true;
+
+    await AppDataSource.manager.save(photo);
+  },
+});
+```

.changeset/nasty-jars-pump.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fixes an issue that was treating v2 trigger directories as v3

.changeset/new-rivers-tell.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Added replayRun function to the SDK

.changeset/ninety-pets-travel.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Improve the SDK function types and expose a new APIError instead of the APIResult type

.changeset/polite-ducks-switch.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix post start hooks

.changeset/polite-rockets-matter.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix dev CLI output when not printing update messages

.changeset/poor-flowers-cross.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Use the dashboard url instead of the API url for the View logs link

.changeset/pre.json

@@ -44,21 +44,68 @@
     "@trigger.dev/yalt": "2.3.18"
   },
   "changesets": [
+    "angry-eagles-trade",
+    "beige-pens-dance",
     "breezy-gorillas-mate",
     "chilled-hornets-move",
     "clean-pianos-listen",
+    "clever-apes-collect",
+    "clever-carrots-travel",
     "cool-glasses-bake",
+    "cuddly-feet-approve",
+    "dry-walls-check",
+    "eight-pumas-float",
+    "eleven-paws-join",
+    "few-students-share",
+    "funny-swans-destroy",
+    "green-bags-wink",
     "khaki-apricots-design",
+    "khaki-poems-lay",
+    "late-icons-lie",
+    "late-steaks-behave",
     "lemon-jobs-repair",
     "light-bulldogs-press",
+    "light-dragons-complain",
+    "loud-actors-remember",
     "many-ligers-pump",
     "mighty-camels-joke",
+    "mighty-flowers-train",
+    "nasty-jars-pump",
+    "new-rivers-tell",
+    "ninety-pets-travel",
     "odd-poets-own",
+    "polite-ducks-switch",
+    "polite-rockets-matter",
+    "poor-flowers-cross",
+    "rare-roses-float",
     "real-planets-stare",
+    "rich-kangaroos-unite",
+    "rotten-beers-refuse",
+    "rotten-dryers-exercise",
+    "selfish-ducks-sort",
+    "shaggy-spoons-taste",
+    "sharp-emus-compare",
+    "sharp-zebras-serve",
+    "shiny-coats-cry",
+    "silly-suits-switch",
+    "slow-buses-own",
+    "smart-needles-move",
     "smart-olives-eat",
+    "spicy-lamps-smoke",
     "strange-ghosts-matter",
+    "strong-lemons-add",
     "stupid-bulldogs-applaud",
     "sweet-lizards-press",
-    "tall-bees-wave"
+    "swift-dragons-peel",
+    "tall-bees-wave",
+    "tame-guests-know",
+    "tender-oranges-rhyme",
+    "tidy-balloons-suffer",
+    "tidy-dryers-sleep",
+    "tiny-doors-type",
+    "tiny-elephants-scream",
+    "tricky-bulldogs-heal",
+    "tricky-ladybugs-unite",
+    "two-pumas-wait"
   ]
 }

.changeset/rare-roses-float.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Add openssl to prod worker image and allow passing auth token via env var for deploy

.changeset/rich-kangaroos-unite.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix a bug where revoking the CLI token would prevent you from ever logging in again with the CLI.

.changeset/rotten-beers-refuse.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Add git to prod worker image which fixes private package installs

.changeset/rotten-dryers-exercise.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Adding task with a triggerSource of schedule

.changeset/selfish-ducks-sort.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+---
+
+Lock SDK and CLI deps on exact core version

.changeset/shaggy-spoons-taste.md

@@ -0,0 +1,56 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Updates the `trigger`, `batchTrigger` and their `*AndWait` variants to use the first parameter for the payload/items, and the second parameter for options.
+
+Before:
+
+```ts
+await yourTask.trigger({ payload: { foo: "bar" }, options: { idempotencyKey: "key_1234" } });
+await yourTask.triggerAndWait({ payload: { foo: "bar" }, options: { idempotencyKey: "key_1234" } });
+
+await yourTask.batchTrigger({ items: [{ payload: { foo: "bar" } }, { payload: { foo: "baz" } }] });
+await yourTask.batchTriggerAndWait({ items: [{ payload: { foo: "bar" } }, { payload: { foo: "baz" } }] });
+```
+
+After:
+
+```ts
+await yourTask.trigger({ foo: "bar" }, { idempotencyKey: "key_1234" });
+await yourTask.triggerAndWait({ foo: "bar" }, { idempotencyKey: "key_1234" });
+
+await yourTask.batchTrigger([{ payload: { foo: "bar" } }, { payload: { foo: "baz" } }]);
+await yourTask.batchTriggerAndWait([{ payload: { foo: "bar" } }, { payload: { foo: "baz" } }]);
+```
+
+We've also changed the API of the `triggerAndWait` result. Before, if the subtask that was triggered finished with an error, we would automatically "rethrow" the error in the parent task.
+
+Now instead we're returning a `TaskRunResult` object that allows you to discriminate between successful and failed runs in the subtask:
+
+Before:
+
+```ts
+try {
+  const result = await yourTask.triggerAndWait({ foo: "bar" });
+
+  // result is the output of your task
+  console.log("result", result);
+
+} catch (error) {
+  // handle subtask errors here
+}
+```
+
+After:
+
+```ts
+const result = await yourTask.triggerAndWait({ foo: "bar" });
+
+if (result.ok) {
+  console.log(`Run ${result.id} succeeded with output`, result.output);
+} else {
+  console.log(`Run ${result.id} failed with error`, result.error);
+}
+```

.changeset/sharp-emus-compare.md

@@ -0,0 +1,13 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+When using idempotency keys, triggerAndWait and batchTriggerAndWait will still work even if the existing runs have already been completed (or even partially completed, in the case of batchTriggerAndWait)
+
+- TaskRunExecutionResult.id is now the run friendlyId, not the attempt friendlyId
+- A single TaskRun can now have many batchItems, in the case of batchTriggerAndWait while using idempotency keys
+- A run’s idempotencyKey is now added to the ctx as well as the TaskEvent and displayed in the span view
+- When resolving batchTriggerAndWait, the runtimes no longer reject promises, leading to an error in the parent task
+

.changeset/sharp-zebras-serve.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Update trigger.dev CLI for new batch otel support

.changeset/shiny-coats-cry.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Add additional logging around cleaning up dev workers, and always kill them after 5 seconds if they haven't already exited

.changeset/silly-suits-switch.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/otlp-importer": patch
+"trigger.dev": patch
+---
+
+Fix package builds and CLI commands on Windows

.changeset/slow-buses-own.md

@@ -0,0 +1,8 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+"@trigger.dev/cli": patch
+---
+
+Move to our global system from AsyncLocalStorage for the current task context storage

.changeset/smart-needles-move.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Remove unimplemented batchOptions

.changeset/spicy-lamps-smoke.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fixing an issue with bundling @trigger.dev/core/v3 in dev when using pnpm

.changeset/strong-lemons-add.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Make optional schedule object fields nullish

.changeset/swift-dragons-peel.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Added JSDocs to the schedule SDK types

.changeset/tame-guests-know.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Changed the binary name from trigger.dev to triggerdev to fix a Windows issue

.changeset/tender-oranges-rhyme.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Stop swallowing deployment errors and display them better

.changeset/tidy-balloons-suffer.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Init command was failing on Windows because of bad template paths

.changeset/tidy-dryers-sleep.md

@@ -0,0 +1,10 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Fixes an issue that caused failed tasks when resuming after calling `triggerAndWait` or `batchTriggerAndWait` in prod/staging (this doesn't effect dev).
+
+The version of Node.js we use for deployed workers (latest 20) would crash with an out-of-memory error when the checkpoint was restored. This crash does not happen on Node 18x or Node21x, so we've decided to upgrade the worker version to Node.js21x, to mitigate this issue.
+
+You'll need to re-deploy to production to fix the issue.

.changeset/tiny-doors-type.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Added cancelRun to the SDK

.changeset/tiny-elephants-scream.md

@@ -0,0 +1,7 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+- Add graceful exit for prod workers
+- Prevent overflow in long waits

.changeset/tricky-bulldogs-heal.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Added a new global - Task Catalog - to better handle task metadata

.changeset/tricky-ladybugs-unite.md

@@ -0,0 +1,8 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Extracting out all the non-SDK related features from the main @trigger.dev/core/v3 export

.changeset/two-pumas-wait.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/cli": patch
+---
+
+Add support for tasks located in subdirectories inside trigger dirs

.github/workflows/pr_checks.yml

@@ -27,6 +27,6 @@ jobs:
     uses: ./.github/workflows/unit-tests.yml
     secrets: inherit
 
-  e2e:
-    uses: ./.github/workflows/e2e.yml
-    secrets: inherit
+  # e2e:
+  #   uses: ./.github/workflows/e2e.yml
+  #   secrets: inherit

.github/workflows/publish.yml

@@ -49,11 +49,11 @@ jobs:
     uses: ./.github/workflows/unit-tests.yml
     secrets: inherit
 
-  e2e:
-    uses: ./.github/workflows/e2e.yml
-    secrets: inherit
+  # e2e:
+  #   uses: ./.github/workflows/e2e.yml
+  #   secrets: inherit
 
   publish:
-    needs: [typecheck, units, e2e]
+    needs: [typecheck, units]
     uses: ./.github/workflows/publish-docker.yml
     secrets: inherit

.vscode/launch.json

@@ -23,7 +23,7 @@
     {
       "type": "node-terminal",
       "request": "launch",
-      "name": "Debug BYO Auth",
+      "name": "Debug v2 job catalog",
       "command": "pnpm run byo-auth",
       "envFile": "${workspaceFolder}/references/job-catalog/.env",
       "cwd": "${workspaceFolder}/references/job-catalog",
@@ -33,7 +33,7 @@
       "type": "node-terminal",
       "request": "launch",
       "name": "Debug V3 Dev CLI",
-      "command": "pnpm exec trigger.dev dev --log-level debug",
+      "command": "pnpm exec triggerdev dev --log-level debug",
       "cwd": "${workspaceFolder}/references/v3-catalog",
       "sourceMaps": true
     },
@@ -41,7 +41,7 @@
       "type": "node-terminal",
       "request": "launch",
       "name": "Debug V3 Deploy CLI",
-      "command": "pnpm exec trigger.dev deploy",
+      "command": "pnpm exec triggerdev deploy",
       "cwd": "${workspaceFolder}/references/v3-catalog",
       "sourceMaps": true
     },

README.md

@@ -7,7 +7,7 @@
   
 ### The open source background jobs framework
 
-[Discord](https://discord.gg/JtBAxBr2m3) | [Website](https://trigger.dev) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Docs](https://trigger.dev/docs)
+[Discord](https://trigger.dev/discord) | [Website](https://trigger.dev) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Docs](https://trigger.dev/docs)
 
 [![Twitter](https://img.shields.io/twitter/url/https/twitter.com/triggerdotdev.svg?style=social&label=Follow%20%40trigger.dev)](https://twitter.com/triggerdotdev)
 [![GitHub Repo stars](https://img.shields.io/github/stars/triggerdotdev/trigger.dev?style=social)](https://github.com/triggerdotdev/trigger.dev)

apps/coordinator/package.json

@@ -21,8 +21,8 @@
     "execa": "^8.0.1",
     "nanoid": "^5.0.6",
     "prom-client": "^15.1.0",
-    "socket.io": "^4.7.4",
-    "socket.io-client": "^4.7.4"
+    "socket.io": "4.7.4",
+    "socket.io-client": "4.7.4"
   },
   "devDependencies": {
     "@types/node": "^18",

apps/coordinator/src/index.ts

@@ -8,9 +8,9 @@ import {
   PlatformToCoordinatorMessages,
   ProdWorkerSocketData,
   ProdWorkerToCoordinatorMessages,
-  ZodNamespace,
-  ZodSocketConnection,
 } from "@trigger.dev/core/v3";
+import { ZodNamespace } from "@trigger.dev/core/v3/zodNamespace";
+import { ZodSocketConnection } from "@trigger.dev/core/v3/zodSocket";
 import { HttpReply, getTextBody, SimpleLogger } from "@trigger.dev/core-apps";
 
 import { collectDefaultMetrics, register, Gauge } from "prom-client";
@@ -18,6 +18,8 @@ collectDefaultMetrics();
 
 const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || 8020);
 const NODE_NAME = process.env.NODE_NAME || "coordinator";
+const DEFAULT_RETRY_DELAY_THRESHOLD_IN_MS = 30_000;
+
 const REGISTRY_HOST = process.env.REGISTRY_HOST || "localhost:5000";
 const CHECKPOINT_PATH = process.env.CHECKPOINT_PATH || "/checkpoints";
 const REGISTRY_TLS_VERIFY = process.env.REGISTRY_TLS_VERIFY === "false" ? "false" : "true";
@@ -334,12 +336,19 @@ class TaskCoordinator {
     { resolve: (value: void) => void; reject: (err?: any) => void }
   >();
 
+  #delayThresholdInMs: number;
+
   constructor(
     private port: number,
     private host = "0.0.0.0"
   ) {
     this.#httpServer = this.#createHttpServer();
     this.#checkpointer.initialize();
+    this.#delayThresholdInMs = this.#getDelayThreshold();
+
+    if (process.env.DELAY_THRESHOLD_IN_MS) {
+      this.#delayThresholdInMs = this.#getDelayThreshold();
+    }
 
     const io = new Server(this.#httpServer);
     this.#prodWorkerNamespace = this.#createProdWorkerNamespace(io);
@@ -356,6 +365,27 @@ class TaskCoordinator {
     register.registerMetric(connectedTasksTotal);
   }
 
+  #getDelayThreshold() {
+    if (!process.env.RETRY_DELAY_THRESHOLD_IN_MS) {
+      return DEFAULT_RETRY_DELAY_THRESHOLD_IN_MS;
+    }
+
+    const threshold = parseInt(process.env.RETRY_DELAY_THRESHOLD_IN_MS);
+
+    if (isNaN(threshold)) {
+      logger.log(
+        "RETRY_DELAY_THRESHOLD_IN_MS parses as NaN, must supply integer. Will use default instead.",
+        {
+          RETRY_DELAY_THRESHOLD_IN_MS: process.env.RETRY_DELAY_THRESHOLD_IN_MS,
+          DEFAULT_DELAY_THRESHOLD_IN_MS: DEFAULT_RETRY_DELAY_THRESHOLD_IN_MS,
+        }
+      );
+      return DEFAULT_RETRY_DELAY_THRESHOLD_IN_MS;
+    }
+
+    return threshold;
+  }
+
   #createPlatformSocket() {
     if (!PLATFORM_ENABLED) {
       console.log("INFO: platform connection disabled");
@@ -629,7 +659,7 @@ class TaskCoordinator {
             return;
           }
 
-          if (completion.retry.delay < 10_000) {
+          if (completion.retry.delay < this.#delayThresholdInMs) {
             completeWithoutCheckpoint(false);
             return;
           }

apps/docker-provider/src/index.ts

@@ -257,6 +257,7 @@ class DockerTaskOperations implements TaskOperations {
       return await execa("docker", [
         "exec",
         containerName,
+        "busybox",
         "wget",
         "-q",
         "-O-",

apps/kubernetes-provider/package.json

@@ -19,6 +19,7 @@
     "@kubernetes/client-node": "^0.20.0",
     "@trigger.dev/core": "workspace:*",
     "@trigger.dev/core-apps": "workspace:*",
+    "p-queue": "^8.0.1",
     "socket.io-client": "^4.7.4"
   },
   "devDependencies": {

apps/kubernetes-provider/src/index.ts

@@ -9,6 +9,7 @@ import {
 } from "@trigger.dev/core-apps";
 import { Machine, PostStartCauses, PreStopCauses, EnvironmentType } from "@trigger.dev/core/v3";
 import { randomUUID } from "crypto";
+import { TaskMonitor } from "./taskMonitor";
 
 const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
 const NODE_NAME = process.env.NODE_NAME || "local";
@@ -24,6 +25,10 @@ type Namespace = {
   };
 };
 
+type ComputeResources = {
+  [K in "cpu" | "memory" | "ephemeral-storage"]?: string;
+};
+
 class KubernetesTaskOperations implements TaskOperations {
   #namespace: Namespace;
   #k8sApi: {
@@ -55,25 +60,15 @@ class KubernetesTaskOperations implements TaskOperations {
           template: {
             metadata: {
               labels: {
+                ...this.#getSharedLabels(opts),
                 app: "task-index",
                 "app.kubernetes.io/part-of": "trigger-worker",
                 "app.kubernetes.io/component": "index",
-                env: opts.envId,
-                envtype: this.#envTypeToLabelValue(opts.envType),
-                org: opts.orgId,
-                project: opts.projectId,
+                deployment: opts.deploymentId,
               },
             },
             spec: {
-              restartPolicy: "Never",
-              imagePullSecrets: [
-                {
-                  name: "registry-trigger",
-                },
-              ],
-              nodeSelector: {
-                nodetype: "worker",
-              },
+              ...this.#defaultPodSpec,
               containers: [
                 {
                   name: this.#getIndexContainerName(opts.shortCode),
@@ -83,12 +78,13 @@ class KubernetesTaskOperations implements TaskOperations {
                       containerPort: 8000,
                     },
                   ],
-                  // resources: {
-                  //   limits: {
-                  //     cpu: "100m",
-                  //     memory: "50Mi",
-                  //   },
-                  // },
+                  resources: {
+                    limits: {
+                      cpu: "250m",
+                      memory: "0.5G",
+                      "ephemeral-storage": "2Gi",
+                    },
+                  },
                   lifecycle: {
                     preStop: {
                       exec: {
@@ -97,10 +93,7 @@ class KubernetesTaskOperations implements TaskOperations {
                     },
                   },
                   env: [
-                    {
-                      name: "DEBUG",
-                      value: "true",
-                    },
+                    ...this.#getSharedEnv(opts.envId),
                     {
                       name: "INDEX_TASKS",
                       value: "true",
@@ -113,42 +106,6 @@ class KubernetesTaskOperations implements TaskOperations {
                       name: "TRIGGER_API_URL",
                       value: opts.apiUrl,
                     },
-                    {
-                      name: "TRIGGER_ENV_ID",
-                      value: opts.envId,
-                    },
-                    {
-                      name: "OTEL_EXPORTER_OTLP_ENDPOINT",
-                      value: OTEL_EXPORTER_OTLP_ENDPOINT,
-                    },
-                    {
-                      name: "HTTP_SERVER_PORT",
-                      value: "8000",
-                    },
-                    {
-                      name: "POD_NAME",
-                      valueFrom: {
-                        fieldRef: {
-                          fieldPath: "metadata.name",
-                        },
-                      },
-                    },
-                    {
-                      name: "COORDINATOR_HOST",
-                      valueFrom: {
-                        fieldRef: {
-                          fieldPath: "status.hostIP",
-                        },
-                      },
-                    },
-                    {
-                      name: "MACHINE_NAME",
-                      valueFrom: {
-                        fieldRef: {
-                          fieldPath: "spec.nodeName",
-                        },
-                      },
-                    },
                   ],
                 },
               ],
@@ -167,26 +124,16 @@ class KubernetesTaskOperations implements TaskOperations {
           name: this.#getRunContainerName(opts.runId),
           namespace: this.#namespace.metadata.name,
           labels: {
+            ...this.#getSharedLabels(opts),
             app: "task-run",
             "app.kubernetes.io/part-of": "trigger-worker",
             "app.kubernetes.io/component": "create",
-            env: opts.envId,
-            envtype: this.#envTypeToLabelValue(opts.envType),
-            org: opts.orgId,
-            project: opts.projectId,
             run: opts.runId,
           },
         },
         spec: {
-          restartPolicy: "Never",
-          imagePullSecrets: [
-            {
-              name: "registry-trigger",
-            },
-          ],
-          nodeSelector: {
-            nodetype: "worker",
-          },
+          ...this.#defaultPodSpec,
+          terminationGracePeriodSeconds: 60 * 60,
           containers: [
             {
               name: this.#getRunContainerName(opts.runId),
@@ -197,14 +144,15 @@ class KubernetesTaskOperations implements TaskOperations {
                 },
               ],
               resources: {
-                limits: this.#getResourcesFromMachineConfig(opts.machine),
+                requests: {
+                  ...this.#defaultResourceRequests,
+                },
+                limits: {
+                  ...this.#defaultResourceLimits,
+                  ...this.#getResourcesFromMachineConfig(opts.machine),
+                },
               },
               lifecycle: {
-                postStart: {
-                  exec: {
-                    command: this.#getLifecycleCommand("postStart", "create"),
-                  },
-                },
                 preStop: {
                   exec: {
                     command: this.#getLifecycleCommand("preStop", "terminate"),
@@ -212,54 +160,11 @@ class KubernetesTaskOperations implements TaskOperations {
                 },
               },
               env: [
-                {
-                  name: "DEBUG",
-                  value: "true",
-                },
-                {
-                  name: "HTTP_SERVER_PORT",
-                  value: "8000",
-                },
-                {
-                  name: "TRIGGER_ENV_ID",
-                  value: opts.envId,
-                },
+                ...this.#getSharedEnv(opts.envId),
                 {
                   name: "TRIGGER_RUN_ID",
                   value: opts.runId,
                 },
-                {
-                  name: "TRIGGER_WORKER_VERSION",
-                  value: opts.version,
-                },
-                {
-                  name: "OTEL_EXPORTER_OTLP_ENDPOINT",
-                  value: OTEL_EXPORTER_OTLP_ENDPOINT,
-                },
-                {
-                  name: "POD_NAME",
-                  valueFrom: {
-                    fieldRef: {
-                      fieldPath: "metadata.name",
-                    },
-                  },
-                },
-                {
-                  name: "COORDINATOR_HOST",
-                  valueFrom: {
-                    fieldRef: {
-                      fieldPath: "status.hostIP",
-                    },
-                  },
-                },
-                {
-                  name: "NODE_NAME",
-                  valueFrom: {
-                    fieldRef: {
-                      fieldPath: "spec.nodeName",
-                    },
-                  },
-                },
               ],
               volumeMounts: [
                 {
@@ -288,27 +193,16 @@ class KubernetesTaskOperations implements TaskOperations {
           name: `${this.#getRunContainerName(opts.runId)}-${randomUUID().slice(0, 8)}`,
           namespace: this.#namespace.metadata.name,
           labels: {
+            ...this.#getSharedLabels(opts),
             app: "task-run",
             "app.kubernetes.io/part-of": "trigger-worker",
             "app.kubernetes.io/component": "restore",
-            env: opts.envId,
-            envtype: this.#envTypeToLabelValue(opts.envType),
-            org: opts.orgId,
-            project: opts.projectId,
             run: opts.runId,
             checkpoint: opts.checkpointId,
           },
         },
         spec: {
-          restartPolicy: "Never",
-          imagePullSecrets: [
-            {
-              name: "registry-trigger",
-            },
-          ],
-          nodeSelector: {
-            nodetype: "worker",
-          },
+          ...this.#defaultPodSpec,
           initContainers: [
             {
               name: "pull-base-image",
@@ -317,7 +211,7 @@ class KubernetesTaskOperations implements TaskOperations {
             },
             {
               name: "populate-taskinfo",
-              image: "busybox",
+              image: "docker.io/library/busybox",
               command: ["/bin/sh", "-c"],
               args: ["printenv COORDINATOR_HOST | tee /etc/taskinfo/coordinator-host"],
               env: [
@@ -348,7 +242,13 @@ class KubernetesTaskOperations implements TaskOperations {
                 },
               ],
               resources: {
-                limits: this.#getResourcesFromMachineConfig(opts.machine),
+                requests: {
+                  ...this.#defaultResourceRequests,
+                },
+                limits: {
+                  ...this.#defaultResourceLimits,
+                  ...this.#getResourcesFromMachineConfig(opts.machine),
+                },
               },
               lifecycle: {
                 postStart: {
@@ -406,7 +306,90 @@ class KubernetesTaskOperations implements TaskOperations {
     }
   }
 
-  #getResourcesFromMachineConfig(config: Machine) {
+  get #defaultPodSpec(): Omit<k8s.V1PodSpec, "containers"> {
+    return {
+      restartPolicy: "Never",
+      automountServiceAccountToken: false,
+      imagePullSecrets: [
+        {
+          name: "registry-trigger",
+        },
+      ],
+      nodeSelector: {
+        nodetype: "worker",
+      },
+    };
+  }
+
+  get #defaultResourceRequests(): ComputeResources {
+    return {
+      "ephemeral-storage": "2Gi",
+    };
+  }
+
+  get #defaultResourceLimits(): ComputeResources {
+    return {
+      "ephemeral-storage": "10Gi",
+    };
+  }
+
+  #getSharedEnv(envId: string): k8s.V1EnvVar[] {
+    return [
+      {
+        name: "TRIGGER_ENV_ID",
+        value: envId,
+      },
+      {
+        name: "DEBUG",
+        value: process.env.DEBUG ? "1" : "0",
+      },
+      {
+        name: "HTTP_SERVER_PORT",
+        value: "8000",
+      },
+      {
+        name: "OTEL_EXPORTER_OTLP_ENDPOINT",
+        value: OTEL_EXPORTER_OTLP_ENDPOINT,
+      },
+      {
+        name: "POD_NAME",
+        valueFrom: {
+          fieldRef: {
+            fieldPath: "metadata.name",
+          },
+        },
+      },
+      {
+        name: "COORDINATOR_HOST",
+        valueFrom: {
+          fieldRef: {
+            fieldPath: "status.hostIP",
+          },
+        },
+      },
+      {
+        name: "MACHINE_NAME",
+        valueFrom: {
+          fieldRef: {
+            fieldPath: "spec.nodeName",
+          },
+        },
+      },
+    ];
+  }
+
+  #getSharedLabels(
+    opts: TaskOperationsIndexOptions | TaskOperationsCreateOptions | TaskOperationsRestoreOptions
+  ): Record<string, string> {
+    return {
+      env: opts.envId,
+      envtype: this.#envTypeToLabelValue(opts.envType),
+      org: opts.orgId,
+      project: opts.projectId,
+    };
+  }
+
+  #getResourcesFromMachineConfig(config: Machine): ComputeResources {
     return {
       cpu: `${config.cpu}`,
       memory: `${config.memory}G`,
@@ -417,7 +400,19 @@ class KubernetesTaskOperations implements TaskOperations {
     type: THookType,
     cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses
   ) {
-    return ["/bin/sh", "-c", `sleep 1; wget -q -O- 127.0.0.1:8000/${type}?cause=${cause}`];
+    const retries = 5;
+
+    // This will retry sending the lifecycle hook up to `retries` times
+    // The sleep is required as this may start running before the HTTP server is up
+    const exec = [
+      "/bin/sh",
+      "-c",
+      `for i in $(seq ${retries}); do sleep 1; busybox wget -q -O- 127.0.0.1:8000/${type}?cause=${cause} && break; done`,
+    ];
+
+    logger.debug("getLifecycleCommand()", { exec });
+
+    return exec;
   }
 
   #getIndexContainerName(suffix: string) {
@@ -516,3 +511,34 @@ const provider = new ProviderShell({
 });
 
 provider.listen();
+
+const taskMonitor = new TaskMonitor({
+  runtimeEnv: RUNTIME_ENV,
+  onIndexFailure: async (deploymentId, failureInfo) => {
+    logger.log("Indexing failed", { deploymentId, failureInfo });
+
+    try {
+      provider.platformSocket.send("INDEXING_FAILED", {
+        deploymentId,
+        error: {
+          name: `Crashed with exit code ${failureInfo.exitCode}`,
+          message: failureInfo.reason,
+          stack: failureInfo.logs,
+        },
+      });
+    } catch (error) {
+      logger.error(error);
+    }
+  },
+  onRunFailure: async (runId, failureInfo) => {
+    logger.log("Run failed:", { runId, failureInfo });
+
+    try {
+      provider.platformSocket.send("WORKER_CRASHED", { runId, ...failureInfo });
+    } catch (error) {
+      logger.error(error);
+    }
+  },
+});
+
+taskMonitor.start();

apps/kubernetes-provider/src/taskMonitor.ts

@@ -0,0 +1,442 @@
+import * as k8s from "@kubernetes/client-node";
+import { SimpleLogger } from "@trigger.dev/core-apps";
+import { setTimeout } from "timers/promises";
+import PQueue from "p-queue";
+
+type IndexFailureHandler = (
+  deploymentId: string,
+  failureInfo: {
+    exitCode: number;
+    reason: string;
+    logs: string;
+  }
+) => Promise<any>;
+
+type RunFailureHandler = (
+  runId: string,
+  failureInfo: {
+    exitCode: number;
+    reason: string;
+    logs: string;
+  }
+) => Promise<any>;
+
+type TaskMonitorOptions = {
+  runtimeEnv: "local" | "kubernetes";
+  onIndexFailure?: IndexFailureHandler;
+  onRunFailure?: RunFailureHandler;
+  namespace?: string;
+};
+
+export class TaskMonitor {
+  #enabled = false;
+  #logger = new SimpleLogger("[TaskMonitor]");
+  #taskInformer: ReturnType<typeof k8s.makeInformer<k8s.V1Pod>>;
+  #processedPods = new Map<string, number>();
+  #queue = new PQueue({ concurrency: 10 });
+  #k8sClient: {
+    core: k8s.CoreV1Api;
+    kubeConfig: k8s.KubeConfig;
+  };
+
+  private namespace = "default";
+  private fieldSelector = "status.phase=Failed";
+  private labelSelector = "app in (task-index, task-run)";
+
+  constructor(private opts: TaskMonitorOptions) {
+    this.#k8sClient = this.#createK8sClient();
+
+    this.#taskInformer = this.#createTaskInformer();
+    this.#taskInformer.on("connect", this.#onInformerConnected.bind(this));
+    this.#taskInformer.on("error", this.#onInformerError.bind(this));
+    this.#taskInformer.on("update", this.#enqueueOnPodUpdated.bind(this));
+  }
+
+  #createTaskInformer() {
+    const listTasks = () =>
+      this.#k8sClient.core.listNamespacedPod(
+        this.namespace,
+        undefined,
+        undefined,
+        undefined,
+        this.fieldSelector,
+        this.labelSelector
+      );
+
+    // Uses watch with local caching
+    // https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes
+    const informer = k8s.makeInformer(
+      this.#k8sClient.kubeConfig,
+      `/api/v1/namespaces/${this.namespace}/pods`,
+      listTasks,
+      this.labelSelector,
+      this.fieldSelector
+    );
+
+    return informer;
+  }
+
+  async #onInformerConnected() {
+    this.#logger.log("Connected");
+  }
+
+  async #onInformerError(error: any) {
+    this.#logger.error("Error:", error);
+
+    // Automatic reconnect
+    await setTimeout(2_000);
+    this.#taskInformer.start();
+  }
+
+  #enqueueOnPodUpdated(pod: k8s.V1Pod) {
+    this.#queue.add(async () => {
+      try {
+        // It would be better to only pass the cache key, but the pod may already be removed from the cache by the time we process it
+        await this.#onPodUpdated(pod);
+      } catch (error) {
+        this.#logger.error("Caught onPodUpdated() error:", error);
+      }
+    });
+  }
+
+  async #onPodUpdated(pod: k8s.V1Pod) {
+    this.#logger.debug(`Updated: ${pod.metadata?.name}`);
+    this.#logger.debug("Updated", JSON.stringify(pod, null, 2));
+
+    // We only care about failures
+    if (pod.status?.phase !== "Failed") {
+      return;
+    }
+
+    const podName = pod.metadata?.name;
+
+    if (!podName) {
+      this.#logger.error("Pod is nameless", { pod });
+      return;
+    }
+
+    const containerStatus = pod.status.containerStatuses?.[0];
+
+    if (!containerStatus?.state) {
+      this.#logger.error("Pod failed, but container status doesn't have state", {
+        status: pod.status,
+      });
+      return;
+    }
+
+    if (this.#processedPods.has(podName)) {
+      this.#logger.debug("Pod update already processed", {
+        podName,
+        timestamp: this.#processedPods.get(podName),
+      });
+      return;
+    }
+
+    this.#processedPods.set(podName, Date.now());
+
+    const podStatus = this.#getPodStatusSummary(pod.status);
+    const containerState = this.#getContainerStateSummary(containerStatus.state);
+    const exitCode = containerState.exitCode ?? -1;
+
+    // We use this special exit code to signal any errors were already handled elsewhere
+    if (exitCode === 111) {
+      return;
+    }
+
+    const rawLogs = await this.#getLogTail(podName);
+
+    this.#logger.log(`${podName} failed with:`, {
+      podStatus,
+      containerState,
+      rawLogs,
+    });
+
+    const rawReason = podStatus.reason ?? containerState.reason ?? "";
+    const message = podStatus.message ?? containerState.message ?? "";
+
+    let reason = rawReason || "Unknown error";
+    let logs = rawLogs || "";
+
+    switch (rawReason) {
+      case "Error":
+        reason = "Unknown error.";
+        break;
+      case "Evicted":
+        if (message.startsWith("Pod ephemeral local storage usage")) {
+          reason = "Storage limit exceeded.";
+        } else if (message) {
+          reason = `Evicted: ${message}`;
+        } else {
+          reason = "Evicted for unknown reason.";
+        }
+
+        if (logs.startsWith("failed to try resolving symlinks")) {
+          logs = "";
+        }
+        break;
+      case "OOMKilled":
+        reason = "Out of memory! Try increasing the memory on this task.";
+        break;
+      default:
+        break;
+    }
+
+    const failureInfo = {
+      exitCode,
+      reason,
+      logs,
+    };
+
+    const app = pod.metadata?.labels?.app;
+
+    switch (app) {
+      case "task-index":
+        const deploymentId = pod.metadata?.labels?.deployment;
+
+        if (!deploymentId) {
+          this.#logger.error("Index is missing ID", { pod });
+          return;
+        }
+
+        if (this.opts.onIndexFailure) {
+          await this.opts.onIndexFailure(deploymentId, failureInfo);
+        }
+        break;
+      case "task-run":
+        const runId = pod.metadata?.labels?.run;
+
+        if (!runId) {
+          this.#logger.error("Run is missing ID", { pod });
+          return;
+        }
+
+        if (this.opts.onRunFailure) {
+          await this.opts.onRunFailure(runId, failureInfo);
+        }
+        break;
+      default:
+        this.#logger.error("Pod has invalid app label", { pod });
+        return;
+    }
+
+    await this.#deletePod(podName);
+  }
+
+  async #getLogTail(podName: string) {
+    try {
+      const logs = await this.#k8sClient.core.readNamespacedPodLog(
+        podName,
+        this.namespace,
+        undefined,
+        undefined,
+        undefined,
+        1024, // limitBytes
+        undefined,
+        undefined,
+        undefined,
+        20 // tailLines
+      );
+
+      const responseBody = logs.body ?? "";
+
+      if (responseBody.startsWith("unable to retrieve container logs")) {
+        return "";
+      }
+
+      // Type is wrong, body may be undefined
+      return responseBody;
+    } catch (error) {
+      this.#logger.error("Log tail error:", error instanceof Error ? error.message : "unknown");
+      return "";
+    }
+  }
+
+  #getPodStatusSummary(status: k8s.V1PodStatus) {
+    return {
+      reason: status.reason,
+      message: status.message,
+    };
+  }
+
+  #getContainerStateSummary(state: k8s.V1ContainerState) {
+    return {
+      reason: state.terminated?.reason,
+      exitCode: state.terminated?.exitCode,
+      message: state.terminated?.message,
+    };
+  }
+
+  #createK8sClient() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (this.opts.runtimeEnv === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (this.opts.runtimeEnv === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      kubeConfig: kubeConfig,
+    };
+  }
+
+  #isRecord(candidate: unknown): candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  #logK8sError(err: unknown, debugOnly = false) {
+    if (debugOnly) {
+      this.#logger.debug("K8s API Error", err);
+    } else {
+      this.#logger.error("K8s API Error", err);
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
+      this.#logK8sError(err);
+      return;
+    }
+
+    this.#logK8sError(err, true);
+
+    if (typeof err.body.message === "string") {
+      this.#logK8sError({ message: err.body.message });
+      return;
+    }
+
+    this.#logK8sError({ body: err.body });
+  }
+
+  #printStats(includeMoreDetails = false) {
+    this.#logger.log("Stats:", {
+      cacheSize: this.#taskInformer.list().length,
+      totalProcessed: this.#processedPods.size,
+      ...(includeMoreDetails && {
+        processedPods: this.#processedPods,
+      }),
+    });
+  }
+
+  async #deletePod(name: string) {
+    this.#logger.debug("Deleting pod:", name);
+
+    await this.#k8sClient.core
+      .deleteNamespacedPod(name, this.namespace)
+      .catch(this.#handleK8sError.bind(this));
+  }
+
+  async start() {
+    this.#enabled = true;
+
+    const interval = setInterval(() => {
+      if (!this.#enabled) {
+        clearInterval(interval);
+        return;
+      }
+
+      this.#printStats();
+    }, 300_000);
+
+    await this.#taskInformer.start();
+
+    // this.#launchTests();
+  }
+
+  async stop() {
+    if (!this.#enabled) {
+      return;
+    }
+
+    this.#enabled = false;
+    this.#logger.log("Shutting down..");
+
+    await this.#taskInformer.stop();
+
+    this.#printStats(true);
+  }
+
+  async #launchTests() {
+    const createPod = async (
+      container: k8s.V1Container,
+      name: string,
+      labels?: Record<string, string>
+    ) => {
+      this.#logger.log("Creating pod:", name);
+
+      const pod = {
+        metadata: {
+          name,
+          labels,
+        },
+        spec: {
+          restartPolicy: "Never",
+          automountServiceAccountToken: false,
+          terminationGracePeriodSeconds: 1,
+          containers: [container],
+        },
+      } satisfies k8s.V1Pod;
+
+      await this.#k8sClient.core
+        .createNamespacedPod(this.namespace, pod)
+        .catch(this.#handleK8sError.bind(this));
+    };
+
+    const createOomPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "polinux/stress",
+        resources: {
+          limits: {
+            memory: "100Mi",
+          },
+        },
+        command: ["stress"],
+        args: ["--vm", "1", "--vm-bytes", "150M", "--vm-hang", "1"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createNonZeroExitPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: ["-c", "exit 1"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createOoDiskPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: [
+          "-c",
+          "echo creating huge-file..; head -c 1000m /dev/zero > huge-file; ls -lh huge-file; sleep infinity",
+        ],
+        resources: {
+          limits: {
+            "ephemeral-storage": "500Mi",
+          },
+        },
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    await createNonZeroExitPod("non-zero-exit-task", { app: "task-run", run: "123" });
+    await createOomPod("oom-task", { app: "task-index", deployment: "456" });
+    await createOoDiskPod("ood-task", { app: "task-run", run: "abc" });
+  }
+}

apps/webapp/app/assets/icons/AISparkleIcon.tsx

@@ -0,0 +1,53 @@
+export function AISparkleIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M14.9806 0.803884C14.8871 0.33646 14.4767 0 14 0C13.5233 0 13.1129 0.33646 13.0194 0.803884L12.7809 1.99644C12.7017 2.3923 12.3923 2.70174 11.9964 2.78091L10.8039 3.01942C10.3365 3.1129 10 3.52332 10 4C10 4.47668 10.3365 4.8871 10.8039 4.98058L11.9964 5.21909C12.3923 5.29826 12.7017 5.6077 12.7809 6.00356L13.0194 7.19612C13.1129 7.66354 13.5233 8 14 8C14.4767 8 14.8871 7.66354 14.9806 7.19612L15.2191 6.00356C15.2983 5.6077 15.6077 5.29826 16.0036 5.21909L17.1961 4.98058C17.6635 4.8871 18 4.47668 18 4C18 3.52332 17.6635 3.1129 17.1961 3.01942L16.0036 2.78091C15.6077 2.70174 15.2983 2.3923 15.2191 1.99644L14.9806 0.803884Z"
+        fill="url(#paint0_linear_11402_36656)"
+      />
+      <path
+        d="M5.94868 4.68377C5.81257 4.27543 5.43043 4 5 4C4.56957 4 4.18743 4.27543 4.05132 4.68377L3.36754 6.73509C3.26801 7.03369 3.03369 7.26801 2.73509 7.36754L0.683772 8.05132C0.27543 8.18743 0 8.56957 0 9C0 9.43043 0.27543 9.81257 0.683772 9.94868L2.73509 10.6325C3.03369 10.732 3.26801 10.9663 3.36754 11.2649L4.05132 13.3162C4.18743 13.7246 4.56957 14 5 14C5.43043 14 5.81257 13.7246 5.94868 13.3162L6.63246 11.2649C6.73199 10.9663 6.96631 10.732 7.26491 10.6325L9.31623 9.94868C9.72457 9.81257 10 9.43043 10 9C10 8.56957 9.72457 8.18743 9.31623 8.05132L7.26491 7.36754C6.96631 7.26801 6.73199 7.03369 6.63246 6.73509L5.94868 4.68377Z"
+        fill="url(#paint1_linear_11402_36656)"
+      />
+      <path
+        d="M12.9487 12.6838C12.8126 12.2754 12.4304 12 12 12C11.5696 12 11.1874 12.2754 11.0513 12.6838L10.8675 13.2351C10.768 13.5337 10.5337 13.768 10.2351 13.8675L9.68377 14.0513C9.27543 14.1874 9 14.5696 9 15C9 15.4304 9.27543 15.8126 9.68377 15.9487L10.2351 16.1325C10.5337 16.232 10.768 16.4663 10.8675 16.7649L11.0513 17.3162C11.1874 17.7246 11.5696 18 12 18C12.4304 18 12.8126 17.7246 12.9487 17.3162L13.1325 16.7649C13.232 16.4663 13.4663 16.232 13.7649 16.1325L14.3162 15.9487C14.7246 15.8126 15 15.4304 15 15C15 14.5696 14.7246 14.1874 14.3162 14.0513L13.7649 13.8675C13.4663 13.768 13.232 13.5337 13.1325 13.2351L12.9487 12.6838Z"
+        fill="url(#paint2_linear_11402_36656)"
+      />
+      <defs>
+        <linearGradient
+          id="paint0_linear_11402_36656"
+          x1="9"
+          y1="0"
+          x2="9"
+          y2="18"
+          gradientUnits="userSpaceOnUse"
+        >
+          <stop stopColor="#E543FF" />
+          <stop offset="1" stopColor="#286399" />
+        </linearGradient>
+        <linearGradient
+          id="paint1_linear_11402_36656"
+          x1="9"
+          y1="0"
+          x2="9"
+          y2="18"
+          gradientUnits="userSpaceOnUse"
+        >
+          <stop stopColor="#E543FF" />
+          <stop offset="1" stopColor="#286399" />
+        </linearGradient>
+        <linearGradient
+          id="paint2_linear_11402_36656"
+          x1="9"
+          y1="0"
+          x2="9"
+          y2="18"
+          gradientUnits="userSpaceOnUse"
+        >
+          <stop stopColor="#E543FF" />
+          <stop offset="1" stopColor="#286399" />
+        </linearGradient>
+      </defs>
+    </svg>
+  );
+}

apps/webapp/app/components/SetupCommands.tsx

@@ -206,3 +206,39 @@ export function TriggerDevStepV3() {
     </ClientTabs>
   );
 }
+
+export function TriggerLoginStepV3() {
+  return (
+    <ClientTabs defaultValue="npm">
+      <ClientTabsList>
+        <ClientTabsTrigger value={"npm"}>npm</ClientTabsTrigger>
+        <ClientTabsTrigger value={"pnpm"}>pnpm</ClientTabsTrigger>
+        <ClientTabsTrigger value={"yarn"}>yarn</ClientTabsTrigger>
+      </ClientTabsList>
+      <ClientTabsContent value={"npm"}>
+        <ClipboardField
+          variant="primary/medium"
+          iconButton
+          className="mb-4"
+          value={`npx trigger.dev@${v3PackageTag} login`}
+        />
+      </ClientTabsContent>
+      <ClientTabsContent value={"pnpm"}>
+        <ClipboardField
+          variant="primary/medium"
+          iconButton
+          className="mb-4"
+          value={`pnpm dlx trigger.dev@${v3PackageTag} login`}
+        />
+      </ClientTabsContent>
+      <ClientTabsContent value={"yarn"}>
+        <ClipboardField
+          variant="primary/medium"
+          iconButton
+          className="mb-4"
+          value={`yarn dlx trigger.dev@${v3PackageTag} login`}
+        />
+      </ClientTabsContent>
+    </ClientTabs>
+  );
+}

apps/webapp/app/components/navigation/SideMenu.tsx

@@ -4,6 +4,7 @@ import {
   ArrowRightOnRectangleIcon,
   BeakerIcon,
   ChartBarIcon,
+  ClockIcon,
   CursorArrowRaysIcon,
   IdentificationIcon,
   KeyIcon,
@@ -47,6 +48,7 @@ import {
   v3ProjectPath,
   v3ProjectSettingsPath,
   v3RunsPath,
+  v3SchedulesPath,
   v3TestPath,
 } from "~/utils/pathBuilder";
 import { Feedback } from "../Feedback";
@@ -57,6 +59,7 @@ import { UserProfilePhoto } from "../UserProfilePhoto";
 import { FreePlanUsage } from "../billing/FreePlanUsage";
 import { Badge } from "../primitives/Badge";
 import { Button } from "../primitives/Buttons";
+import { Callout } from "../primitives/Callout";
 import { ClipboardField } from "../primitives/ClipboardField";
 import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "../primitives/Dialog";
 import { Icon } from "../primitives/Icon";
@@ -187,7 +190,22 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
             />
           </div>
         </div>
+        <div className="m-2">
+          {project.version === "V2" ? (
+            <Callout variant={"info"}>This is a v2 project</Callout>
+          ) : (
+            <Callout variant={"idea"}>This is a v3 project in Developer Preview</Callout>
+          )}
+        </div>
         <div className="flex flex-col gap-1 border-t border-grid-bright p-1">
+          {project.version === "V2" && (
+            <SideMenuItem
+              to="https://trigger.dev/v3-early-access"
+              target="_blank"
+              name="Request access to v3"
+              icon={V3Icon}
+            />
+          )}
           {currentPlan?.subscription?.isPaying === true && (
             <Dialog>
               <DialogTrigger asChild>
@@ -555,6 +573,13 @@ function V3ProjectSideMenu({
         to={v3TestPath(organization, project)}
         data-action="test"
       />
+      <SideMenuItem
+        name="Schedules"
+        icon={ClockIcon}
+        iconColor="text-sun-500"
+        to={v3SchedulesPath(organization, project)}
+        data-action="schedules"
+      />
       <SideMenuItem
         name="API keys"
         icon={KeyIcon}
@@ -586,3 +611,15 @@ function V3ProjectSideMenu({
     </>
   );
 }
+
+function V3Icon() {
+  return (
+    <svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <rect width="16" height="16" rx="8" fill="#A8FF53" />
+      <path
+        d="M7.7488 6.17L5.5818 12H3.6678L1.5008 6.17H3.2058L4.6248 10.339L6.0328 6.17H7.7488ZM11.0527 7.292C11.6357 7.303 12.2297 7.006 12.2297 6.28C12.2297 5.774 11.7787 5.433 11.0527 5.433C10.4147 5.433 9.98567 5.741 9.91967 6.214L8.22567 6.126C8.36867 4.861 9.51267 4.014 11.0857 4.014C12.8457 4.014 13.9567 4.806 13.9567 6.049C13.9567 6.951 13.3847 7.534 12.3067 7.776C13.5387 8.04 14.2207 8.777 14.2207 9.855C14.2207 11.274 13.0107 12.176 11.0857 12.176C9.32567 12.176 8.12667 11.197 8.04967 9.712L9.75467 9.646C9.83167 10.405 10.4917 10.757 11.0967 10.757C11.8007 10.757 12.4937 10.394 12.4937 9.591C12.4937 8.81 11.7897 8.425 11.0527 8.447L10.3817 8.458V7.281L11.0527 7.292Z"
+        fill="#15171A"
+      />
+    </svg>
+  );
+}

apps/webapp/app/components/primitives/Buttons.tsx

@@ -286,8 +286,7 @@ export const Button = forwardRef<HTMLButtonElement, ButtonPropsType>(
 type LinkPropsType = Pick<
   LinkProps,
   "to" | "target" | "onClick" | "onMouseDown" | "onMouseEnter" | "onMouseLeave" | "download"
-> &
-  React.ComponentProps<typeof ButtonContent>;
+> & { disabled?: boolean } & React.ComponentProps<typeof ButtonContent>;
 export const LinkButton = ({
   to,
   onClick,
@@ -295,6 +294,7 @@ export const LinkButton = ({
   onMouseEnter,
   onMouseLeave,
   download,
+  disabled = false,
   ...props
 }: LinkPropsType) => {
   const innerRef = useRef<HTMLAnchorElement>(null);
@@ -309,6 +309,19 @@ export const LinkButton = ({
     });
   }
 
+  if (disabled) {
+    return (
+      <div
+        className={cn(
+          "group pointer-events-none cursor-default opacity-40 outline-none",
+          props.fullWidth ? "w-full" : ""
+        )}
+      >
+        <ButtonContent {...props} />
+      </div>
+    );
+  }
+
   if (to.toString().startsWith("http") || to.toString().startsWith("/resources")) {
     return (
       <ExtLink

apps/webapp/app/components/primitives/Checkbox.tsx

@@ -23,7 +23,7 @@ const variants = {
   },
   "button/small": {
     button:
-      "flex items-center w-fit h-8 pl-2 pr-3 rounded border border-charcoal-800 hover:bg-charcoal-850 hover:border-charcoal-750 transition",
+      "flex items-center w-fit h-8 pl-2 pr-3 rounded border border-charcoal-600 hover:bg-charcoal-850 hover:border-charcoal-500 transition",
     label: "text-sm text-text-bright select-none",
     description: "text-text-dimmed",
     inputPosition: "mt-0",
@@ -32,7 +32,7 @@ const variants = {
   },
   button: {
     button:
-      "w-fit py-2 pl-3 pr-4 rounded border border-charcoal-800 hover:bg-charcoal-850 hover:border-charcoal-750 transition",
+      "w-fit py-2 pl-3 pr-4 rounded border border-charcoal-600 hover:bg-charcoal-850 hover:border-charcoal-500 transition",
     label: "text-text-bright select-none",
     description: "text-text-dimmed",
     inputPosition: "mt-1",
@@ -57,7 +57,7 @@ export type CheckboxProps = Omit<
   name?: string;
   value?: string;
   variant?: keyof typeof variants;
-  label?: string;
+  label?: React.ReactNode;
   description?: string;
   badges?: string[];
   className?: string;
@@ -137,7 +137,7 @@ export const Checkbox = React.forwardRef<HTMLInputElement, CheckboxProps>(
           className={cn(
             inputPositionClasses,
             props.readOnly || disabled ? "cursor-default" : "cursor-pointer",
-            "read-only:border-charcoal-650 disabled:border-charcoal-650 rounded-sm border border-charcoal-700 bg-transparent transition checked:!bg-indigo-500 read-only:!bg-charcoal-700 group-hover:bg-charcoal-900 group-hover:checked:bg-indigo-500 group-focus:ring-1 focus:ring-indigo-500 focus:ring-offset-0 focus:ring-offset-transparent focus-visible:outline-none  focus-visible:ring-indigo-500 disabled:!bg-charcoal-700"
+            "read-only:border-charcoal-650 disabled:border-charcoal-650 rounded-sm border border-charcoal-600 bg-transparent transition checked:!bg-indigo-500 read-only:!bg-charcoal-700 group-hover:bg-charcoal-900 group-hover:checked:bg-indigo-500 group-focus:ring-1 focus:ring-indigo-500 focus:ring-offset-0 focus:ring-offset-transparent focus-visible:outline-none  focus-visible:ring-indigo-500 disabled:!bg-charcoal-700"
           )}
           id={id}
           ref={ref}

apps/webapp/app/components/primitives/DateField.tsx

@@ -1,3 +1,4 @@
+import { BellAlertIcon } from "@heroicons/react/20/solid";
 import { CalendarDateTime, createCalendar } from "@internationalized/date";
 import { useDateField, useDateSegment } from "@react-aria/datepicker";
 import type { DateFieldState, DateSegment } from "@react-stately/datepicker";
@@ -5,11 +6,25 @@ import { useDateFieldState } from "@react-stately/datepicker";
 import { Granularity } from "@react-types/datepicker";
 import { useEffect, useRef, useState } from "react";
 import { cn } from "~/utils/cn";
-import { useLocales } from "./LocaleProvider";
 import { Button } from "./Buttons";
 
+const variants = {
+  small: {
+    fieldStyles: "h-5 text-sm rounded-sm px-0.5",
+    nowButtonVariant: "tertiary/small" as const,
+    clearButtonVariant: "minimal/small" as const,
+  },
+  medium: {
+    fieldStyles: "h-7 text-base rounded px-1",
+    nowButtonVariant: "tertiary/medium" as const,
+    clearButtonVariant: "minimal/medium" as const,
+  },
+};
+
+type Variant = keyof typeof variants;
+
 type DateFieldProps = {
-  label?: string;
+  label: string;
   defaultValue?: Date;
   minValue?: Date;
   maxValue?: Date;
@@ -20,6 +35,7 @@ type DateFieldProps = {
   showNowButton?: boolean;
   showClearButton?: boolean;
   onValueChange?: (value: Date | undefined) => void;
+  variant?: Variant;
 };
 
 export function DateField({
@@ -34,6 +50,7 @@ export function DateField({
   showGuide = false,
   showNowButton = false,
   showClearButton = false,
+  variant = "small",
 }: DateFieldProps) {
   const [value, setValue] = useState<undefined | CalendarDateTime>(
     utcDateToCalendarDate(defaultValue)
@@ -90,48 +107,50 @@ export function DateField({
 
   return (
     <div className={`flex flex-col items-start ${className || ""}`}>
-      <span {...labelProps} className="mb-1 ml-0.5 text-xs text-charcoal-300">
-        {label}
-      </span>
-      <div className="flex flex-row items-center gap-1">
+      <div className="flex flex-row items-center gap-1" aria-label={label}>
         <div
           {...fieldProps}
           ref={ref}
           className={cn(
-            "flex rounded-sm border border-charcoal-800 bg-charcoal-750 p-0.5 px-1.5 transition-colors focus-within:border-charcoal-500 hover:border-charcoal-700 focus-within:hover:border-charcoal-500",
+            "flex rounded-sm border bg-charcoal-700 p-0.5 transition focus-within:border-charcoal-600 hover:border-charcoal-600",
             fieldClassName
           )}
         >
-          <DateSegment segment={yearSegment} state={state} />
-          <DateSegment segment={literalSegment("/")} state={state} />
-          <DateSegment segment={monthSegment} state={state} />
-          <DateSegment segment={literalSegment("/")} state={state} />
-          <DateSegment segment={daySegment} state={state} />
-          <DateSegment segment={literalSegment(", ")} state={state} />
-          <DateSegment segment={hourSegment} state={state} />
-          <DateSegment segment={literalSegment(":")} state={state} />
-          <DateSegment segment={minuteSegment} state={state} />
-          <DateSegment segment={literalSegment(":")} state={state} />
-          <DateSegment segment={secondSegment} state={state} />
-          <DateSegment segment={literalSegment(" ")} state={state} />
-          <DateSegment segment={dayPeriodSegment} state={state} />
+          <DateSegment segment={yearSegment} state={state} variant={variant} />
+          <DateSegment segment={literalSegment("/")} state={state} variant={variant} />
+          <DateSegment segment={monthSegment} state={state} variant={variant} />
+          <DateSegment segment={literalSegment("/")} state={state} variant={variant} />
+          <DateSegment segment={daySegment} state={state} variant={variant} />
+          <DateSegment segment={literalSegment(", ")} state={state} variant={variant} />
+          <DateSegment segment={hourSegment} state={state} variant={variant} />
+          <DateSegment segment={literalSegment(":")} state={state} variant={variant} />
+          <DateSegment segment={minuteSegment} state={state} variant={variant} />
+          <DateSegment segment={literalSegment(":")} state={state} variant={variant} />
+          <DateSegment segment={secondSegment} state={state} variant={variant} />
+          <DateSegment segment={literalSegment(" ")} state={state} variant={variant} />
+          <DateSegment segment={dayPeriodSegment} state={state} variant={variant} />
         </div>
         {showNowButton && (
           <Button
-            variant="tertiary/small"
+            type="button"
+            variant={variants[variant].nowButtonVariant}
+            LeadingIcon={BellAlertIcon}
+            leadingIconClassName="text-text-dimmed group-hover:text-text-bright"
             onClick={() => {
               const now = new Date();
               setValue(utcDateToCalendarDate(new Date()));
               onValueChange?.(now);
             }}
           >
-            Now
+            <span className="text-text-dimmed transition group-hover:text-text-bright">Now</span>
           </Button>
         )}
         {showClearButton && (
           <Button
-            variant="tertiary/small"
+            type="button"
+            variant={variants[variant].clearButtonVariant}
             LeadingIcon={"close"}
+            leadingIconClassName="-mr-2"
             onClick={() => {
               setValue(undefined);
               onValueChange?.(undefined);
@@ -142,7 +161,9 @@ export function DateField({
               state.clearSegment("minute");
               state.clearSegment("second");
             }}
-          />
+          >
+            Clear
+          </Button>
         )}
       </div>
       {showGuide && (
@@ -172,11 +193,13 @@ function utcDateToCalendarDate(date?: Date) {
 type DateSegmentProps = {
   segment: DateSegment;
   state: DateFieldState;
+  variant: Variant;
 };
 
-function DateSegment({ segment, state }: DateSegmentProps) {
+function DateSegment({ segment, state, variant }: DateSegmentProps) {
   const ref = useRef<null | HTMLDivElement>(null);
   const { segmentProps } = useDateSegment(segment, state, ref);
+  const sizeVariant = variants[variant];
 
   return (
     <div
@@ -186,23 +209,27 @@ function DateSegment({ segment, state }: DateSegmentProps) {
         ...segmentProps.style,
         minWidth: minWidthForSegment(segment),
       }}
-      className={`group box-content rounded-sm px-0.5 text-right text-sm tabular-nums outline-none focus:bg-indigo-500 focus:text-white ${
+      className={cn(
+        "group box-content text-center tabular-nums outline-none focus:bg-charcoal-600 focus:text-text-bright",
+        sizeVariant.fieldStyles,
         !segment.isEditable ? "text-charcoal-500" : "text-text-bright"
-      }`}
+      )}
     >
       {/* Always reserve space for the placeholder, to prevent layout shift when editing. */}
       <span
         aria-hidden="true"
-        className="block text-center italic text-charcoal-500 group-focus:text-white"
+        className="flex h-full items-center justify-center text-center text-charcoal-500 group-focus:text-text-bright"
         style={{
           visibility: segment.isPlaceholder ? undefined : "hidden",
-          height: segment.isPlaceholder ? "" : 0,
+          height: segment.isPlaceholder ? undefined : 0,
           pointerEvents: "none",
         }}
       >
         {segment.placeholder}
       </span>
-      {segment.isPlaceholder ? "" : segment.text}
+      <span className="flex h-full items-center justify-center">
+        {segment.isPlaceholder ? "" : segment.text}
+      </span>
     </div>
   );
 }
@@ -231,7 +258,7 @@ function DateSegmentGuide({ segment }: { segment: DateSegment }) {
       style={{
         minWidth: minWidthForSegment(segment),
       }}
-      className={`group box-content rounded-sm px-0.5 text-right text-sm tabular-nums outline-none ${
+      className={`group box-content rounded-sm px-0.5 text-right text-sm tabular-nums text-rose-500 outline-none ${
         !segment.isEditable ? "text-charcoal-500" : "text-text-bright"
       }`}
     >

apps/webapp/app/components/primitives/DateTime.tsx

@@ -10,7 +10,7 @@ type DateTimeProps = {
 
 export const DateTime = ({
   date,
-  timeZone = "UTC",
+  timeZone,
   includeSeconds = true,
   includeTime = true,
 }: DateTimeProps) => {
@@ -20,7 +20,7 @@ export const DateTime = ({
 
   const initialFormattedDateTime = formatDateTime(
     realDate,
-    timeZone,
+    timeZone ?? "UTC",
     locales,
     includeSeconds,
     includeTime
@@ -32,7 +32,13 @@ export const DateTime = ({
     const resolvedOptions = Intl.DateTimeFormat().resolvedOptions();
 
     setFormattedDateTime(
-      formatDateTime(realDate, resolvedOptions.timeZone, locales, includeSeconds, includeTime)
+      formatDateTime(
+        realDate,
+        timeZone ?? resolvedOptions.timeZone,
+        locales,
+        includeSeconds,
+        includeTime
+      )
     );
   }, [locales, includeSeconds, realDate]);
 

apps/webapp/app/components/primitives/Dialog.tsx

@@ -43,7 +43,7 @@ const DialogContent = React.forwardRef<
     <DialogPrimitive.Content
       ref={ref}
       className={cn(
-        "fixed z-50 grid w-full gap-4 rounded-b-lg border bg-background-dimmed px-4 pb-6 pt-3.5 shadow-lg animate-in data-[state=open]:fade-in-90 data-[state=open]:slide-in-from-bottom-10 sm:max-w-lg sm:rounded-lg sm:zoom-in-90 data-[state=open]:sm:slide-in-from-bottom-0",
+        "fixed z-50 grid w-full gap-4 rounded-b-lg border bg-background-dimmed px-4 pb-4 pt-3.5 shadow-lg animate-in data-[state=open]:fade-in-90 data-[state=open]:slide-in-from-bottom-10 sm:max-w-lg sm:rounded-lg sm:zoom-in-90 data-[state=open]:sm:slide-in-from-bottom-0",
         className
       )}
       {...props}

apps/webapp/app/components/primitives/Label.tsx

@@ -1,29 +1,41 @@
 import * as React from "react";
 import { cn } from "~/utils/cn";
+import { InfoIconTooltip, SimpleTooltip } from "./Tooltip";
 
-const labelVariants = {
+const variants = {
   small: {
-    text: "font-sans text-sm font-normal",
+    text: "font-sans text-sm font-normal text-text-bright leading-tight flex items-center gap-1",
   },
   medium: {
-    text: "font-sans text-sm leading-5 font-medium",
+    text: "font-sans text-sm text-text-bright leading-tight flex items-center gap-1",
   },
   large: {
-    text: "font-sans text-base leading-6 font-medium",
+    text: "font-sans text-base font-medium text-text-bright leading-tight flex items-center gap-1",
   },
 };
 
 type LabelProps = React.AllHTMLAttributes<HTMLLabelElement> & {
   className?: string;
   children: React.ReactNode;
-  variant?: keyof typeof labelVariants;
+  variant?: keyof typeof variants;
+  required?: boolean;
+  tooltip?: React.ReactNode;
 };
 
-export function Label({ className, children, variant = "medium", ...props }: LabelProps) {
-  const variation = labelVariants[variant];
+export function Label({
+  className,
+  children,
+  variant = "medium",
+  required = true,
+  tooltip,
+  ...props
+}: LabelProps) {
+  const variation = variants[variant];
   return (
     <label className={cn(variation.text, className)} {...props}>
       {children}
+      {tooltip ? <InfoIconTooltip content={tooltip} /> : null}
+      {!required && <span className="text-text-dimmed"> (optional)</span>}
     </label>
   );
 }

apps/webapp/app/components/primitives/PageHeader.tsx

@@ -15,6 +15,7 @@ import { Tabs, TabsProps } from "./Tabs";
 
 type WithChildren = {
   children: React.ReactNode;
+  className?: string;
 };
 
 export function NavBar({ children }: WithChildren) {
@@ -72,8 +73,8 @@ export function PageAccessories({ children }: WithChildren) {
   return <div className="flex items-center gap-3">{children}</div>;
 }
 
-export function PageInfoRow({ children }: WithChildren) {
-  return <div className="flex w-full items-center gap-2">{children}</div>;
+export function PageInfoRow({ children, className }: WithChildren) {
+  return <div className={cn("flex w-full items-center gap-2", className)}>{children}</div>;
 }
 
 export function PageInfoGroup({

apps/webapp/app/components/primitives/Pagination.tsx

@@ -1,16 +1,18 @@
 import { ChevronRightIcon } from "@heroicons/react/24/outline";
 import { ChevronLeftIcon } from "@heroicons/react/24/solid";
 import { Link, useLocation } from "@remix-run/react";
-import { LinkDisabled } from "./LinkWithDisabled";
 import { cn } from "~/utils/cn";
-import { ButtonContent, LinkButton } from "./Buttons";
+import { ButtonContent } from "./Buttons";
+import { LinkDisabled } from "./LinkWithDisabled";
 
 export function PaginationControls({
   currentPage,
   totalPages,
+  showPageNumbers = true,
 }: {
   currentPage: number;
   totalPages: number;
+  showPageNumbers?: boolean;
 }) {
   const location = useLocation();
   if (totalPages <= 1) {
@@ -30,9 +32,11 @@ export function PaginationControls({
         </ButtonContent>
       </LinkDisabled>
 
-      {calculatePageLinks(currentPage, totalPages).map((page, i) => (
-        <PageLinkComponent page={page} key={i} location={location} />
-      ))}
+      {showPageNumbers
+        ? calculatePageLinks(currentPage, totalPages).map((page, i) => (
+            <PageLinkComponent page={page} key={i} location={location} />
+          ))
+        : null}
 
       <LinkDisabled
         to={pageUrl(location, currentPage + 1)}

apps/webapp/app/components/primitives/RadioButton.tsx

@@ -22,7 +22,7 @@ const variants = {
   },
   "button/small": {
     button:
-      "flex items-center w-fit h-8 pl-2 pr-3 rounded border border-charcoal-800 hover:bg-charcoal-850 hover:border-charcoal-750 transition data-[disabled]:opacity-70 data-[disabled]:hover:bg-transparent data-[state=checked]:bg-charcoal-850",
+      "flex items-center w-fit h-8 pl-2 pr-3 rounded border border-charcoal-600 hover:bg-charcoal-850 hover:border-charcoal-500 transition data-[disabled]:opacity-70 data-[disabled]:hover:bg-transparent data-[state=checked]:bg-charcoal-850",
     label: "text-sm text-text-bright select-none",
     description: "text-text-dimmed",
     inputPosition: "mt-0",
@@ -30,7 +30,7 @@ const variants = {
   },
   button: {
     button:
-      "w-fit py-2 pl-3 pr-4 rounded border border-charcoal-800 hover:bg-charcoal-850 hover:border-charcoal-750 transition data-[state=checked]:bg-charcoal-850 data-[disabled]:opacity-70",
+      "w-fit py-2 pl-3 pr-4 rounded border border-charcoal-600 hover:bg-charcoal-850 hover:border-charcoal-500 transition data-[state=checked]:bg-charcoal-850 data-[disabled]:opacity-70",
     label: "text-text-bright select-none",
     description: "text-text-dimmed",
     inputPosition: "mt-1",
@@ -38,7 +38,7 @@ const variants = {
   },
   description: {
     button:
-      "w-full p-2.5 hover:bg-charcoal-850 transition data-[disabled]:opacity-70 data-[state=checked]:bg-charcoal-850 border-charcoal-800 border rounded-sm",
+      "w-full p-2.5 hover:bg-charcoal-850 transition data-[disabled]:opacity-70 data-[state=checked]:bg-charcoal-850 border-charcoal-600 border rounded-sm",
     label: "text-text-bright font-semibold -mt-1 text-left",
     description: "text-text-dimmed -mt-0 text-left",
     inputPosition: "mt-0",
@@ -46,7 +46,7 @@ const variants = {
   },
   icon: {
     button:
-      "w-full p-2.5 pb-4 hover:bg-charcoal-850 transition data-[disabled]:opacity-70 data-[state=checked]:bg-charcoal-850 border-charcoal-800 border rounded-sm",
+      "w-full p-2.5 pb-4 hover:bg-charcoal-850 transition data-[disabled]:opacity-70 data-[state=checked]:bg-charcoal-850 border-charcoal-600 border rounded-sm",
     label: "text-text-bright font-semibold -mt-1 text-left",
     description: "text-text-dimmed -mt-0 text-left",
     inputPosition: "mt-0",
@@ -70,7 +70,7 @@ export function RadioButtonCircle({
   return (
     <div
       className={cn(
-        "ring-offset-background focus-visible:ring-ring aspect-square h-4 w-4 shrink-0 overflow-hidden rounded-full border border-charcoal-700 focus:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50",
+        "ring-offset-background focus-visible:ring-ring aspect-square h-4 w-4 shrink-0 overflow-hidden rounded-full border border-charcoal-600 focus:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50",
         boxClassName
       )}
     >
@@ -129,7 +129,7 @@ export const RadioGroupItem = React.forwardRef<
       >
         <div
           className={cn(
-            "ring-offset-background focus-visible:ring-ring aspect-square h-4 w-4 shrink-0 overflow-hidden rounded-full border border-charcoal-700 focus:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50",
+            "ring-offset-background focus-visible:ring-ring aspect-square h-4 w-4 shrink-0 overflow-hidden rounded-full border border-charcoal-600 focus:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50",
             variation.inputPosition
           )}
         >

apps/webapp/app/components/primitives/ShortcutKey.tsx

@@ -2,8 +2,14 @@ import { Fragment } from "react";
 import { Modifier, ShortcutDefinition } from "~/hooks/useShortcutKeys";
 import { cn } from "~/utils/cn";
 import { useOperatingSystem } from "./OperatingSystemProvider";
+import {
+  ChevronDownIcon,
+  ChevronLeftIcon,
+  ChevronRightIcon,
+  ChevronUpIcon,
+} from "@heroicons/react/20/solid";
 
-const variants = {
+export const variants = {
   small:
     "text-[0.6rem] font-medium min-w-[17px] rounded-[2px] px-1 ml-1 -mr-0.5 grid place-content-center border border-dimmed/40 text-text-dimmed group-hover:text-text-bright/80 group-hover:border-dimmed/60 transition uppercase",
   medium:
@@ -23,7 +29,7 @@ export function ShortcutKey({ shortcut, variant, className }: ShortcutKeyProps)
   const isMac = platform === "mac";
   let relevantShortcut = "mac" in shortcut ? (isMac ? shortcut.mac : shortcut.windows) : shortcut;
   const modifiers = relevantShortcut.modifiers ?? [];
-  const character = keyString(relevantShortcut.key, isMac);
+  const character = keyString(relevantShortcut.key, isMac, variant);
 
   return (
     <span className={cn(variants[variant], className)}>
@@ -35,10 +41,22 @@ export function ShortcutKey({ shortcut, variant, className }: ShortcutKeyProps)
   );
 }
 
-function keyString(key: String, isMac: boolean) {
+function keyString(key: String, isMac: boolean, size: "small" | "medium") {
+  key = key.toLowerCase();
+
+  const className = size === "small" ? "w-2.5 h-4" : "w-3 h-5";
+
   switch (key) {
     case "enter":
       return isMac ? "↵" : key;
+    case "arrowdown":
+      return <ChevronDownIcon className={className} />;
+    case "arrowup":
+      return <ChevronUpIcon className={className} />;
+    case "arrowleft":
+      return <ChevronLeftIcon className={className} />;
+    case "arrowright":
+      return <ChevronRightIcon className={className} />;
     default:
       return key;
   }

apps/webapp/app/components/primitives/Table.tsx

@@ -49,7 +49,7 @@ export const TableHeader = forwardRef<HTMLTableSectionElement, TableHeaderProps>
 
 type TableBodyProps = {
   className?: string;
-  children: ReactNode;
+  children?: ReactNode;
 };
 
 export const TableBody = forwardRef<HTMLTableSectionElement, TableBodyProps>(

apps/webapp/app/components/primitives/Tooltip.tsx

@@ -1,6 +1,7 @@
 import * as React from "react";
 import * as TooltipPrimitive from "@radix-ui/react-tooltip";
 import { cn } from "~/utils/cn";
+import { InformationCircleIcon } from "@heroicons/react/20/solid";
 
 const variantClasses = {
   basic:
@@ -83,4 +84,22 @@ function SimpleTooltip({
   );
 }
 
+export function InfoIconTooltip({
+  content,
+  buttonClassName,
+}: {
+  content: React.ReactNode;
+  buttonClassName?: string;
+}) {
+  return (
+    <SimpleTooltip
+      button={
+        <InformationCircleIcon className={cn("h-3.5 w-3.5 text-text-dimmed", buttonClassName)} />
+      }
+      content={content}
+      variant="dark"
+    />
+  );
+}
+
 export { Tooltip, TooltipTrigger, TooltipContent, TooltipProvider, TooltipArrow, SimpleTooltip };

apps/webapp/app/components/primitives/TreeView/TreeView.tsx

@@ -1,10 +1,10 @@
 import { VirtualItem, Virtualizer, useVirtualizer } from "@tanstack/react-virtual";
+import { motion } from "framer-motion";
 import { MutableRefObject, RefObject, useCallback, useEffect, useReducer, useRef } from "react";
 import { UnmountClosed } from "react-collapse";
 import { cn } from "~/utils/cn";
 import { NodeState, NodesState, reducer } from "./reducer";
 import { applyFilterToState, concreteStateFromInput, selectedIdFromState } from "./utils";
-import { motion } from "framer-motion";
 
 export type TreeViewProps<TData> = {
   tree: FlatTree<TData>;
@@ -165,6 +165,11 @@ export type UseTreeStateOutput = {
   expandNode: (id: string, scrollToNode?: boolean) => void;
   collapseNode: (id: string) => void;
   toggleExpandNode: (id: string, scrollToNode?: boolean) => void;
+  expandAllBelowDepth: (depth: number) => void;
+  collapseAllBelowDepth: (depth: number) => void;
+  expandLevel: (level: number) => void;
+  collapseLevel: (level: number) => void;
+  toggleExpandLevel: (level: number) => void;
   selectFirstVisibleNode: (scrollToNode?: boolean) => void;
   selectLastVisibleNode: (scrollToNode?: boolean) => void;
   selectNextVisibleNode: (scrollToNode?: boolean) => void;
@@ -333,6 +338,41 @@ export function useTree<TData>({
     [state]
   );
 
+  const expandAllBelowDepth = useCallback(
+    (depth: number) => {
+      dispatch({ type: "EXPAND_ALL_BELOW_DEPTH", payload: { tree, depth } });
+    },
+    [state]
+  );
+
+  const collapseAllBelowDepth = useCallback(
+    (depth: number) => {
+      dispatch({ type: "COLLAPSE_ALL_BELOW_DEPTH", payload: { tree, depth } });
+    },
+    [state]
+  );
+
+  const expandLevel = useCallback(
+    (level: number) => {
+      dispatch({ type: "EXPAND_LEVEL", payload: { tree, level } });
+    },
+    [state]
+  );
+
+  const collapseLevel = useCallback(
+    (level: number) => {
+      dispatch({ type: "COLLAPSE_LEVEL", payload: { tree, level } });
+    },
+    [state]
+  );
+
+  const toggleExpandLevel = useCallback(
+    (level: number) => {
+      dispatch({ type: "TOGGLE_EXPAND_LEVEL", payload: { tree, level } });
+    },
+    [state]
+  );
+
   const getTreeProps = useCallback(() => {
     return {
       role: "tree",
@@ -368,25 +408,48 @@ export function useTree<TData>({
           }
           case "Left":
           case "ArrowLeft": {
+            e.preventDefault();
+
             const selected = selectedIdFromState(state.nodes);
             if (selected) {
               const treeNode = tree.find((node) => node.id === selected);
-              if (treeNode && treeNode.hasChildren && state.nodes[selected].expanded) {
+
+              if (e.altKey) {
+                if (treeNode && treeNode.hasChildren) {
+                  collapseLevel(treeNode.level);
+                }
+                break;
+              }
+
+              const shouldCollapse =
+                treeNode && treeNode.hasChildren && state.nodes[selected].expanded;
+              if (shouldCollapse) {
                 collapseNode(selected);
               } else {
                 selectParentNode(true);
               }
             }
-            e.preventDefault();
+
             break;
           }
           case "Right":
           case "ArrowRight": {
+            e.preventDefault();
+
             const selected = selectedIdFromState(state.nodes);
+
             if (selected) {
+              const treeNode = tree.find((node) => node.id === selected);
+
+              if (e.altKey) {
+                if (treeNode && treeNode.hasChildren) {
+                  expandLevel(treeNode.level);
+                }
+                break;
+              }
+
               expandNode(selected, true);
             }
-            e.preventDefault();
             break;
           }
           case "Escape": {
@@ -427,6 +490,11 @@ export function useTree<TData>({
     expandNode,
     collapseNode,
     toggleExpandNode,
+    expandAllBelowDepth,
+    collapseAllBelowDepth,
+    expandLevel,
+    collapseLevel,
+    toggleExpandLevel,
     selectFirstVisibleNode,
     selectLastVisibleNode,
     selectNextVisibleNode,

apps/webapp/app/components/primitives/TreeView/reducer.ts

@@ -91,6 +91,46 @@ type ToggleExpandNodeAction = {
   } & WithScrollToNode;
 };
 
+type ExpandAllBelowDepthAction = {
+  type: "EXPAND_ALL_BELOW_DEPTH";
+  payload: {
+    depth: number;
+    tree: FlatTree<any>;
+  };
+};
+
+type CollapseAllBelowDepthAction = {
+  type: "COLLAPSE_ALL_BELOW_DEPTH";
+  payload: {
+    depth: number;
+    tree: FlatTree<any>;
+  };
+};
+
+type ExpandLevelAction = {
+  type: "EXPAND_LEVEL";
+  payload: {
+    level: number;
+    tree: FlatTree<any>;
+  };
+};
+
+type CollapseLevelAction = {
+  type: "COLLAPSE_LEVEL";
+  payload: {
+    level: number;
+    tree: FlatTree<any>;
+  };
+};
+
+type ToggleExpandLevelAction = {
+  type: "TOGGLE_EXPAND_LEVEL";
+  payload: {
+    level: number;
+    tree: FlatTree<any>;
+  };
+};
+
 type SelectFirstVisibleNodeAction = {
   type: "SELECT_FIRST_VISIBLE_NODE";
   payload: {
@@ -135,6 +175,11 @@ export type Action =
   | ExpandNodeAction
   | CollapseNodeAction
   | ToggleExpandNodeAction
+  | ExpandAllBelowDepthAction
+  | CollapseAllBelowDepthAction
+  | ExpandLevelAction
+  | CollapseLevelAction
+  | ToggleExpandLevelAction
   | SelectFirstVisibleNodeAction
   | SelectLastVisibleNodeAction
   | SelectNextVisibleNodeAction
@@ -229,6 +274,109 @@ export function reducer(state: TreeState, action: Action): TreeState {
         });
       }
     }
+    case "EXPAND_ALL_BELOW_DEPTH": {
+      const nodesToExpand = action.payload.tree.filter(
+        (n) => n.level >= action.payload.depth && n.hasChildren
+      );
+
+      const newNodes = Object.fromEntries(
+        Object.entries(state.nodes).map(([key, value]) => [
+          key,
+          {
+            ...value,
+            expanded: nodesToExpand.find((n) => n.id === key) ? true : value.expanded,
+          },
+        ])
+      );
+
+      const visibleNodes = applyVisibility(action.payload.tree, newNodes);
+      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+    }
+    case "COLLAPSE_ALL_BELOW_DEPTH": {
+      const nodesToCollapse = action.payload.tree.filter(
+        (n) => n.level >= action.payload.depth && n.hasChildren
+      );
+
+      const newNodes = Object.fromEntries(
+        Object.entries(state.nodes).map(([key, value]) => [
+          key,
+          {
+            ...value,
+            expanded: nodesToCollapse.find((n) => n.id === key) ? false : value.expanded,
+          },
+        ])
+      );
+
+      const visibleNodes = applyVisibility(action.payload.tree, newNodes);
+      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+    }
+    case "EXPAND_LEVEL": {
+      const nodesToExpand = action.payload.tree.filter(
+        (n) => n.level <= action.payload.level && n.hasChildren
+      );
+
+      const newNodes = Object.fromEntries(
+        Object.entries(state.nodes).map(([key, value]) => [
+          key,
+          {
+            ...value,
+            expanded: nodesToExpand.find((n) => n.id === key) ? true : value.expanded,
+          },
+        ])
+      );
+
+      const visibleNodes = applyVisibility(action.payload.tree, newNodes);
+      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+    }
+    case "COLLAPSE_LEVEL": {
+      const nodesToCollapse = action.payload.tree.filter(
+        (n) => n.level === action.payload.level && n.hasChildren
+      );
+
+      const newNodes = Object.fromEntries(
+        Object.entries(state.nodes).map(([key, value]) => [
+          key,
+          {
+            ...value,
+            expanded: nodesToCollapse.find((n) => n.id === key) ? false : value.expanded,
+          },
+        ])
+      );
+
+      const visibleNodes = applyVisibility(action.payload.tree, newNodes);
+      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+    }
+    case "TOGGLE_EXPAND_LEVEL": {
+      //first get the first item at that level in the tree. If it is expanded, collapse all nodes at that level
+      //if it is collapsed, expand all nodes at that level
+      const nodesAtLevel = action.payload.tree.filter(
+        (n) => n.level === action.payload.level && n.hasChildren
+      );
+      const firstNode = nodesAtLevel[0];
+      if (!firstNode) {
+        return state;
+      }
+
+      const currentlyExpanded = state.nodes[firstNode.id]?.expanded ?? true;
+      const currentVisible = state.nodes[firstNode.id]?.visible ?? true;
+      if (currentlyExpanded && currentVisible) {
+        return reducer(state, {
+          type: "COLLAPSE_LEVEL",
+          payload: {
+            level: action.payload.level,
+            tree: action.payload.tree,
+          },
+        });
+      } else {
+        return reducer(state, {
+          type: "EXPAND_LEVEL",
+          payload: {
+            level: action.payload.level,
+            tree: action.payload.tree,
+          },
+        });
+      }
+    }
     case "SELECT_FIRST_VISIBLE_NODE": {
       const node = firstVisibleNode(action.payload.tree, state.nodes);
       if (node) {

apps/webapp/app/components/run/RunOverview.tsx

@@ -124,9 +124,9 @@ export function RunOverview({ run, trigger, showRerun, paths, currentUser }: Run
           {!run.isFinished && <CancelRun runId={run.id} />}
         </PageAccessories>
       </NavBar>
-      <PageBody scrollable={false}>
-        <div className="px-4 pt-4">
-          <PageInfoRow>
+      <PageBody scrollable={false} className="grid grid-rows-[auto_1fr] overflow-hidden">
+        <div className="border-b border-grid-dimmed px-4 py-4">
+          <PageInfoRow className="overflow-hidden">
             <PageInfoGroup>
               <PageInfoProperty
                 icon={<RunStatusIcon status={run.status} className="h-4 w-4" />}
@@ -166,7 +166,7 @@ export function RunOverview({ run, trigger, showRerun, paths, currentUser }: Run
             </PageInfoGroup>
           </PageInfoRow>
         </div>
-        <div className="grid h-full grid-cols-2 gap-2">
+        <div className="grid h-full grid-cols-2 gap-2 overflow-hidden">
           <div className="flex flex-col gap-6 overflow-y-auto py-4 pl-4 pr-2 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600">
             <div>
               {run.status === "SUCCESS" &&

apps/webapp/app/components/runs/RunFilters.tsx

@@ -32,6 +32,7 @@ import {
 import { TimeFrameFilter } from "./TimeFrameFilter";
 import { Button } from "../primitives/Buttons";
 import { useCallback } from "react";
+import assertNever from "assert-never";
 
 export function RunsFilters() {
   const navigate = useNavigate();
@@ -182,8 +183,7 @@ export function FilterStatusIcon({
     case "FAILED":
       return <XCircleIcon className={cn(filterStatusClassNameColor(status), className)} />;
     default: {
-      const _exhaustiveCheck: never = status;
-      throw new Error(`Non-exhaustive match for value: ${status}`);
+      assertNever(status);
     }
   }
 }
@@ -205,8 +205,7 @@ export function filterStatusTitle(status: FilterableStatus): string {
     case "TIMEDOUT":
       return "Timed out";
     default: {
-      const _exhaustiveCheck: never = status;
-      throw new Error(`Non-exhaustive match for value: ${status}`);
+      assertNever(status);
     }
   }
 }
@@ -228,8 +227,7 @@ export function filterStatusClassNameColor(status: FilterableStatus): string {
     case "TIMEDOUT":
       return "text-amber-300";
     default: {
-      const _exhaustiveCheck: never = status;
-      throw new Error(`Non-exhaustive match for value: ${status}`);
+      assertNever(status);
     }
   }
 }

apps/webapp/app/components/runs/RunStatuses.tsx

@@ -11,6 +11,7 @@ import type { JobRunStatus } from "@trigger.dev/database";
 import { cn } from "~/utils/cn";
 import { Spinner } from "../primitives/Spinner";
 import { z } from "zod";
+import assertNever from "assert-never";
 
 export function RunStatus({ status }: { status: JobRunStatus }) {
   return (
@@ -51,8 +52,7 @@ export function RunStatusIcon({ status, className }: { status: JobRunStatus; cla
     case "CANCELED":
       return <NoSymbolIcon className={cn(runStatusClassNameColor(status), className)} />;
     default: {
-      const _exhaustiveCheck: never = status;
-      throw new Error(`Non-exhaustive match for value: ${status}`);
+      assertNever(status);
     }
   }
 }
@@ -89,8 +89,7 @@ export function runStatusTitle(status: JobRunStatus): string {
     case "INVALID_PAYLOAD":
       return "Invalid payload";
     default: {
-      const _exhaustiveCheck: never = status;
-      throw new Error(`Non-exhaustive match for value: ${status}`);
+      assertNever(status);
     }
   }
 }
@@ -123,8 +122,7 @@ export function runStatusClassNameColor(status: JobRunStatus): string {
     case "CANCELED":
       return "text-charcoal-500";
     default: {
-      const _exhaustiveCheck: never = status;
-      throw new Error(`Non-exhaustive match for value: ${status}`);
+      assertNever(status);
     }
   }
 }

apps/webapp/app/components/runs/TimeFrameFilter.tsx

@@ -7,6 +7,7 @@ import { DateField } from "../primitives/DateField";
 import { formatDateTime } from "../primitives/DateTime";
 import { Paragraph } from "../primitives/Paragraph";
 import { Popover, PopoverContent, PopoverTrigger } from "../primitives/Popover";
+import { Label } from "../primitives/Label";
 
 type RunTimeFrameFilterProps = {
   from?: number;
@@ -190,7 +191,7 @@ const timeFrameValues = [
 
 export type RelativeTimeFrameItem = (typeof timeFrameValues)[number];
 
-function AbsoluteTimeFrame({
+export function AbsoluteTimeFrame({
   from,
   to,
   onValueChange,
@@ -202,7 +203,8 @@ function AbsoluteTimeFrame({
   return (
     <div className="flex flex-col gap-2 pt-2">
       <div className="flex flex-col justify-start gap-2">
-        <div className="flex flex-row items-center gap-1">
+        <div className="space-y-1">
+          <Label>From (UTC)</Label>
           <DateField
             label="From (UTC)"
             defaultValue={from}
@@ -214,7 +216,8 @@ function AbsoluteTimeFrame({
             showClearButton
           />
         </div>
-        <div className="flex flex-row items-center gap-1">
+        <div className="space-y-1">
+          <Label>To (UTC)</Label>
           <DateField
             label="To (UTC)"
             defaultValue={to}

apps/webapp/app/components/runs/v3/CancelRunDialog.tsx

@@ -1,5 +1,5 @@
 import { StopCircleIcon } from "@heroicons/react/20/solid";
-import { useFetcher } from "@remix-run/react";
+import { Form, useFetcher, useNavigation } from "@remix-run/react";
 import { Button } from "~/components/primitives/Buttons";
 import {
   DialogContent,
@@ -14,29 +14,32 @@ type CancelRunDialogProps = {
 };
 
 export function CancelRunDialog({ runFriendlyId, redirectPath }: CancelRunDialogProps) {
-  const cancelFetcher = useFetcher();
+  const navigation = useNavigation();
+
+  const formAction = `/resources/taskruns/${runFriendlyId}/cancel`;
+  const isLoading = navigation.formAction === formAction;
 
   return (
-    <DialogContent>
+    <DialogContent key="cancel">
       <DialogHeader>Cancel this run?</DialogHeader>
       <DialogDescription>
         Canceling a run will stop execution. If you want to run this later you will have to replay
         the entire run with the original payload.
       </DialogDescription>
       <DialogFooter>
-        <cancelFetcher.Form action={`/resources/taskruns/${runFriendlyId}/cancel`} method="post">
+        <Form action={`/resources/taskruns/${runFriendlyId}/cancel`} method="post">
           <Button
             type="submit"
             name="redirectUrl"
             value={redirectPath}
             variant="danger/small"
-            LeadingIcon={cancelFetcher.state === "idle" ? StopCircleIcon : "spinner-white"}
-            disabled={cancelFetcher.state !== "idle"}
+            LeadingIcon={isLoading ? "spinner-white" : StopCircleIcon}
+            disabled={isLoading}
             shortcut={{ modifiers: ["meta"], key: "enter" }}
           >
-            {cancelFetcher.state === "idle" ? "Cancel run" : "Canceling..."}
+            {isLoading ? "Canceling..." : "Cancel run"}
           </Button>
-        </cancelFetcher.Form>
+        </Form>
       </DialogFooter>
     </DialogContent>
   );

apps/webapp/app/components/runs/v3/DeploymentError.tsx

@@ -0,0 +1,39 @@
+import { CodeBlock } from "~/components/code/CodeBlock";
+import { Callout } from "~/components/primitives/Callout";
+import { Header2 } from "~/components/primitives/Headers";
+import type { ErrorData } from "~/presenters/v3/DeploymentPresenter.server";
+
+type DeploymentErrorProps = {
+  errorData: ErrorData;
+};
+
+export function DeploymentError({ errorData }: DeploymentErrorProps) {
+  return (
+    <div className="flex flex-col gap-2 rounded-sm border border-rose-500/50 p-3">
+      <DeploymentErrorHeader title={errorData.name ?? "Error"} titleClassName="text-rose-500" />
+      {errorData.message && <Callout variant="error">{errorData.message}</Callout>}
+      {errorData.stack && (
+        <CodeBlock
+          showCopyButton={false}
+          showLineNumbers={false}
+          code={errorData.stack}
+          maxLines={20}
+        />
+      )}
+    </div>
+  );
+}
+
+function DeploymentErrorHeader({
+  title,
+  titleClassName,
+}: {
+  title: string;
+  titleClassName?: string;
+}) {
+  return (
+    <div className="flex items-center justify-between">
+      <Header2 className={titleClassName}>{title}</Header2>
+    </div>
+  );
+}

apps/webapp/app/components/runs/v3/DeploymentStatus.tsx

@@ -5,6 +5,7 @@ import {
   XCircleIcon,
 } from "@heroicons/react/20/solid";
 import { WorkerDeploymentStatus } from "@trigger.dev/database";
+import assertNever from "assert-never";
 import { Spinner } from "~/components/primitives/Spinner";
 import { cn } from "~/utils/cn";
 
@@ -54,8 +55,7 @@ export function DeploymentStatusIcon({
         />
       );
     default: {
-      const _exhaustiveCheck: never = status;
-      throw new Error(`Non-exhaustive match for value: ${status}`);
+      assertNever(status);
     }
   }
 }
@@ -74,8 +74,7 @@ export function deploymentStatusClassNameColor(status: WorkerDeploymentStatus):
     case "FAILED":
       return "text-error";
     default: {
-      const _exhaustiveCheck: never = status;
-      throw new Error(`Non-exhaustive match for value: ${status}`);
+      assertNever(status);
     }
   }
 }
@@ -97,8 +96,7 @@ export function deploymentStatusTitle(status: WorkerDeploymentStatus): string {
     case "FAILED":
       return "Failed";
     default: {
-      const _exhaustiveCheck: never = status;
-      throw new Error(`Non-exhaustive match for value: ${status}`);
+      assertNever(status);
     }
   }
 }

apps/webapp/app/components/runs/v3/EnabledStatus.tsx

@@ -0,0 +1,20 @@
+import { BoltSlashIcon, CheckCircleIcon } from "@heroicons/react/20/solid";
+
+export function EnabledStatus({ enabled }: { enabled: boolean }) {
+  switch (enabled) {
+    case true:
+      return (
+        <div className="flex items-center gap-1 text-xs text-success">
+          <CheckCircleIcon className="h-4 w-4" />
+          Enabled
+        </div>
+      );
+    case false:
+      return (
+        <div className="text-dimmed flex items-center gap-1 text-xs">
+          <BoltSlashIcon className="h-4 w-4" />
+          Disabled
+        </div>
+      );
+  }
+}

apps/webapp/app/components/runs/v3/ReplayRunDialog.tsx

@@ -0,0 +1,44 @@
+import { ArrowPathIcon } from "@heroicons/react/20/solid";
+import { Form, useFetcher, useNavigation } from "@remix-run/react";
+import { Button } from "~/components/primitives/Buttons";
+import {
+  DialogContent,
+  DialogDescription,
+  DialogFooter,
+  DialogHeader,
+} from "~/components/primitives/Dialog";
+
+type ReplayRunDialogProps = {
+  runFriendlyId: string;
+  failedRedirect: string;
+};
+
+export function ReplayRunDialog({ runFriendlyId, failedRedirect }: ReplayRunDialogProps) {
+  const navigation = useNavigation();
+
+  const formAction = `/resources/taskruns/${runFriendlyId}/replay`;
+  const isLoading = navigation.formAction === formAction;
+
+  return (
+    <DialogContent key="replay">
+      <DialogHeader>Replay this run?</DialogHeader>
+      <DialogDescription>
+        Replaying a run will create a new run with the same payload and environment as the original.
+      </DialogDescription>
+      <DialogFooter>
+        <Form action={formAction} method="post">
+          <input type="hidden" name="failedRedirect" value={failedRedirect} />
+          <Button
+            type="submit"
+            variant="primary/small"
+            LeadingIcon={isLoading ? "spinner-white" : ArrowPathIcon}
+            disabled={isLoading}
+            shortcut={{ modifiers: ["meta"], key: "enter" }}
+          >
+            {isLoading ? "Replaying..." : "Replay run"}
+          </Button>
+        </Form>
+      </DialogFooter>
+    </DialogContent>
+  );
+}

apps/webapp/app/components/runs/v3/RunFilters.tsx

@@ -1,4 +1,4 @@
-import { TrashIcon } from "@heroicons/react/20/solid";
+import { XMarkIcon } from "@heroicons/react/20/solid";
 import { useNavigate } from "@remix-run/react";
 import type { TaskRunStatus as TaskRunStatusType } from "@trigger.dev/database";
 import { RuntimeEnvironment, TaskRunAttemptStatus, TaskRunStatus } from "@trigger.dev/database";
@@ -27,6 +27,7 @@ import { TaskRunStatusCombo, descriptionForTaskRunStatus } from "./TaskRunStatus
 
 export const allTaskRunStatuses = [
   "PENDING",
+  "WAITING_FOR_DEPLOY",
   "EXECUTING",
   "RETRYING_AFTER_FAILURE",
   "WAITING_TO_RESUME",
@@ -35,6 +36,7 @@ export const allTaskRunStatuses = [
   "COMPLETED_WITH_ERRORS",
   "INTERRUPTED",
   "SYSTEM_FAILURE",
+  "CRASHED",
 ] as TaskRunStatusType[];
 
 export const TaskAttemptStatus = z.nativeEnum(TaskRunStatus);
@@ -227,23 +229,25 @@ export function RunsFilters({ possibleEnvironments, possibleTasks }: RunFiltersP
                 All tasks
               </Paragraph>
             </SelectItem>
-            {possibleTasks.map((task) => (
-              <SelectItem key={task} value={task}>
-                <Paragraph
-                  variant="extra-small"
-                  className="pl-0.5 transition group-hover:text-text-bright"
-                >
-                  {task}
-                </Paragraph>
-              </SelectItem>
-            ))}
+            {possibleTasks
+              .sort((a, b) => a.localeCompare(b)) // 🔤
+              .map((task) => (
+                <SelectItem key={task} value={task}>
+                  <Paragraph
+                    variant="extra-small"
+                    className="pl-0.5 transition group-hover:text-text-bright"
+                  >
+                    {task}
+                  </Paragraph>
+                </SelectItem>
+              ))}
           </SelectContent>
         </Select>
       </SelectGroup>
 
       <TimeFrameFilter from={from} to={to} onRangeChanged={handleTimeFrameChange} />
 
-      <Button variant="minimal/small" onClick={() => clearFilters()} LeadingIcon={TrashIcon} />
+      <Button variant="minimal/small" onClick={() => clearFilters()} LeadingIcon={XMarkIcon} />
     </div>
   );
 }

apps/webapp/app/components/runs/v3/RunIcon.tsx

@@ -1,6 +1,5 @@
 import {
   ClockIcon,
-  FolderIcon,
   HandRaisedIcon,
   InformationCircleIcon,
   Squares2X2Icon,
@@ -41,6 +40,8 @@ export function RunIcon({ name, className, spanName }: TaskIconProps) {
   switch (name) {
     case "task":
       return <TaskIcon className={cn(className, "text-blue-500")} />;
+    case "scheduled":
+      return <ClockIcon className={cn(className, "text-sun-500")} />;
     case "attempt":
       return <AttemptIcon className={cn(className, "text-text-dimmed")} />;
     case "wait":

apps/webapp/app/components/runs/v3/ScheduleFilters.tsx

@@ -0,0 +1,162 @@
+import { TrashIcon, XMarkIcon } from "@heroicons/react/20/solid";
+import { useNavigate } from "@remix-run/react";
+import { RuntimeEnvironment } from "@trigger.dev/database";
+import { useCallback } from "react";
+import { z } from "zod";
+import { useOptimisticLocation } from "~/hooks/useOptimisticLocation";
+import { EnvironmentLabel } from "../../environments/EnvironmentLabel";
+import { Button } from "../../primitives/Buttons";
+import { Paragraph } from "../../primitives/Paragraph";
+import {
+  Select,
+  SelectContent,
+  SelectGroup,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "../../primitives/Select";
+import { Input } from "~/components/primitives/Input";
+import { useDebounce } from "~/hooks/useDebounce";
+import { useThrottle } from "~/hooks/useThrottle";
+
+export const ScheduleListFilters = z.object({
+  page: z.coerce.number().default(1),
+  tasks: z
+    .string()
+    .optional()
+    .transform((value) => (value ? value.split(",") : undefined)),
+  environments: z
+    .string()
+    .optional()
+    .transform((value) => (value ? value.split(",") : undefined)),
+  search: z.string().optional(),
+});
+
+export type ScheduleListFilters = z.infer<typeof ScheduleListFilters>;
+
+const All = "ALL";
+
+type DisplayableEnvironment = Pick<RuntimeEnvironment, "type" | "id"> & {
+  userName?: string;
+};
+
+type ScheduleFiltersProps = {
+  possibleEnvironments: DisplayableEnvironment[];
+  possibleTasks: string[];
+};
+
+export function ScheduleFilters({ possibleEnvironments, possibleTasks }: ScheduleFiltersProps) {
+  const navigate = useNavigate();
+  const location = useOptimisticLocation();
+  const searchParams = new URLSearchParams(location.search);
+  const { environments, tasks, page, search } = ScheduleListFilters.parse(
+    Object.fromEntries(searchParams.entries())
+  );
+
+  const handleFilterChange = useCallback((filterType: string, value: string | undefined) => {
+    if (value) {
+      searchParams.set(filterType, value);
+    } else {
+      searchParams.delete(filterType);
+    }
+    searchParams.delete("page");
+    navigate(`${location.pathname}?${searchParams.toString()}`);
+  }, []);
+
+  const handleTaskChange = useCallback((value: string | typeof All) => {
+    handleFilterChange("tasks", value === "ALL" ? undefined : value);
+  }, []);
+
+  const handleEnvironmentChange = useCallback((value: string | typeof All) => {
+    handleFilterChange("environments", value === "ALL" ? undefined : value);
+  }, []);
+
+  const handleSearchChange = useThrottle((value: string) => {
+    handleFilterChange("search", value.length === 0 ? undefined : value);
+  }, 300);
+
+  const clearFilters = useCallback(() => {
+    searchParams.delete("page");
+    searchParams.delete("enabled");
+    searchParams.delete("tasks");
+    searchParams.delete("environments");
+    searchParams.delete("search");
+    navigate(`${location.pathname}?${searchParams.toString()}`);
+  }, []);
+
+  return (
+    <div className="flex w-full flex-row">
+      <Input
+        name="search"
+        placeholder="Search schedule id, external id, deduplication id or CRON pattern"
+        icon="search"
+        variant="tertiary"
+        className="grow"
+        defaultValue={search}
+        onChange={(e) => handleSearchChange(e.target.value)}
+      />
+      <SelectGroup>
+        <Select
+          name="environment"
+          value={environments?.at(0) ?? "ALL"}
+          onValueChange={handleEnvironmentChange}
+        >
+          <SelectTrigger size="minimal" width="full">
+            <SelectValue
+              placeholder={"Select environment"}
+              className="ml-2 whitespace-nowrap p-0"
+            />
+          </SelectTrigger>
+          <SelectContent>
+            <SelectItem value={"ALL"}>
+              <Paragraph
+                variant="extra-small"
+                className="whitespace-nowrap pl-0.5 transition group-hover:text-text-bright"
+              >
+                All environments
+              </Paragraph>
+            </SelectItem>
+            {possibleEnvironments.map((env) => (
+              <SelectItem key={env.id} value={env.id}>
+                <div className="flex items-center gap-x-2">
+                  <EnvironmentLabel environment={env} userName={env.userName} />
+                  <Paragraph variant="extra-small">environment</Paragraph>
+                </div>
+              </SelectItem>
+            ))}
+          </SelectContent>
+        </Select>
+      </SelectGroup>
+
+      <SelectGroup>
+        <Select name="tasks" value={tasks?.at(0) ?? "ALL"} onValueChange={handleTaskChange}>
+          <SelectTrigger size="minimal" width="full">
+            <SelectValue placeholder="Select task" className="ml-2 p-0" />
+          </SelectTrigger>
+          <SelectContent>
+            <SelectItem value={"ALL"}>
+              <Paragraph
+                variant="extra-small"
+                className="whitespace-nowrap pl-0.5 transition group-hover:text-text-bright"
+              >
+                All tasks
+              </Paragraph>
+            </SelectItem>
+            {possibleTasks.map((task) => (
+              <SelectItem key={task} value={task}>
+                <Paragraph
+                  variant="extra-small"
+                  className="pl-0.5 transition group-hover:text-text-bright"
+                >
+                  {task}
+                </Paragraph>
+              </SelectItem>
+            ))}
+          </SelectContent>
+        </Select>
+      </SelectGroup>
+
+      <Button variant="minimal/small" onClick={() => clearFilters()} LeadingIcon={XMarkIcon} />
+    </div>
+  );
+}

apps/webapp/app/components/runs/v3/SpanEvents.tsx

@@ -66,7 +66,11 @@ function SpanEventError({
 }) {
   return (
     <div className="flex flex-col gap-2 rounded-sm border border-rose-500/50 p-3">
-      <SpanEventHeader title={"Error"} time={spanEvent.time} titleClassName="text-rose-500" />
+      <SpanEventHeader
+        title={exception.type ?? "Error"}
+        time={spanEvent.time}
+        titleClassName="text-rose-500"
+      />
       {exception.message && <Callout variant="error">{exception.message}</Callout>}
       {exception.stacktrace && (
         <CodeBlock

apps/webapp/app/components/runs/v3/SpanTitle.tsx

@@ -73,14 +73,7 @@ export function SpanCodePathAccessory({
     >
       {accessory.items.map((item, index) => (
         <Fragment key={index}>
-          <span
-            className={cn(
-              "truncate",
-              index === accessory.items.length - 1 ? "text-sun-100" : "text-text-dimmed"
-            )}
-          >
-            {item.text}
-          </span>
+          <span className={cn("truncate", "text-text-dimmed")}>{item.text}</span>
           {index < accessory.items.length - 1 && (
             <span className="text-text-dimmed">
               <ChevronRightIcon className="h-4 w-4" />

apps/webapp/app/components/runs/v3/TaskPath.tsx

@@ -27,7 +27,7 @@ type TaskFunctionNameProps = {
 
 export function TaskFunctionName({ variant, functionName, className }: TaskFunctionNameProps) {
   return (
-    <InlineCode variant={variant} className={cn("text-sun-100", className)}>
+    <InlineCode variant={variant} className={cn("text-text-dimmed", className)}>
       {`${functionName}()`}
     </InlineCode>
   );

apps/webapp/app/components/runs/v3/TaskRunAttemptStatus.tsx

@@ -8,6 +8,7 @@ import {
 } from "@heroicons/react/20/solid";
 import type { TaskRunAttemptStatus as TaskRunAttemptStatusType } from "@trigger.dev/database";
 import { TaskRunAttemptStatus } from "@trigger.dev/database";
+import assertNever from "assert-never";
 import { SnowflakeIcon } from "lucide-react";
 import { Spinner } from "~/components/primitives/Spinner";
 import { cn } from "~/utils/cn";
@@ -72,8 +73,7 @@ export function TaskRunAttemptStatusIcon({
     case "COMPLETED":
       return <CheckCircleIcon className={cn(runAttemptStatusClassNameColor(status), className)} />;
     default: {
-      const _exhaustiveCheck: never = status;
-      throw new Error(`Non-exhaustive match for value: ${status}`);
+      assertNever(status);
     }
   }
 }
@@ -99,8 +99,7 @@ export function runAttemptStatusClassNameColor(status: ExtendedTaskAttemptStatus
     case "COMPLETED":
       return "text-success";
     default: {
-      const _exhaustiveCheck: never = status;
-      throw new Error(`Non-exhaustive match for value: ${status}`);
+      assertNever(status);
     }
   }
 }
@@ -126,8 +125,7 @@ export function runAttemptStatusTitle(status: ExtendedTaskAttemptStatus | null):
     case "COMPLETED":
       return "Completed";
     default: {
-      const _exhaustiveCheck: never = status;
-      throw new Error(`Non-exhaustive match for value: ${status}`);
+      assertNever(status);
     }
   }
 }

apps/webapp/app/components/runs/v3/TaskRunStatus.tsx

@@ -3,19 +3,21 @@ import {
   BoltSlashIcon,
   BugAntIcon,
   CheckCircleIcon,
-  ClockIcon,
+  FireIcon,
   NoSymbolIcon,
   PauseCircleIcon,
   RectangleStackIcon,
   XCircleIcon,
 } from "@heroicons/react/20/solid";
 import { TaskRunStatus } from "@trigger.dev/database";
+import assertNever from "assert-never";
 import { SnowflakeIcon } from "lucide-react";
 import { Spinner } from "~/components/primitives/Spinner";
 import { cn } from "~/utils/cn";
 
 const taskRunStatusDescriptions: Record<TaskRunStatus, string> = {
   PENDING: "Task is waiting to be executed",
+  WAITING_FOR_DEPLOY: "Task needs to be deployed first to start executing",
   EXECUTING: "Task is currently being executed",
   RETRYING_AFTER_FAILURE: "Task is being reattempted after a failure",
   WAITING_TO_RESUME: "Task has been frozen and is waiting to be resumed",
@@ -25,8 +27,17 @@ const taskRunStatusDescriptions: Record<TaskRunStatus, string> = {
   INTERRUPTED: "Task has failed because it was interrupted",
   SYSTEM_FAILURE: "Task has failed due to a system failure",
   PAUSED: "Task has been paused by the user",
+  CRASHED: "Task has crashed and won't be retried",
 };
 
+export const QUEUED_STATUSES: TaskRunStatus[] = ["PENDING", "WAITING_FOR_DEPLOY"];
+
+export const RUNNING_STATUSES: TaskRunStatus[] = [
+  "EXECUTING",
+  "RETRYING_AFTER_FAILURE",
+  "WAITING_TO_RESUME",
+];
+
 export function descriptionForTaskRunStatus(status: TaskRunStatus): string {
   return taskRunStatusDescriptions[status];
 }
@@ -62,6 +73,8 @@ export function TaskRunStatusIcon({
   switch (status) {
     case "PENDING":
       return <RectangleStackIcon className={cn(runStatusClassNameColor(status), className)} />;
+    case "WAITING_FOR_DEPLOY":
+      return <RectangleStackIcon className={cn(runStatusClassNameColor(status), className)} />;
     case "EXECUTING":
       return <Spinner className={cn(runStatusClassNameColor(status), className)} />;
     case "WAITING_TO_RESUME":
@@ -80,10 +93,11 @@ export function TaskRunStatusIcon({
       return <XCircleIcon className={cn(runStatusClassNameColor(status), className)} />;
     case "SYSTEM_FAILURE":
       return <BugAntIcon className={cn(runStatusClassNameColor(status), className)} />;
+    case "CRASHED":
+      return <FireIcon className={cn(runStatusClassNameColor(status), className)} />;
 
     default: {
-      const _exhaustiveCheck: never = status;
-      throw new Error(`Non-exhaustive match for value: ${status}`);
+      assertNever(status);
     }
   }
 }
@@ -92,6 +106,8 @@ export function runStatusClassNameColor(status: TaskRunStatus): string {
   switch (status) {
     case "PENDING":
       return "text-charcoal-500";
+    case "WAITING_FOR_DEPLOY":
+      return "text-amber-500";
     case "EXECUTING":
     case "RETRYING_AFTER_FAILURE":
       return "text-pending";
@@ -109,9 +125,10 @@ export function runStatusClassNameColor(status: TaskRunStatus): string {
       return "text-error";
     case "SYSTEM_FAILURE":
       return "text-error";
+    case "CRASHED":
+      return "text-error";
     default: {
-      const _exhaustiveCheck: never = status;
-      throw new Error(`Non-exhaustive match for value: ${status}`);
+      assertNever(status);
     }
   }
 }
@@ -120,6 +137,8 @@ export function runStatusTitle(status: TaskRunStatus): string {
   switch (status) {
     case "PENDING":
       return "Queued";
+    case "WAITING_FOR_DEPLOY":
+      return "Waiting for deploy";
     case "EXECUTING":
       return "Executing";
     case "WAITING_TO_RESUME":
@@ -138,9 +157,10 @@ export function runStatusTitle(status: TaskRunStatus): string {
       return "Failed";
     case "SYSTEM_FAILURE":
       return "System failure";
+    case "CRASHED":
+      return "Crashed";
     default: {
-      const _exhaustiveCheck: never = status;
-      throw new Error(`Non-exhaustive match for value: ${status}`);
+      assertNever(status);
     }
   }
 }

apps/webapp/app/components/runs/v3/TaskRunsTable.tsx

@@ -1,10 +1,16 @@
+import { ArrowPathIcon, StopCircleIcon } from "@heroicons/react/20/solid";
 import { StopIcon } from "@heroicons/react/24/outline";
 import { BeakerIcon, BookOpenIcon, CheckIcon } from "@heroicons/react/24/solid";
+import { useLocation } from "@remix-run/react";
+import { formatDuration } from "@trigger.dev/core/v3";
 import { User } from "@trigger.dev/database";
+import { Button, LinkButton } from "~/components/primitives/Buttons";
+import { Dialog, DialogTrigger } from "~/components/primitives/Dialog";
+import { useEnvironments } from "~/hooks/useEnvironments";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
 import { RunListAppliedFilters, RunListItem } from "~/presenters/v3/RunListPresenter.server";
-import { docsPath, v3RunPath, v3TestPath } from "~/utils/pathBuilder";
+import { docsPath, v3RunSpanPath, v3TestPath } from "~/utils/pathBuilder";
 import { EnvironmentLabel } from "../../environments/EnvironmentLabel";
 import { DateTime } from "../../primitives/DateTime";
 import { Paragraph } from "../../primitives/Paragraph";
@@ -14,20 +20,14 @@ import {
   TableBlankRow,
   TableBody,
   TableCell,
-  TableCellChevron,
   TableCellMenu,
   TableHeader,
   TableHeaderCell,
   TableRow,
 } from "../../primitives/Table";
-import { formatDuration } from "@trigger.dev/core/v3";
-import { TaskRunStatusCombo } from "./TaskRunStatus";
-import { useEnvironments } from "~/hooks/useEnvironments";
-import { Button, LinkButton } from "~/components/primitives/Buttons";
-import { StopCircleIcon } from "@heroicons/react/20/solid";
-import { Dialog, DialogTrigger } from "~/components/primitives/Dialog";
 import { CancelRunDialog } from "./CancelRunDialog";
-import { useLocation } from "@remix-run/react";
+import { ReplayRunDialog } from "./ReplayRunDialog";
+import { TaskRunStatusCombo } from "./TaskRunStatus";
 
 type RunsTableProps = {
   total: number;
@@ -36,7 +36,6 @@ type RunsTableProps = {
   showJob?: boolean;
   runs: RunListItem[];
   isLoading?: boolean;
-  currentUser: User;
 };
 
 export function TaskRunsTable({
@@ -45,11 +44,9 @@ export function TaskRunsTable({
   filters,
   runs,
   isLoading = false,
-  currentUser,
 }: RunsTableProps) {
   const organization = useOrganization();
   const project = useProject();
-  const location = useLocation();
 
   return (
     <Table>
@@ -78,16 +75,17 @@ export function TaskRunsTable({
           <BlankState isLoading={isLoading} filters={filters} />
         ) : (
           runs.map((run) => {
-            const path = v3RunPath(organization, project, run);
-            const usernameForEnv =
-              currentUser.id !== run.environment.userId ? run.environment.userName : undefined;
+            const path = v3RunSpanPath(organization, project, run, { spanId: run.spanId });
             return (
               <TableRow key={run.id}>
                 <TableCell to={path}>#{run.number}</TableCell>
                 <TableCell to={path}>{run.taskIdentifier}</TableCell>
                 <TableCell to={path}>{run.version ?? "–"}</TableCell>
                 <TableCell to={path}>
-                  <EnvironmentLabel environment={run.environment} userName={usernameForEnv} />
+                  <EnvironmentLabel
+                    environment={run.environment}
+                    userName={run.environment.userName}
+                  />
                 </TableCell>
                 <TableCell to={path}>
                   <TaskRunStatusCombo status={run.status} />
@@ -110,23 +108,7 @@ export function TaskRunsTable({
                 <TableCell to={path}>
                   {run.createdAt ? <DateTime date={run.createdAt} /> : "–"}
                 </TableCell>
-                {run.isCancellable ? (
-                  <TableCellMenu isSticky>
-                    <Dialog>
-                      <DialogTrigger asChild>
-                        <Button variant="small-menu-item" LeadingIcon={StopCircleIcon}>
-                          Cancel run
-                        </Button>
-                      </DialogTrigger>
-                      <CancelRunDialog
-                        runFriendlyId={run.friendlyId}
-                        redirectPath={`${location.pathname}${location.search}`}
-                      />
-                    </Dialog>
-                  </TableCellMenu>
-                ) : (
-                  <TableCell to={path}>{""}</TableCell>
-                )}
+                <RunActionsCell run={run} path={path} />
               </TableRow>
             );
           })
@@ -144,6 +126,43 @@ export function TaskRunsTable({
   );
 }
 
+function RunActionsCell({ run, path }: { run: RunListItem; path: string }) {
+  const location = useLocation();
+
+  if (!run.isCancellable && !run.isReplayable) return <TableCell to={path}>{""}</TableCell>;
+
+  return (
+    <TableCellMenu isSticky>
+      {run.isCancellable && (
+        <Dialog>
+          <DialogTrigger asChild>
+            <Button variant="small-menu-item" LeadingIcon={StopCircleIcon}>
+              Cancel run
+            </Button>
+          </DialogTrigger>
+          <CancelRunDialog
+            runFriendlyId={run.friendlyId}
+            redirectPath={`${location.pathname}${location.search}`}
+          />
+        </Dialog>
+      )}
+      {run.isReplayable && (
+        <Dialog>
+          <DialogTrigger asChild>
+            <Button variant="small-menu-item" LeadingIcon={ArrowPathIcon}>
+              Replay run
+            </Button>
+          </DialogTrigger>
+          <ReplayRunDialog
+            runFriendlyId={run.friendlyId}
+            failedRedirect={`${location.pathname}${location.search}`}
+          />
+        </Dialog>
+      )}
+    </TableCellMenu>
+  );
+}
+
 function NoRuns({ title }: { title: string }) {
   return (
     <div className="flex items-center justify-center">

apps/webapp/app/components/runs/v3/TaskTriggerSource.tsx

@@ -0,0 +1,36 @@
+import { ClockIcon } from "@heroicons/react/20/solid";
+import { TaskTriggerSource } from "@trigger.dev/database";
+import { TaskIcon } from "~/assets/icons/TaskIcon";
+import { cn } from "~/utils/cn";
+
+export function TaskTriggerSourceIcon({
+  source,
+  className,
+}: {
+  source: TaskTriggerSource;
+  className?: string;
+}) {
+  switch (source) {
+    case "STANDARD": {
+      return (
+        <div className={cn("grid size-4 place-items-center text-blue-500", className)}>
+          <TaskIcon className="size-[87.5%]" />
+        </div>
+      );
+    }
+    case "SCHEDULED": {
+      return <ClockIcon className={cn("size-4 text-sun-500", className)} />;
+    }
+  }
+}
+
+export function taskTriggerSourceDescription(source: TaskTriggerSource) {
+  switch (source) {
+    case "STANDARD": {
+      return "Standard task";
+    }
+    case "SCHEDULED": {
+      return "Scheduled task";
+    }
+  }
+}

apps/webapp/app/db.server.ts

@@ -4,6 +4,7 @@ import { z } from "zod";
 import { logger } from "./services/logger.server";
 import { env } from "./env.server";
 import { singleton } from "./utils/singleton";
+import { isValidDatabaseUrl } from "./utils/db";
 
 export type PrismaTransactionClient = Omit<
   PrismaClient,
@@ -69,23 +70,21 @@ export { Prisma };
 
 export const prisma = singleton("prisma", getClient);
 
+export const $replica: Omit<PrismaClient, "$transaction"> = singleton(
+  "replica",
+  () => getReplicaClient() ?? prisma
+);
+
 function getClient() {
   const { DATABASE_URL } = process.env;
   invariant(typeof DATABASE_URL === "string", "DATABASE_URL env var not set");
 
-  const databaseUrl = new URL(DATABASE_URL);
+  const databaseUrl = extendQueryParams(DATABASE_URL, {
+    connection_limit: env.DATABASE_CONNECTION_LIMIT.toString(),
+    pool_timeout: env.DATABASE_POOL_TIMEOUT.toString(),
+  });
 
-  // We need to add the connection_limit and pool_timeout query params to the url, in a way that works if the DATABASE_URL already has query params
-  const query = databaseUrl.searchParams;
-  query.set("connection_limit", env.DATABASE_CONNECTION_LIMIT.toString());
-  query.set("pool_timeout", env.DATABASE_POOL_TIMEOUT.toString());
-  databaseUrl.search = query.toString();
-
-  // Remove the username:password in the url and print that to the console
-  const urlWithoutCredentials = new URL(databaseUrl.href);
-  urlWithoutCredentials.password = "";
-
-  console.log(`🔌 setting up prisma client to ${urlWithoutCredentials.toString()}`);
+  console.log(`🔌 setting up prisma client to ${redactUrlSecrets(databaseUrl)}`);
 
   const client = new PrismaClient({
     datasources: {
@@ -133,8 +132,90 @@ function getClient() {
   return client;
 }
 
+function getReplicaClient() {
+  if (!env.DATABASE_READ_REPLICA_URL) {
+    console.log(`🔌 No database replica, using the regular client`);
+    return;
+  }
+
+  const replicaUrl = extendQueryParams(env.DATABASE_READ_REPLICA_URL, {
+    connection_limit: env.DATABASE_CONNECTION_LIMIT.toString(),
+    pool_timeout: env.DATABASE_POOL_TIMEOUT.toString(),
+  });
+
+  console.log(`🔌 setting up read replica connection to ${redactUrlSecrets(replicaUrl)}`);
+
+  const replicaClient = new PrismaClient({
+    datasources: {
+      db: {
+        url: replicaUrl.href,
+      },
+    },
+    log: [
+      {
+        emit: "stdout",
+        level: "error",
+      },
+      {
+        emit: "stdout",
+        level: "info",
+      },
+      {
+        emit: "stdout",
+        level: "warn",
+      },
+    ],
+  });
+
+  // connect eagerly
+  replicaClient.$connect();
+
+  console.log(`🔌 read replica connected`);
+
+  return replicaClient;
+}
+
+function extendQueryParams(hrefOrUrl: string | URL, queryParams: Record<string, string>) {
+  const url = new URL(hrefOrUrl);
+  const query = url.searchParams;
+
+  for (const [key, val] of Object.entries(queryParams)) {
+    query.set(key, val);
+  }
+
+  url.search = query.toString();
+
+  return url;
+}
+
+function redactUrlSecrets(hrefOrUrl: string | URL) {
+  const url = new URL(hrefOrUrl);
+  url.password = "";
+  return url.href;
+}
+
 export type { PrismaClient } from "@trigger.dev/database";
 
 export const PrismaErrorSchema = z.object({
   code: z.string(),
 });
+
+function getDatabaseSchema() {
+  if (!isValidDatabaseUrl(env.DATABASE_URL)) {
+    throw new Error("Invalid Database URL");
+  }
+
+  const databaseUrl = new URL(env.DATABASE_URL);
+  const schemaFromSearchParam = databaseUrl.searchParams.get("schema");
+
+  if (!schemaFromSearchParam) {
+    console.debug("❗ database schema unspecified, will default to `public` schema");
+    return "public";
+  }
+
+  return schemaFromSearchParam;
+}
+
+export const DATABASE_SCHEMA = singleton("DATABASE_SCHEMA", getDatabaseSchema);
+
+export const sqlDatabaseSchema = Prisma.sql([`${DATABASE_SCHEMA}`]);

apps/webapp/app/env.server.ts

@@ -1,13 +1,25 @@
 import { z } from "zod";
 import { SecretStoreOptionsSchema } from "./services/secrets/secretStore.server";
 import { isValidRegex } from "./utils/regex";
+import { isValidDatabaseUrl } from "./utils/db";
 
 const EnvironmentSchema = z.object({
   NODE_ENV: z.union([z.literal("development"), z.literal("production"), z.literal("test")]),
-  DATABASE_URL: z.string(),
+  DATABASE_URL: z
+    .string()
+    .refine(
+      isValidDatabaseUrl,
+      "DATABASE_URL is invalid, for details please check the additional output above this message."
+    ),
   DATABASE_CONNECTION_LIMIT: z.coerce.number().int().default(10),
   DATABASE_POOL_TIMEOUT: z.coerce.number().int().default(60),
-  DIRECT_URL: z.string(),
+  DIRECT_URL: z
+    .string()
+    .refine(
+      isValidDatabaseUrl,
+      "DIRECT_URL is invalid, for details please check the additional output above this message."
+    ),
+  DATABASE_READ_REPLICA_URL: z.string().optional(),
   SESSION_SECRET: z.string(),
   MAGIC_LINK_SECRET: z.string(),
   ENCRYPTION_KEY: z.string(),
@@ -19,6 +31,7 @@ const EnvironmentSchema = z.object({
   LOGIN_ORIGIN: z.string().default("http://localhost:3030"),
   APP_ORIGIN: z.string().default("http://localhost:3030"),
   APP_ENV: z.string().default(process.env.NODE_ENV),
+  SERVICE_NAME: z.string().default("trigger.dev webapp"),
   SECRET_STORE: SecretStoreOptionsSchema.default("DATABASE"),
   POSTHOG_PROJECT_KEY: z.string().optional(),
   TELEMETRY_TRIGGER_API_KEY: z.string().optional(),
@@ -53,6 +66,7 @@ const EnvironmentSchema = z.object({
   AWS_SQS_BATCH_SIZE: z.coerce.number().int().optional().default(1),
   AWS_SQS_WAIT_TIME_MS: z.coerce.number().int().optional().default(100),
   DISABLE_SSE: z.string().optional(),
+  OPENAI_API_KEY: z.string().optional(),
 
   // Redis options
   REDIS_HOST: z.string().optional(),
@@ -63,7 +77,6 @@ const EnvironmentSchema = z.object({
   REDIS_PASSWORD: z.string().optional(),
   REDIS_TLS_DISABLED: z.string().optional(),
 
-  DEFAULT_QUEUE_EXECUTION_CONCURRENCY_LIMIT: z.coerce.number().int().default(5),
   DEFAULT_ENV_EXECUTION_CONCURRENCY_LIMIT: z.coerce.number().int().default(10),
   DEFAULT_ORG_EXECUTION_CONCURRENCY_LIMIT: z.coerce.number().int().default(10),
   DEFAULT_DEV_ENV_EXECUTION_ATTEMPTS: z.coerce.number().int().positive().default(1),
@@ -87,8 +100,6 @@ const EnvironmentSchema = z.object({
 
   //v3
   V3_ENABLED: z.string().default("false"),
-  IMAGE_REGISTRY: z.string().default("docker.io"),
-  IMAGE_REPO: z.string().default("task"),
   PROVIDER_SECRET: z.string().default("provider-secret"),
   COORDINATOR_SECRET: z.string().default("coordinator-secret"),
   DEPOT_TOKEN: z.string().optional(),
@@ -103,6 +114,9 @@ const EnvironmentSchema = z.object({
   EVENTS_BATCH_SIZE: z.coerce.number().int().default(100),
   EVENTS_BATCH_INTERVAL: z.coerce.number().int().default(1000),
   EVENTS_DEFAULT_LOG_RETENTION: z.coerce.number().int().default(7),
+  SHARED_QUEUE_CONSUMER_POOL_SIZE: z.coerce.number().int().default(10),
+  SHARED_QUEUE_CONSUMER_INTERVAL_MS: z.coerce.number().int().default(100),
+  SHARED_QUEUE_CONSUMER_NEXT_TICK_INTERVAL_MS: z.coerce.number().int().default(100),
 
   // Development OTEL environment variables
   DEV_OTEL_EXPORTER_OTLP_ENDPOINT: z.string().optional(),
@@ -116,6 +130,17 @@ const EnvironmentSchema = z.object({
   DEV_OTEL_LOG_SCHEDULED_DELAY_MILLIS: z.string().default("200"),
   DEV_OTEL_LOG_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"),
   DEV_OTEL_LOG_MAX_QUEUE_SIZE: z.string().default("512"),
+
+  PROD_OTEL_BATCH_PROCESSING_ENABLED: z.string().default("0"),
+  PROD_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE: z.string().default("64"),
+  PROD_OTEL_SPAN_SCHEDULED_DELAY_MILLIS: z.string().default("200"),
+  PROD_OTEL_SPAN_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"),
+  PROD_OTEL_SPAN_MAX_QUEUE_SIZE: z.string().default("512"),
+  PROD_OTEL_LOG_MAX_EXPORT_BATCH_SIZE: z.string().default("64"),
+  PROD_OTEL_LOG_SCHEDULED_DELAY_MILLIS: z.string().default("200"),
+  PROD_OTEL_LOG_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"),
+  PROD_OTEL_LOG_MAX_QUEUE_SIZE: z.string().default("512"),
+
   RUNTIME_WAIT_THRESHOLD_IN_MS: z.coerce.number().int().default(30000),
 
   // Internal OTEL environment variables
@@ -124,8 +149,9 @@ const EnvironmentSchema = z.object({
   INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE: z.string().optional(),
   INTERNAL_OTEL_TRACE_LOGGING_ENABLED: z.string().default("1"),
   // this means 1/20 traces or 5% of traces will be sampled (sampled = recorded)
-  INTERNAL_OTEL_TRACE_SAMPING_RATE: z.string().default("20"),
+  INTERNAL_OTEL_TRACE_SAMPLING_RATE: z.string().default("20"),
   INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED: z.string().default("0"),
+  INTERNAL_OTEL_TRACE_DISABLED: z.string().default("0"),
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/hooks/useEnvironments.ts

@@ -10,20 +10,3 @@ export function useEnvironments(matches?: UIMatch[]) {
 
   return project.environments;
 }
-
-export function useDevEnvironment(matches?: UIMatch[]) {
-  const user = useUser();
-  const environments = useEnvironments(matches);
-  if (!environments) return;
-
-  return environments.find(
-    (environment) => environment.type === "DEVELOPMENT" && environment.userId === user.id
-  );
-}
-
-export function useProdEnvironment(matches?: UIMatch[]) {
-  const environments = useEnvironments(matches);
-  if (!environments) return;
-
-  return environments.find((environment) => environment.type === "PRODUCTION");
-}

apps/webapp/app/hooks/useReplaceLocation.ts

@@ -0,0 +1,32 @@
+import { useCallback, useState } from "react";
+import { useOptimisticLocation } from "./useOptimisticLocation";
+import type { Location } from "@remix-run/react";
+
+export function useReplaceLocation() {
+  const optimisticLocation = useOptimisticLocation();
+  const [location, setLocation] = useState(optimisticLocation);
+
+  const replaceLocation = useCallback((location: Location<any>) => {
+    const fullPath = location.pathname + location.search + location.hash;
+    //replace the URL in the browser
+    history.replaceState(null, "", fullPath);
+    //update the state (new object in case the same location ref was modified)
+    const newLocation = { ...location };
+    setLocation(newLocation);
+  }, []);
+
+  const replaceSearchParam = useCallback(
+    (key: string, value?: string) => {
+      const searchParams = new URLSearchParams(location.search);
+      if (value) {
+        searchParams.set(key, value);
+      } else {
+        searchParams.delete(key);
+      }
+      replaceLocation({ ...optimisticLocation, search: "?" + searchParams.toString() });
+    },
+    [optimisticLocation, replaceLocation]
+  );
+
+  return { location, replaceLocation, replaceSearchParam };
+}

apps/webapp/app/hooks/useThrottle.ts

@@ -1,14 +1,23 @@
-import { useRef } from "react";
+import { useEffect, useRef } from "react";
 
-export function useThrottle<T extends (...args: any[]) => any>(fn: T, delay: number) {
+export function useThrottle(fn: (...args: any[]) => void, duration: number) {
   const timeout = useRef<ReturnType<typeof setTimeout>>();
 
-  return (...args: Parameters<T>) => {
-    if (!timeout.current) {
-      fn(...args);
-      timeout.current = setTimeout(() => {
-        timeout.current = undefined;
-      }, delay);
+  // Clean up when the component is unmounted
+  useEffect(() => {
+    return () => {
+      if (timeout.current) clearTimeout(timeout.current);
+    };
+  }, []);
+
+  return (...args: Parameters<typeof fn>) => {
+    if (timeout.current) {
+      clearTimeout(timeout.current);
     }
+
+    timeout.current = setTimeout(() => {
+      fn(...args);
+      timeout.current = undefined;
+    }, duration);
   };
 }