Merge remote-tracking branch 'origin/main' into feat/compute-workload-manager

Add optional COMPUTE_TRACE_OTLP_ENDPOINT env var to override the OTLP
endpoint for supervisor-emitted spans (defaults to TRIGGER_API_URL/otel).
Useful for sending spans to an OTel collector instead of the webapp.

Also demotes remaining per-run logs in compute workload manager and
workload server to debug/verbose.

.changeset/ai-prompt-management.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Define and manage AI prompts with `prompts.define()`. Create typesafe prompt templates with variables, resolve them at runtime, and manage versions and overrides from the dashboard without redeploying.

.changeset/fix-dev-build-dir-leak.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix dev CLI leaking build directories on rebuild, causing disk space accumulation. Deprecated workers are now pruned (capped at 2 retained) when no active runs reference them. The watchdog process also cleans up `.trigger/tmp/` when the dev CLI is killed ungracefully (e.g. SIGKILL from pnpm).

.changeset/fix-list-deploys-nullable.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Fix `list_deploys` MCP tool failing when deployments have null `runtime` or `runtimeVersion` fields.

.changeset/fix-local-build-load.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix `--load` flag being silently ignored on local/self-hosted builds.

.changeset/llm-metadata-run-tags.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Propagate run tags to span attributes so they can be extracted server-side for LLM cost attribution metadata.

.changeset/mcp-get-span-details.md

@@ -0,0 +1,11 @@
+---
+"@trigger.dev/core": patch
+"trigger.dev": patch
+---
+
+Add `get_span_details` MCP tool for inspecting individual spans within a run trace.
+
+- New `get_span_details` tool returns full span attributes, timing, events, and AI enrichment (model, tokens, cost, speed)
+- Span IDs now shown in `get_run_details` trace output for easy discovery
+- New API endpoint `GET /api/v1/runs/:runId/spans/:spanId`
+- New `retrieveSpan()` method on the API client

.changeset/mcp-query-tools.md

@@ -0,0 +1,42 @@
+---
+"@trigger.dev/core": patch
+"trigger.dev": patch
+---
+
+MCP server improvements: new tools, bug fixes, and new flags.
+
+**New tools:**
+- `get_query_schema` — discover available TRQL tables and columns
+- `query` — execute TRQL queries against your data
+- `list_dashboards` — list built-in dashboards and their widgets
+- `run_dashboard_query` — execute a single dashboard widget query
+- `whoami` — show current profile, user, and API URL
+- `list_profiles` — list all configured CLI profiles
+- `switch_profile` — switch active profile for the MCP session
+- `start_dev_server` — start `trigger dev` in the background and stream output
+- `stop_dev_server` — stop the running dev server
+- `dev_server_status` — check dev server status and view recent logs
+
+**New API endpoints:**
+- `GET /api/v1/query/schema` — query table schema discovery
+- `GET /api/v1/query/dashboards` — list built-in dashboards
+
+**New features:**
+- `--readonly` flag hides write tools (`deploy`, `trigger_task`, `cancel_run`) so the AI cannot make changes
+- `read:query` JWT scope for query endpoint authorization
+- `get_run_details` trace output is now paginated with cursor support
+- MCP tool annotations (`readOnlyHint`, `destructiveHint`) for all tools
+
+**Bug fixes:**
+- Fixed `search_docs` tool failing due to renamed upstream Mintlify tool (`SearchTriggerDev` → `search_trigger_dev`)
+- Fixed `list_deploys` failing when deployments have null `runtime`/`runtimeVersion` fields (#3139)
+- Fixed `list_preview_branches` crashing due to incorrect response shape access
+- Fixed `metrics` table column documented as `value` instead of `metric_value` in query docs
+- Fixed dev CLI leaking build directories on rebuild — deprecated workers now clean up their build dirs when their last run completes
+
+**Context optimizations:**
+- `get_query_schema` now requires a table name and returns only one table's schema (was returning all tables)
+- `get_current_worker` no longer inlines payload schemas; use new `get_task_schema` tool instead
+- Query results formatted as text tables instead of JSON (~50% fewer tokens)
+- `cancel_run`, `list_deploys`, `list_preview_branches` formatted as text instead of raw JSON
+- Schema and dashboard API responses cached to avoid redundant fetches

.changeset/tame-oranges-change.md

@@ -0,0 +1,8 @@
+---
+"@trigger.dev/redis-worker": patch
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Adapted the CLI API client to propagate the trigger source via http headers.

.claude/rules/database-safety.md

@@ -0,0 +1,13 @@
+---
+paths:
+  - "internal-packages/database/**"
+---
+
+# Database Migration Safety
+
+- When adding indexes to **existing tables**, use `CREATE INDEX CONCURRENTLY IF NOT EXISTS` to avoid table locks. These must be in their own separate migration file (one index per file).
+- Indexes on **newly created tables** (same migration as `CREATE TABLE`) do not need CONCURRENTLY.
+- When indexing a **new column on an existing table**, split into two migrations: first `ADD COLUMN IF NOT EXISTS`, then `CREATE INDEX CONCURRENTLY IF NOT EXISTS` in a separate file.
+- After generating a migration with Prisma, remove extraneous lines for: `_BackgroundWorkerToBackgroundWorkerFile`, `_BackgroundWorkerToTaskQueue`, `_TaskRunToTaskRunTag`, `_WaitpointRunConnections`, `_completedWaitpoints`, `SecretStore_key_idx`, and unrelated TaskRun indexes.
+- Never drop columns or tables without explicit approval.
+- New code should target `RunEngineVersion.V2` only.

.claude/rules/docs-writing.md

@@ -0,0 +1,14 @@
+---
+paths:
+  - "docs/**"
+---
+
+# Documentation Writing Rules
+
+- Use Mintlify MDX format. Frontmatter: `title`, `description`, `sidebarTitle` (optional).
+- After creating a new page, add it to `docs.json` navigation under the correct group.
+- Use Mintlify components: `<Note>`, `<Warning>`, `<Info>`, `<Tip>`, `<CodeGroup>`, `<Expandable>`, `<Steps>`/`<Step>`.
+- Code examples should be complete and runnable where possible.
+- Always import from `@trigger.dev/sdk`, never `@trigger.dev/sdk/v3`.
+- Keep paragraphs short. Use headers to break up content.
+- Link to related pages using relative paths (e.g., `[Tasks](/tasks/overview)`).

.claude/rules/legacy-v3-code.md

@@ -0,0 +1,33 @@
+---
+paths:
+  - "apps/webapp/app/v3/**"
+---
+
+# Legacy V1 Engine Code in `app/v3/`
+
+The `v3/` directory name is misleading - most code here is actively used by the current V2 engine. Only the specific files below are legacy V1-only code.
+
+## V1-Only Files - Never Modify
+
+- `marqs/` directory (entire MarQS queue system: sharedQueueConsumer, devQueueConsumer, fairDequeuingStrategy, devPubSub)
+- `legacyRunEngineWorker.server.ts` (V1 background job worker)
+- `services/triggerTaskV1.server.ts` (deprecated V1 task triggering)
+- `services/cancelTaskRunV1.server.ts` (deprecated V1 cancellation)
+- `authenticatedSocketConnection.server.ts` (V1 dev WebSocket using DevQueueConsumer)
+- `sharedSocketConnection.ts` (V1 shared queue socket using SharedQueueConsumer)
+
+## V1/V2 Branching Pattern
+
+Some services act as routers that branch on `RunEngineVersion`:
+- `services/cancelTaskRun.server.ts` - calls V1 service or `engine.cancelRun()` for V2
+- `services/batchTriggerV3.server.ts` - uses marqs for V1 path, run-engine for V2
+
+When editing these shared services, only modify V2 code paths.
+
+## V2 Modern Stack
+
+- **Run lifecycle**: `@internal/run-engine` (internal-packages/run-engine)
+- **Background jobs**: `@trigger.dev/redis-worker` (not graphile-worker/zodworker)
+- **Queue operations**: RunQueue inside run-engine (not MarQS)
+- **V2 engine singleton**: `runEngine.server.ts`, `runEngineHandlers.server.ts`
+- **V2 workers**: `commonWorker.server.ts`, `alertsWorker.server.ts`, `batchTriggerWorker.server.ts`

.claude/rules/sdk-packages.md

@@ -0,0 +1,12 @@
+---
+paths:
+  - "packages/**"
+---
+
+# Public Package Rules
+
+- Changes to `packages/` are **customer-facing**. Always add a changeset: `pnpm run changeset:add`
+- Default to **patch**. Get maintainer approval for minor. Never select major without explicit approval.
+- `@trigger.dev/core`: **Never import the root**. Always use subpath imports (e.g., `@trigger.dev/core/v3`).
+- Do NOT update `rules/` or `.claude/skills/trigger-dev-tasks/` unless explicitly asked. These are maintained in separate dedicated passes.
+- Test changes using `references/hello-world` reference project.

.claude/rules/server-apps.md

@@ -0,0 +1,23 @@
+---
+paths:
+  - "apps/**"
+---
+
+# Server App Changes
+
+When modifying server apps (webapp, supervisor, coordinator, etc.) with **no package changes**, add a `.server-changes/` file instead of a changeset:
+
+```bash
+cat > .server-changes/descriptive-name.md << 'EOF'
+---
+area: webapp
+type: fix
+---
+
+Brief description of what changed and why.
+EOF
+```
+
+- **area**: `webapp` | `supervisor` | `coordinator` | `kubernetes-provider` | `docker-provider`
+- **type**: `feature` | `fix` | `improvement` | `breaking`
+- If the PR also touches `packages/`, just the changeset is sufficient (no `.server-changes/` needed).

.claude/rules/span-timeline-events.md

@@ -0,0 +1,72 @@
+# Span Timeline Events
+
+The trace view's right panel shows a timeline of events for the selected span. These are OTel span events rendered by `app/utils/timelineSpanEvents.ts` and the `SpanTimeline` component.
+
+## How They Work
+
+1. **Span events** in OTel are attached to a parent span. In ClickHouse, they're stored as separate rows with `kind: "SPAN_EVENT"` sharing the parent span's `span_id`. The `#mergeRecordsIntoSpanDetail` method reassembles them into the span's `events` array at query time.
+2. The timeline only renders events whose `name` starts with `trigger.dev/` - all others are silently filtered out.
+3. The **display name** comes from `properties.event` (not the span event name), mapped through `getFriendlyNameForEvent()`.
+4. Events are shown on the **span they belong to** - events on one span don't appear in another span's timeline.
+
+## ClickHouse Storage Constraint
+
+When events are written to ClickHouse, `spanEventsToTaskEventV1Input()` filters out events whose `start_time` is not greater than the parent span's `startTime`. Events at or before the span start are silently dropped. This means span events must have timestamps strictly after the span's own `startTimeUnixNano`.
+
+## Timeline Rendering (SpanTimeline component)
+
+The `SpanTimeline` component in `app/components/run/RunTimeline.tsx` renders:
+
+1. **Events** (thin 1px line with hollow dots) - all events from `createTimelineSpanEventsFromSpanEvents()`
+2. **"Started"** marker (thick cap) - at the span's `startTime`
+3. **Duration bar** (thick 7px line) - from "Started" to "Finished"
+4. **"Finished"** marker (thick cap) - at `startTime + duration`
+
+The thin line before "Started" only appears when there are events with timestamps between the span start and the first child span. For the Attempt span this works well (Dequeued → Pod scheduled → Launched → etc. all happen before execution starts). Events all get `lineVariant: "light"` (thin) while the execution bar gets `variant: "normal"` (thick).
+
+## Trace View Sort Order
+
+Sibling spans (same parent) are sorted by `start_time ASC` from the ClickHouse query. The `createTreeFromFlatItems` function preserves this order. Event timestamps don't affect sort order - only the span's own `start_time`.
+
+## Event Structure
+
+```typescript
+// OTel span event format
+{
+  name: "trigger.dev/run",        // Must start with "trigger.dev/" to render
+  timeUnixNano: "1711200000000000000",
+  attributes: [
+    { key: "event", value: { stringValue: "dequeue" } },  // The actual event type
+    { key: "duration", value: { intValue: 150 } },         // Optional: duration in ms
+  ]
+}
+```
+
+## Admin-Only Events
+
+`getAdminOnlyForEvent()` controls visibility. Events default to **admin-only** (`true`).
+
+| Event | Admin-only | Friendly name |
+|-------|-----------|---------------|
+| `dequeue` | No | Dequeued |
+| `fork` | No | Launched |
+| `import` | No (if no fork event) | Importing task file |
+| `create_attempt` | Yes | Attempt created |
+| `lazy_payload` | Yes | Lazy attempt initialized |
+| `pod_scheduled` | Yes | Pod scheduled |
+| (default) | Yes | (raw event name) |
+
+## Adding New Timeline Events
+
+1. Add OTLP span event with `name: "trigger.dev/<scope>"` and `properties.event: "<type>"`
+2. Event timestamp must be strictly after the parent span's `startTimeUnixNano` (ClickHouse drops earlier events)
+3. Add friendly name in `getFriendlyNameForEvent()` in `app/utils/timelineSpanEvents.ts`
+4. Set admin visibility in `getAdminOnlyForEvent()`
+5. Optionally add help text in `getHelpTextForEvent()`
+
+## Key Files
+
+- `app/utils/timelineSpanEvents.ts` - filtering, naming, admin logic
+- `app/components/run/RunTimeline.tsx` - `SpanTimeline` component (thin line + thick bar rendering)
+- `app/presenters/v3/SpanPresenter.server.ts` - loads span data including events
+- `app/v3/eventRepository/clickhouseEventRepository.server.ts` - `spanEventsToTaskEventV1Input()` (storage filter), `#mergeRecordsIntoSpanDetail` (reassembly)

.claude/skills/trigger-dev-tasks/SKILL.md

@@ -0,0 +1,200 @@
+---
+name: trigger-dev-tasks
+description: Use this skill when writing, designing, or optimizing Trigger.dev background tasks and workflows. This includes creating reliable async tasks, implementing AI workflows, setting up scheduled jobs, structuring complex task hierarchies with subtasks, configuring build extensions for tools like ffmpeg or Puppeteer/Playwright, and handling task schemas with Zod validation.
+allowed-tools: Read, Write, Edit, Glob, Grep, Bash
+---
+
+# Trigger.dev Task Expert
+
+You are an expert Trigger.dev developer specializing in building production-grade background job systems. Tasks deployed to Trigger.dev run in Node.js 21+ and use the `@trigger.dev/sdk` package.
+
+## Critical Rules
+
+1. **Always use `@trigger.dev/sdk`** - Never use `@trigger.dev/sdk/v3` or deprecated `client.defineJob` pattern
+2. **Never use `node-fetch`** - Use the built-in `fetch` function
+3. **Export all tasks** - Every task must be exported, including subtasks
+4. **Never wrap wait/trigger calls in Promise.all** - `triggerAndWait`, `batchTriggerAndWait`, and `wait.*` calls cannot be wrapped in `Promise.all` or `Promise.allSettled`
+
+## Basic Task Pattern
+
+```ts
+import { task } from "@trigger.dev/sdk";
+
+export const processData = task({
+  id: "process-data",
+  retry: {
+    maxAttempts: 10,
+    factor: 1.8,
+    minTimeoutInMs: 500,
+    maxTimeoutInMs: 30_000,
+  },
+  run: async (payload: { userId: string; data: any[] }) => {
+    console.log(`Processing ${payload.data.length} items`);
+    return { processed: payload.data.length };
+  },
+});
+```
+
+## Schema Task (with validation)
+
+```ts
+import { schemaTask } from "@trigger.dev/sdk";
+import { z } from "zod";
+
+export const validatedTask = schemaTask({
+  id: "validated-task",
+  schema: z.object({
+    name: z.string(),
+    email: z.string().email(),
+  }),
+  run: async (payload) => {
+    // Payload is automatically validated and typed
+    return { message: `Hello ${payload.name}` };
+  },
+});
+```
+
+## Triggering Tasks
+
+### From Backend Code (type-only import to prevent dependency leakage)
+
+```ts
+import { tasks } from "@trigger.dev/sdk";
+import type { processData } from "./trigger/tasks";
+
+const handle = await tasks.trigger<typeof processData>("process-data", {
+  userId: "123",
+  data: [{ id: 1 }],
+});
+```
+
+### From Inside Tasks
+
+```ts
+export const parentTask = task({
+  id: "parent-task",
+  run: async (payload) => {
+    // Trigger and wait - returns Result object, NOT direct output
+    const result = await childTask.triggerAndWait({ data: "value" });
+    if (result.ok) {
+      console.log("Output:", result.output);
+    } else {
+      console.error("Failed:", result.error);
+    }
+
+    // Or unwrap directly (throws on error)
+    const output = await childTask.triggerAndWait({ data: "value" }).unwrap();
+  },
+});
+```
+
+## Idempotency (Critical for Retries)
+
+Always use idempotency keys when triggering tasks from inside other tasks:
+
+```ts
+import { idempotencyKeys } from "@trigger.dev/sdk";
+
+export const paymentTask = task({
+  id: "process-payment",
+  run: async (payload: { orderId: string }) => {
+    // Scoped to current run - survives retries
+    const key = await idempotencyKeys.create(`payment-${payload.orderId}`);
+
+    await chargeCustomer.trigger(payload, {
+      idempotencyKey: key,
+      idempotencyKeyTTL: "24h",
+    });
+  },
+});
+```
+
+## Trigger Options
+
+```ts
+await myTask.trigger(payload, {
+  delay: "1h",           // Delay execution
+  ttl: "10m",            // Cancel if not started within TTL
+  idempotencyKey: key,
+  queue: "my-queue",
+  machine: "large-1x",   // micro, small-1x, small-2x, medium-1x, medium-2x, large-1x, large-2x
+  maxAttempts: 3,
+  tags: ["user_123"],    // Max 10 tags
+  debounce: {            // Consolidate rapid triggers
+    key: "unique-key",
+    delay: "5s",
+    mode: "trailing",    // "leading" (default) or "trailing"
+  },
+});
+```
+
+## Debouncing
+
+Consolidate multiple triggers into a single execution:
+
+```ts
+// Rapid triggers with same key = single execution
+await myTask.trigger({ userId: "123" }, {
+  debounce: {
+    key: "user-123-update",
+    delay: "5s",
+  },
+});
+
+// Trailing mode: use payload from LAST trigger
+await myTask.trigger({ data: "latest" }, {
+  debounce: {
+    key: "my-key",
+    delay: "10s",
+    mode: "trailing",
+  },
+});
+```
+
+Use cases: user activity updates, webhook deduplication, search indexing, notification batching.
+
+## Batch Triggering
+
+Up to 1,000 items per batch, 3MB per payload:
+
+```ts
+const results = await myTask.batchTriggerAndWait([
+  { payload: { userId: "1" } },
+  { payload: { userId: "2" } },
+]);
+
+for (const result of results) {
+  if (result.ok) console.log(result.output);
+}
+```
+
+## Machine Presets
+
+| Preset      | vCPU | Memory |
+|-------------|------|--------|
+| micro       | 0.25 | 0.25GB |
+| small-1x    | 0.5  | 0.5GB  |
+| small-2x    | 1    | 1GB    |
+| medium-1x   | 1    | 2GB    |
+| medium-2x   | 2    | 4GB    |
+| large-1x    | 4    | 8GB    |
+| large-2x    | 8    | 16GB   |
+
+## Design Principles
+
+1. **Break complex workflows into subtasks** that can be independently retried and made idempotent
+2. **Don't over-complicate** - Sometimes `Promise.allSettled` inside a single task is better than many subtasks (each task has dedicated process and is charged by millisecond)
+3. **Always configure retries** - Set appropriate `maxAttempts` based on the operation
+4. **Use idempotency keys** - Especially for payment/critical operations
+5. **Group related subtasks** - Keep subtasks only used by one parent in the same file, don't export them
+6. **Use logger** - Log at key execution points with `logger.info()`, `logger.error()`, etc.
+
+## Reference Documentation
+
+For detailed documentation on specific topics, read these files:
+
+- `basic-tasks.md` - Task basics, triggering, waits
+- `advanced-tasks.md` - Tags, queues, concurrency, metadata, error handling
+- `scheduled-tasks.md` - Cron schedules, declarative and imperative
+- `realtime.md` - Real-time subscriptions, streams, React hooks
+- `config.md` - trigger.config.ts, build extensions (Prisma, Playwright, FFmpeg, etc.)

.claude/skills/trigger-dev-tasks/advanced-tasks.md

@@ -0,0 +1,485 @@
+# Trigger.dev Advanced Tasks (v4)
+
+**Advanced patterns and features for writing tasks**
+
+## Tags & Organization
+
+```ts
+import { task, tags } from "@trigger.dev/sdk";
+
+export const processUser = task({
+  id: "process-user",
+  run: async (payload: { userId: string; orgId: string }, { ctx }) => {
+    // Add tags during execution
+    await tags.add(`user_${payload.userId}`);
+    await tags.add(`org_${payload.orgId}`);
+
+    return { processed: true };
+  },
+});
+
+// Trigger with tags
+await processUser.trigger(
+  { userId: "123", orgId: "abc" },
+  { tags: ["priority", "user_123", "org_abc"] } // Max 10 tags per run
+);
+
+// Subscribe to tagged runs
+for await (const run of runs.subscribeToRunsWithTag("user_123")) {
+  console.log(`User task ${run.id}: ${run.status}`);
+}
+```
+
+**Tag Best Practices:**
+
+- Use prefixes: `user_123`, `org_abc`, `video:456`
+- Max 10 tags per run, 1-64 characters each
+- Tags don't propagate to child tasks automatically
+
+## Batch Triggering v2
+
+Enhanced batch triggering with larger payloads and streaming ingestion.
+
+### Limits
+
+- **Maximum batch size**: 1,000 items (increased from 500)
+- **Payload per item**: 3MB each (increased from 1MB combined)
+- Payloads > 512KB automatically offload to object storage
+
+### Rate Limiting (per environment)
+
+| Tier | Bucket Size | Refill Rate |
+|------|-------------|-------------|
+| Free | 1,200 runs | 100 runs/10 sec |
+| Hobby | 5,000 runs | 500 runs/5 sec |
+| Pro | 5,000 runs | 500 runs/5 sec |
+
+### Concurrent Batch Processing
+
+| Tier | Concurrent Batches |
+|------|-------------------|
+| Free | 1 |
+| Hobby | 10 |
+| Pro | 10 |
+
+### Usage
+
+```ts
+import { myTask } from "./trigger/myTask";
+
+// Basic batch trigger (up to 1,000 items)
+const runs = await myTask.batchTrigger([
+  { payload: { userId: "user-1" } },
+  { payload: { userId: "user-2" } },
+  { payload: { userId: "user-3" } },
+]);
+
+// Batch trigger with wait
+const results = await myTask.batchTriggerAndWait([
+  { payload: { userId: "user-1" } },
+  { payload: { userId: "user-2" } },
+]);
+
+for (const result of results) {
+  if (result.ok) {
+    console.log("Result:", result.output);
+  }
+}
+
+// With per-item options
+const batchHandle = await myTask.batchTrigger([
+  {
+    payload: { userId: "123" },
+    options: {
+      idempotencyKey: "user-123-batch",
+      tags: ["priority"],
+    },
+  },
+  {
+    payload: { userId: "456" },
+    options: {
+      idempotencyKey: "user-456-batch",
+    },
+  },
+]);
+```
+
+## Debouncing
+
+Consolidate multiple triggers into a single execution by debouncing task runs with a unique key and delay window.
+
+### Use Cases
+
+- **User activity updates**: Batch rapid user actions into a single run
+- **Webhook deduplication**: Handle webhook bursts without redundant processing
+- **Search indexing**: Combine document updates instead of processing individually
+- **Notification batching**: Group notifications to prevent user spam
+
+### Basic Usage
+
+```ts
+await myTask.trigger(
+  { userId: "123" },
+  {
+    debounce: {
+      key: "user-123-update",  // Unique identifier for debounce group
+      delay: "5s",              // Wait duration ("5s", "1m", or milliseconds)
+    },
+  }
+);
+```
+
+### Execution Modes
+
+**Leading Mode** (default): Uses payload/options from the first trigger; subsequent triggers only reschedule execution time.
+
+```ts
+// First trigger sets the payload
+await myTask.trigger({ action: "first" }, {
+  debounce: { key: "my-key", delay: "10s" }
+});
+
+// Second trigger only reschedules - payload remains "first"
+await myTask.trigger({ action: "second" }, {
+  debounce: { key: "my-key", delay: "10s" }
+});
+// Task executes with { action: "first" }
+```
+
+**Trailing Mode**: Uses payload/options from the most recent trigger.
+
+```ts
+await myTask.trigger(
+  { data: "latest-value" },
+  {
+    debounce: {
+      key: "trailing-example",
+      delay: "10s",
+      mode: "trailing",
+    },
+  }
+);
+```
+
+In trailing mode, these options update with each trigger:
+- `payload` — task input data
+- `metadata` — run metadata
+- `tags` — run tags (replaces existing)
+- `maxAttempts` — retry attempts
+- `maxDuration` — maximum compute time
+- `machine` — machine preset
+
+### Important Notes
+
+- Idempotency keys take precedence over debounce settings
+- Compatible with `triggerAndWait()` — parent runs block correctly on debounced execution
+- Debounce key is scoped to the task
+
+## Concurrency & Queues
+
+```ts
+import { task, queue } from "@trigger.dev/sdk";
+
+// Shared queue for related tasks
+const emailQueue = queue({
+  name: "email-processing",
+  concurrencyLimit: 5, // Max 5 emails processing simultaneously
+});
+
+// Task-level concurrency
+export const oneAtATime = task({
+  id: "sequential-task",
+  queue: { concurrencyLimit: 1 }, // Process one at a time
+  run: async (payload) => {
+    // Critical section - only one instance runs
+  },
+});
+
+// Per-user concurrency
+export const processUserData = task({
+  id: "process-user-data",
+  run: async (payload: { userId: string }) => {
+    // Override queue with user-specific concurrency
+    await childTask.trigger(payload, {
+      queue: {
+        name: `user-${payload.userId}`,
+        concurrencyLimit: 2,
+      },
+    });
+  },
+});
+
+export const emailTask = task({
+  id: "send-email",
+  queue: emailQueue, // Use shared queue
+  run: async (payload: { to: string }) => {
+    // Send email logic
+  },
+});
+```
+
+## Error Handling & Retries
+
+```ts
+import { task, retry, AbortTaskRunError } from "@trigger.dev/sdk";
+
+export const resilientTask = task({
+  id: "resilient-task",
+  retry: {
+    maxAttempts: 10,
+    factor: 1.8, // Exponential backoff multiplier
+    minTimeoutInMs: 500,
+    maxTimeoutInMs: 30_000,
+    randomize: false,
+  },
+  catchError: async ({ error, ctx }) => {
+    // Custom error handling
+    if (error.code === "FATAL_ERROR") {
+      throw new AbortTaskRunError("Cannot retry this error");
+    }
+
+    // Log error details
+    console.error(`Task ${ctx.task.id} failed:`, error);
+
+    // Allow retry by returning nothing
+    return { retryAt: new Date(Date.now() + 60000) }; // Retry in 1 minute
+  },
+  run: async (payload) => {
+    // Retry specific operations
+    const result = await retry.onThrow(
+      async () => {
+        return await unstableApiCall(payload);
+      },
+      { maxAttempts: 3 }
+    );
+
+    // Conditional HTTP retries
+    const response = await retry.fetch("https://api.example.com", {
+      retry: {
+        maxAttempts: 5,
+        condition: (response, error) => {
+          return response?.status === 429 || response?.status >= 500;
+        },
+      },
+    });
+
+    return result;
+  },
+});
+```
+
+## Machines & Performance
+
+```ts
+export const heavyTask = task({
+  id: "heavy-computation",
+  machine: { preset: "large-2x" }, // 8 vCPU, 16 GB RAM
+  maxDuration: 1800, // 30 minutes timeout
+  run: async (payload, { ctx }) => {
+    // Resource-intensive computation
+    if (ctx.machine.preset === "large-2x") {
+      // Use all available cores
+      return await parallelProcessing(payload);
+    }
+
+    return await standardProcessing(payload);
+  },
+});
+
+// Override machine when triggering
+await heavyTask.trigger(payload, {
+  machine: { preset: "medium-1x" }, // Override for this run
+});
+```
+
+**Machine Presets:**
+
+- `micro`: 0.25 vCPU, 0.25 GB RAM
+- `small-1x`: 0.5 vCPU, 0.5 GB RAM (default)
+- `small-2x`: 1 vCPU, 1 GB RAM
+- `medium-1x`: 1 vCPU, 2 GB RAM
+- `medium-2x`: 2 vCPU, 4 GB RAM
+- `large-1x`: 4 vCPU, 8 GB RAM
+- `large-2x`: 8 vCPU, 16 GB RAM
+
+## Idempotency
+
+```ts
+import { task, idempotencyKeys } from "@trigger.dev/sdk";
+
+export const paymentTask = task({
+  id: "process-payment",
+  retry: {
+    maxAttempts: 3,
+  },
+  run: async (payload: { orderId: string; amount: number }) => {
+    // Automatically scoped to this task run, so if the task is retried, the idempotency key will be the same
+    const idempotencyKey = await idempotencyKeys.create(`payment-${payload.orderId}`);
+
+    // Ensure payment is processed only once
+    await chargeCustomer.trigger(payload, {
+      idempotencyKey,
+      idempotencyKeyTTL: "24h", // Key expires in 24 hours
+    });
+  },
+});
+
+// Payload-based idempotency
+import { createHash } from "node:crypto";
+
+function createPayloadHash(payload: any): string {
+  const hash = createHash("sha256");
+  hash.update(JSON.stringify(payload));
+  return hash.digest("hex");
+}
+
+export const deduplicatedTask = task({
+  id: "deduplicated-task",
+  run: async (payload) => {
+    const payloadHash = createPayloadHash(payload);
+    const idempotencyKey = await idempotencyKeys.create(payloadHash);
+
+    await processData.trigger(payload, { idempotencyKey });
+  },
+});
+```
+
+## Metadata & Progress Tracking
+
+```ts
+import { task, metadata } from "@trigger.dev/sdk";
+
+export const batchProcessor = task({
+  id: "batch-processor",
+  run: async (payload: { items: any[] }, { ctx }) => {
+    const totalItems = payload.items.length;
+
+    // Initialize progress metadata
+    metadata
+      .set("progress", 0)
+      .set("totalItems", totalItems)
+      .set("processedItems", 0)
+      .set("status", "starting");
+
+    const results = [];
+
+    for (let i = 0; i < payload.items.length; i++) {
+      const item = payload.items[i];
+
+      // Process item
+      const result = await processItem(item);
+      results.push(result);
+
+      // Update progress
+      const progress = ((i + 1) / totalItems) * 100;
+      metadata
+        .set("progress", progress)
+        .increment("processedItems", 1)
+        .append("logs", `Processed item ${i + 1}/${totalItems}`)
+        .set("currentItem", item.id);
+    }
+
+    // Final status
+    metadata.set("status", "completed");
+
+    return { results, totalProcessed: results.length };
+  },
+});
+
+// Update parent metadata from child task
+export const childTask = task({
+  id: "child-task",
+  run: async (payload, { ctx }) => {
+    // Update parent task metadata
+    metadata.parent.set("childStatus", "processing");
+    metadata.root.increment("childrenCompleted", 1);
+
+    return { processed: true };
+  },
+});
+```
+
+## Logging & Tracing
+
+```ts
+import { task, logger } from "@trigger.dev/sdk";
+
+export const tracedTask = task({
+  id: "traced-task",
+  run: async (payload, { ctx }) => {
+    logger.info("Task started", { userId: payload.userId });
+
+    // Custom trace with attributes
+    const user = await logger.trace(
+      "fetch-user",
+      async (span) => {
+        span.setAttribute("user.id", payload.userId);
+        span.setAttribute("operation", "database-fetch");
+
+        const userData = await database.findUser(payload.userId);
+        span.setAttribute("user.found", !!userData);
+
+        return userData;
+      },
+      { userId: payload.userId }
+    );
+
+    logger.debug("User fetched", { user: user.id });
+
+    try {
+      const result = await processUser(user);
+      logger.info("Processing completed", { result });
+      return result;
+    } catch (error) {
+      logger.error("Processing failed", {
+        error: error.message,
+        userId: payload.userId,
+      });
+      throw error;
+    }
+  },
+});
+```
+
+## Hidden Tasks
+
+```ts
+// Hidden task - not exported, only used internally
+const internalProcessor = task({
+  id: "internal-processor",
+  run: async (payload: { data: string }) => {
+    return { processed: payload.data.toUpperCase() };
+  },
+});
+
+// Public task that uses hidden task
+export const publicWorkflow = task({
+  id: "public-workflow",
+  run: async (payload: { input: string }) => {
+    // Use hidden task internally
+    const result = await internalProcessor.triggerAndWait({
+      data: payload.input,
+    });
+
+    if (result.ok) {
+      return { output: result.output.processed };
+    }
+
+    throw new Error("Internal processing failed");
+  },
+});
+```
+
+## Best Practices
+
+- **Concurrency**: Use queues to prevent overwhelming external services
+- **Retries**: Configure exponential backoff for transient failures
+- **Idempotency**: Always use for payment/critical operations
+- **Metadata**: Track progress for long-running tasks
+- **Machines**: Match machine size to computational requirements
+- **Tags**: Use consistent naming patterns for filtering
+- **Debouncing**: Use for user activity, webhooks, and notification batching
+- **Batch triggering**: Use for bulk operations up to 1,000 items
+- **Error Handling**: Distinguish between retryable and fatal errors
+
+Design tasks to be stateless, idempotent, and resilient to failures. Use metadata for state tracking and queues for resource management.

.claude/skills/trigger-dev-tasks/basic-tasks.md

@@ -0,0 +1,199 @@
+# Trigger.dev Basic Tasks (v4)
+
+**MUST use `@trigger.dev/sdk`, NEVER `client.defineJob`**
+
+## Basic Task
+
+```ts
+import { task } from "@trigger.dev/sdk";
+
+export const processData = task({
+  id: "process-data",
+  retry: {
+    maxAttempts: 10,
+    factor: 1.8,
+    minTimeoutInMs: 500,
+    maxTimeoutInMs: 30_000,
+    randomize: false,
+  },
+  run: async (payload: { userId: string; data: any[] }) => {
+    // Task logic - runs for long time, no timeouts
+    console.log(`Processing ${payload.data.length} items for user ${payload.userId}`);
+    return { processed: payload.data.length };
+  },
+});
+```
+
+## Schema Task (with validation)
+
+```ts
+import { schemaTask } from "@trigger.dev/sdk";
+import { z } from "zod";
+
+export const validatedTask = schemaTask({
+  id: "validated-task",
+  schema: z.object({
+    name: z.string(),
+    age: z.number(),
+    email: z.string().email(),
+  }),
+  run: async (payload) => {
+    // Payload is automatically validated and typed
+    return { message: `Hello ${payload.name}, age ${payload.age}` };
+  },
+});
+```
+
+## Triggering Tasks
+
+### From Backend Code
+
+```ts
+import { tasks } from "@trigger.dev/sdk";
+import type { processData } from "./trigger/tasks";
+
+// Single trigger
+const handle = await tasks.trigger<typeof processData>("process-data", {
+  userId: "123",
+  data: [{ id: 1 }, { id: 2 }],
+});
+
+// Batch trigger (up to 1,000 items, 3MB per payload)
+const batchHandle = await tasks.batchTrigger<typeof processData>("process-data", [
+  { payload: { userId: "123", data: [{ id: 1 }] } },
+  { payload: { userId: "456", data: [{ id: 2 }] } },
+]);
+```
+
+### Debounced Triggering
+
+Consolidate multiple triggers into a single execution:
+
+```ts
+// Multiple rapid triggers with same key = single execution
+await myTask.trigger(
+  { userId: "123" },
+  {
+    debounce: {
+      key: "user-123-update",  // Unique key for debounce group
+      delay: "5s",              // Wait before executing
+    },
+  }
+);
+
+// Trailing mode: use payload from LAST trigger
+await myTask.trigger(
+  { data: "latest-value" },
+  {
+    debounce: {
+      key: "trailing-example",
+      delay: "10s",
+      mode: "trailing",  // Default is "leading" (first payload)
+    },
+  }
+);
+```
+
+**Debounce modes:**
+- `leading` (default): Uses payload from first trigger, subsequent triggers only reschedule
+- `trailing`: Uses payload from most recent trigger
+
+### From Inside Tasks (with Result handling)
+
+```ts
+export const parentTask = task({
+  id: "parent-task",
+  run: async (payload) => {
+    // Trigger and continue
+    const handle = await childTask.trigger({ data: "value" });
+
+    // Trigger and wait - returns Result object, NOT task output
+    const result = await childTask.triggerAndWait({ data: "value" });
+    if (result.ok) {
+      console.log("Task output:", result.output); // Actual task return value
+    } else {
+      console.error("Task failed:", result.error);
+    }
+
+    // Quick unwrap (throws on error)
+    const output = await childTask.triggerAndWait({ data: "value" }).unwrap();
+
+    // Batch trigger and wait
+    const results = await childTask.batchTriggerAndWait([
+      { payload: { data: "item1" } },
+      { payload: { data: "item2" } },
+    ]);
+
+    for (const run of results) {
+      if (run.ok) {
+        console.log("Success:", run.output);
+      } else {
+        console.log("Failed:", run.error);
+      }
+    }
+  },
+});
+
+export const childTask = task({
+  id: "child-task",
+  run: async (payload: { data: string }) => {
+    return { processed: payload.data };
+  },
+});
+```
+
+> Never wrap triggerAndWait or batchTriggerAndWait calls in a Promise.all or Promise.allSettled as this is not supported in Trigger.dev tasks.
+
+## Waits
+
+```ts
+import { task, wait } from "@trigger.dev/sdk";
+
+export const taskWithWaits = task({
+  id: "task-with-waits",
+  run: async (payload) => {
+    console.log("Starting task");
+
+    // Wait for specific duration
+    await wait.for({ seconds: 30 });
+    await wait.for({ minutes: 5 });
+    await wait.for({ hours: 1 });
+    await wait.for({ days: 1 });
+
+    // Wait until specific date
+    await wait.until({ date: new Date("2024-12-25") });
+
+    // Wait for token (from external system)
+    await wait.forToken({
+      token: "user-approval-token",
+      timeoutInSeconds: 3600, // 1 hour timeout
+    });
+
+    console.log("All waits completed");
+    return { status: "completed" };
+  },
+});
+```
+
+> Never wrap wait calls in a Promise.all or Promise.allSettled as this is not supported in Trigger.dev tasks.
+
+## Key Points
+
+- **Result vs Output**: `triggerAndWait()` returns a `Result` object with `ok`, `output`, `error` properties - NOT the direct task output
+- **Type safety**: Use `import type` for task references when triggering from backend
+- **Waits > 5 seconds**: Automatically checkpointed, don't count toward compute usage
+- **Debounce + idempotency**: Idempotency keys take precedence over debounce settings
+
+## NEVER Use (v2 deprecated)
+
+```ts
+// BREAKS APPLICATION
+client.defineJob({
+  id: "job-id",
+  run: async (payload, io) => {
+    /* ... */
+  },
+});
+```
+
+Use SDK (`@trigger.dev/sdk`), check `result.ok` before accessing `result.output`

.claude/skills/trigger-dev-tasks/config.md

@@ -0,0 +1,346 @@
+# Trigger.dev Configuration
+
+**Complete guide to configuring `trigger.config.ts` with build extensions**
+
+## Basic Configuration
+
+```ts
+import { defineConfig } from "@trigger.dev/sdk";
+
+export default defineConfig({
+  project: "<project-ref>", // Required: Your project reference
+  dirs: ["./trigger"], // Task directories
+  runtime: "node", // "node", "node-22", or "bun"
+  logLevel: "info", // "debug", "info", "warn", "error"
+
+  // Default retry settings
+  retries: {
+    enabledInDev: false,
+    default: {
+      maxAttempts: 3,
+      minTimeoutInMs: 1000,
+      maxTimeoutInMs: 10000,
+      factor: 2,
+      randomize: true,
+    },
+  },
+
+  // Build configuration
+  build: {
+    autoDetectExternal: true,
+    keepNames: true,
+    minify: false,
+    extensions: [], // Build extensions go here
+  },
+
+  // Global lifecycle hooks
+  onStartAttempt: async ({ payload, ctx }) => {
+    console.log("Global task start");
+  },
+  onSuccess: async ({ payload, output, ctx }) => {
+    console.log("Global task success");
+  },
+  onFailure: async ({ payload, error, ctx }) => {
+    console.log("Global task failure");
+  },
+});
+```
+
+## Build Extensions
+
+### Database & ORM
+
+#### Prisma
+
+```ts
+import { prismaExtension } from "@trigger.dev/build/extensions/prisma";
+
+extensions: [
+  prismaExtension({
+    schema: "prisma/schema.prisma",
+    version: "5.19.0", // Optional: specify version
+    migrate: true, // Run migrations during build
+    directUrlEnvVarName: "DIRECT_DATABASE_URL",
+    typedSql: true, // Enable TypedSQL support
+  }),
+];
+```
+
+#### TypeScript Decorators (for TypeORM)
+
+```ts
+import { emitDecoratorMetadata } from "@trigger.dev/build/extensions/typescript";
+
+extensions: [
+  emitDecoratorMetadata(), // Enables decorator metadata
+];
+```
+
+### Scripting Languages
+
+#### Python
+
+```ts
+import { pythonExtension } from "@trigger.dev/build/extensions/python";
+
+extensions: [
+  pythonExtension({
+    scripts: ["./python/**/*.py"], // Copy Python files
+    requirementsFile: "./requirements.txt", // Install packages
+    devPythonBinaryPath: ".venv/bin/python", // Dev mode binary
+  }),
+];
+
+// Usage in tasks
+const result = await python.runInline(`print("Hello, world!")`);
+const output = await python.runScript("./python/script.py", ["arg1"]);
+```
+
+### Browser Automation
+
+#### Playwright
+
+```ts
+import { playwright } from "@trigger.dev/build/extensions/playwright";
+
+extensions: [
+  playwright({
+    browsers: ["chromium", "firefox", "webkit"], // Default: ["chromium"]
+    headless: true, // Default: true
+  }),
+];
+```
+
+#### Puppeteer
+
+```ts
+import { puppeteer } from "@trigger.dev/build/extensions/puppeteer";
+
+extensions: [puppeteer()];
+
+// Environment variable needed:
+// PUPPETEER_EXECUTABLE_PATH: "/usr/bin/google-chrome-stable"
+```
+
+#### Lightpanda
+
+```ts
+import { lightpanda } from "@trigger.dev/build/extensions/lightpanda";
+
+extensions: [
+  lightpanda({
+    version: "latest", // or "nightly"
+    disableTelemetry: false,
+  }),
+];
+```
+
+### Media Processing
+
+#### FFmpeg
+
+```ts
+import { ffmpeg } from "@trigger.dev/build/extensions/core";
+
+extensions: [
+  ffmpeg({ version: "7" }), // Static build, or omit for Debian version
+];
+
+// Automatically sets FFMPEG_PATH and FFPROBE_PATH
+// Add fluent-ffmpeg to external packages if using
+```
+
+#### Audio Waveform
+
+```ts
+import { audioWaveform } from "@trigger.dev/build/extensions/audioWaveform";
+
+extensions: [
+  audioWaveform(), // Installs Audio Waveform 1.1.0
+];
+```
+
+### System & Package Management
+
+#### System Packages (apt-get)
+
+```ts
+import { aptGet } from "@trigger.dev/build/extensions/core";
+
+extensions: [
+  aptGet({
+    packages: ["ffmpeg", "imagemagick", "curl=7.68.0-1"], // Can specify versions
+  }),
+];
+```
+
+#### Additional NPM Packages
+
+Only use this for installing CLI tools, NOT packages you import in your code.
+
+```ts
+import { additionalPackages } from "@trigger.dev/build/extensions/core";
+
+extensions: [
+  additionalPackages({
+    packages: ["wrangler"], // CLI tools and specific versions
+  }),
+];
+```
+
+#### Additional Files
+
+```ts
+import { additionalFiles } from "@trigger.dev/build/extensions/core";
+
+extensions: [
+  additionalFiles({
+    files: ["wrangler.toml", "./assets/**", "./fonts/**"], // Glob patterns supported
+  }),
+];
+```
+
+### Environment & Build Tools
+
+#### Environment Variable Sync
+
+```ts
+import { syncEnvVars } from "@trigger.dev/build/extensions/core";
+
+extensions: [
+  syncEnvVars(async (ctx) => {
+    // ctx contains: environment, projectRef, env
+    return [
+      { name: "SECRET_KEY", value: await getSecret(ctx.environment) },
+      { name: "API_URL", value: ctx.environment === "prod" ? "api.prod.com" : "api.dev.com" },
+    ];
+  }),
+];
+```
+
+#### ESBuild Plugins
+
+```ts
+import { esbuildPlugin } from "@trigger.dev/build/extensions";
+import { sentryEsbuildPlugin } from "@sentry/esbuild-plugin";
+
+extensions: [
+  esbuildPlugin(
+    sentryEsbuildPlugin({
+      org: process.env.SENTRY_ORG,
+      project: process.env.SENTRY_PROJECT,
+      authToken: process.env.SENTRY_AUTH_TOKEN,
+    }),
+    { placement: "last", target: "deploy" } // Optional config
+  ),
+];
+```
+
+## Custom Build Extensions
+
+```ts
+import { defineConfig } from "@trigger.dev/sdk";
+
+const customExtension = {
+  name: "my-custom-extension",
+
+  externalsForTarget: (target) => {
+    return ["some-native-module"]; // Add external dependencies
+  },
+
+  onBuildStart: async (context) => {
+    console.log(`Build starting for ${context.target}`);
+    // Register esbuild plugins, modify build context
+  },
+
+  onBuildComplete: async (context, manifest) => {
+    console.log("Build complete, adding layers");
+    // Add build layers, modify deployment
+    context.addLayer({
+      id: "my-layer",
+      files: [{ source: "./custom-file", destination: "/app/custom" }],
+      commands: ["chmod +x /app/custom"],
+    });
+  },
+};
+
+export default defineConfig({
+  project: "my-project",
+  build: {
+    extensions: [customExtension],
+  },
+});
+```
+
+## Advanced Configuration
+
+### Telemetry
+
+```ts
+import { PrismaInstrumentation } from "@prisma/instrumentation";
+import { OpenAIInstrumentation } from "@langfuse/openai";
+
+export default defineConfig({
+  // ... other config
+  telemetry: {
+    instrumentations: [new PrismaInstrumentation(), new OpenAIInstrumentation()],
+    exporters: [customExporter], // Optional custom exporters
+  },
+});
+```
+
+### Machine & Performance
+
+```ts
+export default defineConfig({
+  // ... other config
+  defaultMachine: "large-1x", // Default machine for all tasks
+  maxDuration: 300, // Default max duration (seconds)
+  enableConsoleLogging: true, // Console logging in development
+});
+```
+
+## Common Extension Combinations
+
+### Full-Stack Web App
+
+```ts
+extensions: [
+  prismaExtension({ schema: "prisma/schema.prisma", migrate: true }),
+  additionalFiles({ files: ["./public/**", "./assets/**"] }),
+  syncEnvVars(async (ctx) => [...envVars]),
+];
+```
+
+### AI/ML Processing
+
+```ts
+extensions: [
+  pythonExtension({
+    scripts: ["./ai/**/*.py"],
+    requirementsFile: "./requirements.txt",
+  }),
+  ffmpeg({ version: "7" }),
+  additionalPackages({ packages: ["wrangler"] }),
+];
+```
+
+### Web Scraping
+
+```ts
+extensions: [
+  playwright({ browsers: ["chromium"] }),
+  puppeteer(),
+  additionalFiles({ files: ["./selectors.json", "./proxies.txt"] }),
+];
+```
+
+## Best Practices
+
+- **Use specific versions**: Pin extension versions for reproducible builds
+- **External packages**: Add modules with native addons to the `build.external` array
+- **Environment sync**: Use `syncEnvVars` for dynamic secrets
+- **File paths**: Use glob patterns for flexible file inclusion
+- **Debug builds**: Use `--log-level debug --dry-run` for troubleshooting
+
+Extensions only affect deployment, not local development. Use `external` array for packages that shouldn't be bundled.

.claude/skills/trigger-dev-tasks/realtime.md

@@ -0,0 +1,244 @@
+# Trigger.dev Realtime
+
+**Real-time monitoring and updates for runs**
+
+## Core Concepts
+
+Realtime allows you to:
+
+- Subscribe to run status changes, metadata updates, and streams
+- Build real-time dashboards and UI updates
+- Monitor task progress from frontend and backend
+
+## Authentication
+
+### Public Access Tokens
+
+```ts
+import { auth } from "@trigger.dev/sdk";
+
+// Read-only token for specific runs
+const publicToken = await auth.createPublicToken({
+  scopes: {
+    read: {
+      runs: ["run_123", "run_456"],
+      tasks: ["my-task-1", "my-task-2"],
+    },
+  },
+  expirationTime: "1h", // Default: 15 minutes
+});
+```
+
+### Trigger Tokens (Frontend only)
+
+```ts
+// Single-use token for triggering tasks
+const triggerToken = await auth.createTriggerPublicToken("my-task", {
+  expirationTime: "30m",
+});
+```
+
+## Backend Usage
+
+### Subscribe to Runs
+
+```ts
+import { runs, tasks } from "@trigger.dev/sdk";
+
+// Trigger and subscribe
+const handle = await tasks.trigger("my-task", { data: "value" });
+
+// Subscribe to specific run
+for await (const run of runs.subscribeToRun<typeof myTask>(handle.id)) {
+  console.log(`Status: ${run.status}, Progress: ${run.metadata?.progress}`);
+  if (run.status === "COMPLETED") break;
+}
+
+// Subscribe to runs with tag
+for await (const run of runs.subscribeToRunsWithTag("user-123")) {
+  console.log(`Tagged run ${run.id}: ${run.status}`);
+}
+
+// Subscribe to batch
+for await (const run of runs.subscribeToBatch(batchId)) {
+  console.log(`Batch run ${run.id}: ${run.status}`);
+}
+```
+
+### Realtime Streams v2
+
+```ts
+import { streams, InferStreamType } from "@trigger.dev/sdk";
+
+// 1. Define streams (shared location)
+export const aiStream = streams.define<string>({
+  id: "ai-output",
+});
+
+export type AIStreamPart = InferStreamType<typeof aiStream>;
+
+// 2. Pipe from task
+export const streamingTask = task({
+  id: "streaming-task",
+  run: async (payload) => {
+    const completion = await openai.chat.completions.create({
+      model: "gpt-4",
+      messages: [{ role: "user", content: payload.prompt }],
+      stream: true,
+    });
+
+    const { waitUntilComplete } = aiStream.pipe(completion);
+    await waitUntilComplete();
+  },
+});
+
+// 3. Read from backend
+const stream = await aiStream.read(runId, {
+  timeoutInSeconds: 300,
+  startIndex: 0, // Resume from specific chunk
+});
+
+for await (const chunk of stream) {
+  console.log("Chunk:", chunk); // Fully typed
+}
+```
+
+## React Frontend Usage
+
+### Installation
+
+```bash
+npm add @trigger.dev/react-hooks
+```
+
+### Triggering Tasks
+
+```tsx
+"use client";
+import { useTaskTrigger, useRealtimeTaskTrigger } from "@trigger.dev/react-hooks";
+import type { myTask } from "../trigger/tasks";
+
+function TriggerComponent({ accessToken }: { accessToken: string }) {
+  // Basic trigger
+  const { submit, handle, isLoading } = useTaskTrigger<typeof myTask>("my-task", {
+    accessToken,
+  });
+
+  // Trigger with realtime updates
+  const {
+    submit: realtimeSubmit,
+    run,
+    isLoading: isRealtimeLoading,
+  } = useRealtimeTaskTrigger<typeof myTask>("my-task", { accessToken });
+
+  return (
+    <div>
+      <button onClick={() => submit({ data: "value" })} disabled={isLoading}>
+        Trigger Task
+      </button>
+
+      <button onClick={() => realtimeSubmit({ data: "realtime" })} disabled={isRealtimeLoading}>
+        Trigger with Realtime
+      </button>
+
+      {run && <div>Status: {run.status}</div>}
+    </div>
+  );
+}
+```
+
+### Subscribing to Runs
+
+```tsx
+"use client";
+import { useRealtimeRun, useRealtimeRunsWithTag } from "@trigger.dev/react-hooks";
+import type { myTask } from "../trigger/tasks";
+
+function SubscribeComponent({ runId, accessToken }: { runId: string; accessToken: string }) {
+  // Subscribe to specific run
+  const { run, error } = useRealtimeRun<typeof myTask>(runId, {
+    accessToken,
+    onComplete: (run) => {
+      console.log("Task completed:", run.output);
+    },
+  });
+
+  // Subscribe to tagged runs
+  const { runs } = useRealtimeRunsWithTag("user-123", { accessToken });
+
+  if (error) return <div>Error: {error.message}</div>;
+  if (!run) return <div>Loading...</div>;
+
+  return (
+    <div>
+      <div>Status: {run.status}</div>
+      <div>Progress: {run.metadata?.progress || 0}%</div>
+      {run.output && <div>Result: {JSON.stringify(run.output)}</div>}
+
+      <h3>Tagged Runs:</h3>
+      {runs.map((r) => (
+        <div key={r.id}>
+          {r.id}: {r.status}
+        </div>
+      ))}
+    </div>
+  );
+}
+```
+
+### Realtime Streams with React
+
+```tsx
+"use client";
+import { useRealtimeStream } from "@trigger.dev/react-hooks";
+import { aiStream } from "../trigger/streams";
+
+function StreamComponent({ runId, accessToken }: { runId: string; accessToken: string }) {
+  // Pass defined stream directly for type safety
+  const { parts, error } = useRealtimeStream(aiStream, runId, {
+    accessToken,
+    timeoutInSeconds: 300,
+    throttleInMs: 50, // Control re-render frequency
+  });
+
+  if (error) return <div>Error: {error.message}</div>;
+  if (!parts) return <div>Loading...</div>;
+
+  const text = parts.join(""); // parts is typed as AIStreamPart[]
+
+  return <div>Streamed Text: {text}</div>;
+}
+```
+
+### Wait Tokens
+
+```tsx
+"use client";
+import { useWaitToken } from "@trigger.dev/react-hooks";
+
+function WaitTokenComponent({ tokenId, accessToken }: { tokenId: string; accessToken: string }) {
+  const { complete } = useWaitToken(tokenId, { accessToken });
+
+  return <button onClick={() => complete({ approved: true })}>Approve Task</button>;
+}
+```
+
+## Run Object Properties
+
+Key properties available in run subscriptions:
+
+- `id`: Unique run identifier
+- `status`: `QUEUED`, `EXECUTING`, `COMPLETED`, `FAILED`, `CANCELED`, etc.
+- `payload`: Task input data (typed)
+- `output`: Task result (typed, when completed)
+- `metadata`: Real-time updatable data
+- `createdAt`, `updatedAt`: Timestamps
+- `costInCents`: Execution cost
+
+## Best Practices
+
+- **Use Realtime over SWR**: Recommended for most use cases due to rate limits
+- **Scope tokens properly**: Only grant necessary read/trigger permissions
+- **Handle errors**: Always check for errors in hooks and subscriptions
+- **Type safety**: Use task types for proper payload/output typing
+- **Cleanup subscriptions**: Backend subscriptions auto-complete, frontend hooks auto-cleanup

.claude/skills/trigger-dev-tasks/scheduled-tasks.md

@@ -0,0 +1,113 @@
+# Scheduled Tasks (Cron)
+
+Recurring tasks using cron. For one-off future runs, use the **delay** option.
+
+## Define a Scheduled Task
+
+```ts
+import { schedules } from "@trigger.dev/sdk";
+
+export const task = schedules.task({
+  id: "first-scheduled-task",
+  run: async (payload) => {
+    payload.timestamp; // Date (scheduled time, UTC)
+    payload.lastTimestamp; // Date | undefined
+    payload.timezone; // IANA, e.g. "America/New_York" (default "UTC")
+    payload.scheduleId; // string
+    payload.externalId; // string | undefined
+    payload.upcoming; // Date[]
+
+    payload.timestamp.toLocaleString("en-US", { timeZone: payload.timezone });
+  },
+});
+```
+
+> Scheduled tasks need at least one schedule attached to run.
+
+## Attach Schedules
+
+**Declarative (sync on dev/deploy):**
+
+```ts
+schedules.task({
+  id: "every-2h",
+  cron: "0 */2 * * *", // UTC
+  run: async () => {},
+});
+
+schedules.task({
+  id: "tokyo-5am",
+  cron: { pattern: "0 5 * * *", timezone: "Asia/Tokyo", environments: ["PRODUCTION", "STAGING"] },
+  run: async () => {},
+});
+```
+
+**Imperative (SDK or dashboard):**
+
+```ts
+await schedules.create({
+  task: task.id,
+  cron: "0 0 * * *",
+  timezone: "America/New_York", // DST-aware
+  externalId: "user_123",
+  deduplicationKey: "user_123-daily", // updates if reused
+});
+```
+
+### Dynamic / Multi-tenant Example
+
+```ts
+// /trigger/reminder.ts
+export const reminderTask = schedules.task({
+  id: "todo-reminder",
+  run: async (p) => {
+    if (!p.externalId) throw new Error("externalId is required");
+    const user = await db.getUser(p.externalId);
+    await sendReminderEmail(user);
+  },
+});
+```
+
+```ts
+// app/reminders/route.ts
+export async function POST(req: Request) {
+  const data = await req.json();
+  return Response.json(
+    await schedules.create({
+      task: reminderTask.id,
+      cron: "0 8 * * *",
+      timezone: data.timezone,
+      externalId: data.userId,
+      deduplicationKey: `${data.userId}-reminder`,
+    })
+  );
+}
+```
+
+## Cron Syntax (no seconds)
+
+```
+* * * * *
+| | | | └ day of week (0–7 or 1L–7L; 0/7=Sun; L=last)
+| | | └── month (1–12)
+| | └──── day of month (1–31 or L)
+| └────── hour (0–23)
+└──────── minute (0–59)
+```
+
+## When Schedules Won't Trigger
+
+- **Dev:** only when the dev CLI is running.
+- **Staging/Production:** only for tasks in the **latest deployment**.
+
+## SDK Management
+
+```ts
+await schedules.retrieve(id);
+await schedules.list();
+await schedules.update(id, { cron: "0 0 1 * *", externalId: "ext", deduplicationKey: "key" });
+await schedules.deactivate(id);
+await schedules.activate(id);
+await schedules.del(id);
+await schedules.timezones(); // list of IANA timezones
+```

.cursor/commands/deslop.md

@@ -0,0 +1,11 @@
+# Remove AI code slop
+
+Check the diff against main, and remove all AI generated slop introduced in this branch.
+
+This includes:
+- Extra comments that a human wouldn't add or is inconsistent with the rest of the file
+- Extra defensive checks or try/catch blocks that are abnormal for that area of the codebase (especially if called by trusted / validated codepaths)
+- Casts to any to get around type issues
+- Any other style that is inconsistent with the file
+
+Report at the end with only a 1-3 sentence summary of what you changed

.cursor/mcp.json

@@ -1,7 +1,3 @@
 {
-  "mcpServers": {
-    "trigger.dev": {
-      "url": "http://localhost:3333/sse"
-    }
-  }
-}
+  "mcpServers": {}
+}

.cursor/rules/migrations.mdc

@@ -0,0 +1,6 @@
+---
+description: how to create and apply database migrations
+alwaysApply: false
+---
+
+Follow our [migrations.md](mdc:ai/references/migrations.md) guide for how to create and apply database migrations.

.cursor/rules/otel-metrics.mdc

@@ -0,0 +1,66 @@
+---
+description: Guidelines for creating OpenTelemetry metrics to avoid cardinality issues
+globs:
+  - "**/*.ts"
+---
+
+# OpenTelemetry Metrics Guidelines
+
+When creating or editing OTEL metrics (counters, histograms, gauges), always ensure metric attributes have **low cardinality**.
+
+## What is Cardinality?
+
+Cardinality refers to the number of unique values an attribute can have. Each unique combination of attribute values creates a new time series, which consumes memory and storage in your metrics backend.
+
+## Rules
+
+### DO use low-cardinality attributes:
+- **Enums**: `environment_type` (PRODUCTION, STAGING, DEVELOPMENT, PREVIEW)
+- **Booleans**: `hasFailures`, `streaming`, `success`
+- **Bounded error codes**: A finite, controlled set of error types
+- **Shard IDs**: When sharding is bounded (e.g., 0-15)
+
+### DO NOT use high-cardinality attributes:
+- **UUIDs/IDs**: `envId`, `userId`, `runId`, `projectId`, `organizationId`
+- **Unbounded integers**: `itemCount`, `batchSize`, `retryCount`
+- **Timestamps**: `createdAt`, `startTime`
+- **Free-form strings**: `errorMessage`, `taskName`, `queueName`
+
+## Example
+
+```typescript
+// BAD - High cardinality
+this.counter.add(1, {
+  envId: options.environmentId,        // UUID - unbounded
+  itemCount: options.runCount,         // Integer - unbounded
+});
+
+// GOOD - Low cardinality
+this.counter.add(1, {
+  environment_type: options.environmentType,  // Enum - 4 values
+  streaming: true,                            // Boolean - 2 values
+});
+```
+
+## Prometheus Metric Naming
+
+When metrics are exported via OTLP to Prometheus, the exporter automatically adds unit suffixes to metric names:
+
+| OTel Metric Name | Unit | Prometheus Name |
+|------------------|------|-----------------|
+| `my_duration_ms` | `ms` | `my_duration_ms_milliseconds` |
+| `my_counter` | counter | `my_counter_total` |
+| `items_inserted` | counter | `items_inserted_inserts_total` |
+| `batch_size` | histogram | `batch_size_items_bucket` |
+
+Keep this in mind when writing Grafana dashboards or Prometheus queries—the metric names in Prometheus will differ from the names defined in code.
+
+## Reference
+
+See the schedule engine (`internal-packages/schedule-engine/src/engine/index.ts`) for a good example of low-cardinality metric attributes.
+
+High cardinality metrics can cause:
+- Memory bloat in metrics backends (Axiom, Prometheus, etc.)
+- Slow queries and dashboard timeouts
+- Increased costs (many backends charge per time series)
+- Potential data loss or crashes at scale

.cursor/rules/webapp.mdc

@@ -6,7 +6,7 @@ alwaysApply: false
 
 The main trigger.dev webapp, which powers it's API and dashboard and makes up the docker image that is produced as an OSS image, is a Remix 2.1.0 app that uses an express server, written in TypeScript. The following subsystems are either included in the webapp or are used by the webapp in another part of the monorepo:
 
-- `@trigger.dev/database` exports a Prisma 5.4.1 client that is used extensively in the webapp to access a PostgreSQL instance. The schema file is [schema.prisma](mdc:internal-packages/database/prisma/schema.prisma)
+- `@trigger.dev/database` exports a Prisma 6.14.0 client that is used extensively in the webapp to access a PostgreSQL instance. The schema file is [schema.prisma](mdc:internal-packages/database/prisma/schema.prisma)
 - `@trigger.dev/core` is a published package and is used to share code between the `@trigger.dev/sdk` and the webapp. It includes functionality but also a load of Zod schemas for data validation. When importing from `@trigger.dev/core` in the webapp, we never import the root `@trigger.dev/core` path, instead we favor one of the subpath exports that you can find in [package.json](mdc:packages/core/package.json)
 - `@internal/run-engine` has all the code needed to trigger a run and take it through it's lifecycle to completion.
 - `@trigger.dev/redis-worker` is a custom redis based background job/worker system that's used in the webapp and also used inside the run engine.
@@ -31,7 +31,10 @@ We originally the Trigger.dev "Run Engine" not as a single system, but just spre
 - The batch trigger API endpoint is [api.v1.tasks.batch.ts](mdc:apps/webapp/app/routes/api.v1.tasks.batch.ts)
 - Setup code for the prisma client is in [db.server.ts](mdc:apps/webapp/app/db.server.ts)
 - The run engine is configured in [runEngine.server.ts](mdc:apps/webapp/app/v3/runEngine.server.ts)
-- All the "services" that are found in app/v3/services/**/*.server.ts
+- All the "services" that are found in app/v3/services/\*_/_.server.ts
 - The code for the TaskEvent data, which is the otel data sent from tasks to our servers, is in both the [eventRepository.server.ts](mdc:apps/webapp/app/v3/eventRepository.server.ts) and also the [otlpExporter.server.ts](mdc:apps/webapp/app/v3/otlpExporter.server.ts). The otel endpoints which are hit from production and development otel exporters is [otel.v1.logs.ts](mdc:apps/webapp/app/routes/otel.v1.logs.ts) and [otel.v1.traces.ts](mdc:apps/webapp/app/routes/otel.v1.traces.ts)
-- We use "presenters" to move more complex loader code into a class, and you can find those are app/v3/presenters/**/*.server.ts
+- We use "presenters" to move more complex loader code into a class, and you can find those are app/v3/presenters/\*_/_.server.ts
 
+- All the "services" that are found in app/v3/services/\*_/_.server.ts
+- The code for the TaskEvent data, which is the otel data sent from tasks to our servers, is in both the [eventRepository.server.ts](mdc:apps/webapp/app/v3/eventRepository.server.ts) and also the [otlpExporter.server.ts](mdc:apps/webapp/app/v3/otlpExporter.server.ts). The otel endpoints which are hit from production and development otel exporters is [otel.v1.logs.ts](mdc:apps/webapp/app/routes/otel.v1.logs.ts) and [otel.v1.traces.ts](mdc:apps/webapp/app/routes/otel.v1.traces.ts)
+- We use "presenters" to move more complex loader code into a class, and you can find those are app/v3/presenters/\*_/_.server.ts

.dockerignore

@@ -16,6 +16,8 @@
 **/dist
 **/node_modules
 
+**/generated/prisma
+
 apps/webapp/build
 apps/webapp/public/build
 

.env.example

@@ -13,6 +13,11 @@ APP_ORIGIN=http://localhost:3030
 ELECTRIC_ORIGIN=http://localhost:3060
 NODE_ENV=development
 
+# Clickhouse
+CLICKHOUSE_URL=http://default:password@localhost:8123
+RUN_REPLICATION_CLICKHOUSE_URL=http://default:password@localhost:8123
+RUN_REPLICATION_ENABLED=1
+
 # Set this to UTC because Node.js uses the system timezone
 TZ="UTC"
 
@@ -29,9 +34,9 @@ DEPLOY_REGISTRY_HOST=localhost:5000
 
 # OPTIONAL VARIABLES
 # This is used for validating emails that are allowed to log in. Every email that do not match this regex will be rejected.
-# WHITELISTED_EMAILS="authorized@yahoo\.com|authorized@gmail\.com"
+# WHITELISTED_EMAILS="^(authorized@yahoo\.com|authorized@gmail\.com)$"
 # Accounts with these emails will get global admin rights. This grants access to the admin UI.
-# ADMIN_EMAILS="admin@example\.com|another-admin@example\.com"
+# ADMIN_EMAILS="^(admin@example\.com|another-admin@example\.com)$"
 # This is used for logging in via GitHub. You can leave these commented out if you don't want to use GitHub for authentication.
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
@@ -80,4 +85,10 @@ POSTHOG_PROJECT_KEY=
 # These control the server-side internal telemetry
 # INTERNAL_OTEL_TRACE_EXPORTER_URL=<URL to send traces to>
 # INTERNAL_OTEL_TRACE_LOGGING_ENABLED=1
-# INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED=0,
+# INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED=0
+
+# Enable local observability stack (requires `pnpm run docker` to start otel-collector)
+# Uncomment these to send metrics to the local Prometheus via OTEL Collector:
+# INTERNAL_OTEL_METRIC_EXPORTER_ENABLED=1
+# INTERNAL_OTEL_METRIC_EXPORTER_URL=http://localhost:4318/v1/metrics
+# INTERNAL_OTEL_METRIC_EXPORTER_INTERVAL_MS=15000

.github/ISSUE_TEMPLATE/vouch-request.yml

@@ -0,0 +1,28 @@
+name: Vouch Request
+description: Request to be vouched as a contributor
+labels: ["vouch-request"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        ## Vouch Request
+
+        We use [vouch](https://github.com/mitchellh/vouch) to manage contributor trust. PRs from unvouched users are automatically closed.
+
+        To get vouched, fill out this form. A maintainer will review your request and vouch for you by commenting on this issue.
+  - type: textarea
+    id: context
+    attributes:
+      label: Why do you want to contribute?
+      description: Tell us a bit about yourself and what you'd like to work on.
+      placeholder: "I'd like to fix a bug I found in..."
+    validations:
+      required: true
+  - type: textarea
+    id: prior-work
+    attributes:
+      label: Prior contributions or relevant experience
+      description: Links to previous open source work, relevant projects, or anything that helps us understand your background.
+      placeholder: "https://github.com/..."
+    validations:
+      required: false

.github/VOUCHED.td

@@ -0,0 +1,20 @@
+# Vouched contributors for Trigger.dev
+# See: https://github.com/mitchellh/vouch
+#
+# Org members
+0ski
+D-K-P
+ericallam
+matt-aitken
+mpcgrid
+myftija
+nicktrn
+samejr
+isshaddad
+# Outside contributors
+gautamsi
+capaj
+chengzp
+bharathkumar39293
+bhekanik
+jrossi

.github/workflows/changesets-pr.yml

@@ -0,0 +1,123 @@
+name: 🦋 Changesets PR
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "packages/**"
+      - ".changeset/**"
+      - ".server-changes/**"
+      - "package.json"
+      - "pnpm-lock.yaml"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  release-pr:
+    name: Create Release PR
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    if: github.repository == 'triggerdotdev/trigger.dev'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+
+      - name: Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+          cache: "pnpm"
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Create release PR
+        id: changesets
+        uses: changesets/action@v1
+        with:
+          version: pnpm run changeset:version
+          commit: "chore: release"
+          title: "chore: release"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Update PR title and enhance body
+        if: steps.changesets.outputs.published != 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          PR_NUMBER=$(gh pr list --head changeset-release/main --json number --jq '.[0].number')
+          if [ -n "$PR_NUMBER" ]; then
+            git fetch origin changeset-release/main
+            # we arbitrarily reference the version of the cli package here; it is the same for all package releases
+            VERSION=$(git show origin/changeset-release/main:packages/cli-v3/package.json | jq -r '.version')
+            gh pr edit "$PR_NUMBER" --title "chore: release v$VERSION"
+
+            # Enhance the PR body with a clean, deduplicated summary
+            RAW_BODY=$(gh pr view "$PR_NUMBER" --json body --jq '.body')
+            ENHANCED_BODY=$(CHANGESET_PR_BODY="$RAW_BODY" node scripts/enhance-release-pr.mjs "$VERSION")
+            if [ -n "$ENHANCED_BODY" ]; then
+              gh api repos/triggerdotdev/trigger.dev/pulls/"$PR_NUMBER" \
+                -X PATCH \
+                -f body="$ENHANCED_BODY"
+            fi
+          fi
+
+  update-lockfile:
+    name: Update lockfile on release PR
+    runs-on: ubuntu-latest
+    needs: release-pr
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout release branch
+        uses: actions/checkout@v4
+        with:
+          ref: changeset-release/main
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+
+      - name: Install and update lockfile
+        run: pnpm install --no-frozen-lockfile
+
+      - name: Clean up consumed .server-changes/ files
+        run: |
+          set -e
+          shopt -s nullglob
+          files=(.server-changes/*.md)
+          for f in "${files[@]}"; do
+            if [ "$(basename "$f")" != "README.md" ]; then
+              git rm --ignore-unmatch "$f"
+            fi
+          done
+
+      - name: Commit and push lockfile + server-changes cleanup
+        run: |
+          set -e
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add pnpm-lock.yaml
+          if ! git diff --cached --quiet; then
+            git commit -m "chore: update lockfile and clean up .server-changes/ for release"
+            git push origin changeset-release/main
+          else
+            echo "No changes to commit"
+          fi

.github/workflows/claude-md-audit.yml

@@ -0,0 +1,69 @@
+name: 📝 CLAUDE.md Audit
+
+on:
+  pull_request:
+    types: [opened, ready_for_review, synchronize]
+    paths-ignore:
+      - "docs/**"
+      - ".changeset/**"
+      - ".server-changes/**"
+      - "**/*.md"
+      - "references/**"
+
+concurrency:
+  group: claude-md-audit-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  audit:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
+      issues: write
+      id-token: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Run Claude Code
+        id: claude
+        uses: anthropics/claude-code-action@v1
+        with:
+          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
+          use_sticky_comment: true
+
+          claude_args: |
+            --max-turns 15
+            --allowedTools "Read,Glob,Grep,Bash(git diff:*)"
+
+          prompt: |
+            You are reviewing a PR to check whether any CLAUDE.md files or .claude/rules/ files need updating.
+
+            ## Your task
+
+            1. Run `git diff origin/main...HEAD --name-only` to see which files changed in this PR.
+            2. For each changed directory, check if there's a CLAUDE.md in that directory or a parent directory.
+            3. Determine if any CLAUDE.md or .claude/rules/ file should be updated based on the changes. Consider:
+               - New files/directories that aren't covered by existing documentation
+               - Changed architecture or patterns that contradict current CLAUDE.md guidance
+               - New dependencies, services, or infrastructure that Claude should know about
+               - Renamed or moved files that are referenced in CLAUDE.md
+               - Changes to build commands, test patterns, or development workflows
+
+            ## Response format
+
+            If NO updates are needed, respond with exactly:
+            ✅ CLAUDE.md files look current for this PR.
+
+            If updates ARE needed, respond with a short list:
+            📝 **CLAUDE.md updates suggested:**
+            - `path/to/CLAUDE.md`: [what should be added/changed]
+            - `.claude/rules/file.md`: [what should be added/changed]
+
+            Keep suggestions specific and brief. Only flag things that would actually mislead Claude in future sessions.
+            Do NOT suggest updates for trivial changes (bug fixes, small refactors within existing patterns).
+            Do NOT suggest creating new CLAUDE.md files - only updates to existing ones.

.github/workflows/claude.yml

@@ -0,0 +1,70 @@
+name: Claude Code
+
+on:
+  issue_comment:
+    types: [created]
+  pull_request_review_comment:
+    types: [created]
+  issues:
+    types: [opened, assigned]
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  claude:
+    if: |
+      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
+      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+      issues: read
+      id-token: write
+      actions: read # Required for Claude to read CI results on PRs
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: Run Claude Code
+        id: claude
+        uses: anthropics/claude-code-action@v1
+        with:
+          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
+
+          # This is an optional setting that allows Claude to read CI results on PRs
+          additional_permissions: |
+            actions: read
+
+          claude_args: |
+            --model claude-opus-4-5-20251101
+            --allowedTools "Bash(pnpm:*),Bash(turbo:*),Bash(git:*),Bash(gh:*),Bash(npx:*),Bash(docker:*),Edit,MultiEdit,Read,Write,Glob,Grep,LS,Task"
+
+          # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
+          # prompt: 'Update the pull request description to include a summary of changes.'
+
+          # Optional: Add claude_args to customize behavior and configuration
+          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
+          # or https://code.claude.com/docs/en/cli-reference for available options
+          # claude_args: '--allowed-tools Bash(gh pr:*)'

.github/workflows/e2e.yml

@@ -31,12 +31,12 @@ jobs:
       - name: ⎔ Setup pnpm
         uses: pnpm/action-setup@v4
         with:
-          version: 8.15.5
+          version: 10.23.0
 
       - name: ⎔ Setup node
         uses: buildjet/setup-node@v4
         with:
-          node-version: 20.11.1
+          node-version: 20.20.0
 
       - name: 📥 Download deps
         run: pnpm install --frozen-lockfile --filter trigger.dev...

.github/workflows/helm-pr-prerelease.yml

@@ -0,0 +1,138 @@
+name: 🧭 Helm Chart PR Prerelease
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - "hosting/k8s/helm/**"
+
+concurrency:
+  group: helm-prerelease-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+env:
+  REGISTRY: ghcr.io
+  CHART_NAME: trigger
+
+jobs:
+  lint-and-test:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v4
+        with:
+          version: "3.18.3"
+
+      - name: Build dependencies
+        run: helm dependency build ./hosting/k8s/helm/
+
+      - name: Extract dependency charts
+        run: |
+          cd ./hosting/k8s/helm/
+          for file in ./charts/*.tgz; do echo "Extracting $file"; tar -xzf "$file" -C ./charts; done
+
+      - name: Lint Helm Chart
+        run: |
+          helm lint ./hosting/k8s/helm/
+
+      - name: Render templates
+        run: |
+          helm template test-release ./hosting/k8s/helm/ \
+            --values ./hosting/k8s/helm/values.yaml \
+            --output-dir ./helm-output
+
+      - name: Validate manifests
+        uses: docker://ghcr.io/yannh/kubeconform:v0.7.0
+        with:
+          entrypoint: "/kubeconform"
+          args: "-summary -output json ./helm-output"
+
+  prerelease:
+    needs: lint-and-test
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      pull-requests: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v4
+        with:
+          version: "3.18.3"
+
+      - name: Build dependencies
+        run: helm dependency build ./hosting/k8s/helm/
+
+      - name: Extract dependency charts
+        run: |
+          cd ./hosting/k8s/helm/
+          for file in ./charts/*.tgz; do echo "Extracting $file"; tar -xzf "$file" -C ./charts; done
+
+      - name: Log in to Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Generate prerelease version
+        id: version
+        run: |
+          BASE_VERSION=$(grep '^version:' ./hosting/k8s/helm/Chart.yaml | awk '{print $2}')
+          PR_NUMBER=${{ github.event.pull_request.number }}
+          SHORT_SHA=$(echo "${{ github.event.pull_request.head.sha }}" | cut -c1-7)
+          PRERELEASE_VERSION="${BASE_VERSION}-pr${PR_NUMBER}.${SHORT_SHA}"
+          echo "version=$PRERELEASE_VERSION" >> $GITHUB_OUTPUT
+          echo "Prerelease version: $PRERELEASE_VERSION"
+
+      - name: Update Chart.yaml with prerelease version
+        run: |
+          sed -i "s/^version:.*/version: ${{ steps.version.outputs.version }}/" ./hosting/k8s/helm/Chart.yaml
+
+      - name: Package Helm Chart
+        run: |
+          helm package ./hosting/k8s/helm/ --destination /tmp/
+
+      - name: Push Helm Chart to GHCR
+        run: |
+          VERSION="${{ steps.version.outputs.version }}"
+          CHART_PACKAGE="/tmp/${{ env.CHART_NAME }}-${VERSION}.tgz"
+
+          # Push to GHCR OCI registry
+          helm push "$CHART_PACKAGE" "oci://${{ env.REGISTRY }}/${{ github.repository_owner }}/charts"
+
+      - name: Find existing comment
+        uses: peter-evans/find-comment@v3
+        id: find-comment
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: "github-actions[bot]"
+          body-includes: "Helm Chart Prerelease Published"
+
+      - name: Create or update PR comment
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          comment-id: ${{ steps.find-comment.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            ### 🧭 Helm Chart Prerelease Published
+
+            **Version:** `${{ steps.version.outputs.version }}`
+
+            **Install:**
+            ```bash
+            helm upgrade --install trigger \
+              oci://ghcr.io/${{ github.repository_owner }}/charts/trigger \
+              --version "${{ steps.version.outputs.version }}"
+            ```
+
+            > ⚠️ This is a prerelease for testing. Do not use in production.
+          edit-mode: replace

.github/workflows/pr_checks.yml

@@ -29,3 +29,7 @@ jobs:
     with:
       package: cli-v3
     secrets: inherit
+
+  sdk-compat:
+    uses: ./.github/workflows/sdk-compat.yml
+    secrets: inherit

.github/workflows/publish.yml

@@ -1,6 +1,7 @@
 name: 🚀 Publish Trigger.dev Docker
 
 on:
+  workflow_dispatch:
   workflow_call:
     inputs:
       image_tag:

.github/workflows/release.yml

@@ -1,98 +1,264 @@
 name: 🦋 Changesets Release
 
-permissions:
-  contents: write
-
 on:
-  push:
+  pull_request:
+    types: [closed]
     branches:
       - main
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - ".github/CODEOWNERS"
-      - ".github/ISSUE_TEMPLATE/**"
+  workflow_dispatch:
+    inputs:
+      type:
+        description: "Select release type"
+        required: true
+        type: choice
+        options:
+          - release
+          - prerelease
+        default: "prerelease"
+      ref:
+        description: "The ref (branch, tag, or SHA) to checkout and release from"
+        required: true
+        type: string
+      prerelease_tag:
+        description: "The npm dist-tag for the prerelease (e.g., 'v4-prerelease')"
+        required: false
+        type: string
+        default: "prerelease"
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}
+  cancel-in-progress: false
 
 jobs:
-  release:
-    name: 🦋 Changesets Release
+  show-release-summary:
+    name: 📋 Release Summary
     runs-on: ubuntu-latest
+    if: |
+      github.repository == 'triggerdotdev/trigger.dev' &&
+      github.event_name == 'pull_request' &&
+      github.event.pull_request.merged == true &&
+      github.event.pull_request.head.ref == 'changeset-release/main'
+    steps:
+      - name: Show release summary
+        env:
+          PR_BODY: ${{ github.event.pull_request.body }}
+        run: |
+          echo "$PR_BODY" | sed -n '/^# Releases/,$p' >> $GITHUB_STEP_SUMMARY
+
+  release:
+    name: 🚀 Release npm packages
+    runs-on: ubuntu-latest
+    environment: npm-publish
     permissions:
       contents: write
       packages: write
-      pull-requests: write
-    if: github.repository == 'triggerdotdev/trigger.dev'
+      id-token: write
+    if: |
+      github.repository == 'triggerdotdev/trigger.dev' &&
+      (
+        (github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'release') ||
+        (github.event_name == 'pull_request' && github.event.pull_request.merged == true && github.event.pull_request.head.ref == 'changeset-release/main')
+      )
     outputs:
       published: ${{ steps.changesets.outputs.published }}
       published_packages: ${{ steps.changesets.outputs.publishedPackages }}
       published_package_version: ${{ steps.get_version.outputs.package_version }}
     steps:
-      - name: ⬇️ Checkout repo
+      - name: Checkout repo
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.ref || github.sha }}
 
-      - name: ⎔ Setup pnpm
+      - name: Verify ref is on main
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          if ! git merge-base --is-ancestor ${{ github.event.inputs.ref }} origin/main; then
+            echo "Error: ref must be an ancestor of main (i.e., already merged)"
+            exit 1
+          fi
+
+      - name: Setup pnpm
         uses: pnpm/action-setup@v4
         with:
-          version: 8.15.5
+          version: 10.23.0
 
-      - name: ⎔ Setup node
+      - name: Setup node
         uses: buildjet/setup-node@v4
         with:
-          node-version: 20.11.1
+          node-version: 20.20.0
           cache: "pnpm"
 
-      - name: 📥 Download deps
+      # npm v11.5.1 or newer is required for OIDC support
+      # https://github.blog/changelog/2025-07-31-npm-trusted-publishing-with-oidc-is-generally-available/#whats-new
+      - name: Setup npm 11.x for OIDC
+        run: npm install -g npm@11.6.4
+
+      - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
-      - name: 📀 Generate Prisma Client
+      - name: Generate Prisma client
         run: pnpm run generate
 
-      - name: 🏗️ Build
+      - name: Build
         run: pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev"
 
-      - name: 🔎 Type check
+      - name: Type check
         run: pnpm run typecheck --filter "@trigger.dev/*" --filter "trigger.dev"
 
-      - name: 🔐 Setup npm auth
-        run: |
-          echo "registry=https://registry.npmjs.org" >> ~/.npmrc
-          echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}" >> ~/.npmrc
-
-        # This action has two responsibilities. The first time the workflow runs
-        # (initial push to the `main` branch) it will create a new branch and
-        # then open a PR with the related changes for the new version. After the
-        # PR is merged, the workflow will run again and this action will build +
-        # publish to npm.
-      - name: 🚀 PR / Publish
-        if: ${{ !env.ACT }}
+      - name: Publish
         id: changesets
         uses: changesets/action@v1
         with:
-          version: pnpm run changeset:version
-          commit: "chore: Update version for release"
-          title: "chore: Update version for release"
           publish: pnpm run changeset:release
-          createGithubReleases: true
+          createGithubReleases: false
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
 
-      # - name: 🚀 PR / Publish (mock)
-      #   if: ${{ env.ACT }}
-      #   id: changesets
-      #   run: |
-      #     echo "published=true" >> "$GITHUB_OUTPUT"
-      #     echo "publishedPackages=[{\"name\": \"@xx/xx\", \"version\": \"1.2.0\"}, {\"name\": \"@xx/xy\", \"version\": \"0.8.9\"}]" >> "$GITHUB_OUTPUT"
-
-      - name: 📦 Get package version
+      - name: Show package version
         if: steps.changesets.outputs.published == 'true'
         id: get_version
         run: |
           package_version=$(echo '${{ steps.changesets.outputs.publishedPackages }}' | jq -r '.[0].version')
           echo "package_version=${package_version}" >> "$GITHUB_OUTPUT"
+
+      - name: Create unified GitHub release
+        if: steps.changesets.outputs.published == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RELEASE_PR_BODY: ${{ github.event.pull_request.body }}
+        run: |
+          VERSION="${{ steps.get_version.outputs.package_version }}"
+          node scripts/generate-github-release.mjs "$VERSION" > /tmp/release-body.md
+          gh release create "v${VERSION}" \
+            --title "trigger.dev v${VERSION}" \
+            --notes-file /tmp/release-body.md \
+            --target main
+
+      - name: Create and push Docker tag
+        if: steps.changesets.outputs.published == 'true'
+        run: |
+          set -e
+          git tag "v.docker.${{ steps.get_version.outputs.package_version }}"
+          git push origin "v.docker.${{ steps.get_version.outputs.package_version }}"
+
+  # Trigger Docker builds directly via workflow_call since tags pushed with
+  # GITHUB_TOKEN don't trigger other workflows (GitHub Actions limitation).
+  publish-docker:
+    name: 🐳 Publish Docker images
+    needs: release
+    if: needs.release.outputs.published == 'true'
+    uses: ./.github/workflows/publish.yml
+    secrets: inherit
+    with:
+      image_tag: v${{ needs.release.outputs.published_package_version }}
+
+  # After Docker images are published, update the GitHub release with the exact GHCR tag URL.
+  # The GHCR package version ID is only known after the image is pushed, so we query for it here.
+  update-release:
+    name: 🔗 Update release Docker link
+    needs: [release, publish-docker]
+    if: needs.release.outputs.published == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      packages: read
+    steps:
+      - name: Update GitHub release with Docker image link
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -e
+          VERSION="${{ needs.release.outputs.published_package_version }}"
+          TAG="v${VERSION}"
+
+          # Query GHCR for the version ID matching this tag
+          VERSION_ID=$(gh api --paginate -H "Accept: application/vnd.github+json" \
+            /orgs/triggerdotdev/packages/container/trigger.dev/versions \
+            --jq ".[] | select(.metadata.container.tags[] == \"${TAG}\") | .id" \
+            | head -1)
+
+          if [ -z "$VERSION_ID" ]; then
+            echo "Warning: Could not find GHCR version ID for tag ${TAG}, skipping update"
+            exit 0
+          fi
+
+          DOCKER_URL="https://github.com/triggerdotdev/trigger.dev/pkgs/container/trigger.dev/${VERSION_ID}?tag=${TAG}"
+          GENERIC_URL="https://github.com/triggerdotdev/trigger.dev/pkgs/container/trigger.dev"
+
+          # Get current release body and replace the generic link with the tag-specific one.
+          # Use word boundary after GENERIC_URL (closing paren) to avoid matching URLs that
+          # already have a version ID appended (idempotent on re-runs).
+          gh release view "${TAG}" --repo triggerdotdev/trigger.dev --json body --jq '.body' > /tmp/release-body.md
+          sed -i "s|${GENERIC_URL})|${DOCKER_URL})|g" /tmp/release-body.md
+
+          gh release edit "${TAG}" --repo triggerdotdev/trigger.dev --notes-file /tmp/release-body.md
+
+  # Dispatch changelog entry creation to the marketing site repo.
+  # Runs after update-release so the GitHub release body already has the exact Docker image URL.
+  dispatch-changelog:
+    name: 📝 Dispatch changelog PR
+    needs: [release, update-release]
+    if: needs.release.outputs.published == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: peter-evans/repository-dispatch@v3
+        with:
+          token: ${{ secrets.CROSS_REPO_PAT }}
+          repository: triggerdotdev/trigger.dev-site-v3
+          event-type: new-release
+          client-payload: '{"version": "${{ needs.release.outputs.published_package_version }}"}'
+
+  # The prerelease job needs to be on the same workflow file due to a limitation related to how npm verifies OIDC claims.
+  prerelease:
+    name: 🧪 Prerelease
+    runs-on: ubuntu-latest
+    environment: npm-publish
+    permissions:
+      contents: read
+      id-token: write
+    if: github.repository == 'triggerdotdev/trigger.dev' && github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'prerelease'
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.inputs.ref }}
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+          cache: "pnpm"
+
+      # npm v11.5.1 or newer is required for OIDC support
+      # https://github.blog/changelog/2025-07-31-npm-trusted-publishing-with-oidc-is-generally-available/#whats-new
+      - name: Setup npm 11.x for OIDC
+        run: npm install -g npm@11.6.4
+
+      - name: Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: Generate Prisma Client
+        run: pnpm run generate
+
+      - name: Snapshot version
+        run: pnpm exec changeset version --snapshot ${{ github.event.inputs.prerelease_tag }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Clean
+        run: pnpm run clean --filter "@trigger.dev/*" --filter "trigger.dev"
+
+      - name: Build
+        run: pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev"
+
+      - name: Publish prerelease
+        run: pnpm exec changeset publish --no-git-tag --snapshot --tag ${{ github.event.inputs.prerelease_tag }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/sdk-compat.yml

@@ -0,0 +1,178 @@
+name: "🔌 SDK Compatibility Tests"
+
+permissions:
+  contents: read
+
+on:
+  workflow_call:
+
+jobs:
+  node-compat:
+    name: "Node.js ${{ matrix.node }} (${{ matrix.os }})"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        node: ["20.20", "22.12"]
+
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: ${{ matrix.node }}
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: 🔨 Build SDK dependencies
+        shell: bash
+        run: pnpm run build --filter '@trigger.dev/sdk^...'
+
+      - name: 🔨 Build SDK
+        shell: bash
+        run: pnpm run build --filter '@trigger.dev/sdk'
+
+      - name: 🧪 Run SDK Compatibility Tests
+        shell: bash
+        run: pnpm --filter @internal/sdk-compat-tests test
+
+  bun-compat:
+    name: "Bun Runtime"
+    runs-on: ubuntu-latest
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+          cache: "pnpm"
+
+      - name: 🥟 Setup Bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: 🔨 Build SDK dependencies
+        run: pnpm run build --filter @trigger.dev/sdk^...
+
+      - name: 🔨 Build SDK
+        run: pnpm run build --filter @trigger.dev/sdk
+
+      - name: 🧪 Run Bun Compatibility Test
+        working-directory: internal-packages/sdk-compat-tests/src/fixtures/bun
+        run: bun run test.ts
+
+  deno-compat:
+    name: "Deno Runtime"
+    runs-on: ubuntu-latest
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+          cache: "pnpm"
+
+      - name: 🦕 Setup Deno
+        uses: denoland/setup-deno@v2
+        with:
+          deno-version: v2.x
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: 🔨 Build SDK dependencies
+        run: pnpm run build --filter @trigger.dev/sdk^...
+
+      - name: 🔨 Build SDK
+        run: pnpm run build --filter @trigger.dev/sdk
+
+      - name: 🔗 Link node_modules for Deno fixture
+        working-directory: internal-packages/sdk-compat-tests/src/fixtures/deno
+        run: ln -s ../../../../../node_modules node_modules
+
+      - name: 🧪 Run Deno Compatibility Test
+        working-directory: internal-packages/sdk-compat-tests/src/fixtures/deno
+        run: deno run --allow-read --allow-env --allow-sys test.ts
+
+  cloudflare-compat:
+    name: "Cloudflare Workers"
+    runs-on: ubuntu-latest
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: 🔨 Build SDK dependencies
+        run: pnpm run build --filter @trigger.dev/sdk^...
+
+      - name: 🔨 Build SDK
+        run: pnpm run build --filter @trigger.dev/sdk
+
+      - name: 📥 Install Cloudflare fixture deps
+        working-directory: internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker
+        run: pnpm install
+
+      - name: 🧪 Run Cloudflare Workers Compatibility Test (dry-run)
+        working-directory: internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker
+        run: npx wrangler deploy --dry-run --outdir dist

.github/workflows/typecheck.yml

@@ -19,12 +19,12 @@ jobs:
       - name: ⎔ Setup pnpm
         uses: pnpm/action-setup@v4
         with:
-          version: 8.15.5
+          version: 10.23.0
 
       - name: ⎔ Setup node
         uses: buildjet/setup-node@v4
         with:
-          node-version: 20.11.1
+          node-version: 20.20.0
           cache: "pnpm"
 
       - name: 📥 Download deps
@@ -35,6 +35,8 @@ jobs:
 
       - name: 🔎 Type check
         run: pnpm run typecheck
+        env:
+          NODE_OPTIONS: --max-old-space-size=8192
 
       - name: 🔎 Check exports
         run: pnpm run check-exports

.github/workflows/unit-tests-internal.yml

@@ -53,12 +53,12 @@ jobs:
       - name: ⎔ Setup pnpm
         uses: pnpm/action-setup@v4
         with:
-          version: 8.15.5
+          version: 10.23.0
 
       - name: ⎔ Setup node
         uses: buildjet/setup-node@v4
         with:
-          node-version: 20.11.1
+          node-version: 20.20.0
           cache: "pnpm"
 
       # ..to avoid rate limits when pulling images
@@ -72,6 +72,17 @@ jobs:
         if: ${{ !env.DOCKERHUB_USERNAME }}
         run: echo "DockerHub login skipped because secrets are not available."
 
+      - name: 🐳 Pre-pull testcontainer images
+        if: ${{ env.DOCKERHUB_USERNAME }}
+        run: |
+          echo "Pre-pulling Docker images with authenticated session..."
+          docker pull postgres:14
+          docker pull clickhouse/clickhouse-server:25.4-alpine
+          docker pull redis:7-alpine
+          docker pull testcontainers/ryuk:0.11.0
+          docker pull electricsql/electric:1.2.4
+          echo "Image pre-pull complete"
+
       - name: 📥 Download deps
         run: pnpm install --frozen-lockfile
 
@@ -111,12 +122,12 @@ jobs:
       - name: ⎔ Setup pnpm
         uses: pnpm/action-setup@v4
         with:
-          version: 8.15.5
+          version: 10.23.0
 
       - name: ⎔ Setup node
         uses: buildjet/setup-node@v4
         with:
-          node-version: 20.11.1
+          node-version: 20.20.0
           # no cache enabled, we're not installing deps
 
       - name: Download blob reports from GitHub Actions Artifacts

.github/workflows/unit-tests-packages.yml

@@ -53,12 +53,12 @@ jobs:
       - name: ⎔ Setup pnpm
         uses: pnpm/action-setup@v4
         with:
-          version: 8.15.5
+          version: 10.23.0
 
       - name: ⎔ Setup node
         uses: buildjet/setup-node@v4
         with:
-          node-version: 20.11.1
+          node-version: 20.20.0
           cache: "pnpm"
 
       # ..to avoid rate limits when pulling images
@@ -72,6 +72,17 @@ jobs:
         if: ${{ !env.DOCKERHUB_USERNAME }}
         run: echo "DockerHub login skipped because secrets are not available."
 
+      - name: 🐳 Pre-pull testcontainer images
+        if: ${{ env.DOCKERHUB_USERNAME }}
+        run: |
+          echo "Pre-pulling Docker images with authenticated session..."
+          docker pull postgres:14
+          docker pull clickhouse/clickhouse-server:25.4-alpine
+          docker pull redis:7-alpine
+          docker pull testcontainers/ryuk:0.11.0
+          docker pull electricsql/electric:1.2.4
+          echo "Image pre-pull complete"
+
       - name: 📥 Download deps
         run: pnpm install --frozen-lockfile
 
@@ -111,12 +122,12 @@ jobs:
       - name: ⎔ Setup pnpm
         uses: pnpm/action-setup@v4
         with:
-          version: 8.15.5
+          version: 10.23.0
 
       - name: ⎔ Setup node
         uses: buildjet/setup-node@v4
         with:
-          node-version: 20.11.1
+          node-version: 20.20.0
           # no cache enabled, we're not installing deps
 
       - name: Download blob reports from GitHub Actions Artifacts

.github/workflows/unit-tests-webapp.yml

@@ -53,12 +53,12 @@ jobs:
       - name: ⎔ Setup pnpm
         uses: pnpm/action-setup@v4
         with:
-          version: 8.15.5
+          version: 10.23.0
 
       - name: ⎔ Setup node
         uses: buildjet/setup-node@v4
         with:
-          node-version: 20.11.1
+          node-version: 20.20.0
           cache: "pnpm"
 
       # ..to avoid rate limits when pulling images
@@ -72,6 +72,17 @@ jobs:
         if: ${{ !env.DOCKERHUB_USERNAME }}
         run: echo "DockerHub login skipped because secrets are not available."
 
+      - name: 🐳 Pre-pull testcontainer images
+        if: ${{ env.DOCKERHUB_USERNAME }}
+        run: |
+          echo "Pre-pulling Docker images with authenticated session..."
+          docker pull postgres:14
+          docker pull clickhouse/clickhouse-server:25.4-alpine
+          docker pull redis:7-alpine
+          docker pull testcontainers/ryuk:0.11.0
+          docker pull electricsql/electric:1.2.4
+          echo "Image pre-pull complete"
+
       - name: 📥 Download deps
         run: pnpm install --frozen-lockfile
 
@@ -85,7 +96,7 @@ jobs:
           DIRECT_URL: postgresql://postgres:postgres@localhost:5432/postgres
           SESSION_SECRET: "secret"
           MAGIC_LINK_SECRET: "secret"
-          ENCRYPTION_KEY: "secret"
+          ENCRYPTION_KEY: "dummy-encryption-keeeey-32-bytes"
           DEPLOY_REGISTRY_HOST: "docker.io"
           CLICKHOUSE_URL: "http://default:password@localhost:8123"
 
@@ -119,12 +130,12 @@ jobs:
       - name: ⎔ Setup pnpm
         uses: pnpm/action-setup@v4
         with:
-          version: 8.15.5
+          version: 10.23.0
 
       - name: ⎔ Setup node
         uses: buildjet/setup-node@v4
         with:
-          node-version: 20.11.1
+          node-version: 20.20.0
           # no cache enabled, we're not installing deps
 
       - name: Download blob reports from GitHub Actions Artifacts

.github/workflows/vouch-check-pr.yml

@@ -0,0 +1,44 @@
+name: Vouch - Check PR
+
+on:
+  pull_request_target:
+    types: [opened, reopened]
+
+permissions:
+  contents: read
+  pull-requests: write
+  issues: read
+
+jobs:
+  check-vouch:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: mitchellh/vouch/action/check-pr@c6d80ead49839655b61b422700b7a3bc9d0804a9 # v1.4.2
+        with:
+          pr-number: ${{ github.event.pull_request.number }}
+          auto-close: true
+          require-vouch: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+  require-draft:
+    needs: check-vouch
+    if: >
+      github.event.pull_request.draft == false &&
+      github.event.pull_request.author_association != 'MEMBER' &&
+      github.event.pull_request.author_association != 'OWNER' &&
+      github.event.pull_request.author_association != 'COLLABORATOR'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Close non-draft PR
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          STATE=$(gh pr view ${{ github.event.pull_request.number }} --repo ${{ github.repository }} --json state -q '.state')
+          if [ "$STATE" != "OPEN" ]; then
+            echo "PR is already closed, skipping."
+            exit 0
+          fi
+          gh pr close ${{ github.event.pull_request.number }} \
+            --repo ${{ github.repository }} \
+            --comment "Thanks for your contribution! We require all external PRs to be opened in **draft** status first so you can address CodeRabbit review comments and ensure CI passes before requesting a review. Please re-open this PR as a draft. See [CONTRIBUTING.md](https://github.com/${{ github.repository }}/blob/main/CONTRIBUTING.md#pr-workflow) for details."

.github/workflows/vouch-manage-by-issue.yml

@@ -0,0 +1,24 @@
+name: Vouch - Manage by Issue
+
+on:
+  issue_comment:
+    types: [created]
+
+permissions:
+  contents: write
+  issues: write
+
+jobs:
+  manage:
+    runs-on: ubuntu-latest
+    if: >-
+      contains(github.event.comment.body, 'vouch') ||
+      contains(github.event.comment.body, 'denounce') ||
+      contains(github.event.comment.body, 'unvouch')
+    steps:
+      - uses: mitchellh/vouch/action/manage-by-issue@c6d80ead49839655b61b422700b7a3bc9d0804a9 # v1.4.2
+        with:
+          comment-id: ${{ github.event.comment.id }}
+          issue-id: ${{ github.event.issue.number }}
+        env:
+          GH_TOKEN: ${{ github.token }}

.gitignore

@@ -15,6 +15,9 @@ out/
 dist
 packages/**/dist
 
+# vendored bundles (generated during build)
+packages/**/src/**/vendor
+
 # Tailwind
 apps/**/styles/tailwind.css
 packages/**/styles/tailwind.css
@@ -29,12 +32,10 @@ yarn-debug.log*
 yarn-error.log*
 
 # local env files
-.env.docker
+.env
+.env.*
 .docker/*.env
-.env.local
-.env.development.local
-.env.test.local
-.env.production.local
+!.env.example
 
 # turbo
 .turbo
@@ -63,4 +64,8 @@ apps/**/public/build
 /packages/core/src/package.json
 /packages/trigger-sdk/src/package.json
 /packages/python/src/package.json
-.claude
+**/.claude/settings.local.json
+.mcp.log
+.mcp.json
+.cursor/debug.log
+ailogger-output.log

.npmrc

@@ -1,5 +0,0 @@
-link-workspace-packages=false
-public-hoist-pattern[]=*prisma*
-prefer-workspace-packages=true
-update-notifier=false
-side-effects-cache=false

.nvmrc

@@ -1 +1 @@
-v20.11.1
+v20.20.0

.server-changes/README.md

@@ -0,0 +1,81 @@
+# Server Changes
+
+This directory tracks changes to server-only components (webapp, supervisor, coordinator, etc.) that are not captured by changesets. Changesets only track published npm packages — server changes would otherwise go undocumented.
+
+## When to add a file
+
+**Server-only PRs**: If your PR only changes `apps/webapp/`, `apps/supervisor/`, `apps/coordinator/`, or other server components (and does NOT change anything in `packages/`), add a `.server-changes/` file.
+
+**Mixed PRs** (both packages and server): Just add a changeset as usual. No `.server-changes/` file needed — the changeset covers it.
+
+**Package-only PRs**: Just add a changeset as usual.
+
+## File format
+
+Create a markdown file with a descriptive name:
+
+```
+.server-changes/fix-batch-queue-stalls.md
+```
+
+With this format:
+
+```markdown
+---
+area: webapp
+type: fix
+---
+
+Speed up batch queue processing by removing stalls and fixing retry race
+```
+
+### Fields
+
+- **area** (required): `webapp` | `supervisor` | `coordinator` | `kubernetes-provider` | `docker-provider`
+- **type** (required): `feature` | `fix` | `improvement` | `breaking`
+
+### Description
+
+The body text (below the frontmatter) is a one-line description of the change. Keep it concise — it will appear in release notes.
+
+## Lifecycle
+
+1. Engineer adds a `.server-changes/` file in their PR
+2. Files accumulate on `main` as PRs merge
+3. The changeset release PR includes these in its summary
+4. After the release merges, CI cleans up the consumed files
+
+## Examples
+
+**New feature:**
+
+```markdown
+---
+area: webapp
+type: feature
+---
+
+TRQL query language and the Query page
+```
+
+**Bug fix:**
+
+```markdown
+---
+area: webapp
+type: fix
+---
+
+Fix schedule limit counting for orgs with custom limits
+```
+
+**Improvement:**
+
+```markdown
+---
+area: webapp
+type: improvement
+---
+
+Use the replica for API auth queries to reduce primary load
+```

.server-changes/ai-prompt-management.md

@@ -0,0 +1,30 @@
+---
+area: webapp
+type: feature
+---
+
+AI prompt management dashboard and enhanced span inspectors.
+
+**Prompt management:**
+- Prompts list page with version status, model, override indicators, and 24h usage sparklines
+- Prompt detail page with template viewer, variable preview, version history timeline, and override editor
+- Create, edit, and remove overrides to change prompt content or model without redeploying
+- Promote any code-deployed version to current
+- Generations tab with infinite scroll, live polling, and inline span inspector
+- Per-prompt metrics: total generations, avg tokens, avg cost, latency, with version-level breakdowns
+
+**AI span inspectors:**
+- Custom inspectors for `ai.generateText`, `ai.streamText`, `ai.generateObject`, `ai.streamObject` parent spans
+- `ai.toolCall` inspector showing tool name, call ID, and input arguments
+- `ai.embed` inspector showing model, provider, and input text
+- Prompt tab on AI spans linking to prompt version with template and input variables
+- Compact timestamp and duration header on all AI span inspectors
+
+**AI metrics dashboard:**
+- Operations, Providers, and Prompts filters on the AI Metrics dashboard
+- Cost by prompt widget
+- "AI" section in the sidebar with Prompts and AI Metrics links
+
+**Other improvements:**
+- Resizable panel sizes now persist across page refreshes
+- Fixed `<div>` inside `<p>` DOM nesting warnings in span titles and chat messages

.server-changes/allow-rollbacks-promote-api.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: feature
+---
+
+Add allowRollbacks query param to the promote deployment API to enable version downgrades

.server-changes/ck-index-master-queue-dedup.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: fix
+---
+
+Concurrency-keyed queues now use a single master queue entry per base queue instead of one entry per key. Prevents high-CK-count tenants from consuming the entire parentQueueLimit window and starving other tenants on the same shard.

.server-changes/dev-cli-disconnect-md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: feature
+---
+
+Added `/engine/v1/dev/disconnect` endpoint to auto-cancel runs when the CLI disconnects. Maximum of 500 runs can be cancelled. Uses the bulk action system when there are more than 25 runs to cancel.

.server-changes/fix-batch-waitpoint-lock-contention.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: fix
+---
+
+Reduce lock contention when processing large `batchTriggerAndWait` batches. Previously, each batch item acquired a Redis lock on the parent run to insert a `TaskRunWaitpoint` row, causing `LockAcquisitionTimeoutError` with high concurrency (880 errors/24h in prod). Since `blockRunWithCreatedBatch` already transitions the parent to `EXECUTING_WITH_WAITPOINTS` before items are processed, the per-item lock is unnecessary. The new `blockRunWithWaitpointLockless` method performs only the idempotent CTE insert without acquiring the lock.

.server-changes/fix-clickhouse-query-client-secure-param.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: fix
+---
+
+Strip `secure` query parameter from QUERY_CLICKHOUSE_URL before passing to ClickHouse client. This was already done for the main and logs ClickHouse clients but was missing for the query client, causing a startup crash with `Error: Unknown URL parameters: secure`.

.server-changes/llm-cost-tracking.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: feature
+---
+
+Add automatic LLM cost calculation for spans with GenAI semantic conventions. When a span arrives with `gen_ai.response.model` and token usage data, costs are calculated from an in-memory pricing registry backed by Postgres and dual-written to both span attributes (`trigger.llm.*`) and a new `llm_metrics_v1` ClickHouse table that captures usage, cost, performance (TTFC, tokens/sec), and behavioral (finish reason, operation type) metrics.

.server-changes/mcp-get-span-details.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: feature
+---
+
+Add API endpoint `GET /api/v1/runs/:runId/spans/:spanId` that returns detailed span information including properties, events, AI enrichment (model, tokens, cost), and triggered child runs.

.vouch.yml

@@ -0,0 +1,2 @@
+vouch:
+  - github: edosrecki

.vscode/launch.json

@@ -31,6 +31,15 @@
       "cwd": "${workspaceFolder}/apps/webapp",
       "sourceMaps": true
     },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug opened test file",
+      "command": "pnpm run test -- ./${relativeFile}",
+      "envFile": "${workspaceFolder}/.env",
+      "cwd": "${workspaceFolder}",
+      "sourceMaps": true
+    },
     {
       "type": "chrome",
       "request": "launch",
@@ -59,7 +68,7 @@
       "request": "launch",
       "name": "Debug V3 Dev CLI",
       "command": "pnpm exec trigger dev",
-      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "cwd": "${workspaceFolder}/references/hello-world",
       "sourceMaps": true
     },
     {
@@ -83,7 +92,7 @@
       "request": "launch",
       "name": "Debug V3 Deploy CLI",
       "command": "pnpm exec trigger deploy --self-hosted --load-image",
-      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "cwd": "${workspaceFolder}/references/hello-world",
       "sourceMaps": true
     },
     {
@@ -91,7 +100,7 @@
       "request": "launch",
       "name": "Debug V3 list-profiles CLI",
       "command": "pnpm exec trigger list-profiles --log-level debug",
-      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "cwd": "${workspaceFolder}/references/hello-world",
       "sourceMaps": true
     },
     {
@@ -99,7 +108,7 @@
       "request": "launch",
       "name": "Debug V3 update CLI",
       "command": "pnpm exec trigger update",
-      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "cwd": "${workspaceFolder}/references/hello-world",
       "sourceMaps": true
     },
     {
@@ -107,7 +116,7 @@
       "request": "launch",
       "name": "Debug V3 Management",
       "command": "pnpm run management",
-      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "cwd": "${workspaceFolder}/references/hello-world",
       "sourceMaps": true
     },
     {

.vscode/settings.json

@@ -6,5 +6,6 @@
     "**/node_modules/**": true,
     "packages/cli-v3/e2e": true
   },
-  "vitest.disableWorkspaceWarning": true
+  "vitest.disableWorkspaceWarning": true,
+  "chat.agent.maxRequests": 10000
 }

AGENTS.md

@@ -13,7 +13,7 @@ This repository is a pnpm monorepo managed with Turbo. It contains multiple apps
 See `ai/references/repo.md` for a more complete explanation of the workspaces.
 
 ## Development setup
-1. Install dependencies with `pnpm i` (pnpm `8.15.5` and Node.js `20.11.1` are required).
+1. Install dependencies with `pnpm i` (pnpm `10.23.0` and Node.js `20.20.0` are required).
 2. Copy `.env.example` to `.env` and generate a random 16 byte hex string for `ENCRYPTION_KEY` (`openssl rand -hex 16`). Update other secrets if needed.
 3. Start the local services with Docker:
    ```bash

CHANGESETS.md

@@ -1,24 +1,49 @@
-# Changesets
+# Changesets and Server Changes
 
-Trigger.dev uses [changesets](https://github.com/changesets/changesets) to manage updated our packages and releasing them to npm.
+Trigger.dev uses [changesets](https://github.com/changesets/changesets) to manage package versions and releasing them to npm. For server-only changes, we use a lightweight `.server-changes/` convention.
 
-## Adding a changeset
+## Adding a changeset (package changes)
 
 To add a changeset, use `pnpm run changeset:add` and follow the instructions [here](https://github.com/changesets/changesets/blob/main/docs/adding-a-changeset.md). Please only ever select one of our public packages when adding a changeset.
 
-## Release instructions (local only)
+## Adding a server change (server-only changes)
 
-Based on the instructions [here](https://github.com/changesets/changesets/blob/main/docs/intro-to-using-changesets.md)
+If your PR only changes server components (`apps/webapp/`, `apps/supervisor/`, etc.) and does NOT change any published packages, add a `.server-changes/` file instead of a changeset:
 
-1. Run `pnpm run changeset:version`
-2. Run `pnpm run changeset:release`
+```sh
+cat > .server-changes/fix-batch-queue-stalls.md << 'EOF'
+---
+area: webapp
+type: fix
+---
+
+Speed up batch queue processing by removing stalls and fixing retry race
+EOF
+```
+
+- `area`: `webapp` | `supervisor` | `coordinator` | `kubernetes-provider` | `docker-provider`
+- `type`: `feature` | `fix` | `improvement` | `breaking`
+
+For **mixed PRs** (both packages and server): just add a changeset. No `.server-changes/` file needed.
+
+See `.server-changes/README.md` for full documentation.
+
+## When to add which
+
+| PR changes | What to add |
+|---|---|
+| Only packages (`packages/`) | Changeset (`pnpm run changeset:add`) |
+| Only server (`apps/`) | `.server-changes/` file |
+| Both packages and server | Just the changeset |
 
 ## Release instructions (CI)
 
 Please follow the best-practice of adding changesets in the same commit as the code making the change with `pnpm run changeset:add`, as it will allow our release.yml CI workflow to function properly:
 
-- Anytime new changesets are added in a commit in the `main` branch, the [release.yml](./.github/workflows/release.yml) workflow will run and will automatically create/update a PR with a fresh run of `pnpm run changeset:version`.
-- When the version PR is merged into `main`, the release.yml workflow will automatically run `pnpm run changeset:release` to build and release packages to npm.
+- Anytime new changesets are added in a commit in the `main` branch, the [changesets-pr.yml](./.github/workflows/changesets-pr.yml) workflow will run and will automatically create/update a PR with a fresh run of `pnpm run changeset:version`.
+- The release PR body is automatically enhanced with a clean, deduplicated summary that includes both package changes and `.server-changes/` entries.
+- Consumed `.server-changes/` files are removed on the `changeset-release/main` branch — the same way changesets deletes `.changeset/*.md` files. When the release PR merges, they're gone from main.
+- When the version PR is merged into `main`, the [release.yml](./.github/workflows/release.yml) workflow will automatically build, release packages to npm, and create a single unified GitHub release.
 
 ## Pre-release instructions
 
@@ -30,14 +55,16 @@ Please follow the best-practice of adding changesets in the same commit as the c
 
 ## Snapshot instructions
 
-1. Delete the `.changeset/pre.json` file (if it exists)
+1. Update the `.changeset/config.json` file to set the `"changelog"` field to this:
+
+```json
+"changelog": "@changesets/cli/changelog",
+```
 
 2. Do a temporary commit (do NOT push this, you should undo it after)
 
-3. Copy the `GITHUB_TOKEN` line from the .env file
+3. Run `./scripts/publish-prerelease.sh prerelease`
 
-4. Run `GITHUB_TOKEN=github_pat_12345 ./scripts/publish-prerelease.sh re2`
+You can choose a different tag if you want, but usually `prerelease` is fine.
 
-Make sure to replace the token with yours. `re2` is the tag that will be used for the pre-release.
-
-5. Undo the commit where you deleted the pre.json file.
+5. Undo the commit where you updated the config.json file.

CLAUDE.md

@@ -0,0 +1,243 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code when working with this repository. Subdirectory CLAUDE.md files provide deeper context when you navigate into specific areas.
+
+## Build and Development Commands
+
+This is a pnpm 10.23.0 monorepo using Turborepo. Run commands from root with `pnpm run`.
+
+```bash
+pnpm run docker              # Start Docker services (PostgreSQL, Redis, Electric)
+pnpm run db:migrate           # Run database migrations
+pnpm run db:seed              # Seed the database (required for reference projects)
+
+# Build packages (required before running)
+pnpm run build --filter webapp && pnpm run build --filter trigger.dev && pnpm run build --filter @trigger.dev/sdk
+
+pnpm run dev --filter webapp  # Run webapp (http://localhost:3030)
+pnpm run dev --filter trigger.dev --filter "@trigger.dev/*"  # Watch CLI and packages
+```
+
+### Verifying Changes
+
+The verification command depends on where the change lives:
+
+- **Apps and internal packages** (`apps/*`, `internal-packages/*`): Use `typecheck`. **Never use `build`** for these — building proves almost nothing about correctness.
+- **Public packages** (`packages/*`): Use `build`.
+
+```bash
+# Apps and internal packages — use typecheck
+pnpm run typecheck --filter webapp                  # ~1-2 minutes
+pnpm run typecheck --filter @internal/run-engine
+
+# Public packages — use build
+pnpm run build --filter @trigger.dev/sdk
+pnpm run build --filter @trigger.dev/core
+```
+
+Only run typecheck/build after major changes (new files, significant refactors, schema changes). For small edits, trust the types and let CI catch issues.
+
+## Testing
+
+We use vitest exclusively. **Never mock anything** - use testcontainers instead.
+
+```bash
+pnpm run test --filter webapp                          # All tests for a package
+cd internal-packages/run-engine
+pnpm run test ./src/engine/tests/ttl.test.ts --run     # Single test file
+pnpm run build --filter @internal/run-engine           # May need to build deps first
+```
+
+Test files go next to source files (e.g., `MyService.ts` -> `MyService.test.ts`).
+
+### Testcontainers for Redis/PostgreSQL
+
+```typescript
+import { redisTest, postgresTest, containerTest } from "@internal/testcontainers";
+
+redisTest("should use redis", async ({ redisOptions }) => {
+  /* ... */
+});
+postgresTest("should use postgres", async ({ prisma }) => {
+  /* ... */
+});
+containerTest("should use both", async ({ prisma, redisOptions }) => {
+  /* ... */
+});
+```
+
+## Changesets and Server Changes
+
+When modifying any public package (`packages/*` or `integrations/*`), add a changeset:
+
+```bash
+pnpm run changeset:add
+```
+
+- Default to **patch** for bug fixes and minor changes
+- Confirm with maintainers before selecting **minor** (new features)
+- **Never** select major without explicit approval
+
+When modifying only server components (`apps/webapp/`, `apps/supervisor/`, etc.) with no package changes, add a `.server-changes/` file instead. See `.server-changes/README.md` for format and documentation.
+
+## Architecture Overview
+
+### Request Flow
+
+User API call -> Webapp routes -> Services -> RunEngine -> Redis Queue -> Supervisor -> Container execution -> Results back through RunEngine -> ClickHouse (analytics) + PostgreSQL (state)
+
+### Apps
+
+- **apps/webapp**: Remix 2.1.0 app - main API, dashboard, orchestration. Uses Express server.
+- **apps/supervisor**: Manages task execution containers (Docker/Kubernetes).
+
+### Public Packages
+
+- **packages/trigger-sdk** (`@trigger.dev/sdk`): Main SDK for writing tasks
+- **packages/cli-v3** (`trigger.dev`): CLI - also bundles code that goes into customer task images
+- **packages/core** (`@trigger.dev/core`): Shared types. **Import subpaths only** (never root).
+- **packages/build** (`@trigger.dev/build`): Build extensions and types
+- **packages/react-hooks**: React hooks for realtime and triggering
+- **packages/redis-worker** (`@trigger.dev/redis-worker`): Redis-based background job system
+
+### Internal Packages
+
+- **internal-packages/database**: Prisma 6.14.0 client and schema (PostgreSQL)
+- **internal-packages/clickhouse**: ClickHouse client, schema migrations, analytics queries
+- **internal-packages/run-engine**: "Run Engine 2.0" - core run lifecycle management
+- **internal-packages/redis**: Redis client creation utilities (ioredis)
+- **internal-packages/testcontainers**: Test helpers for Redis/PostgreSQL containers
+- **internal-packages/schedule-engine**: Durable cron scheduling
+- **internal-packages/zodworker**: Graphile-worker wrapper (DEPRECATED - use redis-worker)
+
+### Legacy V1 Engine Code
+
+The `apps/webapp/app/v3/` directory name is misleading - most code there is actively used by V2. Only specific files are V1-only legacy (MarQS queue, triggerTaskV1, cancelTaskRunV1, etc.). See `apps/webapp/CLAUDE.md` for the exact list. When you encounter V1/V2 branching in services, only modify V2 code paths. All new work uses Run Engine 2.0 (`@internal/run-engine`) and redis-worker.
+
+### Documentation
+
+Docs live in `docs/` as a Mintlify site (MDX format). See `docs/CLAUDE.md` for conventions.
+
+### Reference Projects
+
+The `references/` directory contains test workspaces for testing SDK and platform features. Use `references/hello-world` to manually test changes before submitting PRs.
+
+## Docker Image Guidelines
+
+When updating Docker image references:
+
+- **Always use multiplatform/index digests**, not architecture-specific digests
+- Architecture-specific digests cause CI failures on different build environments
+- Use the digest from the main Docker Hub page, not from a specific OS/ARCH variant
+
+## Writing Trigger.dev Tasks
+
+Always import from `@trigger.dev/sdk`. Never use `@trigger.dev/sdk/v3` or deprecated `client.defineJob`.
+
+```typescript
+import { task } from "@trigger.dev/sdk";
+
+export const myTask = task({
+  id: "my-task",
+  run: async (payload: { message: string }) => {
+    // Task logic
+  },
+});
+```
+
+### SDK Documentation Rules
+
+The `rules/` directory contains versioned SDK documentation distributed via the SDK installer. Current version: `rules/manifest.json`. Do NOT update `rules/` or `.claude/skills/trigger-dev-tasks/` unless explicitly asked - these are maintained in separate dedicated passes.
+
+## Testing with hello-world Reference Project
+
+First-time setup:
+
+1. `pnpm run db:seed` to seed the database
+2. Build CLI: `pnpm run build --filter trigger.dev && pnpm i`
+3. Authorize: `cd references/hello-world && pnpm exec trigger login -a http://localhost:3030`
+
+Running: `cd references/hello-world && pnpm exec trigger dev`
+
+## Local Task Testing Workflow
+
+### Step 1: Start Webapp in Background
+
+```bash
+# Run from repo root with run_in_background: true
+pnpm run dev --filter webapp
+curl -s http://localhost:3030/healthcheck  # Verify running
+```
+
+### Step 2: Start Trigger Dev in Background
+
+```bash
+cd references/hello-world && pnpm exec trigger dev
+# Wait for "Local worker ready [node]"
+```
+
+### Step 3: Trigger and Monitor Tasks via MCP
+
+```
+mcp__trigger__get_current_worker(projectRef: "proj_rrkpdguyagvsoktglnod", environment: "dev")
+mcp__trigger__trigger_task(projectRef: "proj_rrkpdguyagvsoktglnod", environment: "dev", taskId: "hello-world", payload: {"message": "Hello"})
+mcp__trigger__list_runs(projectRef: "proj_rrkpdguyagvsoktglnod", environment: "dev", taskIdentifier: "hello-world", limit: 5)
+```
+
+Dashboard: http://localhost:3030/orgs/references-9dfd/projects/hello-world-97DT/env/dev/runs
+
+<!-- intent-skills:start -->
+
+# Skill mappings — when working in these areas, load the linked skill file into context.
+
+skills:
+
+- task: "Using agentcrumbs for debug tracing, adding crumbs, trails, markers, querying traces, or stripping debug code before merge"
+  load: "node_modules/agentcrumbs/skills/agentcrumbs/SKILL.md"
+- task: "Setting up agentcrumbs in the project, initializing namespace catalog, running crumbs init"
+load: "node_modules/agentcrumbs/skills/agentcrumbs/init/SKILL.md"
+<!-- intent-skills:end -->
+
+## agentcrumbs
+
+Add crumbs as you write code — not just when debugging. Mark lines with
+`// @crumbs` or wrap blocks in `// #region @crumbs`. They stay on the
+branch throughout development and are stripped by `agentcrumbs strip`
+before merge.
+
+### Namespaces
+
+| Namespace         | Description                                    | Path                                |
+| ----------------- | ---------------------------------------------- | ----------------------------------- |
+| `webapp`          | Main Remix app — API, dashboard, orchestration | `apps/webapp`                       |
+| `supervisor`      | Task execution container manager               | `apps/supervisor`                   |
+| `sdk`             | Main SDK for writing tasks                     | `packages/trigger-sdk`              |
+| `cli`             | CLI and task image bundling                    | `packages/cli-v3`                   |
+| `core`            | Shared types and subpath exports               | `packages/core`                     |
+| `build`           | Build extensions and types                     | `packages/build`                    |
+| `react-hooks`     | React hooks for realtime/triggering            | `packages/react-hooks`              |
+| `redis-worker`    | Redis-based background job system              | `packages/redis-worker`             |
+| `run-engine`      | Run Engine 2.0 — core run lifecycle            | `internal-packages/run-engine`      |
+| `database`        | Prisma client and schema                       | `internal-packages/database`        |
+| `clickhouse`      | ClickHouse client, migrations, analytics       | `internal-packages/clickhouse`      |
+| `schedule-engine` | Durable cron scheduling                        | `internal-packages/schedule-engine` |
+| `run-queue`       | Run queue management                           | `internal-packages/run-queue`       |
+
+Do not invent new namespaces — pick from this table or ask first.
+
+### For PR reviewers
+
+Do not flag `// @crumbs` markers, `#region @crumbs` blocks, or agentcrumbs
+imports in reviews. These are temporary debug instrumentation stripped
+before merge. Data logged in crumbs (IDs, names, values) never reaches
+production.
+
+### CLI
+
+```bash
+pnpm exec agentcrumbs collect    # start collector (multi-service)
+pnpm exec agentcrumbs tail --app trigger       # live tail
+pnpm exec agentcrumbs clear --app trigger      # remove crumbs before merge
+```
+
+The preferred way to query for crumbs is to use `pnpm exec agentcrumbs query --app trigger` with the `--limit` option and cursor pagination, and clear existing crumbs before reproducing a bug via `pnpm exec agentcrumbs clear --app trigger`.

CONTRIBUTING.md

@@ -2,10 +2,25 @@
 
 Thank you for taking the time to contribute to Trigger.dev. Your involvement is not just welcomed, but we encourage it! 🚀
 
-Please take some time to read this guide to understand contributing best practices for Trigger.dev.
+Please take some time to read this guide to understand contributing best practices for Trigger.dev. Note that we use [vouch](https://github.com/mitchellh/vouch) to manage contributor trust, so you'll need to be vouched before opening a PR.
 
 Thank you for helping us make Trigger.dev even better! 🤩
 
+> **Important:** We only accept PRs that address a single issue. Please do not submit PRs containing multiple unrelated fixes or features. If you have multiple contributions, open a separate PR for each one.
+
+## Getting vouched (required before opening a PR)
+
+We use [vouch](https://github.com/mitchellh/vouch) to manage contributor trust. **PRs from unvouched users are automatically closed.**
+
+Before you open your first pull request, you need to be vouched by a maintainer. Here's how:
+
+1. Open a [Vouch Request](https://github.com/triggerdotdev/trigger.dev/issues/new?template=vouch-request.yml) issue.
+2. Tell us what you'd like to work on and share any relevant background.
+3. A maintainer will review your request and vouch for you by commenting on the issue.
+4. Once vouched, your PRs will be accepted normally.
+
+If you're unsure whether you're already vouched, go ahead and open a PR — the check will tell you.
+
 ## Developing
 
 The development branch is `main`. This is the branch that all pull
@@ -14,8 +29,8 @@ branch are tagged into a release periodically.
 
 ### Prerequisites
 
-- [Node.js](https://nodejs.org/en) version 20.11.1
-- [pnpm package manager](https://pnpm.io/installation) version 8.15.5
+- [Node.js](https://nodejs.org/en) version 20.20.0
+- [pnpm package manager](https://pnpm.io/installation) version 10.23.0
 - [Docker](https://www.docker.com/get-started/)
 - [protobuf](https://github.com/protocolbuffers/protobuf)
 
@@ -34,9 +49,9 @@ branch are tagged into a release periodically.
    ```
    cd trigger.dev
    ```
-3. Ensure you are on the correct version of Node.js (20.11.1). If you are using `nvm`, there is an `.nvmrc` file that will automatically select the correct version of Node.js when you navigate to the repository.
+3. Ensure you are on the correct version of Node.js (20.20.0). If you are using `nvm`, there is an `.nvmrc` file that will automatically select the correct version of Node.js when you navigate to the repository.
 
-4. Run `corepack enable` to use the correct version of pnpm (`8.15.5`) as specified in the root `package.json` file.
+4. Run `corepack enable` to use the correct version of pnpm (`10.23.0`) as specified in the root `package.json` file.
 
 5. Install the required packages using pnpm.
    ```
@@ -84,17 +99,17 @@ branch are tagged into a release periodically.
 
 2. Once the app is running click the magic link button and enter your email. You will automatically be logged in, since you are running locally. Create an Org and your first project in the dashboard.
 
-## Manual testing using v3-catalog
+## Manual testing using hello-world
 
-We use the `<root>/references/v3-catalog` subdirectory as a staging ground for testing changes to the SDK (`@trigger.dev/sdk` at `<root>/packages/trigger-sdk`), the Core package (`@trigger.dev/core` at `<root>packages/core`), the CLI (`trigger.dev` at `<root>/packages/cli-v3`) and the platform (The remix app at `<root>/apps/webapp`). The instructions below will get you started on using the `v3-catalog` for local development of Trigger.dev (v3).
+We use the `<root>/references/hello-world` subdirectory as a staging ground for testing changes to the SDK (`@trigger.dev/sdk` at `<root>/packages/trigger-sdk`), the Core package (`@trigger.dev/core` at `<root>packages/core`), the CLI (`trigger.dev` at `<root>/packages/cli-v3`) and the platform (The remix app at `<root>/apps/webapp`). The instructions below will get you started on using the `hello-world` for local development of Trigger.dev.
 
 ### First-time setup
 
 First, make sure you are running the webapp according to the instructions above. Then:
 
-1. Visit http://localhost:3030 in your browser and create a new V3 project called "v3-catalog".
+1. Visit http://localhost:3030 in your browser and create a new project called "hello-world".
 
-2. In Postgres go to the "Projects" table and for the project you create change the `externalRef` to `yubjwjsfkxnylobaqvqz`.
+2. In Postgres go to the "Projects" table and for the project you create change the `externalRef` to `proj_rrkpdguyagvsoktglnod`.
 
 3. Build the CLI
 
@@ -105,10 +120,10 @@ pnpm run build --filter trigger.dev
 pnpm i
 ```
 
-4. Change into the `<root>/references/v3-catalog` directory and authorize the CLI to the local server:
+4. Change into the `<root>/references/hello-world` directory and authorize the CLI to the local server:
 
 ```sh
-cd references/v3-catalog
+cd references/hello-world
 cp .env.example .env
 pnpm exec trigger login -a http://localhost:3030
 ```
@@ -118,7 +133,7 @@ This will open a new browser window and authorize the CLI against your local use
 You can optionally pass a `--profile` flag to the `login` command, which will allow you to use the CLI with separate accounts/servers. We suggest using a profile called `local` for your local development:
 
 ```sh
-cd references/v3-catalog
+cd references/hello-world
 pnpm exec trigger login -a http://localhost:3030 --profile local
 # later when you run the dev or deploy command:
 pnpm exec trigger dev --profile local
@@ -127,7 +142,7 @@ pnpm exec trigger deploy --profile local
 
 ### Running
 
-The following steps should be followed any time you start working on a new feature you want to test in v3:
+The following steps should be followed any time you start working on a new feature you want to test:
 
 1. Make sure the webapp is running on localhost:3030
 
@@ -137,84 +152,29 @@ The following steps should be followed any time you start working on a new featu
 pnpm run dev --filter trigger.dev --filter "@trigger.dev/*"
 ```
 
-3. Open another terminal window, and change into the `<root>/references/v3-catalog` directory.
+3. Open another terminal window, and change into the `<root>/references/hello-world` directory.
 
-4. You'll need to run the following commands to setup prisma and migrate the database:
+4. Run the `dev` command, which will register all the local tasks with the platform and allow you to start testing task execution:
 
 ```sh
-pnpm exec prisma migrate deploy
-pnpm run generate:prisma
-```
-
-5. Run the `dev` command, which will register all the local tasks with the platform and allow you to start testing task execution:
-
-```sh
-# in <root>/references/v3-catalog
+# in <root>/references/hello-world
 pnpm exec trigger dev
 ```
 
 If you want additional debug logging, you can use the `--log-level debug` flag:
 
 ```sh
-# in <root>/references/v3-catalog
+# in <root>/references/hello-world
 pnpm exec trigger dev --log-level debug
 ```
 
-6. If you make any changes in the CLI/Core/SDK, you'll need to `CTRL+C` to exit the `dev` command and restart it to pickup changes. Any changes to the files inside of the `v3-catalog/src/trigger` dir will automatically be rebuilt by the `dev` command.
+6. If you make any changes in the CLI/Core/SDK, you'll need to `CTRL+C` to exit the `dev` command and restart it to pickup changes. Any changes to the files inside of the `hello-world/src/trigger` dir will automatically be rebuilt by the `dev` command.
 
-7. Navigate to the `v3-catalog` project in your local dashboard at localhost:3030 and you should see the list of tasks.
+7. Navigate to the `hello-world` project in your local dashboard at localhost:3030 and you should see the list of tasks.
 
-8. Go to the "Test" page in the sidebar and select a task. Then enter a payload and click "Run test". You can tell what the payloads should be by looking at the relevant task file inside the `/references/v3-catalog/src/trigger` folder. Many of them accept an empty payload.
+8. Go to the "Test" page in the sidebar and select a task. Then enter a payload and click "Run test". You can tell what the payloads should be by looking at the relevant task file inside the `/references/hello-world/src/trigger` folder. Many of them accept an empty payload.
 
-9. Feel free to add additional files in `v3-catalog/src/trigger` to test out specific aspects of the system, or add in edge cases.
-
-## Running end-to-end webapp tests (deprecated)
-
-To run the end-to-end tests, follow the steps below:
-
-1. Set up environment variables (copy example envs into the correct place)
-
-```sh
-cp ./.env.example ./.env
-cp ./references/nextjs-test/.env.example ./references/nextjs-test/.env.local
-```
-
-2. Set up dependencies
-
-```sh
-# Build packages
-pnpm run build --filter @references/nextjs-test^...
-pnpm --filter @trigger.dev/database generate
-
-# Move trigger-cli bin to correct place
-pnpm install --frozen-lockfile
-
-# Install playwrite browsers (ONE TIME ONLY)
-npx playwright install
-```
-
-3. Set up the database
-
-```sh
-pnpm run docker
-pnpm run db:migrate
-pnpm run db:seed
-```
-
-4. Run the end-to-end tests
-
-```sh
-pnpm run test:e2e
-```
-
-### Cleanup
-
-The end-to-end tests use a `setup` and `teardown` script to seed the database with test data. If the test runner doesn't exit cleanly, then the database can be left in a state where the tests can't run because the `setup` script will try to create data that already exists. If this happens, you can manually delete the `users` and `organizations` from the database using prisma studio:
-
-```sh
-# With the database running (i.e. pnpm run docker)
-pnpm run db:studio
-```
+9. Feel free to add additional files in `hello-world/src/trigger` to test out specific aspects of the system, or add in edge cases.
 
 ## Adding and running migrations
 
@@ -282,9 +242,23 @@ See the [Job Catalog](./references/job-catalog/README.md) file for more.
 
 **If you get errors, be sure to fix them before committing.**
 
-- Be sure to [check the "Allow edits from maintainers" option](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/allowing-changes-to-a-pull-request-branch-created-from-a-fork) while creating you PR.
-- If your PR refers to or fixes an issue, be sure to add `refs #XXX` or `fixes #XXX` to the PR description. Replacing `XXX` with the respective issue number. See more about [Linking a pull request to an issue
-  ](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue).
+> **Note:** We may close PRs if we decide that the cost of integrating the change outweighs the benefits. To improve the chances of your PR getting accepted, follow the guidelines below.
+
+### PR workflow
+
+1. **Always open your PR in draft status first.** Do not mark it as "Ready for Review" until the steps below are complete.
+2. **Address all CodeRabbit code review comments.** Our CI runs an automated code review via CodeRabbit. Go through each comment and either fix the issue or resolve it with a comment explaining why no change is needed.
+3. **Wait for all CI checks to pass.** Do not mark the PR as "Ready for Review" until every check is green.
+4. **Then mark the PR as "Ready for Review"** so a maintainer can take a look.
+
+### Cost/benefit analysis for risky changes
+
+If your change touches core infrastructure, modifies widely-used code paths, or could introduce regressions, consider doing a brief cost/benefit analysis and including it in the PR description. Explain what the benefit is to users and why the risk is worth it. This goes a long way toward helping maintainers evaluate your contribution.
+
+### General guidelines
+
+- Be sure to [check the "Allow edits from maintainers" option](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/allowing-changes-to-a-pull-request-branch-created-from-a-fork) while creating your PR.
+- If your PR refers to or fixes an issue, be sure to add `refs #XXX` or `fixes #XXX` to the PR description. Replacing `XXX` with the respective issue number. See more about [Linking a pull request to an issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue).
 - Be sure to fill the PR Template accordingly.
 
 ## Adding changesets
@@ -307,6 +281,39 @@ You will be prompted to select which packages to include in the changeset. Only
 
 Most of the time the changes you'll make are likely to be categorized as patch releases. If you feel like there is the need for a minor or major release of the package based on the changes being made, add the changeset as such and it will be discussed during PR review.
 
+## Adding server changes
+
+Changesets only track published npm packages. If your PR only changes server components (`apps/webapp/`, `apps/supervisor/`, `apps/coordinator/`, etc.) with no package changes, add a `.server-changes/` file so the change appears in release notes.
+
+Create a markdown file with a descriptive name:
+
+```sh
+cat > .server-changes/fix-batch-queue-stalls.md << 'EOF'
+---
+area: webapp
+type: fix
+---
+
+Speed up batch queue processing by removing stalls and fixing retry race
+EOF
+```
+
+**Fields:**
+- `area` (required): `webapp` | `supervisor` | `coordinator` | `kubernetes-provider` | `docker-provider`
+- `type` (required): `feature` | `fix` | `improvement` | `breaking`
+
+The body text (below the frontmatter) is a one-line description of the change. Keep it concise — it will appear in release notes.
+
+**When to add which:**
+
+| PR changes | What to add |
+|---|---|
+| Only packages (`packages/`) | Changeset |
+| Only server (`apps/`) | `.server-changes/` file |
+| Both packages and server | Just the changeset |
+
+See `.server-changes/README.md` for more details.
+
 ## Troubleshooting
 
 ### EADDRINUSE: address already in use :::3030

README.md

@@ -1,44 +1,76 @@
 <div align="center">
-<picture>
-  <source media="(prefers-color-scheme: dark)" srcset="https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/a45d1fa2-0ae8-4a39-4409-f4f934bfae00/public">
-  <source media="(prefers-color-scheme: light)" srcset="https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/3f5ad4c1-c4c8-4277-b622-290e7f37bd00/public">
-  <img alt="Trigger.dev logo" src="https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/a45d1fa2-0ae8-4a39-4409-f4f934bfae00/public">
-</picture>
-  
-### Open source background jobs and AI infrastructure
 
-[Discord](https://trigger.dev/discord) | [Website](https://trigger.dev) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Docs](https://trigger.dev/docs)
+![Trigger.dev logo](https://content.trigger.dev/github-header-banner.jpg)
 
-[![Twitter](https://img.shields.io/twitter/url/https/twitter.com/triggerdotdev.svg?style=social&label=Follow%20%40trigger.dev)](https://twitter.com/triggerdotdev)
+### Build and deploy fully‑managed AI agents and workflows
+
+[Website](https://trigger.dev) | [Docs](https://trigger.dev/docs) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Example projects](https://github.com/triggerdotdev/examples) | [Feature requests](https://triggerdev.featurebase.app/) | [Public roadmap](https://triggerdev.featurebase.app/roadmap) | [Self-hosting](https://trigger.dev/docs/self-hosting/overview) 
+
+[![Open Source](https://img.shields.io/badge/Open%20Source-%E2%9D%A4-red.svg)](https://github.com/triggerdotdev/trigger.dev)
+[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/triggerdotdev/trigger.dev/blob/main/LICENSE)
+[![npm](https://img.shields.io/npm/v/@trigger.dev/sdk.svg?label=npm)](https://www.npmjs.com/package/@trigger.dev/sdk)
+[![SDK downloads](https://img.shields.io/npm/dm/@trigger.dev/sdk.svg?label=SDK%20downloads)](https://www.npmjs.com/package/@trigger.dev/sdk)
+
+[![Twitter Follow](https://img.shields.io/twitter/follow/triggerdotdev?style=social)](https://twitter.com/triggerdotdev)
+[![Discord](https://img.shields.io/discord/1066956501299777596?logo=discord&logoColor=white&color=7289da)](https://discord.gg/nkqV9xBYWy)
+[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/triggerdotdev/trigger.dev)
+[![GitHub stars](https://img.shields.io/github/stars/triggerdotdev/trigger.dev?style=social)](https://github.com/triggerdotdev/trigger.dev)
 
 </div>
 
 ## About Trigger.dev
 
-Trigger.dev is an open source platform and SDK which allows you to create long-running background jobs. Write normal async code, deploy, and never hit a timeout.
+Trigger.dev is the open-source platform for building AI workflows in TypeScript. Long-running tasks with retries, queues, observability, and elastic scaling.
 
-### Key features:
+## The platform designed for building AI agents
 
-- JavaScript and TypeScript SDK
-- No timeouts
-- Retries (with exponential backoff)
-- Queues and concurrency controls
-- Schedules and crons
-- Full Observability; logs, live trace views, advanced filtering
-- React hooks to interact with the Trigger API from your React app
-- Pipe LLM streams straight to your users through the Realtime API
-- Trigger tasks and display the run status and metadata anywhere in your app
-- Custom alerts, get notified by email, Slack or webhooks
-- No infrastructure to manage
-- Elastic (scaling)
-- Works with your existing tech stack
+Build [AI agents](https://trigger.dev/product/ai-agents) using all the frameworks, services and LLMs you're used to, deploy them to Trigger.dev and get durable, long-running tasks with retries, queues, observability, and elastic scaling out of the box.
 
-## In your codebase
+- **Long-running without timeouts**: Execute your tasks with absolutely no timeouts, unlike AWS Lambda, Vercel, and other serverless platforms.
+
+- **Durability, retries & queues**: Build rock solid agents and AI applications using our durable tasks, retries, queues and idempotency.
+
+- **True runtime freedom**: Customize your deployed tasks with system packages – run browsers, Python scripts, FFmpeg and more.
+
+- **Human-in-the-loop**: Programmatically pause your tasks until a human can approve, reject or give feedback.
+
+- **Realtime apps & streaming**: Move your background jobs to the foreground by subscribing to runs or streaming AI responses to your app.
+
+- **Observability & monitoring**: Each run has full tracing and logs. Configure error alerts to catch bugs fast.
+
+## Key features:
+
+- **[JavaScript and TypeScript SDK](https://trigger.dev/docs/tasks/overview)** - Build background tasks using familiar programming models
+- **[Long-running tasks](https://trigger.dev/docs/runs/max-duration)** - Handle resource-heavy tasks without timeouts
+- **[Durable cron schedules](https://trigger.dev/docs/tasks/scheduled#scheduled-tasks-cron)** - Create and attach recurring schedules of up to a year
+- **[Trigger.dev Realtime](https://trigger.dev/docs/realtime/overview)** - Trigger, subscribe to, and get real-time updates for runs, with LLM streaming support
+- **[Build extensions](https://trigger.dev/docs/config/extensions/overview#build-extensions)** - Hook directly into the build system and customize the build process. Run Python scripts, FFmpeg, browsers, and more.
+- **[React hooks](https://trigger.dev/docs/frontend/react-hooks#react-hooks)** - Interact with the Trigger.dev API on your frontend using our React hooks package
+- **[Batch triggering](https://trigger.dev/docs/triggering#tasks-batchtrigger)** - Use batchTrigger() to initiate multiple runs of a task with custom payloads and options
+- **[Structured inputs / outputs](https://trigger.dev/docs/tasks/schemaTask#schematask)** - Define precise data schemas for your tasks with runtime payload validation
+- **[Waits](https://trigger.dev/docs/wait)** - Add waits to your tasks to pause execution for a specified duration
+- **[Preview branches](https://trigger.dev/docs/deployment/preview-branches)** - Create isolated environments for testing and development. Integrates with Vercel and git workflows
+- **[Waitpoints](https://trigger.dev/docs/wait-for-token#wait-for-token)** - Add human-in-the-loop judgment at critical decision points without disrupting workflow
+- **[Concurrency & queues](https://trigger.dev/docs/queue-concurrency#concurrency-and-queues)** - Set concurrency rules to manage how multiple tasks execute
+- **[Multiple environments](https://trigger.dev/docs/how-it-works#dev-mode)** - Support for DEV, PREVIEW, STAGING, and PROD environments
+- **[No infrastructure to manage](https://trigger.dev/docs/how-it-works#trigger-dev-architecture)** - Auto-scaling infrastructure that eliminates timeouts and server management
+- **[Automatic retries](https://trigger.dev/docs/errors-retrying)** - If your task encounters an uncaught error, we automatically attempt to run it again
+- **[Checkpointing](https://trigger.dev/docs/how-it-works#the-checkpoint-resume-system)** - Tasks are inherently durable, thanks to our checkpointing feature
+- **[Versioning](https://trigger.dev/docs/versioning)** - Atomic versioning allows you to deploy new versions without affecting running tasks
+- **[Machines](https://trigger.dev/docs/machines)** - Configure the number of vCPUs and GBs of RAM you want the task to use
+- **[Observability & monitoring](https://trigger.dev/product/observability-and-monitoring)** - Monitor every aspect of your tasks' performance with comprehensive logging and visualization tools
+- **[Logging & tracing](https://trigger.dev/docs/logging)** - Comprehensive logging and tracing for all your tasks
+- **[Tags](https://trigger.dev/docs/tags#tags)** - Attach up to ten tags to each run, allowing you to filter via the dashboard, realtime, and the SDK
+- **[Run metadata](https://trigger.dev/docs/runs/metadata#run-metadata)** - Attach metadata to runs which updates as the run progresses and is available to use in your frontend for live updates
+- **[Bulk actions](https://trigger.dev/docs/bulk-actions)** - Perform actions on multiple runs simultaneously, including replaying and cancelling
+- **[Real-time alerts](https://trigger.dev/docs/troubleshooting-alerts#alerts)** - Choose your preferred notification method for run failures and deployments
+
+## Write tasks in your codebase
 
 Create tasks where they belong: in your codebase. Version control, localhost, test and review like you're already used to.
 
 ```ts
-import { task } from "@trigger.dev/sdk/v3";
+import { task } from "@trigger.dev/sdk";
 
 //1. You need to export each task
 export const helloWorld = task({
@@ -58,13 +90,13 @@ Use our SDK to write tasks in your codebase. There's no infrastructure to manage
 
 ## Environments
 
-We support `Development`, `Staging`, and `Production` environments, allowing you to test your tasks before deploying them to production.
+We support `Development`, `Staging`, `Preview`, and `Production` environments, allowing you to test your tasks before deploying them to production.
 
 ## Full visibility of every job run
 
 View every task in every run so you can tell exactly what happened. We provide a full trace view of every task run so you can see what happened at every step.
 
-![Trace view image](https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/7c1b347f-004c-4482-38a7-3f6fa9c00d00/public)
+![Trace view image](https://content.trigger.dev/trace-view.png)
 
 # Getting started
 
@@ -73,14 +105,19 @@ The quickest way to get started is to create an account and project in our [web
 ### Useful links:
 
 - [Quick start](https://trigger.dev/docs/quick-start) - get up and running in minutes
-- [How it works](https://trigger.dev/docs/v3/how-it-works) - understand how Trigger.dev works under the hood
+- [How it works](https://trigger.dev/docs/how-it-works) - understand how Trigger.dev works under the hood
 - [Guides and examples](https://trigger.dev/docs/guides/introduction) - walk-through guides and code examples for popular frameworks and use cases
 
 ## Self-hosting
 
-If you prefer to self-host Trigger.dev, you can follow our [self-hosting guide](https://trigger.dev/docs/v3/open-source-self-hosting#overview).
+If you prefer to self-host Trigger.dev, you can follow our [self-hosting guides](https://trigger.dev/docs/self-hosting/overview):
 
-We also have a dedicated self-hosting channel in our [Discord server](https://trigger.dev/discord) for support.
+- [Docker self-hosting guide](https://trigger.dev/docs/self-hosting/docker) - use Docker Compose to spin up a Trigger.dev instance
+- [Kubernetes self-hosting guide](https://trigger.dev/docs/self-hosting/kubernetes) - use our official Helm chart to deploy Trigger.dev to your Kubernetes cluster
+
+## Support and community
+
+We have a large active community in our official [Discord server](https://trigger.dev/discord) for support, including a dedicated channel for self-hosting.
 
 ## Development
 

RELEASE.md

@@ -1,5 +1,28 @@
 ## Guide on releasing a new version
 
+### Automated release (v4+)
+
+Releases are fully automated via CI:
+
+1. PRs merge to `main` with changesets (for package changes) and/or `.server-changes/` files (for server-only changes).
+2. The [changesets-pr.yml](./.github/workflows/changesets-pr.yml) workflow automatically creates/updates the `changeset-release/main` PR with version bumps and an enhanced summary of all changes. Consumed `.server-changes/` files are removed on the release branch (same approach changesets uses for `.changeset/` files — they're deleted on the branch, so merging the PR cleans them up).
+3. When ready to release, merge the changeset release PR into `main`.
+4. The [release.yml](./.github/workflows/release.yml) workflow automatically:
+   - Publishes all packages to npm
+   - Creates a single unified GitHub release (e.g., "trigger.dev v4.3.4")
+   - Tags and triggers Docker image builds
+   - After Docker images are pushed, updates the GitHub release with the exact GHCR tag link
+
+### What engineers need to do
+
+- **Package changes**: Add a changeset with `pnpm run changeset:add`
+- **Server-only changes**: Add a `.server-changes/` file (see `.server-changes/README.md`)
+- **Mixed PRs**: Just the changeset is enough
+
+See `CHANGESETS.md` for full details on changesets and server changes.
+
+### Legacy release (v3)
+
 1. Merge in the changeset PR into main, making sure to cancel both the release and publish github actions from that merge.
 2. Pull the changes locally into main
 3. Run `pnpm i` which will update the pnpm lock file with the new versions

ai/references/migrations.md

@@ -0,0 +1,121 @@
+## Creating and applying migrations
+
+We use prisma migrations to manage the database schema. Please follow the following steps when editing the `internal-packages/database/prisma/schema.prisma` file:
+
+Edit the `schema.prisma` file to add or modify the schema.
+
+Create a new migration file but don't apply it yet:
+
+```bash
+cd internal-packages/database
+pnpm run db:migrate:dev:create --name "add_new_column_to_table"
+```
+
+The migration file will be created in the `prisma/migrations` directory, but it will have a bunch of edits to the schema that are not needed and will need to be removed before we can apply the migration. Here's an example of what the migration file might look like:
+
+```sql
+-- AlterEnum
+ALTER TYPE "public"."TaskRunExecutionStatus" ADD VALUE 'DELAYED';
+
+-- AlterTable
+ALTER TABLE "public"."TaskRun" ADD COLUMN     "debounce" JSONB;
+
+-- AlterTable
+ALTER TABLE "public"."_BackgroundWorkerToBackgroundWorkerFile" ADD CONSTRAINT "_BackgroundWorkerToBackgroundWorkerFile_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_BackgroundWorkerToBackgroundWorkerFile_AB_unique";
+
+-- AlterTable
+ALTER TABLE "public"."_BackgroundWorkerToTaskQueue" ADD CONSTRAINT "_BackgroundWorkerToTaskQueue_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_BackgroundWorkerToTaskQueue_AB_unique";
+
+-- AlterTable
+ALTER TABLE "public"."_TaskRunToTaskRunTag" ADD CONSTRAINT "_TaskRunToTaskRunTag_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_TaskRunToTaskRunTag_AB_unique";
+
+-- AlterTable
+ALTER TABLE "public"."_WaitpointRunConnections" ADD CONSTRAINT "_WaitpointRunConnections_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_WaitpointRunConnections_AB_unique";
+
+-- AlterTable
+ALTER TABLE "public"."_completedWaitpoints" ADD CONSTRAINT "_completedWaitpoints_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_completedWaitpoints_AB_unique";
+
+-- CreateIndex
+CREATE INDEX "SecretStore_key_idx" ON "public"."SecretStore"("key" text_pattern_ops);
+
+-- CreateIndex
+CREATE INDEX "TaskRun_runtimeEnvironmentId_id_idx" ON "public"."TaskRun"("runtimeEnvironmentId", "id" DESC);
+
+-- CreateIndex
+CREATE INDEX "TaskRun_runtimeEnvironmentId_createdAt_idx" ON "public"."TaskRun"("runtimeEnvironmentId", "createdAt" DESC);
+```
+
+All the following lines should be removed:
+
+```sql
+-- AlterTable
+ALTER TABLE "public"."_BackgroundWorkerToBackgroundWorkerFile" ADD CONSTRAINT "_BackgroundWorkerToBackgroundWorkerFile_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_BackgroundWorkerToBackgroundWorkerFile_AB_unique";
+
+-- AlterTable
+ALTER TABLE "public"."_BackgroundWorkerToTaskQueue" ADD CONSTRAINT "_BackgroundWorkerToTaskQueue_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_BackgroundWorkerToTaskQueue_AB_unique";
+
+-- AlterTable
+ALTER TABLE "public"."_TaskRunToTaskRunTag" ADD CONSTRAINT "_TaskRunToTaskRunTag_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_TaskRunToTaskRunTag_AB_unique";
+
+-- AlterTable
+ALTER TABLE "public"."_WaitpointRunConnections" ADD CONSTRAINT "_WaitpointRunConnections_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_WaitpointRunConnections_AB_unique";
+
+-- AlterTable
+ALTER TABLE "public"."_completedWaitpoints" ADD CONSTRAINT "_completedWaitpoints_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_completedWaitpoints_AB_unique";
+
+-- CreateIndex
+CREATE INDEX "SecretStore_key_idx" ON "public"."SecretStore"("key" text_pattern_ops);
+
+-- CreateIndex
+CREATE INDEX "TaskRun_runtimeEnvironmentId_id_idx" ON "public"."TaskRun"("runtimeEnvironmentId", "id" DESC);
+
+-- CreateIndex
+CREATE INDEX "TaskRun_runtimeEnvironmentId_createdAt_idx" ON "public"."TaskRun"("runtimeEnvironmentId", "createdAt" DESC);
+```
+
+Leaving only this:
+
+```sql
+-- AlterEnum
+ALTER TYPE "public"."TaskRunExecutionStatus" ADD VALUE 'DELAYED';
+
+-- AlterTable
+ALTER TABLE "public"."TaskRun" ADD COLUMN     "debounce" JSONB;
+```
+
+After editing the migration file, apply the migration:
+
+```bash
+cd internal-packages/database
+pnpm run db:migrate:deploy && pnpm run generate
+```

ai/references/repo.md

@@ -1,6 +1,6 @@
 ## Repo Overview
 
-This is a pnpm 8.15.5 monorepo that uses turborepo @turbo.json. The following workspaces are relevant
+This is a pnpm 10.23.0 monorepo that uses turborepo @turbo.json. The following workspaces are relevant
 
 ## Apps
 

apps/coordinator/Containerfile

@@ -35,7 +35,7 @@ COPY --from=pruner --chown=node:node /app/out/full/ .
 COPY --from=dev-deps --chown=node:node /app/ .
 COPY --chown=node:node turbo.json turbo.json
 
-RUN pnpm run -r --filter coordinator build:bundle
+RUN pnpm run -r --filter @trigger.dev/core bundle-vendor && pnpm run -r --filter coordinator build:bundle
 
 FROM alpine AS cri-tools
 

apps/docker-provider/Containerfile

@@ -31,7 +31,7 @@ COPY --from=pruner --chown=node:node /app/out/full/ .
 COPY --from=dev-deps --chown=node:node /app/ .
 COPY --chown=node:node turbo.json turbo.json
 
-RUN pnpm run -r --filter docker-provider build:bundle
+RUN pnpm run -r --filter @trigger.dev/core bundle-vendor && pnpm run -r --filter docker-provider build:bundle
 
 FROM base AS runner
 

apps/kubernetes-provider/Containerfile

@@ -31,7 +31,7 @@ COPY --from=pruner --chown=node:node /app/out/full/ .
 COPY --from=dev-deps --chown=node:node /app/ .
 COPY --chown=node:node turbo.json turbo.json
 
-RUN pnpm run -r --filter kubernetes-provider build:bundle
+RUN pnpm run -r --filter @trigger.dev/core bundle-vendor && pnpm run -r --filter kubernetes-provider build:bundle
 
 FROM base AS runner
 

apps/supervisor/CLAUDE.md

@@ -0,0 +1,20 @@
+# Supervisor
+
+Node.js app that manages task execution containers. Receives work from the platform, starts Docker/Kubernetes containers, monitors execution, and reports results.
+
+## Key Directories
+
+- `src/services/` - Core service logic
+- `src/workloadManager/` - Container orchestration abstraction (Docker or Kubernetes)
+- `src/workloadServer/` - HTTP server for workload communication (heartbeats, snapshots)
+- `src/clients/` - Platform communication (webapp/coordinator)
+- `src/env.ts` - Environment configuration
+
+## Architecture
+
+- **WorkloadManager**: Abstracts Docker vs Kubernetes execution
+- **SupervisorSession**: Manages the dequeue loop with EWMA-based dynamic scaling
+- **ResourceMonitor**: Tracks CPU/memory during execution
+- **PodCleaner/FailedPodHandler**: Kubernetes-specific cleanup
+
+Communicates with the platform via Socket.io and HTTP. Receives task assignments through the dequeue protocol from the webapp.

apps/supervisor/Containerfile

@@ -5,7 +5,7 @@ WORKDIR /app
 FROM node-22-alpine AS pruner
 
 COPY --chown=node:node . .
-RUN npx -q turbo@1.10.9 prune --scope=supervisor --docker
+RUN npx -q turbo@2.5.4 prune --scope=supervisor --docker
 
 FROM node-22-alpine AS base
 
@@ -16,7 +16,7 @@ COPY --from=pruner --chown=node:node /app/out/json/ .
 COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
 COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
 
-RUN corepack enable && corepack prepare --activate
+RUN corepack enable && corepack prepare pnpm@10.23.0 --activate
 
 FROM base AS deps-fetcher
 RUN apk add --no-cache python3-dev py3-setuptools make g++ gcc linux-headers
@@ -37,7 +37,7 @@ COPY --chown=node:node scripts/updateVersion.ts scripts/updateVersion.ts
 
 RUN pnpm run generate && \
   pnpm run --filter supervisor... build&& \
-  pnpm deploy --filter=supervisor --prod /prod/supervisor
+  pnpm deploy --legacy --filter=supervisor --prod /prod/supervisor
 
 FROM base AS runner
 

apps/supervisor/package.json

@@ -13,6 +13,7 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
+    "@aws-sdk/client-ecr": "^3.839.0",
     "@kubernetes/client-node": "^1.0.0",
     "@trigger.dev/core": "workspace:*",
     "dockerode": "^4.0.6",

apps/supervisor/src/env.ts

@@ -3,102 +3,176 @@ import { env as stdEnv } from "std-env";
 import { z } from "zod";
 import { AdditionalEnvVars, BoolEnv } from "./envUtil.js";
 
-const Env = z.object({
-  // This will come from `spec.nodeName` in k8s
-  TRIGGER_WORKER_INSTANCE_NAME: z.string().default(randomUUID()),
-  TRIGGER_WORKER_HEARTBEAT_INTERVAL_SECONDS: z.coerce.number().default(30),
+const Env = z
+  .object({
+    // This will come from `spec.nodeName` in k8s
+    TRIGGER_WORKER_INSTANCE_NAME: z.string().default(randomUUID()),
+    TRIGGER_WORKER_HEARTBEAT_INTERVAL_SECONDS: z.coerce.number().default(30),
 
-  // Required settings
-  TRIGGER_API_URL: z.string().url(),
-  TRIGGER_WORKER_TOKEN: z.string(), // accepts file:// path to read from a file
-  MANAGED_WORKER_SECRET: z.string(),
-  OTEL_EXPORTER_OTLP_ENDPOINT: z.string().url(), // set on the runners
+    // Required settings
+    TRIGGER_API_URL: z.string().url(),
+    TRIGGER_WORKER_TOKEN: z.string(), // accepts file:// path to read from a file
+    MANAGED_WORKER_SECRET: z.string(),
+    OTEL_EXPORTER_OTLP_ENDPOINT: z.string().url(), // set on the runners
 
-  // Workload API settings (coordinator mode) - the workload API is what the run controller connects to
-  TRIGGER_WORKLOAD_API_ENABLED: BoolEnv.default(true),
-  TRIGGER_WORKLOAD_API_PROTOCOL: z
-    .string()
-    .transform((s) => z.enum(["http", "https"]).parse(s.toLowerCase()))
-    .default("http"),
-  TRIGGER_WORKLOAD_API_DOMAIN: z.string().optional(), // If unset, will use orchestrator-specific default
-  TRIGGER_WORKLOAD_API_HOST_INTERNAL: z.string().default("0.0.0.0"),
-  TRIGGER_WORKLOAD_API_PORT_INTERNAL: z.coerce.number().default(8020), // This is the port the workload API listens on
-  TRIGGER_WORKLOAD_API_PORT_EXTERNAL: z.coerce.number().default(8020), // This is the exposed port passed to the run controller
+    // Workload API settings (coordinator mode) - the workload API is what the run controller connects to
+    TRIGGER_WORKLOAD_API_ENABLED: BoolEnv.default(true),
+    TRIGGER_WORKLOAD_API_PROTOCOL: z
+      .string()
+      .transform((s) => z.enum(["http", "https"]).parse(s.toLowerCase()))
+      .default("http"),
+    TRIGGER_WORKLOAD_API_DOMAIN: z.string().optional(), // If unset, will use orchestrator-specific default
+    TRIGGER_WORKLOAD_API_HOST_INTERNAL: z.string().default("0.0.0.0"),
+    TRIGGER_WORKLOAD_API_PORT_INTERNAL: z.coerce.number().default(8020), // This is the port the workload API listens on
+    TRIGGER_WORKLOAD_API_PORT_EXTERNAL: z.coerce.number().default(8020), // This is the exposed port passed to the run controller
 
-  // Runner settings
-  RUNNER_HEARTBEAT_INTERVAL_SECONDS: z.coerce.number().optional(),
-  RUNNER_SNAPSHOT_POLL_INTERVAL_SECONDS: z.coerce.number().optional(),
-  RUNNER_ADDITIONAL_ENV_VARS: AdditionalEnvVars, // optional (csv)
-  RUNNER_PRETTY_LOGS: BoolEnv.default(false),
+    // Runner settings
+    RUNNER_HEARTBEAT_INTERVAL_SECONDS: z.coerce.number().optional(),
+    RUNNER_SNAPSHOT_POLL_INTERVAL_SECONDS: z.coerce.number().optional(),
+    RUNNER_ADDITIONAL_ENV_VARS: AdditionalEnvVars, // optional (csv)
+    RUNNER_PRETTY_LOGS: BoolEnv.default(false),
 
-  // Dequeue settings (provider mode)
-  TRIGGER_DEQUEUE_ENABLED: BoolEnv.default(true),
-  TRIGGER_DEQUEUE_INTERVAL_MS: z.coerce.number().int().default(250),
-  TRIGGER_DEQUEUE_IDLE_INTERVAL_MS: z.coerce.number().int().default(1000),
-  TRIGGER_DEQUEUE_MAX_RUN_COUNT: z.coerce.number().int().default(10),
-  TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT: z.coerce.number().int().default(1),
+    // Dequeue settings (provider mode)
+    TRIGGER_DEQUEUE_ENABLED: BoolEnv.default(true),
+    TRIGGER_DEQUEUE_INTERVAL_MS: z.coerce.number().int().default(250),
+    TRIGGER_DEQUEUE_IDLE_INTERVAL_MS: z.coerce.number().int().default(1000),
+    TRIGGER_DEQUEUE_MAX_RUN_COUNT: z.coerce.number().int().default(1),
+    TRIGGER_DEQUEUE_MIN_CONSUMER_COUNT: z.coerce.number().int().default(1),
+    TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT: z.coerce.number().int().default(10),
+    TRIGGER_DEQUEUE_SCALING_STRATEGY: z.enum(["none", "smooth", "aggressive"]).default("none"),
+    TRIGGER_DEQUEUE_SCALING_UP_COOLDOWN_MS: z.coerce.number().int().default(5000), // 5 seconds
+    TRIGGER_DEQUEUE_SCALING_DOWN_COOLDOWN_MS: z.coerce.number().int().default(30000), // 30 seconds
+    TRIGGER_DEQUEUE_SCALING_TARGET_RATIO: z.coerce.number().default(1.0), // Target ratio of queue items to consumers (1.0 = 1 item per consumer)
+    TRIGGER_DEQUEUE_SCALING_EWMA_ALPHA: z.coerce.number().min(0).max(1).default(0.3), // Smooths queue length measurements (0=historical, 1=current)
+    TRIGGER_DEQUEUE_SCALING_BATCH_WINDOW_MS: z.coerce.number().int().positive().default(1000), // Batch window for metrics processing (ms)
+    TRIGGER_DEQUEUE_SCALING_DAMPING_FACTOR: z.coerce.number().min(0).max(1).default(0.7), // Smooths consumer count changes after EWMA (0=no scaling, 1=immediate)
 
-  // Optional services
-  TRIGGER_WARM_START_URL: z.string().optional(),
-  TRIGGER_CHECKPOINT_URL: z.string().optional(),
-  TRIGGER_METADATA_URL: z.string().optional(),
+    // Optional services
+    TRIGGER_WARM_START_URL: z.string().optional(),
+    TRIGGER_CHECKPOINT_URL: z.string().optional(),
+    TRIGGER_METADATA_URL: z.string().optional(),
 
-  // Used by the resource monitor
-  RESOURCE_MONITOR_ENABLED: BoolEnv.default(false),
-  RESOURCE_MONITOR_OVERRIDE_CPU_TOTAL: z.coerce.number().optional(),
-  RESOURCE_MONITOR_OVERRIDE_MEMORY_TOTAL_GB: z.coerce.number().optional(),
+    // Used by the resource monitor
+    RESOURCE_MONITOR_ENABLED: BoolEnv.default(false),
+    RESOURCE_MONITOR_OVERRIDE_CPU_TOTAL: z.coerce.number().optional(),
+    RESOURCE_MONITOR_OVERRIDE_MEMORY_TOTAL_GB: z.coerce.number().optional(),
 
-  // Docker settings
-  DOCKER_API_VERSION: z.string().default("v1.41"),
-  DOCKER_PLATFORM: z.string().optional(), // e.g. linux/amd64, linux/arm64
-  DOCKER_STRIP_IMAGE_DIGEST: BoolEnv.default(true),
-  DOCKER_REGISTRY_USERNAME: z.string().optional(),
-  DOCKER_REGISTRY_PASSWORD: z.string().optional(),
-  DOCKER_REGISTRY_URL: z.string().optional(), // e.g. https://index.docker.io/v1
-  DOCKER_ENFORCE_MACHINE_PRESETS: BoolEnv.default(true),
-  DOCKER_AUTOREMOVE_EXITED_CONTAINERS: BoolEnv.default(true),
-  /**
-   * Network mode to use for all runners. Supported standard values are: `bridge`, `host`, `none`, and `container:<name|id>`.
-   * Any other value is taken as a custom network's name to which all runners should connect to.
-   *
-   * Accepts a list of comma-separated values to attach to multiple networks. Additional networks are interpreted as network names and will be attached after container creation.
-   *
-   * **WARNING**: Specifying multiple networks will slightly increase startup times.
-   *
-   * @default "host"
-   */
-  DOCKER_RUNNER_NETWORKS: z.string().default("host"),
+    // Docker settings
+    DOCKER_API_VERSION: z.string().optional(),
+    DOCKER_PLATFORM: z.string().optional(), // e.g. linux/amd64, linux/arm64
+    DOCKER_STRIP_IMAGE_DIGEST: BoolEnv.default(true),
+    DOCKER_REGISTRY_USERNAME: z.string().optional(),
+    DOCKER_REGISTRY_PASSWORD: z.string().optional(),
+    DOCKER_REGISTRY_URL: z.string().optional(), // e.g. https://index.docker.io/v1
+    DOCKER_ENFORCE_MACHINE_PRESETS: BoolEnv.default(true),
+    DOCKER_AUTOREMOVE_EXITED_CONTAINERS: BoolEnv.default(true),
+    /**
+     * Network mode to use for all runners. Supported standard values are: `bridge`, `host`, `none`, and `container:<name|id>`.
+     * Any other value is taken as a custom network's name to which all runners should connect to.
+     *
+     * Accepts a list of comma-separated values to attach to multiple networks. Additional networks are interpreted as network names and will be attached after container creation.
+     *
+     * **WARNING**: Specifying multiple networks will slightly increase startup times.
+     *
+     * @default "host"
+     */
+    DOCKER_RUNNER_NETWORKS: z.string().default("host"),
 
-  // Kubernetes settings
-  KUBERNETES_FORCE_ENABLED: BoolEnv.default(false),
-  KUBERNETES_NAMESPACE: z.string().default("default"),
-  KUBERNETES_WORKER_NODETYPE_LABEL: z.string().default("v4-worker"),
-  KUBERNETES_IMAGE_PULL_SECRETS: z.string().optional(), // csv
-  KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT: z.string().default("10Gi"),
-  KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST: z.string().default("2Gi"),
+    // Compute settings
+    COMPUTE_GATEWAY_URL: z.string().url().optional(),
+    COMPUTE_GATEWAY_AUTH_TOKEN: z.string().optional(),
+    COMPUTE_GATEWAY_TIMEOUT_MS: z.coerce.number().int().default(30_000),
+    COMPUTE_SNAPSHOTS_ENABLED: BoolEnv.default(false),
+    COMPUTE_TRACE_SPANS_ENABLED: BoolEnv.default(true),
+    COMPUTE_TRACE_OTLP_ENDPOINT: z.string().url().optional(), // Override for span export (derived from TRIGGER_API_URL if unset)
+    COMPUTE_SNAPSHOT_DELAY_MS: z.coerce.number().int().min(0).max(60_000).default(5_000),
 
-  // Placement tags settings
-  PLACEMENT_TAGS_ENABLED: BoolEnv.default(false),
-  PLACEMENT_TAGS_PREFIX: z.string().default("node.cluster.x-k8s.io"),
+    // Kubernetes settings
+    KUBERNETES_FORCE_ENABLED: BoolEnv.default(false),
+    KUBERNETES_NAMESPACE: z.string().default("default"),
+    KUBERNETES_WORKER_NODETYPE_LABEL: z.string().default("v4-worker"),
+    KUBERNETES_IMAGE_PULL_SECRETS: z.string().optional(), // csv
+    KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT: z.string().default("10Gi"),
+    KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST: z.string().default("2Gi"),
+    KUBERNETES_STRIP_IMAGE_DIGEST: BoolEnv.default(false),
+    KUBERNETES_CPU_REQUEST_MIN_CORES: z.coerce.number().min(0).default(0),
+    KUBERNETES_CPU_REQUEST_RATIO: z.coerce.number().min(0).max(1).default(0.75), // Ratio of CPU limit, so 0.75 = 75% of CPU limit
+    KUBERNETES_MEMORY_REQUEST_MIN_GB: z.coerce.number().min(0).default(0),
+    KUBERNETES_MEMORY_REQUEST_RATIO: z.coerce.number().min(0).max(1).default(1), // Ratio of memory limit, so 1 = 100% of memory limit
 
-  // Metrics
-  METRICS_ENABLED: BoolEnv.default(true),
-  METRICS_COLLECT_DEFAULTS: BoolEnv.default(true),
-  METRICS_HOST: z.string().default("127.0.0.1"),
-  METRICS_PORT: z.coerce.number().int().default(9090),
+    // Per-preset overrides of the global KUBERNETES_CPU_REQUEST_RATIO
+    KUBERNETES_CPU_REQUEST_RATIO_MICRO: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_CPU_REQUEST_RATIO_SMALL_1X: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_CPU_REQUEST_RATIO_SMALL_2X: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_CPU_REQUEST_RATIO_MEDIUM_1X: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_CPU_REQUEST_RATIO_MEDIUM_2X: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_CPU_REQUEST_RATIO_LARGE_1X: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_CPU_REQUEST_RATIO_LARGE_2X: z.coerce.number().min(0).max(1).optional(),
 
-  // Pod cleaner
-  POD_CLEANER_ENABLED: BoolEnv.default(true),
-  POD_CLEANER_INTERVAL_MS: z.coerce.number().int().default(10000),
-  POD_CLEANER_BATCH_SIZE: z.coerce.number().int().default(500),
+    // Per-preset overrides of the global KUBERNETES_MEMORY_REQUEST_RATIO
+    KUBERNETES_MEMORY_REQUEST_RATIO_MICRO: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_MEMORY_REQUEST_RATIO_SMALL_1X: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_MEMORY_REQUEST_RATIO_SMALL_2X: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_MEMORY_REQUEST_RATIO_MEDIUM_1X: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_MEMORY_REQUEST_RATIO_MEDIUM_2X: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_MEMORY_REQUEST_RATIO_LARGE_1X: z.coerce.number().min(0).max(1).optional(),
+    KUBERNETES_MEMORY_REQUEST_RATIO_LARGE_2X: z.coerce.number().min(0).max(1).optional(),
 
-  // Failed pod handler
-  FAILED_POD_HANDLER_ENABLED: BoolEnv.default(true),
-  FAILED_POD_HANDLER_RECONNECT_INTERVAL_MS: z.coerce.number().int().default(1000),
+    KUBERNETES_MEMORY_OVERHEAD_GB: z.coerce.number().min(0).optional(), // Optional memory overhead to add to the limit in GB
+    KUBERNETES_SCHEDULER_NAME: z.string().optional(), // Custom scheduler name for pods
+    KUBERNETES_LARGE_MACHINE_POOL_LABEL: z.string().optional(), // if set, large-* presets affinity for machinepool=<value>
 
-  // Debug
-  DEBUG: BoolEnv.default(false),
-  SEND_RUN_DEBUG_LOGS: BoolEnv.default(false),
-});
+    // Project affinity settings - pods from the same project prefer the same node
+    KUBERNETES_PROJECT_AFFINITY_ENABLED: BoolEnv.default(false),
+    KUBERNETES_PROJECT_AFFINITY_WEIGHT: z.coerce.number().int().min(1).max(100).default(50),
+    KUBERNETES_PROJECT_AFFINITY_TOPOLOGY_KEY: z
+      .string()
+      .trim()
+      .min(1)
+      .default("kubernetes.io/hostname"),
+
+    // Placement tags settings
+    PLACEMENT_TAGS_ENABLED: BoolEnv.default(false),
+    PLACEMENT_TAGS_PREFIX: z.string().default("node.cluster.x-k8s.io"),
+
+    // Metrics
+    METRICS_ENABLED: BoolEnv.default(true),
+    METRICS_COLLECT_DEFAULTS: BoolEnv.default(true),
+    METRICS_HOST: z.string().default("127.0.0.1"),
+    METRICS_PORT: z.coerce.number().int().default(9090),
+
+    // Pod cleaner
+    POD_CLEANER_ENABLED: BoolEnv.default(true),
+    POD_CLEANER_INTERVAL_MS: z.coerce.number().int().default(10000),
+    POD_CLEANER_BATCH_SIZE: z.coerce.number().int().default(500),
+
+    // Failed pod handler
+    FAILED_POD_HANDLER_ENABLED: BoolEnv.default(true),
+    FAILED_POD_HANDLER_RECONNECT_INTERVAL_MS: z.coerce.number().int().default(1000),
+
+    // Debug
+    DEBUG: BoolEnv.default(false),
+    SEND_RUN_DEBUG_LOGS: BoolEnv.default(false),
+  })
+  .superRefine((data, ctx) => {
+    if (data.COMPUTE_SNAPSHOTS_ENABLED && !data.TRIGGER_METADATA_URL) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: "TRIGGER_METADATA_URL is required when COMPUTE_SNAPSHOTS_ENABLED is true",
+        path: ["TRIGGER_METADATA_URL"],
+      });
+    }
+    if (data.COMPUTE_SNAPSHOTS_ENABLED && !data.TRIGGER_WORKLOAD_API_DOMAIN) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: "TRIGGER_WORKLOAD_API_DOMAIN is required when COMPUTE_SNAPSHOTS_ENABLED is true",
+        path: ["TRIGGER_WORKLOAD_API_DOMAIN"],
+      });
+    }
+  })
+  .transform((data) => ({
+    ...data,
+    COMPUTE_TRACE_OTLP_ENDPOINT: data.COMPUTE_TRACE_OTLP_ENDPOINT ?? `${data.TRIGGER_API_URL}/otel`,
+  }));
 
 export const env = Env.parse(stdEnv);

apps/supervisor/src/index.ts

@@ -14,6 +14,7 @@ import {
 } from "./resourceMonitor.js";
 import { KubernetesWorkloadManager } from "./workloadManager/kubernetes.js";
 import { DockerWorkloadManager } from "./workloadManager/docker.js";
+import { ComputeWorkloadManager } from "./workloadManager/compute.js";
 import {
   HttpServer,
   CheckpointClient,
@@ -35,9 +36,11 @@ class ManagedSupervisor {
   private readonly metricsServer?: HttpServer;
   private readonly workloadServer: WorkloadServer;
   private readonly workloadManager: WorkloadManager;
+  private readonly computeManager?: ComputeWorkloadManager;
   private readonly logger = new SimpleStructuredLogger("managed-supervisor");
   private readonly resourceMonitor: ResourceMonitor;
   private readonly checkpointClient?: CheckpointClient;
+  private readonly isComputeMode: boolean;
 
   private readonly podCleaner?: PodCleaner;
   private readonly failedPodHandler?: FailedPodHandler;
@@ -77,9 +80,22 @@ class ManagedSupervisor {
         : new DockerResourceMonitor(new Docker())
       : new NoopResourceMonitor();
 
-    this.workloadManager = this.isKubernetes
-      ? new KubernetesWorkloadManager(workloadManagerOptions)
-      : new DockerWorkloadManager(workloadManagerOptions);
+    this.isComputeMode = !!env.COMPUTE_GATEWAY_URL;
+
+    if (env.COMPUTE_GATEWAY_URL) {
+      const computeManager = new ComputeWorkloadManager({
+        ...workloadManagerOptions,
+        gatewayUrl: env.COMPUTE_GATEWAY_URL,
+        gatewayAuthToken: env.COMPUTE_GATEWAY_AUTH_TOKEN,
+        gatewayTimeoutMs: env.COMPUTE_GATEWAY_TIMEOUT_MS,
+      });
+      this.computeManager = computeManager;
+      this.workloadManager = computeManager;
+    } else {
+      this.workloadManager = this.isKubernetes
+        ? new KubernetesWorkloadManager(workloadManagerOptions)
+        : new DockerWorkloadManager(workloadManagerOptions);
+    }
 
     if (this.isKubernetes) {
       if (env.POD_CLEANER_ENABLED) {
@@ -128,7 +144,18 @@ class ManagedSupervisor {
       dequeueIdleIntervalMs: env.TRIGGER_DEQUEUE_IDLE_INTERVAL_MS,
       queueConsumerEnabled: env.TRIGGER_DEQUEUE_ENABLED,
       maxRunCount: env.TRIGGER_DEQUEUE_MAX_RUN_COUNT,
-      maxConsumerCount: env.TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT,
+      metricsRegistry: register,
+      scaling: {
+        strategy: env.TRIGGER_DEQUEUE_SCALING_STRATEGY,
+        minConsumerCount: env.TRIGGER_DEQUEUE_MIN_CONSUMER_COUNT,
+        maxConsumerCount: env.TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT,
+        scaleUpCooldownMs: env.TRIGGER_DEQUEUE_SCALING_UP_COOLDOWN_MS,
+        scaleDownCooldownMs: env.TRIGGER_DEQUEUE_SCALING_DOWN_COOLDOWN_MS,
+        targetRatio: env.TRIGGER_DEQUEUE_SCALING_TARGET_RATIO,
+        ewmaAlpha: env.TRIGGER_DEQUEUE_SCALING_EWMA_ALPHA,
+        batchWindowMs: env.TRIGGER_DEQUEUE_SCALING_BATCH_WINDOW_MS,
+        dampingFactor: env.TRIGGER_DEQUEUE_SCALING_DAMPING_FACTOR,
+      },
       runNotificationsEnabled: env.TRIGGER_WORKLOAD_API_ENABLED,
       heartbeatIntervalSeconds: env.TRIGGER_WORKER_HEARTBEAT_INTERVAL_SECONDS,
       sendRunDebugLogs: env.SEND_RUN_DEBUG_LOGS,
@@ -171,13 +198,13 @@ class ManagedSupervisor {
     }
 
     this.workerSession.on("runNotification", async ({ time, run }) => {
-      this.logger.log("runNotification", { time, run });
+      this.logger.verbose("runNotification", { time, run });
 
       this.workloadServer.notifyRun({ run });
     });
 
-    this.workerSession.on("runQueueMessage", async ({ time, message }) => {
-      this.logger.log(`Received message with timestamp ${time.toLocaleString()}`, message);
+    this.workerSession.on("runQueueMessage", async ({ time, message, dequeueResponseMs, pollingIntervalMs }) => {
+      this.logger.verbose(`Received message with timestamp ${time.toLocaleString()}`, message);
 
       if (message.completedWaitpoints.length > 0) {
         this.logger.debug("Run has completed waitpoints", {
@@ -194,7 +221,39 @@ class ManagedSupervisor {
       const { checkpoint, ...rest } = message;
 
       if (checkpoint) {
-        this.logger.log("Restoring run", { runId: message.run.id });
+        this.logger.debug("Restoring run", { runId: message.run.id });
+
+        if (this.isComputeMode && this.computeManager && env.COMPUTE_SNAPSHOTS_ENABLED) {
+          try {
+            // Derive runnerId unique per restore cycle (matches iceman's pattern)
+            const runIdShort = message.run.friendlyId.replace("run_", "");
+            const checkpointSuffix = checkpoint.id.slice(-8);
+            const runnerId = `runner-${runIdShort}-${checkpointSuffix}`;
+
+            const didRestore = await this.computeManager.restore({
+              snapshotId: checkpoint.location,
+              runnerId,
+              runFriendlyId: message.run.friendlyId,
+              snapshotFriendlyId: message.snapshot.friendlyId,
+              machine: message.run.machine,
+              traceContext: message.run.traceContext,
+              envId: message.environment.id,
+              orgId: message.organization.id,
+              projectId: message.project.id,
+              dequeuedAt: message.dequeuedAt,
+            });
+
+            if (didRestore) {
+              this.logger.debug("Compute restore successful", { runId: message.run.id, runnerId });
+            } else {
+              this.logger.error("Compute restore failed", { runId: message.run.id, runnerId });
+            }
+          } catch (error) {
+            this.logger.error("Failed to restore run (compute)", { error });
+          }
+
+          return;
+        }
 
         if (!this.checkpointClient) {
           this.logger.error("No checkpoint client", { runId: message.run.id });
@@ -212,7 +271,7 @@ class ManagedSupervisor {
           });
 
           if (didRestore) {
-            this.logger.log("Restore successful", { runId: message.run.id });
+            this.logger.debug("Restore successful", { runId: message.run.id });
           } else {
             this.logger.error("Restore failed", { runId: message.run.id });
           }
@@ -223,24 +282,55 @@ class ManagedSupervisor {
         return;
       }
 
-      this.logger.log("Scheduling run", { runId: message.run.id });
+      this.logger.debug("Scheduling run", { runId: message.run.id });
 
+      const warmStartStart = performance.now();
       const didWarmStart = await this.tryWarmStart(message);
+      const warmStartCheckMs = Math.round(performance.now() - warmStartStart);
 
       if (didWarmStart) {
-        this.logger.log("Warm start successful", { runId: message.run.id });
+        this.logger.debug("Warm start successful", { runId: message.run.id });
         return;
       }
 
+      if (env.COMPUTE_TRACE_SPANS_ENABLED) {
+        const traceparent =
+          message.run.traceContext &&
+          "traceparent" in message.run.traceContext &&
+          typeof message.run.traceContext.traceparent === "string"
+            ? message.run.traceContext.traceparent
+            : undefined;
+
+        if (traceparent) {
+          this.workloadServer.registerRunTraceContext(message.run.friendlyId, {
+            traceparent,
+            envId: message.environment.id,
+            orgId: message.organization.id,
+            projectId: message.project.id,
+          });
+        }
+      }
+
       try {
+        if (!message.deployment.friendlyId) {
+          // mostly a type guard, deployments always exists for deployed environments
+          // a proper fix would be to use a discriminated union schema to differentiate between dequeued runs in dev and in deployed environments.
+          throw new Error("Deployment is missing");
+        }
+
         await this.workloadManager.create({
           dequeuedAt: message.dequeuedAt,
+          dequeueResponseMs,
+          pollingIntervalMs,
+          warmStartCheckMs,
           envId: message.environment.id,
           envType: message.environment.type,
           image: message.image,
           machine: message.run.machine,
           orgId: message.organization.id,
           projectId: message.project.id,
+          deploymentFriendlyId: message.deployment.friendlyId,
+          deploymentVersion: message.backgroundWorker.version,
           runId: message.run.id,
           runFriendlyId: message.run.friendlyId,
           version: message.version,
@@ -248,6 +338,7 @@ class ManagedSupervisor {
           snapshotId: message.snapshot.id,
           snapshotFriendlyId: message.snapshot.friendlyId,
           placementTags: message.placementTags,
+          traceContext: message.run.traceContext,
         });
 
         // Disabled for now
@@ -277,6 +368,7 @@ class ManagedSupervisor {
       host: env.TRIGGER_WORKLOAD_API_HOST_INTERNAL,
       workerClient: this.workerSession.httpClient,
       checkpointClient: this.checkpointClient,
+      computeManager: this.computeManager,
     });
 
     this.workloadServer.on("runConnected", this.onRunConnected.bind(this));
@@ -361,6 +453,7 @@ class ManagedSupervisor {
 
   async stop() {
     this.logger.log("Shutting down");
+    await this.workloadServer.stop();
     await this.workerSession.stop();
 
     // Optional services

apps/supervisor/src/otlpPayload.ts

@@ -0,0 +1,63 @@
+import { randomBytes } from "crypto";
+
+export interface OtlpTraceOptions {
+  traceId: string;
+  parentSpanId?: string;
+  spanName: string;
+  startTimeMs: number;
+  endTimeMs: number;
+  resourceAttributes: Record<string, string | number | boolean>;
+  spanAttributes: Record<string, string | number | boolean>;
+}
+
+/** Build an OTLP JSON ExportTraceServiceRequest payload */
+export function buildOtlpTracePayload(opts: OtlpTraceOptions) {
+  const spanId = randomBytes(8).toString("hex");
+
+  return {
+    resourceSpans: [
+      {
+        resource: {
+          attributes: [
+            { key: "$trigger", value: { boolValue: true } },
+            ...toOtlpAttributes(opts.resourceAttributes),
+          ],
+        },
+        scopeSpans: [
+          {
+            scope: { name: "supervisor.compute" },
+            spans: [
+              {
+                traceId: opts.traceId,
+                spanId,
+                parentSpanId: opts.parentSpanId,
+                name: opts.spanName,
+                kind: 3, // SPAN_KIND_CLIENT
+                startTimeUnixNano: String(opts.startTimeMs * 1_000_000),
+                endTimeUnixNano: String(opts.endTimeMs * 1_000_000),
+                attributes: toOtlpAttributes(opts.spanAttributes),
+                status: { code: 1 }, // STATUS_CODE_OK
+              },
+            ],
+          },
+        ],
+      },
+    ],
+  };
+}
+
+function toOtlpAttributes(
+  attrs: Record<string, string | number | boolean>
+): Array<{ key: string; value: Record<string, unknown> }> {
+  return Object.entries(attrs).map(([key, value]) => ({
+    key,
+    value: toOtlpValue(value),
+  }));
+}
+
+function toOtlpValue(value: string | number | boolean): Record<string, unknown> {
+  if (typeof value === "string") return { stringValue: value };
+  if (typeof value === "boolean") return { boolValue: value };
+  if (Number.isInteger(value)) return { intValue: value };
+  return { doubleValue: value };
+}

apps/supervisor/src/otlpTrace.test.ts

@@ -0,0 +1,114 @@
+import { describe, it, expect } from "vitest";
+import { buildOtlpTracePayload } from "./otlpPayload.js";
+
+describe("buildOtlpTracePayload", () => {
+  it("builds valid OTLP JSON with timing attributes", () => {
+    const payload = buildOtlpTracePayload({
+      traceId: "abcd1234abcd1234abcd1234abcd1234",
+      parentSpanId: "1234567890abcdef",
+      spanName: "compute.provision",
+      startTimeMs: 1000,
+      endTimeMs: 1250,
+      resourceAttributes: {
+        "ctx.environment.id": "env_123",
+        "ctx.organization.id": "org_456",
+        "ctx.project.id": "proj_789",
+        "ctx.run.id": "run_abc",
+      },
+      spanAttributes: {
+        "compute.total_ms": 250,
+        "compute.gateway.schedule_ms": 1,
+        "compute.cache.image_cached": true,
+      },
+    });
+
+    expect(payload.resourceSpans).toHaveLength(1);
+
+    const resourceSpan = payload.resourceSpans[0]!;
+
+    // $trigger=true so the webapp accepts it
+    const triggerAttr = resourceSpan.resource.attributes.find((a) => a.key === "$trigger");
+    expect(triggerAttr).toEqual({ key: "$trigger", value: { boolValue: true } });
+
+    // Resource attributes
+    const envAttr = resourceSpan.resource.attributes.find(
+      (a) => a.key === "ctx.environment.id"
+    );
+    expect(envAttr).toEqual({
+      key: "ctx.environment.id",
+      value: { stringValue: "env_123" },
+    });
+
+    // Span basics
+    const span = resourceSpan.scopeSpans[0]!.spans[0]!;
+    expect(span.name).toBe("compute.provision");
+    expect(span.traceId).toBe("abcd1234abcd1234abcd1234abcd1234");
+    expect(span.parentSpanId).toBe("1234567890abcdef");
+
+    // Integer attribute
+    const totalMs = span.attributes.find((a) => a.key === "compute.total_ms");
+    expect(totalMs).toEqual({ key: "compute.total_ms", value: { intValue: 250 } });
+
+    // Boolean attribute
+    const cached = span.attributes.find((a) => a.key === "compute.cache.image_cached");
+    expect(cached).toEqual({ key: "compute.cache.image_cached", value: { boolValue: true } });
+  });
+
+  it("generates a valid 16-char hex span ID", () => {
+    const payload = buildOtlpTracePayload({
+      traceId: "abcd1234abcd1234abcd1234abcd1234",
+      spanName: "test",
+      startTimeMs: 1000,
+      endTimeMs: 1001,
+      resourceAttributes: {},
+      spanAttributes: {},
+    });
+
+    const span = payload.resourceSpans[0]!.scopeSpans[0]!.spans[0]!;
+    expect(span.spanId).toMatch(/^[0-9a-f]{16}$/);
+  });
+
+  it("converts timestamps to nanoseconds", () => {
+    const payload = buildOtlpTracePayload({
+      traceId: "abcd1234abcd1234abcd1234abcd1234",
+      spanName: "test",
+      startTimeMs: 1000,
+      endTimeMs: 1250,
+      resourceAttributes: {},
+      spanAttributes: {},
+    });
+
+    const span = payload.resourceSpans[0]!.scopeSpans[0]!.spans[0]!;
+    expect(span.startTimeUnixNano).toBe("1000000000");
+    expect(span.endTimeUnixNano).toBe("1250000000");
+  });
+
+  it("omits parentSpanId when not provided", () => {
+    const payload = buildOtlpTracePayload({
+      traceId: "abcd1234abcd1234abcd1234abcd1234",
+      spanName: "test",
+      startTimeMs: 1000,
+      endTimeMs: 1001,
+      resourceAttributes: {},
+      spanAttributes: {},
+    });
+
+    const span = payload.resourceSpans[0]!.scopeSpans[0]!.spans[0]!;
+    expect(span.parentSpanId).toBeUndefined();
+  });
+
+  it("handles double values for non-integer numbers", () => {
+    const payload = buildOtlpTracePayload({
+      traceId: "abcd1234abcd1234abcd1234abcd1234",
+      spanName: "test",
+      startTimeMs: 1000,
+      endTimeMs: 1001,
+      resourceAttributes: {},
+      spanAttributes: { "compute.cpu": 0.25 },
+    });
+
+    const span = payload.resourceSpans[0]!.scopeSpans[0]!.spans[0]!;
+    const cpu = span.attributes.find((a) => a.key === "compute.cpu");
+    expect(cpu).toEqual({ key: "compute.cpu", value: { doubleValue: 0.25 } });
+  });
+});

apps/supervisor/src/otlpTrace.ts

@@ -0,0 +1,19 @@
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import { env } from "./env.js";
+import type { buildOtlpTracePayload } from "./otlpPayload.js";
+
+const logger = new SimpleStructuredLogger("otlp-trace");
+
+/** Fire-and-forget: send an OTLP trace payload to the configured endpoint */
+export function sendOtlpTrace(payload: ReturnType<typeof buildOtlpTracePayload>) {
+  fetch(`${env.COMPUTE_TRACE_OTLP_ENDPOINT}/v1/traces`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(payload),
+    signal: AbortSignal.timeout(5_000),
+  }).catch((err) => {
+    logger.warn("failed to send compute trace span", {
+      error: err instanceof Error ? err.message : String(err),
+    });
+  });
+}

apps/supervisor/src/services/failedPodHandler.ts

@@ -25,6 +25,7 @@ export class FailedPodHandler {
 
   private readonly informer: Informer<V1Pod>;
   private readonly reconnectIntervalMs: number;
+  private reconnecting = false;
 
   // Metrics
   private readonly register: Registry;
@@ -150,7 +151,7 @@ export class FailedPodHandler {
   }
 
   private async onPodCompleted(pod: V1Pod) {
-    this.logger.info("pod-completed", this.podSummary(pod));
+    this.logger.debug("pod-completed", this.podSummary(pod));
     this.informerEventsTotal.inc({ namespace: this.namespace, verb: "add" });
 
     if (!pod.metadata?.name) {
@@ -164,7 +165,7 @@ export class FailedPodHandler {
     }
 
     if (pod.metadata?.deletionTimestamp) {
-      this.logger.info("pod-completed: pod is being deleted", this.podSummary(pod));
+      this.logger.verbose("pod-completed: pod is being deleted", this.podSummary(pod));
       return;
     }
 
@@ -187,7 +188,7 @@ export class FailedPodHandler {
   }
 
   private async onPodSucceeded(pod: V1Pod) {
-    this.logger.info("pod-succeeded", this.podSummary(pod));
+    this.logger.debug("pod-succeeded", this.podSummary(pod));
     this.processedPodsTotal.inc({
       namespace: this.namespace,
       status: this.podStatus(pod),
@@ -195,7 +196,7 @@ export class FailedPodHandler {
   }
 
   private async onPodFailed(pod: V1Pod) {
-    this.logger.info("pod-failed", this.podSummary(pod));
+    this.logger.debug("pod-failed", this.podSummary(pod));
 
     try {
       await this.processFailedPod(pod);
@@ -207,7 +208,7 @@ export class FailedPodHandler {
   }
 
   private async processFailedPod(pod: V1Pod) {
-    this.logger.info("pod-failed: processing pod", this.podSummary(pod));
+    this.logger.verbose("pod-failed: processing pod", this.podSummary(pod));
 
     const mainContainer = pod.status?.containerStatuses?.find((c) => c.name === "run-controller");
 
@@ -230,7 +231,7 @@ export class FailedPodHandler {
   }
 
   private async deletePod(pod: V1Pod) {
-    this.logger.info("pod-failed: deleting pod", this.podSummary(pod));
+    this.logger.verbose("pod-failed: deleting pod", this.podSummary(pod));
     try {
       await this.k8s.core.deleteNamespacedPod({
         name: pod.metadata!.name!,
@@ -250,21 +251,48 @@ export class FailedPodHandler {
   }
 
   private makeOnError(informerName: string) {
-    return () => this.onError(informerName);
+    return (err?: unknown) => this.onError(informerName, err);
   }
 
-  private async onError(informerName: string) {
+  private async onError(informerName: string, err?: unknown) {
     if (!this.isRunning) {
       this.logger.warn("onError: informer not running");
       return;
     }
 
-    this.logger.error("error event fired", { informerName });
-    this.informerEventsTotal.inc({ namespace: this.namespace, verb: "error" });
+    // Guard against multiple simultaneous reconnections
+    if (this.reconnecting) {
+      this.logger.debug("onError: reconnection already in progress, skipping", {
+        informerName,
+      });
+      return;
+    }
 
-    // Reconnect on errors
-    await setTimeout(this.reconnectIntervalMs);
-    await this.informer.start();
+    this.reconnecting = true;
+
+    try {
+      const error = err instanceof Error ? err : undefined;
+      this.logger.error("error event fired", {
+        informerName,
+        error: error?.message,
+        errorType: error?.name,
+      });
+      this.informerEventsTotal.inc({ namespace: this.namespace, verb: "error" });
+
+      // Reconnect on errors
+      await setTimeout(this.reconnectIntervalMs);
+      await this.informer.start();
+    } catch (handlerError) {
+      const error = handlerError instanceof Error ? handlerError : undefined;
+      this.logger.error("onError: reconnection attempt failed", {
+        informerName,
+        error: error?.message,
+        errorType: error?.name,
+        errorStack: error?.stack,
+      });
+    } finally {
+      this.reconnecting = false;
+    }
   }
 
   private makeOnConnect(informerName: string) {

apps/supervisor/src/services/podCleaner.ts

@@ -90,7 +90,7 @@ export class PodCleaner {
           status: "succeeded",
         });
 
-        this.logger.info("Deleted batch of pods", { continuationToken });
+        this.logger.debug("Deleted batch of pods", { continuationToken });
       } catch (err) {
         this.logger.error("Failed to delete batch of pods", {
           err: err instanceof Error ? err.message : String(err),

apps/supervisor/src/services/timerWheel.test.ts

@@ -0,0 +1,254 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { TimerWheel } from "./timerWheel.js";
+
+describe("TimerWheel", () => {
+  beforeEach(() => {
+    vi.useFakeTimers();
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  it("dispatches item after delay", () => {
+    const dispatched: string[] = [];
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: (item) => dispatched.push(item.key),
+    });
+
+    wheel.start();
+    wheel.submit("run-1", "snapshot-data");
+
+    // Not yet
+    vi.advanceTimersByTime(2900);
+    expect(dispatched).toEqual([]);
+
+    // After delay
+    vi.advanceTimersByTime(200);
+    expect(dispatched).toEqual(["run-1"]);
+
+    wheel.stop();
+  });
+
+  it("cancels item before it fires", () => {
+    const dispatched: string[] = [];
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: (item) => dispatched.push(item.key),
+    });
+
+    wheel.start();
+    wheel.submit("run-1", "data");
+
+    vi.advanceTimersByTime(1000);
+    expect(wheel.cancel("run-1")).toBe(true);
+
+    vi.advanceTimersByTime(5000);
+    expect(dispatched).toEqual([]);
+    expect(wheel.size).toBe(0);
+
+    wheel.stop();
+  });
+
+  it("cancel returns false for unknown key", () => {
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: () => {},
+    });
+    expect(wheel.cancel("nonexistent")).toBe(false);
+  });
+
+  it("deduplicates: resubmitting same key replaces the entry", () => {
+    const dispatched: { key: string; data: string }[] = [];
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: (item) => dispatched.push({ key: item.key, data: item.data }),
+    });
+
+    wheel.start();
+    wheel.submit("run-1", "old-data");
+
+    vi.advanceTimersByTime(1000);
+    wheel.submit("run-1", "new-data");
+
+    // Original would have fired at t=3000, but was replaced
+    // New one fires at t=1000+3000=4000
+    vi.advanceTimersByTime(2100);
+    expect(dispatched).toEqual([]);
+
+    vi.advanceTimersByTime(1000);
+    expect(dispatched).toEqual([{ key: "run-1", data: "new-data" }]);
+
+    wheel.stop();
+  });
+
+  it("handles many concurrent items", () => {
+    const dispatched: string[] = [];
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: (item) => dispatched.push(item.key),
+    });
+
+    wheel.start();
+
+    for (let i = 0; i < 1000; i++) {
+      wheel.submit(`run-${i}`, `data-${i}`);
+    }
+    expect(wheel.size).toBe(1000);
+
+    vi.advanceTimersByTime(3100);
+    expect(dispatched.length).toBe(1000);
+    expect(wheel.size).toBe(0);
+
+    wheel.stop();
+  });
+
+  it("handles items submitted at different times", () => {
+    const dispatched: string[] = [];
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: (item) => dispatched.push(item.key),
+    });
+
+    wheel.start();
+
+    wheel.submit("run-1", "data");
+    vi.advanceTimersByTime(1000);
+    wheel.submit("run-2", "data");
+    vi.advanceTimersByTime(1000);
+    wheel.submit("run-3", "data");
+
+    // t=2000: nothing yet
+    expect(dispatched).toEqual([]);
+
+    // t=3100: run-1 fires
+    vi.advanceTimersByTime(1100);
+    expect(dispatched).toEqual(["run-1"]);
+
+    // t=4100: run-2 fires
+    vi.advanceTimersByTime(1000);
+    expect(dispatched).toEqual(["run-1", "run-2"]);
+
+    // t=5100: run-3 fires
+    vi.advanceTimersByTime(1000);
+    expect(dispatched).toEqual(["run-1", "run-2", "run-3"]);
+
+    wheel.stop();
+  });
+
+  it("setDelay changes delay for new items only", () => {
+    const dispatched: string[] = [];
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: (item) => dispatched.push(item.key),
+    });
+
+    wheel.start();
+
+    wheel.submit("run-1", "data"); // 3s delay
+
+    vi.advanceTimersByTime(500);
+    wheel.setDelay(1000);
+    wheel.submit("run-2", "data"); // 1s delay
+
+    // t=1500: run-2 should have fired (submitted at t=500 with 1s delay)
+    vi.advanceTimersByTime(1100);
+    expect(dispatched).toEqual(["run-2"]);
+
+    // t=3100: run-1 fires at its original 3s delay
+    vi.advanceTimersByTime(1500);
+    expect(dispatched).toEqual(["run-2", "run-1"]);
+
+    wheel.stop();
+  });
+
+  it("stop returns unprocessed items", () => {
+    const dispatched: string[] = [];
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: (item) => dispatched.push(item.key),
+    });
+
+    wheel.start();
+    wheel.submit("run-1", "data-1");
+    wheel.submit("run-2", "data-2");
+    wheel.submit("run-3", "data-3");
+
+    const remaining = wheel.stop();
+    expect(dispatched).toEqual([]);
+    expect(wheel.size).toBe(0);
+    expect(remaining.length).toBe(3);
+    expect(remaining.map((r) => r.key).sort()).toEqual(["run-1", "run-2", "run-3"]);
+    expect(remaining.find((r) => r.key === "run-1")?.data).toBe("data-1");
+  });
+
+  it("after stop, new submissions are silently dropped", () => {
+    const dispatched: string[] = [];
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: (item) => dispatched.push(item.key),
+    });
+
+    wheel.start();
+    wheel.stop();
+
+    wheel.submit("run-late", "data");
+    expect(dispatched).toEqual([]);
+    expect(wheel.size).toBe(0);
+  });
+
+  it("tracks size correctly through submit/cancel/dispatch", () => {
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: () => {},
+    });
+
+    wheel.start();
+
+    wheel.submit("a", "data");
+    wheel.submit("b", "data");
+    expect(wheel.size).toBe(2);
+
+    wheel.cancel("a");
+    expect(wheel.size).toBe(1);
+
+    vi.advanceTimersByTime(3100);
+    expect(wheel.size).toBe(0);
+
+    wheel.stop();
+  });
+
+  it("clamps delay to valid range", () => {
+    const dispatched: string[] = [];
+
+    // Very small delay (should be at least 1 tick = 100ms)
+    const wheel = new TimerWheel<string>({
+      delayMs: 0,
+      onExpire: (item) => dispatched.push(item.key),
+    });
+
+    wheel.start();
+    wheel.submit("run-1", "data");
+
+    vi.advanceTimersByTime(200);
+    expect(dispatched).toEqual(["run-1"]);
+
+    wheel.stop();
+  });
+
+  it("multiple cancel calls are safe", () => {
+    const wheel = new TimerWheel<string>({
+      delayMs: 3000,
+      onExpire: () => {},
+    });
+
+    wheel.start();
+    wheel.submit("run-1", "data");
+
+    expect(wheel.cancel("run-1")).toBe(true);
+    expect(wheel.cancel("run-1")).toBe(false);
+
+    wheel.stop();
+  });
+});

apps/supervisor/src/services/timerWheel.ts

@@ -0,0 +1,160 @@
+/**
+ * TimerWheel implements a hashed timer wheel for efficiently managing large numbers
+ * of delayed operations with O(1) submit, cancel, and per-item dispatch.
+ *
+ * Used by the supervisor to delay snapshot requests so that short-lived waitpoints
+ * (e.g. triggerAndWait that resolves in <3s) skip the snapshot entirely.
+ *
+ * The wheel is a ring buffer of slots. A single setInterval advances a cursor.
+ * When the cursor reaches a slot, all items in that slot are dispatched.
+ *
+ * Fixed capacity: 600 slots at 100ms tick = 60s max delay.
+ */
+
+const TICK_MS = 100;
+const NUM_SLOTS = 600; // 60s max delay at 100ms tick
+
+export type TimerWheelItem<T> = {
+  key: string;
+  data: T;
+};
+
+export type TimerWheelOptions<T> = {
+  /** Called when an item's delay expires. */
+  onExpire: (item: TimerWheelItem<T>) => void;
+  /** Delay in milliseconds before items fire. Clamped to [100, 60000]. */
+  delayMs: number;
+};
+
+type Entry<T> = {
+  key: string;
+  data: T;
+  slotIndex: number;
+};
+
+export class TimerWheel<T> {
+  private slots: Set<string>[];
+  private entries: Map<string, Entry<T>>;
+  private cursor: number;
+  private intervalId: ReturnType<typeof setInterval> | null;
+  private onExpire: (item: TimerWheelItem<T>) => void;
+  private delaySlots: number;
+
+  constructor(opts: TimerWheelOptions<T>) {
+    this.slots = Array.from({ length: NUM_SLOTS }, () => new Set());
+    this.entries = new Map();
+    this.cursor = 0;
+    this.intervalId = null;
+    this.onExpire = opts.onExpire;
+    this.delaySlots = Math.max(1, Math.min(NUM_SLOTS, Math.round(opts.delayMs / TICK_MS)));
+  }
+
+  /** Start the timer wheel. Must be called before submitting items. */
+  start(): void {
+    if (this.intervalId) return;
+    this.intervalId = setInterval(() => this.tick(), TICK_MS);
+    // Don't hold the process open just for the timer wheel
+    if (this.intervalId && typeof this.intervalId === "object" && "unref" in this.intervalId) {
+      this.intervalId.unref();
+    }
+  }
+
+  /**
+   * Stop the timer wheel and return all unprocessed items.
+   * The wheel keeps running normally during graceful shutdown - call stop()
+   * only when you're ready to tear down. Caller decides what to do with leftovers.
+   */
+  stop(): TimerWheelItem<T>[] {
+    if (this.intervalId) {
+      clearInterval(this.intervalId);
+      this.intervalId = null;
+    }
+
+    const remaining: TimerWheelItem<T>[] = [];
+    for (const [key, entry] of this.entries) {
+      remaining.push({ key, data: entry.data });
+    }
+
+    for (const slot of this.slots) {
+      slot.clear();
+    }
+    this.entries.clear();
+
+    return remaining;
+  }
+
+  /**
+   * Update the delay for future submissions. Already-queued items keep their original timing.
+   * Clamped to [TICK_MS, 60000ms].
+   */
+  setDelay(delayMs: number): void {
+    this.delaySlots = Math.max(1, Math.min(NUM_SLOTS, Math.round(delayMs / TICK_MS)));
+  }
+
+  /**
+   * Submit an item to be dispatched after the configured delay.
+   * If an item with the same key already exists, it is replaced (dedup).
+   * No-op if the wheel is stopped.
+   */
+  submit(key: string, data: T): void {
+    if (!this.intervalId) return;
+
+    // Dedup: remove existing entry for this key
+    this.cancel(key);
+
+    const slotIndex = (this.cursor + this.delaySlots) % NUM_SLOTS;
+    const entry: Entry<T> = { key, data, slotIndex };
+
+    this.entries.set(key, entry);
+    this.slot(slotIndex).add(key);
+  }
+
+  /**
+   * Cancel a pending item. Returns true if the item was found and removed.
+   */
+  cancel(key: string): boolean {
+    const entry = this.entries.get(key);
+    if (!entry) return false;
+
+    this.slot(entry.slotIndex).delete(key);
+    this.entries.delete(key);
+    return true;
+  }
+
+  /** Number of pending items in the wheel. */
+  get size(): number {
+    return this.entries.size;
+  }
+
+  /** Whether the wheel is running. */
+  get running(): boolean {
+    return this.intervalId !== null;
+  }
+
+  /** Get a slot by index. The array is fully initialized so this always returns a Set. */
+  private slot(index: number): Set<string> {
+    const s = this.slots[index];
+    if (!s) throw new Error(`TimerWheel: invalid slot index ${index}`);
+    return s;
+  }
+
+  /** Advance the cursor and dispatch all items in the current slot. */
+  private tick(): void {
+    this.cursor = (this.cursor + 1) % NUM_SLOTS;
+    const slot = this.slot(this.cursor);
+
+    if (slot.size === 0) return;
+
+    // Collect items to dispatch (copy keys since we mutate during iteration)
+    const keys = [...slot];
+    slot.clear();
+
+    for (const key of keys) {
+      const entry = this.entries.get(key);
+      if (!entry) continue;
+
+      this.entries.delete(key);
+      this.onExpire({ key, data: entry.data });
+    }
+  }
+}

apps/supervisor/src/workloadManager/compute.ts

@@ -0,0 +1,451 @@
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import { parseTraceparent } from "@trigger.dev/core/v3/isomorphic";
+import { flattenAttributes } from "@trigger.dev/core/v3/utils/flattenAttributes";
+import {
+  type WorkloadManager,
+  type WorkloadManagerCreateOptions,
+  type WorkloadManagerOptions,
+} from "./types.js";
+import { env } from "../env.js";
+import { getRunnerId } from "../util.js";
+import { buildOtlpTracePayload } from "../otlpPayload.js";
+import { sendOtlpTrace } from "../otlpTrace.js";
+import { tryCatch } from "@trigger.dev/core";
+
+type ComputeWorkloadManagerOptions = WorkloadManagerOptions & {
+  gatewayUrl: string;
+  gatewayAuthToken?: string;
+  gatewayTimeoutMs: number;
+};
+
+export class ComputeWorkloadManager implements WorkloadManager {
+  private readonly logger = new SimpleStructuredLogger("compute-workload-manager");
+
+  constructor(private opts: ComputeWorkloadManagerOptions) {
+    if (opts.workloadApiDomain) {
+      this.logger.warn("⚠️ Custom workload API domain", {
+        domain: opts.workloadApiDomain,
+      });
+    }
+  }
+
+  async create(opts: WorkloadManagerCreateOptions) {
+    const runnerId = getRunnerId(opts.runFriendlyId, opts.nextAttemptNumber);
+
+    const envVars: Record<string, string> = {
+      OTEL_EXPORTER_OTLP_ENDPOINT: env.OTEL_EXPORTER_OTLP_ENDPOINT,
+      TRIGGER_DEQUEUED_AT_MS: String(opts.dequeuedAt.getTime()),
+      TRIGGER_POD_SCHEDULED_AT_MS: String(Date.now()),
+      TRIGGER_ENV_ID: opts.envId,
+      TRIGGER_DEPLOYMENT_ID: opts.deploymentFriendlyId,
+      TRIGGER_DEPLOYMENT_VERSION: opts.deploymentVersion,
+      TRIGGER_RUN_ID: opts.runFriendlyId,
+      TRIGGER_SNAPSHOT_ID: opts.snapshotFriendlyId,
+      TRIGGER_SUPERVISOR_API_PROTOCOL: this.opts.workloadApiProtocol,
+      TRIGGER_SUPERVISOR_API_PORT: String(this.opts.workloadApiPort),
+      TRIGGER_SUPERVISOR_API_DOMAIN: this.opts.workloadApiDomain ?? "",
+      TRIGGER_WORKER_INSTANCE_NAME: env.TRIGGER_WORKER_INSTANCE_NAME,
+      TRIGGER_RUNNER_ID: runnerId,
+      TRIGGER_MACHINE_CPU: String(opts.machine.cpu),
+      TRIGGER_MACHINE_MEMORY: String(opts.machine.memory),
+      PRETTY_LOGS: String(env.RUNNER_PRETTY_LOGS),
+    };
+
+    if (this.opts.warmStartUrl) {
+      envVars.TRIGGER_WARM_START_URL = this.opts.warmStartUrl;
+    }
+
+    if (env.COMPUTE_SNAPSHOTS_ENABLED && this.opts.metadataUrl) {
+      envVars.TRIGGER_METADATA_URL = this.opts.metadataUrl;
+    }
+
+    if (this.opts.heartbeatIntervalSeconds) {
+      envVars.TRIGGER_HEARTBEAT_INTERVAL_SECONDS = String(this.opts.heartbeatIntervalSeconds);
+    }
+
+    if (this.opts.snapshotPollIntervalSeconds) {
+      envVars.TRIGGER_SNAPSHOT_POLL_INTERVAL_SECONDS = String(
+        this.opts.snapshotPollIntervalSeconds
+      );
+    }
+
+    if (this.opts.additionalEnvVars) {
+      Object.assign(envVars, this.opts.additionalEnvVars);
+    }
+
+    const headers: Record<string, string> = {
+      "Content-Type": "application/json",
+    };
+
+    if (this.opts.gatewayAuthToken) {
+      headers["Authorization"] = `Bearer ${this.opts.gatewayAuthToken}`;
+    }
+
+    // Strip image digest — resolve by tag, not digest
+    const imageRef = opts.image.split("@")[0]!;
+
+    const url = `${this.opts.gatewayUrl}/api/instances`;
+
+    // Wide event: single canonical log line emitted in finally
+    const event: Record<string, unknown> = {
+      // High-cardinality identifiers
+      runId: opts.runFriendlyId,
+      runnerId,
+      envId: opts.envId,
+      envType: opts.envType,
+      orgId: opts.orgId,
+      projectId: opts.projectId,
+      deploymentVersion: opts.deploymentVersion,
+      machine: opts.machine.name,
+      // Environment
+      instanceName: env.TRIGGER_WORKER_INSTANCE_NAME,
+      // Supervisor timing
+      dequeueResponseMs: opts.dequeueResponseMs,
+      pollingIntervalMs: opts.pollingIntervalMs,
+      warmStartCheckMs: opts.warmStartCheckMs,
+      // Request
+      image: imageRef,
+      url,
+    };
+
+    const startMs = performance.now();
+
+    try {
+      const [fetchError, response] = await tryCatch(
+        fetch(url, {
+          method: "POST",
+          headers,
+          signal: AbortSignal.timeout(this.opts.gatewayTimeoutMs),
+          body: JSON.stringify({
+            name: runnerId,
+            image: imageRef,
+            env: envVars,
+            cpu: opts.machine.cpu,
+            memory_gb: opts.machine.memory,
+            metadata: {
+              runId: opts.runFriendlyId,
+              envId: opts.envId,
+              envType: opts.envType,
+              orgId: opts.orgId,
+              projectId: opts.projectId,
+              deploymentVersion: opts.deploymentVersion,
+              machine: opts.machine.name,
+            },
+          }),
+        })
+      );
+
+      if (fetchError) {
+        event.error = fetchError instanceof Error ? fetchError.message : String(fetchError);
+        event.errorType =
+          fetchError instanceof DOMException && fetchError.name === "TimeoutError"
+            ? "timeout"
+            : "fetch";
+        return;
+      }
+
+      event.status = response.status;
+
+      if (!response.ok) {
+        const [bodyError, body] = await tryCatch(response.text());
+        event.responseBody = bodyError ? undefined : body;
+        return;
+      }
+
+      const [parseError, data] = await tryCatch(response.json());
+
+      if (parseError) {
+        event.error = parseError instanceof Error ? parseError.message : String(parseError);
+        event.errorType = "parse";
+        return;
+      }
+
+      event.instanceId = data.id;
+      event.ok = true;
+
+      // Parse timing data from compute response (optional - requires gateway timing flag)
+      if (data._timing) {
+        event.timing = data._timing;
+      }
+
+      this.#emitProvisionSpan(opts, startMs, data._timing);
+    } finally {
+      event.durationMs = Math.round(performance.now() - startMs);
+      event.ok ??= false;
+      this.logger.debug("create instance", event);
+    }
+  }
+
+  private get authHeaders(): Record<string, string> {
+    const headers: Record<string, string> = {
+      "Content-Type": "application/json",
+    };
+    if (this.opts.gatewayAuthToken) {
+      headers["Authorization"] = `Bearer ${this.opts.gatewayAuthToken}`;
+    }
+    return headers;
+  }
+
+  async snapshot(opts: {
+    runnerId: string;
+    callbackUrl: string;
+    metadata: Record<string, string>;
+  }): Promise<boolean> {
+    const url = `${this.opts.gatewayUrl}/api/instances/${opts.runnerId}/snapshot`;
+
+    const [error, response] = await tryCatch(
+      fetch(url, {
+        method: "POST",
+        headers: this.authHeaders,
+        signal: AbortSignal.timeout(this.opts.gatewayTimeoutMs),
+        body: JSON.stringify({
+          callback: {
+            url: opts.callbackUrl,
+            metadata: opts.metadata,
+          },
+        }),
+      })
+    );
+
+    if (error) {
+      this.logger.error("snapshot request failed", {
+        runnerId: opts.runnerId,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      return false;
+    }
+
+    if (response.status !== 202) {
+      this.logger.error("snapshot request rejected", {
+        runnerId: opts.runnerId,
+        status: response.status,
+      });
+      return false;
+    }
+
+    this.logger.debug("snapshot request accepted", { runnerId: opts.runnerId });
+    return true;
+  }
+
+  async deleteInstance(runnerId: string): Promise<boolean> {
+    const url = `${this.opts.gatewayUrl}/api/instances/${runnerId}`;
+
+    const [error, response] = await tryCatch(
+      fetch(url, {
+        method: "DELETE",
+        headers: this.authHeaders,
+        signal: AbortSignal.timeout(this.opts.gatewayTimeoutMs),
+      })
+    );
+
+    if (error) {
+      this.logger.error("delete instance failed", {
+        runnerId,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      return false;
+    }
+
+    if (!response.ok) {
+      this.logger.error("delete instance rejected", {
+        runnerId,
+        status: response.status,
+      });
+      return false;
+    }
+
+    this.logger.debug("delete instance success", { runnerId });
+    return true;
+  }
+
+  #emitProvisionSpan(
+    opts: WorkloadManagerCreateOptions,
+    startMs: number,
+    timing?: unknown
+  ) {
+    if (!env.COMPUTE_TRACE_SPANS_ENABLED) return;
+    const traceparent =
+      opts.traceContext &&
+      "traceparent" in opts.traceContext &&
+      typeof opts.traceContext.traceparent === "string"
+        ? opts.traceContext.traceparent
+        : undefined;
+
+    const parsed = parseTraceparent(traceparent);
+    if (!parsed) return;
+
+    const endMs = performance.now();
+    const now = Date.now();
+    const provisionStartEpochMs = now - (endMs - startMs);
+    const endEpochMs = now;
+
+    // Span starts at dequeue time so events (dequeue) render in the thin-line section
+    // before "Started". The actual provision call time is in provisionStartEpochMs.
+    // Subtract 1ms so compute span always sorts before the attempt span (same dequeue time)
+    const startEpochMs = opts.dequeuedAt.getTime() - 1;
+
+    const spanAttributes: Record<string, string | number | boolean> = {
+      "compute.type": "create",
+      "compute.provision_start_ms": provisionStartEpochMs,
+      ...(timing ? (flattenAttributes(timing, "compute") as Record<string, string | number | boolean>) : {}),
+    };
+
+    if (opts.dequeueResponseMs !== undefined) {
+      spanAttributes["supervisor.dequeue_response_ms"] = opts.dequeueResponseMs;
+    }
+    if (opts.warmStartCheckMs !== undefined) {
+      spanAttributes["supervisor.warm_start_check_ms"] = opts.warmStartCheckMs;
+    }
+
+    const payload = buildOtlpTracePayload({
+      traceId: parsed.traceId,
+      parentSpanId: parsed.spanId,
+      spanName: "compute.provision",
+      startTimeMs: startEpochMs,
+      endTimeMs: endEpochMs,
+      resourceAttributes: {
+        "ctx.environment.id": opts.envId,
+        "ctx.organization.id": opts.orgId,
+        "ctx.project.id": opts.projectId,
+        "ctx.run.id": opts.runFriendlyId,
+      },
+      spanAttributes,
+    });
+
+    // Use the platform API URL, not the runner OTLP endpoint (which may be a VM gateway IP)
+    sendOtlpTrace(payload);
+  }
+
+  async restore(opts: {
+    snapshotId: string;
+    runnerId: string;
+    runFriendlyId: string;
+    snapshotFriendlyId: string;
+    machine: { cpu: number; memory: number };
+    // Trace context for OTel span emission
+    traceContext?: Record<string, unknown>;
+    envId?: string;
+    orgId?: string;
+    projectId?: string;
+    dequeuedAt?: Date;
+  }): Promise<boolean> {
+    const url = `${this.opts.gatewayUrl}/api/snapshots/${opts.snapshotId}/restore`;
+
+    const metadata: Record<string, string> = {
+      TRIGGER_RUNNER_ID: opts.runnerId,
+      TRIGGER_RUN_ID: opts.runFriendlyId,
+      TRIGGER_SNAPSHOT_ID: opts.snapshotFriendlyId,
+      TRIGGER_SUPERVISOR_API_PROTOCOL: this.opts.workloadApiProtocol,
+      TRIGGER_SUPERVISOR_API_PORT: String(this.opts.workloadApiPort),
+      TRIGGER_SUPERVISOR_API_DOMAIN: this.opts.workloadApiDomain ?? "",
+      TRIGGER_WORKER_INSTANCE_NAME: env.TRIGGER_WORKER_INSTANCE_NAME,
+    };
+
+    const body = {
+      name: opts.runnerId,
+      metadata,
+      cpu: opts.machine.cpu,
+      memory_mb: opts.machine.memory * 1024,
+    };
+
+    this.logger.verbose("restore request body", { url, body });
+
+    const startMs = performance.now();
+
+    const [error, response] = await tryCatch(
+      fetch(url, {
+        method: "POST",
+        headers: this.authHeaders,
+        signal: AbortSignal.timeout(this.opts.gatewayTimeoutMs),
+        body: JSON.stringify(body),
+      })
+    );
+
+    const durationMs = Math.round(performance.now() - startMs);
+
+    if (error) {
+      this.logger.error("restore request failed", {
+        snapshotId: opts.snapshotId,
+        runnerId: opts.runnerId,
+        error: error instanceof Error ? error.message : String(error),
+        durationMs,
+      });
+      return false;
+    }
+
+    if (!response.ok) {
+      this.logger.error("restore request rejected", {
+        snapshotId: opts.snapshotId,
+        runnerId: opts.runnerId,
+        status: response.status,
+        durationMs,
+      });
+      return false;
+    }
+
+    this.logger.debug("restore request success", {
+      snapshotId: opts.snapshotId,
+      runnerId: opts.runnerId,
+      durationMs,
+    });
+
+    this.#emitRestoreSpan(opts, startMs);
+
+    return true;
+  }
+
+  #emitRestoreSpan(
+    opts: {
+      snapshotId: string;
+      runnerId: string;
+      runFriendlyId: string;
+      traceContext?: Record<string, unknown>;
+      envId?: string;
+      orgId?: string;
+      projectId?: string;
+      dequeuedAt?: Date;
+    },
+    startMs: number
+  ) {
+    if (!env.COMPUTE_TRACE_SPANS_ENABLED) return;
+
+    const traceparent =
+      opts.traceContext &&
+      "traceparent" in opts.traceContext &&
+      typeof opts.traceContext.traceparent === "string"
+        ? opts.traceContext.traceparent
+        : undefined;
+
+    const parsed = parseTraceparent(traceparent);
+    if (!parsed || !opts.envId || !opts.orgId || !opts.projectId) return;
+
+    const endMs = performance.now();
+    const now = Date.now();
+    const restoreStartEpochMs = now - (endMs - startMs);
+    const endEpochMs = now;
+
+    // Subtract 1ms so restore span always sorts before the attempt span
+    const startEpochMs = (opts.dequeuedAt?.getTime() ?? restoreStartEpochMs) - 1;
+
+    const payload = buildOtlpTracePayload({
+      traceId: parsed.traceId,
+      parentSpanId: parsed.spanId,
+      spanName: "compute.restore",
+      startTimeMs: startEpochMs,
+      endTimeMs: endEpochMs,
+      resourceAttributes: {
+        "ctx.environment.id": opts.envId,
+        "ctx.organization.id": opts.orgId,
+        "ctx.project.id": opts.projectId,
+        "ctx.run.id": opts.runFriendlyId,
+      },
+      spanAttributes: {
+        "compute.type": "restore",
+        "compute.snapshot_id": opts.snapshotId,
+      },
+    });
+
+    sendOtlpTrace(payload);
+  }
+}
+

apps/supervisor/src/workloadManager/docker.ts

@@ -8,14 +8,16 @@ import { env } from "../env.js";
 import { getDockerHostDomain, getRunnerId, normalizeDockerHostUrl } from "../util.js";
 import Docker from "dockerode";
 import { tryCatch } from "@trigger.dev/core";
+import { ECRAuthService } from "./ecrAuth.js";
 
 export class DockerWorkloadManager implements WorkloadManager {
   private readonly logger = new SimpleStructuredLogger("docker-workload-manager");
   private readonly docker: Docker;
 
   private readonly runnerNetworks: string[];
-  private readonly auth?: Docker.AuthConfig;
+  private readonly staticAuth?: Docker.AuthConfig;
   private readonly platformOverride?: string;
+  private readonly ecrAuthService?: ECRAuthService;
 
   constructor(private opts: WorkloadManagerOptions) {
     this.docker = new Docker({
@@ -44,18 +46,23 @@ export class DockerWorkloadManager implements WorkloadManager {
         url: env.DOCKER_REGISTRY_URL,
       });
 
-      this.auth = {
+      this.staticAuth = {
         username: env.DOCKER_REGISTRY_USERNAME,
         password: env.DOCKER_REGISTRY_PASSWORD,
         serveraddress: env.DOCKER_REGISTRY_URL,
       };
+    } else if (ECRAuthService.hasAWSCredentials()) {
+      this.logger.info("🐋 AWS credentials found, initializing ECR auth service");
+      this.ecrAuthService = new ECRAuthService();
     } else {
-      this.logger.warn("🐋 No Docker registry credentials provided, skipping auth");
+      this.logger.warn(
+        "🐋 No Docker registry credentials or AWS credentials provided, skipping auth"
+      );
     }
   }
 
   async create(opts: WorkloadManagerCreateOptions) {
-    this.logger.log("create()", { opts });
+    this.logger.verbose("create()", { opts });
 
     const runnerId = getRunnerId(opts.runFriendlyId, opts.nextAttemptNumber);
 
@@ -65,6 +72,8 @@ export class DockerWorkloadManager implements WorkloadManager {
       `TRIGGER_DEQUEUED_AT_MS=${opts.dequeuedAt.getTime()}`,
       `TRIGGER_POD_SCHEDULED_AT_MS=${Date.now()}`,
       `TRIGGER_ENV_ID=${opts.envId}`,
+      `TRIGGER_DEPLOYMENT_ID=${opts.deploymentFriendlyId}`,
+      `TRIGGER_DEPLOYMENT_VERSION=${opts.deploymentVersion}`,
       `TRIGGER_RUN_ID=${opts.runFriendlyId}`,
       `TRIGGER_SNAPSHOT_ID=${opts.snapshotFriendlyId}`,
       `TRIGGER_SUPERVISOR_API_PROTOCOL=${this.opts.workloadApiProtocol}`,
@@ -160,9 +169,12 @@ export class DockerWorkloadManager implements WorkloadManager {
         imageArchitecture: inspectResult?.Architecture,
       });
 
+      // Get auth config (static or ECR)
+      const authConfig = await this.getAuthConfig();
+
       // Ensure the image is present
       const [createImageError, imageResponseReader] = await tryCatch(
-        this.docker.createImage(this.auth, {
+        this.docker.createImage(authConfig, {
           fromImage: imageRef,
           ...(this.platformOverride ? { platform: this.platformOverride } : {}),
         })
@@ -216,6 +228,26 @@ export class DockerWorkloadManager implements WorkloadManager {
     logger.debug("create succeeded", { startResult, containerId: container.id });
   }
 
+  /**
+   * Get authentication config for Docker operations
+   * Uses static credentials if available, otherwise attempts ECR auth
+   */
+  private async getAuthConfig(): Promise<Docker.AuthConfig | undefined> {
+    // Use static credentials if available
+    if (this.staticAuth) {
+      return this.staticAuth;
+    }
+
+    // Use ECR auth if service is available
+    if (this.ecrAuthService) {
+      const ecrAuth = await this.ecrAuthService.getAuthConfig();
+      return ecrAuth || undefined;
+    }
+
+    // No auth available
+    return undefined;
+  }
+
   private async attachContainerToNetworks({
     containerId,
     networkNames,

apps/supervisor/src/workloadManager/ecrAuth.ts

@@ -0,0 +1,144 @@
+import { ECRClient, GetAuthorizationTokenCommand } from "@aws-sdk/client-ecr";
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import { tryCatch } from "@trigger.dev/core";
+import Docker from "dockerode";
+
+interface ECRTokenCache {
+  token: string;
+  username: string;
+  serverAddress: string;
+  expiresAt: Date;
+}
+
+export class ECRAuthService {
+  private readonly logger = new SimpleStructuredLogger("ecr-auth-service");
+  private readonly ecrClient: ECRClient;
+  private tokenCache: ECRTokenCache | null = null;
+
+  constructor() {
+    this.ecrClient = new ECRClient();
+
+    this.logger.info("🔐 ECR Auth Service initialized", {
+      region: this.ecrClient.config.region,
+    });
+  }
+
+  /**
+   * Check if we have AWS credentials configured
+   */
+  static hasAWSCredentials(): boolean {
+    if (process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) {
+      return true;
+    }
+
+    if (
+      process.env.AWS_PROFILE ||
+      process.env.AWS_ROLE_ARN ||
+      process.env.AWS_WEB_IDENTITY_TOKEN_FILE
+    ) {
+      return true;
+    }
+
+    return false;
+  }
+
+  /**
+   * Check if the current token is still valid with a 10-minute buffer
+   */
+  private isTokenValid(): boolean {
+    if (!this.tokenCache) {
+      return false;
+    }
+
+    const now = new Date();
+    const bufferMs = 10 * 60 * 1000; // 10 minute buffer before expiration
+    return now < new Date(this.tokenCache.expiresAt.getTime() - bufferMs);
+  }
+
+  /**
+   * Get a fresh ECR authorization token from AWS
+   */
+  private async fetchNewToken(): Promise<ECRTokenCache | null> {
+    const [error, response] = await tryCatch(
+      this.ecrClient.send(new GetAuthorizationTokenCommand({}))
+    );
+
+    if (error) {
+      this.logger.error("Failed to get ECR authorization token", { error });
+      return null;
+    }
+
+    const authData = response.authorizationData?.[0];
+    if (!authData?.authorizationToken || !authData.proxyEndpoint) {
+      this.logger.error("Invalid ECR authorization response", { authData });
+      return null;
+    }
+
+    // Decode the base64 token to get username:password
+    const decoded = Buffer.from(authData.authorizationToken, "base64").toString("utf-8");
+    const [username, password] = decoded.split(":", 2);
+
+    if (!username || !password) {
+      this.logger.error("Failed to parse ECR authorization token");
+      return null;
+    }
+
+    const expiresAt = authData.expiresAt || new Date(Date.now() + 12 * 60 * 60 * 1000); // Default 12 hours
+
+    const tokenCache: ECRTokenCache = {
+      token: password,
+      username,
+      serverAddress: authData.proxyEndpoint,
+      expiresAt,
+    };
+
+    this.logger.info("🔐 Successfully fetched ECR token", {
+      username,
+      serverAddress: authData.proxyEndpoint,
+      expiresAt: expiresAt.toISOString(),
+    });
+
+    return tokenCache;
+  }
+
+  /**
+   * Get ECR auth config for Docker operations
+   * Returns cached token if valid, otherwise fetches a new one
+   */
+  async getAuthConfig(): Promise<Docker.AuthConfig | null> {
+    // Check if cached token is still valid
+    if (this.isTokenValid()) {
+      this.logger.debug("Using cached ECR token");
+      return {
+        username: this.tokenCache!.username,
+        password: this.tokenCache!.token,
+        serveraddress: this.tokenCache!.serverAddress,
+      };
+    }
+
+    // Fetch new token
+    this.logger.info("Fetching new ECR authorization token");
+    const newToken = await this.fetchNewToken();
+
+    if (!newToken) {
+      return null;
+    }
+
+    // Cache the new token
+    this.tokenCache = newToken;
+
+    return {
+      username: newToken.username,
+      password: newToken.token,
+      serveraddress: newToken.serverAddress,
+    };
+  }
+
+  /**
+   * Clear the cached token (useful for testing or forcing refresh)
+   */
+  clearCache(): void {
+    this.tokenCache = null;
+    this.logger.debug("ECR token cache cleared");
+  }
+}

apps/supervisor/src/workloadManager/kubernetes.ts

@@ -4,7 +4,12 @@ import {
   type WorkloadManagerCreateOptions,
   type WorkloadManagerOptions,
 } from "./types.js";
-import type { EnvironmentType, MachinePreset, PlacementTag } from "@trigger.dev/core/v3";
+import type {
+  EnvironmentType,
+  MachinePreset,
+  MachinePresetName,
+  PlacementTag,
+} from "@trigger.dev/core/v3";
 import { PlacementTagProcessor } from "@trigger.dev/core/v3/serverOnly";
 import { env } from "../env.js";
 import { type K8sApi, createK8sApi, type k8s } from "../clients/kubernetes.js";
@@ -14,12 +19,39 @@ type ResourceQuantities = {
   [K in "cpu" | "memory" | "ephemeral-storage"]?: string;
 };
 
+const cpuRequestRatioByMachinePreset: Record<MachinePresetName, number | undefined> = {
+  micro: env.KUBERNETES_CPU_REQUEST_RATIO_MICRO,
+  "small-1x": env.KUBERNETES_CPU_REQUEST_RATIO_SMALL_1X,
+  "small-2x": env.KUBERNETES_CPU_REQUEST_RATIO_SMALL_2X,
+  "medium-1x": env.KUBERNETES_CPU_REQUEST_RATIO_MEDIUM_1X,
+  "medium-2x": env.KUBERNETES_CPU_REQUEST_RATIO_MEDIUM_2X,
+  "large-1x": env.KUBERNETES_CPU_REQUEST_RATIO_LARGE_1X,
+  "large-2x": env.KUBERNETES_CPU_REQUEST_RATIO_LARGE_2X,
+};
+
+const memoryRequestRatioByMachinePreset: Record<MachinePresetName, number | undefined> = {
+  micro: env.KUBERNETES_MEMORY_REQUEST_RATIO_MICRO,
+  "small-1x": env.KUBERNETES_MEMORY_REQUEST_RATIO_SMALL_1X,
+  "small-2x": env.KUBERNETES_MEMORY_REQUEST_RATIO_SMALL_2X,
+  "medium-1x": env.KUBERNETES_MEMORY_REQUEST_RATIO_MEDIUM_1X,
+  "medium-2x": env.KUBERNETES_MEMORY_REQUEST_RATIO_MEDIUM_2X,
+  "large-1x": env.KUBERNETES_MEMORY_REQUEST_RATIO_LARGE_1X,
+  "large-2x": env.KUBERNETES_MEMORY_REQUEST_RATIO_LARGE_2X,
+};
+
 export class KubernetesWorkloadManager implements WorkloadManager {
   private readonly logger = new SimpleStructuredLogger("kubernetes-workload-provider");
   private k8s: K8sApi;
   private namespace = env.KUBERNETES_NAMESPACE;
   private placementTagProcessor: PlacementTagProcessor;
 
+  // Resource settings
+  private readonly cpuRequestMinCores = env.KUBERNETES_CPU_REQUEST_MIN_CORES;
+  private readonly cpuRequestRatio = env.KUBERNETES_CPU_REQUEST_RATIO;
+  private readonly memoryRequestMinGb = env.KUBERNETES_MEMORY_REQUEST_MIN_GB;
+  private readonly memoryRequestRatio = env.KUBERNETES_MEMORY_REQUEST_RATIO;
+  private readonly memoryOverheadGb = env.KUBERNETES_MEMORY_OVERHEAD_GB;
+
   constructor(private opts: WorkloadManagerOptions) {
     this.k8s = createK8sApi();
     this.placementTagProcessor = new PlacementTagProcessor({
@@ -49,8 +81,26 @@ export class KubernetesWorkloadManager implements WorkloadManager {
     };
   }
 
+  private stripImageDigest(imageRef: string): string {
+    if (!env.KUBERNETES_STRIP_IMAGE_DIGEST) {
+      return imageRef;
+    }
+
+    const atIndex = imageRef.lastIndexOf("@");
+
+    if (atIndex === -1) {
+      return imageRef;
+    }
+
+    return imageRef.substring(0, atIndex);
+  }
+
+  private clamp(value: number, min: number, max: number): number {
+    return Math.min(Math.max(value, min), max);
+  }
+
   async create(opts: WorkloadManagerCreateOptions) {
-    this.logger.log("[KubernetesWorkloadManager] Creating container", { opts });
+    this.logger.verbose("[KubernetesWorkloadManager] Creating container", { opts });
 
     const runnerId = getRunnerId(opts.runFriendlyId, opts.nextAttemptNumber);
 
@@ -70,11 +120,12 @@ export class KubernetesWorkloadManager implements WorkloadManager {
           },
           spec: {
             ...this.addPlacementTags(this.#defaultPodSpec, opts.placementTags),
+            affinity: this.#getAffinity(opts.machine, opts.projectId),
             terminationGracePeriodSeconds: 60 * 60,
             containers: [
               {
                 name: "run-controller",
-                image: opts.image,
+                image: this.stripImageDigest(opts.image),
                 ports: [
                   {
                     containerPort: 8000,
@@ -98,6 +149,14 @@ export class KubernetesWorkloadManager implements WorkloadManager {
                     name: "TRIGGER_ENV_ID",
                     value: opts.envId,
                   },
+                  {
+                    name: "TRIGGER_DEPLOYMENT_ID",
+                    value: opts.deploymentFriendlyId,
+                  },
+                  {
+                    name: "TRIGGER_DEPLOYMENT_VERSION",
+                    value: opts.deploymentVersion,
+                  },
                   {
                     name: "TRIGGER_SNAPSHOT_ID",
                     value: opts.snapshotFriendlyId,
@@ -249,6 +308,11 @@ export class KubernetesWorkloadManager implements WorkloadManager {
       restartPolicy: "Never",
       automountServiceAccountToken: false,
       imagePullSecrets: this.getImagePullSecrets(),
+      ...(env.KUBERNETES_SCHEDULER_NAME
+        ? {
+            schedulerName: env.KUBERNETES_SCHEDULER_NAME,
+          }
+        : {}),
       ...(env.KUBERNETES_WORKER_NODETYPE_LABEL
         ? {
             nodeSelector: {
@@ -277,20 +341,35 @@ export class KubernetesWorkloadManager implements WorkloadManager {
       envtype: this.#envTypeToLabelValue(opts.envType),
       org: opts.orgId,
       project: opts.projectId,
+      machine: opts.machine.name,
     };
   }
 
   #getResourceRequestsForMachine(preset: MachinePreset): ResourceQuantities {
+    const cpuRatio = cpuRequestRatioByMachinePreset[preset.name] ?? this.cpuRequestRatio;
+    const memoryRatio = memoryRequestRatioByMachinePreset[preset.name] ?? this.memoryRequestRatio;
+
+    const cpuRequest = preset.cpu * cpuRatio;
+    const memoryRequest = preset.memory * memoryRatio;
+
+    // Clamp between min and max
+    const clampedCpu = this.clamp(cpuRequest, this.cpuRequestMinCores, preset.cpu);
+    const clampedMemory = this.clamp(memoryRequest, this.memoryRequestMinGb, preset.memory);
+
     return {
-      cpu: `${preset.cpu * 0.75}`,
-      memory: `${preset.memory}G`,
+      cpu: `${clampedCpu}`,
+      memory: `${clampedMemory}G`,
     };
   }
 
   #getResourceLimitsForMachine(preset: MachinePreset): ResourceQuantities {
+    const memoryLimit = this.memoryOverheadGb
+      ? preset.memory + this.memoryOverheadGb
+      : preset.memory;
+
     return {
       cpu: `${preset.cpu}`,
-      memory: `${preset.memory}G`,
+      memory: `${memoryLimit}G`,
     };
   }
 
@@ -306,4 +385,91 @@ export class KubernetesWorkloadManager implements WorkloadManager {
       },
     };
   }
+
+  #isLargeMachine(preset: MachinePreset): boolean {
+    return preset.name.startsWith("large-");
+  }
+
+  #getAffinity(preset: MachinePreset, projectId: string): k8s.V1Affinity | undefined {
+    const nodeAffinity = this.#getNodeAffinityRules(preset);
+    const podAffinity = this.#getProjectPodAffinity(projectId);
+
+    if (!nodeAffinity && !podAffinity) {
+      return undefined;
+    }
+
+    return {
+      ...(nodeAffinity && { nodeAffinity }),
+      ...(podAffinity && { podAffinity }),
+    };
+  }
+
+  #getNodeAffinityRules(preset: MachinePreset): k8s.V1NodeAffinity | undefined {
+    if (!env.KUBERNETES_LARGE_MACHINE_POOL_LABEL) {
+      return undefined;
+    }
+
+    if (this.#isLargeMachine(preset)) {
+      // soft preference for the large-machine pool, falls back to standard if unavailable
+      return {
+        preferredDuringSchedulingIgnoredDuringExecution: [
+          {
+            weight: 100,
+            preference: {
+              matchExpressions: [
+                {
+                  key: "node.cluster.x-k8s.io/machinepool",
+                  operator: "In",
+                  values: [env.KUBERNETES_LARGE_MACHINE_POOL_LABEL],
+                },
+              ],
+            },
+          },
+        ],
+      };
+    }
+
+    // not schedulable in the large-machine pool
+    return {
+      requiredDuringSchedulingIgnoredDuringExecution: {
+        nodeSelectorTerms: [
+          {
+            matchExpressions: [
+              {
+                key: "node.cluster.x-k8s.io/machinepool",
+                operator: "NotIn",
+                values: [env.KUBERNETES_LARGE_MACHINE_POOL_LABEL],
+              },
+            ],
+          },
+        ],
+      },
+    };
+  }
+
+  #getProjectPodAffinity(projectId: string): k8s.V1PodAffinity | undefined {
+    if (!env.KUBERNETES_PROJECT_AFFINITY_ENABLED) {
+      return undefined;
+    }
+
+    return {
+      preferredDuringSchedulingIgnoredDuringExecution: [
+        {
+          weight: env.KUBERNETES_PROJECT_AFFINITY_WEIGHT,
+          podAffinityTerm: {
+            labelSelector: {
+              matchExpressions: [
+                {
+                  key: "project",
+                  operator: "In",
+                  values: [projectId],
+                },
+              ],
+            },
+            topologyKey: env.KUBERNETES_PROJECT_AFFINITY_TOPOLOGY_KEY,
+          },
+        },
+      ],
+    };
+  }
 }

apps/supervisor/src/workloadManager/types.ts

@@ -24,13 +24,21 @@ export interface WorkloadManagerCreateOptions {
   nextAttemptNumber?: number;
   dequeuedAt: Date;
   placementTags?: PlacementTag[];
+  // Timing context (populated by supervisor handler, included in wide event)
+  dequeueResponseMs?: number;
+  pollingIntervalMs?: number;
+  warmStartCheckMs?: number;
   // identifiers
   envId: string;
   envType: EnvironmentType;
   orgId: string;
   projectId: string;
+  deploymentFriendlyId: string;
+  deploymentVersion: string;
   runId: string;
   runFriendlyId: string;
   snapshotId: string;
   snapshotFriendlyId: string;
+  // Trace context for OTel span emission (W3C format: { traceparent: "00-...", tracestate?: "..." })
+  traceContext?: Record<string, unknown>;
 }

apps/supervisor/src/workloadServer/index.ts

@@ -16,7 +16,6 @@ import {
   type WorkloadRunAttemptCompleteResponseBody,
   WorkloadRunAttemptStartRequestBody,
   type WorkloadRunAttemptStartResponseBody,
-  type WorkloadRunLatestSnapshotResponseBody,
   WorkloadRunSnapshotsSinceResponseBody,
   type WorkloadServerToClientEvents,
   type WorkloadSuspendRunResponseBody,
@@ -25,6 +24,11 @@ import { HttpServer, type CheckpointClient } from "@trigger.dev/core/v3/serverOn
 import { type IncomingMessage } from "node:http";
 import { register } from "../metrics.js";
 import { env } from "../env.js";
+import type { ComputeWorkloadManager } from "../workloadManager/compute.js";
+import { TimerWheel } from "../services/timerWheel.js";
+import { parseTraceparent } from "@trigger.dev/core/v3/isomorphic";
+import { buildOtlpTracePayload } from "../otlpPayload.js";
+import { sendOtlpTrace } from "../otlpTrace.js";
 
 // Use the official export when upgrading to socket.io@4.8.0
 interface DefaultEventsMap {
@@ -54,15 +58,39 @@ type WorkloadServerEvents = {
   ];
 };
 
+const ComputeSnapshotCallbackBody = z.object({
+  snapshot_id: z.string(),
+  instance_id: z.string(),
+  status: z.enum(["completed", "failed"]),
+  error: z.string().optional(),
+  metadata: z.record(z.string()).optional(),
+  duration_ms: z.number().optional(),
+});
+
+type DelayedSnapshot = {
+  runnerId: string;
+  runFriendlyId: string;
+  snapshotFriendlyId: string;
+};
+
+type RunTraceContext = {
+  traceparent: string;
+  envId: string;
+  orgId: string;
+  projectId: string;
+};
+
 type WorkloadServerOptions = {
   port: number;
   host?: string;
   workerClient: SupervisorHttpClient;
   checkpointClient?: CheckpointClient;
+  computeManager?: ComputeWorkloadManager;
 };
 
 export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
   private checkpointClient?: CheckpointClient;
+  private computeManager?: ComputeWorkloadManager;
 
   private readonly logger = new SimpleStructuredLogger("workload-server");
 
@@ -85,6 +113,8 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
   >();
 
   private readonly workerClient: SupervisorHttpClient;
+  private readonly runTraceContexts = new Map<string, RunTraceContext>();
+  private readonly snapshotDelayWheel?: TimerWheel<DelayedSnapshot>;
 
   constructor(opts: WorkloadServerOptions) {
     super();
@@ -94,6 +124,15 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
 
     this.workerClient = opts.workerClient;
     this.checkpointClient = opts.checkpointClient;
+    this.computeManager = opts.computeManager;
+
+    if (this.computeManager && env.COMPUTE_SNAPSHOTS_ENABLED) {
+      this.snapshotDelayWheel = new TimerWheel<DelayedSnapshot>({
+        delayMs: env.COMPUTE_SNAPSHOT_DELAY_MS,
+        onExpire: (item) => this.dispatchComputeSnapshot(item.data),
+      });
+      this.snapshotDelayWheel.start();
+    }
 
     this.httpServer = this.createHttpServer({ host, port });
     this.websocketServer = this.createWebsocketServer();
@@ -126,7 +165,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
   }
 
   private createHttpServer({ host, port }: { host: string; port: number }) {
-    return new HttpServer({
+    const httpServer = new HttpServer({
       port,
       host,
       metrics: {
@@ -232,18 +271,6 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
           handler: async ({ reply, params, req }) => {
             this.logger.debug("Suspend request", { params, headers: req.headers });
 
-            if (!this.checkpointClient) {
-              reply.json(
-                {
-                  ok: false,
-                  error: "Checkpoints disabled",
-                } satisfies WorkloadSuspendRunResponseBody,
-                false,
-                400
-              );
-              return;
-            }
-
             const runnerId = this.runnerIdFromRequest(req);
             const deploymentVersion = this.deploymentVersionFromRequest(req);
             const projectRef = this.projectRefFromRequest(req);
@@ -264,6 +291,45 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
               return;
             }
 
+            if (this.snapshotDelayWheel && this.computeManager && env.COMPUTE_SNAPSHOTS_ENABLED) {
+              if (!env.TRIGGER_WORKLOAD_API_DOMAIN) {
+                this.logger.error(
+                  "TRIGGER_WORKLOAD_API_DOMAIN is not set, cannot create snapshot callback URL"
+                );
+                reply.json({ error: "Snapshot callbacks not configured" }, false, 500);
+                return;
+              }
+
+              // Compute mode: delay snapshot to avoid wasted work on short-lived waitpoints.
+              // If the run continues before the delay expires, the snapshot is cancelled.
+              reply.json({ ok: true } satisfies WorkloadSuspendRunResponseBody, false, 202);
+
+              this.snapshotDelayWheel.submit(params.runFriendlyId, {
+                runnerId,
+                runFriendlyId: params.runFriendlyId,
+                snapshotFriendlyId: params.snapshotFriendlyId,
+              });
+
+              this.logger.debug("Snapshot delayed", {
+                runId: params.runFriendlyId,
+                delayMs: env.COMPUTE_SNAPSHOT_DELAY_MS,
+              });
+
+              return;
+            }
+
+            if (!this.checkpointClient) {
+              reply.json(
+                {
+                  ok: false,
+                  error: "Checkpoints disabled",
+                } satisfies WorkloadSuspendRunResponseBody,
+                false,
+                400
+              );
+              return;
+            }
+
             reply.json(
               {
                 ok: true,
@@ -299,6 +365,11 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
           handler: async ({ req, reply, params }) => {
             this.logger.debug("Run continuation request", { params });
 
+            // Cancel any pending delayed snapshot for this run
+            if (this.snapshotDelayWheel?.cancel(params.runFriendlyId)) {
+              this.logger.debug("Cancelled delayed snapshot", { runId: params.runFriendlyId });
+            }
+
             const continuationResult = await this.workerClient.continueRunExecution(
               params.runFriendlyId,
               params.snapshotFriendlyId,
@@ -322,28 +393,6 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
           },
         }
       )
-      .route("/api/v1/workload-actions/runs/:runFriendlyId/snapshots/latest", "GET", {
-        paramsSchema: WorkloadActionParams.pick({ runFriendlyId: true }),
-        handler: async ({ req, reply, params }) => {
-          const latestSnapshotResponse = await this.workerClient.getLatestSnapshot(
-            params.runFriendlyId,
-            this.runnerIdFromRequest(req)
-          );
-
-          if (!latestSnapshotResponse.success) {
-            this.logger.error("Failed to get latest snapshot", {
-              runId: params.runFriendlyId,
-              error: latestSnapshotResponse.error,
-            });
-            reply.empty(500);
-            return;
-          }
-
-          reply.json({
-            execution: latestSnapshotResponse.data.execution,
-          } satisfies WorkloadRunLatestSnapshotResponseBody);
-        },
-      })
       .route(
         "/api/v1/workload-actions/runs/:runFriendlyId/snapshots/since/:snapshotFriendlyId",
         "GET",
@@ -369,23 +418,6 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
           },
         }
       )
-      .route("/api/v1/workload-actions/runs/:runFriendlyId/logs/debug", "POST", {
-        paramsSchema: WorkloadActionParams.pick({ runFriendlyId: true }),
-        bodySchema: WorkloadDebugLogRequestBody,
-        handler: async ({ req, reply, params, body }) => {
-          reply.empty(204);
-
-          if (!env.SEND_RUN_DEBUG_LOGS) {
-            return;
-          }
-
-          await this.workerClient.sendDebugLog(
-            params.runFriendlyId,
-            body,
-            this.runnerIdFromRequest(req)
-          );
-        },
-      })
       .route("/api/v1/workload-actions/deployments/:deploymentId/dequeue", "GET", {
         paramsSchema: z.object({
           deploymentId: z.string(),
@@ -410,6 +442,105 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
           reply.json(dequeueResponse.data satisfies WorkloadDequeueFromVersionResponseBody);
         },
       });
+
+    if (env.SEND_RUN_DEBUG_LOGS) {
+      httpServer.route("/api/v1/workload-actions/runs/:runFriendlyId/logs/debug", "POST", {
+        paramsSchema: WorkloadActionParams.pick({ runFriendlyId: true }),
+        bodySchema: WorkloadDebugLogRequestBody,
+        handler: async ({ req, reply, params, body }) => {
+          reply.empty(204);
+
+          await this.workerClient.sendDebugLog(
+            params.runFriendlyId,
+            body,
+            this.runnerIdFromRequest(req)
+          );
+        },
+      });
+    } else {
+      // Lightweight mock route without schemas
+      httpServer.route("/api/v1/workload-actions/runs/:runFriendlyId/logs/debug", "POST", {
+        handler: async ({ reply }) => {
+          reply.empty(204);
+        },
+      });
+    }
+
+    // Compute snapshot callback endpoint
+    httpServer.route("/api/v1/compute/snapshot-complete", "POST", {
+      bodySchema: ComputeSnapshotCallbackBody,
+      handler: async ({ reply, body }) => {
+        this.logger.debug("Compute snapshot callback", {
+          snapshotId: body.snapshot_id,
+          instanceId: body.instance_id,
+          status: body.status,
+          error: body.error,
+          metadata: body.metadata,
+          durationMs: body.duration_ms,
+        });
+
+        const runId = body.metadata?.runId;
+        const snapshotFriendlyId = body.metadata?.snapshotFriendlyId;
+
+        if (!runId || !snapshotFriendlyId) {
+          this.logger.error("Compute snapshot callback missing metadata", { body });
+          reply.empty(400);
+          return;
+        }
+
+        // Emit snapshot span (best-effort - requires trace context from dequeue on this instance)
+        this.#emitSnapshotSpan(runId, body.duration_ms, body.snapshot_id);
+
+        if (body.status === "completed") {
+          const result = await this.workerClient.submitSuspendCompletion({
+            runId,
+            snapshotId: snapshotFriendlyId,
+            body: {
+              success: true,
+              checkpoint: {
+                type: "KUBERNETES",
+                location: body.snapshot_id,
+              },
+            },
+          });
+
+          if (result.success) {
+            this.logger.debug("Suspend completion submitted", {
+              runId,
+              instanceId: body.instance_id,
+              snapshotId: body.snapshot_id,
+            });
+          } else {
+            this.logger.error("Failed to submit suspend completion", {
+              runId,
+              snapshotFriendlyId,
+              error: result.error,
+            });
+          }
+        } else {
+          const result = await this.workerClient.submitSuspendCompletion({
+            runId,
+            snapshotId: snapshotFriendlyId,
+            body: {
+              success: false,
+              error: body.error ?? "Snapshot failed",
+            },
+          });
+
+          if (!result.success) {
+            this.logger.error("Failed to submit suspend failure", {
+              runId,
+              snapshotFriendlyId,
+              error: result.error,
+            });
+          }
+        }
+
+        reply.empty(200);
+      },
+    });
+
+    return httpServer;
   }
 
   private createWebsocketServer() {
@@ -423,7 +554,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
     > = io.of("/workload");
 
     websocketServer.on("disconnect", (socket) => {
-      this.logger.log("[WS] disconnect", socket.id);
+      this.logger.verbose("[WS] disconnect", socket.id);
     });
     websocketServer.use(async (socket, next) => {
       const setSocketDataFromHeader = (
@@ -505,7 +636,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
         socket.data.runFriendlyId = undefined;
       };
 
-      socketLogger.log("wsServer socket connected", { ...getSocketMetadata() });
+      socketLogger.debug("wsServer socket connected", { ...getSocketMetadata() });
 
       // FIXME: where does this get set?
       if (socket.data.runFriendlyId) {
@@ -513,7 +644,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
       }
 
       socket.on("disconnecting", (reason, description) => {
-        socketLogger.log("Socket disconnecting", { ...getSocketMetadata(), reason, description });
+        socketLogger.verbose("Socket disconnecting", { ...getSocketMetadata(), reason, description });
 
         if (socket.data.runFriendlyId) {
           runDisconnected(socket.data.runFriendlyId);
@@ -521,7 +652,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
       });
 
       socket.on("disconnect", (reason, description) => {
-        socketLogger.log("Socket disconnected", { ...getSocketMetadata(), reason, description });
+        socketLogger.debug("Socket disconnected", { ...getSocketMetadata(), reason, description });
       });
 
       socket.on("error", (error) => {
@@ -542,7 +673,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
           ...message,
         });
 
-        log.log("Handling run:start");
+        log.debug("Handling run:start");
 
         try {
           runConnected(message.run.friendlyId);
@@ -558,10 +689,13 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
           ...message,
         });
 
-        log.log("Handling run:stop");
+        log.debug("Handling run:stop");
 
         try {
           runDisconnected(message.run.friendlyId);
+          // Don't delete trace context here - run:stop fires after each snapshot/shutdown
+          // but the run may be restored on a new VM and snapshot again. Trace context is
+          // re-populated on dequeue, and entries are small (4 strings per run).
         } catch (error) {
           log.error("run:stop error", { error });
         }
@@ -603,11 +737,92 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
     }
   }
 
+  /**
+   * Dispatch a compute snapshot request to the gateway. Called by the timer wheel
+   * when the delay expires, or immediately during drain.
+   */
+  private async dispatchComputeSnapshot(snapshot: DelayedSnapshot): Promise<void> {
+    if (!this.computeManager) return;
+
+    const callbackUrl = `${env.TRIGGER_WORKLOAD_API_PROTOCOL}://${env.TRIGGER_WORKLOAD_API_DOMAIN}:${env.TRIGGER_WORKLOAD_API_PORT_EXTERNAL}/api/v1/compute/snapshot-complete`;
+
+    const result = await this.computeManager.snapshot({
+      runnerId: snapshot.runnerId,
+      callbackUrl,
+      metadata: {
+        runId: snapshot.runFriendlyId,
+        snapshotFriendlyId: snapshot.snapshotFriendlyId,
+      },
+    });
+
+    if (!result) {
+      this.logger.error("Failed to request compute snapshot", {
+        runId: snapshot.runFriendlyId,
+        runnerId: snapshot.runnerId,
+      });
+    }
+  }
+
+  #emitSnapshotSpan(runFriendlyId: string, durationMs?: number, snapshotId?: string) {
+    if (!env.COMPUTE_TRACE_SPANS_ENABLED) return;
+
+    const ctx = this.runTraceContexts.get(runFriendlyId);
+    if (!ctx) return;
+
+    const parsed = parseTraceparent(ctx.traceparent);
+    if (!parsed) return;
+
+    const endEpochMs = Date.now();
+    const startEpochMs = durationMs ? endEpochMs - durationMs : endEpochMs;
+
+    const spanAttributes: Record<string, string | number | boolean> = {
+      "compute.type": "snapshot",
+    };
+
+    if (durationMs !== undefined) {
+      spanAttributes["compute.total_ms"] = durationMs;
+    }
+
+    if (snapshotId) {
+      spanAttributes["compute.snapshot_id"] = snapshotId;
+    }
+
+    const payload = buildOtlpTracePayload({
+      traceId: parsed.traceId,
+      parentSpanId: parsed.spanId,
+      spanName: "compute.snapshot",
+      startTimeMs: startEpochMs,
+      endTimeMs: endEpochMs,
+      resourceAttributes: {
+        "ctx.environment.id": ctx.envId,
+        "ctx.organization.id": ctx.orgId,
+        "ctx.project.id": ctx.projectId,
+        "ctx.run.id": runFriendlyId,
+      },
+      spanAttributes,
+    });
+
+    sendOtlpTrace(payload);
+  }
+
+  registerRunTraceContext(runFriendlyId: string, ctx: RunTraceContext) {
+    this.runTraceContexts.set(runFriendlyId, ctx);
+  }
+
   async start() {
     await this.httpServer.start();
   }
 
   async stop() {
+    const remaining = this.snapshotDelayWheel?.stop() ?? [];
+    if (remaining.length > 0) {
+      this.logger.info("Snapshot delay wheel stopped, dropped pending snapshots", {
+        count: remaining.length,
+      });
+      this.logger.debug("Dropped snapshot details", {
+        runs: remaining.map((item) => item.key),
+      });
+    }
     await this.httpServer.stop();
   }
 }

apps/webapp/.env

@@ -0,0 +1 @@
+../../.env

apps/webapp/.gitignore

@@ -9,7 +9,8 @@ node_modules
 
 /app/styles/tailwind.css
 
-
+# Ensure the .env symlink is not removed by accident
+!.env
 
 # Storybook build outputs
 build-storybook.log

apps/webapp/CLAUDE.md

@@ -0,0 +1,100 @@
+# Webapp
+
+Remix 2.1.0 app serving as the main API, dashboard, and orchestration engine. Uses an Express server (`server.ts`).
+
+## Verifying Changes
+
+**Never run `pnpm run build --filter webapp` to verify changes.** Building proves almost nothing about correctness. The webapp is an app, not a public package — use typecheck from the repo root:
+
+```bash
+pnpm run typecheck --filter webapp   # ~1-2 minutes
+```
+
+Only run typecheck after major changes (new files, significant refactors, schema changes). For small edits, trust the types and let CI catch issues.
+
+Note: Public packages (`packages/*`) use `build` instead. See the root CLAUDE.md for details.
+
+## Testing Dashboard Changes with Chrome DevTools MCP
+
+Use the `chrome-devtools` MCP server to visually verify local dashboard changes. The webapp must be running (`pnpm run dev --filter webapp` from repo root).
+
+### Login
+
+```
+1. mcp__chrome-devtools__new_page(url: "http://localhost:3030")
+   → Redirects to /login
+2. mcp__chrome-devtools__click the "Continue with Email" link
+3. mcp__chrome-devtools__fill the email field with "local@trigger.dev"
+4. mcp__chrome-devtools__click "Send a magic link"
+   → Auto-logs in and redirects to the dashboard (no email verification needed locally)
+```
+
+### Navigating and Verifying
+
+- **take_snapshot**: Get an a11y tree of the page (text content, element UIDs for interaction). Prefer this over screenshots for understanding page structure.
+- **take_screenshot**: Capture what the page looks like visually. Use to verify styling, layout, and visual changes.
+- **navigate_page**: Go to specific URLs, e.g. `http://localhost:3030/orgs/references-bc08/projects/hello-world-SiWs/env/dev/runs`
+- **click / fill**: Interact with elements using UIDs from `take_snapshot`.
+- **evaluate_script**: Run JS in the browser console for debugging.
+- **list_console_messages**: Check for console errors after navigating.
+
+### Tips
+
+- Snapshots can be very large on complex pages (200K+ chars). Use `take_screenshot` first to orient, then `take_snapshot` only when you need element UIDs to interact.
+- The local seeded user email is `local@trigger.dev`.
+- Dashboard URL pattern: `http://localhost:3030/orgs/{orgSlug}/projects/{projectSlug}/env/{envSlug}/{section}`
+
+## Key File Locations
+
+- **Trigger API**: `app/routes/api.v1.tasks.$taskId.trigger.ts`
+- **Batch trigger**: `app/routes/api.v1.tasks.batch.ts`
+- **OTEL endpoints**: `app/routes/otel.v1.logs.ts`, `app/routes/otel.v1.traces.ts`
+- **Prisma setup**: `app/db.server.ts`
+- **Run engine config**: `app/v3/runEngine.server.ts`
+- **Services**: `app/v3/services/**/*.server.ts`
+- **Presenters**: `app/v3/presenters/**/*.server.ts`
+
+## Route Convention
+
+Routes use Remix flat-file convention with dot-separated segments:
+`api.v1.tasks.$taskId.trigger.ts` -> `/api/v1/tasks/:taskId/trigger`
+
+## Environment Variables
+
+Access via `env` export from `app/env.server.ts`. **Never use `process.env` directly.**
+
+For testable code, **never import env.server.ts** in test files. Pass configuration as options instead:
+- `realtimeClient.server.ts` (testable service, takes config as constructor arg)
+- `realtimeClientGlobal.server.ts` (creates singleton with env config)
+
+## Run Engine 2.0
+
+The webapp integrates `@internal/run-engine` via `app/v3/runEngine.server.ts`. This is the singleton engine instance. Services in `app/v3/services/` call engine methods for all run lifecycle operations (triggering, completing, cancelling, etc.).
+
+The `engineVersion.server.ts` file determines V1 vs V2 for a given environment. New code should always target V2.
+
+## Background Workers
+
+Background job workers use `@trigger.dev/redis-worker`:
+- `app/v3/commonWorker.server.ts`
+- `app/v3/alertsWorker.server.ts`
+- `app/v3/batchTriggerWorker.server.ts`
+
+Do NOT add new jobs using zodworker/graphile-worker (legacy).
+
+## Real-time
+
+- Socket.io: `app/v3/handleSocketIo.server.ts`, `app/v3/handleWebsockets.server.ts`
+- Electric SQL: Powers real-time data sync for the dashboard
+
+## Legacy V1 Code
+
+The `app/v3/` directory name is misleading - most code is actively used by V2. Only these specific files are V1-only legacy:
+- `app/v3/marqs/` (old MarQS queue system)
+- `app/v3/legacyRunEngineWorker.server.ts`
+- `app/v3/services/triggerTaskV1.server.ts`
+- `app/v3/services/cancelTaskRunV1.server.ts`
+- `app/v3/authenticatedSocketConnection.server.ts`
+- `app/v3/sharedSocketConnection.ts`
+
+Some services (e.g., `cancelTaskRun.server.ts`, `batchTriggerV3.server.ts`) branch on `RunEngineVersion` to support both V1 and V2. When editing these, only modify V2 code paths.

apps/webapp/app/assets/icons/AbacusIcon.tsx

@@ -0,0 +1,71 @@
+export function AbacusIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <g clipPath="url(#clip0_16909_120578)">
+        <path
+          d="M4 3V21"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M20 21V3"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M8 5L8 6"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M14 5L14 6"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M15 10L15 11"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M9 10L9 11"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M12 15L12 16"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M8 15L8 16"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M3 21H21"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+      </g>
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/AiProviderIcons.tsx

@@ -0,0 +1,177 @@
+type IconProps = { className?: string };
+
+export function OpenAIIcon({ className }: IconProps) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M9.648 9.27004V7.36778C9.648 7.20757 9.70764 7.08738 9.84657 7.00738L13.6396 4.80477C14.1559 4.50443 14.7715 4.36434 15.4069 4.36434C17.7899 4.36434 19.2992 6.22658 19.2992 8.20884C19.2992 8.34898 19.2992 8.50919 19.2793 8.6694L15.3473 6.3466C15.1091 6.20651 14.8707 6.20651 14.6324 6.3466L9.648 9.27004ZM18.5048 16.6789V12.1334C18.5048 11.853 18.3855 11.6528 18.1473 11.5126L13.1629 8.58919L14.7913 7.64801C14.9303 7.56801 15.0495 7.56801 15.1884 7.64801L18.9814 9.85062C20.0737 10.4915 20.8084 11.853 20.8084 13.1745C20.8084 14.6962 19.9148 16.098 18.5048 16.6787V16.6789ZM8.47638 12.6742L6.848 11.7131C6.70907 11.6331 6.64943 11.5128 6.64943 11.3526V6.94746C6.64943 4.80498 8.2778 3.18295 10.4821 3.18295C11.3163 3.18295 12.0906 3.46334 12.7461 3.96392L8.834 6.24669C8.59578 6.38679 8.47658 6.58702 8.47658 6.86746V12.6743L8.47638 12.6742ZM11.9814 14.7165L9.648 13.395V10.5918L11.9814 9.27025L14.3146 10.5918V13.395L11.9814 14.7165ZM13.4807 20.8038C12.6466 20.8038 11.8723 20.5234 11.2168 20.0229L15.1288 17.7401C15.3671 17.6 15.4863 17.3997 15.4863 17.1193V11.3124L17.1346 12.2735C17.2735 12.3535 17.3331 12.4737 17.3331 12.634V17.0391C17.3331 19.1816 15.6848 20.8036 13.4807 20.8036V20.8038ZM8.77424 16.3385L4.9812 14.136C3.88892 13.4951 3.15426 12.1336 3.15426 10.8121C3.15426 9.27025 4.06775 7.88863 5.4776 7.30789V11.8733C5.4776 12.1537 5.59683 12.3539 5.83506 12.494L10.7997 15.3974L9.17134 16.3385C9.03241 16.4185 8.91317 16.4185 8.77424 16.3385ZM8.55592 19.6224C6.31192 19.6224 4.66364 17.9204 4.66364 15.8179C4.66364 15.6577 4.68355 15.4975 4.70329 15.3373L8.61535 17.62C8.85358 17.7602 9.09201 17.7602 9.33023 17.62L14.3146 14.7167V16.619C14.3146 16.7792 14.255 16.8994 14.1161 16.9794L10.3231 19.182C9.80672 19.4823 9.19109 19.6224 8.55571 19.6224H8.55592ZM13.4807 22.0052C15.8836 22.0052 17.8891 20.2832 18.3461 18.0004C20.5701 17.4197 21.9999 15.3172 21.9999 13.1747C21.9999 11.773 21.4043 10.4115 20.3319 9.43025C20.4312 9.00972 20.4908 8.58919 20.4908 8.16886C20.4908 5.30551 18.1872 3.16283 15.5261 3.16283C14.9901 3.16283 14.4737 3.24283 13.9574 3.42316C13.0636 2.54207 11.8324 1.98145 10.4821 1.98145C8.07927 1.98145 6.07369 3.70339 5.61678 5.98616C3.39269 6.5669 1.96289 8.6694 1.96289 10.8119C1.96289 12.2136 2.55857 13.5751 3.63095 14.5563C3.53166 14.9768 3.47207 15.3974 3.47207 15.8177C3.47207 18.6811 5.77567 20.8237 8.43668 20.8237C8.97277 20.8237 9.48911 20.7437 10.0055 20.5634C10.899 21.4445 12.1302 22.0052 13.4807 22.0052Z"
+        fill="currentColor"
+      />
+    </svg>
+  );
+}
+
+export function AnthropicIcon({ className }: IconProps) {
+  return (
+    <svg
+      className={className}
+      viewBox="0 0 24 24"
+      fill="currentColor"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path d="M17.3041 3.541h-3.6718l6.696 16.918H24Zm-10.6082 0L0 20.459h3.7442l1.3693-3.5527h7.0052l1.3693 3.5528h3.7442L10.5363 3.5409Zm-.3712 10.2232 2.2914-5.9456 2.2914 5.9456Z" />
+    </svg>
+  );
+}
+
+export function GeminiIcon({ className }: IconProps) {
+  return (
+    <svg
+      className={className}
+      viewBox="0 0 24 24"
+      fill="currentColor"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path d="M12 0C12 6.627 6.627 12 0 12c6.627 0 12 5.373 12 12 0-6.627 5.373-12 12-12-6.627 0-12-5.373-12-12Z" />
+    </svg>
+  );
+}
+
+export function LlamaIcon({ className }: IconProps) {
+  return (
+    <svg
+      className={className}
+      viewBox="0 0 12 12"
+      fill="currentColor"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        fill-rule="evenodd"
+        clip-rule="evenodd"
+        d="M3.4485 2C4.406 2 5.2065 2.466 6.1635 3.688L6.3045 3.5015C6.3995 3.3785 6.496 3.2595 6.5945 3.1465L6.751 2.9715C7.294 2.394 7.896 2 8.6125 2C9.249 2 9.847 2.2785 10.358 2.758L10.467 2.8645C11.332 3.747 11.9255 5.2195 11.9935 6.8775L11.999 7.0735L12 7.1985C12 7.949 11.86 8.578 11.591 9.0485L11.521 9.1635L11.467 9.24C11.3165 9.45 11.135 9.619 10.924 9.7445L10.7915 9.8155L10.748 9.8355C10.6986 9.85749 10.6482 9.87718 10.597 9.8945C10.3825 9.96542 10.1579 10.0006 9.932 9.9985C9.67 9.9985 9.434 9.965 9.213 9.891C8.906 9.789 8.6315 9.611 8.35 9.333L8.2365 9.2155C7.86 8.8095 7.4695 8.2275 6.99 7.4225L6.275 6.2175L6.003 5.77L5.12 7.335L4.9485 7.631C3.7985 9.578 3.1135 10 2.178 10C1.573 10 1.0755 9.79 0.71 9.409L0.626 9.317C0.384 9.0305 0.2075 8.6615 0.1045 8.2225L0.071 8.0625C0.0323456 7.84982 0.00961585 7.63456 0.003 7.4185L0 7.234C0.001 6.8615 0.03 6.489 0.087 6.119L0.137 5.8325C0.286 5.0675 0.551 4.3535 0.905 3.754L1.0095 3.584C1.598 2.669 2.404 2.0575 3.317 2.004L3.4485 2ZM3.432 3.3075L3.3315 3.3125C2.9165 3.354 2.5285 3.649 2.2055 4.101L2.1365 4.2005L2.1315 4.2095C1.7965 4.718 1.539 5.3985 1.4035 6.132L1.4015 6.143C1.33323 6.5148 1.29859 6.89199 1.298 7.27L1.299 7.364C1.301 7.454 1.3075 7.544 1.319 7.634L1.3405 7.7795C1.3865 8.031 1.469 8.2335 1.5835 8.3835L1.642 8.452C1.7935 8.6135 1.991 8.698 2.227 8.698C2.777 8.698 3.125 8.36 4.075 6.8775L5.1625 5.1775L5.3895 4.827L5.32 4.728C4.555 3.65 4.042 3.3075 3.432 3.3075ZM8.53 3.0315L8.442 3.035C8.1245 3.059 7.8305 3.2145 7.532 3.5015L7.434 3.6005C7.2145 3.8315 6.9905 4.1325 6.7505 4.504L6.8835 4.703C6.9735 4.84 7.0645 4.983 7.1585 5.132L7.305 5.3695L8.003 6.537L8.3505 7.094C8.642 7.557 8.8655 7.894 9.0545 8.135L9.161 8.266C9.302 8.429 9.4255 8.536 9.5495 8.6025L9.6005 8.6275C9.714 8.6775 9.829 8.6965 9.9595 8.6965C10.0475 8.6975 10.1345 8.685 10.2185 8.66C10.3875 8.608 10.5235 8.5 10.625 8.3415L10.6725 8.26L10.711 8.179C10.808 7.9495 10.856 7.649 10.856 7.2865L10.853 7.062C10.813 5.6265 10.384 4.376 9.753 3.663L9.665 3.5685C9.33 3.227 8.943 3.0315 8.53 3.0315Z"
+        fill="currentColor"
+      />
+    </svg>
+  );
+}
+
+export function DeepseekIcon({ className }: IconProps) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <g clip-path="url(#clip0_20374_57805)">
+        <path
+          fill-rule="evenodd"
+          clip-rule="evenodd"
+          d="M23.7479 4.48176C23.4939 4.35776 23.384 4.59476 23.236 4.71576C23.185 4.75476 23.142 4.80576 23.099 4.85176C22.727 5.24876 22.293 5.50876 21.726 5.47776C20.897 5.43176 20.189 5.69176 19.563 6.32576C19.43 5.54376 18.988 5.07776 18.316 4.77776C17.964 4.62176 17.608 4.46676 17.361 4.12776C17.189 3.88676 17.142 3.61776 17.056 3.35376C17.001 3.19376 16.946 3.03076 16.763 3.00376C16.563 2.97276 16.4849 3.13976 16.4069 3.27976C16.094 3.85176 15.9729 4.48176 15.9849 5.11976C16.0119 6.55576 16.618 7.69976 17.823 8.51276C17.96 8.60576 17.995 8.69976 17.952 8.83576C17.87 9.11576 17.772 9.38776 17.686 9.66876C17.631 9.84776 17.549 9.88576 17.357 9.80876C16.7082 9.52995 16.1189 9.12939 15.6209 8.62876C14.7639 7.80076 13.9899 6.88676 13.0239 6.17076C12.8001 6.00537 12.5703 5.84827 12.3349 5.69976C11.3499 4.74276 12.4649 3.95676 12.7229 3.86376C12.9929 3.76576 12.8159 3.43176 11.9439 3.43576C11.0719 3.43976 10.2739 3.73076 9.25695 4.11976C9.10582 4.17767 8.95033 4.22348 8.79195 4.25676C7.84158 4.07769 6.8696 4.0433 5.90895 4.15476C4.02395 4.36476 2.51895 5.25676 1.41195 6.77776C0.0819496 8.60576 -0.23105 10.6838 0.15195 12.8498C0.55495 15.1338 1.72095 17.0248 3.51195 18.5028C5.36995 20.0358 7.50895 20.7868 9.94995 20.6428C11.4319 20.5578 13.0829 20.3588 14.9439 18.7828C15.4139 19.0168 15.906 19.1098 16.724 19.1798C17.354 19.2388 17.96 19.1498 18.429 19.0518C19.164 18.8958 19.1129 18.2148 18.8479 18.0908C16.693 17.0868 17.166 17.4958 16.735 17.1648C17.831 15.8688 19.481 14.5228 20.127 10.1618C20.177 9.81476 20.134 9.59676 20.127 9.31676C20.123 9.14676 20.162 9.07976 20.357 9.06076C20.898 9.00463 21.4228 8.84327 21.902 8.58576C23.298 7.82276 23.862 6.57076 23.995 5.06876C24.015 4.83876 23.9909 4.60276 23.7479 4.48176ZM11.5809 17.9998C9.49195 16.3578 8.47895 15.8168 8.06095 15.8398C7.66895 15.8638 7.73995 16.3108 7.82595 16.6028C7.91595 16.8908 8.03295 17.0888 8.19695 17.3418C8.31095 17.5088 8.38895 17.7578 8.08395 17.9448C7.41095 18.3608 6.24195 17.8048 6.18695 17.7778C4.82595 16.9758 3.68695 15.9178 2.88595 14.4708C2.11195 13.0778 1.66195 11.5838 1.58795 9.98876C1.56795 9.60276 1.68095 9.46676 2.06495 9.39676C2.56906 9.30029 3.08558 9.28711 3.59395 9.35776C5.72595 9.66976 7.53995 10.6228 9.06195 12.1318C9.92995 12.9918 10.5869 14.0188 11.2639 15.0228C11.9839 16.0888 12.7579 17.1048 13.7439 17.9368C14.0919 18.2288 14.3689 18.4508 14.6349 18.6138C13.8329 18.7038 12.4949 18.7238 11.5809 17.9998ZM12.5809 11.5598C12.5808 11.5101 12.5927 11.4611 12.6157 11.4171C12.6387 11.373 12.672 11.3353 12.7129 11.307C12.7538 11.2787 12.8009 11.2609 12.8502 11.2549C12.8995 11.2489 12.9495 11.2551 12.9959 11.2728C13.0551 11.294 13.1062 11.3331 13.142 11.3848C13.1779 11.4364 13.1967 11.4979 13.1959 11.5608C13.1961 11.6014 13.1881 11.6416 13.1726 11.6791C13.157 11.7166 13.1341 11.7506 13.1053 11.7792C13.0764 11.8078 13.0422 11.8303 13.0045 11.8455C12.9669 11.8607 12.9266 11.8683 12.8859 11.8678C12.8457 11.8679 12.8057 11.86 12.7685 11.8445C12.7313 11.829 12.6976 11.8063 12.6693 11.7776C12.641 11.7489 12.6186 11.7149 12.6037 11.6775C12.5887 11.6401 12.5803 11.6 12.5809 11.5598ZM15.6909 13.1558C15.4909 13.2368 15.2919 13.3068 15.1009 13.3158C14.8136 13.3258 14.5316 13.236 14.3029 13.0618C14.0289 12.8318 13.8329 12.7038 13.7509 12.3038C13.7227 12.1083 13.7281 11.9094 13.7669 11.7158C13.8369 11.3888 13.7589 11.1788 13.5279 10.9888C13.3409 10.8328 13.1019 10.7898 12.8399 10.7898C12.7502 10.7845 12.6631 10.7578 12.5859 10.7118C12.4759 10.6578 12.3859 10.5218 12.4719 10.3538C12.4999 10.2998 12.6319 10.1678 12.6639 10.1438C13.0199 9.94176 13.4309 10.0078 13.8099 10.1598C14.1619 10.3038 14.4279 10.5678 14.8109 10.9418C15.2019 11.3928 15.2729 11.5178 15.4959 11.8558C15.6719 12.1208 15.8319 12.3928 15.9409 12.7038C16.0079 12.8988 15.9219 13.0578 15.6909 13.1558Z"
+          fill="currentColor"
+        />
+      </g>
+      <defs>
+        <clipPath id="clip0_20374_57805">
+          <rect width="24" height="24" fill="currentColor" />
+        </clipPath>
+      </defs>
+    </svg>
+  );
+}
+
+export function XAIIcon({ className }: IconProps) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M17.7329 8.44672L18.0784 22.0139H20.8452L21.1911 3.50781L17.7329 8.44672Z"
+        fill="currentColor"
+      />
+      <path d="M21.1911 2H16.9692L10.3442 11.4621L12.4552 14.4768L21.1911 2Z" fill="currentColor" />
+      <path
+        d="M2.95508 22.0136H7.17691L9.28824 18.9989L7.17691 15.9839L2.95508 22.0136Z"
+        fill="currentColor"
+      />
+      <path
+        d="M2.95508 8.44629L12.4546 22.0134H16.6764L7.17691 8.44629H2.95508Z"
+        fill="currentColor"
+      />
+    </svg>
+  );
+}
+
+export function PerplexityIcon({ className }: IconProps) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        fill-rule="evenodd"
+        clip-rule="evenodd"
+        d="M18.4875 2V8.06H20.75V16.6833H18.3042V22L12.44 16.8383V21.9592H11.5308V16.8325L5.66 22V16.6125H3.25V7.99H5.65333V2L11.5308 7.41167V2.15833H12.4392V7.56667L18.4875 2ZM12.44 9.53667V15.6358L17.395 19.9975V14.0333L12.44 9.53667ZM11.5242 9.47L6.56917 13.9683V19.9975L11.5242 15.6358V9.47083V9.47ZM18.3042 15.7867H19.8408V8.9575H13.2167L18.3042 13.5742V15.7867ZM10.8192 8.88667H4.15833V15.7158H5.65833V13.5692L10.8183 8.88583L10.8192 8.88667ZM6.5625 4.06333V7.98833H10.825L6.5625 4.06333ZM17.5783 4.06333L13.3158 7.98833H17.5783V4.06333Z"
+        fill="currentColor"
+      />
+    </svg>
+  );
+}
+
+export function CerebrasIcon({ className }: IconProps) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        fill-rule="evenodd"
+        clip-rule="evenodd"
+        d="M11.6535 20.6834C10.4382 20.6834 9.28717 20.4401 8.23625 20.0036C6.66345 19.3453 5.31944 18.2433 4.36862 16.8551C3.4178 15.4669 2.86732 13.7997 2.86732 11.9964C2.86732 10.7943 3.11039 9.65655 3.56078 8.61182C4.22564 7.0519 5.3409 5.72809 6.7421 4.7907C8.1433 3.85331 9.83047 3.30948 11.6535 3.30948V2C10.2594 2 8.92972 2.27907 7.71437 2.78712C5.8985 3.54562 4.35432 4.81217 3.26767 6.40788C2.17386 8.00356 1.5376 9.92844 1.5376 11.9964C1.5376 13.3774 1.82356 14.6941 2.33114 15.8891C3.09609 17.6851 4.38291 19.2093 5.99144 20.2898C7.60713 21.3703 9.55167 22 11.6463 22V20.6834H11.6535Z"
+        fill="currentColor"
+      />
+      <path
+        fill-rule="evenodd"
+        clip-rule="evenodd"
+        d="M7.14949 17.3272C6.32735 16.6403 5.71253 15.8102 5.2979 14.9014C4.88323 13.9927 4.66877 13.0124 4.66877 12.0249C4.66877 11.2378 4.8046 10.4506 5.06911 9.69931C5.34079 8.94794 5.74113 8.23241 6.29159 7.58122C6.9779 6.7655 7.81433 6.15012 8.72225 5.73508C9.63021 5.32005 10.6239 5.11254 11.6105 5.11254C12.3969 5.11254 13.1904 5.24849 13.9411 5.51325C14.6989 5.78516 15.4138 6.18588 16.0643 6.7297L16.9151 5.72077C16.143 5.07676 15.2851 4.59018 14.3843 4.27533C13.4835 3.95332 12.547 3.7959 11.6105 3.7959C10.4309 3.7959 9.25846 4.04634 8.17178 4.54009C7.08514 5.03382 6.09141 5.77087 5.27643 6.73687C4.62587 7.50967 4.14689 8.36119 3.82518 9.25563C3.50347 10.1501 3.34619 11.0875 3.34619 12.0249C3.34619 13.1984 3.59641 14.3719 4.08969 15.4524C4.58298 16.5329 5.32649 17.5276 6.29876 18.3362L7.14949 17.3272Z"
+        fill="currentColor"
+      />
+      <path
+        fill-rule="evenodd"
+        clip-rule="evenodd"
+        d="M9.18714 16.4758C8.32211 16.0179 7.64298 15.3524 7.17829 14.5724C6.7136 13.7925 6.47052 12.8909 6.47052 11.9821C6.47052 11.1807 6.65641 10.3721 7.06388 9.62074C7.52144 8.75492 8.19345 8.08228 8.97983 7.6243C9.76624 7.1592 10.667 6.9159 11.5821 6.9159C12.3828 6.9159 13.1978 7.10194 13.9556 7.50269L14.5704 6.33631C13.6196 5.83541 12.5901 5.59212 11.5749 5.59927C10.424 5.59927 9.28725 5.90697 8.30069 6.48655C7.31412 7.06618 6.46339 7.92487 5.89146 9.00535C5.39101 9.95706 5.14795 10.9803 5.14795 11.9821C5.14795 13.127 5.45536 14.2576 6.04159 15.2379C6.62782 16.2254 7.48568 17.0626 8.57236 17.635L9.18714 16.4758Z"
+        fill="currentColor"
+      />
+      <path
+        fill-rule="evenodd"
+        clip-rule="evenodd"
+        d="M11.6608 15.2165C11.2104 15.2165 10.7815 15.1235 10.3955 14.9589C9.80924 14.7156 9.31596 14.3005 8.96564 13.7782C8.61536 13.2558 8.40804 12.6333 8.40804 11.9606C8.40804 11.5098 8.50095 11.0805 8.66537 10.6941C8.90845 10.1145 9.32309 9.61359 9.84496 9.26297C10.3669 8.91235 10.9888 8.70484 11.6608 8.70484V7.38818C11.0317 7.38818 10.4312 7.517 9.88072 7.74597C9.05858 8.09659 8.36511 8.66905 7.87183 9.39892C7.37142 10.136 7.08545 11.0233 7.08545 11.9678C7.08545 12.5975 7.21412 13.1986 7.4429 13.7496C7.79322 14.5725 8.37228 15.2666 9.10147 15.7603C9.83067 16.2469 10.71 16.5331 11.6608 16.5331V15.2165Z"
+        fill="currentColor"
+      />
+      <path
+        fill-rule="evenodd"
+        clip-rule="evenodd"
+        d="M12.7332 10.9234C12.5831 10.766 12.4187 10.6372 12.2542 10.5442C12.0898 10.4511 11.9183 10.401 11.7395 10.401C11.4965 10.401 11.2891 10.444 11.0961 10.5299C10.9102 10.6157 10.7458 10.7302 10.61 10.8805C10.4741 11.0236 10.374 11.1953 10.3026 11.3814C10.2311 11.5674 10.2025 11.7678 10.2025 11.9681C10.2025 12.1685 10.2382 12.3689 10.3026 12.5549C10.374 12.7409 10.4741 12.9127 10.61 13.0558C10.7458 13.1989 10.9031 13.3206 11.0961 13.4064C11.282 13.4923 11.4965 13.5352 11.7395 13.5352C11.9397 13.5352 12.1327 13.4923 12.3043 13.4136C12.4759 13.3277 12.626 13.2061 12.7475 13.0486L13.6197 13.986C13.491 14.1148 13.3409 14.2293 13.1693 14.3223C12.9978 14.4154 12.8262 14.4941 12.6546 14.5513C12.483 14.6086 12.3114 14.6515 12.1542 14.673C11.9969 14.7016 11.8539 14.7087 11.7395 14.7087C11.3463 14.7087 10.9746 14.6443 10.6314 14.5155C10.2811 14.3868 9.98084 14.2007 9.73064 13.9574C9.47326 13.7213 9.27311 13.4279 9.12298 13.0916C8.97285 12.7553 8.90137 12.376 8.90137 11.9681C8.90137 11.5531 8.97285 11.181 9.12298 10.8447C9.27311 10.5084 9.47326 10.2222 9.73064 9.97887C9.98801 9.74274 10.2883 9.55669 10.6314 9.42075C10.9817 9.29193 11.3535 9.22754 11.7395 9.22754C12.0755 9.22754 12.4115 9.29193 12.7475 9.42075C13.0835 9.54953 13.3838 9.7499 13.634 10.0218L12.7332 10.9234Z"
+        fill="currentColor"
+      />
+    </svg>
+  );
+}
+
+export function MistralIcon({ className }: IconProps) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
+      <path d="M7.28516 3.74658H4.1416V6.8809H7.28516V3.74658Z" />
+      <path d="M19.8579 3.74658H16.7144V6.8809H19.8579V3.74658Z" />
+      <path d="M10.4277 6.88086H4.1416V10.0152H10.4277V6.88086Z" />
+      <path d="M19.8588 6.88086H13.5728V10.0152H19.8588V6.88086Z" />
+      <path d="M19.8564 10.0137H4.1416V13.148H19.8564V10.0137Z" />
+      <path d="M7.28516 13.1484H4.1416V16.2828H7.28516V13.1484Z" />
+      <path d="M13.5723 13.1484H10.4287V16.2828H13.5723V13.1484Z" />
+      <path d="M19.8579 13.1484H16.7144V16.2828H19.8579V13.1484Z" />
+      <path d="M10.4286 16.2812H1V19.4157H10.4286V16.2812Z" />
+      <path d="M23.0024 16.2812H13.5728V19.4157H23.0024V16.2812Z" />
+    </svg>
+  );
+}
+
+export function AzureIcon({ className }: IconProps) {
+  return (
+    <svg
+      className={className}
+      viewBox="0 0 24 24"
+      fill="currentColor"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path d="M13.05 4.24L6.56 18.05l2.77-.46 1.89-4.55 4.2 5.19.03-.01 2.99.53L13.05 4.24zm-4.1 7.51L2.49 20.21h5.23l1.23-3.04v-5.42z" />
+    </svg>
+  );
+}
+

apps/webapp/app/assets/icons/AnthropicLogoIcon.tsx

@@ -0,0 +1,12 @@
+export function AnthropicLogoIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      viewBox="0 0 24 24"
+      fill="currentColor"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path d="M17.3041 3.541h-3.6718l6.696 16.918H24Zm-10.6082 0L0 20.459h3.7442l1.3693-3.5527h7.0052l1.3693 3.5528h3.7442L10.5363 3.5409Zm-.3712 10.2232 2.2914-5.9456 2.2914 5.9456Z" />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/ArrowTopRightBottomLeftIcon.tsx

@@ -0,0 +1,22 @@
+export function ArrowTopRightBottomLeftIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M14.8258 10.5L20.125 5.20083V8.5625C20.125 9.08027 20.5447 9.5 21.0625 9.5C21.5803 9.5 22 9.08027 22 8.5625V2.9375C22 2.41973 21.5803 2 21.0625 2H15.4375C14.9197 2 14.5 2.41973 14.5 2.9375C14.5 3.45527 14.9197 3.875 15.4375 3.875H18.7992L13.5 9.17417C13.1339 9.54029 13.1339 10.1339 13.5 10.5C13.8661 10.8661 14.4597 10.8661 14.8258 10.5Z"
+        fill="currentColor"
+      />
+      <path
+        d="M2 21.0625V15.4375C2 14.9197 2.41973 14.5 2.9375 14.5C3.45527 14.5 3.875 14.9197 3.875 15.4375V18.7992L9.17417 13.5C9.54029 13.1339 10.1339 13.1339 10.5 13.5C10.8661 13.8661 10.8661 14.4597 10.5 14.8258L5.20083 20.125H8.5625C9.08027 20.125 9.5 20.5447 9.5 21.0625C9.5 21.5803 9.08027 22 8.5625 22H2.9375C2.69757 22 2.45765 21.9085 2.27459 21.7254C2.1847 21.6355 2.11689 21.5319 2.07114 21.4214C2.0253 21.3108 2 21.1896 2 21.0625Z"
+        fill="currentColor"
+      />
+      <path
+        d="M14.8258 10.5L20.125 5.20083V10C20.125 10.5178 20.5447 10.9375 21.0625 10.9375C21.5803 10.9375 22 10.5178 22 10V2.9375C22 2.41973 21.5803 2 21.0625 2H14C13.4822 2 13.0625 2.41973 13.0625 2.9375C13.0625 3.45527 13.4822 3.875 14 3.875H18.7992L13.5 9.17417C13.1339 9.54029 13.1339 10.1339 13.5 10.5C13.8661 10.8661 14.4597 10.8661 14.8258 10.5Z"
+        fill="currentColor"
+      />
+      <path
+        d="M2 21.0625V13.9375C2 13.4197 2.41973 13 2.9375 13C3.45527 13 3.875 13.4197 3.875 13.9375V18.7992L9.17417 13.5C9.54029 13.1339 10.1339 13.1339 10.5 13.5C10.8661 13.8661 10.8661 14.4597 10.5 14.8258L5.20083 20.125H10.0625C10.5803 20.125 11 20.5447 11 21.0625C11 21.5803 10.5803 22 10.0625 22H2.9375C2.69757 22 2.45765 21.9085 2.27459 21.7254C2.1847 21.6355 2.11689 21.5319 2.07114 21.4214C2.0253 21.3108 2 21.1896 2 21.0625Z"
+        fill="currentColor"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/ChevronExtraSmallDown.tsx

@@ -0,0 +1,13 @@
+export function ChevronExtraSmallDown({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M15 6L9.75926 12.1142C9.36016 12.5798 8.63984 12.5798 8.24074 12.1142L3 6"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeMiterlimit="1.00244"
+        strokeLinecap="round"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/ChevronExtraSmallUp.tsx

@@ -0,0 +1,13 @@
+export function ChevronExtraSmallUp({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M3 12L8.24074 5.8858C8.63984 5.42019 9.36016 5.42019 9.75926 5.8858L15 12"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeMiterlimit="1.00244"
+        strokeLinecap="round"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/ConcurrencyIcon.tsx

@@ -0,0 +1,13 @@
+export function ConcurrencyIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <circle cx="3.75" cy="3.75" r="2.25" fill="currentColor" />
+      <circle cx="9" cy="3.75" r="2.25" fill="currentColor" />
+      <circle cx="14.25" cy="3.75" r="2.25" fill="currentColor" />
+      <circle cx="3.75" cy="9" r="2.25" fill="currentColor" />
+      <circle cx="9" cy="9" r="2.25" fill="currentColor" />
+      <circle cx="9" cy="14.25" r="1.75" stroke="currentColor" />
+      <circle cx="14.25" cy="9" r="2.25" fill="currentColor" />
+    </svg>
+  );
+}