Merge branch 'main' into fix/tri-6732-hover-vertical-timeline-does-not-update-on-task-runs

## Summary
- **Fix Docker publish automation**: The `v.docker.*` tags pushed by the
release workflow using `GITHUB_TOKEN` don't trigger the publish workflow
(GitHub Actions limitation to prevent infinite loops). Added a
`workflow_call` to `publish.yml` directly from the release job so Docker
images are built automatically after npm publish. Tags are still pushed
for reference.
- **Fix worker Containerfiles**: The coordinator, docker-provider, and
kubernetes-provider builds have been failing since the superjson
vendoring change in `@trigger.dev/core` (#2949). The Containerfiles now
run `bundle-vendor` before `build:bundle` to generate the vendor files
that esbuild needs.

### Context
- Docker images on GHCR have been stuck at v4.3.0 — v4.3.1, v4.3.2,
v4.3.3 tags existed on GitHub but never triggered publish runs
- The worker builds (publish-worker) have been failing on every push to
main since Jan 30

## Test plan
- [x] Verified kubernetes-provider Containerfile builds locally with the
fix
- [x] Manually dispatched publish workflow for v4.3.1 — all jobs
succeeded
<!-- devin-review-badge-begin -->

---

<a
href="https://app.devin.ai/review/triggerdotdev/trigger.dev/pull/3013"
target="_blank">
  <picture>
<source media="(prefers-color-scheme: dark)"
srcset="https://static.devin.ai/assets/gh-open-in-devin-review-dark.svg?v=1">
<img
src="https://static.devin.ai/assets/gh-open-in-devin-review-light.svg?v=1"
alt="Open with Devin">
  </picture>
</a>
<!-- devin-review-badge-end -->

.changeset/add-debounce-maxdelay.md

@@ -0,0 +1,16 @@
+---
+"@trigger.dev/core": patch
+"@trigger.dev/sdk": patch
+---
+
+Add `maxDelay` option to debounce feature. This allows setting a maximum time limit for how long a debounced run can be delayed, ensuring execution happens within a specified window even with continuous triggers.
+
+```typescript
+await myTask.trigger(payload, {
+  debounce: {
+    key: "my-key",
+    delay: "5s",
+    maxDelay: "30m", // Execute within 30 minutes regardless of continuous triggers
+  },
+});
+```

.changeset/calm-hooks-wait.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/react-hooks": patch
+---
+
+Fix `onComplete` callback firing prematurely when the realtime stream disconnects before the run finishes.

.changeset/config.json

@@ -1,24 +1,25 @@
 {
   "$schema": "https://unpkg.com/@changesets/config@2.2.0/schema.json",
-  "changelog": "@changesets/cli/changelog",
-  "commit": false,
-  "fixed": [
-    [
-      "@trigger.dev/*"
-    ]
+  "changelog": [
+    "@remix-run/changelog-github",
+    {
+      "repo": "triggerdotdev/trigger.dev"
+    }
   ],
+  "commit": false,
+  "fixed": [["@trigger.dev/*", "trigger.dev"]],
   "linked": [],
   "access": "public",
   "baseBranch": "main",
   "updateInternalDependencies": "patch",
   "ignore": [
     "webapp",
-    "emails",
-    "proxy",
-    "yalt",
-    "@trigger.dev/database"
+    "coordinator",
+    "docker-provider",
+    "kubernetes-provider",
+    "supervisor"
   ],
   "___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH": {
     "onlyUpdatePeerDependentsWhenOutOfRange": true
   }
-}
+}

.changeset/consistent-stream-targets.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Aligned the SDK's `getRunIdForOptions` logic with the Core package to handle semantic targets (`root`, `parent`) in root tasks.

.changeset/export-start-attempt-hook-type.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Export `AnyOnStartAttemptHookFunction` type to allow defining `onStartAttempt` hooks for individual tasks.

.changeset/fix-dead-process-execute-hang.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix runner getting stuck indefinitely when `execute()` is called on a dead child process.

.changeset/vendor-superjson-esm-fix.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/core": patch
+---
+
+fix: vendor superjson to fix ESM/CJS compatibility
+
+Bundle superjson during build to avoid `ERR_REQUIRE_ESM` errors on Node.js versions that don't support `require(ESM)` by default (< 22.12.0) and AWS Lambda which intentionally disables it.

.claude/skills/trigger-dev-tasks/SKILL.md

@@ -0,0 +1,200 @@
+---
+name: trigger-dev-tasks
+description: Use this skill when writing, designing, or optimizing Trigger.dev background tasks and workflows. This includes creating reliable async tasks, implementing AI workflows, setting up scheduled jobs, structuring complex task hierarchies with subtasks, configuring build extensions for tools like ffmpeg or Puppeteer/Playwright, and handling task schemas with Zod validation.
+allowed-tools: Read, Write, Edit, Glob, Grep, Bash
+---
+
+# Trigger.dev Task Expert
+
+You are an expert Trigger.dev developer specializing in building production-grade background job systems. Tasks deployed to Trigger.dev run in Node.js 21+ and use the `@trigger.dev/sdk` package.
+
+## Critical Rules
+
+1. **Always use `@trigger.dev/sdk`** - Never use `@trigger.dev/sdk/v3` or deprecated `client.defineJob` pattern
+2. **Never use `node-fetch`** - Use the built-in `fetch` function
+3. **Export all tasks** - Every task must be exported, including subtasks
+4. **Never wrap wait/trigger calls in Promise.all** - `triggerAndWait`, `batchTriggerAndWait`, and `wait.*` calls cannot be wrapped in `Promise.all` or `Promise.allSettled`
+
+## Basic Task Pattern
+
+```ts
+import { task } from "@trigger.dev/sdk";
+
+export const processData = task({
+  id: "process-data",
+  retry: {
+    maxAttempts: 10,
+    factor: 1.8,
+    minTimeoutInMs: 500,
+    maxTimeoutInMs: 30_000,
+  },
+  run: async (payload: { userId: string; data: any[] }) => {
+    console.log(`Processing ${payload.data.length} items`);
+    return { processed: payload.data.length };
+  },
+});
+```
+
+## Schema Task (with validation)
+
+```ts
+import { schemaTask } from "@trigger.dev/sdk";
+import { z } from "zod";
+
+export const validatedTask = schemaTask({
+  id: "validated-task",
+  schema: z.object({
+    name: z.string(),
+    email: z.string().email(),
+  }),
+  run: async (payload) => {
+    // Payload is automatically validated and typed
+    return { message: `Hello ${payload.name}` };
+  },
+});
+```
+
+## Triggering Tasks
+
+### From Backend Code (type-only import to prevent dependency leakage)
+
+```ts
+import { tasks } from "@trigger.dev/sdk";
+import type { processData } from "./trigger/tasks";
+
+const handle = await tasks.trigger<typeof processData>("process-data", {
+  userId: "123",
+  data: [{ id: 1 }],
+});
+```
+
+### From Inside Tasks
+
+```ts
+export const parentTask = task({
+  id: "parent-task",
+  run: async (payload) => {
+    // Trigger and wait - returns Result object, NOT direct output
+    const result = await childTask.triggerAndWait({ data: "value" });
+    if (result.ok) {
+      console.log("Output:", result.output);
+    } else {
+      console.error("Failed:", result.error);
+    }
+
+    // Or unwrap directly (throws on error)
+    const output = await childTask.triggerAndWait({ data: "value" }).unwrap();
+  },
+});
+```
+
+## Idempotency (Critical for Retries)
+
+Always use idempotency keys when triggering tasks from inside other tasks:
+
+```ts
+import { idempotencyKeys } from "@trigger.dev/sdk";
+
+export const paymentTask = task({
+  id: "process-payment",
+  run: async (payload: { orderId: string }) => {
+    // Scoped to current run - survives retries
+    const key = await idempotencyKeys.create(`payment-${payload.orderId}`);
+
+    await chargeCustomer.trigger(payload, {
+      idempotencyKey: key,
+      idempotencyKeyTTL: "24h",
+    });
+  },
+});
+```
+
+## Trigger Options
+
+```ts
+await myTask.trigger(payload, {
+  delay: "1h",           // Delay execution
+  ttl: "10m",            // Cancel if not started within TTL
+  idempotencyKey: key,
+  queue: "my-queue",
+  machine: "large-1x",   // micro, small-1x, small-2x, medium-1x, medium-2x, large-1x, large-2x
+  maxAttempts: 3,
+  tags: ["user_123"],    // Max 10 tags
+  debounce: {            // Consolidate rapid triggers
+    key: "unique-key",
+    delay: "5s",
+    mode: "trailing",    // "leading" (default) or "trailing"
+  },
+});
+```
+
+## Debouncing
+
+Consolidate multiple triggers into a single execution:
+
+```ts
+// Rapid triggers with same key = single execution
+await myTask.trigger({ userId: "123" }, {
+  debounce: {
+    key: "user-123-update",
+    delay: "5s",
+  },
+});
+
+// Trailing mode: use payload from LAST trigger
+await myTask.trigger({ data: "latest" }, {
+  debounce: {
+    key: "my-key",
+    delay: "10s",
+    mode: "trailing",
+  },
+});
+```
+
+Use cases: user activity updates, webhook deduplication, search indexing, notification batching.
+
+## Batch Triggering
+
+Up to 1,000 items per batch, 3MB per payload:
+
+```ts
+const results = await myTask.batchTriggerAndWait([
+  { payload: { userId: "1" } },
+  { payload: { userId: "2" } },
+]);
+
+for (const result of results) {
+  if (result.ok) console.log(result.output);
+}
+```
+
+## Machine Presets
+
+| Preset      | vCPU | Memory |
+|-------------|------|--------|
+| micro       | 0.25 | 0.25GB |
+| small-1x    | 0.5  | 0.5GB  |
+| small-2x    | 1    | 1GB    |
+| medium-1x   | 1    | 2GB    |
+| medium-2x   | 2    | 4GB    |
+| large-1x    | 4    | 8GB    |
+| large-2x    | 8    | 16GB   |
+
+## Design Principles
+
+1. **Break complex workflows into subtasks** that can be independently retried and made idempotent
+2. **Don't over-complicate** - Sometimes `Promise.allSettled` inside a single task is better than many subtasks (each task has dedicated process and is charged by millisecond)
+3. **Always configure retries** - Set appropriate `maxAttempts` based on the operation
+4. **Use idempotency keys** - Especially for payment/critical operations
+5. **Group related subtasks** - Keep subtasks only used by one parent in the same file, don't export them
+6. **Use logger** - Log at key execution points with `logger.info()`, `logger.error()`, etc.
+
+## Reference Documentation
+
+For detailed documentation on specific topics, read these files:
+
+- `basic-tasks.md` - Task basics, triggering, waits
+- `advanced-tasks.md` - Tags, queues, concurrency, metadata, error handling
+- `scheduled-tasks.md` - Cron schedules, declarative and imperative
+- `realtime.md` - Real-time subscriptions, streams, React hooks
+- `config.md` - trigger.config.ts, build extensions (Prisma, Playwright, FFmpeg, etc.)

.claude/skills/trigger-dev-tasks/advanced-tasks.md

@@ -0,0 +1,485 @@
+# Trigger.dev Advanced Tasks (v4)
+
+**Advanced patterns and features for writing tasks**
+
+## Tags & Organization
+
+```ts
+import { task, tags } from "@trigger.dev/sdk";
+
+export const processUser = task({
+  id: "process-user",
+  run: async (payload: { userId: string; orgId: string }, { ctx }) => {
+    // Add tags during execution
+    await tags.add(`user_${payload.userId}`);
+    await tags.add(`org_${payload.orgId}`);
+
+    return { processed: true };
+  },
+});
+
+// Trigger with tags
+await processUser.trigger(
+  { userId: "123", orgId: "abc" },
+  { tags: ["priority", "user_123", "org_abc"] } // Max 10 tags per run
+);
+
+// Subscribe to tagged runs
+for await (const run of runs.subscribeToRunsWithTag("user_123")) {
+  console.log(`User task ${run.id}: ${run.status}`);
+}
+```
+
+**Tag Best Practices:**
+
+- Use prefixes: `user_123`, `org_abc`, `video:456`
+- Max 10 tags per run, 1-64 characters each
+- Tags don't propagate to child tasks automatically
+
+## Batch Triggering v2
+
+Enhanced batch triggering with larger payloads and streaming ingestion.
+
+### Limits
+
+- **Maximum batch size**: 1,000 items (increased from 500)
+- **Payload per item**: 3MB each (increased from 1MB combined)
+- Payloads > 512KB automatically offload to object storage
+
+### Rate Limiting (per environment)
+
+| Tier | Bucket Size | Refill Rate |
+|------|-------------|-------------|
+| Free | 1,200 runs | 100 runs/10 sec |
+| Hobby | 5,000 runs | 500 runs/5 sec |
+| Pro | 5,000 runs | 500 runs/5 sec |
+
+### Concurrent Batch Processing
+
+| Tier | Concurrent Batches |
+|------|-------------------|
+| Free | 1 |
+| Hobby | 10 |
+| Pro | 10 |
+
+### Usage
+
+```ts
+import { myTask } from "./trigger/myTask";
+
+// Basic batch trigger (up to 1,000 items)
+const runs = await myTask.batchTrigger([
+  { payload: { userId: "user-1" } },
+  { payload: { userId: "user-2" } },
+  { payload: { userId: "user-3" } },
+]);
+
+// Batch trigger with wait
+const results = await myTask.batchTriggerAndWait([
+  { payload: { userId: "user-1" } },
+  { payload: { userId: "user-2" } },
+]);
+
+for (const result of results) {
+  if (result.ok) {
+    console.log("Result:", result.output);
+  }
+}
+
+// With per-item options
+const batchHandle = await myTask.batchTrigger([
+  {
+    payload: { userId: "123" },
+    options: {
+      idempotencyKey: "user-123-batch",
+      tags: ["priority"],
+    },
+  },
+  {
+    payload: { userId: "456" },
+    options: {
+      idempotencyKey: "user-456-batch",
+    },
+  },
+]);
+```
+
+## Debouncing
+
+Consolidate multiple triggers into a single execution by debouncing task runs with a unique key and delay window.
+
+### Use Cases
+
+- **User activity updates**: Batch rapid user actions into a single run
+- **Webhook deduplication**: Handle webhook bursts without redundant processing
+- **Search indexing**: Combine document updates instead of processing individually
+- **Notification batching**: Group notifications to prevent user spam
+
+### Basic Usage
+
+```ts
+await myTask.trigger(
+  { userId: "123" },
+  {
+    debounce: {
+      key: "user-123-update",  // Unique identifier for debounce group
+      delay: "5s",              // Wait duration ("5s", "1m", or milliseconds)
+    },
+  }
+);
+```
+
+### Execution Modes
+
+**Leading Mode** (default): Uses payload/options from the first trigger; subsequent triggers only reschedule execution time.
+
+```ts
+// First trigger sets the payload
+await myTask.trigger({ action: "first" }, {
+  debounce: { key: "my-key", delay: "10s" }
+});
+
+// Second trigger only reschedules - payload remains "first"
+await myTask.trigger({ action: "second" }, {
+  debounce: { key: "my-key", delay: "10s" }
+});
+// Task executes with { action: "first" }
+```
+
+**Trailing Mode**: Uses payload/options from the most recent trigger.
+
+```ts
+await myTask.trigger(
+  { data: "latest-value" },
+  {
+    debounce: {
+      key: "trailing-example",
+      delay: "10s",
+      mode: "trailing",
+    },
+  }
+);
+```
+
+In trailing mode, these options update with each trigger:
+- `payload` — task input data
+- `metadata` — run metadata
+- `tags` — run tags (replaces existing)
+- `maxAttempts` — retry attempts
+- `maxDuration` — maximum compute time
+- `machine` — machine preset
+
+### Important Notes
+
+- Idempotency keys take precedence over debounce settings
+- Compatible with `triggerAndWait()` — parent runs block correctly on debounced execution
+- Debounce key is scoped to the task
+
+## Concurrency & Queues
+
+```ts
+import { task, queue } from "@trigger.dev/sdk";
+
+// Shared queue for related tasks
+const emailQueue = queue({
+  name: "email-processing",
+  concurrencyLimit: 5, // Max 5 emails processing simultaneously
+});
+
+// Task-level concurrency
+export const oneAtATime = task({
+  id: "sequential-task",
+  queue: { concurrencyLimit: 1 }, // Process one at a time
+  run: async (payload) => {
+    // Critical section - only one instance runs
+  },
+});
+
+// Per-user concurrency
+export const processUserData = task({
+  id: "process-user-data",
+  run: async (payload: { userId: string }) => {
+    // Override queue with user-specific concurrency
+    await childTask.trigger(payload, {
+      queue: {
+        name: `user-${payload.userId}`,
+        concurrencyLimit: 2,
+      },
+    });
+  },
+});
+
+export const emailTask = task({
+  id: "send-email",
+  queue: emailQueue, // Use shared queue
+  run: async (payload: { to: string }) => {
+    // Send email logic
+  },
+});
+```
+
+## Error Handling & Retries
+
+```ts
+import { task, retry, AbortTaskRunError } from "@trigger.dev/sdk";
+
+export const resilientTask = task({
+  id: "resilient-task",
+  retry: {
+    maxAttempts: 10,
+    factor: 1.8, // Exponential backoff multiplier
+    minTimeoutInMs: 500,
+    maxTimeoutInMs: 30_000,
+    randomize: false,
+  },
+  catchError: async ({ error, ctx }) => {
+    // Custom error handling
+    if (error.code === "FATAL_ERROR") {
+      throw new AbortTaskRunError("Cannot retry this error");
+    }
+
+    // Log error details
+    console.error(`Task ${ctx.task.id} failed:`, error);
+
+    // Allow retry by returning nothing
+    return { retryAt: new Date(Date.now() + 60000) }; // Retry in 1 minute
+  },
+  run: async (payload) => {
+    // Retry specific operations
+    const result = await retry.onThrow(
+      async () => {
+        return await unstableApiCall(payload);
+      },
+      { maxAttempts: 3 }
+    );
+
+    // Conditional HTTP retries
+    const response = await retry.fetch("https://api.example.com", {
+      retry: {
+        maxAttempts: 5,
+        condition: (response, error) => {
+          return response?.status === 429 || response?.status >= 500;
+        },
+      },
+    });
+
+    return result;
+  },
+});
+```
+
+## Machines & Performance
+
+```ts
+export const heavyTask = task({
+  id: "heavy-computation",
+  machine: { preset: "large-2x" }, // 8 vCPU, 16 GB RAM
+  maxDuration: 1800, // 30 minutes timeout
+  run: async (payload, { ctx }) => {
+    // Resource-intensive computation
+    if (ctx.machine.preset === "large-2x") {
+      // Use all available cores
+      return await parallelProcessing(payload);
+    }
+
+    return await standardProcessing(payload);
+  },
+});
+
+// Override machine when triggering
+await heavyTask.trigger(payload, {
+  machine: { preset: "medium-1x" }, // Override for this run
+});
+```
+
+**Machine Presets:**
+
+- `micro`: 0.25 vCPU, 0.25 GB RAM
+- `small-1x`: 0.5 vCPU, 0.5 GB RAM (default)
+- `small-2x`: 1 vCPU, 1 GB RAM
+- `medium-1x`: 1 vCPU, 2 GB RAM
+- `medium-2x`: 2 vCPU, 4 GB RAM
+- `large-1x`: 4 vCPU, 8 GB RAM
+- `large-2x`: 8 vCPU, 16 GB RAM
+
+## Idempotency
+
+```ts
+import { task, idempotencyKeys } from "@trigger.dev/sdk";
+
+export const paymentTask = task({
+  id: "process-payment",
+  retry: {
+    maxAttempts: 3,
+  },
+  run: async (payload: { orderId: string; amount: number }) => {
+    // Automatically scoped to this task run, so if the task is retried, the idempotency key will be the same
+    const idempotencyKey = await idempotencyKeys.create(`payment-${payload.orderId}`);
+
+    // Ensure payment is processed only once
+    await chargeCustomer.trigger(payload, {
+      idempotencyKey,
+      idempotencyKeyTTL: "24h", // Key expires in 24 hours
+    });
+  },
+});
+
+// Payload-based idempotency
+import { createHash } from "node:crypto";
+
+function createPayloadHash(payload: any): string {
+  const hash = createHash("sha256");
+  hash.update(JSON.stringify(payload));
+  return hash.digest("hex");
+}
+
+export const deduplicatedTask = task({
+  id: "deduplicated-task",
+  run: async (payload) => {
+    const payloadHash = createPayloadHash(payload);
+    const idempotencyKey = await idempotencyKeys.create(payloadHash);
+
+    await processData.trigger(payload, { idempotencyKey });
+  },
+});
+```
+
+## Metadata & Progress Tracking
+
+```ts
+import { task, metadata } from "@trigger.dev/sdk";
+
+export const batchProcessor = task({
+  id: "batch-processor",
+  run: async (payload: { items: any[] }, { ctx }) => {
+    const totalItems = payload.items.length;
+
+    // Initialize progress metadata
+    metadata
+      .set("progress", 0)
+      .set("totalItems", totalItems)
+      .set("processedItems", 0)
+      .set("status", "starting");
+
+    const results = [];
+
+    for (let i = 0; i < payload.items.length; i++) {
+      const item = payload.items[i];
+
+      // Process item
+      const result = await processItem(item);
+      results.push(result);
+
+      // Update progress
+      const progress = ((i + 1) / totalItems) * 100;
+      metadata
+        .set("progress", progress)
+        .increment("processedItems", 1)
+        .append("logs", `Processed item ${i + 1}/${totalItems}`)
+        .set("currentItem", item.id);
+    }
+
+    // Final status
+    metadata.set("status", "completed");
+
+    return { results, totalProcessed: results.length };
+  },
+});
+
+// Update parent metadata from child task
+export const childTask = task({
+  id: "child-task",
+  run: async (payload, { ctx }) => {
+    // Update parent task metadata
+    metadata.parent.set("childStatus", "processing");
+    metadata.root.increment("childrenCompleted", 1);
+
+    return { processed: true };
+  },
+});
+```
+
+## Logging & Tracing
+
+```ts
+import { task, logger } from "@trigger.dev/sdk";
+
+export const tracedTask = task({
+  id: "traced-task",
+  run: async (payload, { ctx }) => {
+    logger.info("Task started", { userId: payload.userId });
+
+    // Custom trace with attributes
+    const user = await logger.trace(
+      "fetch-user",
+      async (span) => {
+        span.setAttribute("user.id", payload.userId);
+        span.setAttribute("operation", "database-fetch");
+
+        const userData = await database.findUser(payload.userId);
+        span.setAttribute("user.found", !!userData);
+
+        return userData;
+      },
+      { userId: payload.userId }
+    );
+
+    logger.debug("User fetched", { user: user.id });
+
+    try {
+      const result = await processUser(user);
+      logger.info("Processing completed", { result });
+      return result;
+    } catch (error) {
+      logger.error("Processing failed", {
+        error: error.message,
+        userId: payload.userId,
+      });
+      throw error;
+    }
+  },
+});
+```
+
+## Hidden Tasks
+
+```ts
+// Hidden task - not exported, only used internally
+const internalProcessor = task({
+  id: "internal-processor",
+  run: async (payload: { data: string }) => {
+    return { processed: payload.data.toUpperCase() };
+  },
+});
+
+// Public task that uses hidden task
+export const publicWorkflow = task({
+  id: "public-workflow",
+  run: async (payload: { input: string }) => {
+    // Use hidden task internally
+    const result = await internalProcessor.triggerAndWait({
+      data: payload.input,
+    });
+
+    if (result.ok) {
+      return { output: result.output.processed };
+    }
+
+    throw new Error("Internal processing failed");
+  },
+});
+```
+
+## Best Practices
+
+- **Concurrency**: Use queues to prevent overwhelming external services
+- **Retries**: Configure exponential backoff for transient failures
+- **Idempotency**: Always use for payment/critical operations
+- **Metadata**: Track progress for long-running tasks
+- **Machines**: Match machine size to computational requirements
+- **Tags**: Use consistent naming patterns for filtering
+- **Debouncing**: Use for user activity, webhooks, and notification batching
+- **Batch triggering**: Use for bulk operations up to 1,000 items
+- **Error Handling**: Distinguish between retryable and fatal errors
+
+Design tasks to be stateless, idempotent, and resilient to failures. Use metadata for state tracking and queues for resource management.

.claude/skills/trigger-dev-tasks/basic-tasks.md

@@ -0,0 +1,199 @@
+# Trigger.dev Basic Tasks (v4)
+
+**MUST use `@trigger.dev/sdk`, NEVER `client.defineJob`**
+
+## Basic Task
+
+```ts
+import { task } from "@trigger.dev/sdk";
+
+export const processData = task({
+  id: "process-data",
+  retry: {
+    maxAttempts: 10,
+    factor: 1.8,
+    minTimeoutInMs: 500,
+    maxTimeoutInMs: 30_000,
+    randomize: false,
+  },
+  run: async (payload: { userId: string; data: any[] }) => {
+    // Task logic - runs for long time, no timeouts
+    console.log(`Processing ${payload.data.length} items for user ${payload.userId}`);
+    return { processed: payload.data.length };
+  },
+});
+```
+
+## Schema Task (with validation)
+
+```ts
+import { schemaTask } from "@trigger.dev/sdk";
+import { z } from "zod";
+
+export const validatedTask = schemaTask({
+  id: "validated-task",
+  schema: z.object({
+    name: z.string(),
+    age: z.number(),
+    email: z.string().email(),
+  }),
+  run: async (payload) => {
+    // Payload is automatically validated and typed
+    return { message: `Hello ${payload.name}, age ${payload.age}` };
+  },
+});
+```
+
+## Triggering Tasks
+
+### From Backend Code
+
+```ts
+import { tasks } from "@trigger.dev/sdk";
+import type { processData } from "./trigger/tasks";
+
+// Single trigger
+const handle = await tasks.trigger<typeof processData>("process-data", {
+  userId: "123",
+  data: [{ id: 1 }, { id: 2 }],
+});
+
+// Batch trigger (up to 1,000 items, 3MB per payload)
+const batchHandle = await tasks.batchTrigger<typeof processData>("process-data", [
+  { payload: { userId: "123", data: [{ id: 1 }] } },
+  { payload: { userId: "456", data: [{ id: 2 }] } },
+]);
+```
+
+### Debounced Triggering
+
+Consolidate multiple triggers into a single execution:
+
+```ts
+// Multiple rapid triggers with same key = single execution
+await myTask.trigger(
+  { userId: "123" },
+  {
+    debounce: {
+      key: "user-123-update",  // Unique key for debounce group
+      delay: "5s",              // Wait before executing
+    },
+  }
+);
+
+// Trailing mode: use payload from LAST trigger
+await myTask.trigger(
+  { data: "latest-value" },
+  {
+    debounce: {
+      key: "trailing-example",
+      delay: "10s",
+      mode: "trailing",  // Default is "leading" (first payload)
+    },
+  }
+);
+```
+
+**Debounce modes:**
+- `leading` (default): Uses payload from first trigger, subsequent triggers only reschedule
+- `trailing`: Uses payload from most recent trigger
+
+### From Inside Tasks (with Result handling)
+
+```ts
+export const parentTask = task({
+  id: "parent-task",
+  run: async (payload) => {
+    // Trigger and continue
+    const handle = await childTask.trigger({ data: "value" });
+
+    // Trigger and wait - returns Result object, NOT task output
+    const result = await childTask.triggerAndWait({ data: "value" });
+    if (result.ok) {
+      console.log("Task output:", result.output); // Actual task return value
+    } else {
+      console.error("Task failed:", result.error);
+    }
+
+    // Quick unwrap (throws on error)
+    const output = await childTask.triggerAndWait({ data: "value" }).unwrap();
+
+    // Batch trigger and wait
+    const results = await childTask.batchTriggerAndWait([
+      { payload: { data: "item1" } },
+      { payload: { data: "item2" } },
+    ]);
+
+    for (const run of results) {
+      if (run.ok) {
+        console.log("Success:", run.output);
+      } else {
+        console.log("Failed:", run.error);
+      }
+    }
+  },
+});
+
+export const childTask = task({
+  id: "child-task",
+  run: async (payload: { data: string }) => {
+    return { processed: payload.data };
+  },
+});
+```
+
+> Never wrap triggerAndWait or batchTriggerAndWait calls in a Promise.all or Promise.allSettled as this is not supported in Trigger.dev tasks.
+
+## Waits
+
+```ts
+import { task, wait } from "@trigger.dev/sdk";
+
+export const taskWithWaits = task({
+  id: "task-with-waits",
+  run: async (payload) => {
+    console.log("Starting task");
+
+    // Wait for specific duration
+    await wait.for({ seconds: 30 });
+    await wait.for({ minutes: 5 });
+    await wait.for({ hours: 1 });
+    await wait.for({ days: 1 });
+
+    // Wait until specific date
+    await wait.until({ date: new Date("2024-12-25") });
+
+    // Wait for token (from external system)
+    await wait.forToken({
+      token: "user-approval-token",
+      timeoutInSeconds: 3600, // 1 hour timeout
+    });
+
+    console.log("All waits completed");
+    return { status: "completed" };
+  },
+});
+```
+
+> Never wrap wait calls in a Promise.all or Promise.allSettled as this is not supported in Trigger.dev tasks.
+
+## Key Points
+
+- **Result vs Output**: `triggerAndWait()` returns a `Result` object with `ok`, `output`, `error` properties - NOT the direct task output
+- **Type safety**: Use `import type` for task references when triggering from backend
+- **Waits > 5 seconds**: Automatically checkpointed, don't count toward compute usage
+- **Debounce + idempotency**: Idempotency keys take precedence over debounce settings
+
+## NEVER Use (v2 deprecated)
+
+```ts
+// BREAKS APPLICATION
+client.defineJob({
+  id: "job-id",
+  run: async (payload, io) => {
+    /* ... */
+  },
+});
+```
+
+Use SDK (`@trigger.dev/sdk`), check `result.ok` before accessing `result.output`

.claude/skills/trigger-dev-tasks/config.md

@@ -0,0 +1,346 @@
+# Trigger.dev Configuration
+
+**Complete guide to configuring `trigger.config.ts` with build extensions**
+
+## Basic Configuration
+
+```ts
+import { defineConfig } from "@trigger.dev/sdk";
+
+export default defineConfig({
+  project: "<project-ref>", // Required: Your project reference
+  dirs: ["./trigger"], // Task directories
+  runtime: "node", // "node", "node-22", or "bun"
+  logLevel: "info", // "debug", "info", "warn", "error"
+
+  // Default retry settings
+  retries: {
+    enabledInDev: false,
+    default: {
+      maxAttempts: 3,
+      minTimeoutInMs: 1000,
+      maxTimeoutInMs: 10000,
+      factor: 2,
+      randomize: true,
+    },
+  },
+
+  // Build configuration
+  build: {
+    autoDetectExternal: true,
+    keepNames: true,
+    minify: false,
+    extensions: [], // Build extensions go here
+  },
+
+  // Global lifecycle hooks
+  onStartAttempt: async ({ payload, ctx }) => {
+    console.log("Global task start");
+  },
+  onSuccess: async ({ payload, output, ctx }) => {
+    console.log("Global task success");
+  },
+  onFailure: async ({ payload, error, ctx }) => {
+    console.log("Global task failure");
+  },
+});
+```
+
+## Build Extensions
+
+### Database & ORM
+
+#### Prisma
+
+```ts
+import { prismaExtension } from "@trigger.dev/build/extensions/prisma";
+
+extensions: [
+  prismaExtension({
+    schema: "prisma/schema.prisma",
+    version: "5.19.0", // Optional: specify version
+    migrate: true, // Run migrations during build
+    directUrlEnvVarName: "DIRECT_DATABASE_URL",
+    typedSql: true, // Enable TypedSQL support
+  }),
+];
+```
+
+#### TypeScript Decorators (for TypeORM)
+
+```ts
+import { emitDecoratorMetadata } from "@trigger.dev/build/extensions/typescript";
+
+extensions: [
+  emitDecoratorMetadata(), // Enables decorator metadata
+];
+```
+
+### Scripting Languages
+
+#### Python
+
+```ts
+import { pythonExtension } from "@trigger.dev/build/extensions/python";
+
+extensions: [
+  pythonExtension({
+    scripts: ["./python/**/*.py"], // Copy Python files
+    requirementsFile: "./requirements.txt", // Install packages
+    devPythonBinaryPath: ".venv/bin/python", // Dev mode binary
+  }),
+];
+
+// Usage in tasks
+const result = await python.runInline(`print("Hello, world!")`);
+const output = await python.runScript("./python/script.py", ["arg1"]);
+```
+
+### Browser Automation
+
+#### Playwright
+
+```ts
+import { playwright } from "@trigger.dev/build/extensions/playwright";
+
+extensions: [
+  playwright({
+    browsers: ["chromium", "firefox", "webkit"], // Default: ["chromium"]
+    headless: true, // Default: true
+  }),
+];
+```
+
+#### Puppeteer
+
+```ts
+import { puppeteer } from "@trigger.dev/build/extensions/puppeteer";
+
+extensions: [puppeteer()];
+
+// Environment variable needed:
+// PUPPETEER_EXECUTABLE_PATH: "/usr/bin/google-chrome-stable"
+```
+
+#### Lightpanda
+
+```ts
+import { lightpanda } from "@trigger.dev/build/extensions/lightpanda";
+
+extensions: [
+  lightpanda({
+    version: "latest", // or "nightly"
+    disableTelemetry: false,
+  }),
+];
+```
+
+### Media Processing
+
+#### FFmpeg
+
+```ts
+import { ffmpeg } from "@trigger.dev/build/extensions/core";
+
+extensions: [
+  ffmpeg({ version: "7" }), // Static build, or omit for Debian version
+];
+
+// Automatically sets FFMPEG_PATH and FFPROBE_PATH
+// Add fluent-ffmpeg to external packages if using
+```
+
+#### Audio Waveform
+
+```ts
+import { audioWaveform } from "@trigger.dev/build/extensions/audioWaveform";
+
+extensions: [
+  audioWaveform(), // Installs Audio Waveform 1.1.0
+];
+```
+
+### System & Package Management
+
+#### System Packages (apt-get)
+
+```ts
+import { aptGet } from "@trigger.dev/build/extensions/core";
+
+extensions: [
+  aptGet({
+    packages: ["ffmpeg", "imagemagick", "curl=7.68.0-1"], // Can specify versions
+  }),
+];
+```
+
+#### Additional NPM Packages
+
+Only use this for installing CLI tools, NOT packages you import in your code.
+
+```ts
+import { additionalPackages } from "@trigger.dev/build/extensions/core";
+
+extensions: [
+  additionalPackages({
+    packages: ["wrangler"], // CLI tools and specific versions
+  }),
+];
+```
+
+#### Additional Files
+
+```ts
+import { additionalFiles } from "@trigger.dev/build/extensions/core";
+
+extensions: [
+  additionalFiles({
+    files: ["wrangler.toml", "./assets/**", "./fonts/**"], // Glob patterns supported
+  }),
+];
+```
+
+### Environment & Build Tools
+
+#### Environment Variable Sync
+
+```ts
+import { syncEnvVars } from "@trigger.dev/build/extensions/core";
+
+extensions: [
+  syncEnvVars(async (ctx) => {
+    // ctx contains: environment, projectRef, env
+    return [
+      { name: "SECRET_KEY", value: await getSecret(ctx.environment) },
+      { name: "API_URL", value: ctx.environment === "prod" ? "api.prod.com" : "api.dev.com" },
+    ];
+  }),
+];
+```
+
+#### ESBuild Plugins
+
+```ts
+import { esbuildPlugin } from "@trigger.dev/build/extensions";
+import { sentryEsbuildPlugin } from "@sentry/esbuild-plugin";
+
+extensions: [
+  esbuildPlugin(
+    sentryEsbuildPlugin({
+      org: process.env.SENTRY_ORG,
+      project: process.env.SENTRY_PROJECT,
+      authToken: process.env.SENTRY_AUTH_TOKEN,
+    }),
+    { placement: "last", target: "deploy" } // Optional config
+  ),
+];
+```
+
+## Custom Build Extensions
+
+```ts
+import { defineConfig } from "@trigger.dev/sdk";
+
+const customExtension = {
+  name: "my-custom-extension",
+
+  externalsForTarget: (target) => {
+    return ["some-native-module"]; // Add external dependencies
+  },
+
+  onBuildStart: async (context) => {
+    console.log(`Build starting for ${context.target}`);
+    // Register esbuild plugins, modify build context
+  },
+
+  onBuildComplete: async (context, manifest) => {
+    console.log("Build complete, adding layers");
+    // Add build layers, modify deployment
+    context.addLayer({
+      id: "my-layer",
+      files: [{ source: "./custom-file", destination: "/app/custom" }],
+      commands: ["chmod +x /app/custom"],
+    });
+  },
+};
+
+export default defineConfig({
+  project: "my-project",
+  build: {
+    extensions: [customExtension],
+  },
+});
+```
+
+## Advanced Configuration
+
+### Telemetry
+
+```ts
+import { PrismaInstrumentation } from "@prisma/instrumentation";
+import { OpenAIInstrumentation } from "@langfuse/openai";
+
+export default defineConfig({
+  // ... other config
+  telemetry: {
+    instrumentations: [new PrismaInstrumentation(), new OpenAIInstrumentation()],
+    exporters: [customExporter], // Optional custom exporters
+  },
+});
+```
+
+### Machine & Performance
+
+```ts
+export default defineConfig({
+  // ... other config
+  defaultMachine: "large-1x", // Default machine for all tasks
+  maxDuration: 300, // Default max duration (seconds)
+  enableConsoleLogging: true, // Console logging in development
+});
+```
+
+## Common Extension Combinations
+
+### Full-Stack Web App
+
+```ts
+extensions: [
+  prismaExtension({ schema: "prisma/schema.prisma", migrate: true }),
+  additionalFiles({ files: ["./public/**", "./assets/**"] }),
+  syncEnvVars(async (ctx) => [...envVars]),
+];
+```
+
+### AI/ML Processing
+
+```ts
+extensions: [
+  pythonExtension({
+    scripts: ["./ai/**/*.py"],
+    requirementsFile: "./requirements.txt",
+  }),
+  ffmpeg({ version: "7" }),
+  additionalPackages({ packages: ["wrangler"] }),
+];
+```
+
+### Web Scraping
+
+```ts
+extensions: [
+  playwright({ browsers: ["chromium"] }),
+  puppeteer(),
+  additionalFiles({ files: ["./selectors.json", "./proxies.txt"] }),
+];
+```
+
+## Best Practices
+
+- **Use specific versions**: Pin extension versions for reproducible builds
+- **External packages**: Add modules with native addons to the `build.external` array
+- **Environment sync**: Use `syncEnvVars` for dynamic secrets
+- **File paths**: Use glob patterns for flexible file inclusion
+- **Debug builds**: Use `--log-level debug --dry-run` for troubleshooting
+
+Extensions only affect deployment, not local development. Use `external` array for packages that shouldn't be bundled.

.claude/skills/trigger-dev-tasks/realtime.md

@@ -0,0 +1,244 @@
+# Trigger.dev Realtime
+
+**Real-time monitoring and updates for runs**
+
+## Core Concepts
+
+Realtime allows you to:
+
+- Subscribe to run status changes, metadata updates, and streams
+- Build real-time dashboards and UI updates
+- Monitor task progress from frontend and backend
+
+## Authentication
+
+### Public Access Tokens
+
+```ts
+import { auth } from "@trigger.dev/sdk";
+
+// Read-only token for specific runs
+const publicToken = await auth.createPublicToken({
+  scopes: {
+    read: {
+      runs: ["run_123", "run_456"],
+      tasks: ["my-task-1", "my-task-2"],
+    },
+  },
+  expirationTime: "1h", // Default: 15 minutes
+});
+```
+
+### Trigger Tokens (Frontend only)
+
+```ts
+// Single-use token for triggering tasks
+const triggerToken = await auth.createTriggerPublicToken("my-task", {
+  expirationTime: "30m",
+});
+```
+
+## Backend Usage
+
+### Subscribe to Runs
+
+```ts
+import { runs, tasks } from "@trigger.dev/sdk";
+
+// Trigger and subscribe
+const handle = await tasks.trigger("my-task", { data: "value" });
+
+// Subscribe to specific run
+for await (const run of runs.subscribeToRun<typeof myTask>(handle.id)) {
+  console.log(`Status: ${run.status}, Progress: ${run.metadata?.progress}`);
+  if (run.status === "COMPLETED") break;
+}
+
+// Subscribe to runs with tag
+for await (const run of runs.subscribeToRunsWithTag("user-123")) {
+  console.log(`Tagged run ${run.id}: ${run.status}`);
+}
+
+// Subscribe to batch
+for await (const run of runs.subscribeToBatch(batchId)) {
+  console.log(`Batch run ${run.id}: ${run.status}`);
+}
+```
+
+### Realtime Streams v2
+
+```ts
+import { streams, InferStreamType } from "@trigger.dev/sdk";
+
+// 1. Define streams (shared location)
+export const aiStream = streams.define<string>({
+  id: "ai-output",
+});
+
+export type AIStreamPart = InferStreamType<typeof aiStream>;
+
+// 2. Pipe from task
+export const streamingTask = task({
+  id: "streaming-task",
+  run: async (payload) => {
+    const completion = await openai.chat.completions.create({
+      model: "gpt-4",
+      messages: [{ role: "user", content: payload.prompt }],
+      stream: true,
+    });
+
+    const { waitUntilComplete } = aiStream.pipe(completion);
+    await waitUntilComplete();
+  },
+});
+
+// 3. Read from backend
+const stream = await aiStream.read(runId, {
+  timeoutInSeconds: 300,
+  startIndex: 0, // Resume from specific chunk
+});
+
+for await (const chunk of stream) {
+  console.log("Chunk:", chunk); // Fully typed
+}
+```
+
+## React Frontend Usage
+
+### Installation
+
+```bash
+npm add @trigger.dev/react-hooks
+```
+
+### Triggering Tasks
+
+```tsx
+"use client";
+import { useTaskTrigger, useRealtimeTaskTrigger } from "@trigger.dev/react-hooks";
+import type { myTask } from "../trigger/tasks";
+
+function TriggerComponent({ accessToken }: { accessToken: string }) {
+  // Basic trigger
+  const { submit, handle, isLoading } = useTaskTrigger<typeof myTask>("my-task", {
+    accessToken,
+  });
+
+  // Trigger with realtime updates
+  const {
+    submit: realtimeSubmit,
+    run,
+    isLoading: isRealtimeLoading,
+  } = useRealtimeTaskTrigger<typeof myTask>("my-task", { accessToken });
+
+  return (
+    <div>
+      <button onClick={() => submit({ data: "value" })} disabled={isLoading}>
+        Trigger Task
+      </button>
+
+      <button onClick={() => realtimeSubmit({ data: "realtime" })} disabled={isRealtimeLoading}>
+        Trigger with Realtime
+      </button>
+
+      {run && <div>Status: {run.status}</div>}
+    </div>
+  );
+}
+```
+
+### Subscribing to Runs
+
+```tsx
+"use client";
+import { useRealtimeRun, useRealtimeRunsWithTag } from "@trigger.dev/react-hooks";
+import type { myTask } from "../trigger/tasks";
+
+function SubscribeComponent({ runId, accessToken }: { runId: string; accessToken: string }) {
+  // Subscribe to specific run
+  const { run, error } = useRealtimeRun<typeof myTask>(runId, {
+    accessToken,
+    onComplete: (run) => {
+      console.log("Task completed:", run.output);
+    },
+  });
+
+  // Subscribe to tagged runs
+  const { runs } = useRealtimeRunsWithTag("user-123", { accessToken });
+
+  if (error) return <div>Error: {error.message}</div>;
+  if (!run) return <div>Loading...</div>;
+
+  return (
+    <div>
+      <div>Status: {run.status}</div>
+      <div>Progress: {run.metadata?.progress || 0}%</div>
+      {run.output && <div>Result: {JSON.stringify(run.output)}</div>}
+
+      <h3>Tagged Runs:</h3>
+      {runs.map((r) => (
+        <div key={r.id}>
+          {r.id}: {r.status}
+        </div>
+      ))}
+    </div>
+  );
+}
+```
+
+### Realtime Streams with React
+
+```tsx
+"use client";
+import { useRealtimeStream } from "@trigger.dev/react-hooks";
+import { aiStream } from "../trigger/streams";
+
+function StreamComponent({ runId, accessToken }: { runId: string; accessToken: string }) {
+  // Pass defined stream directly for type safety
+  const { parts, error } = useRealtimeStream(aiStream, runId, {
+    accessToken,
+    timeoutInSeconds: 300,
+    throttleInMs: 50, // Control re-render frequency
+  });
+
+  if (error) return <div>Error: {error.message}</div>;
+  if (!parts) return <div>Loading...</div>;
+
+  const text = parts.join(""); // parts is typed as AIStreamPart[]
+
+  return <div>Streamed Text: {text}</div>;
+}
+```
+
+### Wait Tokens
+
+```tsx
+"use client";
+import { useWaitToken } from "@trigger.dev/react-hooks";
+
+function WaitTokenComponent({ tokenId, accessToken }: { tokenId: string; accessToken: string }) {
+  const { complete } = useWaitToken(tokenId, { accessToken });
+
+  return <button onClick={() => complete({ approved: true })}>Approve Task</button>;
+}
+```
+
+## Run Object Properties
+
+Key properties available in run subscriptions:
+
+- `id`: Unique run identifier
+- `status`: `QUEUED`, `EXECUTING`, `COMPLETED`, `FAILED`, `CANCELED`, etc.
+- `payload`: Task input data (typed)
+- `output`: Task result (typed, when completed)
+- `metadata`: Real-time updatable data
+- `createdAt`, `updatedAt`: Timestamps
+- `costInCents`: Execution cost
+
+## Best Practices
+
+- **Use Realtime over SWR**: Recommended for most use cases due to rate limits
+- **Scope tokens properly**: Only grant necessary read/trigger permissions
+- **Handle errors**: Always check for errors in hooks and subscriptions
+- **Type safety**: Use task types for proper payload/output typing
+- **Cleanup subscriptions**: Backend subscriptions auto-complete, frontend hooks auto-cleanup

.claude/skills/trigger-dev-tasks/scheduled-tasks.md

@@ -0,0 +1,113 @@
+# Scheduled Tasks (Cron)
+
+Recurring tasks using cron. For one-off future runs, use the **delay** option.
+
+## Define a Scheduled Task
+
+```ts
+import { schedules } from "@trigger.dev/sdk";
+
+export const task = schedules.task({
+  id: "first-scheduled-task",
+  run: async (payload) => {
+    payload.timestamp; // Date (scheduled time, UTC)
+    payload.lastTimestamp; // Date | undefined
+    payload.timezone; // IANA, e.g. "America/New_York" (default "UTC")
+    payload.scheduleId; // string
+    payload.externalId; // string | undefined
+    payload.upcoming; // Date[]
+
+    payload.timestamp.toLocaleString("en-US", { timeZone: payload.timezone });
+  },
+});
+```
+
+> Scheduled tasks need at least one schedule attached to run.
+
+## Attach Schedules
+
+**Declarative (sync on dev/deploy):**
+
+```ts
+schedules.task({
+  id: "every-2h",
+  cron: "0 */2 * * *", // UTC
+  run: async () => {},
+});
+
+schedules.task({
+  id: "tokyo-5am",
+  cron: { pattern: "0 5 * * *", timezone: "Asia/Tokyo", environments: ["PRODUCTION", "STAGING"] },
+  run: async () => {},
+});
+```
+
+**Imperative (SDK or dashboard):**
+
+```ts
+await schedules.create({
+  task: task.id,
+  cron: "0 0 * * *",
+  timezone: "America/New_York", // DST-aware
+  externalId: "user_123",
+  deduplicationKey: "user_123-daily", // updates if reused
+});
+```
+
+### Dynamic / Multi-tenant Example
+
+```ts
+// /trigger/reminder.ts
+export const reminderTask = schedules.task({
+  id: "todo-reminder",
+  run: async (p) => {
+    if (!p.externalId) throw new Error("externalId is required");
+    const user = await db.getUser(p.externalId);
+    await sendReminderEmail(user);
+  },
+});
+```
+
+```ts
+// app/reminders/route.ts
+export async function POST(req: Request) {
+  const data = await req.json();
+  return Response.json(
+    await schedules.create({
+      task: reminderTask.id,
+      cron: "0 8 * * *",
+      timezone: data.timezone,
+      externalId: data.userId,
+      deduplicationKey: `${data.userId}-reminder`,
+    })
+  );
+}
+```
+
+## Cron Syntax (no seconds)
+
+```
+* * * * *
+| | | | └ day of week (0–7 or 1L–7L; 0/7=Sun; L=last)
+| | | └── month (1–12)
+| | └──── day of month (1–31 or L)
+| └────── hour (0–23)
+└──────── minute (0–59)
+```
+
+## When Schedules Won't Trigger
+
+- **Dev:** only when the dev CLI is running.
+- **Staging/Production:** only for tasks in the **latest deployment**.
+
+## SDK Management
+
+```ts
+await schedules.retrieve(id);
+await schedules.list();
+await schedules.update(id, { cron: "0 0 1 * *", externalId: "ext", deduplicationKey: "key" });
+await schedules.deactivate(id);
+await schedules.activate(id);
+await schedules.del(id);
+await schedules.timezones(); // list of IANA timezones
+```

.configs/tsconfig.base.json

@@ -0,0 +1,36 @@
+{
+  "compilerOptions": {
+    "target": "es2022",
+    "lib": ["ES2022", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "moduleDetection": "force",
+    "verbatimModuleSyntax": false,
+    "jsx": "react",
+
+    "strict": true,
+    "alwaysStrict": true,
+    "strictPropertyInitialization": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noImplicitAny": true,
+    "noImplicitReturns": true,
+    "noImplicitThis": true,
+
+    "noFallthroughCasesInSwitch": true,
+    "resolveJsonModule": true,
+
+    "removeComments": false,
+    "esModuleInterop": true,
+    "emitDecoratorMetadata": false,
+    "experimentalDecorators": false,
+    "downlevelIteration": true,
+    "isolatedModules": true,
+    "noUncheckedIndexedAccess": true,
+
+    "pretty": true,
+    "customConditions": ["@triggerdotdev/source"]
+  }
+}

.cursor/commands/deslop.md

@@ -0,0 +1,11 @@
+# Remove AI code slop
+
+Check the diff against main, and remove all AI generated slop introduced in this branch.
+
+This includes:
+- Extra comments that a human wouldn't add or is inconsistent with the rest of the file
+- Extra defensive checks or try/catch blocks that are abnormal for that area of the codebase (especially if called by trusted / validated codepaths)
+- Casts to any to get around type issues
+- Any other style that is inconsistent with the file
+
+Report at the end with only a 1-3 sentence summary of what you changed

.cursor/mcp.json

@@ -0,0 +1,3 @@
+{
+  "mcpServers": {}
+}

.cursor/rules/executing-commands.mdc

@@ -0,0 +1,24 @@
+---
+description: how to run commands in the monorepo
+globs: 
+alwaysApply: true
+---
+Almost all commands in the monorepo should be executed when `pnpm run ...` from the root of the monorepo. For example, running tests for the `@internal/run-engine` internal package:
+
+```
+pnpm run dev --filter webapp
+```
+
+But often, when running tests, it's better to `cd` into the directory and then run tests:
+
+```
+cd apps/webapp
+pnpm run test --run
+```
+
+This way you can run for a single file easily:
+
+```
+cd internal-packages/run-engine
+pnpm run test ./src/engine/tests/ttl.test.ts --run
+```

.cursor/rules/migrations.mdc

@@ -0,0 +1,6 @@
+---
+description: how to create and apply database migrations
+alwaysApply: false
+---
+
+Follow our [migrations.md](mdc:ai/references/migrations.md) guide for how to create and apply database migrations.

.cursor/rules/otel-metrics.mdc

@@ -0,0 +1,66 @@
+---
+description: Guidelines for creating OpenTelemetry metrics to avoid cardinality issues
+globs:
+  - "**/*.ts"
+---
+
+# OpenTelemetry Metrics Guidelines
+
+When creating or editing OTEL metrics (counters, histograms, gauges), always ensure metric attributes have **low cardinality**.
+
+## What is Cardinality?
+
+Cardinality refers to the number of unique values an attribute can have. Each unique combination of attribute values creates a new time series, which consumes memory and storage in your metrics backend.
+
+## Rules
+
+### DO use low-cardinality attributes:
+- **Enums**: `environment_type` (PRODUCTION, STAGING, DEVELOPMENT, PREVIEW)
+- **Booleans**: `hasFailures`, `streaming`, `success`
+- **Bounded error codes**: A finite, controlled set of error types
+- **Shard IDs**: When sharding is bounded (e.g., 0-15)
+
+### DO NOT use high-cardinality attributes:
+- **UUIDs/IDs**: `envId`, `userId`, `runId`, `projectId`, `organizationId`
+- **Unbounded integers**: `itemCount`, `batchSize`, `retryCount`
+- **Timestamps**: `createdAt`, `startTime`
+- **Free-form strings**: `errorMessage`, `taskName`, `queueName`
+
+## Example
+
+```typescript
+// BAD - High cardinality
+this.counter.add(1, {
+  envId: options.environmentId,        // UUID - unbounded
+  itemCount: options.runCount,         // Integer - unbounded
+});
+
+// GOOD - Low cardinality
+this.counter.add(1, {
+  environment_type: options.environmentType,  // Enum - 4 values
+  streaming: true,                            // Boolean - 2 values
+});
+```
+
+## Prometheus Metric Naming
+
+When metrics are exported via OTLP to Prometheus, the exporter automatically adds unit suffixes to metric names:
+
+| OTel Metric Name | Unit | Prometheus Name |
+|------------------|------|-----------------|
+| `my_duration_ms` | `ms` | `my_duration_ms_milliseconds` |
+| `my_counter` | counter | `my_counter_total` |
+| `items_inserted` | counter | `items_inserted_inserts_total` |
+| `batch_size` | histogram | `batch_size_items_bucket` |
+
+Keep this in mind when writing Grafana dashboards or Prometheus queries—the metric names in Prometheus will differ from the names defined in code.
+
+## Reference
+
+See the schedule engine (`internal-packages/schedule-engine/src/engine/index.ts`) for a good example of low-cardinality metric attributes.
+
+High cardinality metrics can cause:
+- Memory bloat in metrics backends (Axiom, Prometheus, etc.)
+- Slow queries and dashboard timeouts
+- Increased costs (many backends charge per time series)
+- Potential data loss or crashes at scale

.cursor/rules/repo.mdc

@@ -0,0 +1,6 @@
+---
+description: understanding the structure of the monorepo
+globs: 
+alwaysApply: true
+---
+We've documented the structure of our monorepo here: [repo.md](mdc:ai/references/repo.md)

.cursor/rules/webapp.mdc

@@ -0,0 +1,40 @@
+---
+description: Making updates to the main trigger.dev remix webapp
+globs: apps/webapp/**/*.tsx,apps/webapp/**/*.ts
+alwaysApply: false
+---
+
+The main trigger.dev webapp, which powers it's API and dashboard and makes up the docker image that is produced as an OSS image, is a Remix 2.1.0 app that uses an express server, written in TypeScript. The following subsystems are either included in the webapp or are used by the webapp in another part of the monorepo:
+
+- `@trigger.dev/database` exports a Prisma 6.14.0 client that is used extensively in the webapp to access a PostgreSQL instance. The schema file is [schema.prisma](mdc:internal-packages/database/prisma/schema.prisma)
+- `@trigger.dev/core` is a published package and is used to share code between the `@trigger.dev/sdk` and the webapp. It includes functionality but also a load of Zod schemas for data validation. When importing from `@trigger.dev/core` in the webapp, we never import the root `@trigger.dev/core` path, instead we favor one of the subpath exports that you can find in [package.json](mdc:packages/core/package.json)
+- `@internal/run-engine` has all the code needed to trigger a run and take it through it's lifecycle to completion.
+- `@trigger.dev/redis-worker` is a custom redis based background job/worker system that's used in the webapp and also used inside the run engine.
+
+## Environment variables and testing
+
+In the webapp, all environment variables are accessed through the `env` export of [env.server.ts](mdc:apps/webapp/app/env.server.ts), instead of directly accessing `process.env`.
+
+Ideally, the `env.server.ts` file would never be imported into a test file, either directly or indirectly. Tests should only imported classes and functions from a file matching `app/**/*.ts` of the webapp, and that file should not use environment variables, everything should be passed through as options instead. This "service/configuration" separation is important, and can be seen in a few places in the code for examples:
+
+- [realtimeClient.server.ts](mdc:apps/webapp/app/services/realtimeClient.server.ts) is the testable service, and [realtimeClientGlobal.server.ts](mdc:apps/webapp/app/services/realtimeClientGlobal.server.ts) is the configuration
+
+Also for writing tests in the webapp, checkout our [tests.md](mdc:ai/references/tests.md) guide
+
+## Legacy run engine vs Run Engine 2.0
+
+We originally the Trigger.dev "Run Engine" not as a single system, but just spread out all over the codebase, with no real separate or encapsulation. And we didn't even call it a "Run Engine". With Run Engine 2.0, we've completely rewritten big parts of the way the system works, and moved it over to an internal package called `@internal/run-engine`. So we've retroactively named the previous run engine "Legacy run engine". We're focused almost exclusively now on moving to Run Engine 2.0 and will be deprecating and removing the legacy run engine code eventually.
+
+## Where to look for code
+
+- The trigger API endpoint is [api.v1.tasks.$taskId.trigger.ts](mdc:apps/webapp/app/routes/api.v1.tasks.$taskId.trigger.ts)
+- The batch trigger API endpoint is [api.v1.tasks.batch.ts](mdc:apps/webapp/app/routes/api.v1.tasks.batch.ts)
+- Setup code for the prisma client is in [db.server.ts](mdc:apps/webapp/app/db.server.ts)
+- The run engine is configured in [runEngine.server.ts](mdc:apps/webapp/app/v3/runEngine.server.ts)
+- All the "services" that are found in app/v3/services/\*_/_.server.ts
+- The code for the TaskEvent data, which is the otel data sent from tasks to our servers, is in both the [eventRepository.server.ts](mdc:apps/webapp/app/v3/eventRepository.server.ts) and also the [otlpExporter.server.ts](mdc:apps/webapp/app/v3/otlpExporter.server.ts). The otel endpoints which are hit from production and development otel exporters is [otel.v1.logs.ts](mdc:apps/webapp/app/routes/otel.v1.logs.ts) and [otel.v1.traces.ts](mdc:apps/webapp/app/routes/otel.v1.traces.ts)
+- We use "presenters" to move more complex loader code into a class, and you can find those are app/v3/presenters/\*_/_.server.ts
+
+- All the "services" that are found in app/v3/services/\*_/_.server.ts
+- The code for the TaskEvent data, which is the otel data sent from tasks to our servers, is in both the [eventRepository.server.ts](mdc:apps/webapp/app/v3/eventRepository.server.ts) and also the [otlpExporter.server.ts](mdc:apps/webapp/app/v3/otlpExporter.server.ts). The otel endpoints which are hit from production and development otel exporters is [otel.v1.logs.ts](mdc:apps/webapp/app/routes/otel.v1.logs.ts) and [otel.v1.traces.ts](mdc:apps/webapp/app/routes/otel.v1.traces.ts)
+- We use "presenters" to move more complex loader code into a class, and you can find those are app/v3/presenters/\*_/_.server.ts

.cursor/rules/writing-tests.mdc

@@ -0,0 +1,6 @@
+---
+description: How to write tests in the monorepo
+globs: 
+alwaysApply: true
+---
+Follow our [tests.md](mdc:ai/references/tests.md) guide for how to write tests in the monorepo.

.cursorignore

@@ -0,0 +1,9 @@
+apps/docker-provider/
+apps/kubernetes-provider/
+apps/proxy/
+apps/coordinator/
+packages/rsc/
+.changeset
+.zed
+.env
+!.env.example

.dockerignore

@@ -1,43 +1,25 @@
-\*.log
-.git
-.github
+**/*.log
+**/*.pem
+**/*.tsbuildinfo
 
-# editor
+**/.cache
+**/.env
+**/.next
+**/.output
+**/.trigger
+**/.tshy
+**/.tshy-build
+**/.turbo
+**/.vercel
+**/.wrangler
 
-.idea
-.vscode
+**/dist
+**/node_modules
 
-# dependencies
+**/generated/prisma
 
-node_modules
-.pnp
-.pnp.js
-
-# testing
-
-coverage
-
-# next.js
-
-.next/
-build
-
-# packages
-
-build
-dist
-packages/\*\*/dist
-
-# misc
-
-.DS_Store
-\*.pem
-
-.turbo
-.vercel
-.cache
-.output
-apps/\*\*/public/build
+apps/webapp/build
+apps/webapp/public/build
 
 cypress/screenshots
 cypress/videos
@@ -46,8 +28,21 @@ apps/**/styles/tailwind.css
 packages/**/styles/tailwind.css
 
 .changeset
-references
+.DS_Store
+.git
+.github
+.idea
+.pnp
+.pnp.js
+.vscode
+
+coverage
+build
+docs
 examples
+out
+references
+
 CHANGESETS.md
 CONTRIBUTING.md
 README.md

.env.example

@@ -10,36 +10,85 @@ DIRECT_URL=${DATABASE_URL}
 REMIX_APP_PORT=3030
 APP_ENV=development
 APP_ORIGIN=http://localhost:3030
+ELECTRIC_ORIGIN=http://localhost:3060
 NODE_ENV=development
 
-# Redis is used for concurrency control
-# REDIS_HOST="localhost"
-# REDIS_PORT="6379"
-# REDIS_TLS_DISABLED="true"
+# Clickhouse
+CLICKHOUSE_URL=http://default:password@localhost:8123
+RUN_REPLICATION_CLICKHOUSE_URL=http://default:password@localhost:8123
+RUN_REPLICATION_ENABLED=1
+
+# Set this to UTC because Node.js uses the system timezone
+TZ="UTC"
+
+# Redis is used for the v3 queuing and v2 concurrency control
+REDIS_HOST="localhost"
+REDIS_PORT="6379"
+REDIS_TLS_DISABLED="true"
+
+DEV_OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:3030/otel"
+DEV_OTEL_BATCH_PROCESSING_ENABLED="0"
+
+# When the domain is set to `localhost` the CLI deploy command will only --load the image by default and not --push it
+DEPLOY_REGISTRY_HOST=localhost:5000
 
 # OPTIONAL VARIABLES
 # This is used for validating emails that are allowed to log in. Every email that do not match this regex will be rejected.
-# WHITELISTED_EMAILS="authorized@yahoo\.com|authorized@gmail\.com"
+# WHITELISTED_EMAILS="^(authorized@yahoo\.com|authorized@gmail\.com)$"
+# Accounts with these emails will get global admin rights. This grants access to the admin UI.
+# ADMIN_EMAILS="^(admin@example\.com|another-admin@example\.com)$"
 # This is used for logging in via GitHub. You can leave these commented out if you don't want to use GitHub for authentication.
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
 
-# Resend is an email service used for signing in to Trigger.dev via a Magic Link.
-# Emails will print to the console if you leave these commented out
+# Configure an email transport to allow users to sign in to Trigger.dev via a Magic Link.
+# If none are configured, emails will print to the console instead.
+# Uncomment one of the following blocks to allow delivery of
+
+# Resend
 ### Visit https://resend.com, create an account and get your API key. Then insert it below along with your From and Reply To email addresses. Visit https://resend.com/docs for more information.
-# RESEND_API_KEY=<api_key>
+# EMAIL_TRANSPORT=resend
+# FROM_EMAIL=
+# REPLY_TO_EMAIL=
+# RESEND_API_KEY=
+
+# Generic SMTP
+### Enter the configuration provided by your mail provider. Visit https://nodemailer.com/smtp/ for more information
+### SMTP_SECURE = false will use STARTTLS when connecting to a server that supports it (usually port 587)
+# EMAIL_TRANSPORT=smtp
+# FROM_EMAIL=
+# REPLY_TO_EMAIL=
+# SMTP_HOST=
+# SMTP_PORT=587
+# SMTP_SECURE=false
+# SMTP_USER=
+# SMTP_PASSWORD=
+
+# AWS Simple Email Service
+### Authentication is configured using the default Node.JS credentials provider chain (https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/Package/-aws-sdk-credential-providers/#fromnodeproviderchain)
+# EMAIL_TRANSPORT=aws-ses
 # FROM_EMAIL=
 # REPLY_TO_EMAIL=
 
 # CLOUD VARIABLES
 POSTHOG_PROJECT_KEY=
-PLAIN_API_KEY= 
-CLOUD_AIRTABLE_CLIENT_ID=
-CLOUD_AIRTABLE_CLIENT_SECRET=
-CLOUD_GITHUB_CLIENT_ID=
-CLOUD_GITHUB_CLIENT_SECRET=
-CLOUD_LINEAR_CLIENT_ID=
-CLOUD_LINEAR_CLIENT_SECRET=
-CLOUD_SLACK_APP_HOST=
-CLOUD_SLACK_CLIENT_ID=
-CLOUD_SLACK_CLIENT_SECRET= 
+
+# DEPOT_ORG_ID=<Depot org id>
+# DEPOT_TOKEN=<Depot org token>
+# DEV_OTEL_EXPORTER_OTLP_ENDPOINT="http://0.0.0.0:4318"
+# These are needed for the object store (for handling large payloads/outputs)
+# OBJECT_STORE_BASE_URL="https://{bucket}.{accountId}.r2.cloudflarestorage.com"
+# OBJECT_STORE_ACCESS_KEY_ID=
+# OBJECT_STORE_SECRET_ACCESS_KEY=
+# CHECKPOINT_THRESHOLD_IN_MS=10000
+
+# These control the server-side internal telemetry
+# INTERNAL_OTEL_TRACE_EXPORTER_URL=<URL to send traces to>
+# INTERNAL_OTEL_TRACE_LOGGING_ENABLED=1
+# INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED=0
+
+# Enable local observability stack (requires `pnpm run docker` to start otel-collector)
+# Uncomment these to send metrics to the local Prometheus via OTEL Collector:
+# INTERNAL_OTEL_METRIC_EXPORTER_ENABLED=1
+# INTERNAL_OTEL_METRIC_EXPORTER_URL=http://localhost:4318/v1/metrics
+# INTERNAL_OTEL_METRIC_EXPORTER_INTERVAL_MS=15000

.eslintrc.js

@@ -1,14 +0,0 @@
-module.exports = {
-  root: true,
-  // This tells ESLint to load the config from the package `eslint-config-custom`
-  extends: ["custom"],
-  settings: {
-    next: {
-      rootDir: ["apps/*/"],
-    },
-  },
-  parserOptions: {
-    sourceType: "module",
-    ecmaVersion: 2020,
-  },
-};

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [triggerdotdev]

.github/ISSUE_TEMPLATE/instrumentation_request.yml

@@ -0,0 +1,21 @@
+name: OpenTelemetry Auto-Instrumentation Request
+description: Suggest an SDK that you'd like to be auto-instrumented in the Run log view
+title: "auto-instrumentation: "
+labels: ["🌟 enhancement"]
+body:
+  - type: textarea
+    attributes:
+      label: What API or SDK would you to have automatic spans for?
+      description: A clear description of which API or SDK you'd like, and links to it.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Is there an existing OpenTelemetry auto-instrumentation package?
+      description: You can search for existing ones – https://opentelemetry.io/ecosystem/registry/?component=instrumentation&language=js
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Additional information
+      description: Add any other information related to the feature here. If your feature request is related to any issues or discussions, link them here.

.github/actions/get-image-tag/action.yml

@@ -0,0 +1,89 @@
+name: "#️⃣ Get image tag (action)"
+
+description: This action gets the image tag from the commit ref or input (if provided)
+
+outputs:
+  tag:
+    description: The image tag
+    value: ${{ steps.get_tag.outputs.tag }}
+  is_semver:
+    description: Whether the tag is a semantic version
+    value: ${{ steps.check_semver.outputs.is_semver }}
+
+inputs:
+  tag:
+    description: The image tag. If this is set it will return the tag as is.
+    required: false
+    default: ""
+
+runs:
+  using: "composite"
+  steps:
+    - name: "#️⃣ Get image tag (step)"
+      id: get_tag
+      shell: bash
+      run: |
+        if [[ -n "${{ inputs.tag }}" ]]; then
+          tag="${{ inputs.tag }}"
+        elif [[ "${{ github.ref_type }}" == "tag" ]]; then
+          if [[ "${{ github.ref_name }}" == infra-*-* ]]; then
+            env=$(echo ${{ github.ref_name }} | cut -d- -f2)
+            sha=$(echo ${{ github.sha }} | head -c7)
+            ts=$(date +%s)
+            tag=${env}-${sha}-${ts}
+          elif [[ "${{ github.ref_name }}" == re2-*-* ]]; then
+            env=$(echo ${{ github.ref_name }} | cut -d- -f2)
+            sha=$(echo ${{ github.sha }} | head -c7)
+            ts=$(date +%s)
+            tag=${env}-${sha}-${ts}
+          elif [[ "${{ github.ref_name }}" == v.docker.* ]]; then
+            version="${GITHUB_REF_NAME#v.docker.}"
+            tag="v${version}"
+          elif [[ "${{ github.ref_name }}" == build-* ]]; then
+            tag="${GITHUB_REF_NAME#build-}"
+          else
+            echo "Invalid git tag: ${{ github.ref_name }}"
+            exit 1
+          fi
+        elif [[ "${{ github.ref_name }}" == "main" ]]; then
+          tag="main"
+        else
+          echo "Invalid git ref: ${{ github.ref }}"
+          exit 1
+        fi
+        echo "tag=${tag}" >> "$GITHUB_OUTPUT"
+
+    - name: 🔍 Check for validity
+      id: check_validity
+      shell: bash
+      env:
+        tag: ${{ steps.get_tag.outputs.tag }}
+      run: |
+        if [[ "${tag}" =~ ^[a-z0-9]+([._-][a-z0-9]+)*$ ]]; then
+          echo "Tag is valid: ${tag}"
+        else
+          echo "Tag is not valid: ${tag}"
+          exit 1
+        fi
+
+    - name: 🆚 Check for semver
+      id: check_semver
+      shell: bash
+      env:
+        tag: ${{ steps.get_tag.outputs.tag }}
+      # Will match most semver formats except build metadata, i.e. v1.2.3+build.1
+      # Valid matches:
+      # v1.2.3
+      # v1.2.3-alpha
+      # v1.2.3-alpha.1
+      # v1.2.3-rc.1
+      # v1.2.3-beta-1
+      run: |
+        if [[ "${tag}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ ]]; then
+          echo "Tag is a semantic version: ${tag}"
+          is_semver=true
+        else
+          echo "Tag is not a semantic version: ${tag}"
+          is_semver=false
+        fi
+        echo "is_semver=${is_semver}" >> "$GITHUB_OUTPUT"

.github/copilot-instructions.md

@@ -0,0 +1 @@
+This is the repo for Trigger.dev, a background jobs platform written in TypeScript. Our webapp at apps/webapp is a Remix 2.1 app that uses Node.js v20. Our SDK is an isomorphic TypeScript SDK at packages/trigger-sdk. Always prefer using isomorphic code like fetch, ReadableStream, etc. instead of Node.js specific code. Our tests are all vitest. We use prisma in internal-packages/database for our database interactions using PostgreSQL. For TypeScript, we usually use types over interfaces. We use zod a lot in packages/core and in the webapp. Avoid enums. Use strict mode. No default exports, use function declarations.

.github/test/README.md

@@ -0,0 +1,70 @@
+# GitHub Action Tests
+
+This directory contains necessary files to allow local testing of GitHub Actions workflows, composite actions, etc. You will need to install [act](https://github.com/nektos/act) to perform tests.
+
+## Workflow tests
+
+Trigger specific workflow files by specifying their full path:
+
+```
+act -W .github/workflow/release.yml
+```
+
+You will likely need to override any custom runners we use, e.g. buildjet. For example:
+
+```
+override=catthehacker/ubuntu:act-latest
+
+act -W .github/workflow/release.yml \
+    -P buildjet-8vcpu-ubuntu-2204=$override
+
+# override multiple images at the same time
+act -W .github/workflow/release.yml \
+    -P buildjet-8vcpu-ubuntu-2204=$override \
+    -P buildjet-16vcpu-ubuntu-2204=$override
+```
+
+Trigger with specific event payloads to test pushing to branches or tags:
+
+```
+override=catthehacker/ubuntu:act-latest
+
+# simulate push to main
+act -W .github/workflow/publish.yml \
+    -P buildjet-8vcpu-ubuntu-2204=$override \
+    -P buildjet-16vcpu-ubuntu-2204=$override \
+    -e .github/events/push-tag-main.json
+
+# simulate a `build-` prefixed tag
+act -W .github/workflow/publish.yml \
+    -P buildjet-8vcpu-ubuntu-2204=$override \
+    -P buildjet-16vcpu-ubuntu-2204=$override \
+    -e .github/events/push-tag-buld.json
+```
+
+By default, `act` will send a push event. To trigger a different event:
+
+```
+# basic syntax
+act <EVENT> ...
+
+# simulate a pull request
+act pull_request
+
+# only trigger a specific workflow
+act pull_request -W .github/workflow/pr_checks.yml
+```
+
+## Composite action tests
+
+The composite (custom) action tests can be run by triggering the `test-actions` workflow:
+
+```
+act -W .github/test/test-actions.yml
+```
+
+## Helpful flags
+
+- `--pull=false` - perform fully offline tests if all images are already present
+- `-j <job_name>` - run the specified job only
+- `-l push` - list all workflows with push triggers

.github/test/events/push-main.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/heads/main"
+}

.github/test/events/push-tag-build.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/build-buildtag"
+}

.github/test/events/push-tag-docker-nonsemver.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/v.docker.nonsemver"
+}

.github/test/events/push-tag-docker.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/v.docker.1.2.3"
+}

.github/test/events/push-tag-infra-prod.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/infra-prod-anything"
+}

.github/test/events/push-tag-infra-test.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/infra-test-anything"
+}

.github/test/events/push-tag-semver.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/1.2.3"
+}

.github/test/events/push-tag.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/standard-tag"
+}

.github/test/test-actions.yml

@@ -0,0 +1,152 @@
+name: Test Actions
+
+on:
+  workflow_dispatch:
+
+jobs:
+  get-image-tag-none:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Log current ref
+        run: |
+          echo "ref: ${{ github.ref }}"
+          echo "ref_type: ${{ github.ref_type }}"
+          echo "ref_name: ${{ github.ref_name }}"
+
+      - name: Run without input tag
+        id: get_tag
+        # this step may fail depending on the current ref
+        continue-on-error: true
+        uses: ./.github/actions/get-image-tag
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-null:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Log current ref
+        run: |
+          echo "ref: ${{ github.ref }}"
+          echo "ref_type: ${{ github.ref_type }}"
+          echo "ref_name: ${{ github.ref_name }}"
+
+      - name: Run without input tag
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        # this step may fail depending on the current ref
+        continue-on-error: true
+        with:
+          # this should behave exactly as when no tag is provided
+          tag: null
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-override:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with tag override
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: "abc-123"
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-invalid-string:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with invalid string
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        # this step is expected to fail
+        continue-on-error: true
+        with:
+          # does not end with alphanumeric character
+          tag: "abc-123-"
+
+      - name: Fail job if previous step did not fail
+        if: steps.get_tag.outcome != 'failure'
+        run: exit 1
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-prerelease:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with prerelease semver
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: "v1.2.3-beta.4"
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-semver:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with basic semver
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: "v1.2.3"
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-invalid-semver:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with invalid semver
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        # this step is expected to fail
+        continue-on-error: true
+        with:
+          tag: "v1.2.3-"
+
+      - name: Fail job if previous step did not fail
+        if: steps.get_tag.outcome != 'failure'
+        run: exit 1
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"

.github/workflows/changesets-pr.yml

@@ -0,0 +1,102 @@
+name: 🦋 Changesets PR
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "packages/**"
+      - ".changeset/**"
+      - "package.json"
+      - "pnpm-lock.yaml"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  release-pr:
+    name: Create Release PR
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    if: github.repository == 'triggerdotdev/trigger.dev'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+
+      - name: Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+          cache: "pnpm"
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Create release PR
+        id: changesets
+        uses: changesets/action@v1
+        with:
+          version: pnpm run changeset:version
+          commit: "chore: release"
+          title: "chore: release"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Update PR title with version
+        if: steps.changesets.outputs.published != 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          PR_NUMBER=$(gh pr list --head changeset-release/main --json number --jq '.[0].number')
+          if [ -n "$PR_NUMBER" ]; then
+            git fetch origin changeset-release/main
+            # we arbitrarily reference the version of the cli package here; it is the same for all package releases
+            VERSION=$(git show origin/changeset-release/main:packages/cli-v3/package.json | jq -r '.version')
+            gh pr edit "$PR_NUMBER" --title "chore: release v$VERSION"
+          fi
+
+  update-lockfile:
+    name: Update lockfile on release PR
+    runs-on: ubuntu-latest
+    needs: release-pr
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout release branch
+        uses: actions/checkout@v4
+        with:
+          ref: changeset-release/main
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+
+      - name: Install and update lockfile
+        run: pnpm install --no-frozen-lockfile
+
+      - name: Commit and push lockfile
+        run: |
+          set -e
+          if git diff --quiet pnpm-lock.yaml; then
+            echo "No lockfile changes"
+          else
+            git config user.name "github-actions[bot]"
+            git config user.email "github-actions[bot]@users.noreply.github.com"
+            git add pnpm-lock.yaml
+            git commit -m "chore: update lockfile for release"
+            git push origin changeset-release/main
+          fi

.github/workflows/claude.yml

@@ -0,0 +1,70 @@
+name: Claude Code
+
+on:
+  issue_comment:
+    types: [created]
+  pull_request_review_comment:
+    types: [created]
+  issues:
+    types: [opened, assigned]
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  claude:
+    if: |
+      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
+      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+      issues: read
+      id-token: write
+      actions: read # Required for Claude to read CI results on PRs
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: Run Claude Code
+        id: claude
+        uses: anthropics/claude-code-action@v1
+        with:
+          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
+
+          # This is an optional setting that allows Claude to read CI results on PRs
+          additional_permissions: |
+            actions: read
+
+          claude_args: |
+            --model claude-opus-4-5-20251101
+            --allowedTools "Bash(pnpm:*),Bash(turbo:*),Bash(git:*),Bash(gh:*),Bash(npx:*),Bash(docker:*),Edit,MultiEdit,Read,Write,Glob,Grep,LS,Task"
+
+          # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
+          # prompt: 'Update the pull request description to include a summary of changes.'
+
+          # Optional: Add claude_args to customize behavior and configuration
+          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
+          # or https://code.claude.com/docs/en/cli-reference for available options
+          # claude_args: '--allowed-tools Bash(gh pr:*)'

.github/workflows/docs.yml

@@ -0,0 +1,42 @@
+name: 📚 Docs Checks
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "docs/**"
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - "docs/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  check-broken-links:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./docs
+    steps:
+      - name: 📥 Checkout repository
+        uses: actions/checkout@v4
+
+      - name: 📦 Cache npm
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.npm
+          key: |
+            ${{ runner.os }}-mintlify
+          restore-keys: |
+            ${{ runner.os }}-mintlify
+
+      - name: 🔗 Check for broken links
+        run: npx mintlify@4.0.393 broken-links

.github/workflows/e2e.yml

@@ -1,66 +1,59 @@
-name: "🧪 E2E Tests"
+name: "E2E"
+
+permissions:
+  contents: read
+
 on:
   workflow_call:
-jobs:
-  e2e:
-    name: "🧪 E2E Tests"
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    steps:
-      - name: 🐳 Login to Docker Hub
-        if: github.event_name == 'push'
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
+    inputs:
+      package:
+        description: The identifier of the job to run
+        default: webapp
+        required: false
+        type: string
 
+jobs:
+  cli-v3:
+    name: "🧪 CLI v3 tests (${{ matrix.os }} - ${{ matrix.package-manager }})"
+    if: inputs.package == 'cli-v3' || inputs.package == ''
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        package-manager: ["npm", "pnpm"]
+    steps:
       - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
         with:
-          version: 7.18
+          version: 10.23.0
 
       - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
+        uses: buildjet/setup-node@v4
         with:
-          node-version: 18
-          cache: "pnpm"
+          node-version: 20.20.0
 
       - name: 📥 Download deps
-        run: pnpm install --frozen-lockfile
+        run: pnpm install --frozen-lockfile --filter trigger.dev...
 
-      - name: Install Playwright Browsers
-        run: npx playwright install --with-deps
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
 
-      - name: Run Playwright tests
+      - name: 🔧 Build v3 cli monorepo dependencies
+        run: pnpm run build --filter trigger.dev^...
+
+      - name: 🔧 Build worker template files
+        run: pnpm --filter trigger.dev run build:workers
+
+      - name: Enable corepack
+        run: corepack enable
+
+      - name: Run E2E Tests
+        shell: bash
         run: |
-          # Setup environment variables
-          cp ./.env.example ./.env
-          cp ./references/nextjs-test/.env.example ./references/nextjs-test/.env.local
-
-          # Build packages
-          pnpm run build --filter @references/nextjs-test^...
-          pnpm --filter @trigger.dev/database generate
-
-          # Move trigger-cli bin to correct place
-          pnpm install --frozen-lockfile
-
-          # Execute tests
-          pnpm run docker
-          pnpm run db:migrate
-          pnpm run db:seed
-          pnpm run test:e2e
-
-          # Cleanup
-          pnpm run docker:stop
-
-      - name: Upload Playwright report
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: playwright-report
-          path: playwright-report/
-          retention-days: 30
+          LOG=debug PM=${{ matrix.package-manager }} pnpm --filter trigger.dev run test:e2e

.github/workflows/helm-pr-prerelease.yml

@@ -0,0 +1,138 @@
+name: 🧭 Helm Chart PR Prerelease
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - "hosting/k8s/helm/**"
+
+concurrency:
+  group: helm-prerelease-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+env:
+  REGISTRY: ghcr.io
+  CHART_NAME: trigger
+
+jobs:
+  lint-and-test:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v4
+        with:
+          version: "3.18.3"
+
+      - name: Build dependencies
+        run: helm dependency build ./hosting/k8s/helm/
+
+      - name: Extract dependency charts
+        run: |
+          cd ./hosting/k8s/helm/
+          for file in ./charts/*.tgz; do echo "Extracting $file"; tar -xzf "$file" -C ./charts; done
+
+      - name: Lint Helm Chart
+        run: |
+          helm lint ./hosting/k8s/helm/
+
+      - name: Render templates
+        run: |
+          helm template test-release ./hosting/k8s/helm/ \
+            --values ./hosting/k8s/helm/values.yaml \
+            --output-dir ./helm-output
+
+      - name: Validate manifests
+        uses: docker://ghcr.io/yannh/kubeconform:v0.7.0
+        with:
+          entrypoint: "/kubeconform"
+          args: "-summary -output json ./helm-output"
+
+  prerelease:
+    needs: lint-and-test
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      pull-requests: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v4
+        with:
+          version: "3.18.3"
+
+      - name: Build dependencies
+        run: helm dependency build ./hosting/k8s/helm/
+
+      - name: Extract dependency charts
+        run: |
+          cd ./hosting/k8s/helm/
+          for file in ./charts/*.tgz; do echo "Extracting $file"; tar -xzf "$file" -C ./charts; done
+
+      - name: Log in to Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Generate prerelease version
+        id: version
+        run: |
+          BASE_VERSION=$(grep '^version:' ./hosting/k8s/helm/Chart.yaml | awk '{print $2}')
+          PR_NUMBER=${{ github.event.pull_request.number }}
+          SHORT_SHA=$(echo "${{ github.event.pull_request.head.sha }}" | cut -c1-7)
+          PRERELEASE_VERSION="${BASE_VERSION}-pr${PR_NUMBER}.${SHORT_SHA}"
+          echo "version=$PRERELEASE_VERSION" >> $GITHUB_OUTPUT
+          echo "Prerelease version: $PRERELEASE_VERSION"
+
+      - name: Update Chart.yaml with prerelease version
+        run: |
+          sed -i "s/^version:.*/version: ${{ steps.version.outputs.version }}/" ./hosting/k8s/helm/Chart.yaml
+
+      - name: Package Helm Chart
+        run: |
+          helm package ./hosting/k8s/helm/ --destination /tmp/
+
+      - name: Push Helm Chart to GHCR
+        run: |
+          VERSION="${{ steps.version.outputs.version }}"
+          CHART_PACKAGE="/tmp/${{ env.CHART_NAME }}-${VERSION}.tgz"
+
+          # Push to GHCR OCI registry
+          helm push "$CHART_PACKAGE" "oci://${{ env.REGISTRY }}/${{ github.repository_owner }}/charts"
+
+      - name: Find existing comment
+        uses: peter-evans/find-comment@v3
+        id: find-comment
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: "github-actions[bot]"
+          body-includes: "Helm Chart Prerelease Published"
+
+      - name: Create or update PR comment
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          comment-id: ${{ steps.find-comment.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            ### 🧭 Helm Chart Prerelease Published
+
+            **Version:** `${{ steps.version.outputs.version }}`
+
+            **Install:**
+            ```bash
+            helm upgrade --install trigger \
+              oci://ghcr.io/${{ github.repository_owner }}/charts/trigger \
+              --version "${{ steps.version.outputs.version }}"
+            ```
+
+            > ⚠️ This is a prerelease for testing. Do not use in production.
+          edit-mode: replace

.github/workflows/pr_checks.yml

@@ -2,13 +2,10 @@ name: 🤖 PR Checks
 
 on:
   pull_request:
-    branches:
-      - main
+    types: [opened, synchronize, reopened]
     paths-ignore:
-      - "**.md"
-      - "**.mdx"
-      - ".github/CODEOWNERS"
-      - ".github/ISSUE_TEMPLATE/**"
+      - "docs/**"
+      - ".changeset/**"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -29,4 +26,10 @@ jobs:
 
   e2e:
     uses: ./.github/workflows/e2e.yml
+    with:
+      package: cli-v3
+    secrets: inherit
+
+  sdk-compat:
+    uses: ./.github/workflows/sdk-compat.yml
     secrets: inherit

.github/workflows/publish-docker.yml

@@ -1,84 +0,0 @@
-name: "🐳 Publish Docker"
-on:
-  workflow_call:
-jobs:
-  publish:
-    strategy:
-      fail-fast: true # when a job fails, all remaining ones will be cancelled
-      matrix:
-        runs-on: [buildjet-4vcpu-ubuntu-2204, buildjet-4vcpu-ubuntu-2204-arm]
-    name: ${{matrix.runs-on}}
-    runs-on: ${{matrix.runs-on}}
-    outputs:
-      version: ${{ steps.get_version.outputs.version }}
-      short_sha: ${{ steps.get_commit.outputs.sha_short }}
-    steps:
-      - name: 🐳 Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
-
-      - name: 🆚 Get the version
-        id: get_version
-        run: |
-          IMAGE_TAG="${GITHUB_REF#refs/tags/}"
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-            if [[ $IMAGE_TAG == v.docker.* ]]; then
-              ORIGINAL_VERSION="${IMAGE_TAG#v.docker.}"
-              IMAGE_TAG="v${ORIGINAL_VERSION}"
-            elif [[ $IMAGE_TAG == build-* ]]; then
-              IMAGE_TAG="${IMAGE_TAG#build-}"
-            fi
-            echo "IMAGE_TAG=${IMAGE_TAG}"
-          elif [[ $GITHUB_REF == refs/heads/main ]]; then
-            # Handle main branch specifically
-            IMAGE_TAG="main"
-            echo "IMAGE_TAG=${IMAGE_TAG}"
-          else
-            echo "Invalid reference: ${GITHUB_REF}"
-            exit 1
-          fi
-          if [[ ${{matrix.runs-on}} == *-arm ]]; then
-            IMAGE_TAG="${IMAGE_TAG}-arm"
-          fi
-          echo "::set-output name=version::${IMAGE_TAG}"
-      - name: 🔢 Get the commit hash
-        id: get_commit
-        run: |
-          echo ::set-output name=sha_short::$(echo ${{ github.sha }} | cut -c1-7)
-
-      - name: 🐳 Build Docker Image
-        run: |
-          docker build -t release_build_image -f ./docker/Dockerfile .
-
-      - name: 🐙 Login to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: 🐙 Push to GitHub Container Registry
-        run: |
-          docker tag release_build_image $REGISTRY/$REPOSITORY:$IMAGE_TAG
-          docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
-        env:
-          REGISTRY: ghcr.io/triggerdotdev
-          REPOSITORY: trigger.dev
-          IMAGE_TAG: ${{ steps.get_version.outputs.version }}
-
-      - name: 🐙 Push 'latest' to GitHub Container Registry
-        if: startsWith(github.ref, 'refs/tags/v.docker')
-        run: |
-          LATEST=latest
-          if [[ ${{matrix.runs-on}} == *-arm ]]; then
-            LATEST="${LATEST}-arm"
-          fi
-          docker tag release_build_image $REGISTRY/$REPOSITORY:$LATEST
-          docker push $REGISTRY/$REPOSITORY:$LATEST
-        env:
-          REGISTRY: ghcr.io/triggerdotdev
-          REPOSITORY: trigger.dev

.github/workflows/publish-webapp.yml

@@ -0,0 +1,93 @@
+name: "🐳 Publish Webapp"
+
+permissions:
+  contents: read
+  packages: write
+  id-token: write
+
+on:
+  workflow_call:
+    inputs:
+      image_tag:
+        description: The image tag to publish
+        type: string
+        required: false
+        default: ""
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    env:
+      PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING: 1
+    outputs:
+      version: ${{ steps.get_tag.outputs.tag }}
+      short_sha: ${{ steps.get_commit.outputs.sha_short }}
+    steps:
+      - name: 🏭 Setup Depot CLI
+        uses: depot/setup-action@v1
+
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: "#️⃣ Get the image tag"
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: ${{ inputs.image_tag }}
+
+      - name: 🔢 Get the commit hash
+        id: get_commit
+        run: |
+          echo "sha_short=$(echo ${{ github.sha }} | cut -c1-7)" >> "$GITHUB_OUTPUT"
+
+      - name: 📛 Set the tags
+        id: set_tags
+        run: |
+          ref_without_tag=ghcr.io/triggerdotdev/trigger.dev
+          image_tags=$ref_without_tag:${{ steps.get_tag.outputs.tag }}
+
+          # if tag is a semver, also tag it as v4
+          if [[ "${{ steps.get_tag.outputs.is_semver }}" == true ]]; then
+            # TODO: switch to v4 tag on GA
+            image_tags=$image_tags,$ref_without_tag:v4-beta
+          fi
+
+          echo "image_tags=${image_tags}" >> "$GITHUB_OUTPUT"
+
+      - name: 📝 Set the build info
+        id: set_build_info
+        run: |
+          tag=${{ steps.get_tag.outputs.tag }}
+          if [[ "${{ steps.get_tag.outputs.is_semver }}" == true ]]; then
+            echo "BUILD_APP_VERSION=${tag}" >> "$GITHUB_OUTPUT"
+          fi
+          echo "BUILD_GIT_SHA=${{ github.sha }}" >> "$GITHUB_OUTPUT"
+          echo "BUILD_GIT_REF_NAME=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
+          echo "BUILD_TIMESTAMP_SECONDS=$(date +%s)" >> "$GITHUB_OUTPUT"
+
+      - name: 🐙 Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 🐳 Build image and push to GitHub Container Registry
+        uses: depot/build-push-action@v1
+        with:
+          file: ./docker/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.set_tags.outputs.image_tags }}
+          push: true
+          build-args: |
+            BUILD_APP_VERSION=${{ steps.set_build_info.outputs.BUILD_APP_VERSION }}
+            BUILD_GIT_SHA=${{ steps.set_build_info.outputs.BUILD_GIT_SHA }}
+            BUILD_GIT_REF_NAME=${{ steps.set_build_info.outputs.BUILD_GIT_REF_NAME }}
+            BUILD_TIMESTAMP_SECONDS=${{ steps.set_build_info.outputs.BUILD_TIMESTAMP_SECONDS }}
+            SENTRY_RELEASE=${{ steps.set_build_info.outputs.BUILD_GIT_SHA }}
+            SENTRY_ORG=triggerdev
+            SENTRY_PROJECT=trigger-cloud
+          secrets: |
+            sentry_auth_token=${{ secrets.SENTRY_AUTH_TOKEN }}

.github/workflows/publish-worker-v4.yml

@@ -0,0 +1,89 @@
+name: "⚒️ Publish Worker (v4)"
+
+on:
+  workflow_call:
+    inputs:
+      image_tag:
+        description: The image tag to publish
+        type: string
+        required: false
+        default: ""
+  push:
+    tags:
+      - "re2-test-*"
+      - "re2-prod-*"
+
+permissions:
+  id-token: write
+  packages: write
+  contents: read
+
+jobs:
+  # check-branch:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Fail if re2-prod-* is pushed from a non-main branch
+  #       if: startsWith(github.ref_name, 're2-prod-') && github.base_ref != 'main'
+  #       run: |
+  #         echo "🚫 re2-prod-* tags can only be pushed from the main branch."
+  #         exit 1
+  build:
+    # needs: check-branch
+    strategy:
+      matrix:
+        package: [supervisor]
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: "1"
+    steps:
+      - name: 🏭 Setup Depot CLI
+        uses: depot/setup-action@v1
+
+      - name: ⬇️ Checkout git repo
+        uses: actions/checkout@v4
+
+      - name: 📦 Get image repo
+        id: get_repository
+        run: |
+          if [[ "${{ matrix.package }}" == *-provider ]]; then
+            provider_type=$(echo "${{ matrix.package }}" | cut -d- -f1)
+            repo=provider/${provider_type}
+          else
+            repo="${{ matrix.package }}"
+          fi
+          echo "repo=${repo}" >> "$GITHUB_OUTPUT"
+
+      - name: "#️⃣ Get image tag"
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: ${{ inputs.image_tag }}
+
+      - name: 📛 Set tags to push
+        id: set_tags
+        run: |
+          ref_without_tag=ghcr.io/triggerdotdev/${{ steps.get_repository.outputs.repo }}
+          image_tags=$ref_without_tag:${{ steps.get_tag.outputs.tag }}
+
+          # if tag is a semver, also tag it as v4
+          if [[ "${{ steps.get_tag.outputs.is_semver }}" == true ]]; then
+            # TODO: switch to v4 tag on GA
+            image_tags=$image_tags,$ref_without_tag:v4-beta
+          fi
+
+          echo "image_tags=${image_tags}" >> "$GITHUB_OUTPUT"
+
+      - name: 🐙 Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 🐳 Build image and push to GitHub Container Registry
+        uses: depot/build-push-action@v1
+        with:
+          file: ./apps/${{ matrix.package }}/Containerfile
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.set_tags.outputs.image_tags }}
+          push: true

.github/workflows/publish-worker.yml

@@ -0,0 +1,87 @@
+name: "⚒️ Publish Worker"
+
+on:
+  workflow_call:
+    inputs:
+      image_tag:
+        description: The image tag to publish
+        type: string
+        required: false
+        default: ""
+  push:
+    tags:
+      - "infra-dev-*"
+      - "infra-test-*"
+      - "infra-prod-*"
+
+permissions:
+  packages: write
+  contents: read
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        package: [coordinator, docker-provider, kubernetes-provider]
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: "1"
+    steps:
+      - name: ⬇️ Checkout git repo
+        uses: actions/checkout@v4
+
+      - name: 📦 Get image repo
+        id: get_repository
+        run: |
+          if [[ "${{ matrix.package }}" == *-provider ]]; then
+            provider_type=$(echo "${{ matrix.package }}" | cut -d- -f1)
+            repo=provider/${provider_type}
+          else
+            repo="${{ matrix.package }}"
+          fi
+          echo "repo=${repo}" >> "$GITHUB_OUTPUT"
+
+      - id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: ${{ inputs.image_tag }}
+
+      - name: 🐋 Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # ..to avoid rate limits when pulling images
+      - name: 🐳 Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: 🚢 Build Container Image
+        run: |
+          docker build -t infra_image -f ./apps/${{ matrix.package }}/Containerfile .
+
+      # ..to push image
+      - name: 🐙 Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 🐙 Push to GitHub Container Registry
+        run: |
+          docker tag infra_image "$REGISTRY/$REPOSITORY:$IMAGE_TAG"
+          docker push "$REGISTRY/$REPOSITORY:$IMAGE_TAG"
+        env:
+          REGISTRY: ghcr.io/triggerdotdev
+          REPOSITORY: ${{ steps.get_repository.outputs.repo }}
+          IMAGE_TAG: ${{ steps.get_tag.outputs.tag }}
+
+      # - name: 🐙 Push 'v3' tag to GitHub Container Registry
+      #   if: steps.get_tag.outputs.is_semver == 'true'
+      #   run: |
+      #     docker tag infra_image "$REGISTRY/$REPOSITORY:v3"
+      #     docker push "$REGISTRY/$REPOSITORY:v3"
+      #   env:
+      #     REGISTRY: ghcr.io/triggerdotdev
+      #     REPOSITORY: ${{ steps.get_repository.outputs.repo }}

.github/workflows/publish.yml

@@ -1,6 +1,13 @@
 name: 🚀 Publish Trigger.dev Docker
 
 on:
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      image_tag:
+        description: The image tag to publish
+        required: true
+        type: string
   push:
     branches:
       - main
@@ -8,16 +15,20 @@ on:
       - "v.docker.*"
       - "build-*"
     paths:
+      - ".github/actions/**/*.yml"
       - ".github/workflows/publish.yml"
+      - ".github/workflows/typecheck.yml"
+      - ".github/workflows/unit-tests.yml"
+      - ".github/workflows/e2e.yml"
+      - ".github/workflows/publish-webapp.yml"
+      - ".github/workflows/publish-worker.yml"
       - "packages/**"
       - "!packages/**/*.md"
       - "!packages/**/*.eslintrc"
+      - "internal-packages/**"
       - "apps/**"
       - "!apps/**/*.md"
       - "!apps/**/*.eslintrc"
-      - "integrations/**"
-      - "!integrations/**/*.md"
-      - "!integrations/**/*.eslintrc"
       - "pnpm-lock.yaml"
       - "pnpm-workspace.yaml"
       - "turbo.json"
@@ -45,11 +56,23 @@ jobs:
     uses: ./.github/workflows/unit-tests.yml
     secrets: inherit
 
-  e2e:
-    uses: ./.github/workflows/e2e.yml
+  publish-webapp:
+    needs: [typecheck]
+    uses: ./.github/workflows/publish-webapp.yml
     secrets: inherit
+    with:
+      image_tag: ${{ inputs.image_tag }}
 
-  publish:
-    needs: [typecheck, units, e2e]
-    uses: ./.github/workflows/publish-docker.yml
+  publish-worker:
+    needs: [typecheck]
+    uses: ./.github/workflows/publish-worker.yml
     secrets: inherit
+    with:
+      image_tag: ${{ inputs.image_tag }}
+
+  publish-worker-v4:
+    needs: [typecheck]
+    uses: ./.github/workflows/publish-worker-v4.yml
+    secrets: inherit
+    with:
+      image_tag: ${{ inputs.image_tag }}

.github/workflows/release-helm.yml

@@ -0,0 +1,143 @@
+name: 🧭 Helm Chart Release
+
+on:
+  push:
+    tags:
+      - 'helm-v*'
+  workflow_dispatch:
+    inputs:
+      chart_version:
+        description: 'Chart version to release'
+        required: true
+        type: string
+
+env:
+  REGISTRY: ghcr.io
+  CHART_NAME: trigger
+
+jobs:
+  lint-and-test:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v4
+        with:
+          version: "3.18.3"
+
+      - name: Build dependencies
+        run: helm dependency build ./hosting/k8s/helm/
+
+      - name: Extract dependency charts
+        run: |
+          cd ./hosting/k8s/helm/
+          for file in ./charts/*.tgz; do echo "Extracting $file"; tar -xzf "$file" -C ./charts; done
+
+      - name: Lint Helm Chart
+        run: |
+          helm lint ./hosting/k8s/helm/
+
+      - name: Render templates
+        run: |
+          helm template test-release ./hosting/k8s/helm/ \
+            --values ./hosting/k8s/helm/values.yaml \
+            --output-dir ./helm-output
+
+      - name: Validate manifests
+        uses: docker://ghcr.io/yannh/kubeconform:v0.7.0
+        with:
+          entrypoint: '/kubeconform'
+          args: "-summary -output json ./helm-output"
+
+  release:
+    needs: lint-and-test
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write # for gh-release
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v4
+        with:
+          version: "3.18.3"
+
+      - name: Build dependencies
+        run: helm dependency build ./hosting/k8s/helm/
+
+      - name: Extract dependency charts
+        run: |
+          cd ./hosting/k8s/helm/
+          for file in ./charts/*.tgz; do echo "Extracting $file"; tar -xzf "$file" -C ./charts; done
+
+      - name: Log in to Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract version from tag or input
+        id: version
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            VERSION="${{ github.event.inputs.chart_version }}"
+          else
+            VERSION="${{ github.ref_name }}"
+            VERSION="${VERSION#helm-v}"
+          fi
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "Releasing version: $VERSION"
+
+      - name: Check Chart.yaml version matches release version
+        run: |
+          VERSION="${{ steps.version.outputs.version }}"
+          CHART_VERSION=$(grep '^version:' ./hosting/k8s/helm/Chart.yaml | awk '{print $2}')
+          echo "Chart.yaml version: $CHART_VERSION"
+          echo "Release version: $VERSION"
+          if [ "$CHART_VERSION" != "$VERSION" ]; then
+            echo "❌ Chart.yaml version does not match release version!"
+            exit 1
+          fi
+          echo "✅ Chart.yaml version matches release version."
+
+      - name: Package Helm Chart
+        run: |
+          helm package ./hosting/k8s/helm/ --destination /tmp/
+
+      - name: Push Helm Chart to GHCR
+        run: |
+          VERSION="${{ steps.version.outputs.version }}"
+          CHART_PACKAGE="/tmp/${{ env.CHART_NAME }}-${VERSION}.tgz"
+          
+          # Push to GHCR OCI registry
+          helm push "$CHART_PACKAGE" "oci://${{ env.REGISTRY }}/${{ github.repository_owner }}/charts"
+
+      - name: Create GitHub Release
+        id: release
+        uses: softprops/action-gh-release@v1
+        if: github.event_name == 'push'
+        with:
+          tag_name: ${{ github.ref_name }}
+          name: "Helm Chart ${{ steps.version.outputs.version }}"
+          body: |
+            ### Installation
+            ```bash
+            helm upgrade --install trigger \
+              oci://${{ env.REGISTRY }}/${{ github.repository_owner }}/charts/${{ env.CHART_NAME }} \
+              --version "${{ steps.version.outputs.version }}"
+            ```
+            
+            ### Changes
+            See commit history for detailed changes in this release.
+          files: |
+            /tmp/${{ env.CHART_NAME }}-${{ steps.version.outputs.version }}.tgz
+          token: ${{ secrets.GITHUB_TOKEN }}
+          draft: true
+          prerelease: true

.github/workflows/release.yml

@@ -1,72 +1,195 @@
 name: 🦋 Changesets Release
 
 on:
-  push:
+  pull_request:
+    types: [closed]
     branches:
       - main
-    paths-ignore:
-      - "**.md"
-      - ".github/CODEOWNERS"
-      - ".github/ISSUE_TEMPLATE/**"
-    
+  workflow_dispatch:
+    inputs:
+      type:
+        description: "Select release type"
+        required: true
+        type: choice
+        options:
+          - release
+          - prerelease
+        default: "prerelease"
+      ref:
+        description: "The ref (branch, tag, or SHA) to checkout and release from"
+        required: true
+        type: string
+      prerelease_tag:
+        description: "The npm dist-tag for the prerelease (e.g., 'v4-prerelease')"
+        required: false
+        type: string
+        default: "prerelease"
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: false
 
 jobs:
-  release:
-    name: 🦋 Changesets Release
-    runs-on: buildjet-4vcpu-ubuntu-2204
+  show-release-summary:
+    name: 📋 Release Summary
+    runs-on: ubuntu-latest
     if: |
-      github.repository == 'triggerdotdev/trigger.dev'
-    outputs:
-      published_packages: ${{ steps.changesets.outputs.publishedPackages }}
-      published: ${{ steps.changesets.outputs.published }}
+      github.repository == 'triggerdotdev/trigger.dev' &&
+      github.event_name == 'pull_request' &&
+      github.event.pull_request.merged == true &&
+      github.event.pull_request.head.ref == 'changeset-release/main'
     steps:
-      - name: 🛑 Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.11.0
+      - name: Show release summary
+        env:
+          PR_BODY: ${{ github.event.pull_request.body }}
+        run: |
+          echo "$PR_BODY" | sed -n '/^# Releases/,$p' >> $GITHUB_STEP_SUMMARY
 
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
+  release:
+    name: 🚀 Release npm packages
+    runs-on: ubuntu-latest
+    environment: npm-publish
+    permissions:
+      contents: write
+      packages: write
+      id-token: write
+    if: |
+      github.repository == 'triggerdotdev/trigger.dev' &&
+      (
+        (github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'release') ||
+        (github.event_name == 'pull_request' && github.event.pull_request.merged == true && github.event.pull_request.head.ref == 'changeset-release/main')
+      )
+    outputs:
+      published: ${{ steps.changesets.outputs.published }}
+      published_packages: ${{ steps.changesets.outputs.publishedPackages }}
+      published_package_version: ${{ steps.get_version.outputs.package_version }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.ref || github.sha }}
 
-      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
-        with:
-          version: 7.18
+      - name: Verify ref is on main
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          if ! git merge-base --is-ancestor ${{ github.event.inputs.ref }} origin/main; then
+            echo "Error: ref must be an ancestor of main (i.e., already merged)"
+            exit 1
+          fi
 
-      - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
         with:
-          node-version: 18
+          version: 10.23.0
+
+      - name: Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
           cache: "pnpm"
 
-      - name: 📥 Download deps
+      # npm v11.5.1 or newer is required for OIDC support
+      # https://github.blog/changelog/2025-07-31-npm-trusted-publishing-with-oidc-is-generally-available/#whats-new
+      - name: Setup npm 11.x for OIDC
+        run: npm install -g npm@11.6.4
+
+      - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
-      - name: 📀 Generate Prisma Client
+      - name: Generate Prisma client
         run: pnpm run generate
 
-      - name: 🔎 Type check
-        run: pnpm run typecheck
+      - name: Build
+        run: pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev"
 
-      - name: 🔐 Setup npm auth
-        run: |
-          echo "registry=https://registry.npmjs.org" >> ~/.npmrc
-          echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}" >> ~/.npmrc
+      - name: Type check
+        run: pnpm run typecheck --filter "@trigger.dev/*" --filter "trigger.dev"
 
-        # This action has two responsibilities. The first time the workflow runs
-        # (initial push to the `main` branch) it will create a new branch and
-        # then open a PR with the related changes for the new version. After the
-        # PR is merged, the workflow will run again and this action will build +
-        # publish to npm.
-      - name: 🚀 PR / Publish
+      - name: Publish
         id: changesets
         uses: changesets/action@v1
         with:
-          version: pnpm run changeset:version
-          commit: "chore: Update version for release"
-          title: "chore: Update version for release"
           publish: pnpm run changeset:release
-          createGithubReleases: false
+          createGithubReleases: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Show package version
+        if: steps.changesets.outputs.published == 'true'
+        id: get_version
+        run: |
+          package_version=$(echo '${{ steps.changesets.outputs.publishedPackages }}' | jq -r '.[0].version')
+          echo "package_version=${package_version}" >> "$GITHUB_OUTPUT"
+
+      - name: Create and push Docker tag
+        if: steps.changesets.outputs.published == 'true'
+        run: |
+          set -e
+          git tag "v.docker.${{ steps.get_version.outputs.package_version }}"
+          git push origin "v.docker.${{ steps.get_version.outputs.package_version }}"
+
+  # Trigger Docker builds directly via workflow_call since tags pushed with
+  # GITHUB_TOKEN don't trigger other workflows (GitHub Actions limitation).
+  publish-docker:
+    name: 🐳 Publish Docker images
+    needs: release
+    if: needs.release.outputs.published == 'true'
+    uses: ./.github/workflows/publish.yml
+    secrets: inherit
+    with:
+      image_tag: v${{ needs.release.outputs.published_package_version }}
+
+  # The prerelease job needs to be on the same workflow file due to a limitation related to how npm verifies OIDC claims.
+  prerelease:
+    name: 🧪 Prerelease
+    runs-on: ubuntu-latest
+    environment: npm-publish
+    permissions:
+      contents: read
+      id-token: write
+    if: github.repository == 'triggerdotdev/trigger.dev' && github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'prerelease'
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.inputs.ref }}
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+          cache: "pnpm"
+
+      # npm v11.5.1 or newer is required for OIDC support
+      # https://github.blog/changelog/2025-07-31-npm-trusted-publishing-with-oidc-is-generally-available/#whats-new
+      - name: Setup npm 11.x for OIDC
+        run: npm install -g npm@11.6.4
+
+      - name: Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: Generate Prisma Client
+        run: pnpm run generate
+
+      - name: Snapshot version
+        run: pnpm exec changeset version --snapshot ${{ github.event.inputs.prerelease_tag }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Clean
+        run: pnpm run clean --filter "@trigger.dev/*" --filter "trigger.dev"
+
+      - name: Build
+        run: pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev"
+
+      - name: Publish prerelease
+        run: pnpm exec changeset publish --no-git-tag --snapshot --tag ${{ github.event.inputs.prerelease_tag }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/sdk-compat.yml

@@ -0,0 +1,178 @@
+name: "🔌 SDK Compatibility Tests"
+
+permissions:
+  contents: read
+
+on:
+  workflow_call:
+
+jobs:
+  node-compat:
+    name: "Node.js ${{ matrix.node }} (${{ matrix.os }})"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        node: ["20.20", "22.12"]
+
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: ${{ matrix.node }}
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: 🔨 Build SDK dependencies
+        shell: bash
+        run: pnpm run build --filter '@trigger.dev/sdk^...'
+
+      - name: 🔨 Build SDK
+        shell: bash
+        run: pnpm run build --filter '@trigger.dev/sdk'
+
+      - name: 🧪 Run SDK Compatibility Tests
+        shell: bash
+        run: pnpm --filter @internal/sdk-compat-tests test
+
+  bun-compat:
+    name: "Bun Runtime"
+    runs-on: ubuntu-latest
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+          cache: "pnpm"
+
+      - name: 🥟 Setup Bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: 🔨 Build SDK dependencies
+        run: pnpm run build --filter @trigger.dev/sdk^...
+
+      - name: 🔨 Build SDK
+        run: pnpm run build --filter @trigger.dev/sdk
+
+      - name: 🧪 Run Bun Compatibility Test
+        working-directory: internal-packages/sdk-compat-tests/src/fixtures/bun
+        run: bun run test.ts
+
+  deno-compat:
+    name: "Deno Runtime"
+    runs-on: ubuntu-latest
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+          cache: "pnpm"
+
+      - name: 🦕 Setup Deno
+        uses: denoland/setup-deno@v2
+        with:
+          deno-version: v2.x
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: 🔨 Build SDK dependencies
+        run: pnpm run build --filter @trigger.dev/sdk^...
+
+      - name: 🔨 Build SDK
+        run: pnpm run build --filter @trigger.dev/sdk
+
+      - name: 🔗 Link node_modules for Deno fixture
+        working-directory: internal-packages/sdk-compat-tests/src/fixtures/deno
+        run: ln -s ../../../../../node_modules node_modules
+
+      - name: 🧪 Run Deno Compatibility Test
+        working-directory: internal-packages/sdk-compat-tests/src/fixtures/deno
+        run: deno run --allow-read --allow-env --allow-sys test.ts
+
+  cloudflare-compat:
+    name: "Cloudflare Workers"
+    runs-on: ubuntu-latest
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: 🔨 Build SDK dependencies
+        run: pnpm run build --filter @trigger.dev/sdk^...
+
+      - name: 🔨 Build SDK
+        run: pnpm run build --filter @trigger.dev/sdk
+
+      - name: 📥 Install Cloudflare fixture deps
+        working-directory: internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker
+        run: pnpm install
+
+      - name: 🧪 Run Cloudflare Workers Compatibility Test (dry-run)
+        working-directory: internal-packages/sdk-compat-tests/src/fixtures/cloudflare-worker
+        run: npx wrangler deploy --dry-run --outdir dist

.github/workflows/typecheck.yml

@@ -1,25 +1,30 @@
 name: "ʦ TypeScript"
+
 on:
   workflow_call:
+
+permissions:
+  contents: read
+
 jobs:
   typecheck:
-    runs-on: buildjet-4vcpu-ubuntu-2204
+    runs-on: ubuntu-latest
 
     steps:
       - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
         with:
-          version: 7.18
+          version: 10.23.0
 
       - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
+        uses: buildjet/setup-node@v4
         with:
-          node-version: 18
+          node-version: 20.20.0
           cache: "pnpm"
 
       - name: 📥 Download deps
@@ -29,4 +34,9 @@ jobs:
         run: pnpm run generate
 
       - name: 🔎 Type check
-        run: pnpm run typecheck --filter webapp
+        run: pnpm run typecheck
+        env:
+          NODE_OPTIONS: --max-old-space-size=8192
+
+      - name: 🔎 Check exports
+        run: pnpm run check-exports

.github/workflows/unit-tests-internal.yml

@@ -0,0 +1,141 @@
+name: "🧪 Unit Tests: Internal"
+
+permissions:
+  contents: read
+
+on:
+  workflow_call:
+
+jobs:
+  unitTests:
+    name: "🧪 Unit Tests: Internal"
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        shardIndex: [1, 2, 3, 4, 5, 6, 7, 8]
+        shardTotal: [8]
+    env:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      SHARD_INDEX: ${{ matrix.shardIndex }}
+      SHARD_TOTAL: ${{ matrix.shardTotal }}
+    steps:
+      - name: 🔧 Disable IPv6
+        run: |
+          sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
+          sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1
+          sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=1
+
+      - name: 🔧 Configure docker address pool
+        run: |
+          CONFIG='{
+            "default-address-pools" : [
+              {
+                "base" : "172.17.0.0/12",
+                "size" : 20
+              },
+              {
+                "base" : "192.168.0.0/16",
+                "size" : 24
+              }
+            ]
+          }'
+          mkdir -p /etc/docker
+          echo "$CONFIG" | sudo tee /etc/docker/daemon.json
+
+      - name: 🔧 Restart docker daemon
+        run: sudo systemctl restart docker
+
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+          cache: "pnpm"
+
+      # ..to avoid rate limits when pulling images
+      - name: 🐳 Login to DockerHub
+        if: ${{ env.DOCKERHUB_USERNAME }}
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: 🐳 Skipping DockerHub login (no secrets available)
+        if: ${{ !env.DOCKERHUB_USERNAME }}
+        run: echo "DockerHub login skipped because secrets are not available."
+
+      - name: 🐳 Pre-pull testcontainer images
+        if: ${{ env.DOCKERHUB_USERNAME }}
+        run: |
+          echo "Pre-pulling Docker images with authenticated session..."
+          docker pull postgres:14
+          docker pull clickhouse/clickhouse-server:25.4-alpine
+          docker pull redis:7-alpine
+          docker pull testcontainers/ryuk:0.11.0
+          docker pull electricsql/electric:1.2.4
+          echo "Image pre-pull complete"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: 🧪 Run Internal Unit Tests
+        run: pnpm run test:internal --reporter=default --reporter=blob --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+
+      - name: Gather all reports
+        if: ${{ !cancelled() }}
+        run: |
+          mkdir -p .vitest-reports
+          find . -type f -path '*/.vitest-reports/blob-*.json' \
+            -exec bash -c 'src="$1"; basename=$(basename "$src"); pkg=$(dirname "$src" | sed "s|^\./||;s|/\.vitest-reports$||;s|/|_|g"); cp "$src" ".vitest-reports/${pkg}-${basename}"' _ {} \;
+
+      - name: Upload blob reports to GitHub Actions Artifacts
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: internal-blob-report-${{ matrix.shardIndex }}
+          path: .vitest-reports/*
+          include-hidden-files: true
+          retention-days: 1
+
+  merge-reports:
+    name: "📊 Merge Reports"
+    if: ${{ !cancelled() }}
+    needs: [unitTests]
+    runs-on: ubuntu-latest
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+          # no cache enabled, we're not installing deps
+
+      - name: Download blob reports from GitHub Actions Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: .vitest-reports
+          pattern: internal-blob-report-*
+          merge-multiple: true
+
+      - name: Merge reports
+        run: pnpm dlx vitest@3.1.4 run --merge-reports --pass-with-no-tests

.github/workflows/unit-tests-packages.yml

@@ -0,0 +1,141 @@
+name: "🧪 Unit Tests: Packages"
+
+permissions:
+  contents: read
+
+on:
+  workflow_call:
+
+jobs:
+  unitTests:
+    name: "🧪 Unit Tests: Packages"
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        shardIndex: [1]
+        shardTotal: [1]
+    env:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      SHARD_INDEX: ${{ matrix.shardIndex }}
+      SHARD_TOTAL: ${{ matrix.shardTotal }}
+    steps:
+      - name: 🔧 Disable IPv6
+        run: |
+          sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
+          sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1
+          sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=1
+
+      - name: 🔧 Configure docker address pool
+        run: |
+          CONFIG='{
+            "default-address-pools" : [
+              {
+                "base" : "172.17.0.0/12",
+                "size" : 20
+              },
+              {
+                "base" : "192.168.0.0/16",
+                "size" : 24
+              }
+            ]
+          }'
+          mkdir -p /etc/docker
+          echo "$CONFIG" | sudo tee /etc/docker/daemon.json
+
+      - name: 🔧 Restart docker daemon
+        run: sudo systemctl restart docker
+
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+          cache: "pnpm"
+
+      # ..to avoid rate limits when pulling images
+      - name: 🐳 Login to DockerHub
+        if: ${{ env.DOCKERHUB_USERNAME }}
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: 🐳 Skipping DockerHub login (no secrets available)
+        if: ${{ !env.DOCKERHUB_USERNAME }}
+        run: echo "DockerHub login skipped because secrets are not available."
+
+      - name: 🐳 Pre-pull testcontainer images
+        if: ${{ env.DOCKERHUB_USERNAME }}
+        run: |
+          echo "Pre-pulling Docker images with authenticated session..."
+          docker pull postgres:14
+          docker pull clickhouse/clickhouse-server:25.4-alpine
+          docker pull redis:7-alpine
+          docker pull testcontainers/ryuk:0.11.0
+          docker pull electricsql/electric:1.2.4
+          echo "Image pre-pull complete"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: 🧪 Run Package Unit Tests
+        run: pnpm run test:packages --reporter=default --reporter=blob --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+
+      - name: Gather all reports
+        if: ${{ !cancelled() }}
+        run: |
+          mkdir -p .vitest-reports
+          find . -type f -path '*/.vitest-reports/blob-*.json' \
+            -exec bash -c 'src="$1"; basename=$(basename "$src"); pkg=$(dirname "$src" | sed "s|^\./||;s|/\.vitest-reports$||;s|/|_|g"); cp "$src" ".vitest-reports/${pkg}-${basename}"' _ {} \;
+
+      - name: Upload blob reports to GitHub Actions Artifacts
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: packages-blob-report-${{ matrix.shardIndex }}
+          path: .vitest-reports/*
+          include-hidden-files: true
+          retention-days: 1
+
+  merge-reports:
+    name: "📊 Merge Reports"
+    if: ${{ !cancelled() }}
+    needs: [unitTests]
+    runs-on: ubuntu-latest
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+          # no cache enabled, we're not installing deps
+
+      - name: Download blob reports from GitHub Actions Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: .vitest-reports
+          pattern: packages-blob-report-*
+          merge-multiple: true
+
+      - name: Merge reports
+        run: pnpm dlx vitest@3.1.4 run --merge-reports --pass-with-no-tests

.github/workflows/unit-tests-webapp.yml

@@ -0,0 +1,149 @@
+name: "🧪 Unit Tests: Webapp"
+
+permissions:
+  contents: read
+
+on:
+  workflow_call:
+
+jobs:
+  unitTests:
+    name: "🧪 Unit Tests: Webapp"
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        shardIndex: [1, 2, 3, 4, 5, 6, 7, 8]
+        shardTotal: [8]
+    env:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      SHARD_INDEX: ${{ matrix.shardIndex }}
+      SHARD_TOTAL: ${{ matrix.shardTotal }}
+    steps:
+      - name: 🔧 Disable IPv6
+        run: |
+          sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
+          sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1
+          sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=1
+
+      - name: 🔧 Configure docker address pool
+        run: |
+          CONFIG='{
+            "default-address-pools" : [
+              {
+                "base" : "172.17.0.0/12",
+                "size" : 20
+              },
+              {
+                "base" : "192.168.0.0/16",
+                "size" : 24
+              }
+            ]
+          }'
+          mkdir -p /etc/docker
+          echo "$CONFIG" | sudo tee /etc/docker/daemon.json
+
+      - name: 🔧 Restart docker daemon
+        run: sudo systemctl restart docker
+
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+          cache: "pnpm"
+
+      # ..to avoid rate limits when pulling images
+      - name: 🐳 Login to DockerHub
+        if: ${{ env.DOCKERHUB_USERNAME }}
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: 🐳 Skipping DockerHub login (no secrets available)
+        if: ${{ !env.DOCKERHUB_USERNAME }}
+        run: echo "DockerHub login skipped because secrets are not available."
+
+      - name: 🐳 Pre-pull testcontainer images
+        if: ${{ env.DOCKERHUB_USERNAME }}
+        run: |
+          echo "Pre-pulling Docker images with authenticated session..."
+          docker pull postgres:14
+          docker pull clickhouse/clickhouse-server:25.4-alpine
+          docker pull redis:7-alpine
+          docker pull testcontainers/ryuk:0.11.0
+          docker pull electricsql/electric:1.2.4
+          echo "Image pre-pull complete"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: 🧪 Run Webapp Unit Tests
+        run: pnpm run test:webapp --reporter=default --reporter=blob --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+        env:
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
+          DIRECT_URL: postgresql://postgres:postgres@localhost:5432/postgres
+          SESSION_SECRET: "secret"
+          MAGIC_LINK_SECRET: "secret"
+          ENCRYPTION_KEY: "dummy-encryption-keeeey-32-bytes"
+          DEPLOY_REGISTRY_HOST: "docker.io"
+          CLICKHOUSE_URL: "http://default:password@localhost:8123"
+
+      - name: Gather all reports
+        if: ${{ !cancelled() }}
+        run: |
+          mkdir -p .vitest-reports
+          find . -type f -path '*/.vitest-reports/blob-*.json' \
+            -exec bash -c 'src="$1"; basename=$(basename "$src"); pkg=$(dirname "$src" | sed "s|^\./||;s|/\.vitest-reports$||;s|/|_|g"); cp "$src" ".vitest-reports/${pkg}-${basename}"' _ {} \;
+
+      - name: Upload blob reports to GitHub Actions Artifacts
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: webapp-blob-report-${{ matrix.shardIndex }}
+          path: .vitest-reports/*
+          include-hidden-files: true
+          retention-days: 1
+
+  merge-reports:
+    name: "📊 Merge Reports"
+    if: ${{ !cancelled() }}
+    needs: [unitTests]
+    runs-on: ubuntu-latest
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+          # no cache enabled, we're not installing deps
+
+      - name: Download blob reports from GitHub Actions Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: .vitest-reports
+          pattern: webapp-blob-report-*
+          merge-multiple: true
+
+      - name: Merge reports
+        run: pnpm dlx vitest@3.1.4 run --merge-reports --pass-with-no-tests

.github/workflows/unit-tests.yml

@@ -1,40 +1,18 @@
 name: "🧪 Unit Tests"
+
+permissions:
+  contents: read
+
 on:
   workflow_call:
+
 jobs:
-  unitTests:
-    name: "🧪 Unit Tests"
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    steps:
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
-        with:
-          version: 7.18
-
-      - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
-        with:
-          node-version: 18
-          cache: "pnpm"
-
-      - name: ⎔ Setup Deno
-        uses: denoland/setup-deno@v1
-        with:
-          deno-version: v1.x
-
-      - name: ⎔ Setup bun
-        uses: oven-sh/setup-bun@v1
-        with:
-          bun-version: "1.0.15"
-
-      - name: 📥 Download deps
-        run: pnpm install --frozen-lockfile
-
-      - name: Run Unit Tests
-        run: |
-          pnpm run test
+  webapp:
+    uses: ./.github/workflows/unit-tests-webapp.yml
+    secrets: inherit
+  packages:
+    uses: ./.github/workflows/unit-tests-packages.yml
+    secrets: inherit
+  internal:
+    uses: ./.github/workflows/unit-tests-internal.yml
+    secrets: inherit

.gitignore

@@ -12,10 +12,12 @@ coverage
 # next.js
 .next/
 out/
-build
 dist
 packages/**/dist
 
+# vendored bundles (generated during build)
+packages/**/src/**/vendor
+
 # Tailwind
 apps/**/styles/tailwind.css
 packages/**/styles/tailwind.css
@@ -30,12 +32,10 @@ yarn-debug.log*
 yarn-error.log*
 
 # local env files
-.env.docker
+.env
+.env.*
 .docker/*.env
-.env.local
-.env.development.local
-.env.test.local
-.env.production.local
+!.env.example
 
 # turbo
 .turbo
@@ -53,4 +53,18 @@ apps/**/public/build
 /playwright-report/
 /playwright/.cache/
 
-.cosine
+.cosine
+.trigger
+.tshy*
+.yarn
+*.tsbuildinfo
+/packages/cli-v3/src/package.json
+.husky
+/packages/react-hooks/src/package.json
+/packages/core/src/package.json
+/packages/trigger-sdk/src/package.json
+/packages/python/src/package.json
+**/.claude/settings.local.json
+.mcp.log
+.mcp.json
+.cursor/debug.log

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "internal-packages/otlp-importer/protos"]
+	path = internal-packages/otlp-importer/protos
+	url = https://github.com/open-telemetry/opentelemetry-proto.git

.npmrc

@@ -1,2 +0,0 @@
-link-workspace-packages=false
-public-hoist-pattern[]=*prisma*

.nvmrc

@@ -1 +1 @@
-v18.18.0
+v20.20.0

.prettierignore

@@ -7,5 +7,4 @@ tailwind.css
 **/.react-email/
 **/storybook-static/
 **/.changeset/
-**/build/
 **/dist/

.vscode/extensions.json

@@ -1,9 +1,4 @@
 {
-	"recommendations": [
-        "astro-build.astro-vscode",
-        "denoland.vscode-deno"
-    ],
-	"unwantedRecommendations": [
-		
-	]
-}
+  "recommendations": ["bierner.comment-tagged-templates"],
+  "unwantedRecommendations": []
+}

.vscode/launch.json

@@ -9,10 +9,28 @@
       "request": "launch",
       "name": "Debug WebApp",
       "command": "pnpm run dev --filter webapp",
-      "envFile": "${workspaceFolder}/apps/webapp/.env",
+      "envFile": "${workspaceFolder}/.env",
       "cwd": "${workspaceFolder}",
       "sourceMaps": true
     },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug realtimeStreams.test.ts",
+      "command": "pnpm run test -t RealtimeStreams",
+      "envFile": "${workspaceFolder}/.env",
+      "cwd": "${workspaceFolder}/apps/webapp",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug triggerTask.test.ts",
+      "command": "pnpm run test --run ./test/engine/triggerTask.test.ts",
+      "envFile": "${workspaceFolder}/.env",
+      "cwd": "${workspaceFolder}/apps/webapp",
+      "sourceMaps": true
+    },
     {
       "type": "chrome",
       "request": "launch",
@@ -23,10 +41,121 @@
     {
       "type": "node-terminal",
       "request": "launch",
-      "name": "Debug BYO Auth",
-      "command": "pnpm run byo-auth",
-      "envFile": "${workspaceFolder}/references/job-catalog/.env",
-      "cwd": "${workspaceFolder}/references/job-catalog",
+      "name": "Debug V3 init CLI",
+      "command": "pnpm exec trigger init",
+      "cwd": "${workspaceFolder}/references/init-shell",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 init dev CLI",
+      "command": "pnpm exec trigger dev",
+      "cwd": "${workspaceFolder}/references/init-shell",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 Dev CLI",
+      "command": "pnpm exec trigger dev",
+      "cwd": "${workspaceFolder}/references/hello-world",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug Dev Next.js Realtime",
+      "command": "pnpm exec trigger dev",
+      "cwd": "${workspaceFolder}/references/nextjs-realtime",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug prisma-catalog deploy CLI",
+      "command": "pnpm exec trigger deploy --self-hosted --load-image",
+      "cwd": "${workspaceFolder}/references/prisma-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 Deploy CLI",
+      "command": "pnpm exec trigger deploy --self-hosted --load-image",
+      "cwd": "${workspaceFolder}/references/hello-world",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 list-profiles CLI",
+      "command": "pnpm exec trigger list-profiles --log-level debug",
+      "cwd": "${workspaceFolder}/references/hello-world",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 update CLI",
+      "command": "pnpm exec trigger update",
+      "cwd": "${workspaceFolder}/references/hello-world",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 Management",
+      "command": "pnpm run management",
+      "cwd": "${workspaceFolder}/references/hello-world",
+      "sourceMaps": true
+    },
+    {
+      "type": "node",
+      "request": "attach",
+      "name": "Attach to Trigger.dev CLI (v3)",
+      "port": 9229,
+      "restart": true,
+      "skipFiles": ["<node_internals>/**"]
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug CLI e2e tests",
+      "command": "MOD=otel-telemetry-loader pnpm run test:e2e",
+      "cwd": "${workspaceFolder}/packages/cli-v3",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "debug v3 hello-world dev",
+      "command": "pnpm exec trigger dev",
+      "cwd": "${workspaceFolder}/references/hello-world",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug RunEngine tests",
+      "command": "pnpm run test ./src/engine/tests/releaseConcurrencyTokenBucketQueue.test.ts -t 'Should retrieve metrics for all queues via getQueueMetrics'",
+      "cwd": "${workspaceFolder}/internal-packages/run-engine",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug RunQueue tests",
+      "command": "pnpm run test ./src/run-queue/index.test.ts --run",
+      "cwd": "${workspaceFolder}/internal-packages/run-engine",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug d3-demo",
+      "command": "pnpm exec trigger dev",
+      "cwd": "${workspaceFolder}/references/d3-demo",
       "sourceMaps": true
     }
   ]

.vscode/settings.json

@@ -1,4 +1,11 @@
 {
   "deno.enablePaths": ["references/deno-reference", "runtime_tests/tests/deno"],
-  "debug.toolBarLocation": "commandCenter"
+  "debug.toolBarLocation": "commandCenter",
+  "typescript.tsdk": "node_modules/typescript/lib",
+  "search.exclude": {
+    "**/node_modules/**": true,
+    "packages/cli-v3/e2e": true
+  },
+  "vitest.disableWorkspaceWarning": true,
+  "typescript.experimental.useTsgo": false
 }

.zed/tasks.json

@@ -0,0 +1,45 @@
+[
+  {
+    "label": "Build packages",
+    "command": "pnpm run build --filter \"@trigger.dev/*\" --filter trigger.dev",
+    //"args": [],
+    // Env overrides for the command, will be appended to the terminal's environment from the settings.
+    "env": { "foo": "bar" },
+    // Current working directory to spawn the command into, defaults to current project root.
+    //"cwd": "/path/to/working/directory",
+    // Whether to use a new terminal tab or reuse the existing one to spawn the process, defaults to `false`.
+    "use_new_terminal": false,
+    // Whether to allow multiple instances of the same task to be run, or rather wait for the existing ones to finish, defaults to `false`.
+    "allow_concurrent_runs": false,
+    // What to do with the terminal pane and tab, after the command was started:
+    // * `always` — always show the task's pane, and focus the corresponding tab in it (default)
+    // * `no_focus` — always show the task's pane, add the task's tab in it, but don't focus it
+    // * `never` — do not alter focus, but still add/reuse the task's tab in its pane
+    "reveal": "always",
+    // What to do with the terminal pane and tab, after the command has finished:
+    // * `never` — Do nothing when the command finishes (default)
+    // * `always` — always hide the terminal tab, hide the pane also if it was the last tab in it
+    // * `on_success` — hide the terminal tab on task success only, otherwise behaves similar to `always`
+    "hide": "never",
+    // Which shell to use when running a task inside the terminal.
+    // May take 3 values:
+    // 1. (default) Use the system's default terminal configuration in /etc/passwd
+    //      "shell": "system"
+    // 2. A program:
+    //      "shell": {
+    //        "program": "sh"
+    //      }
+    // 3. A program with arguments:
+    //     "shell": {
+    //         "with_arguments": {
+    //           "program": "/bin/bash",
+    //           "args": ["--login"]
+    //         }
+    //     }
+    "shell": "system",
+    // Whether to show the task line in the output of the spawned task, defaults to `true`.
+    "show_summary": true,
+    // Whether to show the command line in the output of the spawned task, defaults to `true`.
+    "show_output": true
+  }
+]

AGENTS.md

@@ -0,0 +1,68 @@
+# Guidance for Coding Agents
+
+This repository is a pnpm monorepo managed with Turbo. It contains multiple apps and packages that make up the Trigger.dev platform and SDK.
+
+## Repository layout
+- `apps/webapp` – Remix application that serves as the main API and dashboard.
+- `apps/supervisor` – Node application for executing built tasks.
+- `packages/*` – Published packages such as `@trigger.dev/sdk`, the CLI (`trigger.dev`), and shared libraries.
+- `internal-packages/*` – Internal-only packages used by the webapp and other apps.
+- `references/*` – Example projects for manual testing and development of new features.
+- `ai/references` – Contains additional documentation including an overview (`repo.md`) and testing guidelines (`tests.md`).
+
+See `ai/references/repo.md` for a more complete explanation of the workspaces.
+
+## Development setup
+1. Install dependencies with `pnpm i` (pnpm `10.23.0` and Node.js `20.20.0` are required).
+2. Copy `.env.example` to `.env` and generate a random 16 byte hex string for `ENCRYPTION_KEY` (`openssl rand -hex 16`). Update other secrets if needed.
+3. Start the local services with Docker:
+   ```bash
+   pnpm run docker
+   ```
+4. Run database migrations:
+   ```bash
+   pnpm run db:migrate
+   ```
+5. Build the webapp, CLI and SDK packages:
+   ```bash
+   pnpm run build --filter webapp && pnpm run build --filter trigger.dev && pnpm run build --filter @trigger.dev/sdk
+   ```
+6. Launch the development server:
+   ```bash
+   pnpm run dev --filter webapp
+   ```
+   The webapp runs on <http://localhost:3030>.
+
+For full setup instructions see `CONTRIBUTING.md`.
+
+## Running tests
+- Unit tests use **vitest**. Run all tests:
+  ```bash
+  pnpm run test
+  ```
+- Run tests for a specific workspace (example for `webapp`):
+  ```bash
+  pnpm run test --filter webapp
+  ```
+- Prefer running a single test file from within its directory:
+  ```bash
+  cd apps/webapp
+  pnpm run test ./src/components/Button.test.ts
+  ```
+  If packages in that workspace need to be built first, run `pnpm run build --filter webapp`.
+
+Refer to `ai/references/tests.md` for details on writing tests. Tests should avoid mocks or stubs and use the helpers from `@internal/testcontainers` when Redis or Postgres are needed.
+
+## Coding style
+- Formatting is enforced using Prettier. Run `pnpm run format` before committing.
+- Follow the existing project conventions. Test files live beside the files under test and use descriptive `describe` and `it` blocks.
+- Do not commit directly to the `main` branch. All changes should be made in a separate branch and go through a pull request.
+
+## Additional docs
+- The root `README.md` describes Trigger.dev and links to documentation.
+- The `docs` workspace contains our documentation site, which can be run locally with:
+  ```bash
+  pnpm run dev --filter docs
+  ```
+- `references/README.md` explains how to create new reference projects for manual testing.
+

CHANGESETS.md

@@ -30,28 +30,16 @@ Please follow the best-practice of adding changesets in the same commit as the c
 
 ## Snapshot instructions
 
-!MAKE SURE TO UPDATE THE TAG IN THE INSTRUCTIONS BELOW!
+1. Update the `.changeset/config.json` file to set the `"changelog"` field to this:
 
-1. Add changesets as usual
-
-```sh
-pnpm run changeset:add
+```json
+"changelog": "@changesets/cli/changelog",
 ```
 
-2. Create a snapshot version (replace "prerelease" with your tag)
+2. Do a temporary commit (do NOT push this, you should undo it after)
 
-```sh
-pnpm exec changeset version --snapshot prerelease
-```
+3. Run `./scripts/publish-prerelease.sh prerelease`
 
-3. Build the packages:
+You can choose a different tag if you want, but usually `prerelease` is fine.
 
-```sh
-pnpm run build --filter "@trigger.dev/*"
-```
-
-4. Publish the snapshot (replace "dev" with your tag)
-
-```sh
-pnpm exec changeset publish --no-git-tag --snapshot --tag prerelease
-```
+5. Undo the commit where you updated the config.json file.

CLAUDE.md

@@ -0,0 +1,303 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Build and Development Commands
+
+This is a pnpm 10.23.0 monorepo using Turborepo. Run commands from root with `pnpm run`.
+
+### Essential Commands
+
+```bash
+# Start Docker services (PostgreSQL, Redis, Electric)
+pnpm run docker
+
+# Run database migrations
+pnpm run db:migrate
+
+# Seed the database (required for reference projects)
+pnpm run db:seed
+
+# Build packages (required before running)
+pnpm run build --filter webapp && pnpm run build --filter trigger.dev && pnpm run build --filter @trigger.dev/sdk
+
+# Run webapp in development mode (http://localhost:3030)
+pnpm run dev --filter webapp
+
+# Build and watch for changes (CLI and packages)
+pnpm run dev --filter trigger.dev --filter "@trigger.dev/*"
+```
+
+### Testing
+
+We use vitest exclusively. **Never mock anything** - use testcontainers instead.
+
+```bash
+# Run all tests for a package
+pnpm run test --filter webapp
+
+# Run a single test file (preferred - cd into directory first)
+cd internal-packages/run-engine
+pnpm run test ./src/engine/tests/ttl.test.ts --run
+
+# May need to build dependencies first
+pnpm run build --filter @internal/run-engine
+```
+
+Test files go next to source files (e.g., `MyService.ts` → `MyService.test.ts`).
+
+#### Testcontainers for Redis/PostgreSQL
+
+```typescript
+import { redisTest, postgresTest, containerTest } from "@internal/testcontainers";
+
+// Redis only
+redisTest("should use redis", async ({ redisOptions }) => {
+  /* ... */
+});
+
+// PostgreSQL only
+postgresTest("should use postgres", async ({ prisma }) => {
+  /* ... */
+});
+
+// Both Redis and PostgreSQL
+containerTest("should use both", async ({ prisma, redisOptions }) => {
+  /* ... */
+});
+```
+
+### Changesets
+
+When modifying any public package (`packages/*` or `integrations/*`), add a changeset:
+
+```bash
+pnpm run changeset:add
+```
+
+- Default to **patch** for bug fixes and minor changes
+- Confirm with maintainers before selecting **minor** (new features)
+- **Never** select major (breaking changes) without explicit approval
+
+## Architecture Overview
+
+### Apps
+
+- **apps/webapp**: Remix 2.1.0 app - main API, dashboard, and Docker image. Uses Express server.
+- **apps/supervisor**: Node.js app handling task execution, interfacing with Docker/Kubernetes.
+
+### Public Packages
+
+- **packages/trigger-sdk** (`@trigger.dev/sdk`): Main SDK
+- **packages/cli-v3** (`trigger.dev`): CLI package
+- **packages/core** (`@trigger.dev/core`): Shared code between SDK and webapp. Import subpaths only (never root).
+- **packages/build**: Build extensions and types
+- **packages/react-hooks**: React hooks for realtime and triggering
+- **packages/redis-worker** (`@trigger.dev/redis-worker`): Custom Redis-based background job system
+
+### Internal Packages
+
+- **internal-packages/database** (`@trigger.dev/database`): Prisma 6.14.0 client and schema
+- **internal-packages/clickhouse** (`@internal/clickhouse`): ClickHouse client and schema migrations
+- **internal-packages/run-engine** (`@internal/run-engine`): "Run Engine 2.0" - run lifecycle management
+- **internal-packages/redis** (`@internal/redis`): Redis client creation utilities
+- **internal-packages/testcontainers** (`@internal/testcontainers`): Test helpers for Redis/PostgreSQL containers
+- **internal-packages/zodworker** (`@internal/zodworker`): Graphile-worker wrapper (being replaced by redis-worker)
+
+### Reference Projects
+
+The `references/` directory contains test workspaces for developing and testing new SDK and platform features. Use these projects (e.g., `references/hello-world`) to manually test changes to the CLI, SDK, core packages, and webapp before submitting PRs.
+
+## Webapp Development
+
+### Key Locations
+
+- Trigger API: `apps/webapp/app/routes/api.v1.tasks.$taskId.trigger.ts`
+- Batch trigger: `apps/webapp/app/routes/api.v1.tasks.batch.ts`
+- Prisma setup: `apps/webapp/app/db.server.ts`
+- Run engine config: `apps/webapp/app/v3/runEngine.server.ts`
+- Services: `apps/webapp/app/v3/services/**/*.server.ts`
+- Presenters: `apps/webapp/app/v3/presenters/**/*.server.ts`
+- OTEL endpoints: `apps/webapp/app/routes/otel.v1.logs.ts`, `otel.v1.traces.ts`
+
+### Environment Variables
+
+Access via `env` export from `apps/webapp/app/env.server.ts`, never `process.env` directly.
+
+For testable code, **never import env.server.ts** in test files. Pass configuration as options instead. Example pattern:
+
+- `realtimeClient.server.ts` (testable service)
+- `realtimeClientGlobal.server.ts` (configuration)
+
+### Legacy vs Run Engine 2.0
+
+The codebase is transitioning from the "legacy run engine" (spread across codebase) to "Run Engine 2.0" (`@internal/run-engine`). Focus on Run Engine 2.0 for new work.
+
+## Docker Image Guidelines
+
+When updating Docker image references in `docker/Dockerfile` or other container files:
+
+- **Always use multiplatform/index digests**, not architecture-specific digests
+- Architecture-specific digests (e.g., for `linux/amd64` only) will cause CI failures on different build environments
+- On Docker Hub, the multiplatform digest is shown on the main image page, while architecture-specific digests are listed under "OS/ARCH"
+- Example: Use `node:20.20-bullseye-slim@sha256:abc123...` where the digest is from the multiplatform index, not from a specific OS/ARCH variant
+
+## Database Migrations (PostgreSQL)
+
+1. Edit `internal-packages/database/prisma/schema.prisma`
+2. Create migration:
+   ```bash
+   cd internal-packages/database
+   pnpm run db:migrate:dev:create --name "add_new_column"
+   ```
+3. **Important**: Generated migration includes extraneous changes. Remove lines related to:
+   - `_BackgroundWorkerToBackgroundWorkerFile`
+   - `_BackgroundWorkerToTaskQueue`
+   - `_TaskRunToTaskRunTag`
+   - `_WaitpointRunConnections`
+   - `_completedWaitpoints`
+   - `SecretStore_key_idx`
+   - Various `TaskRun` indexes unless you added them
+4. Apply migration:
+   ```bash
+   pnpm run db:migrate:deploy && pnpm run generate
+   ```
+
+### Index Migration Rules
+
+- Indexes **must use CONCURRENTLY** to avoid table locks
+- **CONCURRENTLY indexes must be in their own separate migration file** - they cannot be combined with other schema changes
+
+## ClickHouse Migrations
+
+ClickHouse migrations use Goose format and live in `internal-packages/clickhouse/schema/`.
+
+1. Create a new numbered SQL file (e.g., `010_add_new_column.sql`)
+2. Use Goose markers:
+
+   ```sql
+   -- +goose Up
+   ALTER TABLE trigger_dev.your_table
+   ADD COLUMN new_column String DEFAULT '';
+
+   -- +goose Down
+   ALTER TABLE trigger_dev.your_table
+   DROP COLUMN new_column;
+   ```
+
+Follow naming conventions in `internal-packages/clickhouse/README.md`:
+
+- `raw_` prefix for input tables
+- `_v1`, `_v2` suffixes for versioning
+- `_mv_v1` suffix for materialized views
+
+## Writing Trigger.dev Tasks
+
+Always import from `@trigger.dev/sdk`. Never use `@trigger.dev/sdk/v3` or deprecated `client.defineJob` pattern.
+
+```typescript
+import { task } from "@trigger.dev/sdk";
+
+// Every task must be exported
+export const myTask = task({
+  id: "my-task", // Unique ID
+  run: async (payload: { message: string }) => {
+    // Task logic - no timeouts
+  },
+});
+```
+
+### SDK Documentation Rules
+
+The `rules/` directory contains versioned documentation for writing Trigger.dev tasks, distributed to users via the SDK installer. Current version is defined in `rules/manifest.json`.
+
+- `rules/4.3.0/` - Latest: batch trigger v2 (1,000 items, 3MB payloads), debouncing
+- `rules/4.1.0/` - Realtime streams v2, updated config
+- `rules/4.0.0/` - Base v4 SDK documentation
+
+When adding new SDK features, create a new version directory with only the files that changed from the previous version. Update `manifest.json` to point unchanged files to previous versions.
+
+### Claude Code Skill
+
+The `.claude/skills/trigger-dev-tasks/` skill provides Claude Code with Trigger.dev task expertise. It includes:
+
+- `SKILL.md` - Core instructions and patterns
+- Reference files for basic tasks, advanced tasks, scheduled tasks, realtime, and config
+
+Keep the skill in sync with the latest rules version when SDK features change.
+
+## Testing with hello-world Reference Project
+
+First-time setup:
+
+1. Run `pnpm run db:seed` to seed the database (creates the hello-world project)
+2. Build CLI: `pnpm run build --filter trigger.dev && pnpm i`
+3. Authorize CLI: `cd references/hello-world && pnpm exec trigger login -a http://localhost:3030`
+
+Running:
+
+```bash
+cd references/hello-world
+pnpm exec trigger dev  # or with --log-level debug
+```
+
+## Local Task Testing Workflow
+
+This workflow enables Claude Code to run the webapp and trigger dev simultaneously, trigger tasks, and inspect results for testing code changes.
+
+### Step 1: Start Webapp in Background
+
+```bash
+# Run from repo root with run_in_background: true
+pnpm run dev --filter webapp
+```
+
+Verify webapp is running:
+
+```bash
+curl -s http://localhost:3030/healthcheck  # Should return 200
+```
+
+### Step 2: Start Trigger Dev in Background
+
+```bash
+# Run from hello-world directory with run_in_background: true
+cd references/hello-world && pnpm exec trigger dev
+```
+
+The worker will build and register tasks. Check output for "Local worker ready [node]" message.
+
+### Step 3: Trigger and Monitor Tasks via MCP
+
+Use the Trigger.dev MCP tools to interact with tasks:
+
+```
+# Get current worker and registered tasks
+mcp__trigger__get_current_worker(projectRef: "proj_rrkpdguyagvsoktglnod", environment: "dev")
+
+# Trigger a task
+mcp__trigger__trigger_task(
+  projectRef: "proj_rrkpdguyagvsoktglnod",
+  environment: "dev",
+  taskId: "hello-world",
+  payload: {"message": "Hello from Claude"}
+)
+
+# List runs to see status
+mcp__trigger__list_runs(
+  projectRef: "proj_rrkpdguyagvsoktglnod",
+  environment: "dev",
+  taskIdentifier: "hello-world",
+  limit: 5
+)
+```
+
+### Step 4: Monitor Execution
+
+- Check trigger dev output file for real-time execution logs
+- Successful runs show: `Task | Run ID | Success (Xms)`
+- Dashboard available at: http://localhost:3030/orgs/references-9dfd/projects/hello-world-97DT/env/dev/runs
+
+### Key Project Refs
+
+- hello-world: `proj_rrkpdguyagvsoktglnod`

CONTRIBUTING.md

@@ -10,13 +10,14 @@ Thank you for helping us make Trigger.dev even better! 🤩
 
 The development branch is `main`. This is the branch that all pull
 requests should be made against. The changes on the `main`
-branch are tagged into a release monthly.
+branch are tagged into a release periodically.
 
 ### Prerequisites
 
-- [Node.js](https://nodejs.org/en) version >=16.x
-- [pnpm package manager](https://pnpm.io/installation) version 7
+- [Node.js](https://nodejs.org/en) version 20.20.0
+- [pnpm package manager](https://pnpm.io/installation) version 10.23.0
 - [Docker](https://www.docker.com/get-started/)
+- [protobuf](https://github.com/protocolbuffers/protobuf)
 
 ### Setup
 
@@ -33,15 +34,19 @@ branch are tagged into a release monthly.
    ```
    cd trigger.dev
    ```
-3. Install the required packages using pnpm.
+3. Ensure you are on the correct version of Node.js (20.20.0). If you are using `nvm`, there is an `.nvmrc` file that will automatically select the correct version of Node.js when you navigate to the repository.
+
+4. Run `corepack enable` to use the correct version of pnpm (`10.23.0`) as specified in the root `package.json` file.
+
+5. Install the required packages using pnpm.
    ```
    pnpm i
    ```
-4. Create your `.env` file
+6. Create your `.env` file
    ```
    cp .env.example .env
    ```
-5. Open it and generate a new value for `ENCRYPTION_KEY`:
+7. Open it and generate a new value for `ENCRYPTION_KEY`:
 
    `ENCRYPTION_KEY` is used to two-way encrypt OAuth access tokens and so you'll probably want to actually generate a unique value, and it must be a random 16 byte hex string. You can generate one with the following command:
 
@@ -51,27 +56,21 @@ branch are tagged into a release monthly.
 
    Feel free to update `SESSION_SECRET` and `MAGIC_LINK_SECRET` as well using the same method.
 
-6. Start Docker. This starts the required services like Postgres. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)
+8. Start Docker. This starts the required services like Postgres & Redis. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)
 
    ```
    pnpm run docker
    ```
 
-   This will also start and run a local instance of [pgAdmin](https://www.pgadmin.org/) on [localhost:5480](http://localhost:5480), preconfigured with email `admin@example.com` and pwd `admin`. Then use `postgres` as the password to the Trigger.dev server.
-
-7. Migrate the database
+9. Migrate the database
    ```
    pnpm run db:migrate
    ```
-8. Build the app
-   ```
-   pnpm run build --filter webapp
-   ```
-9. Run the seed script
-   ```
-   pnpm run db:seed
-   ```
-10. Run the app. See the section below.
+10. Build everything
+    ```
+    pnpm run build --filter webapp && pnpm run build --filter trigger.dev && pnpm run build --filter @trigger.dev/sdk
+    ```
+11. Run the app. See the section below.
 
 ## Running
 
@@ -83,145 +82,113 @@ branch are tagged into a release monthly.
 
    It should run on port `3030`: [http://localhost:3030](http://localhost:3030/)
 
-2. Once the app is running click the magic link button and enter your email.
-3. Check your terminal, the magic link email should have printed out as following:
+2. Once the app is running click the magic link button and enter your email. You will automatically be logged in, since you are running locally. Create an Org and your first project in the dashboard.
 
-   ```sh
-   webapp:dev: Log in to Trigger.dev
-   webapp:dev:
-   webapp:dev: Click here to log in with this magic link
-   webapp:dev: [http://localhost:3030/magic?token=U2FsdGVkX18OvB0JxgaswTLCSbaRz%2FY82TN0EZWhSzFyZYwgG%2BIzKVTkeiaOtWfotPw7F8RwFzCHh53aBpMEu%2B%2B%2FItb%2FcJYh89MSjc3Pz92bevoEjqxSQ%2Ff%2BZbks09JOpqlBbYC3FzGWC8vuSVFBlxqLXxteSDLthZSUaC%2BS2LaA%2BJgp%2BLO7hgjAaC2lXbCHrM7MTgTdXOFt7i0Dvvuwz6%2BWY25RnfomZOPqDsyH0xz8Q2rzPTz0Xu53WSXrZ1hd]
-   webapp:dev:
-   webapp:dev: If you didn't try to log in, you can safely ignore this email.
-   ```
+## Manual testing using hello-world
 
-   Paste the magic link shown in your terminal into your browser to login.
+We use the `<root>/references/hello-world` subdirectory as a staging ground for testing changes to the SDK (`@trigger.dev/sdk` at `<root>/packages/trigger-sdk`), the Core package (`@trigger.dev/core` at `<root>packages/core`), the CLI (`trigger.dev` at `<root>/packages/cli-v3`) and the platform (The remix app at `<root>/apps/webapp`). The instructions below will get you started on using the `hello-world` for local development of Trigger.dev.
+
+### First-time setup
+
+First, make sure you are running the webapp according to the instructions above. Then:
+
+1. Visit http://localhost:3030 in your browser and create a new project called "hello-world".
+
+2. In Postgres go to the "Projects" table and for the project you create change the `externalRef` to `proj_rrkpdguyagvsoktglnod`.
+
+3. Build the CLI
+
+```sh
+# Build the CLI
+pnpm run build --filter trigger.dev
+# Make it accessible to `pnpm exec`
+pnpm i
+```
+
+4. Change into the `<root>/references/hello-world` directory and authorize the CLI to the local server:
+
+```sh
+cd references/hello-world
+cp .env.example .env
+pnpm exec trigger login -a http://localhost:3030
+```
+
+This will open a new browser window and authorize the CLI against your local user account.
+
+You can optionally pass a `--profile` flag to the `login` command, which will allow you to use the CLI with separate accounts/servers. We suggest using a profile called `local` for your local development:
+
+```sh
+cd references/hello-world
+pnpm exec trigger login -a http://localhost:3030 --profile local
+# later when you run the dev or deploy command:
+pnpm exec trigger dev --profile local
+pnpm exec trigger deploy --profile local
+```
+
+### Running
+
+The following steps should be followed any time you start working on a new feature you want to test:
+
+1. Make sure the webapp is running on localhost:3030
+
+2. Open a terminal window and build the CLI and packages and watch for changes
+
+```sh
+pnpm run dev --filter trigger.dev --filter "@trigger.dev/*"
+```
+
+3. Open another terminal window, and change into the `<root>/references/hello-world` directory.
+
+4. Run the `dev` command, which will register all the local tasks with the platform and allow you to start testing task execution:
+
+```sh
+# in <root>/references/hello-world
+pnpm exec trigger dev
+```
+
+If you want additional debug logging, you can use the `--log-level debug` flag:
+
+```sh
+# in <root>/references/hello-world
+pnpm exec trigger dev --log-level debug
+```
+
+6. If you make any changes in the CLI/Core/SDK, you'll need to `CTRL+C` to exit the `dev` command and restart it to pickup changes. Any changes to the files inside of the `hello-world/src/trigger` dir will automatically be rebuilt by the `dev` command.
+
+7. Navigate to the `hello-world` project in your local dashboard at localhost:3030 and you should see the list of tasks.
+
+8. Go to the "Test" page in the sidebar and select a task. Then enter a payload and click "Run test". You can tell what the payloads should be by looking at the relevant task file inside the `/references/hello-world/src/trigger` folder. Many of them accept an empty payload.
+
+9. Feel free to add additional files in `hello-world/src/trigger` to test out specific aspects of the system, or add in edge cases.
 
 ## Adding and running migrations
 
-1. Modify packages/database/prisma/schema.prisma file
+1. Modify internal-packages/database/prisma/schema.prisma file
 2. Change directory to the packages/database folder
+
    ```sh
    cd packages/database
    ```
-3. Generate the Prisma client
 
-   ```sh
-   pnpm run generate
-   ```
-
-   The above updates the prisma client generated into node_modules/.prisma/client folder. This helps with typing of relevant prisma models. It ensures typescript
-   recognizes fields added or removed from a model and type-checks appropriately.
-
-4. Create and apply the migrations
+3. Create a migration
 
    ```
-   pnpm run db:migrate:dev
+   pnpm run db:migrate:dev:create
    ```
 
-   This creates a migration file and executes the migrations against your database and applies changes to the database schema(s)
+   This creates a migration file. Check the migration file does only what you want. If you're adding any database indexes they must use `CONCURRENTLY`, otherwise they'll lock the table when executed.
 
-5. Commit generated migrations as well as changes to the schema.prisma file
-6. If you're using VSCode you may need to restart the Typescript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`.
+4. Run the migration.
 
-## Testing CLI changes
-
-To test CLI changes, follow the steps below:
-
-1. Build the CLI and watch for changes
-
-```sh
-cd packages/cli
-pnpm run dev
+```
+pnpm run db:migrate:deploy
+pnpm run generate
 ```
 
-2. Open a new Terminal window and run the webapp locally and then create a new project in the dashboard. Copy out the dev API key.
+This executes the migrations against your database and applies changes to the database schema(s), and then regenerates the Prisma client.
 
-3. Create a new temporary Next.js app in references directory
-
-```sh
-cd ./references
-pnpm create next-app@latest test-cli --ts --no-eslint --tailwind --app --src-dir --import-alias "@/*"
-```
-
-4. Then once that's finished, add the `@trigger.dev/cli` to the `devDependencies` of the newly created Next.js app's `package.json` file, like so:
-
-```json
-{
-  // other package.json properties
-  "devDependencies": { "@trigger.dev/cli": "workspace:*" }
-}
-```
-
-5. Back in the terminal, navigate into the reference, and initialize the CLI. When prompted, select `self-hosted` and enter `localhost:3030` if you are testing against the local instance of Trigger.dev, or you can just use the Trigger.dev cloud. When asked for an API key, use the key you copied earlier.
-
-```sh
-cd ./test-cli
-pnpm i
-pnpm exec trigger-cli init
-```
-
-6. If you are just testing the `init` command, you can stop here. If you'd like to test the `dev` command, first start the Next.js app on port 3000:
-
-```sh
-pnpm run dev
-```
-
-7. Open a new terminal window, and then run the `dev` command like so:
-
-```sh
-pnpm exec trigger-cli dev
-```
-
-8. Please remember to delete the temporary project you created after you've tested the changes, and before you raise a PR.
-
-## Running end-to-end webapp tests
-
-To run the end-to-end tests, follow the steps below:
-
-1. Set up environment variables (copy example envs into the correct place)
-
-```sh
-cp ./.env.example ./.env
-cp ./references/nextjs-test/.env.example ./references/nextjs-test/.env.local
-```
-
-2. Set up dependencies
-
-```sh
-# Build packages
-pnpm run build --filter @references/nextjs-test^...
-pnpm --filter @trigger.dev/database generate
-
-# Move trigger-cli bin to correct place
-pnpm install --frozen-lockfile
-
-# Install playwrite browsers (ONE TIME ONLY)
-npx playwright install
-```
-
-3. Set up the database
-
-```sh
-pnpm run docker
-pnpm run db:migrate
-pnpm run db:seed
-```
-
-4. Run the end-to-end tests
-
-```sh
-pnpm run test:e2e
-```
-
-### Cleanup
-
-The end-to-end tests use a `setup` and `teardown` script to seed the database with test data. If the test runner doesn't exit cleanly, then the database can be left in a state where the tests can't run because the `setup` script will try to create data that already exists. If this happens, you can manually delete the `users` and `organizations` from the database using prisma studio:
-
-```sh
-# With the database running (i.e. pnpm run docker)
-pnpm run db:studio
-```
+4. Commit generated migrations as well as changes to the schema.prisma file
+5. If you're using VSCode you may need to restart the Typescript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`.
 
 ## Add sample jobs
 

DOCKER_INSTALLATION.md

@@ -8,47 +8,56 @@ If you don't have Docker installed on your machine, you'll run into some complic
 
 Below are the steps on how you can avoid that.
 
-First you need to setup docker-compose as it is an underlying tool that this command: `pnpm run docker` fires behind the scene.
+First you need to setup docker compose as it is an underlying tool that this command: `pnpm run docker` fires behind the scene.
 
 ## Linux
 
-To install Docker Compose on Linux Ubuntu via the terminal, you can follow these steps:
+To install Docker Compose on Linux Ubuntu, you can follow these steps:
 
-1. Update the package index on your system by running the following command:
+1. Create the Docker config directory and cli-plugins subdirectory:
 
    ```shell
-   sudo apt update
+   DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker}
+   mkdir -p $DOCKER_CONFIG/cli-plugins
    ```
 
-2. Install the required dependencies by running the following command:
+2. Download the Docker Compose plugin:
 
    ```shell
-   sudo apt install curl
+   curl -SL "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o $DOCKER_CONFIG/cli-plugins/docker-compose
    ```
 
-3. Download the Docker Compose binary into the `/usr/local/bin` directory using the `curl` command:
+   Note:
+
+   - To install for all users, replace `$DOCKER_CONFIG/cli-plugins` with `/usr/local/lib/docker/cli-plugins`
+
+3. Set the appropriate permissions to make the Docker Compose plugin executable:
 
    ```shell
-   sudo curl -L "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+   chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose
    ```
 
-4. Set the appropriate permissions to make the `docker-compose` binary executable:
+   If you installed for all users:
 
    ```shell
-   sudo chmod +x /usr/local/bin/docker-compose
+   sudo chmod +x /usr/local/lib/docker/cli-plugins/docker-compose
    ```
 
-5. Verify that Docker Compose has been successfully installed by running the following command:
+4. Verify that Docker Compose has been successfully installed:
 
    ```shell
-   docker-compose --version
+   docker compose version
    ```
 
-   This command should display the version information of Docker Compose without any errors.
+   You should see output similar to:
 
-After following these steps, you should have Docker Compose installed on your Ubuntu system, and you can use it by running `docker-compose` commands in the terminal.
+   ```
+   Docker Compose version vX.Y.Z
+   ```
 
-When you've verified that the `docker-compose` package is installed and you proceed to start Docker with `pnpm run docker`.
+After following these steps, you should have Docker Compose installed on your Ubuntu system, and you can use it by running `docker compose` commands in the terminal.
+
+When you've verified that the `docker compose` package is installed and you proceed to start Docker with `pnpm run docker`.
 
 You'll probably get an error similar to the one below:
 

README.md

@@ -1,90 +1,129 @@
 <div align="center">
-<picture>
-  <source media="(prefers-color-scheme: dark)" srcset="https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/a45d1fa2-0ae8-4a39-4409-f4f934bfae00/public">
-  <source media="(prefers-color-scheme: light)" srcset="https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/3f5ad4c1-c4c8-4277-b622-290e7f37bd00/public">
-  <img alt="Trigger.dev logo" src="https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/a45d1fa2-0ae8-4a39-4409-f4f934bfae00/public">
-</picture>
-  
-### The open source background jobs framework
 
-[Discord](https://discord.gg/JtBAxBr2m3) | [Website](https://trigger.dev) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Docs](https://trigger.dev/docs)
+![Trigger.dev logo](https://content.trigger.dev/github-header-banner.jpg)
 
-[![Twitter](https://img.shields.io/twitter/url/https/twitter.com/triggerdotdev.svg?style=social&label=Follow%20%40trigger.dev)](https://twitter.com/triggerdotdev)
-[![GitHub Repo stars](https://img.shields.io/github/stars/triggerdotdev/trigger.dev?style=social)](https://github.com/triggerdotdev/trigger.dev)
+### Build and deploy fully‑managed AI agents and workflows
+
+[Website](https://trigger.dev) | [Docs](https://trigger.dev/docs) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Example projects](https://github.com/triggerdotdev/examples) | [Feature requests](https://triggerdev.featurebase.app/) | [Public roadmap](https://triggerdev.featurebase.app/roadmap) | [Self-hosting](https://trigger.dev/docs/self-hosting/overview) 
+
+[![Open Source](https://img.shields.io/badge/Open%20Source-%E2%9D%A4-red.svg)](https://github.com/triggerdotdev/trigger.dev)
+[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/triggerdotdev/trigger.dev/blob/main/LICENSE)
+[![npm](https://img.shields.io/npm/v/@trigger.dev/sdk.svg?label=npm)](https://www.npmjs.com/package/@trigger.dev/sdk)
+[![SDK downloads](https://img.shields.io/npm/dm/@trigger.dev/sdk.svg?label=SDK%20downloads)](https://www.npmjs.com/package/@trigger.dev/sdk)
+
+[![Twitter Follow](https://img.shields.io/twitter/follow/triggerdotdev?style=social)](https://twitter.com/triggerdotdev)
+[![Discord](https://img.shields.io/discord/1066956501299777596?logo=discord&logoColor=white&color=7289da)](https://discord.gg/nkqV9xBYWy)
+[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/triggerdotdev/trigger.dev)
+[![GitHub stars](https://img.shields.io/github/stars/triggerdotdev/trigger.dev?style=social)](https://github.com/triggerdotdev/trigger.dev)
 
 </div>
 
-# About Trigger.dev
+## About Trigger.dev
 
-Create long-running jobs directly in your codebase with features like API integrations, webhooks, scheduling and delays.
+Trigger.dev is the open-source platform for building AI workflows in TypeScript. Long-running tasks with retries, queues, observability, and elastic scaling.
 
-## Long running Jobs on serverless
+## The platform designed for building AI agents
 
-Reliably run jobs and don’t worry about function timeouts, we handle those for you.
+Build [AI agents](https://trigger.dev/product/ai-agents) using all the frameworks, services and LLMs you're used to, deploy them to Trigger.dev and get durable, long-running tasks with retries, queues, observability, and elastic scaling out of the box.
 
-- Auto-resume after a function timeout
-- Auto-resume after a server outage
-- Add delays of up to a year
+- **Long-running without timeouts**: Execute your tasks with absolutely no timeouts, unlike AWS Lambda, Vercel, and other serverless platforms.
 
-## In your codebase
+- **Durability, retries & queues**: Build rock solid agents and AI applications using our durable tasks, retries, queues and idempotency.
 
-Create Jobs where they belong: in your codebase. Version control, localhost, test, review, and deploy like you're already used to.
+- **True runtime freedom**: Customize your deployed tasks with system packages – run browsers, Python scripts, FFmpeg and more.
 
-## Secure by design
+- **Human-in-the-loop**: Programmatically pause your tasks until a human can approve, reject or give feedback.
 
-We only receive Triggers and the data you choose to send to us. You can even completely self-host the entire platform.
+- **Realtime apps & streaming**: Move your background jobs to the foreground by subscribing to runs or streaming AI responses to your app.
 
-## Don't worry about deployment
+- **Observability & monitoring**: Each run has full tracing and logs. Configure error alerts to catch bugs fast.
 
-Just use our SDK to write Jobs in your codebase. There's nothing extra to deploy and no CI to configure, your Jobs just connect to our cloud. Or you can always self-host.
+## Key features:
+
+- **[JavaScript and TypeScript SDK](https://trigger.dev/docs/tasks/overview)** - Build background tasks using familiar programming models
+- **[Long-running tasks](https://trigger.dev/docs/runs/max-duration)** - Handle resource-heavy tasks without timeouts
+- **[Durable cron schedules](https://trigger.dev/docs/tasks/scheduled#scheduled-tasks-cron)** - Create and attach recurring schedules of up to a year
+- **[Trigger.dev Realtime](https://trigger.dev/docs/realtime/overview)** - Trigger, subscribe to, and get real-time updates for runs, with LLM streaming support
+- **[Build extensions](https://trigger.dev/docs/config/extensions/overview#build-extensions)** - Hook directly into the build system and customize the build process. Run Python scripts, FFmpeg, browsers, and more.
+- **[React hooks](https://trigger.dev/docs/frontend/react-hooks#react-hooks)** - Interact with the Trigger.dev API on your frontend using our React hooks package
+- **[Batch triggering](https://trigger.dev/docs/triggering#tasks-batchtrigger)** - Use batchTrigger() to initiate multiple runs of a task with custom payloads and options
+- **[Structured inputs / outputs](https://trigger.dev/docs/tasks/schemaTask#schematask)** - Define precise data schemas for your tasks with runtime payload validation
+- **[Waits](https://trigger.dev/docs/wait)** - Add waits to your tasks to pause execution for a specified duration
+- **[Preview branches](https://trigger.dev/docs/deployment/preview-branches)** - Create isolated environments for testing and development. Integrates with Vercel and git workflows
+- **[Waitpoints](https://trigger.dev/docs/wait-for-token#wait-for-token)** - Add human-in-the-loop judgment at critical decision points without disrupting workflow
+- **[Concurrency & queues](https://trigger.dev/docs/queue-concurrency#concurrency-and-queues)** - Set concurrency rules to manage how multiple tasks execute
+- **[Multiple environments](https://trigger.dev/docs/how-it-works#dev-mode)** - Support for DEV, PREVIEW, STAGING, and PROD environments
+- **[No infrastructure to manage](https://trigger.dev/docs/how-it-works#trigger-dev-architecture)** - Auto-scaling infrastructure that eliminates timeouts and server management
+- **[Automatic retries](https://trigger.dev/docs/errors-retrying)** - If your task encounters an uncaught error, we automatically attempt to run it again
+- **[Checkpointing](https://trigger.dev/docs/how-it-works#the-checkpoint-resume-system)** - Tasks are inherently durable, thanks to our checkpointing feature
+- **[Versioning](https://trigger.dev/docs/versioning)** - Atomic versioning allows you to deploy new versions without affecting running tasks
+- **[Machines](https://trigger.dev/docs/machines)** - Configure the number of vCPUs and GBs of RAM you want the task to use
+- **[Observability & monitoring](https://trigger.dev/product/observability-and-monitoring)** - Monitor every aspect of your tasks' performance with comprehensive logging and visualization tools
+- **[Logging & tracing](https://trigger.dev/docs/logging)** - Comprehensive logging and tracing for all your tasks
+- **[Tags](https://trigger.dev/docs/tags#tags)** - Attach up to ten tags to each run, allowing you to filter via the dashboard, realtime, and the SDK
+- **[Run metadata](https://trigger.dev/docs/runs/metadata#run-metadata)** - Attach metadata to runs which updates as the run progresses and is available to use in your frontend for live updates
+- **[Bulk actions](https://trigger.dev/docs/bulk-actions)** - Perform actions on multiple runs simultaneously, including replaying and cancelling
+- **[Real-time alerts](https://trigger.dev/docs/troubleshooting-alerts#alerts)** - Choose your preferred notification method for run failures and deployments
+
+## Write tasks in your codebase
+
+Create tasks where they belong: in your codebase. Version control, localhost, test and review like you're already used to.
+
+```ts
+import { task } from "@trigger.dev/sdk";
+
+//1. You need to export each task
+export const helloWorld = task({
+  //2. Use a unique id for each task
+  id: "hello-world",
+  //3. The run function is the main function of the task
+  run: async (payload: { message: string }) => {
+    //4. You can write code that runs for a long time here, there are no timeouts
+    console.log(payload.message);
+  },
+});
+```
+
+## Deployment
+
+Use our SDK to write tasks in your codebase. There's no infrastructure to manage, your tasks automatically scale and connect to our cloud. Or you can always self-host.
+
+## Environments
+
+We support `Development`, `Staging`, `Preview`, and `Production` environments, allowing you to test your tasks before deploying them to production.
 
 ## Full visibility of every job run
 
-View every Task in every Run so you can tell exactly what happened.
+View every task in every run so you can tell exactly what happened. We provide a full trace view of every task run so you can see what happened at every step.
 
-![image](https://www.trigger.dev/build/_assets/web-app-2QFKXFLW.png)
-
-## Built-in integrations
-
-Easily integrate with hundreds of third-party APIs – including your own. Use API keys (which never leave your server) or let us handle OAuth for you. Install our integration packages and easily subscribe to webhooks and perform common tasks, or you can easily use your existing favorite Node.JS SDKs and get resumability and idempotency through our `runTask` function.
-
-## Our progress
-
-We’re building the most comprehensive and easy-to-use background jobs framework for developers.
-
-Click the links to join the discussions about our upcoming features.
-
-| Feature                                                                              | What it does                                        | Status |
-| ------------------------------------------------------------------------------------ | --------------------------------------------------- | ------ |
-| Integration kit                                                                      | Official Trigger.dev integrations or build your own | ✅     |
-| Self-hosting                                                                         | Host the platform yourself                          | ✅     |
-| Cloud                                                                                | Just write code, no deployment required             | ✅     |
-| Dashboard                                                                            | View every Task in every Run                        | ✅     |
-| Serverless                                                                           | Long-running Jobs on your serverless backend        | ✅     |
-| React hooks                                                                          | Easily update your UI with Job progress             | ✅     |
-| React frameworks                                                                     | Support for Remix, Astro, RedwoodJS & more          | ✅     |
-| [Background tasks](https://github.com/triggerdotdev/trigger.dev/discussions/400)     | Offload long or intense Tasks to our infrastructure | 🛠️     |
-| [Long-running servers](https://github.com/triggerdotdev/trigger.dev/discussions/430) | Run Jobs on your long-running backend               | 🛠️     |
-| Polling Triggers                                                                     | Subscribe to changes without webhooks               | 🕝     |
-| Vercel integration                                                                   | Easy deploy and preview environment support         | 🕝     |
-| Streaming                                                                            | Receive data from your Jobs in realtime             | 🕝     |
-| 100+ integrations                                                                    | Comprehensive support for popular APIs              | 🕝     |
-| [Trigger.dev Connect](https://github.com/triggerdotdev/trigger.dev/discussions/441)  | Use integrations signed in as your users            | 🕝     |
-| File IO                                                                              | Create Tasks that have file outputs                 | 🕝     |
+![Trace view image](https://content.trigger.dev/trace-view.png)
 
 # Getting started
 
-Visit our docs [here](https://trigger.dev/docs).
+The quickest way to get started is to create an account and project in our [web app](https://cloud.trigger.dev), and follow the instructions in the onboarding. Build and deploy your first task in minutes.
 
-## Self-host
+### Useful links:
 
-We provide an official trigger.dev docker image you can use to easily self-host the platform. We're working on more extensive guides but we currently provide a [Fly.io example repository](https://github.com/triggerdotdev/fly.io) with instructions in the README for deploying and using a self-hosted instance of Trigger.dev on Fly.io.
+- [Quick start](https://trigger.dev/docs/quick-start) - get up and running in minutes
+- [How it works](https://trigger.dev/docs/how-it-works) - understand how Trigger.dev works under the hood
+- [Guides and examples](https://trigger.dev/docs/guides/introduction) - walk-through guides and code examples for popular frameworks and use cases
+
+## Self-hosting
+
+If you prefer to self-host Trigger.dev, you can follow our [self-hosting guides](https://trigger.dev/docs/self-hosting/overview):
+
+- [Docker self-hosting guide](https://trigger.dev/docs/self-hosting/docker) - use Docker Compose to spin up a Trigger.dev instance
+- [Kubernetes self-hosting guide](https://trigger.dev/docs/self-hosting/kubernetes) - use our official Helm chart to deploy Trigger.dev to your Kubernetes cluster
+
+## Support and community
+
+We have a large active community in our official [Discord server](https://trigger.dev/discord) for support, including a dedicated channel for self-hosting.
 
 ## Development
 
 To setup and develop locally or contribute to the open source project, follow our [development guide](./CONTRIBUTING.md).
 
-## Meet the Amazing People Behind This Project 🚀
+## Meet the Amazing People Behind This Project:
 
 <a href="https://github.com/triggerdotdev/trigger.dev/graphs/contributors">
   <img src="https://contrib.rocks/image?repo=triggerdotdev/trigger.dev" />

RELEASE.md

@@ -0,0 +1,13 @@
+## Guide on releasing a new version
+
+1. Merge in the changeset PR into main, making sure to cancel both the release and publish github actions from that merge.
+2. Pull the changes locally into main
+3. Run `pnpm i` which will update the pnpm lock file with the new versions
+4. create a commit with "Release 3.x.x" and push. This will build and release the packages
+5. Create a git tag on that release commit with v.docker.3.x.x and push the tag to origin. This will publish the `v3.x.x` docker image to GitHub Container Registry.
+6. Once the image is built and pushed, create a new GitHub release and select the tag you just created, along with the previous tag that was released.
+7. This will generate some release notes. Edit out the package changes and leave only the server changes.
+8. Name the release `@trigger.dev/docker@3.x.x`
+9. Include the package link (e.g. https://github.com/triggerdotdev/trigger.dev/pkgs/container/trigger.dev/278459584?tag=v3.x.x)
+10. Once the packages have been published, head over to the [v2-legacy repo](https://github.com/triggerdotdev/v2-legacy.trigger.dev) and follow the instructions in the README for creating a matching release.
+11. Before deploying to cloud, compare the differences in the previously created release and double check to see if there are any migrations with indexes created concurrently, and make sure to run those before deploying.

ai/references/migrations.md

@@ -0,0 +1,121 @@
+## Creating and applying migrations
+
+We use prisma migrations to manage the database schema. Please follow the following steps when editing the `internal-packages/database/prisma/schema.prisma` file:
+
+Edit the `schema.prisma` file to add or modify the schema.
+
+Create a new migration file but don't apply it yet:
+
+```bash
+cd internal-packages/database
+pnpm run db:migrate:dev:create --name "add_new_column_to_table"
+```
+
+The migration file will be created in the `prisma/migrations` directory, but it will have a bunch of edits to the schema that are not needed and will need to be removed before we can apply the migration. Here's an example of what the migration file might look like:
+
+```sql
+-- AlterEnum
+ALTER TYPE "public"."TaskRunExecutionStatus" ADD VALUE 'DELAYED';
+
+-- AlterTable
+ALTER TABLE "public"."TaskRun" ADD COLUMN     "debounce" JSONB;
+
+-- AlterTable
+ALTER TABLE "public"."_BackgroundWorkerToBackgroundWorkerFile" ADD CONSTRAINT "_BackgroundWorkerToBackgroundWorkerFile_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_BackgroundWorkerToBackgroundWorkerFile_AB_unique";
+
+-- AlterTable
+ALTER TABLE "public"."_BackgroundWorkerToTaskQueue" ADD CONSTRAINT "_BackgroundWorkerToTaskQueue_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_BackgroundWorkerToTaskQueue_AB_unique";
+
+-- AlterTable
+ALTER TABLE "public"."_TaskRunToTaskRunTag" ADD CONSTRAINT "_TaskRunToTaskRunTag_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_TaskRunToTaskRunTag_AB_unique";
+
+-- AlterTable
+ALTER TABLE "public"."_WaitpointRunConnections" ADD CONSTRAINT "_WaitpointRunConnections_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_WaitpointRunConnections_AB_unique";
+
+-- AlterTable
+ALTER TABLE "public"."_completedWaitpoints" ADD CONSTRAINT "_completedWaitpoints_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_completedWaitpoints_AB_unique";
+
+-- CreateIndex
+CREATE INDEX "SecretStore_key_idx" ON "public"."SecretStore"("key" text_pattern_ops);
+
+-- CreateIndex
+CREATE INDEX "TaskRun_runtimeEnvironmentId_id_idx" ON "public"."TaskRun"("runtimeEnvironmentId", "id" DESC);
+
+-- CreateIndex
+CREATE INDEX "TaskRun_runtimeEnvironmentId_createdAt_idx" ON "public"."TaskRun"("runtimeEnvironmentId", "createdAt" DESC);
+```
+
+All the following lines should be removed:
+
+```sql
+-- AlterTable
+ALTER TABLE "public"."_BackgroundWorkerToBackgroundWorkerFile" ADD CONSTRAINT "_BackgroundWorkerToBackgroundWorkerFile_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_BackgroundWorkerToBackgroundWorkerFile_AB_unique";
+
+-- AlterTable
+ALTER TABLE "public"."_BackgroundWorkerToTaskQueue" ADD CONSTRAINT "_BackgroundWorkerToTaskQueue_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_BackgroundWorkerToTaskQueue_AB_unique";
+
+-- AlterTable
+ALTER TABLE "public"."_TaskRunToTaskRunTag" ADD CONSTRAINT "_TaskRunToTaskRunTag_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_TaskRunToTaskRunTag_AB_unique";
+
+-- AlterTable
+ALTER TABLE "public"."_WaitpointRunConnections" ADD CONSTRAINT "_WaitpointRunConnections_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_WaitpointRunConnections_AB_unique";
+
+-- AlterTable
+ALTER TABLE "public"."_completedWaitpoints" ADD CONSTRAINT "_completedWaitpoints_AB_pkey" PRIMARY KEY ("A", "B");
+
+-- DropIndex
+DROP INDEX "public"."_completedWaitpoints_AB_unique";
+
+-- CreateIndex
+CREATE INDEX "SecretStore_key_idx" ON "public"."SecretStore"("key" text_pattern_ops);
+
+-- CreateIndex
+CREATE INDEX "TaskRun_runtimeEnvironmentId_id_idx" ON "public"."TaskRun"("runtimeEnvironmentId", "id" DESC);
+
+-- CreateIndex
+CREATE INDEX "TaskRun_runtimeEnvironmentId_createdAt_idx" ON "public"."TaskRun"("runtimeEnvironmentId", "createdAt" DESC);
+```
+
+Leaving only this:
+
+```sql
+-- AlterEnum
+ALTER TYPE "public"."TaskRunExecutionStatus" ADD VALUE 'DELAYED';
+
+-- AlterTable
+ALTER TABLE "public"."TaskRun" ADD COLUMN     "debounce" JSONB;
+```
+
+After editing the migration file, apply the migration:
+
+```bash
+cd internal-packages/database
+pnpm run db:migrate:deploy && pnpm run generate
+```

ai/references/repo.md

@@ -0,0 +1,37 @@
+## Repo Overview
+
+This is a pnpm 10.23.0 monorepo that uses turborepo @turbo.json. The following workspaces are relevant
+
+## Apps
+
+- <root>/apps/webapp is a remix app that is the main API and dashboard for trigger.dev
+- <root>/apps/supervisor is a node.js app that handles the execution of built tasks, interaction with the webapp through internal "engine" APIs, as well as interfacing with things like docker or kubernetes, to execute the code.
+
+## Public Packages
+
+- <root>/packages/trigger-sdk is the `@trigger.dev/sdk` main SDK package.
+- <root>/packages/cli-v3 is the `trigger.dev` CLI package. See our [CLI dev command](https://trigger.dev/docs/cli-dev.md) and [Deployment](https://trigger.dev/docs/deployment/overview.md) docs for more information.
+- <root>/packages/core is the `@trigger.dev/core` package that is shared across the SDK and other packages
+- <root>/packages/build defines the types and prebuilt build extensions for trigger.dev. See our [build extensions docs](https://trigger.dev/docs/config/extensions/overview.md) for more information.
+- <root>/packages/react-hooks defines some useful react hooks like our realtime hooks. See our [Realtime hooks](https://trigger.dev/docs/frontend/react-hooks/realtime.md) and our [Trigger hooks](https://trigger.dev/docs/frontend/react-hooks/triggering.md) for more information.
+- <root>/packages/redis-worker is the `@trigger.dev/redis-worker` package that implements a custom background job/worker sytem powered by redis for offloading work to the background, used in the webapp and also in the Run Engine 2.0.
+
+## Internal Packages
+
+- <root>/internal-packages/\* are packages that are used internally only, not published, and usually they have a tsc build step and are used in the webapp
+- <root>/internal-packages/database is the `@trigger.dev/database` package that exports a prisma client, has the schema file, and exports a few other helpers.
+- <root>/internal-packages/run-engine is the `@internal/run-engine` package that is "Run Engine 2.0" and handles moving a run all the way through it's lifecycle
+- <root>/internal-packages/redis is the `@internal/redis` package that exports Redis types and the `createRedisClient` function to unify how we create redis clients in the repo. It's not used everywhere yet, but it's the preferred way to create redis clients from now on.
+- <root>/internal-packages/testcontainers is the `@internal/testcontainers` package that exports a few useful functions for spinning up local testcontainers when writing vitest tests. See our [tests.md](./tests.md) file for more information.
+- <root>/internal-packages/zodworker is the `@internal/zodworker` package that implements a wrapper around graphile-worker that allows us to use zod to validate our background jobs. We are moving away from using graphile-worker as our background job system, replacing it with our own redis-worker package.
+
+## References
+
+- <root>/references/\* are test workspaces that we use to write and test the system. Not quite e2e tests or automated, but just a useful place to help develop new features
+
+## Other
+
+- <root>/docs is our trigger.dev/docs mintlify documentation site
+- <root>/docker/Dockerfile is the one that creates the main trigger.dev published image
+- <root>/docker/docker-compose.yml is the file we run locally to start postgresql, redis, and electric when we are doing local development. You can run it with `pnpm run docker`
+- <root>/CONTRIBUTING.md defines the steps it takes for OSS contributors to start contributing.

ai/references/tests.md

@@ -0,0 +1,86 @@
+## Running Tests
+
+We use vitest exclusively for testing. To execute tests for a particular workspace, run the following command:
+
+```bash
+pnpm run test --filter webapp
+```
+
+Prefer running tests on a single file (and first cding into the directory):
+
+```bash
+cd apps/webapp
+pnpm run test ./src/components/Button.test.ts
+```
+
+If you are cd'ing into a directory, you may have to build dependencies first:
+
+```bash
+pnpm run build --filter webapp
+cd apps/webapp
+pnpm run test ./src/components/Button.test.ts
+```
+
+## Writing Tests
+
+We use vitest for testing. We almost NEVER mock anything. Start with a top-level "describe", and have multiple "it" statements inside of it.
+
+New test files should be placed right next to the file being tested. For example:
+
+- Source file: `./src/services/MyService.ts`
+- Test file: `./src/services/MyService.test.ts`
+
+When writing anything that needs redis or postgresql, we have some internal "testcontainers" that are used to spin up a local instance, redis, or both.
+
+redisTest:
+
+```typescript
+import { redisTest } from "@internal/testcontainers";
+import { createRedisClient } from "@internal/redis";
+
+describe("redisTest", () => {
+  redisTest("should use redis", async ({ redisOptions }) => {
+    const redis = createRedisClient(redisOptions);
+
+    await redis.set("test", "test");
+    const result = await redis.get("test");
+    expect(result).toEqual("test");
+  });
+});
+```
+
+postgresTest:
+
+```typescript
+import { postgresTest } from "@internal/testcontainers";
+
+describe("postgresTest", () => {
+  postgresTest("should use postgres", async ({ prisma }) => {
+    // prisma is an instance of PrismaClient
+  });
+});
+```
+
+containerTest:
+
+```typescript
+import { containerTest } from "@internal/testcontainers";
+
+describe("containerTest", () => {
+  containerTest("should use container", async ({ prisma, redisOptions }) => {
+    // container has both prisma and redis
+  });
+});
+```
+
+## Dos and Dont's
+
+- Do not mock anything.
+- Do not use mocks in tests.
+- Do not use spies in tests.
+- Do not use stubs in tests.
+- Do not use fakes in tests.
+- Do not use sinon in tests.
+- Structure each test with a setup, action, and assertion style.
+- Feel free to write long test names.
+- If there is any randomness in the code under test, use `seedrandom` to make it deterministic by allowing the caller to provide a seed.

apps/coordinator/.env.example

@@ -0,0 +1,4 @@
+HTTP_SERVER_PORT=8020
+PLATFORM_ENABLED=true
+PLATFORM_WS_PORT=3030
+SECURE_CONNECTION=false

apps/coordinator/.gitignore

@@ -0,0 +1,3 @@
+dist/
+node_modules/
+.env

apps/coordinator/Containerfile

@@ -0,0 +1,60 @@
+# syntax=docker/dockerfile:labs
+
+FROM node:20-bookworm-slim@sha256:72f2f046a5f8468db28730b990b37de63ce93fd1a72a40f531d6aa82afdf0d46 AS node-20
+
+WORKDIR /app
+
+FROM node-20 AS pruner
+
+COPY --chown=node:node . .
+RUN npx -q turbo@1.10.9 prune --scope=coordinator --docker
+RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
+
+FROM node-20 AS base
+
+RUN apt-get update \
+  && apt-get install -y buildah ca-certificates dumb-init docker.io busybox \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY --chown=node:node .gitignore .gitignore
+COPY --from=pruner --chown=node:node /app/out/json/ .
+COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
+COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
+
+FROM base AS dev-deps
+RUN corepack enable
+ENV NODE_ENV development
+
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
+
+FROM base AS builder
+RUN corepack enable
+
+COPY --from=pruner --chown=node:node /app/out/full/ .
+COPY --from=dev-deps --chown=node:node /app/ .
+COPY --chown=node:node turbo.json turbo.json
+
+RUN pnpm run -r --filter @trigger.dev/core bundle-vendor && pnpm run -r --filter coordinator build:bundle
+
+FROM alpine AS cri-tools
+
+WORKDIR /cri-tools
+
+ARG CRICTL_VERSION=v1.29.0
+ARG CRICTL_CHECKSUM=sha256:d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb
+ADD --checksum=${CRICTL_CHECKSUM} \
+  https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz .
+RUN tar zxvf crictl-${CRICTL_VERSION}-linux-amd64.tar.gz
+
+FROM base AS runner
+
+RUN corepack enable
+ENV NODE_ENV production
+
+COPY --from=cri-tools --chown=node:node /cri-tools/crictl /usr/local/bin
+COPY --from=builder --chown=node:node /app/apps/coordinator/dist/index.mjs ./index.mjs
+
+EXPOSE 8000
+
+CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]

apps/coordinator/README.md

@@ -0,0 +1,3 @@
+# Coordinator
+
+Sits between the platform and tasks. Facilitates communication and checkpointing, amongst other things.

apps/coordinator/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "coordinator",
+  "private": true,
+  "version": "0.0.1",
+  "description": "",
+  "main": "dist/index.cjs",
+  "scripts": {
+    "build": "npm run build:bundle",
+    "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"",
+    "build:image": "docker build -f Containerfile . -t coordinator",
+    "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
+    "start": "tsx src/index.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@trigger.dev/core": "workspace:*",
+    "nanoid": "^5.0.6",
+    "prom-client": "^15.1.0",
+    "socket.io": "4.7.4",
+    "tinyexec": "^0.3.0"
+  },
+  "devDependencies": {
+    "dotenv": "^16.4.2",
+    "esbuild": "^0.19.11",
+    "tsx": "^4.7.0"
+  }
+}

apps/coordinator/src/chaosMonkey.ts

@@ -0,0 +1,88 @@
+import { setTimeout as timeout } from "node:timers/promises";
+
+class ChaosMonkeyError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ChaosMonkeyError";
+  }
+}
+
+export class ChaosMonkey {
+  private chaosEventRate = 0.2;
+  private delayInSeconds = 45;
+
+  constructor(
+    private enabled = false,
+    private disableErrors = false,
+    private disableDelays = false
+  ) {
+    if (this.enabled) {
+      console.log("🍌 Chaos monkey enabled");
+    }
+  }
+
+  static Error = ChaosMonkeyError;
+
+  enable() {
+    this.enabled = true;
+    console.log("🍌 Chaos monkey enabled");
+  }
+
+  disable() {
+    this.enabled = false;
+    console.log("🍌 Chaos monkey disabled");
+  }
+
+  async call({
+    throwErrors = !this.disableErrors,
+    addDelays = !this.disableDelays,
+  }: {
+    throwErrors?: boolean;
+    addDelays?: boolean;
+  } = {}) {
+    if (!this.enabled) {
+      return;
+    }
+
+    const random = Math.random();
+
+    if (random > this.chaosEventRate) {
+      // Don't interfere with normal operation
+      return;
+    }
+
+    const chaosEvents: Array<() => Promise<any>> = [];
+
+    if (addDelays) {
+      chaosEvents.push(async () => {
+        console.log("🍌 Chaos monkey: Add delay");
+
+        await timeout(this.delayInSeconds * 1000);
+      });
+    }
+
+    if (throwErrors) {
+      chaosEvents.push(async () => {
+        console.log("🍌 Chaos monkey: Throw error");
+
+        throw new ChaosMonkey.Error("🍌 Chaos monkey: Throw error");
+      });
+    }
+
+    if (chaosEvents.length === 0) {
+      console.error("🍌 Chaos monkey: No events selected");
+      return;
+    }
+
+    const randomIndex = Math.floor(Math.random() * chaosEvents.length);
+
+    const chaosEvent = chaosEvents[randomIndex];
+
+    if (!chaosEvent) {
+      console.error("🍌 Chaos monkey: No event found");
+      return;
+    }
+
+    await chaosEvent();
+  }
+}

apps/coordinator/src/checkpointer.ts

@@ -0,0 +1,708 @@
+import { ExponentialBackoff } from "@trigger.dev/core/v3/apps";
+import { testDockerCheckpoint } from "@trigger.dev/core/v3/serverOnly";
+import { nanoid } from "nanoid";
+import fs from "node:fs/promises";
+import { ChaosMonkey } from "./chaosMonkey";
+import { Buildah, Crictl, Exec } from "./exec";
+import { setTimeout } from "node:timers/promises";
+import { TempFileCleaner } from "./cleaner";
+import { numFromEnv, boolFromEnv } from "./util";
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+
+type CheckpointerInitializeReturn = {
+  canCheckpoint: boolean;
+  willSimulate: boolean;
+};
+
+type CheckpointAndPushOptions = {
+  runId: string;
+  leaveRunning?: boolean;
+  projectRef: string;
+  deploymentVersion: string;
+  shouldHeartbeat?: boolean;
+  attemptNumber?: number;
+};
+
+type CheckpointAndPushResult =
+  | { success: true; checkpoint: CheckpointData }
+  | {
+      success: false;
+      reason?: "CANCELED" | "ERROR" | "SKIP_RETRYING";
+    };
+
+type CheckpointData = {
+  location: string;
+  docker: boolean;
+};
+
+type CheckpointerOptions = {
+  dockerMode: boolean;
+  forceSimulate: boolean;
+  heartbeat: (runId: string) => void;
+  registryHost?: string;
+  registryNamespace?: string;
+  registryTlsVerify?: boolean;
+  disableCheckpointSupport?: boolean;
+  checkpointPath?: string;
+  simulateCheckpointFailure?: boolean;
+  simulateCheckpointFailureSeconds?: number;
+  simulatePushFailure?: boolean;
+  simulatePushFailureSeconds?: number;
+  chaosMonkey?: ChaosMonkey;
+};
+
+async function getFileSize(filePath: string): Promise<number> {
+  try {
+    const stats = await fs.stat(filePath);
+    return stats.size;
+  } catch (error) {
+    console.error("Error getting file size:", error);
+    return -1;
+  }
+}
+
+async function getParsedFileSize(filePath: string) {
+  const sizeInBytes = await getFileSize(filePath);
+
+  let message = `Size in bytes: ${sizeInBytes}`;
+
+  if (sizeInBytes > 1024 * 1024) {
+    const sizeInMB = (sizeInBytes / 1024 / 1024).toFixed(2);
+    message = `Size in MB (rounded): ${sizeInMB}`;
+  } else if (sizeInBytes > 1024) {
+    const sizeInKB = (sizeInBytes / 1024).toFixed(2);
+    message = `Size in KB (rounded): ${sizeInKB}`;
+  }
+
+  return {
+    path: filePath,
+    sizeInBytes,
+    message,
+  };
+}
+
+export class Checkpointer {
+  #initialized = false;
+  #canCheckpoint = false;
+  #dockerMode: boolean;
+
+  #logger = new SimpleStructuredLogger("checkpointer");
+
+  #failedCheckpoints = new Map<string, unknown>();
+
+  // Indexed by run ID
+  #runAbortControllers = new Map<
+    string,
+    { signal: AbortSignal; abort: AbortController["abort"] }
+  >();
+
+  private registryHost: string;
+  private registryNamespace: string;
+  private registryTlsVerify: boolean;
+
+  private disableCheckpointSupport: boolean;
+
+  private simulateCheckpointFailure: boolean;
+  private simulateCheckpointFailureSeconds: number;
+  private simulatePushFailure: boolean;
+  private simulatePushFailureSeconds: number;
+
+  private chaosMonkey: ChaosMonkey;
+  private tmpCleaner?: TempFileCleaner;
+
+  constructor(private opts: CheckpointerOptions) {
+    this.#dockerMode = opts.dockerMode;
+
+    this.registryHost = opts.registryHost ?? "localhost:5000";
+    this.registryNamespace = opts.registryNamespace ?? "trigger";
+    this.registryTlsVerify = opts.registryTlsVerify ?? true;
+
+    this.disableCheckpointSupport = opts.disableCheckpointSupport ?? false;
+
+    this.simulateCheckpointFailure = opts.simulateCheckpointFailure ?? false;
+    this.simulateCheckpointFailureSeconds = opts.simulateCheckpointFailureSeconds ?? 300;
+    this.simulatePushFailure = opts.simulatePushFailure ?? false;
+    this.simulatePushFailureSeconds = opts.simulatePushFailureSeconds ?? 300;
+
+    this.chaosMonkey = opts.chaosMonkey ?? new ChaosMonkey(!!process.env.CHAOS_MONKEY_ENABLED);
+    this.tmpCleaner = this.#createTmpCleaner();
+  }
+
+  async init(): Promise<CheckpointerInitializeReturn> {
+    if (this.#initialized) {
+      return this.#getInitReturn(this.#canCheckpoint);
+    }
+
+    this.#logger.log(`${this.#dockerMode ? "Docker" : "Kubernetes"} mode`);
+
+    if (this.#dockerMode) {
+      const testCheckpoint = await testDockerCheckpoint();
+
+      if (testCheckpoint.ok) {
+        return this.#getInitReturn(true);
+      }
+
+      this.#logger.error(testCheckpoint.message, { error: testCheckpoint.error });
+      return this.#getInitReturn(false);
+    }
+
+    const canLogin = await Buildah.canLogin(this.registryHost);
+
+    if (!canLogin) {
+      this.#logger.error(`No checkpoint support: Not logged in to registry ${this.registryHost}`);
+    }
+
+    return this.#getInitReturn(canLogin);
+  }
+
+  #getInitReturn(canCheckpoint: boolean): CheckpointerInitializeReturn {
+    this.#canCheckpoint = canCheckpoint;
+
+    if (canCheckpoint) {
+      if (!this.#initialized) {
+        this.#logger.log("Full checkpoint support!");
+      }
+    }
+
+    this.#initialized = true;
+
+    const willSimulate = this.#dockerMode && (!this.#canCheckpoint || this.opts.forceSimulate);
+
+    if (willSimulate) {
+      this.#logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", {
+        forceSimulate: this.opts.forceSimulate,
+      });
+    }
+
+    return {
+      canCheckpoint,
+      willSimulate,
+    };
+  }
+
+  #getImageRef(projectRef: string, deploymentVersion: string, shortCode: string) {
+    return `${this.registryHost}/${this.registryNamespace}/${projectRef}:${deploymentVersion}.prod-${shortCode}`;
+  }
+
+  #getExportLocation(projectRef: string, deploymentVersion: string, shortCode: string) {
+    const basename = `${projectRef}-${deploymentVersion}-${shortCode}`;
+
+    if (this.#dockerMode) {
+      return basename;
+    } else {
+      return Crictl.getExportLocation(basename);
+    }
+  }
+
+  async checkpointAndPush(
+    opts: CheckpointAndPushOptions,
+    delayMs?: number
+  ): Promise<CheckpointData | undefined> {
+    const start = performance.now();
+    this.#logger.log(`checkpointAndPush() start`, { start, opts });
+
+    const { runId } = opts;
+
+    let interval: NodeJS.Timer | undefined;
+    if (opts.shouldHeartbeat) {
+      interval = setInterval(() => {
+        this.#logger.log("Sending heartbeat", { runId });
+        this.opts.heartbeat(runId);
+      }, 20_000);
+    }
+
+    const controller = new AbortController();
+    const signal = controller.signal;
+    const abort = controller.abort.bind(controller);
+
+    const onAbort = () => {
+      this.#logger.error("Checkpoint aborted", { runId, options: opts });
+    };
+
+    signal.addEventListener("abort", onAbort, { once: true });
+
+    const removeCurrentAbortController = () => {
+      const controller = this.#runAbortControllers.get(runId);
+
+      // Ensure only the current controller is removed
+      if (controller && controller.signal === signal) {
+        this.#runAbortControllers.delete(runId);
+      }
+
+      // Remove the abort listener in case it hasn't fired
+      signal.removeEventListener("abort", onAbort);
+    };
+
+    if (!this.#dockerMode && !this.#canCheckpoint) {
+      this.#logger.error("No checkpoint support. Simulation requires docker.");
+      this.#failCheckpoint(runId, "NO_SUPPORT");
+      return;
+    }
+
+    if (this.#isRunCheckpointing(runId)) {
+      this.#logger.error("Checkpoint procedure already in progress", { options: opts });
+      this.#failCheckpoint(runId, "IN_PROGRESS");
+      return;
+    }
+
+    // This is a new checkpoint, clear any last failure for this run
+    this.#clearFailedCheckpoint(runId);
+
+    if (this.disableCheckpointSupport) {
+      this.#logger.error("Checkpoint support disabled", { options: opts });
+      this.#failCheckpoint(runId, "DISABLED");
+      return;
+    }
+
+    this.#runAbortControllers.set(runId, { signal, abort });
+
+    try {
+      const result = await this.#checkpointAndPushWithBackoff(opts, { delayMs, signal });
+
+      const end = performance.now();
+      this.#logger.log(`checkpointAndPush() end`, {
+        start,
+        end,
+        diff: end - start,
+        diffWithoutDelay: end - start - (delayMs ?? 0),
+        opts,
+        success: result.success,
+        delayMs,
+      });
+
+      if (!result.success) {
+        return;
+      }
+
+      return result.checkpoint;
+    } finally {
+      if (opts.shouldHeartbeat) {
+        // @ts-ignore - Some kind of node incompatible type issue
+        clearInterval(interval);
+      }
+      removeCurrentAbortController();
+    }
+  }
+
+  #isRunCheckpointing(runId: string) {
+    return this.#runAbortControllers.has(runId);
+  }
+
+  cancelAllCheckpointsForRun(runId: string): boolean {
+    this.#logger.log("cancelAllCheckpointsForRun: call", { runId });
+
+    // If the last checkpoint failed, pretend we canceled it
+    // This ensures tasks don't wait for external resume messages to continue
+    if (this.#hasFailedCheckpoint(runId)) {
+      this.#logger.log("cancelAllCheckpointsForRun: hasFailedCheckpoint", { runId });
+      this.#clearFailedCheckpoint(runId);
+      return true;
+    }
+
+    const controller = this.#runAbortControllers.get(runId);
+
+    if (!controller) {
+      this.#logger.debug("cancelAllCheckpointsForRun: no abort controller", { runId });
+      return false;
+    }
+
+    const { abort, signal } = controller;
+
+    if (signal.aborted) {
+      this.#logger.debug("cancelAllCheckpointsForRun: signal already aborted", { runId });
+      return false;
+    }
+
+    abort("cancelCheckpoint()");
+    this.#runAbortControllers.delete(runId);
+
+    return true;
+  }
+
+  async #checkpointAndPushWithBackoff(
+    {
+      runId,
+      leaveRunning = true, // This mirrors kubernetes behaviour more accurately
+      projectRef,
+      deploymentVersion,
+      attemptNumber,
+    }: CheckpointAndPushOptions,
+    { delayMs, signal }: { delayMs?: number; signal: AbortSignal }
+  ): Promise<CheckpointAndPushResult> {
+    if (delayMs && delayMs > 0) {
+      this.#logger.log("Delaying checkpoint", { runId, delayMs });
+
+      try {
+        await setTimeout(delayMs, undefined, { signal });
+      } catch (error) {
+        this.#logger.log("Checkpoint canceled during initial delay", { runId });
+        return { success: false, reason: "CANCELED" };
+      }
+    }
+
+    this.#logger.log("Checkpointing with backoff", {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+    });
+
+    const backoff = new ExponentialBackoff()
+      .type("EqualJitter")
+      .base(3)
+      .max(3 * 3600)
+      .maxElapsed(48 * 3600);
+
+    for await (const { delay, retry } of backoff) {
+      try {
+        if (retry > 0) {
+          this.#logger.error("Retrying checkpoint", {
+            runId,
+            retry,
+            delay,
+          });
+
+          try {
+            await setTimeout(delay.milliseconds, undefined, { signal });
+          } catch (error) {
+            this.#logger.log("Checkpoint canceled during retry delay", { runId });
+            return { success: false, reason: "CANCELED" };
+          }
+        }
+
+        const result = await this.#checkpointAndPush(
+          {
+            runId,
+            leaveRunning,
+            projectRef,
+            deploymentVersion,
+            attemptNumber,
+          },
+          { signal }
+        );
+
+        if (result.success) {
+          return result;
+        }
+
+        if (result.reason === "CANCELED") {
+          this.#logger.log("Checkpoint canceled, won't retry", { runId });
+          // Don't fail the checkpoint, as it was canceled
+          return result;
+        }
+
+        if (result.reason === "SKIP_RETRYING") {
+          this.#logger.log("Skipping retrying", { runId });
+          return result;
+        }
+
+        continue;
+      } catch (error) {
+        this.#logger.error("Checkpoint error", {
+          retry,
+          runId,
+          delay,
+          error: error instanceof Error ? error.message : error,
+        });
+      }
+    }
+
+    this.#logger.error(`Checkpoint failed after exponential backoff`, {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+    });
+    this.#failCheckpoint(runId, "ERROR");
+
+    return { success: false, reason: "ERROR" };
+  }
+
+  async #checkpointAndPush(
+    {
+      runId,
+      leaveRunning = true, // This mirrors kubernetes behaviour more accurately
+      projectRef,
+      deploymentVersion,
+      attemptNumber,
+    }: CheckpointAndPushOptions,
+    { signal }: { signal: AbortSignal }
+  ): Promise<CheckpointAndPushResult> {
+    await this.init();
+
+    const options = {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+      attemptNumber,
+    };
+
+    const shortCode = nanoid(8);
+    const imageRef = this.#getImageRef(projectRef, deploymentVersion, shortCode);
+    const exportLocation = this.#getExportLocation(projectRef, deploymentVersion, shortCode);
+
+    const buildah = new Buildah({ id: `${runId}-${shortCode}`, abortSignal: signal });
+    const crictl = new Crictl({ id: `${runId}-${shortCode}`, abortSignal: signal });
+
+    const cleanup = async () => {
+      const metadata = {
+        runId,
+        exportLocation,
+        imageRef,
+      };
+
+      if (this.#dockerMode) {
+        this.#logger.debug("Skipping cleanup in docker mode", metadata);
+        return;
+      }
+
+      this.#logger.log("Cleaning up", metadata);
+
+      try {
+        await buildah.cleanup();
+        await crictl.cleanup();
+      } catch (error) {
+        this.#logger.error("Error during cleanup", { ...metadata, error });
+      }
+    };
+
+    try {
+      await this.chaosMonkey.call();
+
+      this.#logger.log("checkpointAndPush: checkpointing", { options });
+
+      const containterName = this.#getRunContainerName(runId);
+
+      // Create checkpoint (docker)
+      if (this.#dockerMode) {
+        await this.#createDockerCheckpoint(
+          signal,
+          runId,
+          exportLocation,
+          leaveRunning,
+          attemptNumber
+        );
+
+        this.#logger.log("checkpointAndPush: checkpoint created", {
+          runId,
+          location: exportLocation,
+        });
+
+        return {
+          success: true,
+          checkpoint: {
+            location: exportLocation,
+            docker: true,
+          },
+        };
+      }
+
+      // Create checkpoint (CRI)
+      if (!this.#canCheckpoint) {
+        this.#logger.error("No checkpoint support in kubernetes mode.");
+        return { success: false, reason: "SKIP_RETRYING" };
+      }
+
+      const containerId = await crictl.ps(containterName, true);
+
+      if (!containerId.stdout) {
+        this.#logger.error("could not find container id", { options, containterName });
+        return { success: false, reason: "SKIP_RETRYING" };
+      }
+
+      const start = performance.now();
+
+      if (this.simulateCheckpointFailure) {
+        if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) {
+          this.#logger.error("Simulating checkpoint failure", { options });
+          throw new Error("SIMULATE_CHECKPOINT_FAILURE");
+        }
+      }
+
+      // Create checkpoint
+      await crictl.checkpoint(containerId.stdout, exportLocation);
+      const postCheckpoint = performance.now();
+
+      // Print checkpoint size
+      const size = await getParsedFileSize(exportLocation);
+      this.#logger.log("checkpoint archive created", { size, options });
+
+      // Create image from checkpoint
+      const workingContainer = await buildah.from("scratch");
+      const postFrom = performance.now();
+
+      await buildah.add(workingContainer.stdout, exportLocation, "/");
+      const postAdd = performance.now();
+
+      await buildah.config(workingContainer.stdout, [
+        `io.kubernetes.cri-o.annotations.checkpoint.name=${shortCode}`,
+      ]);
+      const postConfig = performance.now();
+
+      await buildah.commit(workingContainer.stdout, imageRef);
+      const postCommit = performance.now();
+
+      if (this.simulatePushFailure) {
+        if (performance.now() < this.simulatePushFailureSeconds * 1000) {
+          this.#logger.error("Simulating push failure", { options });
+          throw new Error("SIMULATE_PUSH_FAILURE");
+        }
+      }
+
+      // Push checkpoint image
+      await buildah.push(imageRef, this.registryTlsVerify);
+      const postPush = performance.now();
+
+      const perf = {
+        "crictl checkpoint": postCheckpoint - start,
+        "buildah from": postFrom - postCheckpoint,
+        "buildah add": postAdd - postFrom,
+        "buildah config": postConfig - postAdd,
+        "buildah commit": postCommit - postConfig,
+        "buildah push": postPush - postCommit,
+      };
+
+      this.#logger.log("Checkpointed and pushed image to:", { location: imageRef, perf });
+
+      return {
+        success: true,
+        checkpoint: {
+          location: imageRef,
+          docker: false,
+        },
+      };
+    } catch (error) {
+      if (error instanceof Exec.Result) {
+        if (error.aborted) {
+          this.#logger.error("Checkpoint canceled: Exec", { options });
+
+          return { success: false, reason: "CANCELED" };
+        } else {
+          this.#logger.error("Checkpoint command error", { options, error });
+
+          return { success: false, reason: "ERROR" };
+        }
+      }
+
+      this.#logger.error("Unhandled checkpoint error", {
+        options,
+        error: error instanceof Error ? error.message : error,
+      });
+
+      return { success: false, reason: "ERROR" };
+    } finally {
+      await cleanup();
+
+      if (signal.aborted) {
+        this.#logger.error("Checkpoint canceled: Cleanup", { options });
+
+        // Overrides any prior return value
+        return { success: false, reason: "CANCELED" };
+      }
+    }
+  }
+
+  async unpause(runId: string, attemptNumber?: number): Promise<void> {
+    try {
+      const containterNameWithAttempt = this.#getRunContainerName(runId, attemptNumber);
+      const exec = new Exec({ logger: this.#logger });
+      await exec.x("docker", ["unpause", containterNameWithAttempt]);
+    } catch (error) {
+      this.#logger.error("[Docker] Error during unpause", { runId, attemptNumber, error });
+    }
+  }
+
+  async #createDockerCheckpoint(
+    abortSignal: AbortSignal,
+    runId: string,
+    exportLocation: string,
+    leaveRunning: boolean,
+    attemptNumber?: number
+  ) {
+    const containterNameWithAttempt = this.#getRunContainerName(runId, attemptNumber);
+    const exec = new Exec({ logger: this.#logger, abortSignal });
+
+    try {
+      if (this.opts.forceSimulate || !this.#canCheckpoint) {
+        this.#logger.log("Simulating checkpoint");
+
+        await exec.x("docker", ["pause", containterNameWithAttempt]);
+
+        return;
+      }
+
+      if (this.simulateCheckpointFailure) {
+        if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) {
+          this.#logger.error("Simulating checkpoint failure", {
+            runId,
+            exportLocation,
+            leaveRunning,
+            attemptNumber,
+          });
+
+          throw new Error("SIMULATE_CHECKPOINT_FAILURE");
+        }
+      }
+
+      const args = ["checkpoint", "create"];
+
+      if (leaveRunning) {
+        args.push("--leave-running");
+      }
+
+      args.push(containterNameWithAttempt, exportLocation);
+
+      await exec.x("docker", args);
+    } catch (error) {
+      this.#logger.error("Failed while creating docker checkpoint", { exportLocation });
+      throw error;
+    }
+  }
+
+  #failCheckpoint(runId: string, error: unknown) {
+    this.#failedCheckpoints.set(runId, error);
+  }
+
+  #clearFailedCheckpoint(runId: string) {
+    this.#failedCheckpoints.delete(runId);
+  }
+
+  #hasFailedCheckpoint(runId: string) {
+    return this.#failedCheckpoints.has(runId);
+  }
+
+  #getRunContainerName(suffix: string, attemptNumber?: number) {
+    return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
+  }
+
+  #createTmpCleaner() {
+    if (!boolFromEnv("TMP_CLEANER_ENABLED", false)) {
+      return;
+    }
+
+    const defaultPaths = [Buildah.tmpDir, Crictl.checkpointDir].filter(Boolean);
+    const pathsOverride = process.env.TMP_CLEANER_PATHS_OVERRIDE?.split(",").filter(Boolean) ?? [];
+    const paths = pathsOverride.length ? pathsOverride : defaultPaths;
+
+    if (paths.length === 0) {
+      this.#logger.error("TempFileCleaner enabled but no paths to clean", {
+        defaultPaths,
+        pathsOverride,
+        TMP_CLEANER_PATHS_OVERRIDE: process.env.TMP_CLEANER_PATHS_OVERRIDE,
+      });
+
+      return;
+    }
+    const cleaner = new TempFileCleaner({
+      paths,
+      maxAgeMinutes: numFromEnv("TMP_CLEANER_MAX_AGE_MINUTES", 60),
+      intervalSeconds: numFromEnv("TMP_CLEANER_INTERVAL_SECONDS", 300),
+      leadingEdge: boolFromEnv("TMP_CLEANER_LEADING_EDGE", false),
+    });
+
+    cleaner.start();
+
+    return cleaner;
+  }
+}

apps/coordinator/src/cleaner.ts

@@ -0,0 +1,106 @@
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import { Exec } from "./exec";
+import { setTimeout } from "timers/promises";
+
+interface TempFileCleanerOptions {
+  paths: string[];
+  maxAgeMinutes: number;
+  intervalSeconds: number;
+  leadingEdge?: boolean;
+}
+
+export class TempFileCleaner {
+  private enabled = false;
+
+  private logger: SimpleStructuredLogger;
+  private exec: Exec;
+
+  constructor(private opts: TempFileCleanerOptions) {
+    this.logger = new SimpleStructuredLogger("tmp-cleaner", undefined, { ...this.opts });
+    this.exec = new Exec({ logger: this.logger });
+  }
+
+  async start() {
+    this.logger.log("TempFileCleaner.start");
+    this.enabled = true;
+
+    if (!this.opts.leadingEdge) {
+      await this.wait();
+    }
+
+    while (this.enabled) {
+      try {
+        await this.clean();
+      } catch (error) {
+        this.logger.error("error during tick", { error });
+      }
+
+      await this.wait();
+    }
+  }
+
+  stop() {
+    this.logger.log("TempFileCleaner.stop");
+    this.enabled = false;
+  }
+
+  private wait() {
+    return setTimeout(this.opts.intervalSeconds * 1000);
+  }
+
+  private async clean() {
+    for (const path of this.opts.paths) {
+      try {
+        await this.cleanSingle(path);
+      } catch (error) {
+        this.logger.error("error while cleaning", { path, error });
+      }
+    }
+  }
+
+  private async cleanSingle(startingPoint: string) {
+    const maxAgeMinutes = this.opts.maxAgeMinutes;
+
+    const ignoreStartingPoint = ["!", "-path", startingPoint];
+    const onlyDirectDescendants = ["-maxdepth", "1"];
+    const onlyOldFiles = ["-mmin", `+${maxAgeMinutes}`];
+
+    const baseArgs = [
+      startingPoint,
+      ...ignoreStartingPoint,
+      ...onlyDirectDescendants,
+      ...onlyOldFiles,
+    ];
+
+    const duArgs = ["-exec", "du", "-ch", "{}", "+"];
+    const rmArgs = ["-exec", "rm", "-rf", "{}", "+"];
+
+    const du = this.x("find", [...baseArgs, ...duArgs]);
+    const duOutput = await du;
+
+    const duLines = duOutput.stdout.trim().split("\n");
+    const fileCount = duLines.length - 1; // last line is the total
+    const fileSize = duLines.at(-1)?.trim().split(/\s+/)[0];
+
+    if (fileCount === 0) {
+      this.logger.log("nothing to delete", { startingPoint, maxAgeMinutes });
+      return;
+    }
+
+    this.logger.log("deleting old files", { fileCount, fileSize, startingPoint, maxAgeMinutes });
+
+    const rm = this.x("find", [...baseArgs, ...rmArgs]);
+    const rmOutput = await rm;
+
+    if (rmOutput.stderr.length > 0) {
+      this.logger.error("delete unsuccessful", { rmOutput });
+      return;
+    }
+
+    this.logger.log("deleted old files", { fileCount, fileSize, startingPoint, maxAgeMinutes });
+  }
+
+  private get x() {
+    return this.exec.x.bind(this.exec);
+  }
+}

apps/coordinator/src/exec.ts

@@ -0,0 +1,293 @@
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import { randomUUID } from "crypto";
+import { homedir } from "os";
+import { type Result, x } from "tinyexec";
+
+class TinyResult {
+  pid?: number;
+  exitCode?: number;
+  aborted: boolean;
+  killed: boolean;
+
+  constructor(result: Result) {
+    this.pid = result.pid;
+    this.exitCode = result.exitCode;
+    this.aborted = result.aborted;
+    this.killed = result.killed;
+  }
+}
+
+interface ExecOptions {
+  logger?: SimpleStructuredLogger;
+  abortSignal?: AbortSignal;
+  logOutput?: boolean;
+  trimArgs?: boolean;
+  neverThrow?: boolean;
+}
+
+export class Exec {
+  private logger: SimpleStructuredLogger;
+  private abortSignal: AbortSignal | undefined;
+
+  private logOutput: boolean;
+  private trimArgs: boolean;
+  private neverThrow: boolean;
+
+  constructor(opts: ExecOptions) {
+    this.logger = opts.logger ?? new SimpleStructuredLogger("exec");
+    this.abortSignal = opts.abortSignal;
+
+    this.logOutput = opts.logOutput ?? true;
+    this.trimArgs = opts.trimArgs ?? true;
+    this.neverThrow = opts.neverThrow ?? false;
+  }
+
+  async x(
+    command: string,
+    args?: string[],
+    opts?: { neverThrow?: boolean; ignoreAbort?: boolean }
+  ) {
+    const argsTrimmed = this.trimArgs ? args?.map((arg) => arg.trim()) : args;
+
+    const commandWithFirstArg = `${command}${argsTrimmed?.length ? ` ${argsTrimmed[0]}` : ""}`;
+    this.logger.debug(`exec: ${commandWithFirstArg}`, { command, args, argsTrimmed });
+
+    const result = x(command, argsTrimmed, {
+      signal: opts?.ignoreAbort ? undefined : this.abortSignal,
+      // We don't use this as it doesn't cover killed and aborted processes
+      // throwOnError: true,
+    });
+
+    const output = await result;
+
+    const metadata = {
+      command,
+      argsRaw: args,
+      argsTrimmed,
+      globalOpts: {
+        trimArgs: this.trimArgs,
+        neverThrow: this.neverThrow,
+        hasAbortSignal: !!this.abortSignal,
+      },
+      localOpts: opts,
+      stdout: output.stdout,
+      stderr: output.stderr,
+      pid: result.pid,
+      exitCode: result.exitCode,
+      aborted: result.aborted,
+      killed: result.killed,
+    };
+
+    if (this.logOutput) {
+      this.logger.debug(`output: ${commandWithFirstArg}`, metadata);
+    }
+
+    if (this.neverThrow || opts?.neverThrow) {
+      return output;
+    }
+
+    if (result.aborted) {
+      this.logger.error(`aborted: ${commandWithFirstArg}`, metadata);
+      throw new TinyResult(result);
+    }
+
+    if (result.killed) {
+      this.logger.error(`killed: ${commandWithFirstArg}`, metadata);
+      throw new TinyResult(result);
+    }
+
+    if (result.exitCode !== 0) {
+      this.logger.error(`non-zero exit: ${commandWithFirstArg}`, metadata);
+      throw new TinyResult(result);
+    }
+
+    return output;
+  }
+
+  static Result = TinyResult;
+}
+
+interface BuildahOptions {
+  id?: string;
+  abortSignal?: AbortSignal;
+}
+
+export class Buildah {
+  private id: string;
+  private logger: SimpleStructuredLogger;
+  private exec: Exec;
+
+  private containers = new Set<string>();
+  private images = new Set<string>();
+
+  constructor(opts: BuildahOptions) {
+    this.id = opts.id ?? randomUUID();
+    this.logger = new SimpleStructuredLogger("buildah", undefined, { id: this.id });
+
+    this.exec = new Exec({
+      logger: this.logger,
+      abortSignal: opts.abortSignal,
+    });
+
+    this.logger.log("initiaized", { opts });
+  }
+
+  private get x() {
+    return this.exec.x.bind(this.exec);
+  }
+
+  async from(baseImage: string) {
+    const output = await this.x("buildah", ["from", baseImage]);
+    this.containers.add(output.stdout);
+    return output;
+  }
+
+  async add(container: string, src: string, dest: string) {
+    return await this.x("buildah", ["add", container, src, dest]);
+  }
+
+  async config(container: string, annotations: string[]) {
+    const args = ["config"];
+
+    for (const annotation of annotations) {
+      args.push(`--annotation=${annotation}`);
+    }
+
+    args.push(container);
+
+    return await this.x("buildah", args);
+  }
+
+  async commit(container: string, imageRef: string) {
+    const output = await this.x("buildah", ["commit", container, imageRef]);
+    this.images.add(output.stdout);
+    return output;
+  }
+
+  async push(imageRef: string, registryTlsVerify?: boolean) {
+    return await this.x("buildah", [
+      "push",
+      `--tls-verify=${String(!!registryTlsVerify)}`,
+      imageRef,
+    ]);
+  }
+
+  async cleanup() {
+    if (this.containers.size > 0) {
+      try {
+        const output = await this.x("buildah", ["rm", ...this.containers], { ignoreAbort: true });
+        this.containers.clear();
+
+        if (output.stderr.length > 0) {
+          this.logger.error("failed to remove some containers", { output });
+        }
+      } catch (error) {
+        this.logger.error("failed to clean up containers", { error, containers: this.containers });
+      }
+    } else {
+      this.logger.debug("no containers to clean up");
+    }
+
+    if (this.images.size > 0) {
+      try {
+        const output = await this.x("buildah", ["rmi", ...this.images], { ignoreAbort: true });
+        this.images.clear();
+
+        if (output.stderr.length > 0) {
+          this.logger.error("failed to remove some images", { output });
+        }
+      } catch (error) {
+        this.logger.error("failed to clean up images", { error, images: this.images });
+      }
+    } else {
+      this.logger.debug("no images to clean up");
+    }
+  }
+
+  static async canLogin(registryHost: string) {
+    try {
+      await x("buildah", ["login", "--get-login", registryHost], { throwOnError: true });
+      return true;
+    } catch (error) {
+      return false;
+    }
+  }
+
+  static get tmpDir() {
+    return process.env.TMPDIR ?? "/var/tmp";
+  }
+
+  static get storageRootDir() {
+    return process.getuid?.() === 0
+      ? "/var/lib/containers/storage"
+      : `${homedir()}/.local/share/containers/storage`;
+  }
+}
+
+interface CrictlOptions {
+  id?: string;
+  abortSignal?: AbortSignal;
+}
+
+export class Crictl {
+  private id: string;
+  private logger: SimpleStructuredLogger;
+  private exec: Exec;
+
+  private archives = new Set<string>();
+
+  constructor(opts: CrictlOptions) {
+    this.id = opts.id ?? randomUUID();
+    this.logger = new SimpleStructuredLogger("crictl", undefined, { id: this.id });
+
+    this.exec = new Exec({
+      logger: this.logger,
+      abortSignal: opts.abortSignal,
+    });
+
+    this.logger.log("initiaized", { opts });
+  }
+
+  private get x() {
+    return this.exec.x.bind(this.exec);
+  }
+
+  async ps(containerName: string, quiet?: boolean) {
+    return await this.x("crictl", ["ps", "--name", containerName, quiet ? "--quiet" : ""]);
+  }
+
+  async checkpoint(containerId: string, exportLocation: string) {
+    const output = await this.x("crictl", [
+      "checkpoint",
+      `--export=${exportLocation}`,
+      containerId,
+    ]);
+    this.archives.add(exportLocation);
+    return output;
+  }
+
+  async cleanup() {
+    if (this.archives.size > 0) {
+      try {
+        const output = await this.x("rm", ["-v", ...this.archives], { ignoreAbort: true });
+        this.archives.clear();
+
+        if (output.stderr.length > 0) {
+          this.logger.error("failed to remove some archives", { output });
+        }
+      } catch (error) {
+        this.logger.error("failed to clean up archives", { error, archives: this.archives });
+      }
+    } else {
+      this.logger.debug("no archives to clean up");
+    }
+  }
+
+  static getExportLocation(identifier: string) {
+    return `${this.checkpointDir}/${identifier}.tar`;
+  }
+
+  static get checkpointDir() {
+    return process.env.CRI_CHECKPOINT_DIR ?? "/checkpoints";
+  }
+}

apps/coordinator/src/util.ts

@@ -0,0 +1,31 @@
+export const boolFromEnv = (env: string, defaultValue: boolean): boolean => {
+  const value = process.env[env];
+
+  if (!value) {
+    return defaultValue;
+  }
+
+  return ["1", "true"].includes(value);
+};
+
+export const numFromEnv = (env: string, defaultValue: number): number => {
+  const value = process.env[env];
+
+  if (!value) {
+    return defaultValue;
+  }
+
+  return parseInt(value, 10);
+};
+
+export function safeJsonParse(json?: string): unknown {
+  if (!json) {
+    return;
+  }
+
+  try {
+    return JSON.parse(json);
+  } catch (e) {
+    return null;
+  }
+}

apps/coordinator/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "es2020",
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "resolveJsonModule": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "paths": {
+      "@trigger.dev/core/v3": ["../../packages/core/src/v3"],
+      "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"]
+    }
+  }
+}

apps/docker-provider/.env.example

@@ -0,0 +1,11 @@
+HTTP_SERVER_PORT=8050
+
+PLATFORM_WS_PORT=3030
+PLATFORM_SECRET=provider-secret
+SECURE_CONNECTION=false
+
+OTEL_EXPORTER_OTLP_ENDPOINT=http://0.0.0.0:3030/otel
+
+# Use this if you are on macOS
+# COORDINATOR_HOST="host.docker.internal"
+# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318"

apps/docker-provider/.gitignore

@@ -0,0 +1,3 @@
+dist/
+node_modules/
+.env

apps/docker-provider/Containerfile

@@ -0,0 +1,47 @@
+FROM node:20-alpine@sha256:7a91aa397f2e2dfbfcdad2e2d72599f374e0b0172be1d86eeb73f1d33f36a4b2 AS node-20-alpine
+
+WORKDIR /app
+
+FROM node-20-alpine AS pruner
+
+COPY --chown=node:node . .
+RUN npx -q turbo@1.10.9 prune --scope=docker-provider --docker
+RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
+
+FROM node-20-alpine AS base
+
+RUN apk add --no-cache dumb-init docker
+
+COPY --chown=node:node .gitignore .gitignore
+COPY --from=pruner --chown=node:node /app/out/json/ .
+COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
+COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
+
+FROM base AS dev-deps
+RUN corepack enable
+ENV NODE_ENV development
+
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
+
+FROM base AS builder
+RUN corepack enable
+
+COPY --from=pruner --chown=node:node /app/out/full/ .
+COPY --from=dev-deps --chown=node:node /app/ .
+COPY --chown=node:node turbo.json turbo.json
+
+RUN pnpm run -r --filter @trigger.dev/core bundle-vendor && pnpm run -r --filter docker-provider build:bundle
+
+FROM base AS runner
+
+RUN corepack enable
+ENV NODE_ENV production
+
+COPY --from=builder --chown=node:node /app/apps/docker-provider/dist/index.mjs ./index.mjs
+
+EXPOSE 8000
+
+USER node
+
+CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]

apps/docker-provider/README.md

@@ -0,0 +1,3 @@
+# Docker provider
+
+The `docker-provider` allows the platform to be orchestrator-agnostic. The platform can perform actions such as `INDEX_TASKS` or `INVOKE_TASK` which the provider translates into Docker actions.

apps/docker-provider/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "docker-provider",
+  "private": true,
+  "version": "0.0.1",
+  "description": "",
+  "main": "dist/index.cjs",
+  "scripts": {
+    "build": "npm run build:bundle",
+    "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"",
+    "build:image": "docker build -f Containerfile . -t docker-provider",
+    "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
+    "start": "tsx src/index.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@trigger.dev/core": "workspace:*",
+    "execa": "^8.0.1"
+  },
+  "devDependencies": {
+    "dotenv": "^16.4.2",
+    "esbuild": "^0.19.11",
+    "tsx": "^4.7.0"
+  }
+}

apps/docker-provider/src/index.ts

@@ -0,0 +1,297 @@
+import { $, type ExecaChildProcess, execa } from "execa";
+import {
+  ProviderShell,
+  TaskOperations,
+  TaskOperationsCreateOptions,
+  TaskOperationsIndexOptions,
+  TaskOperationsRestoreOptions,
+} from "@trigger.dev/core/v3/apps";
+import { SimpleLogger } from "@trigger.dev/core/v3/apps";
+import { isExecaChildProcess } from "@trigger.dev/core/v3/apps";
+import { testDockerCheckpoint } from "@trigger.dev/core/v3/serverOnly";
+import { setTimeout } from "node:timers/promises";
+import { PostStartCauses, PreStopCauses } from "@trigger.dev/core/v3";
+
+const MACHINE_NAME = process.env.MACHINE_NAME || "local";
+const COORDINATOR_PORT = process.env.COORDINATOR_PORT || 8020;
+const COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
+const DOCKER_NETWORK = process.env.DOCKER_NETWORK || "host";
+
+const OTEL_EXPORTER_OTLP_ENDPOINT =
+  process.env.OTEL_EXPORTER_OTLP_ENDPOINT || "http://0.0.0.0:4318";
+
+const FORCE_CHECKPOINT_SIMULATION = ["1", "true"].includes(
+  process.env.FORCE_CHECKPOINT_SIMULATION ?? "false"
+);
+
+const logger = new SimpleLogger(`[${MACHINE_NAME}]`);
+
+type TaskOperationsInitReturn = {
+  canCheckpoint: boolean;
+  willSimulate: boolean;
+};
+
+class DockerTaskOperations implements TaskOperations {
+  #initialized = false;
+  #canCheckpoint = false;
+
+  constructor(private opts = { forceSimulate: false }) {}
+
+  async init(): Promise<TaskOperationsInitReturn> {
+    if (this.#initialized) {
+      return this.#getInitReturn(this.#canCheckpoint);
+    }
+
+    logger.log("Initializing task operations");
+
+    const testCheckpoint = await testDockerCheckpoint();
+
+    if (testCheckpoint.ok) {
+      return this.#getInitReturn(true);
+    }
+
+    logger.error(testCheckpoint.message, testCheckpoint.error);
+    return this.#getInitReturn(false);
+  }
+
+  #getInitReturn(canCheckpoint: boolean): TaskOperationsInitReturn {
+    this.#canCheckpoint = canCheckpoint;
+
+    if (canCheckpoint) {
+      if (!this.#initialized) {
+        logger.log("Full checkpoint support!");
+      }
+    }
+
+    this.#initialized = true;
+
+    const willSimulate = !canCheckpoint || this.opts.forceSimulate;
+
+    if (willSimulate) {
+      logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", {
+        forceSimulate: this.opts.forceSimulate,
+      });
+    }
+
+    return {
+      canCheckpoint,
+      willSimulate,
+    };
+  }
+
+  async index(opts: TaskOperationsIndexOptions) {
+    await this.init();
+
+    const containerName = this.#getIndexContainerName(opts.shortCode);
+
+    logger.log(`Indexing task ${opts.imageRef}`, {
+      host: COORDINATOR_HOST,
+      port: COORDINATOR_PORT,
+    });
+
+    logger.debug(
+      await execa("docker", [
+        "run",
+        `--network=${DOCKER_NETWORK}`,
+        "--rm",
+        `--env=INDEX_TASKS=true`,
+        `--env=TRIGGER_SECRET_KEY=${opts.apiKey}`,
+        `--env=TRIGGER_API_URL=${opts.apiUrl}`,
+        `--env=TRIGGER_ENV_ID=${opts.envId}`,
+        `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
+        `--env=POD_NAME=${containerName}`,
+        `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
+        `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
+        `--name=${containerName}`,
+        `${opts.imageRef}`,
+      ])
+    );
+  }
+
+  async create(opts: TaskOperationsCreateOptions) {
+    await this.init();
+
+    const containerName = this.#getRunContainerName(opts.runId, opts.nextAttemptNumber);
+
+    const runArgs = [
+      "run",
+      `--network=${DOCKER_NETWORK}`,
+      "--detach",
+      `--env=TRIGGER_ENV_ID=${opts.envId}`,
+      `--env=TRIGGER_RUN_ID=${opts.runId}`,
+      `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
+      `--env=POD_NAME=${containerName}`,
+      `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
+      `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
+      `--env=TRIGGER_POD_SCHEDULED_AT_MS=${Date.now()}`,
+      `--name=${containerName}`,
+    ];
+
+    if (process.env.ENFORCE_MACHINE_PRESETS) {
+      runArgs.push(`--cpus=${opts.machine.cpu}`, `--memory=${opts.machine.memory}G`);
+    }
+
+    if (opts.dequeuedAt) {
+      runArgs.push(`--env=TRIGGER_RUN_DEQUEUED_AT_MS=${opts.dequeuedAt}`);
+    }
+
+    runArgs.push(`${opts.image}`);
+
+    try {
+      logger.debug(await execa("docker", runArgs));
+    } catch (error) {
+      if (!isExecaChildProcess(error)) {
+        throw error;
+      }
+
+      logger.error("Create failed:", {
+        opts,
+        exitCode: error.exitCode,
+        escapedCommand: error.escapedCommand,
+        stdout: error.stdout,
+        stderr: error.stderr,
+      });
+    }
+  }
+
+  async restore(opts: TaskOperationsRestoreOptions) {
+    await this.init();
+
+    const containerName = this.#getRunContainerName(opts.runId, opts.attemptNumber);
+
+    if (!this.#canCheckpoint || this.opts.forceSimulate) {
+      logger.log("Simulating restore");
+
+      const unpause = logger.debug(await $`docker unpause ${containerName}`);
+
+      if (unpause.exitCode !== 0) {
+        throw new Error("docker unpause command failed");
+      }
+
+      await this.#sendPostStart(containerName);
+      return;
+    }
+
+    const { exitCode } = logger.debug(
+      await $`docker start --checkpoint=${opts.checkpointRef} ${containerName}`
+    );
+
+    if (exitCode !== 0) {
+      throw new Error("docker start command failed");
+    }
+
+    await this.#sendPostStart(containerName);
+  }
+
+  async delete(opts: { runId: string }) {
+    await this.init();
+
+    const containerName = this.#getRunContainerName(opts.runId);
+    await this.#sendPreStop(containerName);
+
+    logger.log("noop: delete");
+  }
+
+  async get(opts: { runId: string }) {
+    await this.init();
+
+    logger.log("noop: get");
+  }
+
+  #getIndexContainerName(suffix: string) {
+    return `task-index-${suffix}`;
+  }
+
+  #getRunContainerName(suffix: string, attemptNumber?: number) {
+    return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
+  }
+
+  async #sendPostStart(containerName: string): Promise<void> {
+    try {
+      const port = await this.#getHttpServerPort(containerName);
+      logger.debug(await this.#runLifecycleCommand(containerName, port, "postStart", "restore"));
+    } catch (error) {
+      logger.error("postStart error", { error });
+      throw new Error("postStart command failed");
+    }
+  }
+
+  async #sendPreStop(containerName: string): Promise<void> {
+    try {
+      const port = await this.#getHttpServerPort(containerName);
+      logger.debug(await this.#runLifecycleCommand(containerName, port, "preStop", "terminate"));
+    } catch (error) {
+      logger.error("preStop error", { error });
+      throw new Error("preStop command failed");
+    }
+  }
+
+  async #getHttpServerPort(containerName: string): Promise<number> {
+    // We first get the correct port, which is random during dev as we run with host networking and need to avoid clashes
+    // FIXME: Skip this in prod
+    const logs = logger.debug(await $`docker logs ${containerName}`);
+    const matches = logs.stdout.match(/http server listening on port (?<port>[0-9]+)/);
+
+    const port = Number(matches?.groups?.port);
+
+    if (!port) {
+      throw new Error("failed to extract port from logs");
+    }
+
+    return port;
+  }
+
+  async #runLifecycleCommand<THookType extends "postStart" | "preStop">(
+    containerName: string,
+    port: number,
+    type: THookType,
+    cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses,
+    retryCount = 0
+  ): Promise<ExecaChildProcess> {
+    try {
+      return await execa("docker", [
+        "exec",
+        containerName,
+        "busybox",
+        "wget",
+        "-q",
+        "-O-",
+        `127.0.0.1:${port}/${type}?cause=${cause}`,
+      ]);
+    } catch (error: any) {
+      if (type === "postStart" && retryCount < 6) {
+        logger.debug(`retriable ${type} error`, { retryCount, message: error?.message });
+        await setTimeout(exponentialBackoff(retryCount + 1, 2, 50, 1150, 50));
+
+        return this.#runLifecycleCommand(containerName, port, type, cause, retryCount + 1);
+      }
+
+      logger.error(`final ${type} error`, { message: error?.message });
+      throw new Error(`${type} command failed after ${retryCount - 1} retries`);
+    }
+  }
+}
+
+const provider = new ProviderShell({
+  tasks: new DockerTaskOperations({ forceSimulate: FORCE_CHECKPOINT_SIMULATION }),
+  type: "docker",
+});
+
+provider.listen();
+
+function exponentialBackoff(
+  retryCount: number,
+  exponential: number,
+  minDelay: number,
+  maxDelay: number,
+  jitter: number
+): number {
+  // Calculate the delay using the exponential backoff formula
+  const delay = Math.min(Math.pow(exponential, retryCount) * minDelay, maxDelay);
+
+  // Calculate the jitter
+  const jitterValue = Math.random() * jitter;
+
+  // Return the calculated delay with jitter
+  return delay + jitterValue;
+}

apps/docker-provider/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "es2020",
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "paths": {
+      "@trigger.dev/core/v3": ["../../packages/core/src/v3"],
+      "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"]
+    }
+  }
+}

apps/kubernetes-provider/.env.example

@@ -0,0 +1,9 @@
+HTTP_SERVER_PORT=8060
+
+PLATFORM_WS_PORT=3030
+PLATFORM_SECRET=provider-secret
+SECURE_CONNECTION=false
+
+# Use this if you are on macOS
+# COORDINATOR_HOST="host.docker.internal"
+# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318"

apps/kubernetes-provider/.gitignore

@@ -0,0 +1,3 @@
+dist/
+node_modules/
+.env