Fix pnpm lock file

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.changeset/angry-eagles-trade.md

@@ -0,0 +1,9 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+- Fix additionalFiles that aren't decendants
+- Stop swallowing uncaught exceptions in prod
+- Improve warnings and errors, fail early on critical warnings
+- New arg to --save-logs even for successful builds

.changeset/beige-pens-dance.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+v3 CLI update command and package manager detection fix

.changeset/breezy-gorillas-mate.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+better handle task metadata parse errors, and display nicely formatted errors

.changeset/chilled-hornets-move.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Vastly improved dev command output

.changeset/clean-pianos-listen.md

@@ -0,0 +1,8 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+add machine config and secure zod connection

.changeset/config.json

@@ -1,15 +1,11 @@
 {
   "$schema": "https://unpkg.com/@changesets/config@2.2.0/schema.json",
-  "changelog": [
-    "@remix-run/changelog-github",
-    {
-      "repo": "triggerdotdev/trigger.dev"
-    }
-  ],
+  "changelog": "@changesets/cli/changelog",
   "commit": false,
   "fixed": [
     [
-      "@trigger.dev/*"
+      "@trigger.dev/*",
+      "trigger.dev"
     ]
   ],
   "linked": [],
@@ -19,7 +15,12 @@
   "ignore": [
     "webapp",
     "emails",
-    "@trigger.dev/database"
+    "proxy",
+    "yalt",
+    "@trigger.dev/database",
+    "coordinator",
+    "docker-provider",
+    "kubernetes-provider"
   ],
   "___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH": {
     "onlyUpdatePeerDependentsWhenOutOfRange": true

.changeset/cool-glasses-bake.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Configurable log levels in the config file and via env var

.changeset/cuddly-feet-approve.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Don’t swallow some error messages when deploying

.changeset/dry-walls-check.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Add option to print console logs in the dev CLI locally (issue #1014)

.changeset/eight-pumas-float.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Fixed batch otel flushing

.changeset/few-students-share.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix permissions inside node_modules

.changeset/green-bags-wink.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/react": patch
+---
+
+Fix for shared queryKey between useRunDetails and useRunStatuses

.changeset/khaki-apricots-design.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Retry 429, 500, and connection error API requests to the trigger.dev server

.changeset/khaki-poems-lay.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Export queue from the SDK

.changeset/late-icons-lie.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Display errors for runs and deployments

.changeset/late-steaks-behave.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+v3: fix digest extraction

.changeset/lemon-jobs-repair.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/nestjs": patch
+---
+
+fix: [nestjs integration] fastify HTTP adapter detection now works correctly for response headers

.changeset/light-bulldogs-press.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Changed "Worker" to "Version" in the dev command key

.changeset/light-dragons-complain.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Correctly handle self-hosted deploy command errors

.changeset/loud-actors-remember.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Default to retrying enabled in dev when running init

.changeset/many-ligers-pump.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Handle string and non-stringifiable outputs like functions

.changeset/mighty-camels-joke.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Improve error messages during dev/deploy and handle deploy image build issues

.changeset/new-rivers-tell.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Added replayRun function to the SDK

.changeset/ninety-pets-travel.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Improve the SDK function types and expose a new APIError instead of the APIResult type

.changeset/odd-poets-own.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Added a Node.js runtime check for the CLI

.changeset/polite-ducks-switch.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix post start hooks

.changeset/poor-flowers-cross.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Use the dashboard url instead of the API url for the View logs link

.changeset/pre.json

@@ -0,0 +1,98 @@
+{
+  "mode": "pre",
+  "tag": "beta",
+  "initialVersions": {
+    "coordinator": "0.0.1",
+    "docker-provider": "0.0.1",
+    "kubernetes-provider": "0.0.1",
+    "proxy": "0.0.11",
+    "webapp": "1.0.0",
+    "yalt": "0.0.1",
+    "@trigger.dev/airtable": "2.3.18",
+    "@trigger.dev/github": "2.3.18",
+    "@trigger.dev/linear": "2.3.18",
+    "@trigger.dev/openai": "2.3.18",
+    "@trigger.dev/plain": "2.3.18",
+    "@trigger.dev/replicate": "2.3.18",
+    "@trigger.dev/resend": "2.3.18",
+    "@trigger.dev/sendgrid": "2.3.18",
+    "@trigger.dev/shopify": "2.3.18",
+    "@trigger.dev/slack": "2.3.18",
+    "@trigger.dev/stripe": "2.3.18",
+    "@trigger.dev/supabase": "2.3.18",
+    "@trigger.dev/typeform": "2.3.18",
+    "@trigger.dev/astro": "2.3.18",
+    "@trigger.dev/cli": "2.3.18",
+    "trigger.dev": "2.3.18",
+    "@trigger.dev/core": "2.3.18",
+    "@trigger.dev/core-apps": "0.0.0",
+    "@trigger.dev/core-backend": "2.3.18",
+    "@trigger.dev/database": "0.0.1",
+    "emails": "1.0.0",
+    "@trigger.dev/eslint-plugin": "2.3.18",
+    "@trigger.dev/express": "2.3.18",
+    "@trigger.dev/hono": "2.3.18",
+    "@trigger.dev/integration-kit": "2.3.18",
+    "@trigger.dev/nestjs": "2.3.18",
+    "@trigger.dev/nextjs": "2.3.18",
+    "@trigger.dev/otlp-importer": "2.3.11",
+    "@trigger.dev/react": "2.3.18",
+    "@trigger.dev/remix": "2.3.18",
+    "@trigger.dev/sveltekit": "2.3.18",
+    "@trigger.dev/testing": "2.3.18",
+    "@trigger.dev/sdk": "2.3.18",
+    "@trigger.dev/yalt": "2.3.18"
+  },
+  "changesets": [
+    "angry-eagles-trade",
+    "beige-pens-dance",
+    "breezy-gorillas-mate",
+    "chilled-hornets-move",
+    "clean-pianos-listen",
+    "cool-glasses-bake",
+    "cuddly-feet-approve",
+    "dry-walls-check",
+    "eight-pumas-float",
+    "few-students-share",
+    "green-bags-wink",
+    "khaki-apricots-design",
+    "khaki-poems-lay",
+    "late-icons-lie",
+    "late-steaks-behave",
+    "lemon-jobs-repair",
+    "light-bulldogs-press",
+    "light-dragons-complain",
+    "loud-actors-remember",
+    "many-ligers-pump",
+    "mighty-camels-joke",
+    "new-rivers-tell",
+    "ninety-pets-travel",
+    "odd-poets-own",
+    "polite-ducks-switch",
+    "poor-flowers-cross",
+    "rare-roses-float",
+    "real-planets-stare",
+    "rotten-dryers-exercise",
+    "shaggy-spoons-taste",
+    "sharp-emus-compare",
+    "sharp-zebras-serve",
+    "shiny-coats-cry",
+    "silly-suits-switch",
+    "smart-needles-move",
+    "smart-olives-eat",
+    "spicy-lamps-smoke",
+    "strange-ghosts-matter",
+    "strong-lemons-add",
+    "stupid-bulldogs-applaud",
+    "sweet-lizards-press",
+    "swift-dragons-peel",
+    "tall-bees-wave",
+    "tame-guests-know",
+    "tender-oranges-rhyme",
+    "tidy-balloons-suffer",
+    "tidy-dryers-sleep",
+    "tiny-doors-type",
+    "tiny-elephants-scream",
+    "tricky-bulldogs-heal"
+  ]
+}

.changeset/rare-roses-float.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Add openssl to prod worker image and allow passing auth token via env var for deploy

.changeset/real-planets-stare.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Fixed incorrect span timings around checkpoints by implementing a precise wall clock that resets after restores

.changeset/rotten-dryers-exercise.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Adding task with a triggerSource of schedule

.changeset/shaggy-spoons-taste.md

@@ -0,0 +1,56 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Updates the `trigger`, `batchTrigger` and their `*AndWait` variants to use the first parameter for the payload/items, and the second parameter for options.
+
+Before:
+
+```ts
+await yourTask.trigger({ payload: { foo: "bar" }, options: { idempotencyKey: "key_1234" } });
+await yourTask.triggerAndWait({ payload: { foo: "bar" }, options: { idempotencyKey: "key_1234" } });
+
+await yourTask.batchTrigger({ items: [{ payload: { foo: "bar" } }, { payload: { foo: "baz" } }] });
+await yourTask.batchTriggerAndWait({ items: [{ payload: { foo: "bar" } }, { payload: { foo: "baz" } }] });
+```
+
+After:
+
+```ts
+await yourTask.trigger({ foo: "bar" }, { idempotencyKey: "key_1234" });
+await yourTask.triggerAndWait({ foo: "bar" }, { idempotencyKey: "key_1234" });
+
+await yourTask.batchTrigger([{ payload: { foo: "bar" } }, { payload: { foo: "baz" } }]);
+await yourTask.batchTriggerAndWait([{ payload: { foo: "bar" } }, { payload: { foo: "baz" } }]);
+```
+
+We've also changed the API of the `triggerAndWait` result. Before, if the subtask that was triggered finished with an error, we would automatically "rethrow" the error in the parent task.
+
+Now instead we're returning a `TaskRunResult` object that allows you to discriminate between successful and failed runs in the subtask:
+
+Before:
+
+```ts
+try {
+  const result = await yourTask.triggerAndWait({ foo: "bar" });
+
+  // result is the output of your task
+  console.log("result", result);
+
+} catch (error) {
+  // handle subtask errors here
+}
+```
+
+After:
+
+```ts
+const result = await yourTask.triggerAndWait({ foo: "bar" });
+
+if (result.ok) {
+  console.log(`Run ${result.id} succeeded with output`, result.output);
+} else {
+  console.log(`Run ${result.id} failed with error`, result.error);
+}
+```

.changeset/sharp-emus-compare.md

@@ -0,0 +1,13 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+When using idempotency keys, triggerAndWait and batchTriggerAndWait will still work even if the existing runs have already been completed (or even partially completed, in the case of batchTriggerAndWait)
+
+- TaskRunExecutionResult.id is now the run friendlyId, not the attempt friendlyId
+- A single TaskRun can now have many batchItems, in the case of batchTriggerAndWait while using idempotency keys
+- A run’s idempotencyKey is now added to the ctx as well as the TaskEvent and displayed in the span view
+- When resolving batchTriggerAndWait, the runtimes no longer reject promises, leading to an error in the parent task
+

.changeset/sharp-zebras-serve.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Update trigger.dev CLI for new batch otel support

.changeset/shiny-coats-cry.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Add additional logging around cleaning up dev workers, and always kill them after 5 seconds if they haven't already exited

.changeset/silly-suits-switch.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/otlp-importer": patch
+"trigger.dev": patch
+---
+
+Fix package builds and CLI commands on Windows

.changeset/smart-needles-move.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Remove unimplemented batchOptions

.changeset/smart-olives-eat.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix CLI logout and add list-profiles command

.changeset/spicy-lamps-smoke.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fixing an issue with bundling @trigger.dev/core/v3 in dev when using pnpm

.changeset/strange-ghosts-matter.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Added DEBUG to the ignored env vars

.changeset/strong-lemons-add.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Make optional schedule object fields nullish

.changeset/stupid-bulldogs-applaud.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fixed an issue where the trigger.dev package was not being built before publishing to npm

.changeset/sweet-lizards-press.md

@@ -0,0 +1,8 @@
+---
+"trigger.dev": major
+"@trigger.dev/core": major
+"@trigger.dev/otlp-importer": major
+"@trigger.dev/sdk": major
+---
+
+Updates to support Trigger.dev v3

.changeset/swift-dragons-peel.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Added JSDocs to the schedule SDK types

.changeset/tall-bees-wave.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Dynamically import superjson and fix some bundling issues

.changeset/tame-guests-know.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Changed the binary name from trigger.dev to triggerdev to fix a Windows issue

.changeset/tender-oranges-rhyme.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Stop swallowing deployment errors and display them better

.changeset/tidy-balloons-suffer.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Init command was failing on Windows because of bad template paths

.changeset/tidy-dryers-sleep.md

@@ -0,0 +1,10 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Fixes an issue that caused failed tasks when resuming after calling `triggerAndWait` or `batchTriggerAndWait` in prod/staging (this doesn't effect dev).
+
+The version of Node.js we use for deployed workers (latest 20) would crash with an out-of-memory error when the checkpoint was restored. This crash does not happen on Node 18x or Node21x, so we've decided to upgrade the worker version to Node.js21x, to mitigate this issue.
+
+You'll need to re-deploy to production to fix the issue.

.changeset/tiny-doors-type.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Added cancelRun to the SDK

.changeset/tiny-elephants-scream.md

@@ -0,0 +1,7 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+- Add graceful exit for prod workers
+- Prevent overflow in long waits

.changeset/tricky-bulldogs-heal.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Added a new global - Task Catalog - to better handle task metadata

.dockerignore

@@ -1,35 +1,43 @@
-*.log
+\*.log
 .git
 .github
+
 # editor
+
 .idea
 .vscode
+
 # dependencies
+
 node_modules
 .pnp
 .pnp.js
 
 # testing
+
 coverage
 
 # next.js
+
 .next/
 build
 
 # packages
+
 build
 dist
-packages/**/dist
+packages/\*\*/dist
 
 # misc
+
 .DS_Store
-*.pem
+\*.pem
 
 .turbo
 .vercel
 .cache
 .output
-apps/**/public/build
+apps/\*\*/public/build
 
 cypress/screenshots
 cypress/videos
@@ -38,6 +46,7 @@ apps/**/styles/tailwind.css
 packages/**/styles/tailwind.css
 
 .changeset
+references
 examples
 CHANGESETS.md
 CONTRIBUTING.md

.env.example

@@ -12,7 +12,14 @@ APP_ENV=development
 APP_ORIGIN=http://localhost:3030
 NODE_ENV=development
 
+# Redis is used for concurrency control
+# REDIS_HOST="localhost"
+# REDIS_PORT="6379"
+# REDIS_TLS_DISABLED="true"
+
 # OPTIONAL VARIABLES
+# This is used for validating emails that are allowed to log in. Every email that do not match this regex will be rejected.
+# WHITELISTED_EMAILS="authorized@yahoo\.com|authorized@gmail\.com"
 # This is used for logging in via GitHub. You can leave these commented out if you don't want to use GitHub for authentication.
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
@@ -24,6 +31,9 @@ NODE_ENV=development
 # FROM_EMAIL=
 # REPLY_TO_EMAIL=
 
+# Remove the following line to enable logging telemetry traces to the console
+LOG_TELEMETRY="false"
+
 # CLOUD VARIABLES
 POSTHOG_PROJECT_KEY=
 PLAIN_API_KEY= 
@@ -31,6 +41,35 @@ CLOUD_AIRTABLE_CLIENT_ID=
 CLOUD_AIRTABLE_CLIENT_SECRET=
 CLOUD_GITHUB_CLIENT_ID=
 CLOUD_GITHUB_CLIENT_SECRET=
+CLOUD_LINEAR_CLIENT_ID=
+CLOUD_LINEAR_CLIENT_SECRET=
 CLOUD_SLACK_APP_HOST=
 CLOUD_SLACK_CLIENT_ID=
-CLOUD_SLACK_CLIENT_SECRET= 
+CLOUD_SLACK_CLIENT_SECRET= 
+
+# v3 variables
+PROVIDER_SECRET=provider-secret # generate the actual secret with `openssl rand -hex 32`
+COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl rand -hex 32`
+
+# Uncomment the following line to enable the registry proxy
+# ENABLE_REGISTRY_PROXY=true
+# DEPOT_TOKEN=<Depot org token>
+# DEPOT_PROJECT_ID=<Depot project id>
+# DEPLOY_REGISTRY_HOST=${APP_ORIGIN} # This is the host that the deploy CLI will use to push images to the registry
+# CONTAINER_REGISTRY_ORIGIN=<Container registry origin e.g. https://registry.digitalocean.com>
+# CONTAINER_REGISTRY_USERNAME=<Container registry username e.g. Digital ocean email address>
+# CONTAINER_REGISTRY_PASSWORD=<Container registry password e.g. Digital ocean PAT>
+# DEV_OTEL_EXPORTER_OTLP_ENDPOINT="http://0.0.0.0:4318"
+# These are needed for the object store (for handling large payloads/outputs)
+# OBJECT_STORE_BASE_URL="https://{bucket}.{accountId}.r2.cloudflarestorage.com"
+# OBJECT_STORE_ACCESS_KEY_ID=
+# OBJECT_STORE_SECRET_ACCESS_KEY=
+# RUNTIME_WAIT_THRESHOLD_IN_MS=10000
+
+# These control the server-side internal telemetry
+# INTERNAL_OTEL_TRACE_EXPORTER_URL=<URL to send traces to>
+# INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_NAME=<Header name for the auth token>
+# INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE=<Auth token value>
+# INTERNAL_OTEL_TRACE_LOGGING_ENABLED=1
+# INTERNAL_OTEL_TRACE_SAMPING_RATE=20 # this means 1/20 traces or 5% of traces will be sampled (sampled = recorded)
+# INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED=0,

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,38 @@
+name: 🐞 Bug Report
+description: Create a bug report to help us improve
+title: "bug: "
+labels: ["🐞 unconfirmed bug"]
+body:
+  - type: textarea
+    attributes:
+      label: Provide environment information
+      description: |
+        Run this command in your project root and paste the results:
+        ```bash
+        npx envinfo --system --binaries
+        ```
+
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of the bug, as well as what you expected to happen when encountering it.
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Reproduction repo
+      description: If applicable, please provide a link to a reproduction repo or a Stackblitz / CodeSandbox project. Your issue may be closed if this is not provided and we are unable to reproduce the issue. If your bug is a docs issue, link the appropriate page.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: To reproduce
+      description: Describe how to reproduce your bug. Steps, code snippets, reproduction repos etc.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Additional information
+      description: Add any other information related to the bug here, screenshots if applicable.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Ask a Question
+    url: https://trigger.dev/discord
+    about: Ask questions and discuss with other community members

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,27 @@
+name: Feature Request
+description: Suggest an idea for this project
+title: "feat: "
+labels: ["🌟 enhancement"]
+body:
+  - type: textarea
+    attributes:
+      label: Is your feature request related to a problem? Please describe.
+      description: A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Describe the solution you'd like to see
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Describe alternate solutions
+      description: A clear and concise description of any alternative solutions or features you've considered.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Additional information
+      description: Add any other information related to the feature here. If your feature request is related to any issues or discussions, link them here.

.github/ISSUE_TEMPLATE/instrumentation_request.yml

@@ -0,0 +1,21 @@
+name: OpenTelemetry Auto-Instrumentation Request
+description: Suggest an SDK that you'd like to be auto-instrumented in the Run log view
+title: "auto-instrumentation: "
+labels: ["🌟 enhancement"]
+body:
+  - type: textarea
+    attributes:
+      label: What API or SDK would you to have automatic spans for?
+      description: A clear description of which API or SDK you'd like, and links to it.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Is there an existing OpenTelemetry auto-instrumentation package?
+      description: You can search for existing ones – https://opentelemetry.io/ecosystem/registry/?component=instrumentation&language=js
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Additional information
+      description: Add any other information related to the feature here. If your feature request is related to any issues or discussions, link them here.

.github/labeler.yml

@@ -0,0 +1,12 @@
+"📌 area: cli":
+  - any: ["cli/**/*"]
+
+"📌 area: t3-app":
+  - any: ["cli/template/**/*"]
+
+"📚 documentation":
+  - any: ["www/**/*"]
+  - any: ["**/*.md"]
+
+"📌 area: ci":
+  - any: [".github/**/*"]

.github/pull_request_template.md

@@ -0,0 +1,27 @@
+Closes #<issue>
+
+## ✅ Checklist
+
+- [ ] I have followed every step in the [contributing guide](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md)
+- [ ] The PR title follows the convention.
+- [ ] I ran and tested the code works
+
+---
+
+## Testing
+
+_[Describe the steps you took to test this change]_
+
+---
+
+## Changelog
+
+_[Short description of what has changed]_
+
+---
+
+## Screenshots
+
+_[Screenshots]_
+
+💯

.github/workflows/e2e.yml

@@ -0,0 +1,74 @@
+name: "🧪 E2E Tests"
+on:
+  workflow_call:
+jobs:
+  e2e:
+    name: "🧪 E2E Tests"
+    runs-on: buildjet-16vcpu-ubuntu-2204
+    steps:
+      - name: 🐳 Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME || vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN || vars.DOCKERHUB_TOKEN }}
+
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: recursive
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v2.2.4
+        with:
+          version: 8.15.5
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v3
+        with:
+          node-version: 20.11.1
+          cache: "pnpm"
+
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: Install Playwright Browsers
+        run: npx playwright install --with-deps
+
+      - name: Run Playwright tests
+        run: |
+          # Setup environment variables
+          cp ./.env.example ./.env
+          cp ./references/nextjs-test/.env.example ./references/nextjs-test/.env.local
+
+          # Build packages
+          pnpm run build --filter @references/nextjs-test^...
+          cd apps/webapp && pnpm run build:server
+          cd ../..
+          pnpm --filter @trigger.dev/database generate
+          pnpm --filter @trigger.dev/otlp-importer generate
+
+          # Move trigger-cli bin to correct place
+          pnpm install --frozen-lockfile
+
+          # Execute tests
+          pnpm run docker
+          pnpm run db:migrate
+          pnpm run db:seed
+          pnpm run test:e2e
+
+          # Cleanup
+          pnpm run docker:stop
+
+      - name: Upload Playwright report
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30

.github/workflows/pr_checks.yml

@@ -0,0 +1,32 @@
+name: 🤖 PR Checks
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - "**.md"
+      - "**.mdx"
+      - ".github/CODEOWNERS"
+      - ".github/ISSUE_TEMPLATE/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  typecheck:
+    uses: ./.github/workflows/typecheck.yml
+    secrets: inherit
+
+  units:
+    uses: ./.github/workflows/unit-tests.yml
+    secrets: inherit
+
+  e2e:
+    uses: ./.github/workflows/e2e.yml
+    secrets: inherit

.github/workflows/publish-docker.yml

@@ -0,0 +1,59 @@
+name: "🐳 Publish Docker"
+on:
+  workflow_call:
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.get_version.outputs.version }}
+      short_sha: ${{ steps.get_commit.outputs.sha_short }}
+    steps:
+      - name: Setup Depot CLI
+        uses: depot/setup-action@v1
+        
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: 🆚 Get the version
+        id: get_version
+        run: |
+          IMAGE_TAG="${GITHUB_REF#refs/tags/}"
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            if [[ $IMAGE_TAG == v.docker.* ]]; then
+              ORIGINAL_VERSION="${IMAGE_TAG#v.docker.}"
+              IMAGE_TAG="v${ORIGINAL_VERSION}"
+            elif [[ $IMAGE_TAG == build-* ]]; then
+              IMAGE_TAG="${IMAGE_TAG#build-}"
+            fi
+            echo "IMAGE_TAG=${IMAGE_TAG}"
+          elif [[ $GITHUB_REF == refs/heads/main ]]; then
+            # Handle main branch specifically
+            IMAGE_TAG="main"
+            echo "IMAGE_TAG=${IMAGE_TAG}"
+          else
+            echo "Invalid reference: ${GITHUB_REF}"
+            exit 1
+          fi
+          echo "::set-output name=version::${IMAGE_TAG}"
+      - name: 🔢 Get the commit hash
+        id: get_commit
+        run: |
+          echo ::set-output name=sha_short::$(echo ${{ github.sha }} | cut -c1-7)
+
+      - name: 🐙 Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 🐳 Build image and push to GitHub Container Registry
+        uses: depot/build-push-action@v1
+        with:
+          file: ./docker/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          tags: |
+            ghcr.io/triggerdotdev/trigger.dev:${{ steps.get_version.outputs.version }}
+          push: true

.github/workflows/publish-infra.yml

@@ -0,0 +1,94 @@
+name: "🚢 Publish Infra Images"
+
+on:
+  push:
+    tags:
+      - "infra-dev-*"
+      - "infra-test-*"
+      - "infra-prod-*"
+    paths:
+      - ".github/workflows/publish.yml"
+      - "packages/**"
+      - "!packages/**/*.md"
+      - "!packages/**/*.eslintrc"
+      - "apps/**"
+      - "!apps/**/*.md"
+      - "!apps/**/*.eslintrc"
+      - "integrations/**"
+      - "!integrations/**/*.md"
+      - "!integrations/**/*.eslintrc"
+      - "pnpm-lock.yaml"
+      - "pnpm-workspace.yaml"
+      - "turbo.json"
+      - "docker/Dockerfile"
+      - "docker/scripts/**"
+      - "tests/**"
+
+permissions:
+  id-token: write
+  packages: write
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+env:
+  AWS_REGION: us-east-1
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        package: [coordinator, kubernetes-provider]
+    runs-on: buildjet-16vcpu-ubuntu-2204
+    env:
+      DOCKER_BUILDKIT: "1"
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Generate image reference
+        id: prep
+        # WARNING: This step expects the workflow to have been triggered by a specific tag format of: infra-${env}-*
+        run: |
+          env=$(echo ${{ github.ref_name }} | cut -d- -f2)
+          sha=${GITHUB_SHA::7}
+          ts=$(date +%s)
+          if [[ "${{ matrix.package }}" == *-provider ]]; then
+            provider_type=$(echo ${{ matrix.package }} | cut -d- -f1)
+            repository=provider/${provider_type}
+          else
+            repository=${{ matrix.package }}
+          fi
+          echo "IMAGE_TAG=${env}-${sha}-${ts}" >> "$GITHUB_OUTPUT"
+          echo "REPOSITORY=${repository}" >> "$GITHUB_OUTPUT"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # ..to avoid rate limits when pulling images
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: 🚢 Build Container Image
+        run: |
+          docker build -t infra_image -f ./apps/${{ matrix.package }}/Containerfile .
+
+      # ..to push image
+      - name: 🐙 Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 🐙 Push to GitHub Container Registry
+        run: |
+          docker tag infra_image $REGISTRY/$REPOSITORY:$IMAGE_TAG
+          docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
+        env:
+          REGISTRY: ghcr.io/triggerdotdev
+          REPOSITORY: ${{ steps.prep.outputs.REPOSITORY }}
+          IMAGE_TAG: ${{ steps.prep.outputs.IMAGE_TAG }}

.github/workflows/publish.yml

@@ -4,11 +4,15 @@ on:
   push:
     branches:
       - main
-      - improvements/*
     tags:
       - "v.docker.*"
+      - "build-*"
     paths:
       - ".github/workflows/publish.yml"
+      - ".github/workflows/typecheck.yml"
+      - ".github/workflows/unit-tests.yml"
+      - ".github/workflows/e2e.yml"
+      - ".github/workflows/publish-docker.yml"
       - "packages/**"
       - "!packages/**/*.md"
       - "!packages/**/*.eslintrc"
@@ -23,6 +27,7 @@ on:
       - "turbo.json"
       - "docker/Dockerfile"
       - "docker/scripts/**"
+      - "tests/**"
 
 permissions:
   id-token: write
@@ -37,190 +42,18 @@ env:
 
 jobs:
   typecheck:
-    name: ʦ TypeScript
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    steps:
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
+    uses: ./.github/workflows/typecheck.yml
+    secrets: inherit
 
-      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
-        with:
-          version: 7.18
-
-      - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
-        with:
-          node-version: 18
-          cache: "pnpm"
-
-      - name: 📥 Download deps
-        run: pnpm install --frozen-lockfile
-
-      - name: 📀 Generate Prisma Client
-        run: pnpm run generate
-
-      - name: 🔎 Type check
-        run: pnpm run typecheck --filter webapp
-
-  unitTests:
-    name: Unit Tests
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    steps:
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
-        with:
-          version: 7.18
-
-      - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
-        with:
-          node-version: 18
-          cache: "pnpm"
-
-      - name: 📥 Download deps
-        run: pnpm install --frozen-lockfile
-
-      - name: Run Unit Tests
-        run: |
-          pnpm run test
+  units:
+    uses: ./.github/workflows/unit-tests.yml
+    secrets: inherit
 
   e2e:
-    name: e2e Tests
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    steps:
-      - name: 🐳 Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
-        with:
-          version: 7.18
-
-      - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
-        with:
-          node-version: 18
-          cache: "pnpm"
-
-      - name: 📥 Download deps
-        run: pnpm install --frozen-lockfile
-
-      - name: Install Playwright Browsers
-        run: npx playwright install --with-deps
-
-      - name: Run Playwright tests
-        run: |
-          # Setup environment variables
-          cp ./.env.example ./.env
-          cp ./examples/nextjs-test/.env.example ./examples/nextjs-test/.env.local
-
-          # Build packages
-          pnpm run build --filter @examples/nextjs-test^...
-          pnpm --filter @trigger.dev/database generate
-
-          # Move trigger-cli bin to correct place
-          pnpm install --frozen-lockfile
-
-          # Execute tests
-          pnpm run docker
-          pnpm run db:migrate
-          pnpm run db:seed
-          pnpm run test:e2e
-
-          # Cleanup
-          pnpm run docker:stop
-
-      - name: Upload Playwright report
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: playwright-report
-          path: playwright-report/
-          retention-days: 30
+    uses: ./.github/workflows/e2e.yml
+    secrets: inherit
 
   publish:
-    needs: [typecheck, unitTests, e2e]
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    outputs:
-      version: ${{ steps.get_version.outputs.version }}
-      short_sha: ${{ steps.get_commit.outputs.sha_short }}
-    steps:
-      - name: 🐳 Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
-
-      - name: 🆚 Get the version
-        id: get_version
-        run: |
-          IMAGE_TAG="${GITHUB_REF#refs/tags/}"
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-            if [[ $IMAGE_TAG == v.docker.* ]]; then
-              ORIGINAL_VERSION="${IMAGE_TAG#v.docker.}"
-              IMAGE_TAG="v${ORIGINAL_VERSION}"
-            fi
-            echo "IMAGE_TAG=${IMAGE_TAG}"
-          elif [[ $GITHUB_REF == refs/heads/improvements/* ]]; then
-            ORIGINAL_VERSION="${GITHUB_REF#refs/heads/improvements/}"
-            IMAGE_TAG="${ORIGINAL_VERSION}.rc"
-            echo "IMAGE_TAG=${IMAGE_TAG}"
-          elif [[ $GITHUB_REF == refs/heads/* ]]; then
-            IMAGE_TAG="${GITHUB_REF#refs/heads/}"
-            echo "IMAGE_TAG=${IMAGE_TAG}"
-          else
-            echo "Invalid reference: ${GITHUB_REF}"
-            exit 1
-          fi
-          echo "::set-output name=version::${IMAGE_TAG}"
-      - name: 🔢 Get the commit hash
-        id: get_commit
-        run: |
-          echo ::set-output name=sha_short::$(echo ${{ github.sha }} | cut -c1-7)
-
-      - name: 🐳 Build Docker Image
-        run: |
-          docker build -t release_build_image -f ./docker/Dockerfile .
-
-      - name: 🐙 Login to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: 🐙 Push to GitHub Container Registry
-        run: |
-          docker tag release_build_image $REGISTRY/$REPOSITORY:$IMAGE_TAG
-          docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
-        env:
-          REGISTRY: ghcr.io/triggerdotdev
-          REPOSITORY: trigger.dev
-          IMAGE_TAG: ${{ steps.get_version.outputs.version }}
-
-      - name: 🐙 Push 'latest' to GitHub Container Registry
-        if: startsWith(github.ref, 'refs/tags/')
-        run: |
-          docker tag release_build_image $REGISTRY/$REPOSITORY:latest
-          docker push $REGISTRY/$REPOSITORY:latest
-        env:
-          REGISTRY: ghcr.io/triggerdotdev
-          REPOSITORY: trigger.dev
+    needs: [typecheck, units, e2e]
+    uses: ./.github/workflows/publish-docker.yml
+    secrets: inherit

.github/workflows/release.yml

@@ -4,21 +4,15 @@ on:
   push:
     branches:
       - main
-    paths:
-      - ".github/workflows/release.yml"
-      - "packages/**"
-      - "!packages/**/*.md"
-      - ".changeset/**"
-      - "integrations/**"
-      - "!integrations/**/*.md"
-      - "pnpm-lock.yaml"
-      - "pnpm-workspace.yaml"
-      - "turbo.json"
+    paths-ignore:
+      - "**.md"
+      - ".github/CODEOWNERS"
+      - ".github/ISSUE_TEMPLATE/**"
 
 jobs:
   release:
     name: 🦋 Changesets Release
-    runs-on: buildjet-4vcpu-ubuntu-2204
+    runs-on: buildjet-8vcpu-ubuntu-2204
     if: |
       github.repository == 'triggerdotdev/trigger.dev'
     outputs:
@@ -36,14 +30,19 @@ jobs:
       - name: ⎔ Setup pnpm
         uses: pnpm/action-setup@v2.2.4
         with:
-          version: 7.18
+          version: 8.15.5
 
       - name: ⎔ Setup node
         uses: buildjet/setup-node@v3
         with:
-          node-version: 18
+          node-version: 20.11.1
           cache: "pnpm"
 
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
       - name: 📥 Download deps
         run: pnpm install --frozen-lockfile
 
@@ -51,7 +50,7 @@ jobs:
         run: pnpm run generate
 
       - name: 🔎 Type check
-        run: pnpm run typecheck
+        run: pnpm run typecheck --filter "@trigger.dev/*" --filter "trigger.dev"
 
       - name: 🔐 Setup npm auth
         run: |

.github/workflows/typecheck.yml

@@ -0,0 +1,37 @@
+name: "ʦ TypeScript"
+on:
+  workflow_call:
+jobs:
+  typecheck:
+    runs-on: buildjet-8vcpu-ubuntu-2204
+
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v2.2.4
+        with:
+          version: 8.15.5
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v3
+        with:
+          node-version: 20.11.1
+          cache: "pnpm"
+
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: 🔎 Type check
+        run: pnpm run typecheck --filter webapp

.github/workflows/unit-tests.yml

@@ -0,0 +1,45 @@
+name: "🧪 Unit Tests"
+on:
+  workflow_call:
+jobs:
+  unitTests:
+    name: "🧪 Unit Tests"
+    runs-on: buildjet-8vcpu-ubuntu-2204
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v2.2.4
+        with:
+          version: 8.15.5
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v3
+        with:
+          node-version: 20.11.1
+          cache: "pnpm"
+
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: ⎔ Setup Deno
+        uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
+
+      - name: ⎔ Setup bun
+        uses: oven-sh/setup-bun@v1
+        with:
+          bun-version: "1.0.15"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: Run Unit Tests
+        run: |
+          pnpm run test

.gitignore

@@ -52,3 +52,6 @@ apps/**/public/build
 /test-results/
 /playwright-report/
 /playwright/.cache/
+
+.cosine
+.trigger/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "packages/otlp-importer/protos"]
+	path = packages/otlp-importer/protos
+	url = https://github.com/open-telemetry/opentelemetry-proto.git

.nvmrc

@@ -1 +1 @@
-v18.12.1
+v20.11.1

.vscode/extensions.json

@@ -0,0 +1,9 @@
+{
+	"recommendations": [
+        "astro-build.astro-vscode",
+        "denoland.vscode-deno"
+    ],
+	"unwantedRecommendations": [
+		
+	]
+}

.vscode/launch.json

@@ -9,7 +9,7 @@
       "request": "launch",
       "name": "Debug WebApp",
       "command": "pnpm run dev --filter webapp",
-      "envFile": "${workspaceFolder}/apps/webapp/.env",
+      "envFile": "${workspaceFolder}/.env",
       "cwd": "${workspaceFolder}",
       "sourceMaps": true
     },
@@ -19,6 +19,39 @@
       "name": "Chrome webapp",
       "url": "http://localhost:3030",
       "webRoot": "${workspaceFolder}/apps/webapp/app"
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug v2 job catalog",
+      "command": "pnpm run byo-auth",
+      "envFile": "${workspaceFolder}/references/job-catalog/.env",
+      "cwd": "${workspaceFolder}/references/job-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 Dev CLI",
+      "command": "pnpm exec triggerdev dev --log-level debug",
+      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 Deploy CLI",
+      "command": "pnpm exec triggerdev deploy",
+      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node",
+      "request": "attach",
+      "name": "Attach to Trigger.dev CLI (v3)",
+      "port": 9229,
+      "restart": true,
+      "skipFiles": ["<node_internals>/**"]
     }
   ]
 }

.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+  "deno.enablePaths": ["references/deno-reference", "runtime_tests/tests/deno"],
+  "debug.toolBarLocation": "commandCenter"
+}

CHANGESETS.md

@@ -27,3 +27,31 @@ Please follow the best-practice of adding changesets in the same commit as the c
 3. Create version `pnpm run changeset:version`
 4. Release `pnpm run changeset:release`
 5. Switch back to normal mode by running `pnpm run changeset:normal`
+
+## Snapshot instructions
+
+!MAKE SURE TO UPDATE THE TAG IN THE INSTRUCTIONS BELOW!
+
+1. Add changesets as usual
+
+```sh
+pnpm run changeset:add
+```
+
+2. Create a snapshot version (replace "prerelease" with your tag)
+
+```sh
+pnpm exec changeset version --snapshot prerelease
+```
+
+3. Build the packages:
+
+```sh
+pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev"
+```
+
+4. Publish the snapshot (replace "dev" with your tag)
+
+```sh
+pnpm exec changeset publish --no-git-tag --snapshot --tag prerelease
+```

CODE_OF_CONDUCT.md

@@ -0,0 +1,43 @@
+# Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment include:
+
+- Using welcoming and inclusive language
+- Being respectful of differing viewpoints and experiences
+- Gracefully accepting constructive criticism
+- Focusing on what is best for the community
+- Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+- The use of sexualized language or imagery and unwelcome sexual attention or advances
+- Trolling, insulting/derogatory comments, and personal or political attacks
+- Public or private harassment
+- Publishing others' private information, such as a physical or electronic address, without explicit permission
+- Other conduct that could reasonably be considered inappropriate in a professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned with this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project email address, posting via an official social media account, or acting as an appointed representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team  All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the Contributor Covenant, version 2.0, available at [https://www.contributor-covenant.org/version/2/0/code_of_conduct.html](https://www.contributor-covenant.org/version/2/0/code_of_conduct.html).

CONTRIBUTING.md

@@ -23,11 +23,11 @@ branch are tagged into a release monthly.
 1. Clone the repo into a public GitHub repository or [fork the repo](https://github.com/triggerdotdev/trigger.dev/fork). If you plan to distribute the code, keep the source code public to comply with the [Apache Licence 2.0](https://github.com/triggerdotdev/trigger.dev/blob/main/LICENSE).
 
    ```
-   git clone https://github.com/triggerdotdev/trigger.dev.git
+   git clone https://github.com/<github_username>/trigger.dev.git
    ```
 
    > If you are on windows, run the following command on gitbash with admin privileges:
-   > `git clone -c core.symlinks=true https://github.com/triggerdotdev/trigger.dev.git`
+   > `git clone -c core.symlinks=true https://github.com/<github_username>/trigger.dev.git`
 
 2. Navigate to the project folder
    ```
@@ -52,9 +52,13 @@ branch are tagged into a release monthly.
    Feel free to update `SESSION_SECRET` and `MAGIC_LINK_SECRET` as well using the same method.
 
 6. Start Docker. This starts the required services like Postgres. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)
+
    ```
    pnpm run docker
    ```
+
+   This will also start and run a local instance of [pgAdmin](https://www.pgadmin.org/) on [localhost:5480](http://localhost:5480), preconfigured with email `admin@example.com` and pwd `admin`. Then use `postgres` as the password to the Trigger.dev server.
+
 7. Migrate the database
    ```
    pnpm run db:migrate
@@ -133,10 +137,10 @@ pnpm run dev
 
 2. Open a new Terminal window and run the webapp locally and then create a new project in the dashboard. Copy out the dev API key.
 
-3. Create a new temporary Next.js app in examples directory
+3. Create a new temporary Next.js app in references directory
 
 ```sh
-cd ./examples
+cd ./references
 pnpm create next-app@latest test-cli --ts --no-eslint --tailwind --app --src-dir --import-alias "@/*"
 ```
 
@@ -149,7 +153,7 @@ pnpm create next-app@latest test-cli --ts --no-eslint --tailwind --app --src-dir
 }
 ```
 
-5. Back in the terminal, navigate into the example, and initialize the CLI. When prompted, select `self-hosted` and enter `localhost:3030` if you are testing against the local instance of Trigger.dev, or you can just use the Trigger.dev cloud. When asked for an API key, use the key you copied earlier.
+5. Back in the terminal, navigate into the reference, and initialize the CLI. When prompted, select `self-hosted` and enter `localhost:3030` if you are testing against the local instance of Trigger.dev, or you can just use the Trigger.dev cloud. When asked for an API key, use the key you copied earlier.
 
 ```sh
 cd ./test-cli
@@ -179,14 +183,14 @@ To run the end-to-end tests, follow the steps below:
 
 ```sh
 cp ./.env.example ./.env
-cp ./examples/nextjs-test/.env.example ./examples/nextjs-test/.env.local
+cp ./references/nextjs-test/.env.example ./references/nextjs-test/.env.local
 ```
 
 2. Set up dependencies
 
 ```sh
 # Build packages
-pnpm run build --filter @examples/nextjs-test^...
+pnpm run build --filter @references/nextjs-test^...
 pnpm --filter @trigger.dev/database generate
 
 # Move trigger-cli bin to correct place
@@ -221,11 +225,11 @@ pnpm run db:studio
 
 ## Add sample jobs
 
-The [examples/jobs-starter](./examples/jobs-starter/) project defines simple jobs you can get started with.
+The [references/job-catalog](./references/job-catalog/) project defines simple jobs you can get started with.
 
-1. `cd` into `examples/jobs-starter`
-2. Create a `.env.local` file with the following content,
-   replacing `[TRIGGER_DEV_API_KEY]` with an actual key:
+1. `cd` into `references/job-catalog`
+2. Create a `.env` file with the following content,
+   replacing `<TRIGGER_DEV_API_KEY>` with an actual key:
 
 ```env
 TRIGGER_API_KEY=[TRIGGER_DEV_API_KEY]
@@ -235,12 +239,20 @@ TRIGGER_API_URL=http://localhost:3030
 `TRIGGER_API_URL` is used to configure the URL for your Trigger.dev instance,
 where the jobs will be registered.
 
-3. Run the `jobs-starter` app:
+3. Run one of the the `job-catalog` files:
 
 ```sh
-pnpm dev
+pnpm run events
 ```
 
+This will open up a local server using `express` on port 8080. Then in a new terminal window you can run the trigger-cli dev command:
+
+```sh
+pnpm run dev:trigger
+```
+
+See the [Job Catalog](./references/job-catalog/README.md) file for more.
+
 4. Navigate to your trigger.dev instance ([http://localhost:3030](http://localhost:3030/)), to see the jobs.
    You can use the test feature to trigger them.
 

README.md

@@ -5,9 +5,9 @@
   <img alt="Trigger.dev logo" src="https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/a45d1fa2-0ae8-4a39-4409-f4f934bfae00/public">
 </picture>
   
-### The background jobs framework for Next.js
+### The open source background jobs framework
 
-[Discord](https://discord.gg/JtBAxBr2m3) | [Website](https://trigger.dev) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Docs](https://trigger.dev/docs)
+[Discord](https://trigger.dev/discord) | [Website](https://trigger.dev) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Docs](https://trigger.dev/docs)
 
 [![Twitter](https://img.shields.io/twitter/url/https/twitter.com/triggerdotdev.svg?style=social&label=Follow%20%40trigger.dev)](https://twitter.com/triggerdotdev)
 [![GitHub Repo stars](https://img.shields.io/github/stars/triggerdotdev/trigger.dev?style=social)](https://github.com/triggerdotdev/trigger.dev)
@@ -18,7 +18,7 @@
 
 Create long-running jobs directly in your codebase with features like API integrations, webhooks, scheduling and delays.
 
-### Long running Jobs on serverless
+## Long running Jobs on serverless
 
 Reliably run jobs and don’t worry about function timeouts, we handle those for you.
 
@@ -26,31 +26,51 @@ Reliably run jobs and don’t worry about function timeouts, we handle those for
 - Auto-resume after a server outage
 - Add delays of up to a year
 
-### In your codebase
+## In your codebase
 
 Create Jobs where they belong: in your codebase. Version control, localhost, test, review, and deploy like you're already used to.
 
-### Secure by design
+## Secure by design
 
 We only receive Triggers and the data you choose to send to us. You can even completely self-host the entire platform.
 
-### Don't worry about deployment
+## Don't worry about deployment
 
-Just use our SDK to write Jobs in your Next.js codebase. There's nothing extra to deploy and no CI to configure, your Jobs just connect to our cloud. Or you can always self-host.
+Just use our SDK to write Jobs in your codebase. There's nothing extra to deploy and no CI to configure, your Jobs just connect to our cloud. Or you can always self-host.
 
-### Full visibility of every job run
+## Full visibility of every job run
 
 View every Task in every Run so you can tell exactly what happened.
 
 ![image](https://www.trigger.dev/build/_assets/web-app-2QFKXFLW.png)
 
-### Built-in integrations
+## Built-in integrations
 
 Easily integrate with hundreds of third-party APIs – including your own. Use API keys (which never leave your server) or let us handle OAuth for you. Install our integration packages and easily subscribe to webhooks and perform common tasks, or you can easily use your existing favorite Node.JS SDKs and get resumability and idempotency through our `runTask` function.
 
-### Trigger.dev Connect (coming soon)
+## Our progress
 
-Easily add integrations for your users.
+We’re building the most comprehensive and easy-to-use background jobs framework for developers.
+
+Click the links to join the discussions about our upcoming features.
+
+| Feature                                                                              | What it does                                        | Status |
+| ------------------------------------------------------------------------------------ | --------------------------------------------------- | ------ |
+| Integration kit                                                                      | Official Trigger.dev integrations or build your own | ✅     |
+| Self-hosting                                                                         | Host the platform yourself                          | ✅     |
+| Cloud                                                                                | Just write code, no deployment required             | ✅     |
+| Dashboard                                                                            | View every Task in every Run                        | ✅     |
+| Serverless                                                                           | Long-running Jobs on your serverless backend        | ✅     |
+| React hooks                                                                          | Easily update your UI with Job progress             | ✅     |
+| React frameworks                                                                     | Support for Remix, Astro, RedwoodJS & more          | ✅     |
+| [Background tasks](https://github.com/triggerdotdev/trigger.dev/discussions/400)     | Offload long or intense Tasks to our infrastructure | 🛠️     |
+| [Long-running servers](https://github.com/triggerdotdev/trigger.dev/discussions/430) | Run Jobs on your long-running backend               | 🛠️     |
+| Polling Triggers                                                                     | Subscribe to changes without webhooks               | 🕝     |
+| Vercel integration                                                                   | Easy deploy and preview environment support         | 🕝     |
+| Streaming                                                                            | Receive data from your Jobs in realtime             | 🕝     |
+| 100+ integrations                                                                    | Comprehensive support for popular APIs              | 🕝     |
+| [Trigger.dev Connect](https://github.com/triggerdotdev/trigger.dev/discussions/441)  | Use integrations signed in as your users            | 🕝     |
+| File IO                                                                              | Create Tasks that have file outputs                 | 🕝     |
 
 # Getting started
 
@@ -63,3 +83,9 @@ We provide an official trigger.dev docker image you can use to easily self-host
 ## Development
 
 To setup and develop locally or contribute to the open source project, follow our [development guide](./CONTRIBUTING.md).
+
+## Meet the Amazing People Behind This Project 🚀
+
+<a href="https://github.com/triggerdotdev/trigger.dev/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=triggerdotdev/trigger.dev" />
+</a>

apps/coordinator/.env.example

@@ -0,0 +1,4 @@
+HTTP_SERVER_PORT=8020
+PLATFORM_ENABLED=true
+PLATFORM_WS_PORT=3030
+SECURE_CONNECTION=false

apps/coordinator/.gitignore

@@ -0,0 +1,3 @@
+dist/
+node_modules/
+.env

apps/coordinator/Containerfile

@@ -0,0 +1,60 @@
+# syntax=docker/dockerfile:labs
+
+FROM node:18-bullseye-slim@sha256:a4edd54dcfdcacc8a4100fee71498e8671d99556a1acf5614539214a70092426 AS node-18
+
+WORKDIR /app
+
+FROM node-18 AS pruner
+
+COPY --chown=node:node . .
+RUN npx -q turbo@1.10.9 prune --scope=coordinator --docker
+RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
+
+FROM node-18 AS base
+
+RUN apt-get update \
+  && apt-get install -y buildah ca-certificates dumb-init \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY --chown=node:node .gitignore .gitignore
+COPY --from=pruner --chown=node:node /app/out/json/ .
+COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
+COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
+
+FROM base AS dev-deps
+RUN corepack enable
+ENV NODE_ENV development
+
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
+
+FROM base AS builder
+RUN corepack enable
+
+COPY --from=pruner --chown=node:node /app/out/full/ .
+COPY --from=dev-deps --chown=node:node /app/ .
+COPY --chown=node:node turbo.json turbo.json
+
+RUN pnpm run -r --filter coordinator build:bundle
+
+FROM alpine AS cri-tools
+
+WORKDIR /cri-tools
+
+ARG CRICTL_VERSION=v1.29.0
+ARG CRICTL_CHECKSUM=sha256:d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb
+ADD --checksum=${CRICTL_CHECKSUM} \
+  https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz .
+RUN tar zxvf crictl-${CRICTL_VERSION}-linux-amd64.tar.gz
+
+FROM base AS runner
+
+RUN corepack enable
+ENV NODE_ENV production
+
+COPY --from=cri-tools --chown=node:node /cri-tools/crictl /usr/local/bin
+COPY --from=builder --chown=node:node /app/apps/coordinator/dist/index.mjs ./index.mjs
+
+EXPOSE 8000
+
+CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]

apps/coordinator/README.md

@@ -0,0 +1,3 @@
+# Coordinator
+
+Sits between the platform and tasks. Facilitates communication and checkpointing, amongst other things.

apps/coordinator/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "coordinator",
+  "private": true,
+  "version": "0.0.1",
+  "description": "",
+  "main": "dist/index.cjs",
+  "scripts": {
+    "build": "npm run build:bundle",
+    "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"",
+    "build:image": "docker build -f Containerfile . -t coordinator",
+    "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
+    "start": "tsx src/index.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@trigger.dev/core": "workspace:*",
+    "@trigger.dev/core-apps": "workspace:*",
+    "execa": "^8.0.1",
+    "nanoid": "^5.0.6",
+    "prom-client": "^15.1.0",
+    "socket.io": "^4.7.4",
+    "socket.io-client": "^4.7.4"
+  },
+  "devDependencies": {
+    "@types/node": "^18",
+    "dotenv": "^16.4.2",
+    "esbuild": "^0.19.11",
+    "tsx": "^4.7.0",
+    "typescript": "^5.3.3"
+  }
+}

apps/coordinator/src/index.ts

@@ -0,0 +1,983 @@
+import { createServer } from "node:http";
+import { $ } from "execa";
+import { nanoid } from "nanoid";
+import { Server } from "socket.io";
+import {
+  CoordinatorToPlatformMessages,
+  CoordinatorToProdWorkerMessages,
+  PlatformToCoordinatorMessages,
+  ProdWorkerSocketData,
+  ProdWorkerToCoordinatorMessages,
+  ZodNamespace,
+  ZodSocketConnection,
+} from "@trigger.dev/core/v3";
+import { HttpReply, getTextBody, SimpleLogger } from "@trigger.dev/core-apps";
+
+import { collectDefaultMetrics, register, Gauge } from "prom-client";
+collectDefaultMetrics();
+
+const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || 8020);
+const NODE_NAME = process.env.NODE_NAME || "coordinator";
+const DEFAULT_RETRY_DELAY_THRESHOLD_IN_MS = 30_000;
+
+const REGISTRY_HOST = process.env.REGISTRY_HOST || "localhost:5000";
+const CHECKPOINT_PATH = process.env.CHECKPOINT_PATH || "/checkpoints";
+const REGISTRY_TLS_VERIFY = process.env.REGISTRY_TLS_VERIFY === "false" ? "false" : "true";
+
+const PLATFORM_ENABLED = ["1", "true"].includes(process.env.PLATFORM_ENABLED ?? "true");
+const PLATFORM_HOST = process.env.PLATFORM_HOST || "127.0.0.1";
+const PLATFORM_WS_PORT = process.env.PLATFORM_WS_PORT || 3030;
+const PLATFORM_SECRET = process.env.PLATFORM_SECRET || "coordinator-secret";
+const SECURE_CONNECTION = ["1", "true"].includes(process.env.SECURE_CONNECTION ?? "false");
+
+const logger = new SimpleLogger(`[${NODE_NAME}]`);
+
+type CheckpointerInitializeReturn = {
+  canCheckpoint: boolean;
+  willSimulate: boolean;
+};
+
+type CheckpointAndPushOptions = {
+  runId: string;
+  leaveRunning?: boolean;
+  projectRef: string;
+  deploymentVersion: string;
+};
+
+type CheckpointData = {
+  location: string;
+  docker: boolean;
+};
+
+class Checkpointer {
+  #initialized = false;
+  #canCheckpoint = false;
+  #dockerMode = !process.env.KUBERNETES_PORT;
+
+  #logger = new SimpleLogger("[checkptr]");
+  #abortControllers = new Map<string, AbortController>();
+
+  constructor(private opts = { forceSimulate: false }) {}
+
+  async initialize(): Promise<CheckpointerInitializeReturn> {
+    if (this.#initialized) {
+      return this.#getInitializeReturn();
+    }
+
+    this.#logger.log(`${this.#dockerMode ? "Docker" : "Kubernetes"} mode`);
+
+    if (this.#dockerMode) {
+      try {
+        await $`criu --version`;
+      } catch (error) {
+        this.#logger.error("No checkpoint support: Missing CRIU binary");
+        this.#logger.error("Will simulate instead");
+        this.#canCheckpoint = false;
+        this.#initialized = true;
+
+        return this.#getInitializeReturn();
+      }
+
+      try {
+        await $`docker checkpoint`;
+      } catch (error) {
+        this.#logger.error(
+          "No checkpoint support: Docker needs to have experimental features enabled"
+        );
+        this.#logger.error("Will simulate instead");
+        this.#canCheckpoint = false;
+        this.#initialized = true;
+
+        return this.#getInitializeReturn();
+      }
+    } else {
+      try {
+        await $`buildah login --get-login ${REGISTRY_HOST}`;
+      } catch (error) {
+        this.#logger.error(`No checkpoint support: Not logged in to registry ${REGISTRY_HOST}`);
+        this.#canCheckpoint = false;
+        this.#initialized = true;
+
+        return this.#getInitializeReturn();
+      }
+    }
+
+    this.#logger.log(
+      `Full checkpoint support${
+        this.#dockerMode && this.opts.forceSimulate ? " with forced simulation enabled." : "!"
+      }`
+    );
+
+    this.#initialized = true;
+    this.#canCheckpoint = true;
+
+    return this.#getInitializeReturn();
+  }
+
+  #getInitializeReturn(): CheckpointerInitializeReturn {
+    return {
+      canCheckpoint: this.#canCheckpoint,
+      willSimulate: this.#dockerMode && (!this.#canCheckpoint || this.opts.forceSimulate),
+    };
+  }
+
+  #getImageRef(projectRef: string, deploymentVersion: string, shortCode: string) {
+    return `${REGISTRY_HOST}/trigger/${projectRef}:${deploymentVersion}.prod-${shortCode}`;
+  }
+
+  #getExportLocation(projectRef: string, deploymentVersion: string, shortCode: string) {
+    const basename = `${projectRef}-${deploymentVersion}-${shortCode}`;
+
+    if (this.#dockerMode) {
+      return basename;
+    } else {
+      return `${CHECKPOINT_PATH}/${basename}.tar`;
+    }
+  }
+
+  async checkpointAndPush(opts: CheckpointAndPushOptions): Promise<CheckpointData | undefined> {
+    const start = performance.now();
+    logger.log(`checkpointAndPush() start`, { start, opts });
+
+    const result = await this.#checkpointAndPush(opts);
+
+    const end = performance.now();
+    logger.log(`checkpointAndPush() end`, {
+      start,
+      end,
+      diff: end - start,
+      opts,
+      success: !!result,
+    });
+
+    return result;
+  }
+
+  isCheckpointing(runId: string) {
+    return this.#abortControllers.has(runId);
+  }
+
+  cancelCheckpoint(runId: string) {
+    const controller = this.#abortControllers.get(runId);
+
+    if (!controller) {
+      logger.debug("Nothing to cancel", { runId });
+      return;
+    }
+
+    controller.abort("cancelCheckpointing()");
+    this.#abortControllers.delete(runId);
+  }
+
+  async #checkpointAndPush({
+    runId,
+    leaveRunning = true, // This mirrors kubernetes behaviour more accurately
+    projectRef,
+    deploymentVersion,
+  }: CheckpointAndPushOptions): Promise<CheckpointData | undefined> {
+    await this.initialize();
+
+    if (!this.#dockerMode && !this.#canCheckpoint) {
+      this.#logger.error("No checkpoint support. Simulation requires docker.");
+      return;
+    }
+
+    if (this.#abortControllers.has(runId)) {
+      logger.error("Checkpoint procedure already in progress", {
+        options: {
+          runId,
+          leaveRunning,
+          projectRef,
+          deploymentVersion,
+        },
+      });
+      return;
+    }
+
+    const controller = new AbortController();
+    this.#abortControllers.set(runId, controller);
+
+    const $$ = $({ signal: controller.signal });
+
+    try {
+      const shortCode = nanoid(8);
+      const imageRef = this.#getImageRef(projectRef, deploymentVersion, shortCode);
+      const exportLocation = this.#getExportLocation(projectRef, deploymentVersion, shortCode);
+
+      this.#logger.log("Checkpointing:", {
+        options: {
+          runId,
+          leaveRunning,
+          projectRef,
+          deploymentVersion,
+        },
+      });
+
+      const containterName = this.#getRunContainerName(runId);
+
+      // Create checkpoint (docker)
+      if (this.#dockerMode) {
+        try {
+          if (this.opts.forceSimulate || !this.#canCheckpoint) {
+            this.#logger.log("Simulating checkpoint");
+            this.#logger.debug(await $$`docker pause ${containterName}`);
+          } else {
+            if (leaveRunning) {
+              this.#logger.debug(
+                await $$`docker checkpoint create --leave-running ${containterName} ${exportLocation}`
+              );
+            } else {
+              this.#logger.debug(
+                await $$`docker checkpoint create ${containterName} ${exportLocation}`
+              );
+            }
+          }
+        } catch (error: any) {
+          this.#logger.error(error.stderr);
+          return;
+        }
+
+        this.#logger.log("checkpoint created:", {
+          runId,
+          location: exportLocation,
+        });
+
+        return {
+          location: exportLocation,
+          docker: true,
+        };
+      }
+
+      // Create checkpoint (CRI)
+      if (!this.#canCheckpoint) {
+        throw new Error("No checkpoint support in kubernetes mode.");
+      }
+
+      const containerId = this.#logger.debug(
+        // @ts-expect-error
+        await $$`crictl ps`
+          .pipeStdout($$({ stdin: "pipe" })`grep ${containterName}`)
+          .pipeStdout($$({ stdin: "pipe" })`cut -f1 ${"-d "}`)
+      );
+
+      if (!containerId.stdout) {
+        throw new Error("could not find container id");
+      }
+
+      this.#logger.debug(await $$`crictl checkpoint --export=${exportLocation} ${containerId}`);
+
+      // Create image from checkpoint
+      const container = this.#logger.debug(await $$`buildah from scratch`);
+      this.#logger.debug(await $$`buildah add ${container} ${exportLocation} /`);
+      this.#logger.debug(
+        await $$`buildah config --annotation=io.kubernetes.cri-o.annotations.checkpoint.name=counter ${container}`
+      );
+      this.#logger.debug(await $$`buildah commit ${container} ${imageRef}`);
+      this.#logger.debug(await $$`buildah rm ${container}`);
+
+      // Push checkpoint image
+      this.#logger.debug(await $$`buildah push --tls-verify=${REGISTRY_TLS_VERIFY} ${imageRef}`);
+
+      this.#logger.log("Checkpointed and pushed image to:", { location: imageRef });
+
+      try {
+        await $$`rm ${exportLocation}`;
+        this.#logger.log("Deleted checkpoint archive", { exportLocation });
+
+        // Disabled for now as this will increase restore time by having to pull the image again
+        // await $`buildah rmi ${imageRef}`;
+        // this.#logger.log("Deleted checkpoint image", { imageRef });
+      } catch (error) {
+        this.#logger.error("Failed during checkpoint cleanup", { exportLocation });
+        this.#logger.debug(error);
+      }
+
+      return {
+        location: imageRef,
+        docker: false,
+      };
+    } catch (error) {
+      this.#logger.error("checkpoint failed", {
+        options: {
+          runId,
+          leaveRunning,
+          projectRef,
+          deploymentVersion,
+        },
+        error,
+      });
+      return;
+    } finally {
+      this.#abortControllers.delete(runId);
+    }
+  }
+
+  #getRunContainerName(suffix: string) {
+    return `task-run-${suffix}`;
+  }
+}
+
+class TaskCoordinator {
+  #httpServer: ReturnType<typeof createServer>;
+  #checkpointer = new Checkpointer({ forceSimulate: true });
+
+  #prodWorkerNamespace: ZodNamespace<
+    typeof ProdWorkerToCoordinatorMessages,
+    typeof CoordinatorToProdWorkerMessages,
+    typeof ProdWorkerSocketData
+  >;
+  #platformSocket?: ZodSocketConnection<
+    typeof CoordinatorToPlatformMessages,
+    typeof PlatformToCoordinatorMessages
+  >;
+
+  #checkpointableTasks = new Map<
+    string,
+    { resolve: (value: void) => void; reject: (err?: any) => void }
+  >();
+
+  #delayThresholdInMs: number;
+
+  constructor(
+    private port: number,
+    private host = "0.0.0.0"
+  ) {
+    this.#httpServer = this.#createHttpServer();
+    this.#checkpointer.initialize();
+    this.#delayThresholdInMs = this.#getDelayThreshold();
+
+    if (process.env.DELAY_THRESHOLD_IN_MS) {
+      this.#delayThresholdInMs = this.#getDelayThreshold();
+    }
+
+    const io = new Server(this.#httpServer);
+    this.#prodWorkerNamespace = this.#createProdWorkerNamespace(io);
+
+    this.#platformSocket = this.#createPlatformSocket();
+
+    const connectedTasksTotal = new Gauge({
+      name: "daemon_connected_tasks_total", // don't change this without updating dashboard config
+      help: "The number of tasks currently connected.",
+      collect: () => {
+        connectedTasksTotal.set(this.#prodWorkerNamespace.namespace.sockets.size);
+      },
+    });
+    register.registerMetric(connectedTasksTotal);
+  }
+
+  #getDelayThreshold() {
+    if (!process.env.RETRY_DELAY_THRESHOLD_IN_MS) {
+      return DEFAULT_RETRY_DELAY_THRESHOLD_IN_MS;
+    }
+
+    const threshold = parseInt(process.env.RETRY_DELAY_THRESHOLD_IN_MS);
+
+    if (isNaN(threshold)) {
+      logger.log(
+        "RETRY_DELAY_THRESHOLD_IN_MS parses as NaN, must supply integer. Will use default instead.",
+        {
+          RETRY_DELAY_THRESHOLD_IN_MS: process.env.RETRY_DELAY_THRESHOLD_IN_MS,
+          DEFAULT_DELAY_THRESHOLD_IN_MS: DEFAULT_RETRY_DELAY_THRESHOLD_IN_MS,
+        }
+      );
+      return DEFAULT_RETRY_DELAY_THRESHOLD_IN_MS;
+    }
+
+    return threshold;
+  }
+
+  #createPlatformSocket() {
+    if (!PLATFORM_ENABLED) {
+      console.log("INFO: platform connection disabled");
+      return;
+    }
+
+    const platformConnection = new ZodSocketConnection({
+      namespace: "coordinator",
+      host: PLATFORM_HOST,
+      port: Number(PLATFORM_WS_PORT),
+      secure: SECURE_CONNECTION,
+      clientMessages: CoordinatorToPlatformMessages,
+      serverMessages: PlatformToCoordinatorMessages,
+      authToken: PLATFORM_SECRET,
+      handlers: {
+        RESUME_AFTER_DEPENDENCY: async (message) => {
+          const taskSocket = await this.#getAttemptSocket(message.attemptFriendlyId);
+
+          if (!taskSocket) {
+            logger.log("Socket for attempt not found", {
+              attemptFriendlyId: message.attemptFriendlyId,
+            });
+            return;
+          }
+
+          // In case the task resumed faster than we could checkpoint
+          this.#cancelCheckpoint(message.runId);
+
+          taskSocket.emit("RESUME_AFTER_DEPENDENCY", message);
+        },
+        RESUME_AFTER_DURATION: async (message) => {
+          const taskSocket = await this.#getAttemptSocket(message.attemptFriendlyId);
+
+          if (!taskSocket) {
+            logger.log("Socket for attempt not found", {
+              attemptFriendlyId: message.attemptFriendlyId,
+            });
+            return;
+          }
+
+          taskSocket.emit("RESUME_AFTER_DURATION", message);
+        },
+        REQUEST_ATTEMPT_CANCELLATION: async (message) => {
+          const taskSocket = await this.#getAttemptSocket(message.attemptFriendlyId);
+
+          if (!taskSocket) {
+            logger.log("Socket for attempt not found", {
+              attemptFriendlyId: message.attemptFriendlyId,
+            });
+            return;
+          }
+
+          taskSocket.emit("REQUEST_ATTEMPT_CANCELLATION", message);
+        },
+        READY_FOR_RETRY: async (message) => {
+          const taskSocket = await this.#getRunSocket(message.runId);
+
+          if (!taskSocket) {
+            logger.log("Socket for attempt not found", {
+              runId: message.runId,
+            });
+            return;
+          }
+
+          taskSocket.emit("READY_FOR_RETRY", message);
+        },
+      },
+    });
+
+    return platformConnection;
+  }
+
+  async #getRunSocket(runId: string) {
+    const sockets = await this.#prodWorkerNamespace.fetchSockets();
+
+    for (const socket of sockets) {
+      if (socket.data.runId === runId) {
+        return socket;
+      }
+    }
+  }
+
+  async #getAttemptSocket(attemptFriendlyId: string) {
+    const sockets = await this.#prodWorkerNamespace.fetchSockets();
+
+    for (const socket of sockets) {
+      if (socket.data.attemptFriendlyId === attemptFriendlyId) {
+        return socket;
+      }
+    }
+  }
+
+  #createProdWorkerNamespace(io: Server) {
+    const provider = new ZodNamespace({
+      io,
+      name: "prod-worker",
+      clientMessages: ProdWorkerToCoordinatorMessages,
+      serverMessages: CoordinatorToProdWorkerMessages,
+      socketData: ProdWorkerSocketData,
+      postAuth: async (socket, next, logger) => {
+        function setSocketDataFromHeader(
+          dataKey: keyof typeof socket.data,
+          headerName: string,
+          required: boolean = true
+        ) {
+          const value = socket.handshake.headers[headerName];
+
+          if (value) {
+            socket.data[dataKey] = Array.isArray(value) ? value[0] : value;
+            return;
+          }
+
+          if (required) {
+            logger.error("missing required header", { headerName });
+            throw new Error("missing header");
+          }
+        }
+
+        try {
+          setSocketDataFromHeader("podName", "x-pod-name");
+          setSocketDataFromHeader("contentHash", "x-trigger-content-hash");
+          setSocketDataFromHeader("projectRef", "x-trigger-project-ref");
+          setSocketDataFromHeader("runId", "x-trigger-run-id");
+          setSocketDataFromHeader("attemptFriendlyId", "x-trigger-attempt-friendly-id", false);
+          setSocketDataFromHeader("envId", "x-trigger-env-id");
+          setSocketDataFromHeader("deploymentId", "x-trigger-deployment-id");
+          setSocketDataFromHeader("deploymentVersion", "x-trigger-deployment-version");
+        } catch (error) {
+          logger.error("setSocketDataFromHeader error", { error });
+          socket.disconnect(true);
+          return;
+        }
+
+        logger.debug("success", socket.data);
+
+        next();
+      },
+      onConnection: async (socket, handler, sender) => {
+        const logger = new SimpleLogger(`[prod-worker][${socket.id}]`);
+
+        const checkpointInProgress = () => {
+          return this.#checkpointableTasks.has(socket.data.runId);
+        };
+
+        const readyToCheckpoint = async (): Promise<
+          { success: true } | { success: false; reason?: string }
+        > => {
+          if (checkpointInProgress()) {
+            return {
+              success: false,
+              reason: "checkpoint in progress",
+            };
+          }
+
+          const isCheckpointable = new Promise((resolve, reject) => {
+            // We set a reasonable timeout to prevent waiting forever
+            // TODO: We may also want to cancel the task as it's unlikely to recover
+            setTimeout(() => reject("timeout"), 10_000);
+
+            this.#checkpointableTasks.set(socket.data.runId, { resolve, reject });
+          });
+
+          try {
+            await isCheckpointable;
+            this.#checkpointableTasks.delete(socket.data.runId);
+
+            return {
+              success: true,
+            };
+          } catch (error) {
+            logger.error("Error while waiting for checkpointable state", { error });
+
+            return {
+              success: false,
+              reason: typeof error === "string" ? error : "unknown",
+            };
+          }
+        };
+
+        this.#platformSocket?.send("LOG", {
+          metadata: socket.data,
+          text: "connected",
+        });
+
+        socket.on("LOG", (message, callback) => {
+          logger.log("[LOG]", message.text);
+
+          callback();
+
+          this.#platformSocket?.send("LOG", {
+            version: "v1",
+            metadata: socket.data,
+            text: message.text,
+          });
+        });
+
+        socket.on("READY_FOR_EXECUTION", async (message) => {
+          logger.log("[READY_FOR_EXECUTION]", message);
+
+          try {
+            const executionAck = await this.#platformSocket?.sendWithAck(
+              "READY_FOR_EXECUTION",
+              message
+            );
+
+            if (!executionAck) {
+              logger.error("no execution ack", { runId: socket.data.runId });
+
+              socket.emit("REQUEST_EXIT", {
+                version: "v1",
+              });
+
+              return;
+            }
+
+            if (!executionAck.success) {
+              logger.error("failed to get execution payload", { runId: socket.data.runId });
+
+              socket.emit("REQUEST_EXIT", {
+                version: "v1",
+              });
+
+              return;
+            }
+
+            socket.emit("EXECUTE_TASK_RUN", {
+              version: "v1",
+              executionPayload: executionAck.payload,
+            });
+
+            socket.data.attemptFriendlyId = executionAck.payload.execution.attempt.id;
+          } catch (error) {
+            logger.error("Error", { error });
+          }
+        });
+
+        socket.on("READY_FOR_RESUME", async (message) => {
+          logger.log("[READY_FOR_RESUME]", message);
+
+          socket.data.attemptFriendlyId = message.attemptFriendlyId;
+          this.#platformSocket?.send("READY_FOR_RESUME", message);
+        });
+
+        socket.on("TASK_RUN_COMPLETED", async ({ completion, execution }, callback) => {
+          logger.log("completed task", { completionId: completion.id });
+
+          const completeWithoutCheckpoint = (shouldExit: boolean) => {
+            this.#platformSocket?.send("TASK_RUN_COMPLETED", {
+              version: "v1",
+              execution,
+              completion,
+            });
+            callback({ willCheckpointAndRestore: false, shouldExit });
+          };
+
+          if (completion.ok) {
+            completeWithoutCheckpoint(true);
+            return;
+          }
+
+          if (
+            completion.error.type === "INTERNAL_ERROR" &&
+            completion.error.code === "TASK_RUN_CANCELLED"
+          ) {
+            completeWithoutCheckpoint(true);
+            return;
+          }
+
+          if (completion.retry === undefined) {
+            completeWithoutCheckpoint(true);
+            return;
+          }
+
+          if (completion.retry.delay < this.#delayThresholdInMs) {
+            completeWithoutCheckpoint(false);
+            return;
+          }
+
+          const { canCheckpoint, willSimulate } = await this.#checkpointer.initialize();
+
+          const willCheckpointAndRestore = canCheckpoint || willSimulate;
+
+          if (!willCheckpointAndRestore) {
+            completeWithoutCheckpoint(false);
+            return;
+          }
+
+          // The worker will then put itself in a checkpointable state
+          callback({ willCheckpointAndRestore: true, shouldExit: false });
+
+          const ready = await readyToCheckpoint();
+
+          if (!ready.success) {
+            logger.error("Failed to become checkpointable", {
+              runId: socket.data.runId,
+              reason: ready.reason,
+            });
+            return;
+          }
+
+          const checkpoint = await this.#checkpointer.checkpointAndPush({
+            runId: socket.data.runId,
+            projectRef: socket.data.projectRef,
+            deploymentVersion: socket.data.deploymentVersion,
+          });
+
+          if (!checkpoint) {
+            logger.error("Failed to checkpoint", { runId: socket.data.runId });
+            completeWithoutCheckpoint(false);
+            return;
+          }
+
+          this.#platformSocket?.send("TASK_RUN_COMPLETED", {
+            version: "v1",
+            execution,
+            completion,
+            checkpoint,
+          });
+
+          if (!checkpoint.docker || !willSimulate) {
+            socket.emit("REQUEST_EXIT", {
+              version: "v1",
+            });
+          }
+        });
+
+        socket.on("READY_FOR_CHECKPOINT", async (message) => {
+          logger.log("[READY_FOR_CHECKPOINT]", message);
+
+          const checkpointable = this.#checkpointableTasks.get(socket.data.runId);
+
+          if (!checkpointable) {
+            logger.error("No checkpoint scheduled", { runId: socket.data.runId });
+            return;
+          }
+
+          checkpointable.resolve();
+        });
+
+        socket.on("CANCEL_CHECKPOINT", async (message) => {
+          logger.log("[CANCEL_CHECKPOINT]", message);
+
+          this.#cancelCheckpoint(socket.data.runId);
+        });
+
+        socket.on("WAIT_FOR_DURATION", async (message, callback) => {
+          logger.log("[WAIT_FOR_DURATION]", message);
+
+          if (checkpointInProgress()) {
+            logger.error("Checkpoint already in progress", { runId: socket.data.runId });
+            callback({ willCheckpointAndRestore: false });
+            return;
+          }
+
+          const { canCheckpoint, willSimulate } = await this.#checkpointer.initialize();
+
+          const willCheckpointAndRestore = canCheckpoint || willSimulate;
+
+          callback({ willCheckpointAndRestore });
+
+          if (!willCheckpointAndRestore) {
+            return;
+          }
+
+          const ready = await readyToCheckpoint();
+
+          if (!ready.success) {
+            logger.error("Failed to become checkpointable", {
+              runId: socket.data.runId,
+              reason: ready.reason,
+            });
+            return;
+          }
+
+          const checkpoint = await this.#checkpointer.checkpointAndPush({
+            runId: socket.data.runId,
+            projectRef: socket.data.projectRef,
+            deploymentVersion: socket.data.deploymentVersion,
+          });
+
+          if (!checkpoint) {
+            // The task container will keep running until the wait duration has elapsed
+            logger.error("Failed to checkpoint", { runId: socket.data.runId });
+            return;
+          }
+
+          if (!checkpoint.docker || !willSimulate) {
+            socket.emit("REQUEST_EXIT", {
+              version: "v1",
+            });
+          }
+
+          this.#platformSocket?.send("CHECKPOINT_CREATED", {
+            version: "v1",
+            attemptFriendlyId: message.attemptFriendlyId,
+            docker: checkpoint.docker,
+            location: checkpoint.location,
+            reason: {
+              type: "WAIT_FOR_DURATION",
+              ms: message.ms,
+              now: message.now,
+            },
+          });
+        });
+
+        socket.on("WAIT_FOR_TASK", async (message, callback) => {
+          logger.log("[WAIT_FOR_TASK]", message);
+
+          const { canCheckpoint, willSimulate } = await this.#checkpointer.initialize();
+
+          const willCheckpointAndRestore = canCheckpoint || willSimulate;
+
+          callback({ willCheckpointAndRestore });
+
+          if (!willCheckpointAndRestore) {
+            return;
+          }
+
+          const checkpoint = await this.#checkpointer.checkpointAndPush({
+            runId: socket.data.runId,
+            projectRef: socket.data.projectRef,
+            deploymentVersion: socket.data.deploymentVersion,
+          });
+
+          if (!checkpoint) {
+            logger.error("Failed to checkpoint", { runId: socket.data.runId });
+            return;
+          }
+
+          if (!checkpoint.docker || !willSimulate) {
+            socket.emit("REQUEST_EXIT", {
+              version: "v1",
+            });
+          }
+
+          this.#platformSocket?.send("CHECKPOINT_CREATED", {
+            version: "v1",
+            attemptFriendlyId: message.attemptFriendlyId,
+            docker: checkpoint.docker,
+            location: checkpoint.location,
+            reason: {
+              type: "WAIT_FOR_TASK",
+              friendlyId: message.friendlyId,
+            },
+          });
+        });
+
+        socket.on("WAIT_FOR_BATCH", async (message, callback) => {
+          logger.log("[WAIT_FOR_BATCH]", message);
+
+          const { canCheckpoint, willSimulate } = await this.#checkpointer.initialize();
+
+          const willCheckpointAndRestore = canCheckpoint || willSimulate;
+
+          callback({ willCheckpointAndRestore });
+
+          if (!willCheckpointAndRestore) {
+            return;
+          }
+
+          const checkpoint = await this.#checkpointer.checkpointAndPush({
+            runId: socket.data.runId,
+            projectRef: socket.data.projectRef,
+            deploymentVersion: socket.data.deploymentVersion,
+          });
+
+          if (!checkpoint) {
+            logger.error("Failed to checkpoint", { runId: socket.data.runId });
+            return;
+          }
+
+          if (!checkpoint.docker || !willSimulate) {
+            socket.emit("REQUEST_EXIT", {
+              version: "v1",
+            });
+          }
+
+          this.#platformSocket?.send("CHECKPOINT_CREATED", {
+            version: "v1",
+            attemptFriendlyId: message.attemptFriendlyId,
+            docker: checkpoint.docker,
+            location: checkpoint.location,
+            reason: {
+              type: "WAIT_FOR_BATCH",
+              batchFriendlyId: message.batchFriendlyId,
+              runFriendlyIds: message.runFriendlyIds,
+            },
+          });
+        });
+
+        socket.on("INDEX_TASKS", async (message, callback) => {
+          logger.log("[INDEX_TASKS]", message);
+
+          const workerAck = await this.#platformSocket?.sendWithAck("CREATE_WORKER", {
+            version: "v1",
+            projectRef: socket.data.projectRef,
+            envId: socket.data.envId,
+            deploymentId: message.deploymentId,
+            metadata: {
+              contentHash: socket.data.contentHash,
+              packageVersion: message.packageVersion,
+              tasks: message.tasks,
+            },
+          });
+
+          if (!workerAck) {
+            logger.debug("no worker ack while indexing", message);
+          }
+
+          callback({ success: !!workerAck?.success });
+        });
+
+        socket.on("INDEXING_FAILED", async (message) => {
+          logger.log("[INDEXING_FAILED]", message);
+
+          this.#platformSocket?.send("INDEXING_FAILED", {
+            version: "v1",
+            deploymentId: message.deploymentId,
+            error: message.error,
+          });
+        });
+      },
+      onDisconnect: async (socket, handler, sender, logger) => {
+        this.#platformSocket?.send("LOG", {
+          metadata: socket.data,
+          text: "disconnect",
+        });
+      },
+      handlers: {
+        TASK_HEARTBEAT: async (message) => {
+          this.#platformSocket?.send("TASK_HEARTBEAT", message);
+        },
+      },
+    });
+
+    return provider;
+  }
+
+  #cancelCheckpoint(runId: string) {
+    const checkpointWait = this.#checkpointableTasks.get(runId);
+
+    if (checkpointWait) {
+      // Stop waiting for task to reach checkpointable state
+      checkpointWait.reject("Checkpoint cancelled");
+    }
+
+    // Cancel checkpointing procedure
+    this.#checkpointer.cancelCheckpoint(runId);
+  }
+
+  #createHttpServer() {
+    const httpServer = createServer(async (req, res) => {
+      logger.log(`[${req.method}]`, req.url);
+
+      const reply = new HttpReply(res);
+
+      switch (req.url) {
+        case "/health": {
+          return reply.text("ok");
+        }
+        case "/metrics": {
+          return reply.text(await register.metrics(), 200, register.contentType);
+        }
+        case "/whoami": {
+          return reply.text(NODE_NAME);
+        }
+        case "/checkpoint": {
+          const body = await getTextBody(req);
+          // await this.#checkpointer.checkpointAndPush(body);
+          return reply.text(`sent restore request: ${body}`);
+        }
+        default: {
+          return reply.empty(404);
+        }
+      }
+    });
+
+    httpServer.on("clientError", (err, socket) => {
+      socket.end("HTTP/1.1 400 Bad Request\r\n\r\n");
+    });
+
+    httpServer.on("listening", () => {
+      logger.log("server listening on port", HTTP_SERVER_PORT);
+    });
+
+    return httpServer;
+  }
+
+  listen() {
+    this.#httpServer.listen(this.port, this.host);
+  }
+}
+
+const coordinator = new TaskCoordinator(HTTP_SERVER_PORT);
+coordinator.listen();

apps/coordinator/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "include": ["./src/**/*.ts"],
+  "exclude": ["node_modules"],
+  "compilerOptions": {
+    "target": "es2016",
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "resolveJsonModule": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "paths": {
+      "@trigger.dev/core/v3": ["../../packages/core/src/v3"],
+      "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"],
+      "@trigger.dev/core-apps": ["../../packages/core-apps/src"],
+      "@trigger.dev/core-apps/*": ["../../packages/core-apps/src/*"]
+    }
+  }
+}

apps/docker-provider/.env.example

@@ -0,0 +1,9 @@
+HTTP_SERVER_PORT=8050
+
+PLATFORM_WS_PORT=3030
+PLATFORM_SECRET=provider-secret
+SECURE_CONNECTION=false
+
+# Use this if you are on macOS
+# COORDINATOR_HOST="host.docker.internal"
+# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318"

apps/docker-provider/.gitignore

@@ -0,0 +1,3 @@
+dist/
+node_modules/
+.env

apps/docker-provider/Containerfile

@@ -0,0 +1,16 @@
+# syntax=docker/dockerfile:labs
+
+FROM node:18-slim AS base
+
+RUN apt-get update \
+  && apt-get install -y dumb-init
+
+FROM base
+
+WORKDIR /app
+
+COPY --chown=node dist/index.mjs /app/
+
+EXPOSE 8000
+
+ENTRYPOINT [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "/app/index.mjs" ]

apps/docker-provider/README.md

@@ -0,0 +1,3 @@
+# Docker provider
+
+The `docker-provider` allows the platform to be orchestrator-agnostic. The platform can perform actions such as `INDEX_TASKS` or `INVOKE_TASK` which the provider translates into Docker actions.

apps/docker-provider/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "docker-provider",
+  "private": true,
+  "version": "0.0.1",
+  "description": "",
+  "main": "dist/index.cjs",
+  "scripts": {
+    "build": "npm run build:bundle",
+    "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"",
+    "build:image": "docker build -f Containerfile . -t docker-provider",
+    "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
+    "start": "tsx src/index.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@trigger.dev/core": "workspace:*",
+    "@trigger.dev/core-apps": "workspace:*",
+    "execa": "^8.0.1",
+    "socket.io-client": "^4.7.4"
+  },
+  "devDependencies": {
+    "@types/node": "^18.19.8",
+    "dotenv": "^16.4.2",
+    "esbuild": "^0.19.11",
+    "tsx": "^4.7.0",
+    "typescript": "^5.3.3"
+  }
+}

apps/docker-provider/src/index.ts

@@ -0,0 +1,302 @@
+import { $, type ExecaChildProcess, execa } from "execa";
+import {
+  SimpleLogger,
+  TaskOperations,
+  ProviderShell,
+  TaskOperationsRestoreOptions,
+  TaskOperationsCreateOptions,
+  TaskOperationsIndexOptions,
+} from "@trigger.dev/core-apps";
+import { setTimeout } from "node:timers/promises";
+import { PostStartCauses, PreStopCauses } from "@trigger.dev/core/v3";
+
+const MACHINE_NAME = process.env.MACHINE_NAME || "local";
+const COORDINATOR_PORT = process.env.COORDINATOR_PORT || 8020;
+const COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
+const OTEL_EXPORTER_OTLP_ENDPOINT =
+  process.env.OTEL_EXPORTER_OTLP_ENDPOINT || "http://0.0.0.0:4318";
+
+const logger = new SimpleLogger(`[${MACHINE_NAME}]`);
+
+type InitializeReturn = {
+  canCheckpoint: boolean;
+  willSimulate: boolean;
+};
+
+function isExecaChildProcess(maybeExeca: unknown): maybeExeca is Awaited<ExecaChildProcess> {
+  return typeof maybeExeca === "object" && maybeExeca !== null && "escapedCommand" in maybeExeca;
+}
+
+class DockerTaskOperations implements TaskOperations {
+  #initialized = false;
+  #canCheckpoint = false;
+
+  constructor(private opts = { forceSimulate: false }) {}
+
+  async #initialize(): Promise<InitializeReturn> {
+    if (this.#initialized) {
+      return this.#getInitializeReturn();
+    }
+
+    logger.log("Initializing task operations");
+
+    if (this.opts.forceSimulate) {
+      logger.log("Forced simulation enabled. Will simulate regardless of checkpoint support.");
+    }
+
+    try {
+      await $`criu --version`;
+    } catch (error) {
+      logger.error("No checkpoint support: Missing CRIU binary. Will simulate instead.");
+      this.#canCheckpoint = false;
+      this.#initialized = true;
+
+      return this.#getInitializeReturn();
+    }
+
+    try {
+      await $`docker checkpoint`;
+    } catch (error) {
+      logger.error("No checkpoint support: Docker needs to have experimental features enabled");
+      logger.error("Will simulate instead");
+      this.#canCheckpoint = false;
+      this.#initialized = true;
+
+      return this.#getInitializeReturn();
+    }
+
+    logger.log("Full checkpoint support!");
+
+    this.#initialized = true;
+    this.#canCheckpoint = true;
+
+    return this.#getInitializeReturn();
+  }
+
+  #getInitializeReturn(): InitializeReturn {
+    return {
+      canCheckpoint: this.#canCheckpoint,
+      willSimulate: !this.#canCheckpoint || this.opts.forceSimulate,
+    };
+  }
+
+  async index(opts: TaskOperationsIndexOptions) {
+    await this.#initialize();
+
+    const containerName = this.#getIndexContainerName(opts.shortCode);
+
+    logger.log(`Indexing task ${opts.imageRef}`, {
+      host: COORDINATOR_HOST,
+      port: COORDINATOR_PORT,
+    });
+
+    try {
+      logger.debug(
+        await execa("docker", [
+          "run",
+          "--network=host",
+          "--rm",
+          `--env=INDEX_TASKS=true`,
+          `--env=TRIGGER_SECRET_KEY=${opts.apiKey}`,
+          `--env=TRIGGER_API_URL=${opts.apiUrl}`,
+          `--env=TRIGGER_ENV_ID=${opts.envId}`,
+          `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
+          `--env=POD_NAME=${containerName}`,
+          `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
+          `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
+          `--name=${containerName}`,
+          `${opts.imageRef}`,
+        ])
+      );
+    } catch (error: any) {
+      if (!isExecaChildProcess(error)) {
+        throw error;
+      }
+
+      logger.error("Index failed:", {
+        opts,
+        exitCode: error.exitCode,
+        escapedCommand: error.escapedCommand,
+        stdout: error.stdout,
+        stderr: error.stderr,
+      });
+    }
+  }
+
+  async create(opts: TaskOperationsCreateOptions) {
+    await this.#initialize();
+
+    const containerName = this.#getRunContainerName(opts.runId);
+
+    try {
+      logger.debug(
+        await execa("docker", [
+          "run",
+          "--network=host",
+          "--detach",
+          `--env=TRIGGER_ENV_ID=${opts.envId}`,
+          `--env=TRIGGER_RUN_ID=${opts.runId}`,
+          `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
+          `--env=POD_NAME=${containerName}`,
+          `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
+          `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
+          `--name=${containerName}`,
+          `${opts.image}`,
+        ])
+      );
+    } catch (error) {
+      if (!isExecaChildProcess(error)) {
+        throw error;
+      }
+
+      logger.error("Create failed:", {
+        opts,
+        exitCode: error.exitCode,
+        escapedCommand: error.escapedCommand,
+        stdout: error.stdout,
+        stderr: error.stderr,
+      });
+    }
+  }
+
+  async restore(opts: TaskOperationsRestoreOptions) {
+    await this.#initialize();
+
+    const containerName = this.#getRunContainerName(opts.runId);
+
+    if (!this.#canCheckpoint || this.opts.forceSimulate) {
+      logger.log("Simulating restore");
+
+      const unpause = logger.debug(await $`docker unpause ${containerName}`);
+
+      if (unpause.exitCode !== 0) {
+        throw new Error("docker unpause command failed");
+      }
+
+      await this.#sendPostStart(containerName);
+      return;
+    }
+
+    const { exitCode } = logger.debug(
+      await $`docker start --checkpoint=${opts.checkpointRef} ${containerName}`
+    );
+
+    if (exitCode !== 0) {
+      throw new Error("docker start command failed");
+    }
+
+    await this.#sendPostStart(containerName);
+  }
+
+  async delete(opts: { runId: string }) {
+    await this.#initialize();
+
+    const containerName = this.#getRunContainerName(opts.runId);
+    await this.#sendPreStop(containerName);
+
+    logger.log("noop: delete");
+  }
+
+  async get(opts: { runId: string }) {
+    await this.#initialize();
+
+    logger.log("noop: get");
+  }
+
+  #getIndexContainerName(suffix: string) {
+    return `task-index-${suffix}`;
+  }
+
+  #getRunContainerName(suffix: string) {
+    return `task-run-${suffix}`;
+  }
+
+  async #sendPostStart(containerName: string): Promise<void> {
+    try {
+      const port = await this.#getHttpServerPort(containerName);
+      logger.debug(await this.#runLifecycleCommand(containerName, port, "postStart", "restore"));
+    } catch (error) {
+      logger.error("postStart error", { error });
+      throw new Error("postStart command failed");
+    }
+  }
+
+  async #sendPreStop(containerName: string): Promise<void> {
+    try {
+      const port = await this.#getHttpServerPort(containerName);
+      logger.debug(await this.#runLifecycleCommand(containerName, port, "preStop", "terminate"));
+    } catch (error) {
+      logger.error("preStop error", { error });
+      throw new Error("preStop command failed");
+    }
+  }
+
+  async #getHttpServerPort(containerName: string): Promise<number> {
+    // We first get the correct port, which is random during dev as we run with host networking and need to avoid clashes
+    // FIXME: Skip this in prod
+    const logs = logger.debug(await $`docker logs ${containerName}`);
+    const matches = logs.stdout.match(/http server listening on port (?<port>[0-9]+)/);
+
+    const port = Number(matches?.groups?.port);
+
+    if (!port) {
+      throw new Error("failed to extract port from logs");
+    }
+
+    return port;
+  }
+
+  async #runLifecycleCommand<THookType extends "postStart" | "preStop">(
+    containerName: string,
+    port: number,
+    type: THookType,
+    cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses,
+    retryCount = 0
+  ): Promise<ExecaChildProcess> {
+    try {
+      return await execa("docker", [
+        "exec",
+        containerName,
+        "busybox",
+        "wget",
+        "-q",
+        "-O-",
+        `127.0.0.1:${port}/${type}?cause=${cause}`,
+      ]);
+    } catch (error: any) {
+      if (type === "postStart" && retryCount < 6) {
+        logger.debug(`retriable ${type} error`, { retryCount, message: error?.message });
+        await setTimeout(exponentialBackoff(retryCount + 1, 2, 50, 1150, 50));
+
+        return this.#runLifecycleCommand(containerName, port, type, cause, retryCount + 1);
+      }
+
+      logger.error(`final ${type} error`, { message: error?.message });
+      throw new Error(`${type} command failed after ${retryCount - 1} retries`);
+    }
+  }
+}
+
+const provider = new ProviderShell({
+  tasks: new DockerTaskOperations({ forceSimulate: true }),
+  type: "docker",
+});
+
+provider.listen();
+
+function exponentialBackoff(
+  retryCount: number,
+  exponential: number,
+  minDelay: number,
+  maxDelay: number,
+  jitter: number
+): number {
+  // Calculate the delay using the exponential backoff formula
+  const delay = Math.min(Math.pow(exponential, retryCount) * minDelay, maxDelay);
+
+  // Calculate the jitter
+  const jitterValue = Math.random() * jitter;
+
+  // Return the calculated delay with jitter
+  return delay + jitterValue;
+}

apps/docker-provider/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "compilerOptions": {
+    "target": "es2016",
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "paths": {
+      "@trigger.dev/core/v3": ["../../packages/core/src/v3"],
+      "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"],
+      "@trigger.dev/core-apps": ["../../packages/core-apps/src"],
+      "@trigger.dev/core-apps/*": ["../../packages/core-apps/src/*"]
+    }
+  }
+}

apps/kubernetes-provider/.env.example

@@ -0,0 +1,9 @@
+HTTP_SERVER_PORT=8060
+
+PLATFORM_WS_PORT=3030
+PLATFORM_SECRET=provider-secret
+SECURE_CONNECTION=false
+
+# Use this if you are on macOS
+# COORDINATOR_HOST="host.docker.internal"
+# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318"

apps/kubernetes-provider/.gitignore

@@ -0,0 +1,3 @@
+dist/
+node_modules/
+.env

apps/kubernetes-provider/Containerfile

@@ -0,0 +1,47 @@
+FROM node:18-alpine@sha256:ca9f6cb0466f9638e59e0c249d335a07c867cd50c429b5c7830dda1bed584649 AS node-18-alpine
+
+WORKDIR /app
+
+FROM node-18-alpine AS pruner
+
+COPY --chown=node:node . .
+RUN npx -q turbo@1.10.9 prune --scope=kubernetes-provider --docker
+RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
+
+FROM node-18-alpine AS base
+
+RUN apk add --no-cache dumb-init
+
+COPY --chown=node:node .gitignore .gitignore
+COPY --from=pruner --chown=node:node /app/out/json/ .
+COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
+COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
+
+FROM base AS dev-deps
+RUN corepack enable
+ENV NODE_ENV development
+
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
+
+FROM base AS builder
+RUN corepack enable
+
+COPY --from=pruner --chown=node:node /app/out/full/ .
+COPY --from=dev-deps --chown=node:node /app/ .
+COPY --chown=node:node turbo.json turbo.json
+
+RUN pnpm run -r --filter kubernetes-provider build:bundle
+
+FROM base AS runner
+
+RUN corepack enable
+ENV NODE_ENV production
+
+COPY --from=builder --chown=node:node /app/apps/kubernetes-provider/dist/index.mjs ./index.mjs
+
+EXPOSE 8000
+
+USER node
+
+CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]

apps/kubernetes-provider/README.md

@@ -0,0 +1,3 @@
+# Kubernetes provider
+
+The `kubernetes-provider` allows the platform to be orchestrator-agnostic. The platform can perform actions such as `INDEX_TASKS` or `INVOKE_TASK` which the provider translates into Kubernetes actions.

apps/kubernetes-provider/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "kubernetes-provider",
+  "private": true,
+  "version": "0.0.1",
+  "description": "",
+  "main": "dist/index.cjs",
+  "scripts": {
+    "build": "npm run build:bundle",
+    "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"",
+    "build:image": "docker build -f Containerfile . -t kubernetes-provider",
+    "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
+    "start": "tsx src/index.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@kubernetes/client-node": "^0.20.0",
+    "@trigger.dev/core": "workspace:*",
+    "@trigger.dev/core-apps": "workspace:*",
+    "p-queue": "^8.0.1",
+    "socket.io-client": "^4.7.4"
+  },
+  "devDependencies": {
+    "dotenv": "^16.4.2",
+    "esbuild": "^0.19.11",
+    "tsx": "^4.7.0",
+    "typescript": "^5.3.3"
+  }
+}

apps/kubernetes-provider/src/index.ts

@@ -0,0 +1,544 @@
+import * as k8s from "@kubernetes/client-node";
+import {
+  ProviderShell,
+  SimpleLogger,
+  TaskOperations,
+  TaskOperationsCreateOptions,
+  TaskOperationsIndexOptions,
+  TaskOperationsRestoreOptions,
+} from "@trigger.dev/core-apps";
+import { Machine, PostStartCauses, PreStopCauses, EnvironmentType } from "@trigger.dev/core/v3";
+import { randomUUID } from "crypto";
+import { TaskMonitor } from "./taskMonitor";
+
+const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
+const NODE_NAME = process.env.NODE_NAME || "local";
+const OTEL_EXPORTER_OTLP_ENDPOINT =
+  process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318";
+
+const logger = new SimpleLogger(`[${NODE_NAME}]`);
+logger.log(`running in ${RUNTIME_ENV} mode`);
+
+type Namespace = {
+  metadata: {
+    name: string;
+  };
+};
+
+type ComputeResources = {
+  [K in "cpu" | "memory" | "ephemeral-storage"]?: string;
+};
+
+class KubernetesTaskOperations implements TaskOperations {
+  #namespace: Namespace;
+  #k8sApi: {
+    core: k8s.CoreV1Api;
+    batch: k8s.BatchV1Api;
+  };
+
+  constructor(namespace = "default") {
+    this.#namespace = {
+      metadata: {
+        name: namespace,
+      },
+    };
+
+    this.#k8sApi = this.#createK8sApi();
+  }
+
+  async index(opts: TaskOperationsIndexOptions) {
+    await this.#createJob(
+      {
+        metadata: {
+          name: this.#getIndexContainerName(opts.shortCode),
+          namespace: this.#namespace.metadata.name,
+        },
+        spec: {
+          completions: 1,
+          backoffLimit: 0,
+          ttlSecondsAfterFinished: 300,
+          template: {
+            metadata: {
+              labels: {
+                ...this.#getSharedLabels(opts),
+                app: "task-index",
+                "app.kubernetes.io/part-of": "trigger-worker",
+                "app.kubernetes.io/component": "index",
+                deployment: opts.deploymentId,
+              },
+            },
+            spec: {
+              ...this.#defaultPodSpec,
+              containers: [
+                {
+                  name: this.#getIndexContainerName(opts.shortCode),
+                  image: opts.imageRef,
+                  ports: [
+                    {
+                      containerPort: 8000,
+                    },
+                  ],
+                  resources: {
+                    limits: {
+                      cpu: "250m",
+                      memory: "0.5G",
+                      "ephemeral-storage": "2Gi",
+                    },
+                  },
+                  lifecycle: {
+                    preStop: {
+                      exec: {
+                        command: this.#getLifecycleCommand("preStop", "terminate"),
+                      },
+                    },
+                  },
+                  env: [
+                    ...this.#getSharedEnv(opts.envId),
+                    {
+                      name: "INDEX_TASKS",
+                      value: "true",
+                    },
+                    {
+                      name: "TRIGGER_SECRET_KEY",
+                      value: opts.apiKey,
+                    },
+                    {
+                      name: "TRIGGER_API_URL",
+                      value: opts.apiUrl,
+                    },
+                  ],
+                },
+              ],
+            },
+          },
+        },
+      },
+      this.#namespace
+    );
+  }
+
+  async create(opts: TaskOperationsCreateOptions) {
+    await this.#createPod(
+      {
+        metadata: {
+          name: this.#getRunContainerName(opts.runId),
+          namespace: this.#namespace.metadata.name,
+          labels: {
+            ...this.#getSharedLabels(opts),
+            app: "task-run",
+            "app.kubernetes.io/part-of": "trigger-worker",
+            "app.kubernetes.io/component": "create",
+            run: opts.runId,
+          },
+        },
+        spec: {
+          ...this.#defaultPodSpec,
+          terminationGracePeriodSeconds: 60 * 60,
+          containers: [
+            {
+              name: this.#getRunContainerName(opts.runId),
+              image: opts.image,
+              ports: [
+                {
+                  containerPort: 8000,
+                },
+              ],
+              resources: {
+                requests: {
+                  ...this.#defaultResourceRequests,
+                },
+                limits: {
+                  ...this.#defaultResourceLimits,
+                  ...this.#getResourcesFromMachineConfig(opts.machine),
+                },
+              },
+              lifecycle: {
+                preStop: {
+                  exec: {
+                    command: this.#getLifecycleCommand("preStop", "terminate"),
+                  },
+                },
+              },
+              env: [
+                ...this.#getSharedEnv(opts.envId),
+                {
+                  name: "TRIGGER_RUN_ID",
+                  value: opts.runId,
+                },
+              ],
+              volumeMounts: [
+                {
+                  name: "taskinfo",
+                  mountPath: "/etc/taskinfo",
+                },
+              ],
+            },
+          ],
+          volumes: [
+            {
+              name: "taskinfo",
+              emptyDir: {},
+            },
+          ],
+        },
+      },
+      this.#namespace
+    );
+  }
+
+  async restore(opts: TaskOperationsRestoreOptions) {
+    await this.#createPod(
+      {
+        metadata: {
+          name: `${this.#getRunContainerName(opts.runId)}-${randomUUID().slice(0, 8)}`,
+          namespace: this.#namespace.metadata.name,
+          labels: {
+            ...this.#getSharedLabels(opts),
+            app: "task-run",
+            "app.kubernetes.io/part-of": "trigger-worker",
+            "app.kubernetes.io/component": "restore",
+            run: opts.runId,
+            checkpoint: opts.checkpointId,
+          },
+        },
+        spec: {
+          ...this.#defaultPodSpec,
+          initContainers: [
+            {
+              name: "pull-base-image",
+              image: opts.imageRef,
+              command: ["sleep", "0"],
+            },
+            {
+              name: "populate-taskinfo",
+              image: "busybox",
+              command: ["/bin/sh", "-c"],
+              args: ["printenv COORDINATOR_HOST | tee /etc/taskinfo/coordinator-host"],
+              env: [
+                {
+                  name: "COORDINATOR_HOST",
+                  valueFrom: {
+                    fieldRef: {
+                      fieldPath: "status.hostIP",
+                    },
+                  },
+                },
+              ],
+              volumeMounts: [
+                {
+                  name: "taskinfo",
+                  mountPath: "/etc/taskinfo",
+                },
+              ],
+            },
+          ],
+          containers: [
+            {
+              name: this.#getRunContainerName(opts.runId),
+              image: opts.checkpointRef,
+              ports: [
+                {
+                  containerPort: 8000,
+                },
+              ],
+              resources: {
+                requests: {
+                  ...this.#defaultResourceRequests,
+                },
+                limits: {
+                  ...this.#defaultResourceLimits,
+                  ...this.#getResourcesFromMachineConfig(opts.machine),
+                },
+              },
+              lifecycle: {
+                postStart: {
+                  exec: {
+                    command: this.#getLifecycleCommand("postStart", "restore"),
+                  },
+                },
+                preStop: {
+                  exec: {
+                    command: this.#getLifecycleCommand("preStop", "terminate"),
+                  },
+                },
+              },
+              volumeMounts: [
+                {
+                  name: "taskinfo",
+                  mountPath: "/etc/taskinfo",
+                },
+              ],
+            },
+          ],
+          volumes: [
+            {
+              name: "taskinfo",
+              emptyDir: {},
+            },
+          ],
+        },
+      },
+      this.#namespace
+    );
+  }
+
+  async delete(opts: { runId: string }) {
+    await this.#deletePod({
+      runId: opts.runId,
+      namespace: this.#namespace,
+    });
+  }
+
+  async get(opts: { runId: string }) {
+    await this.#getPod(opts.runId, this.#namespace);
+  }
+
+  #envTypeToLabelValue(type: EnvironmentType) {
+    switch (type) {
+      case "PRODUCTION":
+        return "prod";
+      case "STAGING":
+        return "stg";
+      case "DEVELOPMENT":
+        return "dev";
+      case "PREVIEW":
+        return "preview";
+    }
+  }
+
+  get #defaultPodSpec(): Omit<k8s.V1PodSpec, "containers"> {
+    return {
+      restartPolicy: "Never",
+      automountServiceAccountToken: false,
+      imagePullSecrets: [
+        {
+          name: "registry-trigger",
+        },
+      ],
+      nodeSelector: {
+        nodetype: "worker",
+      },
+    };
+  }
+
+  get #defaultResourceRequests(): ComputeResources {
+    return {
+      "ephemeral-storage": "2Gi",
+    };
+  }
+
+  get #defaultResourceLimits(): ComputeResources {
+    return {
+      "ephemeral-storage": "10Gi",
+    };
+  }
+
+  #getSharedEnv(envId: string): k8s.V1EnvVar[] {
+    return [
+      {
+        name: "TRIGGER_ENV_ID",
+        value: envId,
+      },
+      {
+        name: "DEBUG",
+        value: process.env.DEBUG ? "1" : "0",
+      },
+      {
+        name: "HTTP_SERVER_PORT",
+        value: "8000",
+      },
+      {
+        name: "OTEL_EXPORTER_OTLP_ENDPOINT",
+        value: OTEL_EXPORTER_OTLP_ENDPOINT,
+      },
+      {
+        name: "POD_NAME",
+        valueFrom: {
+          fieldRef: {
+            fieldPath: "metadata.name",
+          },
+        },
+      },
+      {
+        name: "COORDINATOR_HOST",
+        valueFrom: {
+          fieldRef: {
+            fieldPath: "status.hostIP",
+          },
+        },
+      },
+      {
+        name: "MACHINE_NAME",
+        valueFrom: {
+          fieldRef: {
+            fieldPath: "spec.nodeName",
+          },
+        },
+      },
+    ];
+  }
+
+  #getSharedLabels(
+    opts: TaskOperationsIndexOptions | TaskOperationsCreateOptions | TaskOperationsRestoreOptions
+  ): Record<string, string> {
+    return {
+      env: opts.envId,
+      envtype: this.#envTypeToLabelValue(opts.envType),
+      org: opts.orgId,
+      project: opts.projectId,
+    };
+  }
+
+  #getResourcesFromMachineConfig(config: Machine): ComputeResources {
+    return {
+      cpu: `${config.cpu}`,
+      memory: `${config.memory}G`,
+    };
+  }
+
+  #getLifecycleCommand<THookType extends "postStart" | "preStop">(
+    type: THookType,
+    cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses
+  ) {
+    const retries = 5;
+
+    // This will retry sending the lifecycle hook up to `retries` times
+    // The sleep is required as this may start running before the HTTP server is up
+    const exec = [
+      "/bin/sh",
+      "-c",
+      `for i in $(seq ${retries}); do sleep 1; busybox wget -q -O- 127.0.0.1:8000/${type}?cause=${cause} && break; done`,
+    ];
+
+    logger.debug("getLifecycleCommand()", { exec });
+
+    return exec;
+  }
+
+  #getIndexContainerName(suffix: string) {
+    return `task-index-${suffix}`;
+  }
+
+  #getRunContainerName(suffix: string) {
+    return `task-run-${suffix}`;
+  }
+
+  #createK8sApi() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (RUNTIME_ENV === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (RUNTIME_ENV === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${RUNTIME_ENV}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      batch: kubeConfig.makeApiClient(k8s.BatchV1Api),
+    };
+  }
+
+  async #createPod(pod: k8s.V1Pod, namespace: Namespace) {
+    try {
+      const res = await this.#k8sApi.core.createNamespacedPod(namespace.metadata.name, pod);
+      logger.debug(res.body);
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
+    }
+  }
+
+  async #deletePod(opts: { runId: string; namespace: Namespace }) {
+    try {
+      const res = await this.#k8sApi.core.deleteNamespacedPod(
+        opts.runId,
+        opts.namespace.metadata.name
+      );
+      logger.debug(res.body);
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
+    }
+  }
+
+  async #getPod(runId: string, namespace: Namespace) {
+    try {
+      const res = await this.#k8sApi.core.readNamespacedPod(runId, namespace.metadata.name);
+      logger.debug(res.body);
+      return res.body;
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
+    }
+  }
+
+  async #createJob(job: k8s.V1Job, namespace: Namespace) {
+    try {
+      const res = await this.#k8sApi.batch.createNamespacedJob(namespace.metadata.name, job);
+      logger.debug(res.body);
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
+    }
+  }
+
+  #throwUnlessRecord(candidate: unknown): asserts candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      throw candidate;
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    this.#throwUnlessRecord(err);
+
+    if ("body" in err && err.body) {
+      logger.error(err.body);
+      this.#throwUnlessRecord(err.body);
+
+      if (typeof err.body.message === "string") {
+        throw new Error(err.body?.message);
+      } else {
+        throw err.body;
+      }
+    } else {
+      logger.error(err);
+      throw err;
+    }
+  }
+}
+
+const provider = new ProviderShell({
+  tasks: new KubernetesTaskOperations(),
+  type: "kubernetes",
+});
+
+provider.listen();
+
+const taskMonitor = new TaskMonitor({
+  runtimeEnv: RUNTIME_ENV,
+  onIndexFailure: async (deploymentId, failureInfo) => {
+    logger.log("Indexing failed", { deploymentId, failureInfo });
+
+    try {
+      provider.platformSocket.send("INDEXING_FAILED", {
+        deploymentId,
+        error: {
+          name: `Crashed with exit code ${failureInfo.exitCode}`,
+          message: failureInfo.reason,
+          stack: failureInfo.logs,
+        },
+      });
+    } catch (error) {
+      logger.error(error);
+    }
+  },
+  onRunFailure: async (runId, failureInfo) => {
+    logger.log("Run failed:", { runId, failureInfo });
+
+    try {
+      provider.platformSocket.send("WORKER_CRASHED", { runId, ...failureInfo });
+    } catch (error) {
+      logger.error(error);
+    }
+  },
+});
+
+taskMonitor.start();

apps/kubernetes-provider/src/taskMonitor.ts

@@ -0,0 +1,442 @@
+import * as k8s from "@kubernetes/client-node";
+import { SimpleLogger } from "@trigger.dev/core-apps";
+import { setTimeout } from "timers/promises";
+import PQueue from "p-queue";
+
+type IndexFailureHandler = (
+  deploymentId: string,
+  failureInfo: {
+    exitCode: number;
+    reason: string;
+    logs: string;
+  }
+) => Promise<any>;
+
+type RunFailureHandler = (
+  runId: string,
+  failureInfo: {
+    exitCode: number;
+    reason: string;
+    logs: string;
+  }
+) => Promise<any>;
+
+type TaskMonitorOptions = {
+  runtimeEnv: "local" | "kubernetes";
+  onIndexFailure?: IndexFailureHandler;
+  onRunFailure?: RunFailureHandler;
+  namespace?: string;
+};
+
+export class TaskMonitor {
+  #enabled = false;
+  #logger = new SimpleLogger("[TaskMonitor]");
+  #taskInformer: ReturnType<typeof k8s.makeInformer<k8s.V1Pod>>;
+  #processedPods = new Map<string, number>();
+  #queue = new PQueue({ concurrency: 10 });
+  #k8sClient: {
+    core: k8s.CoreV1Api;
+    kubeConfig: k8s.KubeConfig;
+  };
+
+  private namespace = "default";
+  private fieldSelector = "status.phase=Failed";
+  private labelSelector = "app in (task-index, task-run)";
+
+  constructor(private opts: TaskMonitorOptions) {
+    this.#k8sClient = this.#createK8sClient();
+
+    this.#taskInformer = this.#createTaskInformer();
+    this.#taskInformer.on("connect", this.#onInformerConnected.bind(this));
+    this.#taskInformer.on("error", this.#onInformerError.bind(this));
+    this.#taskInformer.on("update", this.#enqueueOnPodUpdated.bind(this));
+  }
+
+  #createTaskInformer() {
+    const listTasks = () =>
+      this.#k8sClient.core.listNamespacedPod(
+        this.namespace,
+        undefined,
+        undefined,
+        undefined,
+        this.fieldSelector,
+        this.labelSelector
+      );
+
+    // Uses watch with local caching
+    // https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes
+    const informer = k8s.makeInformer(
+      this.#k8sClient.kubeConfig,
+      `/api/v1/namespaces/${this.namespace}/pods`,
+      listTasks,
+      this.labelSelector,
+      this.fieldSelector
+    );
+
+    return informer;
+  }
+
+  async #onInformerConnected() {
+    this.#logger.log("Connected");
+  }
+
+  async #onInformerError(error: any) {
+    this.#logger.error("Error:", error);
+
+    // Automatic reconnect
+    await setTimeout(2_000);
+    this.#taskInformer.start();
+  }
+
+  #enqueueOnPodUpdated(pod: k8s.V1Pod) {
+    this.#queue.add(async () => {
+      try {
+        // It would be better to only pass the cache key, but the pod may already be removed from the cache by the time we process it
+        await this.#onPodUpdated(pod);
+      } catch (error) {
+        this.#logger.error("Caught onPodUpdated() error:", error);
+      }
+    });
+  }
+
+  async #onPodUpdated(pod: k8s.V1Pod) {
+    this.#logger.debug(`Updated: ${pod.metadata?.name}`);
+    this.#logger.debug("Updated", JSON.stringify(pod, null, 2));
+
+    // We only care about failures
+    if (pod.status?.phase !== "Failed") {
+      return;
+    }
+
+    const podName = pod.metadata?.name;
+
+    if (!podName) {
+      this.#logger.error("Pod is nameless", { pod });
+      return;
+    }
+
+    const containerStatus = pod.status.containerStatuses?.[0];
+
+    if (!containerStatus?.state) {
+      this.#logger.error("Pod failed, but container status doesn't have state", {
+        status: pod.status,
+      });
+      return;
+    }
+
+    if (this.#processedPods.has(podName)) {
+      this.#logger.debug("Pod update already processed", {
+        podName,
+        timestamp: this.#processedPods.get(podName),
+      });
+      return;
+    }
+
+    this.#processedPods.set(podName, Date.now());
+
+    const podStatus = this.#getPodStatusSummary(pod.status);
+    const containerState = this.#getContainerStateSummary(containerStatus.state);
+    const exitCode = containerState.exitCode ?? -1;
+
+    // We use this special exit code to signal any errors were already handled elsewhere
+    if (exitCode === 111) {
+      return;
+    }
+
+    const rawLogs = await this.#getLogTail(podName);
+
+    this.#logger.log(`${podName} failed with:`, {
+      podStatus,
+      containerState,
+      rawLogs,
+    });
+
+    const rawReason = podStatus.reason ?? containerState.reason ?? "";
+    const message = podStatus.message ?? containerState.message ?? "";
+
+    let reason = rawReason || "Unknown error";
+    let logs = rawLogs || "";
+
+    switch (rawReason) {
+      case "Error":
+        reason = "Unknown error.";
+        break;
+      case "Evicted":
+        if (message.startsWith("Pod ephemeral local storage usage")) {
+          reason = "Storage limit exceeded.";
+        } else if (message) {
+          reason = `Evicted: ${message}`;
+        } else {
+          reason = "Evicted for unknown reason.";
+        }
+
+        if (logs.startsWith("failed to try resolving symlinks")) {
+          logs = "";
+        }
+        break;
+      case "OOMKilled":
+        reason = "Out of memory! Try increasing the memory on this task.";
+        break;
+      default:
+        break;
+    }
+
+    const failureInfo = {
+      exitCode,
+      reason,
+      logs,
+    };
+
+    const app = pod.metadata?.labels?.app;
+
+    switch (app) {
+      case "task-index":
+        const deploymentId = pod.metadata?.labels?.deployment;
+
+        if (!deploymentId) {
+          this.#logger.error("Index is missing ID", { pod });
+          return;
+        }
+
+        if (this.opts.onIndexFailure) {
+          await this.opts.onIndexFailure(deploymentId, failureInfo);
+        }
+        break;
+      case "task-run":
+        const runId = pod.metadata?.labels?.run;
+
+        if (!runId) {
+          this.#logger.error("Run is missing ID", { pod });
+          return;
+        }
+
+        if (this.opts.onRunFailure) {
+          await this.opts.onRunFailure(runId, failureInfo);
+        }
+        break;
+      default:
+        this.#logger.error("Pod has invalid app label", { pod });
+        return;
+    }
+
+    await this.#deletePod(podName);
+  }
+
+  async #getLogTail(podName: string) {
+    try {
+      const logs = await this.#k8sClient.core.readNamespacedPodLog(
+        podName,
+        this.namespace,
+        undefined,
+        undefined,
+        undefined,
+        1024, // limitBytes
+        undefined,
+        undefined,
+        undefined,
+        20 // tailLines
+      );
+
+      const responseBody = logs.body ?? "";
+
+      if (responseBody.startsWith("unable to retrieve container logs")) {
+        return "";
+      }
+
+      // Type is wrong, body may be undefined
+      return responseBody;
+    } catch (error) {
+      this.#logger.error("Log tail error:", error instanceof Error ? error.message : "unknown");
+      return "";
+    }
+  }
+
+  #getPodStatusSummary(status: k8s.V1PodStatus) {
+    return {
+      reason: status.reason,
+      message: status.message,
+    };
+  }
+
+  #getContainerStateSummary(state: k8s.V1ContainerState) {
+    return {
+      reason: state.terminated?.reason,
+      exitCode: state.terminated?.exitCode,
+      message: state.terminated?.message,
+    };
+  }
+
+  #createK8sClient() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (this.opts.runtimeEnv === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (this.opts.runtimeEnv === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      kubeConfig: kubeConfig,
+    };
+  }
+
+  #isRecord(candidate: unknown): candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  #logK8sError(err: unknown, debugOnly = false) {
+    if (debugOnly) {
+      this.#logger.debug("K8s API Error", err);
+    } else {
+      this.#logger.error("K8s API Error", err);
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
+      this.#logK8sError(err);
+      return;
+    }
+
+    this.#logK8sError(err, true);
+
+    if (typeof err.body.message === "string") {
+      this.#logK8sError({ message: err.body.message });
+      return;
+    }
+
+    this.#logK8sError({ body: err.body });
+  }
+
+  #printStats(includeMoreDetails = false) {
+    this.#logger.log("Stats:", {
+      cacheSize: this.#taskInformer.list().length,
+      totalProcessed: this.#processedPods.size,
+      ...(includeMoreDetails && {
+        processedPods: this.#processedPods,
+      }),
+    });
+  }
+
+  async #deletePod(name: string) {
+    this.#logger.debug("Deleting pod:", name);
+
+    await this.#k8sClient.core
+      .deleteNamespacedPod(name, this.namespace)
+      .catch(this.#handleK8sError.bind(this));
+  }
+
+  async start() {
+    this.#enabled = true;
+
+    const interval = setInterval(() => {
+      if (!this.#enabled) {
+        clearInterval(interval);
+        return;
+      }
+
+      this.#printStats();
+    }, 300_000);
+
+    await this.#taskInformer.start();
+
+    // this.#launchTests();
+  }
+
+  async stop() {
+    if (!this.#enabled) {
+      return;
+    }
+
+    this.#enabled = false;
+    this.#logger.log("Shutting down..");
+
+    await this.#taskInformer.stop();
+
+    this.#printStats(true);
+  }
+
+  async #launchTests() {
+    const createPod = async (
+      container: k8s.V1Container,
+      name: string,
+      labels?: Record<string, string>
+    ) => {
+      this.#logger.log("Creating pod:", name);
+
+      const pod = {
+        metadata: {
+          name,
+          labels,
+        },
+        spec: {
+          restartPolicy: "Never",
+          automountServiceAccountToken: false,
+          terminationGracePeriodSeconds: 1,
+          containers: [container],
+        },
+      } satisfies k8s.V1Pod;
+
+      await this.#k8sClient.core
+        .createNamespacedPod(this.namespace, pod)
+        .catch(this.#handleK8sError.bind(this));
+    };
+
+    const createOomPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "polinux/stress",
+        resources: {
+          limits: {
+            memory: "100Mi",
+          },
+        },
+        command: ["stress"],
+        args: ["--vm", "1", "--vm-bytes", "150M", "--vm-hang", "1"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createNonZeroExitPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "busybox",
+        command: ["sh"],
+        args: ["-c", "exit 1"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createOoDiskPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "busybox",
+        command: ["sh"],
+        args: [
+          "-c",
+          "echo creating huge-file..; head -c 1000m /dev/zero > huge-file; ls -lh huge-file; sleep infinity",
+        ],
+        resources: {
+          limits: {
+            "ephemeral-storage": "500Mi",
+          },
+        },
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    await createNonZeroExitPod("non-zero-exit-task", { app: "task-run", run: "123" });
+    await createOomPod("oom-task", { app: "task-index", deployment: "456" });
+    await createOoDiskPod("ood-task", { app: "task-run", run: "abc" });
+  }
+}

apps/kubernetes-provider/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "compilerOptions": {
+    "target": "es2016",
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "paths": {
+      "@trigger.dev/core/v3": ["../../packages/core/src/v3"],
+      "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"],
+      "@trigger.dev/core-apps": ["../../packages/core-apps/src"],
+      "@trigger.dev/core-apps/*": ["../../packages/core-apps/src/*"]
+    }
+  }
+}