Release 4.0.0-v4-beta.16 (#2053)

* chore: Update version for release (v4-beta)

* Release 4.0.0-v4-beta.16

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: nicktrn <55853254+nicktrn@users.noreply.github.com>

.changeset/blue-eyes-tickle.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+The dev command will now use the platform-provided engine URL

.changeset/breezy-turtles-talk.md

@@ -0,0 +1,10 @@
+---
+"@trigger.dev/react-hooks": patch
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/build": patch
+"@trigger.dev/core": patch
+"@trigger.dev/rsc": patch
+---
+
+Run Engine 2.0 (alpha)

.changeset/config.json

@@ -1,28 +1,25 @@
 {
   "$schema": "https://unpkg.com/@changesets/config@2.2.0/schema.json",
-  "changelog": "@changesets/cli/changelog",
-  "commit": false,
-  "fixed": [
-    [
-      "@trigger.dev/*",
-      "trigger.dev"
-    ]
+  "changelog": [
+    "@remix-run/changelog-github",
+    {
+      "repo": "triggerdotdev/trigger.dev"
+    }
   ],
+  "commit": false,
+  "fixed": [["@trigger.dev/*", "trigger.dev"]],
   "linked": [],
   "access": "public",
   "baseBranch": "main",
   "updateInternalDependencies": "patch",
   "ignore": [
     "webapp",
-    "emails",
-    "proxy",
-    "@trigger.dev/database",
     "coordinator",
     "docker-provider",
     "kubernetes-provider",
-    "@trigger.dev/otlp-importer"
+    "supervisor"
   ],
   "___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH": {
     "onlyUpdatePeerDependentsWhenOutOfRange": true
   }
-}
+}

.changeset/cuddly-boats-press.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Add external log exporters and fix missing external trace exporters in deployed tasks

.changeset/curvy-dogs-share.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+When you create a Waitpoint token using `wait.createToken()` you get a URL back that can be used to complete it by making an HTTP POST request.

.changeset/eighty-rings-divide.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Configurable queue consumer count in supervisor session

.changeset/four-needles-add.md

@@ -0,0 +1,12 @@
+---
+"@trigger.dev/redis-worker": major
+"@trigger.dev/react-hooks": major
+"@trigger.dev/sdk": major
+"trigger.dev": major
+"@trigger.dev/python": major
+"@trigger.dev/build": major
+"@trigger.dev/core": major
+"@trigger.dev/rsc": major
+---
+
+Trigger.dev v4 release. Please see our upgrade to v4 docs to view the full changelog: https://trigger.dev/docs/upgrade-to-v4

.changeset/gentle-waves-suffer.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Fixed an issue with realtime streams that timeout and resume streaming dropping chunks

.changeset/green-lions-relate.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+The envvars.list() and retrieve() functions receive isSecret for each value. Secret values are always redacted.

.changeset/hip-cups-wave.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Fix issue where realtime streams would cut off after 5 minutes

.changeset/honest-files-decide.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Deprecate toolTask and replace with `ai.tool(mySchemaTask)`

.changeset/itchy-frogs-care.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Log images sizes for self-hosted deploys

.changeset/itchy-games-sort.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+---
+
+Display clickable links in Cursor terminal

.changeset/late-chairs-ring.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix init.ts in custom trigger dirs

.changeset/moody-squids-count.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Init command will now correctly install v4-beta packages

.changeset/nice-colts-boil.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Improve warm start times by eagerly creating the child TaskRunProcess when a previous run as completed

.changeset/orange-rocks-grow.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix init.ts detection when using the sentry esbuild plugin

.changeset/plenty-dolphins-act.md

@@ -0,0 +1,8 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+- Correctly resolve waitpoints that come in early
+- Ensure correct state before requesting suspension
+- Fix race conditions in snapshot processing

.changeset/polite-impalas-care.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fixes runLimiter check on #dequeueRuns

.changeset/polite-lies-fix.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Update nypm package to support test-based bun.lock files

.changeset/pre.json

@@ -0,0 +1,60 @@
+{
+  "mode": "pre",
+  "tag": "v4-beta",
+  "initialVersions": {
+    "coordinator": "0.0.1",
+    "docker-provider": "0.0.1",
+    "kubernetes-provider": "0.0.1",
+    "supervisor": "0.0.1",
+    "webapp": "1.0.0",
+    "@trigger.dev/build": "3.3.17",
+    "trigger.dev": "3.3.17",
+    "@trigger.dev/core": "3.3.17",
+    "@trigger.dev/python": "3.3.17",
+    "@trigger.dev/react-hooks": "3.3.17",
+    "@trigger.dev/redis-worker": "3.3.17",
+    "@trigger.dev/rsc": "3.3.17",
+    "@trigger.dev/sdk": "3.3.17"
+  },
+  "changesets": [
+    "blue-eyes-tickle",
+    "breezy-turtles-talk",
+    "cuddly-boats-press",
+    "curvy-dogs-share",
+    "eighty-rings-divide",
+    "four-needles-add",
+    "gentle-waves-suffer",
+    "green-lions-relate",
+    "hip-cups-wave",
+    "honest-files-decide",
+    "itchy-frogs-care",
+    "itchy-games-sort",
+    "late-chairs-ring",
+    "moody-squids-count",
+    "nice-colts-boil",
+    "orange-rocks-grow",
+    "plenty-dolphins-act",
+    "polite-impalas-care",
+    "polite-lies-fix",
+    "real-rats-drop",
+    "red-chairs-begin",
+    "red-wasps-cover",
+    "shiny-kiwis-beam",
+    "smart-coins-hammer",
+    "sour-mirrors-accept",
+    "spotty-ducks-punch",
+    "spotty-pants-wink",
+    "sweet-dolphins-invent",
+    "tender-jobs-collect",
+    "thick-bikes-laugh",
+    "tidy-books-smell",
+    "tiny-buckets-teach",
+    "tricky-houses-invite",
+    "twelve-actors-hide",
+    "two-tigers-dream",
+    "weak-jobs-hide",
+    "wet-deers-think",
+    "wet-steaks-reflect",
+    "witty-donkeys-unite"
+  ]
+}

.changeset/real-rats-drop.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Add onCancel lifecycle hook

.changeset/red-chairs-begin.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Added AI assistance link when you have build errors

.changeset/red-wasps-cover.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Suppress external instrumentation for fetch calls from ApiClient

.changeset/shiny-kiwis-beam.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Handle flush errors gracefully in dev

.changeset/smart-coins-hammer.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+fix: Realtime streams: prevent enqueuing into closed ReadableStream

.changeset/sour-mirrors-accept.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Improve usage flushing

.changeset/spotty-ducks-punch.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+fix: default machine config indexing now works

.changeset/spotty-pants-wink.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Prevent large outputs from overwriting each other

.changeset/sweet-dolphins-invent.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Always print full deploy logs in CI

.changeset/tender-jobs-collect.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+TriggerApiError 4xx errors will no longer cause tasks to be retried

.changeset/thick-bikes-laugh.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/build": patch
+---
+
+Add ffmpeg v7 support to existing extension: `ffmpeg({ version: "7" })`

.changeset/tidy-books-smell.md

@@ -0,0 +1,8 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+- Fix polling interval reset bug that could create duplicate intervals
+- Protect against unexpected attempt number changes
+- Prevent run execution zombies after warm starts

.changeset/tiny-buckets-teach.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix stalled run detection

.changeset/tricky-houses-invite.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Managed run controller performance and reliability improvements

.changeset/twelve-actors-hide.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix init.ts auto-import for deployed workers

.changeset/two-tigers-dream.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+maintain proper context in metadata.root and parent getters

.changeset/weak-jobs-hide.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+v4: New lifecycle hooks

.changeset/wet-deers-think.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Fix QUEUED status snapshot handler

.changeset/wet-steaks-reflect.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+If you pass a directory when calling deploy we validate it exists and give helpful hints

.changeset/witty-donkeys-unite.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/build": patch
+---
+
+Add playwright extension

.configs/tsconfig.base.json

@@ -1,7 +1,7 @@
 {
   "compilerOptions": {
     "target": "es2022",
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "lib": ["ES2022", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
     "module": "NodeNext",
     "moduleResolution": "NodeNext",
     "moduleDetection": "force",
@@ -10,7 +10,7 @@
 
     "strict": true,
     "alwaysStrict": true,
-    "strictPropertyInitialization": false,
+    "strictPropertyInitialization": true,
     "skipLibCheck": true,
     "forceConsistentCasingInFileNames": true,
     "noUnusedLocals": false,

.cursor/mcp.json

@@ -0,0 +1,7 @@
+{
+  "mcpServers": {
+    "trigger.dev": {
+      "url": "http://localhost:3333/sse"
+    }
+  }
+}

.cursor/rules/executing-commands.mdc

@@ -0,0 +1,24 @@
+---
+description: how to run commands in the monorepo
+globs: 
+alwaysApply: true
+---
+Almost all commands in the monorepo should be executed when `pnpm run ...` from the root of the monorepo. For example, running tests for the `@internal/run-engine` internal package:
+
+```
+pnpm run dev --filter webapp
+```
+
+But often, when running tests, it's better to `cd` into the directory and then run tests:
+
+```
+cd apps/webapp
+pnpm run test --run
+```
+
+This way you can run for a single file easily:
+
+```
+cd internal-packages/run-engine
+pnpm run test ./src/engine/tests/ttl.test.ts --run
+```

.cursor/rules/repo.mdc

@@ -0,0 +1,6 @@
+---
+description: understanding the structure of the monorepo
+globs: 
+alwaysApply: true
+---
+We've documented the structure of our monorepo here: [repo.md](mdc:ai/references/repo.md)

.cursor/rules/webapp.mdc

@@ -0,0 +1,37 @@
+---
+description: Making updates to the main trigger.dev remix webapp
+globs: apps/webapp/**/*.tsx,apps/webapp/**/*.ts
+alwaysApply: false
+---
+
+The main trigger.dev webapp, which powers it's API and dashboard and makes up the docker image that is produced as an OSS image, is a Remix 2.1.0 app that uses an express server, written in TypeScript. The following subsystems are either included in the webapp or are used by the webapp in another part of the monorepo:
+
+- `@trigger.dev/database` exports a Prisma 5.4.1 client that is used extensively in the webapp to access a PostgreSQL instance. The schema file is [schema.prisma](mdc:internal-packages/database/prisma/schema.prisma)
+- `@trigger.dev/core` is a published package and is used to share code between the `@trigger.dev/sdk` and the webapp. It includes functionality but also a load of Zod schemas for data validation. When importing from `@trigger.dev/core` in the webapp, we never import the root `@trigger.dev/core` path, instead we favor one of the subpath exports that you can find in [package.json](mdc:packages/core/package.json)
+- `@internal/run-engine` has all the code needed to trigger a run and take it through it's lifecycle to completion.
+- `@trigger.dev/redis-worker` is a custom redis based background job/worker system that's used in the webapp and also used inside the run engine.
+
+## Environment variables and testing
+
+In the webapp, all environment variables are accessed through the `env` export of [env.server.ts](mdc:apps/webapp/app/env.server.ts), instead of directly accessing `process.env`.
+
+Ideally, the `env.server.ts` file would never be imported into a test file, either directly or indirectly. Tests should only imported classes and functions from a file matching `app/**/*.ts` of the webapp, and that file should not use environment variables, everything should be passed through as options instead. This "service/configuration" separation is important, and can be seen in a few places in the code for examples:
+
+- [realtimeClient.server.ts](mdc:apps/webapp/app/services/realtimeClient.server.ts) is the testable service, and [realtimeClientGlobal.server.ts](mdc:apps/webapp/app/services/realtimeClientGlobal.server.ts) is the configuration
+
+Also for writing tests in the webapp, checkout our [tests.md](mdc:ai/references/tests.md) guide
+
+## Legacy run engine vs Run Engine 2.0
+
+We originally the Trigger.dev "Run Engine" not as a single system, but just spread out all over the codebase, with no real separate or encapsulation. And we didn't even call it a "Run Engine". With Run Engine 2.0, we've completely rewritten big parts of the way the system works, and moved it over to an internal package called `@internal/run-engine`. So we've retroactively named the previous run engine "Legacy run engine". We're focused almost exclusively now on moving to Run Engine 2.0 and will be deprecating and removing the legacy run engine code eventually.
+
+## Where to look for code
+
+- The trigger API endpoint is [api.v1.tasks.$taskId.trigger.ts](mdc:apps/webapp/app/routes/api.v1.tasks.$taskId.trigger.ts)
+- The batch trigger API endpoint is [api.v1.tasks.batch.ts](mdc:apps/webapp/app/routes/api.v1.tasks.batch.ts)
+- Setup code for the prisma client is in [db.server.ts](mdc:apps/webapp/app/db.server.ts)
+- The run engine is configured in [runEngine.server.ts](mdc:apps/webapp/app/v3/runEngine.server.ts)
+- All the "services" that are found in app/v3/services/**/*.server.ts
+- The code for the TaskEvent data, which is the otel data sent from tasks to our servers, is in both the [eventRepository.server.ts](mdc:apps/webapp/app/v3/eventRepository.server.ts) and also the [otlpExporter.server.ts](mdc:apps/webapp/app/v3/otlpExporter.server.ts). The otel endpoints which are hit from production and development otel exporters is [otel.v1.logs.ts](mdc:apps/webapp/app/routes/otel.v1.logs.ts) and [otel.v1.traces.ts](mdc:apps/webapp/app/routes/otel.v1.traces.ts)
+- We use "presenters" to move more complex loader code into a class, and you can find those are app/v3/presenters/**/*.server.ts
+

.cursor/rules/writing-tests.mdc

@@ -0,0 +1,6 @@
+---
+description: How to write tests in the monorepo
+globs: 
+alwaysApply: true
+---
+Follow our [tests.md](mdc:ai/references/tests.md) guide for how to write tests in the monorepo.

.cursorignore

@@ -0,0 +1,7 @@
+apps/docker-provider/
+apps/kubernetes-provider/
+apps/proxy/
+apps/coordinator/
+packages/rsc/
+.changeset
+.zed

.dockerignore

@@ -1,44 +1,23 @@
-\*.log
-.git
-.github
+**/*.log
+**/*.pem
+**/*.tsbuildinfo
 
-# editor
+**/.cache
+**/.env
+**/.next
+**/.output
+**/.trigger
+**/.tshy
+**/.tshy-build
+**/.turbo
+**/.vercel
+**/.wrangler
 
-.idea
-.vscode
+**/dist
+**/node_modules
 
-# dependencies
-
-node_modules
-.pnp
-.pnp.js
-
-# testing
-
-coverage
-
-# next.js
-
-.next/
-build
-
-# packages
-
-build
-dist
-packages/\*\*/dist
-
-# misc
-
-.DS_Store
-\*.pem
-
-.turbo
-.vercel
-.cache
-.output
-.trigger
-apps/\*\*/public/build
+apps/webapp/build
+apps/webapp/public/build
 
 cypress/screenshots
 cypress/videos
@@ -47,8 +26,21 @@ apps/**/styles/tailwind.css
 packages/**/styles/tailwind.css
 
 .changeset
-references
+.DS_Store
+.git
+.github
+.idea
+.pnp
+.pnp.js
+.vscode
+
+coverage
+build
+docs
 examples
+out
+references
+
 CHANGESETS.md
 CONTRIBUTING.md
 README.md

.env.example

@@ -31,10 +31,32 @@ DEV_OTEL_BATCH_PROCESSING_ENABLED="0"
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
 
-# Resend is an email service used for signing in to Trigger.dev via a Magic Link.
-# Emails will print to the console if you leave these commented out
+# Configure an email transport to allow users to sign in to Trigger.dev via a Magic Link.
+# If none are configured, emails will print to the console instead.
+# Uncomment one of the following blocks to allow delivery of
+
+# Resend
 ### Visit https://resend.com, create an account and get your API key. Then insert it below along with your From and Reply To email addresses. Visit https://resend.com/docs for more information.
-# RESEND_API_KEY=<api_key>
+# EMAIL_TRANSPORT=resend
+# FROM_EMAIL=
+# REPLY_TO_EMAIL=
+# RESEND_API_KEY=
+
+# Generic SMTP
+### Enter the configuration provided by your mail provider. Visit https://nodemailer.com/smtp/ for more information
+### SMTP_SECURE = false will use STARTTLS when connecting to a server that supports it (usually port 587)
+# EMAIL_TRANSPORT=smtp
+# FROM_EMAIL=
+# REPLY_TO_EMAIL=
+# SMTP_HOST=
+# SMTP_PORT=587
+# SMTP_SECURE=false
+# SMTP_USER=
+# SMTP_PASSWORD=
+
+# AWS Simple Email Service
+### Authentication is configured using the default Node.JS credentials provider chain (https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/Package/-aws-sdk-credential-providers/#fromnodeproviderchain)
+# EMAIL_TRANSPORT=aws-ses
 # FROM_EMAIL=
 # REPLY_TO_EMAIL=
 

.eslintrc.js

@@ -1,14 +0,0 @@
-module.exports = {
-  root: true,
-  // This tells ESLint to load the config from the package `eslint-config-custom`
-  extends: ["custom"],
-  settings: {
-    next: {
-      rootDir: ["apps/*/"],
-    },
-  },
-  parserOptions: {
-    sourceType: "module",
-    ecmaVersion: 2020,
-  },
-};

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [triggerdotdev]

.github/actions/get-image-tag/action.yml

@@ -0,0 +1,89 @@
+name: "#️⃣ Get image tag (action)"
+
+description: This action gets the image tag from the commit ref or input (if provided)
+
+outputs:
+  tag:
+    description: The image tag
+    value: ${{ steps.get_tag.outputs.tag }}
+  is_semver:
+    description: Whether the tag is a semantic version
+    value: ${{ steps.check_semver.outputs.is_semver }}
+
+inputs:
+  tag:
+    description: The image tag. If this is set it will return the tag as is.
+    required: false
+    default: ""
+
+runs:
+  using: "composite"
+  steps:
+    - name: "#️⃣ Get image tag (step)"
+      id: get_tag
+      shell: bash
+      run: |
+        if [[ -n "${{ inputs.tag }}" ]]; then
+          tag="${{ inputs.tag }}"
+        elif [[ "${{ github.ref_type }}" == "tag" ]]; then
+          if [[ "${{ github.ref_name }}" == infra-*-* ]]; then
+            env=$(echo ${{ github.ref_name }} | cut -d- -f2)
+            sha=$(echo ${{ github.sha }} | head -c7)
+            ts=$(date +%s)
+            tag=${env}-${sha}-${ts}
+          elif [[ "${{ github.ref_name }}" == re2-*-* ]]; then
+            env=$(echo ${{ github.ref_name }} | cut -d- -f2)
+            sha=$(echo ${{ github.sha }} | head -c7)
+            ts=$(date +%s)
+            tag=${env}-${sha}-${ts}
+          elif [[ "${{ github.ref_name }}" == v.docker.* ]]; then
+            version="${GITHUB_REF_NAME#v.docker.}"
+            tag="v${version}"
+          elif [[ "${{ github.ref_name }}" == build-* ]]; then
+            tag="${GITHUB_REF_NAME#build-}"
+          else
+            echo "Invalid git tag: ${{ github.ref_name }}"
+            exit 1
+          fi
+        elif [[ "${{ github.ref_name }}" == "main" ]]; then
+          tag="main"
+        else
+          echo "Invalid git ref: ${{ github.ref }}"
+          exit 1
+        fi
+        echo "tag=${tag}" >> "$GITHUB_OUTPUT"
+
+    - name: 🔍 Check for validity
+      id: check_validity
+      shell: bash
+      env:
+        tag: ${{ steps.get_tag.outputs.tag }}
+      run: |
+        if [[ "${tag}" =~ ^[a-z0-9]+([._-][a-z0-9]+)*$ ]]; then
+          echo "Tag is valid: ${tag}"
+        else
+          echo "Tag is not valid: ${tag}"
+          exit 1
+        fi
+
+    - name: 🆚 Check for semver
+      id: check_semver
+      shell: bash
+      env:
+        tag: ${{ steps.get_tag.outputs.tag }}
+      # Will match most semver formats except build metadata, i.e. v1.2.3+build.1
+      # Valid matches:
+      # v1.2.3
+      # v1.2.3-alpha
+      # v1.2.3-alpha.1
+      # v1.2.3-rc.1
+      # v1.2.3-beta-1
+      run: |
+        if [[ "${tag}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ ]]; then
+          echo "Tag is a semantic version: ${tag}"
+          is_semver=true
+        else
+          echo "Tag is not a semantic version: ${tag}"
+          is_semver=false
+        fi
+        echo "is_semver=${is_semver}" >> "$GITHUB_OUTPUT"

.github/copilot-instructions.md

@@ -0,0 +1 @@
+This is the repo for Trigger.dev, a background jobs platform written in TypeScript. Our webapp at apps/webapp is a Remix 2.1 app that uses Node.js v20. Our SDK is an isomorphic TypeScript SDK at packages/trigger-sdk. Always prefer using isomorphic code like fetch, ReadableStream, etc. instead of Node.js specific code. Our tests are all vitest. We use prisma in internal-packages/database for our database interactions using PostgreSQL. For TypeScript, we usually use types over interfaces. We use zod a lot in packages/core and in the webapp. Avoid enums. Use strict mode. No default exports, use function declarations.

.github/test/README.md

@@ -0,0 +1,70 @@
+# GitHub Action Tests
+
+This directory contains necessary files to allow local testing of GitHub Actions workflows, composite actions, etc. You will need to install [act](https://github.com/nektos/act) to perform tests.
+
+## Workflow tests
+
+Trigger specific workflow files by specifying their full path:
+
+```
+act -W .github/workflow/release.yml
+```
+
+You will likely need to override any custom runners we use, e.g. buildjet. For example:
+
+```
+override=catthehacker/ubuntu:act-latest
+
+act -W .github/workflow/release.yml \
+    -P buildjet-8vcpu-ubuntu-2204=$override
+
+# override multiple images at the same time
+act -W .github/workflow/release.yml \
+    -P buildjet-8vcpu-ubuntu-2204=$override \
+    -P buildjet-16vcpu-ubuntu-2204=$override
+```
+
+Trigger with specific event payloads to test pushing to branches or tags:
+
+```
+override=catthehacker/ubuntu:act-latest
+
+# simulate push to main
+act -W .github/workflow/publish.yml \
+    -P buildjet-8vcpu-ubuntu-2204=$override \
+    -P buildjet-16vcpu-ubuntu-2204=$override \
+    -e .github/events/push-tag-main.json
+
+# simulate a `build-` prefixed tag
+act -W .github/workflow/publish.yml \
+    -P buildjet-8vcpu-ubuntu-2204=$override \
+    -P buildjet-16vcpu-ubuntu-2204=$override \
+    -e .github/events/push-tag-buld.json
+```
+
+By default, `act` will send a push event. To trigger a different event:
+
+```
+# basic syntax
+act <EVENT> ...
+
+# simulate a pull request
+act pull_request
+
+# only trigger a specific workflow
+act pull_request -W .github/workflow/pr_checks.yml
+```
+
+## Composite action tests
+
+The composite (custom) action tests can be run by triggering the `test-actions` workflow:
+
+```
+act -W .github/test/test-actions.yml
+```
+
+## Helpful flags
+
+- `--pull=false` - perform fully offline tests if all images are already present
+- `-j <job_name>` - run the specified job only
+- `-l push` - list all workflows with push triggers

.github/test/events/push-main.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/heads/main"
+}

.github/test/events/push-tag-build.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/build-buildtag"
+}

.github/test/events/push-tag-docker-nonsemver.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/v.docker.nonsemver"
+}

.github/test/events/push-tag-docker.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/v.docker.1.2.3"
+}

.github/test/events/push-tag-infra-prod.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/infra-prod-anything"
+}

.github/test/events/push-tag-infra-test.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/infra-test-anything"
+}

.github/test/events/push-tag-semver.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/1.2.3"
+}

.github/test/events/push-tag.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/standard-tag"
+}

.github/test/test-actions.yml

@@ -0,0 +1,152 @@
+name: Test Actions
+
+on:
+  workflow_dispatch:
+
+jobs:
+  get-image-tag-none:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Log current ref
+        run: |
+          echo "ref: ${{ github.ref }}"
+          echo "ref_type: ${{ github.ref_type }}"
+          echo "ref_name: ${{ github.ref_name }}"
+
+      - name: Run without input tag
+        id: get_tag
+        # this step may fail depending on the current ref
+        continue-on-error: true
+        uses: ./.github/actions/get-image-tag
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-null:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Log current ref
+        run: |
+          echo "ref: ${{ github.ref }}"
+          echo "ref_type: ${{ github.ref_type }}"
+          echo "ref_name: ${{ github.ref_name }}"
+
+      - name: Run without input tag
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        # this step may fail depending on the current ref
+        continue-on-error: true
+        with:
+          # this should behave exactly as when no tag is provided
+          tag: null
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-override:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with tag override
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: "abc-123"
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-invalid-string:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with invalid string
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        # this step is expected to fail
+        continue-on-error: true
+        with:
+          # does not end with alphanumeric character
+          tag: "abc-123-"
+
+      - name: Fail job if previous step did not fail
+        if: steps.get_tag.outcome != 'failure'
+        run: exit 1
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-prerelease:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with prerelease semver
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: "v1.2.3-beta.4"
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-semver:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with basic semver
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: "v1.2.3"
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-invalid-semver:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with invalid semver
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        # this step is expected to fail
+        continue-on-error: true
+        with:
+          tag: "v1.2.3-"
+
+      - name: Fail job if previous step did not fail
+        if: steps.get_tag.outcome != 'failure'
+        run: exit 1
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"

.github/workflows/docs.yml

@@ -0,0 +1,42 @@
+name: 📚 Docs Checks
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "docs/**"
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - "docs/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  check-broken-links:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./docs
+    steps:
+      - name: 📥 Checkout repository
+        uses: actions/checkout@v4
+
+      - name: 📦 Cache npm
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.npm
+          key: |
+            ${{ runner.os }}-mintlify
+          restore-keys: |
+            ${{ runner.os }}-mintlify
+
+      - name: 🔗 Check for broken links
+        run: npx mintlify@4.0.393 broken-links

.github/workflows/e2e.yml

@@ -1,4 +1,5 @@
 name: "E2E"
+
 on:
   workflow_call:
     inputs:
@@ -7,6 +8,7 @@ on:
         default: webapp
         required: false
         type: string
+
 jobs:
   cli-v3:
     name: "🧪 CLI v3 tests (${{ matrix.os }} - ${{ matrix.package-manager }})"
@@ -15,11 +17,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [buildjet-8vcpu-ubuntu-2204, windows-latest]
+        os: [ubuntu-latest, windows-latest]
         package-manager: ["npm", "pnpm"]
     steps:
       - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -29,13 +31,16 @@ jobs:
           version: 8.15.5
 
       - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
+        uses: buildjet/setup-node@v4
         with:
           node-version: 20.11.1
 
       - name: 📥 Download deps
         run: pnpm install --frozen-lockfile --filter trigger.dev...
 
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
       - name: 🔧 Build v3 cli monorepo dependencies
         run: pnpm run build --filter trigger.dev^...
 
@@ -49,5 +54,3 @@ jobs:
         shell: bash
         run: |
           LOG=debug PM=${{ matrix.package-manager }} pnpm --filter trigger.dev run test:e2e
-
-

.github/workflows/pr_checks.yml

@@ -3,6 +3,8 @@ name: 🤖 PR Checks
 on:
   pull_request:
     types: [opened, synchronize, reopened]
+    paths-ignore:
+      - "docs/**"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -26,35 +28,3 @@ jobs:
     with:
       package: cli-v3
     secrets: inherit
-
-  preview-release:
-    name: Preview Release
-    needs: [typecheck, units, e2e]
-    if: |
-      github.repository == 'triggerdotdev/trigger.dev'
-    runs-on: buildjet-8vcpu-ubuntu-2204
-    steps:
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          version: 8.15.5
-
-      - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
-        with:
-          node-version: 20.11.1
-          cache: "pnpm"
-
-      - name: 📥 Download deps
-        run: pnpm install --frozen-lockfile
-
-      - name: 🏗️ Build
-        run: pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev"
-
-      - name: ⚡ publish preview release
-        run: npx pkg-pr-new publish --no-template $(ls -d ./packages/*)

.github/workflows/publish-docker.yml

@@ -1,74 +0,0 @@
-name: "🐳 Publish Docker"
-on:
-  workflow_call:
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    env:
-      PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING: 1
-    outputs:
-      version: ${{ steps.get_version.outputs.version }}
-      short_sha: ${{ steps.get_commit.outputs.sha_short }}
-    steps:
-      - name: Setup Depot CLI
-        uses: depot/setup-action@v1
-
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
-        with:
-          submodules: recursive
-
-      - name: 🆚 Get the version
-        id: get_version
-        run: |
-          IMAGE_TAG="${GITHUB_REF#refs/tags/}"
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-            if [[ $IMAGE_TAG == v.docker.* ]]; then
-              ORIGINAL_VERSION="${IMAGE_TAG#v.docker.}"
-              IMAGE_TAG="v${ORIGINAL_VERSION}"
-            elif [[ $IMAGE_TAG == build-* ]]; then
-              IMAGE_TAG="${IMAGE_TAG#build-}"
-            fi
-            echo "IMAGE_TAG=${IMAGE_TAG}"
-          elif [[ $GITHUB_REF == refs/heads/main ]]; then
-            # Handle main branch specifically
-            IMAGE_TAG="main"
-            echo "IMAGE_TAG=${IMAGE_TAG}"
-          else
-            echo "Invalid reference: ${GITHUB_REF}"
-            exit 1
-          fi
-          echo "::set-output name=version::${IMAGE_TAG}"
-
-      - name: 🔢 Get the commit hash
-        id: get_commit
-        run: |
-          echo ::set-output name=sha_short::$(echo ${{ github.sha }} | cut -c1-7)
-
-      - name: 📛 Set the tags
-        id: set_tags
-        run: |
-          ref_without_tag=ghcr.io/triggerdotdev/trigger.dev
-          image_tags=$ref_without_tag:${{ steps.get_version.outputs.version }}
-
-          # if it's a versioned tag, also tag it as v3
-          if [[ "${{ github.ref_name }}" == v.docker.* ]]; then
-            image_tags=$image_tags,$ref_without_tag:v3
-          fi
-          
-          echo "IMAGE_TAGS=${image_tags}" >> "$GITHUB_OUTPUT"
-
-      - name: 🐙 Login to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: 🐳 Build image and push to GitHub Container Registry
-        uses: depot/build-push-action@v1
-        with:
-          file: ./docker/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          tags: ${{ steps.set_tags.outputs.IMAGE_TAGS }}
-          push: true

.github/workflows/publish-infra.yml

@@ -1,121 +0,0 @@
-name: "🚢 Publish Infra Images"
-
-on:
-  workflow_call:
-  push:
-    tags:
-      - "infra-dev-*"
-      - "infra-test-*"
-      - "infra-prod-*"
-    paths:
-      - ".github/workflows/publish.yml"
-      - "packages/**"
-      - "!packages/**/*.md"
-      - "!packages/**/*.eslintrc"
-      - "apps/**"
-      - "!apps/**/*.md"
-      - "!apps/**/*.eslintrc"
-      - "integrations/**"
-      - "!integrations/**/*.md"
-      - "!integrations/**/*.eslintrc"
-      - "pnpm-lock.yaml"
-      - "pnpm-workspace.yaml"
-      - "turbo.json"
-      - "docker/Dockerfile"
-      - "docker/scripts/**"
-      - "tests/**"
-
-permissions:
-  id-token: write
-  packages: write
-  contents: read
-
-env:
-  AWS_REGION: us-east-1
-
-jobs:
-  build:
-    strategy:
-      matrix:
-        package: [coordinator, docker-provider, kubernetes-provider]
-    runs-on: buildjet-16vcpu-ubuntu-2204
-    env:
-      DOCKER_BUILDKIT: "1"
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Generate image reference
-        id: prep
-        run: |
-          # set image repo
-          if [[ "${{ matrix.package }}" == *-provider ]]; then
-            provider_type=$(echo "${{ matrix.package }}" | cut -d- -f1)
-            repository=provider/${provider_type}
-          else
-            repository="${{ matrix.package }}"
-          fi
-          echo "REPOSITORY=${repository}" >> "$GITHUB_OUTPUT"
-
-          # set image tag
-          if [[ "${{ github.ref_type }}" == "tag" ]]; then
-            if [[ "${{ github.ref_name }}" == infra-*-* ]]; then
-              env=$(echo ${{ github.ref_name }} | cut -d- -f2)
-              sha=$(echo ${{ github.sha }} | head -c7)
-              ts=$(date +%s)
-              image_tag=${env}-${sha}-${ts}
-            elif [[ "${{ github.ref_name }}" == v.docker.* ]]; then
-              version="${GITHUB_REF_NAME#v.docker.}"
-              image_tag="v${version}"
-            elif [[ "${{ github.ref_name }}" == build-* ]]; then
-              image_tag="${GITHUB_REF_NAME#build-}"
-            else
-              echo "Invalid tag: ${{ github.ref_name }}"
-              exit 1
-            fi
-          elif [[ "${{ github.ref_name }}" == "main" ]]; then
-            image_tag="main"
-          else
-            echo "Invalid reference: ${{ github.ref }}"
-            exit 1
-          fi
-          echo "IMAGE_TAG=${image_tag}" >> "$GITHUB_OUTPUT"
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      # ..to avoid rate limits when pulling images
-      - name: Login to DockerHub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: 🚢 Build Container Image
-        run: |
-          docker build -t infra_image -f ./apps/${{ matrix.package }}/Containerfile .
-
-      # ..to push image
-      - name: 🐙 Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: 🐙 Push to GitHub Container Registry
-        run: |
-          docker tag infra_image $REGISTRY/$REPOSITORY:$IMAGE_TAG
-          docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
-        env:
-          REGISTRY: ghcr.io/triggerdotdev
-          REPOSITORY: ${{ steps.prep.outputs.REPOSITORY }}
-          IMAGE_TAG: ${{ steps.prep.outputs.IMAGE_TAG }}
-
-      - name: 🐙 Push 'v3' tag to GitHub Container Registry
-        if: startsWith(github.ref_name, 'v.docker.')
-        run: |
-          docker tag infra_image $REGISTRY/$REPOSITORY:v3
-          docker push $REGISTRY/$REPOSITORY:v3
-        env:
-          REGISTRY: ghcr.io/triggerdotdev
-          REPOSITORY: ${{ steps.prep.outputs.REPOSITORY }}

.github/workflows/publish-webapp.yml

@@ -0,0 +1,67 @@
+name: "🐳 Publish Webapp"
+
+on:
+  workflow_call:
+    inputs:
+      image_tag:
+        description: The image tag to publish
+        type: string
+        required: false
+        default: ""
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    env:
+      PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING: 1
+    outputs:
+      version: ${{ steps.get_tag.outputs.tag }}
+      short_sha: ${{ steps.get_commit.outputs.sha_short }}
+    steps:
+      - name: 🏭 Setup Depot CLI
+        uses: depot/setup-action@v1
+
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: "#️⃣ Get the image tag"
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: ${{ inputs.image_tag }}
+
+      - name: 🔢 Get the commit hash
+        id: get_commit
+        run: |
+          echo "sha_short=$(echo ${{ github.sha }} | cut -c1-7)" >> "$GITHUB_OUTPUT"
+
+      - name: 📛 Set the tags
+        id: set_tags
+        run: |
+          ref_without_tag=ghcr.io/triggerdotdev/trigger.dev
+          image_tags=$ref_without_tag:${{ steps.get_tag.outputs.tag }}
+
+          # if tag is a semver, also tag it as v4
+          if [[ "${{ steps.get_tag.outputs.is_semver }}" == true ]]; then
+            # TODO: switch to v4 tag on GA
+            image_tags=$image_tags,$ref_without_tag:v4-beta
+          fi
+
+          echo "image_tags=${image_tags}" >> "$GITHUB_OUTPUT"
+
+      - name: 🐙 Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 🐳 Build image and push to GitHub Container Registry
+        uses: depot/build-push-action@v1
+        with:
+          file: ./docker/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.set_tags.outputs.image_tags }}
+          push: true

.github/workflows/publish-worker-v4.yml

@@ -0,0 +1,89 @@
+name: "⚒️ Publish Worker (v4)"
+
+on:
+  workflow_call:
+    inputs:
+      image_tag:
+        description: The image tag to publish
+        type: string
+        required: false
+        default: ""
+  push:
+    tags:
+      - "re2-test-*"
+      - "re2-prod-*"
+
+permissions:
+  id-token: write
+  packages: write
+  contents: read
+
+jobs:
+  # check-branch:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Fail if re2-prod-* is pushed from a non-main branch
+  #       if: startsWith(github.ref_name, 're2-prod-') && github.base_ref != 'main'
+  #       run: |
+  #         echo "🚫 re2-prod-* tags can only be pushed from the main branch."
+  #         exit 1
+  build:
+    # needs: check-branch
+    strategy:
+      matrix:
+        package: [supervisor]
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: "1"
+    steps:
+      - name: 🏭 Setup Depot CLI
+        uses: depot/setup-action@v1
+
+      - name: ⬇️ Checkout git repo
+        uses: actions/checkout@v4
+
+      - name: 📦 Get image repo
+        id: get_repository
+        run: |
+          if [[ "${{ matrix.package }}" == *-provider ]]; then
+            provider_type=$(echo "${{ matrix.package }}" | cut -d- -f1)
+            repo=provider/${provider_type}
+          else
+            repo="${{ matrix.package }}"
+          fi
+          echo "repo=${repo}" >> "$GITHUB_OUTPUT"
+
+      - name: "#️⃣ Get image tag"
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: ${{ inputs.image_tag }}
+
+      - name: 📛 Set tags to push
+        id: set_tags
+        run: |
+          ref_without_tag=ghcr.io/triggerdotdev/${{ steps.get_repository.outputs.repo }}
+          image_tags=$ref_without_tag:${{ steps.get_tag.outputs.tag }}
+
+          # if tag is a semver, also tag it as v4
+          if [[ "${{ steps.get_tag.outputs.is_semver }}" == true ]]; then
+            # TODO: switch to v4 tag on GA
+            image_tags=$image_tags,$ref_without_tag:v4-beta
+          fi
+
+          echo "image_tags=${image_tags}" >> "$GITHUB_OUTPUT"
+
+      - name: 🐙 Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 🐳 Build image and push to GitHub Container Registry
+        uses: depot/build-push-action@v1
+        with:
+          file: ./apps/${{ matrix.package }}/Containerfile
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.set_tags.outputs.image_tags }}
+          push: true

.github/workflows/publish-worker.yml

@@ -0,0 +1,87 @@
+name: "⚒️ Publish Worker"
+
+on:
+  workflow_call:
+    inputs:
+      image_tag:
+        description: The image tag to publish
+        type: string
+        required: false
+        default: ""
+  push:
+    tags:
+      - "infra-dev-*"
+      - "infra-test-*"
+      - "infra-prod-*"
+
+permissions:
+  packages: write
+  contents: read
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        package: [coordinator, docker-provider, kubernetes-provider]
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: "1"
+    steps:
+      - name: ⬇️ Checkout git repo
+        uses: actions/checkout@v4
+
+      - name: 📦 Get image repo
+        id: get_repository
+        run: |
+          if [[ "${{ matrix.package }}" == *-provider ]]; then
+            provider_type=$(echo "${{ matrix.package }}" | cut -d- -f1)
+            repo=provider/${provider_type}
+          else
+            repo="${{ matrix.package }}"
+          fi
+          echo "repo=${repo}" >> "$GITHUB_OUTPUT"
+
+      - id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: ${{ inputs.image_tag }}
+
+      - name: 🐋 Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # ..to avoid rate limits when pulling images
+      - name: 🐳 Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: 🚢 Build Container Image
+        run: |
+          docker build -t infra_image -f ./apps/${{ matrix.package }}/Containerfile .
+
+      # ..to push image
+      - name: 🐙 Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 🐙 Push to GitHub Container Registry
+        run: |
+          docker tag infra_image "$REGISTRY/$REPOSITORY:$IMAGE_TAG"
+          docker push "$REGISTRY/$REPOSITORY:$IMAGE_TAG"
+        env:
+          REGISTRY: ghcr.io/triggerdotdev
+          REPOSITORY: ${{ steps.get_repository.outputs.repo }}
+          IMAGE_TAG: ${{ steps.get_tag.outputs.tag }}
+
+      # - name: 🐙 Push 'v3' tag to GitHub Container Registry
+      #   if: steps.get_tag.outputs.is_semver == 'true'
+      #   run: |
+      #     docker tag infra_image "$REGISTRY/$REPOSITORY:v3"
+      #     docker push "$REGISTRY/$REPOSITORY:v3"
+      #   env:
+      #     REGISTRY: ghcr.io/triggerdotdev
+      #     REPOSITORY: ${{ steps.get_repository.outputs.repo }}

.github/workflows/publish.yml

@@ -1,6 +1,12 @@
 name: 🚀 Publish Trigger.dev Docker
 
 on:
+  workflow_call:
+    inputs:
+      image_tag:
+        description: The image tag to publish
+        required: true
+        type: string
   push:
     branches:
       - main
@@ -8,20 +14,20 @@ on:
       - "v.docker.*"
       - "build-*"
     paths:
+      - ".github/actions/**/*.yml"
       - ".github/workflows/publish.yml"
       - ".github/workflows/typecheck.yml"
       - ".github/workflows/unit-tests.yml"
       - ".github/workflows/e2e.yml"
-      - ".github/workflows/publish-docker.yml"
+      - ".github/workflows/publish-webapp.yml"
+      - ".github/workflows/publish-worker.yml"
       - "packages/**"
       - "!packages/**/*.md"
       - "!packages/**/*.eslintrc"
+      - "internal-packages/**"
       - "apps/**"
       - "!apps/**/*.md"
       - "!apps/**/*.eslintrc"
-      - "integrations/**"
-      - "!integrations/**/*.md"
-      - "!integrations/**/*.eslintrc"
       - "pnpm-lock.yaml"
       - "pnpm-workspace.yaml"
       - "turbo.json"
@@ -49,12 +55,23 @@ jobs:
     uses: ./.github/workflows/unit-tests.yml
     secrets: inherit
 
-  publish:
+  publish-webapp:
     needs: [typecheck, units]
-    uses: ./.github/workflows/publish-docker.yml
+    uses: ./.github/workflows/publish-webapp.yml
     secrets: inherit
+    with:
+      image_tag: ${{ inputs.image_tag }}
 
-  publish-infra:
+  publish-worker:
     needs: [typecheck, units]
-    uses: ./.github/workflows/publish-infra.yml
+    uses: ./.github/workflows/publish-worker.yml
     secrets: inherit
+    with:
+      image_tag: ${{ inputs.image_tag }}
+
+  publish-worker-v4:
+    needs: [typecheck, units]
+    uses: ./.github/workflows/publish-worker-v4.yml
+    secrets: inherit
+    with:
+      image_tag: ${{ inputs.image_tag }}

.github/workflows/release.yml

@@ -5,26 +5,27 @@ on:
     branches:
       - main
     paths-ignore:
+      - "docs/**"
       - "**.md"
-      - "**.mdx"
       - ".github/CODEOWNERS"
       - ".github/ISSUE_TEMPLATE/**"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   release:
     name: 🦋 Changesets Release
-    runs-on: buildjet-8vcpu-ubuntu-2204
-    if: |
-      github.repository == 'triggerdotdev/trigger.dev'
+    runs-on: ubuntu-latest
+    if: github.repository == 'triggerdotdev/trigger.dev'
     outputs:
-      published_packages: ${{ steps.changesets.outputs.publishedPackages }}
       published: ${{ steps.changesets.outputs.published }}
+      published_packages: ${{ steps.changesets.outputs.publishedPackages }}
+      published_package_version: ${{ steps.get_version.outputs.package_version }}
     steps:
-      - name: 🛑 Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.11.0
-
       - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -34,7 +35,7 @@ jobs:
           version: 8.15.5
 
       - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
+        uses: buildjet/setup-node@v4
         with:
           node-version: 20.11.1
           cache: "pnpm"
@@ -62,6 +63,7 @@ jobs:
         # PR is merged, the workflow will run again and this action will build +
         # publish to npm.
       - name: 🚀 PR / Publish
+        if: ${{ !env.ACT }}
         id: changesets
         uses: changesets/action@v1
         with:
@@ -69,7 +71,31 @@ jobs:
           commit: "chore: Update version for release"
           title: "chore: Update version for release"
           publish: pnpm run changeset:release
-          createGithubReleases: false
+          createGithubReleases: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+      # - name: 🚀 PR / Publish (mock)
+      #   if: ${{ env.ACT }}
+      #   id: changesets
+      #   run: |
+      #     echo "published=true" >> "$GITHUB_OUTPUT"
+      #     echo "publishedPackages=[{\"name\": \"@xx/xx\", \"version\": \"1.2.0\"}, {\"name\": \"@xx/xy\", \"version\": \"0.8.9\"}]" >> "$GITHUB_OUTPUT"
+
+      - name: 📦 Get package version
+        if: steps.changesets.outputs.published == 'true'
+        id: get_version
+        run: |
+          package_version=$(echo '${{ steps.changesets.outputs.publishedPackages }}' | jq -r '.[0].version')
+          echo "package_version=${package_version}" >> "$GITHUB_OUTPUT"
+
+  publish:
+    needs: release
+    uses: ./.github/workflows/publish.yml
+    secrets: inherit
+    # if: needs.release.outputs.published == 'true'
+    # disable automatic publishing for now
+    if: false
+    with:
+      image_tag: v${{ needs.release.outputs.published_package_version }}

.github/workflows/typecheck.yml

@@ -1,13 +1,15 @@
 name: "ʦ TypeScript"
+
 on:
   workflow_call:
+
 jobs:
   typecheck:
-    runs-on: buildjet-8vcpu-ubuntu-2204
+    runs-on: ubuntu-latest
 
     steps:
       - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -17,7 +19,7 @@ jobs:
           version: 8.15.5
 
       - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
+        uses: buildjet/setup-node@v4
         with:
           node-version: 20.11.1
           cache: "pnpm"
@@ -30,6 +32,6 @@ jobs:
 
       - name: 🔎 Type check
         run: pnpm run typecheck
-      
+
       - name: 🔎 Check exports
         run: pnpm run check-exports

.github/workflows/unit-tests.yml

@@ -1,13 +1,43 @@
 name: "🧪 Unit Tests"
+
 on:
   workflow_call:
+
 jobs:
   unitTests:
     name: "🧪 Unit Tests"
-    runs-on: buildjet-8vcpu-ubuntu-2204
+    runs-on: ubuntu-latest
+    env:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
     steps:
+      - name: 🔧 Disable IPv6
+        run: |
+          sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
+          sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1
+          sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=1
+
+      - name: 🔧 Configure docker address pool
+        run: |
+          CONFIG='{
+            "default-address-pools" : [
+              {
+                "base" : "172.17.0.0/12",
+                "size" : 20
+              },
+              {
+                "base" : "192.168.0.0/16",
+                "size" : 24
+              }
+            ]
+          }'
+          mkdir -p /etc/docker
+          echo "$CONFIG" | sudo tee /etc/docker/daemon.json
+
+      - name: 🔧 Restart docker daemon
+        run: sudo systemctl restart docker
+
       - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -17,14 +47,39 @@ jobs:
           version: 8.15.5
 
       - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
+        uses: buildjet/setup-node@v4
         with:
           node-version: 20.11.1
           cache: "pnpm"
 
+      # ..to avoid rate limits when pulling images
+      - name: 🐳 Login to DockerHub
+        if: ${{ env.DOCKERHUB_USERNAME }}
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: 🐳 Skipping DockerHub login (no secrets available)
+        if: ${{ !env.DOCKERHUB_USERNAME }}
+        run: echo "DockerHub login skipped because secrets are not available."
+
       - name: 📥 Download deps
         run: pnpm install --frozen-lockfile
 
-      - name: Run Unit Tests
-        run: |
-          pnpm run test
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: 🧪 Run Webapp Unit Tests
+        run: pnpm run test:webapp
+        env:
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
+          DIRECT_URL: postgresql://postgres:postgres@localhost:5432/postgres
+          SESSION_SECRET: "secret"
+          MAGIC_LINK_SECRET: "secret"
+          ENCRYPTION_KEY: "secret"
+      
+      - name: 🧪 Run Package Unit Tests
+        run: pnpm run test:packages
+      
+      - name: 🧪 Run Internal Unit Tests
+        run: pnpm run test:internal

.gitignore

@@ -58,3 +58,8 @@ apps/**/public/build
 .yarn
 *.tsbuildinfo
 /packages/cli-v3/src/package.json
+.husky
+/packages/react-hooks/src/package.json
+/packages/core/src/package.json
+/packages/trigger-sdk/src/package.json
+/packages/python/src/package.json

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "packages/otlp-importer/protos"]
-	path = packages/otlp-importer/protos
+[submodule "internal-packages/otlp-importer/protos"]
+	path = internal-packages/otlp-importer/protos
 	url = https://github.com/open-telemetry/opentelemetry-proto.git

.npmrc

@@ -1,2 +1,5 @@
 link-workspace-packages=false
-public-hoist-pattern[]=*prisma*
+public-hoist-pattern[]=*prisma*
+prefer-workspace-packages=true
+update-notifier=false
+side-effects-cache=false

.vscode/extensions.json

@@ -1,8 +1,4 @@
 {
-	"recommendations": [
-        "denoland.vscode-deno"
-    ],
-	"unwantedRecommendations": [
-		
-	]
+  "recommendations": ["bierner.comment-tagged-templates"],
+  "unwantedRecommendations": []
 }

.vscode/launch.json

@@ -13,6 +13,24 @@
       "cwd": "${workspaceFolder}",
       "sourceMaps": true
     },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug realtimeStreams.test.ts",
+      "command": "pnpm run test -t RealtimeStreams",
+      "envFile": "${workspaceFolder}/.env",
+      "cwd": "${workspaceFolder}/apps/webapp",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug triggerTask.test.ts",
+      "command": "pnpm run test --run ./test/engine/triggerTask.test.ts",
+      "envFile": "${workspaceFolder}/.env",
+      "cwd": "${workspaceFolder}/apps/webapp",
+      "sourceMaps": true
+    },
     {
       "type": "chrome",
       "request": "launch",
@@ -24,7 +42,15 @@
       "type": "node-terminal",
       "request": "launch",
       "name": "Debug V3 init CLI",
-      "command": "pnpm exec triggerdev init",
+      "command": "pnpm exec trigger init",
+      "cwd": "${workspaceFolder}/references/init-shell",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 init dev CLI",
+      "command": "pnpm exec trigger dev",
       "cwd": "${workspaceFolder}/references/init-shell",
       "sourceMaps": true
     },
@@ -32,15 +58,31 @@
       "type": "node-terminal",
       "request": "launch",
       "name": "Debug V3 Dev CLI",
-      "command": "pnpm exec triggerdev dev",
+      "command": "pnpm exec trigger dev",
       "cwd": "${workspaceFolder}/references/v3-catalog",
       "sourceMaps": true
     },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug Dev Next.js Realtime",
+      "command": "pnpm exec trigger dev",
+      "cwd": "${workspaceFolder}/references/nextjs-realtime",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug prisma-catalog deploy CLI",
+      "command": "pnpm exec trigger deploy --self-hosted --load-image",
+      "cwd": "${workspaceFolder}/references/prisma-catalog",
+      "sourceMaps": true
+    },
     {
       "type": "node-terminal",
       "request": "launch",
       "name": "Debug V3 Deploy CLI",
-      "command": "pnpm exec triggerdev deploy --self-hosted --load-image",
+      "command": "pnpm exec trigger deploy --self-hosted --load-image",
       "cwd": "${workspaceFolder}/references/v3-catalog",
       "sourceMaps": true
     },
@@ -48,7 +90,7 @@
       "type": "node-terminal",
       "request": "launch",
       "name": "Debug V3 list-profiles CLI",
-      "command": "pnpm exec triggerdev list-profiles --log-level debug",
+      "command": "pnpm exec trigger list-profiles --log-level debug",
       "cwd": "${workspaceFolder}/references/v3-catalog",
       "sourceMaps": true
     },
@@ -56,7 +98,7 @@
       "type": "node-terminal",
       "request": "launch",
       "name": "Debug V3 update CLI",
-      "command": "pnpm exec triggerdev update",
+      "command": "pnpm exec trigger update",
       "cwd": "${workspaceFolder}/references/v3-catalog",
       "sourceMaps": true
     },
@@ -88,9 +130,33 @@
       "type": "node-terminal",
       "request": "launch",
       "name": "debug v3 hello-world dev",
-      "command": "pnpm exec triggerdev dev",
+      "command": "pnpm exec trigger dev",
       "cwd": "${workspaceFolder}/references/hello-world",
       "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug RunEngine tests",
+      "command": "pnpm run test ./src/engine/tests/releaseConcurrencyTokenBucketQueue.test.ts -t 'Should retrieve metrics for all queues via getQueueMetrics'",
+      "cwd": "${workspaceFolder}/internal-packages/run-engine",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug RunQueue tests",
+      "command": "pnpm run test ./src/run-queue/index.test.ts",
+      "cwd": "${workspaceFolder}/internal-packages/run-engine",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug d3-demo",
+      "command": "pnpm exec trigger dev",
+      "cwd": "${workspaceFolder}/references/d3-demo",
+      "sourceMaps": true
     }
   ]
 }

.vscode/settings.json

@@ -5,5 +5,6 @@
   "search.exclude": {
     "**/node_modules/**": true,
     "packages/cli-v3/e2e": true
-  }
+  },
+  "vitest.disableWorkspaceWarning": true
 }

.zed/tasks.json

@@ -0,0 +1,45 @@
+[
+  {
+    "label": "Build packages",
+    "command": "pnpm run build --filter \"@trigger.dev/*\" --filter trigger.dev",
+    //"args": [],
+    // Env overrides for the command, will be appended to the terminal's environment from the settings.
+    "env": { "foo": "bar" },
+    // Current working directory to spawn the command into, defaults to current project root.
+    //"cwd": "/path/to/working/directory",
+    // Whether to use a new terminal tab or reuse the existing one to spawn the process, defaults to `false`.
+    "use_new_terminal": false,
+    // Whether to allow multiple instances of the same task to be run, or rather wait for the existing ones to finish, defaults to `false`.
+    "allow_concurrent_runs": false,
+    // What to do with the terminal pane and tab, after the command was started:
+    // * `always` — always show the task's pane, and focus the corresponding tab in it (default)
+    // * `no_focus` — always show the task's pane, add the task's tab in it, but don't focus it
+    // * `never` — do not alter focus, but still add/reuse the task's tab in its pane
+    "reveal": "always",
+    // What to do with the terminal pane and tab, after the command has finished:
+    // * `never` — Do nothing when the command finishes (default)
+    // * `always` — always hide the terminal tab, hide the pane also if it was the last tab in it
+    // * `on_success` — hide the terminal tab on task success only, otherwise behaves similar to `always`
+    "hide": "never",
+    // Which shell to use when running a task inside the terminal.
+    // May take 3 values:
+    // 1. (default) Use the system's default terminal configuration in /etc/passwd
+    //      "shell": "system"
+    // 2. A program:
+    //      "shell": {
+    //        "program": "sh"
+    //      }
+    // 3. A program with arguments:
+    //     "shell": {
+    //         "with_arguments": {
+    //           "program": "/bin/bash",
+    //           "args": ["--login"]
+    //         }
+    //     }
+    "shell": "system",
+    // Whether to show the task line in the output of the spawned task, defaults to `true`.
+    "show_summary": true,
+    // Whether to show the command line in the output of the spawned task, defaults to `true`.
+    "show_output": true
+  }
+]

CONTRIBUTING.md

@@ -114,7 +114,7 @@ pnpm i
 ```sh
 cd references/v3-catalog
 cp .env.example .env
-pnpm exec triggerdev login -a http://localhost:3030
+pnpm exec trigger login -a http://localhost:3030
 ```
 
 This will open a new browser window and authorize the CLI against your local user account.
@@ -123,10 +123,10 @@ You can optionally pass a `--profile` flag to the `login` command, which will al
 
 ```sh
 cd references/v3-catalog
-pnpm exec triggerdev login -a http://localhost:3030 --profile local
+pnpm exec trigger login -a http://localhost:3030 --profile local
 # later when you run the dev or deploy command:
-pnpm exec triggerdev dev --profile local
-pnpm exec triggerdev deploy --profile local
+pnpm exec trigger dev --profile local
+pnpm exec trigger deploy --profile local
 ```
 
 ### Running
@@ -135,43 +135,42 @@ The following steps should be followed any time you start working on a new featu
 
 1. Make sure the webapp is running on localhost:3030
 
-2. Open a terminal window and build the CLI and watch for changes
+2. Open a terminal window and build the CLI and packages and watch for changes
 
 ```sh
-pnpm run dev --filter trigger.dev
+pnpm run dev --filter trigger.dev --filter "@trigger.dev/*"
 ```
 
-2. Open a new terminal window, and anytime changes are made to the `@trigger.dev/core` package, you'll need to manually rebuild the CLI:
-
-```sh
-pnpm run build --filter trigger.dev
-```
-
-Note: You do not need to do the same for `@trigger.dev/sdk`, just core.
-
 3. Open another terminal window, and change into the `<root>/references/v3-catalog` directory.
 
-4. Run the `dev` command, which will register all the local tasks with the platform and allow you to start testing task execution:
+4. You'll need to run the following commands to setup prisma and migrate the database:
+
+```sh
+pnpm exec prisma migrate deploy
+pnpm run generate:prisma
+```
+
+5. Run the `dev` command, which will register all the local tasks with the platform and allow you to start testing task execution:
 
 ```sh
 # in <root>/references/v3-catalog
-pnpm exec triggerdev dev
+pnpm exec trigger dev
 ```
 
 If you want additional debug logging, you can use the `--log-level debug` flag:
 
 ```sh
 # in <root>/references/v3-catalog
-pnpm exec triggerdev dev --log-level debug
+pnpm exec trigger dev --log-level debug
 ```
 
-5. If you make any changes in the CLI/Core/SDK, you'll need to `CTRL+C` to exit the `dev` command and restart it to pickup changes. Any changes to the files inside of the `v3-catalog/src/trigger` dir will automatically be rebuilt by the `dev` command.
+6. If you make any changes in the CLI/Core/SDK, you'll need to `CTRL+C` to exit the `dev` command and restart it to pickup changes. Any changes to the files inside of the `v3-catalog/src/trigger` dir will automatically be rebuilt by the `dev` command.
 
-6. Navigate to the `v3-catalog` project in your local dashboard at localhost:3030 and you should see the list of tasks.
+7. Navigate to the `v3-catalog` project in your local dashboard at localhost:3030 and you should see the list of tasks.
 
-7. Go to the "Test" page in the sidebar and select a task. Then enter a payload and click "Run test". You can tell what the payloads should be by looking at the relevant task file inside the `/references/v3-catalog/src/trigger` folder. Many of them accept an empty payload.
+8. Go to the "Test" page in the sidebar and select a task. Then enter a payload and click "Run test". You can tell what the payloads should be by looking at the relevant task file inside the `/references/v3-catalog/src/trigger` folder. Many of them accept an empty payload.
 
-8. Feel free to add additional files in `v3-catalog/src/trigger` to test out specific aspects of the system, or add in edge cases.
+9. Feel free to add additional files in `v3-catalog/src/trigger` to test out specific aspects of the system, or add in edge cases.
 
 ## Running end-to-end webapp tests (deprecated)
 
@@ -223,20 +222,29 @@ pnpm run db:studio
 
 ## Adding and running migrations
 
-1. Modify packages/database/prisma/schema.prisma file
+1. Modify internal-packages/database/prisma/schema.prisma file
 2. Change directory to the packages/database folder
 
    ```sh
    cd packages/database
    ```
 
-3. Create and apply the migrations
+3. Create a migration
 
    ```
-   pnpm run db:migrate:dev
+   pnpm run db:migrate:dev:create
    ```
 
-   This creates a migration file and executes the migrations against your database and applies changes to the database schema(s)
+   This creates a migration file. Check the migration file does only what you want. If you're adding any database indexes they must use `CONCURRENTLY`, otherwise they'll lock the table when executed.
+
+4. Run the migration.
+
+```
+pnpm run db:migrate:deploy
+pnpm run generate
+```
+
+This executes the migrations against your database and applies changes to the database schema(s), and then regenerates the Prisma client.
 
 4. Commit generated migrations as well as changes to the schema.prisma file
 5. If you're using VSCode you may need to restart the Typescript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`.

DOCKER_INSTALLATION.md

@@ -8,47 +8,56 @@ If you don't have Docker installed on your machine, you'll run into some complic
 
 Below are the steps on how you can avoid that.
 
-First you need to setup docker-compose as it is an underlying tool that this command: `pnpm run docker` fires behind the scene.
+First you need to setup docker compose as it is an underlying tool that this command: `pnpm run docker` fires behind the scene.
 
 ## Linux
 
-To install Docker Compose on Linux Ubuntu via the terminal, you can follow these steps:
+To install Docker Compose on Linux Ubuntu, you can follow these steps:
 
-1. Update the package index on your system by running the following command:
+1. Create the Docker config directory and cli-plugins subdirectory:
 
    ```shell
-   sudo apt update
+   DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker}
+   mkdir -p $DOCKER_CONFIG/cli-plugins
    ```
 
-2. Install the required dependencies by running the following command:
+2. Download the Docker Compose plugin:
 
    ```shell
-   sudo apt install curl
+   curl -SL "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o $DOCKER_CONFIG/cli-plugins/docker-compose
    ```
 
-3. Download the Docker Compose binary into the `/usr/local/bin` directory using the `curl` command:
+   Note:
+
+   - To install for all users, replace `$DOCKER_CONFIG/cli-plugins` with `/usr/local/lib/docker/cli-plugins`
+
+3. Set the appropriate permissions to make the Docker Compose plugin executable:
 
    ```shell
-   sudo curl -L "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+   chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose
    ```
 
-4. Set the appropriate permissions to make the `docker-compose` binary executable:
+   If you installed for all users:
 
    ```shell
-   sudo chmod +x /usr/local/bin/docker-compose
+   sudo chmod +x /usr/local/lib/docker/cli-plugins/docker-compose
    ```
 
-5. Verify that Docker Compose has been successfully installed by running the following command:
+4. Verify that Docker Compose has been successfully installed:
 
    ```shell
-   docker-compose --version
+   docker compose version
    ```
 
-   This command should display the version information of Docker Compose without any errors.
+   You should see output similar to:
 
-After following these steps, you should have Docker Compose installed on your Ubuntu system, and you can use it by running `docker-compose` commands in the terminal.
+   ```
+   Docker Compose version vX.Y.Z
+   ```
 
-When you've verified that the `docker-compose` package is installed and you proceed to start Docker with `pnpm run docker`.
+After following these steps, you should have Docker Compose installed on your Ubuntu system, and you can use it by running `docker compose` commands in the terminal.
+
+When you've verified that the `docker compose` package is installed and you proceed to start Docker with `pnpm run docker`.
 
 You'll probably get an error similar to the one below:
 

README.md

@@ -5,7 +5,7 @@
   <img alt="Trigger.dev logo" src="https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/a45d1fa2-0ae8-4a39-4409-f4f934bfae00/public">
 </picture>
   
-### Open source background jobs with no timeouts
+### Open source background jobs and AI infrastructure
 
 [Discord](https://trigger.dev/discord) | [Website](https://trigger.dev) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Docs](https://trigger.dev/docs)
 
@@ -15,13 +15,22 @@
 
 ## About Trigger.dev
 
-Trigger.dev is an open source platform and SDK which allows you to create long-running background jobs with no timeouts. Write normal async code, deploy, and never hit a timeout.
+Trigger.dev is an open source platform and SDK which allows you to create long-running background jobs. Write normal async code, deploy, and never hit a timeout.
 
-#### Features:
+### Key features:
 
 - JavaScript and TypeScript SDK
-- Write reliable code by default
+- No timeouts
+- Retries (with exponential backoff)
+- Queues and concurrency controls
+- Schedules and crons
+- Full Observability; logs, live trace views, advanced filtering
+- React hooks to interact with the Trigger API from your React app
+- Pipe LLM streams straight to your users through the Realtime API
+- Trigger tasks and display the run status and metadata anywhere in your app
+- Custom alerts, get notified by email, Slack or webhooks
 - No infrastructure to manage
+- Elastic (scaling)
 - Works with your existing tech stack
 
 ## In your codebase
@@ -45,7 +54,7 @@ export const helloWorld = task({
 
 ## Deployment
 
-Use our SDK to write tasks in your codebase. There's no infrastructure to manage, your tasks automatically scale and connect to our cloud. Or you can always [self-host](https://trigger.dev/docs/v3/open-source-self-hosting#overview).
+Use our SDK to write tasks in your codebase. There's no infrastructure to manage, your tasks automatically scale and connect to our cloud. Or you can always self-host.
 
 ## Environments
 
@@ -59,11 +68,19 @@ View every task in every run so you can tell exactly what happened. We provide a
 
 # Getting started
 
-Visit our docs [here](https://trigger.dev/docs/v3/introduction) for a full guide on how to get started with Trigger.dev.
+The quickest way to get started is to create an account and project in our [web app](https://cloud.trigger.dev), and follow the instructions in the onboarding. Build and deploy your first task in minutes.
 
-## Self-host
+### Useful links:
 
-If you prefer to self-host, you can follow our [self-hosting guide](https://trigger.dev/docs/v3/open-source-self-hosting#overview).
+- [Quick start](https://trigger.dev/docs/quick-start) - get up and running in minutes
+- [How it works](https://trigger.dev/docs/v3/how-it-works) - understand how Trigger.dev works under the hood
+- [Guides and examples](https://trigger.dev/docs/guides/introduction) - walk-through guides and code examples for popular frameworks and use cases
+
+## Self-hosting
+
+If you prefer to self-host Trigger.dev, you can follow our [self-hosting guide](https://trigger.dev/docs/v3/open-source-self-hosting#overview).
+
+We also have a dedicated self-hosting channel in our [Discord server](https://trigger.dev/discord) for support.
 
 ## Development
 

RELEASE.md

@@ -0,0 +1,13 @@
+## Guide on releasing a new version
+
+1. Merge in the changeset PR into main, making sure to cancel both the release and publish github actions from that merge.
+2. Pull the changes locally into main
+3. Run `pnpm i` which will update the pnpm lock file with the new versions
+4. create a commit with "Release 3.x.x" and push. This will build and release the packages
+5. Create a git tag on that release commit with v.docker.3.x.x and push the tag to origin. This will publish the `v3.x.x` docker image to GitHub Container Registry.
+6. Once the image is built and pushed, create a new GitHub release and select the tag you just created, along with the previous tag that was released.
+7. This will generate some release notes. Edit out the package changes and leave only the server changes.
+8. Name the release `@trigger.dev/docker@3.x.x`
+9. Include the package link (e.g. https://github.com/triggerdotdev/trigger.dev/pkgs/container/trigger.dev/278459584?tag=v3.x.x)
+10. Once the packages have been published, head over to the [v2-legacy repo](https://github.com/triggerdotdev/v2-legacy.trigger.dev) and follow the instructions in the README for creating a matching release.
+11. Before deploying to cloud, compare the differences in the previously created release and double check to see if there are any migrations with indexes created concurrently, and make sure to run those before deploying.

ai/references/repo.md

@@ -0,0 +1,37 @@
+## Repo Overview
+
+This is a pnpm 8.15.5 monorepo that uses turborepo @turbo.json. The following workspaces are relevant
+
+## Apps
+
+- <root>/apps/webapp is a remix app that is the main API and dashboard for trigger.dev
+- <root>/apps/supervisor is a node.js app that handles the execution of built tasks, interaction with the webapp through internal "engine" APIs, as well as interfacing with things like docker or kubernetes, to execute the code.
+
+## Public Packages
+
+- <root>/packages/trigger-sdk is the `@trigger.dev/sdk` main SDK package.
+- <root>/packages/cli-v3 is the `trigger.dev` CLI package. See our [CLI dev command](https://trigger.dev/docs/cli-dev.md) and [Deployment](https://trigger.dev/docs/deployment/overview.md) docs for more information.
+- <root>/packages/core is the `@trigger.dev/core` package that is shared across the SDK and other packages
+- <root>/packages/build defines the types and prebuilt build extensions for trigger.dev. See our [build extensions docs](https://trigger.dev/docs/config/extensions/overview.md) for more information.
+- <root>/packages/react-hooks defines some useful react hooks like our realtime hooks. See our [Realtime hooks](https://trigger.dev/docs/frontend/react-hooks/realtime.md) and our [Trigger hooks](https://trigger.dev/docs/frontend/react-hooks/triggering.md) for more information.
+- <root>/packages/redis-worker is the `@trigger.dev/redis-worker` package that implements a custom background job/worker sytem powered by redis for offloading work to the background, used in the webapp and also in the Run Engine 2.0.
+
+## Internal Packages
+
+- <root>/internal-packages/\* are packages that are used internally only, not published, and usually they have a tsc build step and are used in the webapp
+- <root>/internal-packages/database is the `@trigger.dev/database` package that exports a prisma client, has the schema file, and exports a few other helpers.
+- <root>/internal-packages/run-engine is the `@internal/run-engine` package that is "Run Engine 2.0" and handles moving a run all the way through it's lifecycle
+- <root>/internal-packages/redis is the `@internal/redis` package that exports Redis types and the `createRedisClient` function to unify how we create redis clients in the repo. It's not used everywhere yet, but it's the preferred way to create redis clients from now on.
+- <root>/internal-packages/testcontainers is the `@internal/testcontainers` package that exports a few useful functions for spinning up local testcontainers when writing vitest tests. See our [tests.md](./tests.md) file for more information.
+- <root>/internal-packages/zodworker is the `@internal/zodworker` package that implements a wrapper around graphile-worker that allows us to use zod to validate our background jobs. We are moving away from using graphile-worker as our background job system, replacing it with our own redis-worker package.
+
+## References
+
+- <root>/references/\* are test workspaces that we use to write and test the system. Not quite e2e tests or automated, but just a useful place to help develop new features
+
+## Other
+
+- <root>/docs is our trigger.dev/docs mintlify documentation site
+- <root>/docker/Dockerfile is the one that creates the main trigger.dev published image
+- <root>/docker/docker-compose.yml is the file we run locally to start postgresql, redis, and electric when we are doing local development. You can run it with `pnpm run docker`
+- <root>/CONTRIBUTING.md defines the steps it takes for OSS contributors to start contributing.

ai/references/tests.md

@@ -0,0 +1,86 @@
+## Running Tests
+
+We use vitest exclusively for testing. To execute tests for a particular workspace, run the following command:
+
+```bash
+pnpm run test --filter webapp
+```
+
+Prefer running tests on a single file (and first cding into the directory):
+
+```bash
+cd apps/webapp
+pnpm run test ./src/components/Button.test.ts
+```
+
+If you are cd'ing into a directory, you may have to build dependencies first:
+
+```bash
+pnpm run build --filter webapp
+cd apps/webapp
+pnpm run test ./src/components/Button.test.ts
+```
+
+## Writing Tests
+
+We use vitest for testing. We almost NEVER mock anything. Start with a top-level "describe", and have multiple "it" statements inside of it.
+
+New test files should be placed right next to the file being tested. For example:
+
+- Source file: `./src/services/MyService.ts`
+- Test file: `./src/services/MyService.test.ts`
+
+When writing anything that needs redis or postgresql, we have some internal "testcontainers" that are used to spin up a local instance, redis, or both.
+
+redisTest:
+
+```typescript
+import { redisTest } from "@internal/testcontainers";
+import { createRedisClient } from "@internal/redis";
+
+describe("redisTest", () => {
+  redisTest("should use redis", async ({ redisOptions }) => {
+    const redis = createRedisClient(redisOptions);
+
+    await redis.set("test", "test");
+    const result = await redis.get("test");
+    expect(result).toEqual("test");
+  });
+});
+```
+
+postgresTest:
+
+```typescript
+import { postgresTest } from "@internal/testcontainers";
+
+describe("postgresTest", () => {
+  postgresTest("should use postgres", async ({ prisma }) => {
+    // prisma is an instance of PrismaClient
+  });
+});
+```
+
+containerTest:
+
+```typescript
+import { containerTest } from "@internal/testcontainers";
+
+describe("containerTest", () => {
+  containerTest("should use container", async ({ prisma, redisOptions }) => {
+    // container has both prisma and redis
+  });
+});
+```
+
+## Dos and Dont's
+
+- Do not mock anything.
+- Do not use mocks in tests.
+- Do not use spies in tests.
+- Do not use stubs in tests.
+- Do not use fakes in tests.
+- Do not use sinon in tests.
+- Structure each test with a setup, action, and assertion style.
+- Feel free to write long test names.
+- If there is any randomness in the code under test, use `seedrandom` to make it deterministic by allowing the caller to provide a seed.

apps/coordinator/Containerfile

@@ -13,7 +13,7 @@ RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
 FROM node-20 AS base
 
 RUN apt-get update \
-  && apt-get install -y buildah ca-certificates dumb-init docker.io \
+  && apt-get install -y buildah ca-certificates dumb-init docker.io busybox \
   && rm -rf /var/lib/apt/lists/*
 
 COPY --chown=node:node .gitignore .gitignore

apps/coordinator/package.json

@@ -17,16 +17,14 @@
   "license": "MIT",
   "dependencies": {
     "@trigger.dev/core": "workspace:*",
-    "execa": "^8.0.1",
     "nanoid": "^5.0.6",
     "prom-client": "^15.1.0",
-    "socket.io": "4.7.4"
+    "socket.io": "4.7.4",
+    "tinyexec": "^0.3.0"
   },
   "devDependencies": {
-    "@types/node": "^18",
     "dotenv": "^16.4.2",
     "esbuild": "^0.19.11",
-    "tsx": "^4.7.0",
-    "typescript": "^5.3.3"
+    "tsx": "^4.7.0"
   }
 }

apps/coordinator/src/chaosMonkey.ts

@@ -1,4 +1,3 @@
-import type { Execa$ } from "execa";
 import { setTimeout as timeout } from "node:timers/promises";
 
 class ChaosMonkeyError extends Error {
@@ -35,11 +34,9 @@ export class ChaosMonkey {
   }
 
   async call({
-    $,
     throwErrors = !this.disableErrors,
     addDelays = !this.disableDelays,
   }: {
-    $?: Execa$<string>;
     throwErrors?: boolean;
     addDelays?: boolean;
   } = {}) {
@@ -60,11 +57,7 @@ export class ChaosMonkey {
       chaosEvents.push(async () => {
         console.log("🍌 Chaos monkey: Add delay");
 
-        if ($) {
-          await $`sleep ${this.delayInSeconds}`;
-        } else {
-          await timeout(this.delayInSeconds * 1000);
-        }
+        await timeout(this.delayInSeconds * 1000);
       });
     }
 
@@ -72,11 +65,7 @@ export class ChaosMonkey {
       chaosEvents.push(async () => {
         console.log("🍌 Chaos monkey: Throw error");
 
-        if ($) {
-          await $`false`;
-        } else {
-          throw new ChaosMonkey.Error("🍌 Chaos monkey: Throw error");
-        }
+        throw new ChaosMonkey.Error("🍌 Chaos monkey: Throw error");
       });
     }
 

apps/coordinator/src/checkpointer.ts

@@ -1,10 +1,13 @@
 import { ExponentialBackoff } from "@trigger.dev/core/v3/apps";
-import { testDockerCheckpoint, isExecaChildProcess } from "@trigger.dev/core/v3/apps";
-import { SimpleLogger } from "@trigger.dev/core/v3/apps";
-import { $ } from "execa";
+import { testDockerCheckpoint } from "@trigger.dev/core/v3/serverOnly";
 import { nanoid } from "nanoid";
 import fs from "node:fs/promises";
 import { ChaosMonkey } from "./chaosMonkey";
+import { Buildah, Crictl, Exec } from "./exec";
+import { setTimeout } from "node:timers/promises";
+import { TempFileCleaner } from "./cleaner";
+import { numFromEnv, boolFromEnv } from "./util";
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
 
 type CheckpointerInitializeReturn = {
   canCheckpoint: boolean;
@@ -24,7 +27,7 @@ type CheckpointAndPushResult =
   | { success: true; checkpoint: CheckpointData }
   | {
       success: false;
-      reason?: "CANCELED" | "DISABLED" | "ERROR" | "IN_PROGRESS" | "NO_SUPPORT" | "SKIP_RETRYING";
+      reason?: "CANCELED" | "ERROR" | "SKIP_RETRYING";
     };
 
 type CheckpointData = {
@@ -83,17 +86,21 @@ export class Checkpointer {
   #canCheckpoint = false;
   #dockerMode: boolean;
 
-  #logger = new SimpleLogger("[checkptr]");
-  #abortControllers = new Map<string, AbortController>();
+  #logger = new SimpleStructuredLogger("checkpointer");
+
   #failedCheckpoints = new Map<string, unknown>();
-  #waitingForRetry = new Set<string>();
+
+  // Indexed by run ID
+  #runAbortControllers = new Map<
+    string,
+    { signal: AbortSignal; abort: AbortController["abort"] }
+  >();
 
   private registryHost: string;
   private registryNamespace: string;
   private registryTlsVerify: boolean;
 
   private disableCheckpointSupport: boolean;
-  private checkpointPath: string;
 
   private simulateCheckpointFailure: boolean;
   private simulateCheckpointFailureSeconds: number;
@@ -101,6 +108,7 @@ export class Checkpointer {
   private simulatePushFailureSeconds: number;
 
   private chaosMonkey: ChaosMonkey;
+  private tmpCleaner?: TempFileCleaner;
 
   constructor(private opts: CheckpointerOptions) {
     this.#dockerMode = opts.dockerMode;
@@ -110,7 +118,6 @@ export class Checkpointer {
     this.registryTlsVerify = opts.registryTlsVerify ?? true;
 
     this.disableCheckpointSupport = opts.disableCheckpointSupport ?? false;
-    this.checkpointPath = opts.checkpointPath ?? "/checkpoints";
 
     this.simulateCheckpointFailure = opts.simulateCheckpointFailure ?? false;
     this.simulateCheckpointFailureSeconds = opts.simulateCheckpointFailureSeconds ?? 300;
@@ -118,6 +125,7 @@ export class Checkpointer {
     this.simulatePushFailureSeconds = opts.simulatePushFailureSeconds ?? 300;
 
     this.chaosMonkey = opts.chaosMonkey ?? new ChaosMonkey(!!process.env.CHAOS_MONKEY_ENABLED);
+    this.tmpCleaner = this.#createTmpCleaner();
   }
 
   async init(): Promise<CheckpointerInitializeReturn> {
@@ -134,18 +142,17 @@ export class Checkpointer {
         return this.#getInitReturn(true);
       }
 
-      this.#logger.error(testCheckpoint.message, testCheckpoint.error ?? "");
+      this.#logger.error(testCheckpoint.message, { error: testCheckpoint.error });
       return this.#getInitReturn(false);
-    } else {
-      try {
-        await $`buildah login --get-login ${this.registryHost}`;
-      } catch (error) {
-        this.#logger.error(`No checkpoint support: Not logged in to registry ${this.registryHost}`);
-        return this.#getInitReturn(false);
-      }
     }
 
-    return this.#getInitReturn(true);
+    const canLogin = await Buildah.canLogin(this.registryHost);
+
+    if (!canLogin) {
+      this.#logger.error(`No checkpoint support: Not logged in to registry ${this.registryHost}`);
+    }
+
+    return this.#getInitReturn(canLogin);
   }
 
   #getInitReturn(canCheckpoint: boolean): CheckpointerInitializeReturn {
@@ -183,33 +190,84 @@ export class Checkpointer {
     if (this.#dockerMode) {
       return basename;
     } else {
-      return `${this.checkpointPath}/${basename}.tar`;
+      return Crictl.getExportLocation(basename);
     }
   }
 
-  async checkpointAndPush(opts: CheckpointAndPushOptions): Promise<CheckpointData | undefined> {
+  async checkpointAndPush(
+    opts: CheckpointAndPushOptions,
+    delayMs?: number
+  ): Promise<CheckpointData | undefined> {
     const start = performance.now();
     this.#logger.log(`checkpointAndPush() start`, { start, opts });
 
-    let interval: NodeJS.Timer | undefined;
+    const { runId } = opts;
 
+    let interval: NodeJS.Timer | undefined;
     if (opts.shouldHeartbeat) {
       interval = setInterval(() => {
-        this.#logger.log("Sending heartbeat", { runId: opts.runId });
-        this.opts.heartbeat(opts.runId);
+        this.#logger.log("Sending heartbeat", { runId });
+        this.opts.heartbeat(runId);
       }, 20_000);
     }
 
+    const controller = new AbortController();
+    const signal = controller.signal;
+    const abort = controller.abort.bind(controller);
+
+    const onAbort = () => {
+      this.#logger.error("Checkpoint aborted", { runId, options: opts });
+    };
+
+    signal.addEventListener("abort", onAbort, { once: true });
+
+    const removeCurrentAbortController = () => {
+      const controller = this.#runAbortControllers.get(runId);
+
+      // Ensure only the current controller is removed
+      if (controller && controller.signal === signal) {
+        this.#runAbortControllers.delete(runId);
+      }
+
+      // Remove the abort listener in case it hasn't fired
+      signal.removeEventListener("abort", onAbort);
+    };
+
+    if (!this.#dockerMode && !this.#canCheckpoint) {
+      this.#logger.error("No checkpoint support. Simulation requires docker.");
+      this.#failCheckpoint(runId, "NO_SUPPORT");
+      return;
+    }
+
+    if (this.#isRunCheckpointing(runId)) {
+      this.#logger.error("Checkpoint procedure already in progress", { options: opts });
+      this.#failCheckpoint(runId, "IN_PROGRESS");
+      return;
+    }
+
+    // This is a new checkpoint, clear any last failure for this run
+    this.#clearFailedCheckpoint(runId);
+
+    if (this.disableCheckpointSupport) {
+      this.#logger.error("Checkpoint support disabled", { options: opts });
+      this.#failCheckpoint(runId, "DISABLED");
+      return;
+    }
+
+    this.#runAbortControllers.set(runId, { signal, abort });
+
     try {
-      const result = await this.#checkpointAndPushWithBackoff(opts);
+      const result = await this.#checkpointAndPushWithBackoff(opts, { delayMs, signal });
 
       const end = performance.now();
       this.#logger.log(`checkpointAndPush() end`, {
         start,
         end,
         diff: end - start,
+        diffWithoutDelay: end - start - (delayMs ?? 0),
         opts,
         success: result.success,
+        delayMs,
       });
 
       if (!result.success) {
@@ -219,48 +277,69 @@ export class Checkpointer {
       return result.checkpoint;
     } finally {
       if (opts.shouldHeartbeat) {
+        // @ts-ignore - Some kind of node incompatible type issue
         clearInterval(interval);
       }
+      removeCurrentAbortController();
     }
   }
 
-  isCheckpointing(runId: string) {
-    return this.#abortControllers.has(runId) || this.#waitingForRetry.has(runId);
+  #isRunCheckpointing(runId: string) {
+    return this.#runAbortControllers.has(runId);
   }
 
-  cancelCheckpoint(runId: string): boolean {
+  cancelAllCheckpointsForRun(runId: string): boolean {
+    this.#logger.log("cancelAllCheckpointsForRun: call", { runId });
+
     // If the last checkpoint failed, pretend we canceled it
     // This ensures tasks don't wait for external resume messages to continue
     if (this.#hasFailedCheckpoint(runId)) {
+      this.#logger.log("cancelAllCheckpointsForRun: hasFailedCheckpoint", { runId });
       this.#clearFailedCheckpoint(runId);
       return true;
     }
 
-    if (this.#waitingForRetry.has(runId)) {
-      this.#waitingForRetry.delete(runId);
-      return true;
-    }
-
-    const controller = this.#abortControllers.get(runId);
+    const controller = this.#runAbortControllers.get(runId);
 
     if (!controller) {
-      this.#logger.debug("Nothing to cancel", { runId });
+      this.#logger.debug("cancelAllCheckpointsForRun: no abort controller", { runId });
       return false;
     }
 
-    controller.abort("cancelCheckpointing()");
-    this.#abortControllers.delete(runId);
+    const { abort, signal } = controller;
+
+    if (signal.aborted) {
+      this.#logger.debug("cancelAllCheckpointsForRun: signal already aborted", { runId });
+      return false;
+    }
+
+    abort("cancelCheckpoint()");
+    this.#runAbortControllers.delete(runId);
 
     return true;
   }
 
-  async #checkpointAndPushWithBackoff({
-    runId,
-    leaveRunning = true, // This mirrors kubernetes behaviour more accurately
-    projectRef,
-    deploymentVersion,
-    attemptNumber,
-  }: CheckpointAndPushOptions): Promise<CheckpointAndPushResult> {
+  async #checkpointAndPushWithBackoff(
+    {
+      runId,
+      leaveRunning = true, // This mirrors kubernetes behaviour more accurately
+      projectRef,
+      deploymentVersion,
+      attemptNumber,
+    }: CheckpointAndPushOptions,
+    { delayMs, signal }: { delayMs?: number; signal: AbortSignal }
+  ): Promise<CheckpointAndPushResult> {
+    if (delayMs && delayMs > 0) {
+      this.#logger.log("Delaying checkpoint", { runId, delayMs });
+
+      try {
+        await setTimeout(delayMs, undefined, { signal });
+      } catch (error) {
+        this.#logger.log("Checkpoint canceled during initial delay", { runId });
+        return { success: false, reason: "CANCELED" };
+      }
+    }
+
     this.#logger.log("Checkpointing with backoff", {
       runId,
       leaveRunning,
@@ -283,24 +362,24 @@ export class Checkpointer {
             delay,
           });
 
-          this.#waitingForRetry.add(runId);
-          await new Promise((resolve) => setTimeout(resolve, delay.milliseconds));
-
-          if (!this.#waitingForRetry.has(runId)) {
-            this.#logger.log("Checkpoint canceled while waiting for retry", { runId });
+          try {
+            await setTimeout(delay.milliseconds, undefined, { signal });
+          } catch (error) {
+            this.#logger.log("Checkpoint canceled during retry delay", { runId });
             return { success: false, reason: "CANCELED" };
-          } else {
-            this.#waitingForRetry.delete(runId);
           }
         }
 
-        const result = await this.#checkpointAndPush({
-          runId,
-          leaveRunning,
-          projectRef,
-          deploymentVersion,
-          attemptNumber,
-        });
+        const result = await this.#checkpointAndPush(
+          {
+            runId,
+            leaveRunning,
+            projectRef,
+            deploymentVersion,
+            attemptNumber,
+          },
+          { signal }
+        );
 
         if (result.success) {
           return result;
@@ -312,24 +391,6 @@ export class Checkpointer {
           return result;
         }
 
-        if (result.reason === "IN_PROGRESS") {
-          this.#logger.log("Checkpoint already in progress, won't retry", { runId });
-          this.#failCheckpoint(runId, result.reason);
-          return result;
-        }
-
-        if (result.reason === "NO_SUPPORT") {
-          this.#logger.log("No checkpoint support, won't retry", { runId });
-          this.#failCheckpoint(runId, result.reason);
-          return result;
-        }
-
-        if (result.reason === "DISABLED") {
-          this.#logger.log("Checkpoint support disabled, won't retry", { runId });
-          this.#failCheckpoint(runId, result.reason);
-          return result;
-        }
-
         if (result.reason === "SKIP_RETRYING") {
           this.#logger.log("Skipping retrying", { runId });
           return result;
@@ -357,13 +418,16 @@ export class Checkpointer {
     return { success: false, reason: "ERROR" };
   }
 
-  async #checkpointAndPush({
-    runId,
-    leaveRunning = true, // This mirrors kubernetes behaviour more accurately
-    projectRef,
-    deploymentVersion,
-    attemptNumber,
-  }: CheckpointAndPushOptions): Promise<CheckpointAndPushResult> {
+  async #checkpointAndPush(
+    {
+      runId,
+      leaveRunning = true, // This mirrors kubernetes behaviour more accurately
+      projectRef,
+      deploymentVersion,
+      attemptNumber,
+    }: CheckpointAndPushOptions,
+    { signal }: { signal: AbortSignal }
+  ): Promise<CheckpointAndPushResult> {
     await this.init();
 
     const options = {
@@ -374,87 +438,53 @@ export class Checkpointer {
       attemptNumber,
     };
 
-    if (!this.#dockerMode && !this.#canCheckpoint) {
-      this.#logger.error("No checkpoint support. Simulation requires docker.");
-      return { success: false, reason: "NO_SUPPORT" };
-    }
-
-    if (this.isCheckpointing(runId)) {
-      this.#logger.error("Checkpoint procedure already in progress", { options });
-      return { success: false, reason: "IN_PROGRESS" };
-    }
-
-    // This is a new checkpoint, clear any last failure for this run
-    this.#clearFailedCheckpoint(runId);
-
-    if (this.disableCheckpointSupport) {
-      this.#logger.error("Checkpoint support disabled", { options });
-      return { success: false, reason: "DISABLED" };
-    }
-
-    const controller = new AbortController();
-    this.#abortControllers.set(runId, controller);
-
-    const $$ = $({ signal: controller.signal });
-
     const shortCode = nanoid(8);
     const imageRef = this.#getImageRef(projectRef, deploymentVersion, shortCode);
     const exportLocation = this.#getExportLocation(projectRef, deploymentVersion, shortCode);
 
+    const buildah = new Buildah({ id: `${runId}-${shortCode}`, abortSignal: signal });
+    const crictl = new Crictl({ id: `${runId}-${shortCode}`, abortSignal: signal });
+
     const cleanup = async () => {
+      const metadata = {
+        runId,
+        exportLocation,
+        imageRef,
+      };
+
       if (this.#dockerMode) {
+        this.#logger.debug("Skipping cleanup in docker mode", metadata);
         return;
       }
 
-      try {
-        await $`rm ${exportLocation}`;
-        this.#logger.log("Deleted checkpoint archive", { exportLocation });
+      this.#logger.log("Cleaning up", metadata);
 
-        await $`buildah rmi ${imageRef}`;
-        this.#logger.log("Deleted checkpoint image", { imageRef });
+      try {
+        await buildah.cleanup();
+        await crictl.cleanup();
       } catch (error) {
-        this.#logger.error("Failure during checkpoint cleanup", { exportLocation, error });
+        this.#logger.error("Error during cleanup", { ...metadata, error });
       }
     };
 
     try {
-      await this.chaosMonkey.call({ $: $$ });
+      await this.chaosMonkey.call();
 
-      this.#logger.log("Checkpointing:", { options });
+      this.#logger.log("checkpointAndPush: checkpointing", { options });
 
       const containterName = this.#getRunContainerName(runId);
-      const containterNameWithAttempt = this.#getRunContainerName(runId, attemptNumber);
 
       // Create checkpoint (docker)
       if (this.#dockerMode) {
-        try {
-          if (this.opts.forceSimulate || !this.#canCheckpoint) {
-            this.#logger.log("Simulating checkpoint");
-            this.#logger.debug(await $$`docker pause ${containterNameWithAttempt}`);
-          } else {
-            if (this.simulateCheckpointFailure) {
-              if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) {
-                this.#logger.error("Simulating checkpoint failure", { options });
-                throw new Error("SIMULATE_CHECKPOINT_FAILURE");
-              }
-            }
+        await this.#createDockerCheckpoint(
+          signal,
+          runId,
+          exportLocation,
+          leaveRunning,
+          attemptNumber
+        );
 
-            if (leaveRunning) {
-              this.#logger.debug(
-                await $$`docker checkpoint create --leave-running ${containterNameWithAttempt} ${exportLocation}`
-              );
-            } else {
-              this.#logger.debug(
-                await $$`docker checkpoint create ${containterNameWithAttempt} ${exportLocation}`
-              );
-            }
-          }
-        } catch (error) {
-          this.#logger.error("Failed while creating docker checkpoint", { exportLocation });
-          throw error;
-        }
-
-        this.#logger.log("checkpoint created:", {
+        this.#logger.log("checkpointAndPush: checkpoint created", {
           runId,
           location: exportLocation,
         });
@@ -474,12 +504,7 @@ export class Checkpointer {
         return { success: false, reason: "SKIP_RETRYING" };
       }
 
-      const containerId = this.#logger.debug(
-        // @ts-expect-error
-        await $$`crictl ps`
-          .pipeStdout($$({ stdin: "pipe" })`grep ${containterName}`)
-          .pipeStdout($$({ stdin: "pipe" })`cut -f1 ${"-d "}`)
-      );
+      const containerId = await crictl.ps(containterName, true);
 
       if (!containerId.stdout) {
         this.#logger.error("could not find container id", { options, containterName });
@@ -496,7 +521,7 @@ export class Checkpointer {
       }
 
       // Create checkpoint
-      this.#logger.debug(await $$`crictl checkpoint --export=${exportLocation} ${containerId}`);
+      await crictl.checkpoint(containerId.stdout, exportLocation);
       const postCheckpoint = performance.now();
 
       // Print checkpoint size
@@ -504,23 +529,20 @@ export class Checkpointer {
       this.#logger.log("checkpoint archive created", { size, options });
 
       // Create image from checkpoint
-      const container = this.#logger.debug(await $$`buildah from scratch`);
+      const workingContainer = await buildah.from("scratch");
       const postFrom = performance.now();
 
-      this.#logger.debug(await $$`buildah add ${container} ${exportLocation} /`);
+      await buildah.add(workingContainer.stdout, exportLocation, "/");
       const postAdd = performance.now();
 
-      this.#logger.debug(
-        await $$`buildah config --annotation=io.kubernetes.cri-o.annotations.checkpoint.name=counter ${container}`
-      );
+      await buildah.config(workingContainer.stdout, [
+        `io.kubernetes.cri-o.annotations.checkpoint.name=${shortCode}`,
+      ]);
       const postConfig = performance.now();
 
-      this.#logger.debug(await $$`buildah commit ${container} ${imageRef}`);
+      await buildah.commit(workingContainer.stdout, imageRef);
       const postCommit = performance.now();
 
-      this.#logger.debug(await $$`buildah rm ${container}`);
-      const postRm = performance.now();
-
       if (this.simulatePushFailure) {
         if (performance.now() < this.simulatePushFailureSeconds * 1000) {
           this.#logger.error("Simulating push failure", { options });
@@ -529,9 +551,7 @@ export class Checkpointer {
       }
 
       // Push checkpoint image
-      this.#logger.debug(
-        await $$`buildah push --tls-verify=${String(this.registryTlsVerify)} ${imageRef}`
-      );
+      await buildah.push(imageRef, this.registryTlsVerify);
       const postPush = performance.now();
 
       const perf = {
@@ -540,8 +560,7 @@ export class Checkpointer {
         "buildah add": postAdd - postFrom,
         "buildah config": postConfig - postAdd,
         "buildah commit": postCommit - postConfig,
-        "buildah rm": postRm - postCommit,
-        "buildah push": postPush - postRm,
+        "buildah push": postPush - postCommit,
       };
 
       this.#logger.log("Checkpointed and pushed image to:", { location: imageRef, perf });
@@ -554,24 +573,90 @@ export class Checkpointer {
         },
       };
     } catch (error) {
-      if (isExecaChildProcess(error)) {
-        if (error.isCanceled) {
-          this.#logger.error("Checkpoint canceled", { options, error });
+      if (error instanceof Exec.Result) {
+        if (error.aborted) {
+          this.#logger.error("Checkpoint canceled: Exec", { options });
 
           return { success: false, reason: "CANCELED" };
+        } else {
+          this.#logger.error("Checkpoint command error", { options, error });
+
+          return { success: false, reason: "ERROR" };
         }
-
-        this.#logger.error("Checkpoint command error", { options, error });
-
-        return { success: false, reason: "ERROR" };
       }
 
-      this.#logger.error("Unhandled checkpoint error", { options, error });
+      this.#logger.error("Unhandled checkpoint error", {
+        options,
+        error: error instanceof Error ? error.message : error,
+      });
 
       return { success: false, reason: "ERROR" };
     } finally {
-      this.#abortControllers.delete(runId);
       await cleanup();
+
+      if (signal.aborted) {
+        this.#logger.error("Checkpoint canceled: Cleanup", { options });
+
+        // Overrides any prior return value
+        return { success: false, reason: "CANCELED" };
+      }
+    }
+  }
+
+  async unpause(runId: string, attemptNumber?: number): Promise<void> {
+    try {
+      const containterNameWithAttempt = this.#getRunContainerName(runId, attemptNumber);
+      const exec = new Exec({ logger: this.#logger });
+      await exec.x("docker", ["unpause", containterNameWithAttempt]);
+    } catch (error) {
+      this.#logger.error("[Docker] Error during unpause", { runId, attemptNumber, error });
+    }
+  }
+
+  async #createDockerCheckpoint(
+    abortSignal: AbortSignal,
+    runId: string,
+    exportLocation: string,
+    leaveRunning: boolean,
+    attemptNumber?: number
+  ) {
+    const containterNameWithAttempt = this.#getRunContainerName(runId, attemptNumber);
+    const exec = new Exec({ logger: this.#logger, abortSignal });
+
+    try {
+      if (this.opts.forceSimulate || !this.#canCheckpoint) {
+        this.#logger.log("Simulating checkpoint");
+
+        await exec.x("docker", ["pause", containterNameWithAttempt]);
+
+        return;
+      }
+
+      if (this.simulateCheckpointFailure) {
+        if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) {
+          this.#logger.error("Simulating checkpoint failure", {
+            runId,
+            exportLocation,
+            leaveRunning,
+            attemptNumber,
+          });
+
+          throw new Error("SIMULATE_CHECKPOINT_FAILURE");
+        }
+      }
+
+      const args = ["checkpoint", "create"];
+
+      if (leaveRunning) {
+        args.push("--leave-running");
+      }
+
+      args.push(containterNameWithAttempt, exportLocation);
+
+      await exec.x("docker", args);
+    } catch (error) {
+      this.#logger.error("Failed while creating docker checkpoint", { exportLocation });
+      throw error;
     }
   }
 
@@ -590,4 +675,34 @@ export class Checkpointer {
   #getRunContainerName(suffix: string, attemptNumber?: number) {
     return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
   }
+
+  #createTmpCleaner() {
+    if (!boolFromEnv("TMP_CLEANER_ENABLED", false)) {
+      return;
+    }
+
+    const defaultPaths = [Buildah.tmpDir, Crictl.checkpointDir].filter(Boolean);
+    const pathsOverride = process.env.TMP_CLEANER_PATHS_OVERRIDE?.split(",").filter(Boolean) ?? [];
+    const paths = pathsOverride.length ? pathsOverride : defaultPaths;
+
+    if (paths.length === 0) {
+      this.#logger.error("TempFileCleaner enabled but no paths to clean", {
+        defaultPaths,
+        pathsOverride,
+        TMP_CLEANER_PATHS_OVERRIDE: process.env.TMP_CLEANER_PATHS_OVERRIDE,
+      });
+
+      return;
+    }
+    const cleaner = new TempFileCleaner({
+      paths,
+      maxAgeMinutes: numFromEnv("TMP_CLEANER_MAX_AGE_MINUTES", 60),
+      intervalSeconds: numFromEnv("TMP_CLEANER_INTERVAL_SECONDS", 300),
+      leadingEdge: boolFromEnv("TMP_CLEANER_LEADING_EDGE", false),
+    });
+
+    cleaner.start();
+
+    return cleaner;
+  }
 }

apps/coordinator/src/cleaner.ts

@@ -0,0 +1,106 @@
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import { Exec } from "./exec";
+import { setTimeout } from "timers/promises";
+
+interface TempFileCleanerOptions {
+  paths: string[];
+  maxAgeMinutes: number;
+  intervalSeconds: number;
+  leadingEdge?: boolean;
+}
+
+export class TempFileCleaner {
+  private enabled = false;
+
+  private logger: SimpleStructuredLogger;
+  private exec: Exec;
+
+  constructor(private opts: TempFileCleanerOptions) {
+    this.logger = new SimpleStructuredLogger("tmp-cleaner", undefined, { ...this.opts });
+    this.exec = new Exec({ logger: this.logger });
+  }
+
+  async start() {
+    this.logger.log("TempFileCleaner.start");
+    this.enabled = true;
+
+    if (!this.opts.leadingEdge) {
+      await this.wait();
+    }
+
+    while (this.enabled) {
+      try {
+        await this.clean();
+      } catch (error) {
+        this.logger.error("error during tick", { error });
+      }
+
+      await this.wait();
+    }
+  }
+
+  stop() {
+    this.logger.log("TempFileCleaner.stop");
+    this.enabled = false;
+  }
+
+  private wait() {
+    return setTimeout(this.opts.intervalSeconds * 1000);
+  }
+
+  private async clean() {
+    for (const path of this.opts.paths) {
+      try {
+        await this.cleanSingle(path);
+      } catch (error) {
+        this.logger.error("error while cleaning", { path, error });
+      }
+    }
+  }
+
+  private async cleanSingle(startingPoint: string) {
+    const maxAgeMinutes = this.opts.maxAgeMinutes;
+
+    const ignoreStartingPoint = ["!", "-path", startingPoint];
+    const onlyDirectDescendants = ["-maxdepth", "1"];
+    const onlyOldFiles = ["-mmin", `+${maxAgeMinutes}`];
+
+    const baseArgs = [
+      startingPoint,
+      ...ignoreStartingPoint,
+      ...onlyDirectDescendants,
+      ...onlyOldFiles,
+    ];
+
+    const duArgs = ["-exec", "du", "-ch", "{}", "+"];
+    const rmArgs = ["-exec", "rm", "-rf", "{}", "+"];
+
+    const du = this.x("find", [...baseArgs, ...duArgs]);
+    const duOutput = await du;
+
+    const duLines = duOutput.stdout.trim().split("\n");
+    const fileCount = duLines.length - 1; // last line is the total
+    const fileSize = duLines.at(-1)?.trim().split(/\s+/)[0];
+
+    if (fileCount === 0) {
+      this.logger.log("nothing to delete", { startingPoint, maxAgeMinutes });
+      return;
+    }
+
+    this.logger.log("deleting old files", { fileCount, fileSize, startingPoint, maxAgeMinutes });
+
+    const rm = this.x("find", [...baseArgs, ...rmArgs]);
+    const rmOutput = await rm;
+
+    if (rmOutput.stderr.length > 0) {
+      this.logger.error("delete unsuccessful", { rmOutput });
+      return;
+    }
+
+    this.logger.log("deleted old files", { fileCount, fileSize, startingPoint, maxAgeMinutes });
+  }
+
+  private get x() {
+    return this.exec.x.bind(this.exec);
+  }
+}

apps/coordinator/src/exec.ts

@@ -0,0 +1,293 @@
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import { randomUUID } from "crypto";
+import { homedir } from "os";
+import { type Result, x } from "tinyexec";
+
+class TinyResult {
+  pid?: number;
+  exitCode?: number;
+  aborted: boolean;
+  killed: boolean;
+
+  constructor(result: Result) {
+    this.pid = result.pid;
+    this.exitCode = result.exitCode;
+    this.aborted = result.aborted;
+    this.killed = result.killed;
+  }
+}
+
+interface ExecOptions {
+  logger?: SimpleStructuredLogger;
+  abortSignal?: AbortSignal;
+  logOutput?: boolean;
+  trimArgs?: boolean;
+  neverThrow?: boolean;
+}
+
+export class Exec {
+  private logger: SimpleStructuredLogger;
+  private abortSignal: AbortSignal | undefined;
+
+  private logOutput: boolean;
+  private trimArgs: boolean;
+  private neverThrow: boolean;
+
+  constructor(opts: ExecOptions) {
+    this.logger = opts.logger ?? new SimpleStructuredLogger("exec");
+    this.abortSignal = opts.abortSignal;
+
+    this.logOutput = opts.logOutput ?? true;
+    this.trimArgs = opts.trimArgs ?? true;
+    this.neverThrow = opts.neverThrow ?? false;
+  }
+
+  async x(
+    command: string,
+    args?: string[],
+    opts?: { neverThrow?: boolean; ignoreAbort?: boolean }
+  ) {
+    const argsTrimmed = this.trimArgs ? args?.map((arg) => arg.trim()) : args;
+
+    const commandWithFirstArg = `${command}${argsTrimmed?.length ? ` ${argsTrimmed[0]}` : ""}`;
+    this.logger.debug(`exec: ${commandWithFirstArg}`, { command, args, argsTrimmed });
+
+    const result = x(command, argsTrimmed, {
+      signal: opts?.ignoreAbort ? undefined : this.abortSignal,
+      // We don't use this as it doesn't cover killed and aborted processes
+      // throwOnError: true,
+    });
+
+    const output = await result;
+
+    const metadata = {
+      command,
+      argsRaw: args,
+      argsTrimmed,
+      globalOpts: {
+        trimArgs: this.trimArgs,
+        neverThrow: this.neverThrow,
+        hasAbortSignal: !!this.abortSignal,
+      },
+      localOpts: opts,
+      stdout: output.stdout,
+      stderr: output.stderr,
+      pid: result.pid,
+      exitCode: result.exitCode,
+      aborted: result.aborted,
+      killed: result.killed,
+    };
+
+    if (this.logOutput) {
+      this.logger.debug(`output: ${commandWithFirstArg}`, metadata);
+    }
+
+    if (this.neverThrow || opts?.neverThrow) {
+      return output;
+    }
+
+    if (result.aborted) {
+      this.logger.error(`aborted: ${commandWithFirstArg}`, metadata);
+      throw new TinyResult(result);
+    }
+
+    if (result.killed) {
+      this.logger.error(`killed: ${commandWithFirstArg}`, metadata);
+      throw new TinyResult(result);
+    }
+
+    if (result.exitCode !== 0) {
+      this.logger.error(`non-zero exit: ${commandWithFirstArg}`, metadata);
+      throw new TinyResult(result);
+    }
+
+    return output;
+  }
+
+  static Result = TinyResult;
+}
+
+interface BuildahOptions {
+  id?: string;
+  abortSignal?: AbortSignal;
+}
+
+export class Buildah {
+  private id: string;
+  private logger: SimpleStructuredLogger;
+  private exec: Exec;
+
+  private containers = new Set<string>();
+  private images = new Set<string>();
+
+  constructor(opts: BuildahOptions) {
+    this.id = opts.id ?? randomUUID();
+    this.logger = new SimpleStructuredLogger("buildah", undefined, { id: this.id });
+
+    this.exec = new Exec({
+      logger: this.logger,
+      abortSignal: opts.abortSignal,
+    });
+
+    this.logger.log("initiaized", { opts });
+  }
+
+  private get x() {
+    return this.exec.x.bind(this.exec);
+  }
+
+  async from(baseImage: string) {
+    const output = await this.x("buildah", ["from", baseImage]);
+    this.containers.add(output.stdout);
+    return output;
+  }
+
+  async add(container: string, src: string, dest: string) {
+    return await this.x("buildah", ["add", container, src, dest]);
+  }
+
+  async config(container: string, annotations: string[]) {
+    const args = ["config"];
+
+    for (const annotation of annotations) {
+      args.push(`--annotation=${annotation}`);
+    }
+
+    args.push(container);
+
+    return await this.x("buildah", args);
+  }
+
+  async commit(container: string, imageRef: string) {
+    const output = await this.x("buildah", ["commit", container, imageRef]);
+    this.images.add(output.stdout);
+    return output;
+  }
+
+  async push(imageRef: string, registryTlsVerify?: boolean) {
+    return await this.x("buildah", [
+      "push",
+      `--tls-verify=${String(!!registryTlsVerify)}`,
+      imageRef,
+    ]);
+  }
+
+  async cleanup() {
+    if (this.containers.size > 0) {
+      try {
+        const output = await this.x("buildah", ["rm", ...this.containers], { ignoreAbort: true });
+        this.containers.clear();
+
+        if (output.stderr.length > 0) {
+          this.logger.error("failed to remove some containers", { output });
+        }
+      } catch (error) {
+        this.logger.error("failed to clean up containers", { error, containers: this.containers });
+      }
+    } else {
+      this.logger.debug("no containers to clean up");
+    }
+
+    if (this.images.size > 0) {
+      try {
+        const output = await this.x("buildah", ["rmi", ...this.images], { ignoreAbort: true });
+        this.images.clear();
+
+        if (output.stderr.length > 0) {
+          this.logger.error("failed to remove some images", { output });
+        }
+      } catch (error) {
+        this.logger.error("failed to clean up images", { error, images: this.images });
+      }
+    } else {
+      this.logger.debug("no images to clean up");
+    }
+  }
+
+  static async canLogin(registryHost: string) {
+    try {
+      await x("buildah", ["login", "--get-login", registryHost], { throwOnError: true });
+      return true;
+    } catch (error) {
+      return false;
+    }
+  }
+
+  static get tmpDir() {
+    return process.env.TMPDIR ?? "/var/tmp";
+  }
+
+  static get storageRootDir() {
+    return process.getuid?.() === 0
+      ? "/var/lib/containers/storage"
+      : `${homedir()}/.local/share/containers/storage`;
+  }
+}
+
+interface CrictlOptions {
+  id?: string;
+  abortSignal?: AbortSignal;
+}
+
+export class Crictl {
+  private id: string;
+  private logger: SimpleStructuredLogger;
+  private exec: Exec;
+
+  private archives = new Set<string>();
+
+  constructor(opts: CrictlOptions) {
+    this.id = opts.id ?? randomUUID();
+    this.logger = new SimpleStructuredLogger("crictl", undefined, { id: this.id });
+
+    this.exec = new Exec({
+      logger: this.logger,
+      abortSignal: opts.abortSignal,
+    });
+
+    this.logger.log("initiaized", { opts });
+  }
+
+  private get x() {
+    return this.exec.x.bind(this.exec);
+  }
+
+  async ps(containerName: string, quiet?: boolean) {
+    return await this.x("crictl", ["ps", "--name", containerName, quiet ? "--quiet" : ""]);
+  }
+
+  async checkpoint(containerId: string, exportLocation: string) {
+    const output = await this.x("crictl", [
+      "checkpoint",
+      `--export=${exportLocation}`,
+      containerId,
+    ]);
+    this.archives.add(exportLocation);
+    return output;
+  }
+
+  async cleanup() {
+    if (this.archives.size > 0) {
+      try {
+        const output = await this.x("rm", ["-v", ...this.archives], { ignoreAbort: true });
+        this.archives.clear();
+
+        if (output.stderr.length > 0) {
+          this.logger.error("failed to remove some archives", { output });
+        }
+      } catch (error) {
+        this.logger.error("failed to clean up archives", { error, archives: this.archives });
+      }
+    } else {
+      this.logger.debug("no archives to clean up");
+    }
+  }
+
+  static getExportLocation(identifier: string) {
+    return `${this.checkpointDir}/${identifier}.tar`;
+  }
+
+  static get checkpointDir() {
+    return process.env.CRI_CHECKPOINT_DIR ?? "/checkpoints";
+  }
+}

apps/coordinator/src/util.ts

@@ -0,0 +1,31 @@
+export const boolFromEnv = (env: string, defaultValue: boolean): boolean => {
+  const value = process.env[env];
+
+  if (!value) {
+    return defaultValue;
+  }
+
+  return ["1", "true"].includes(value);
+};
+
+export const numFromEnv = (env: string, defaultValue: number): number => {
+  const value = process.env[env];
+
+  if (!value) {
+    return defaultValue;
+  }
+
+  return parseInt(value, 10);
+};
+
+export function safeJsonParse(json?: string): unknown {
+  if (!json) {
+    return;
+  }
+
+  try {
+    return JSON.parse(json);
+  } catch (e) {
+    return null;
+  }
+}

apps/coordinator/tsconfig.json

@@ -1,8 +1,6 @@
 {
-  "include": ["./src/**/*.ts"],
-  "exclude": ["node_modules"],
   "compilerOptions": {
-    "target": "es2016",
+    "target": "es2018",
     "module": "commonjs",
     "esModuleInterop": true,
     "resolveJsonModule": true,

apps/docker-provider/package.json

@@ -20,10 +20,8 @@
     "execa": "^8.0.1"
   },
   "devDependencies": {
-    "@types/node": "^18.19.8",
     "dotenv": "^16.4.2",
     "esbuild": "^0.19.11",
-    "tsx": "^4.7.0",
-    "typescript": "^5.3.3"
+    "tsx": "^4.7.0"
   }
 }