Release 3.3.10 (#1601)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.changeset/config.json

@@ -7,21 +7,13 @@
     }
   ],
   "commit": false,
-  "fixed": [
-    [
-      "@trigger.dev/*"
-    ]
-  ],
+  "fixed": [["@trigger.dev/*", "trigger.dev"]],
   "linked": [],
   "access": "public",
   "baseBranch": "main",
   "updateInternalDependencies": "patch",
-  "ignore": [
-    "webapp",
-    "emails",
-    "@trigger.dev/database"
-  ],
+  "ignore": ["webapp", "proxy", "coordinator", "docker-provider", "kubernetes-provider"],
   "___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH": {
     "onlyUpdatePeerDependentsWhenOutOfRange": true
   }
-}
+}

.configs/tsconfig.base.json

@@ -0,0 +1,36 @@
+{
+  "compilerOptions": {
+    "target": "es2022",
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "moduleDetection": "force",
+    "verbatimModuleSyntax": false,
+    "jsx": "react",
+
+    "strict": true,
+    "alwaysStrict": true,
+    "strictPropertyInitialization": false,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noImplicitAny": true,
+    "noImplicitReturns": true,
+    "noImplicitThis": true,
+
+    "noFallthroughCasesInSwitch": true,
+    "resolveJsonModule": true,
+
+    "removeComments": false,
+    "esModuleInterop": true,
+    "emitDecoratorMetadata": false,
+    "experimentalDecorators": false,
+    "downlevelIteration": true,
+    "isolatedModules": true,
+    "noUncheckedIndexedAccess": true,
+
+    "pretty": true,
+    "customConditions": ["@triggerdotdev/source"]
+  }
+}

.dockerignore

@@ -1,35 +1,44 @@
-*.log
+\*.log
 .git
 .github
+
 # editor
+
 .idea
 .vscode
+
 # dependencies
+
 node_modules
 .pnp
 .pnp.js
 
 # testing
+
 coverage
 
 # next.js
+
 .next/
 build
 
 # packages
+
 build
 dist
-packages/**/dist
+packages/\*\*/dist
 
 # misc
+
 .DS_Store
-*.pem
+\*.pem
 
 .turbo
 .vercel
 .cache
 .output
-apps/**/public/build
+.trigger
+apps/\*\*/public/build
 
 cypress/screenshots
 cypress/videos
@@ -38,6 +47,7 @@ apps/**/styles/tailwind.css
 packages/**/styles/tailwind.css
 
 .changeset
+references
 examples
 CHANGESETS.md
 CONTRIBUTING.md

.env.example

@@ -10,19 +10,58 @@ DIRECT_URL=${DATABASE_URL}
 REMIX_APP_PORT=3030
 APP_ENV=development
 APP_ORIGIN=http://localhost:3030
+ELECTRIC_ORIGIN=http://localhost:3060
 NODE_ENV=development
+V3_ENABLED=true
+
+# Redis is used for the v3 queuing and v2 concurrency control
+REDIS_HOST="localhost"
+REDIS_PORT="6379"
+REDIS_TLS_DISABLED="true"
+
+DEV_OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:3030/otel"
+DEV_OTEL_BATCH_PROCESSING_ENABLED="0"
 
 # OPTIONAL VARIABLES
+# This is used for validating emails that are allowed to log in. Every email that do not match this regex will be rejected.
+# WHITELISTED_EMAILS="authorized@yahoo\.com|authorized@gmail\.com"
+# Accounts with these emails will get global admin rights. This grants access to the admin UI.
+# ADMIN_EMAILS="admin@example\.com|another-admin@example\.com"
 # This is used for logging in via GitHub. You can leave these commented out if you don't want to use GitHub for authentication.
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
 
-# Resend is an email service used for signing in to Trigger.dev via a Magic Link.
-# Emails will print to the console if you leave these commented out
+# Configure an email transport to allow users to sign in to Trigger.dev via a Magic Link.
+# If none are configured, emails will print to the console instead.
+# Uncomment one of the following blocks to allow delivery of
+
+# Resend
 ### Visit https://resend.com, create an account and get your API key. Then insert it below along with your From and Reply To email addresses. Visit https://resend.com/docs for more information.
-# RESEND_API_KEY=<api_key>
+# EMAIL_TRANSPORT=resend
 # FROM_EMAIL=
 # REPLY_TO_EMAIL=
+# RESEND_API_KEY=
+
+# Generic SMTP
+### Enter the configuration provided by your mail provider. Visit https://nodemailer.com/smtp/ for more information
+### SMTP_SECURE = false will use STARTTLS when connecting to a server that supports it (usually port 587)
+# EMAIL_TRANSPORT=smtp
+# FROM_EMAIL=
+# REPLY_TO_EMAIL=
+# SMTP_HOST=
+# SMTP_PORT=587
+# SMTP_SECURE=false
+# SMTP_USER=
+# SMTP_PASSWORD=
+
+# AWS Simple Email Service
+### Authentication is configured using the default Node.JS credentials provider chain (https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/Package/-aws-sdk-credential-providers/#fromnodeproviderchain)
+# EMAIL_TRANSPORT=aws-ses
+# FROM_EMAIL=
+# REPLY_TO_EMAIL=
+
+# Remove the following line to enable logging telemetry traces to the console
+LOG_TELEMETRY="false"
 
 # CLOUD VARIABLES
 POSTHOG_PROJECT_KEY=
@@ -31,6 +70,35 @@ CLOUD_AIRTABLE_CLIENT_ID=
 CLOUD_AIRTABLE_CLIENT_SECRET=
 CLOUD_GITHUB_CLIENT_ID=
 CLOUD_GITHUB_CLIENT_SECRET=
+CLOUD_LINEAR_CLIENT_ID=
+CLOUD_LINEAR_CLIENT_SECRET=
 CLOUD_SLACK_APP_HOST=
 CLOUD_SLACK_CLIENT_ID=
-CLOUD_SLACK_CLIENT_SECRET= 
+CLOUD_SLACK_CLIENT_SECRET= 
+
+# v3 variables
+PROVIDER_SECRET=provider-secret # generate the actual secret with `openssl rand -hex 32`
+COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl rand -hex 32`
+
+# Uncomment the following line to enable the registry proxy
+# ENABLE_REGISTRY_PROXY=true
+# DEPOT_TOKEN=<Depot org token>
+# DEPOT_PROJECT_ID=<Depot project id>
+# DEPLOY_REGISTRY_HOST=${APP_ORIGIN} # This is the host that the deploy CLI will use to push images to the registry
+# CONTAINER_REGISTRY_ORIGIN=<Container registry origin e.g. https://registry.digitalocean.com>
+# CONTAINER_REGISTRY_USERNAME=<Container registry username e.g. Digital ocean email address>
+# CONTAINER_REGISTRY_PASSWORD=<Container registry password e.g. Digital ocean PAT>
+# DEV_OTEL_EXPORTER_OTLP_ENDPOINT="http://0.0.0.0:4318"
+# These are needed for the object store (for handling large payloads/outputs)
+# OBJECT_STORE_BASE_URL="https://{bucket}.{accountId}.r2.cloudflarestorage.com"
+# OBJECT_STORE_ACCESS_KEY_ID=
+# OBJECT_STORE_SECRET_ACCESS_KEY=
+# CHECKPOINT_THRESHOLD_IN_MS=10000
+
+# These control the server-side internal telemetry
+# INTERNAL_OTEL_TRACE_EXPORTER_URL=<URL to send traces to>
+# INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_NAME=<Header name for the auth token>
+# INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE=<Auth token value>
+# INTERNAL_OTEL_TRACE_LOGGING_ENABLED=1
+# INTERNAL_OTEL_TRACE_SAMPING_RATE=20 # this means 1/20 traces or 5% of traces will be sampled (sampled = recorded)
+# INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED=0,

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [triggerdotdev]

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,38 @@
+name: 🐞 Bug Report
+description: Create a bug report to help us improve
+title: "bug: "
+labels: ["🐞 unconfirmed bug"]
+body:
+  - type: textarea
+    attributes:
+      label: Provide environment information
+      description: |
+        Run this command in your project root and paste the results:
+        ```bash
+        npx envinfo --system --binaries
+        ```
+
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of the bug, as well as what you expected to happen when encountering it.
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Reproduction repo
+      description: If applicable, please provide a link to a reproduction repo or a Stackblitz / CodeSandbox project. Your issue may be closed if this is not provided and we are unable to reproduce the issue. If your bug is a docs issue, link the appropriate page.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: To reproduce
+      description: Describe how to reproduce your bug. Steps, code snippets, reproduction repos etc.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Additional information
+      description: Add any other information related to the bug here, screenshots if applicable.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Ask a Question
+    url: https://trigger.dev/discord
+    about: Ask questions and discuss with other community members

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,27 @@
+name: Feature Request
+description: Suggest an idea for this project
+title: "feat: "
+labels: ["🌟 enhancement"]
+body:
+  - type: textarea
+    attributes:
+      label: Is your feature request related to a problem? Please describe.
+      description: A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Describe the solution you'd like to see
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Describe alternate solutions
+      description: A clear and concise description of any alternative solutions or features you've considered.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Additional information
+      description: Add any other information related to the feature here. If your feature request is related to any issues or discussions, link them here.

.github/ISSUE_TEMPLATE/instrumentation_request.yml

@@ -0,0 +1,21 @@
+name: OpenTelemetry Auto-Instrumentation Request
+description: Suggest an SDK that you'd like to be auto-instrumented in the Run log view
+title: "auto-instrumentation: "
+labels: ["🌟 enhancement"]
+body:
+  - type: textarea
+    attributes:
+      label: What API or SDK would you to have automatic spans for?
+      description: A clear description of which API or SDK you'd like, and links to it.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Is there an existing OpenTelemetry auto-instrumentation package?
+      description: You can search for existing ones – https://opentelemetry.io/ecosystem/registry/?component=instrumentation&language=js
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Additional information
+      description: Add any other information related to the feature here. If your feature request is related to any issues or discussions, link them here.

.github/actions/get-image-tag/action.yml

@@ -0,0 +1,84 @@
+name: "#️⃣ Get image tag (action)"
+
+description: This action gets the image tag from the commit ref or input (if provided)
+
+outputs:
+  tag:
+    description: The image tag
+    value: ${{ steps.get_tag.outputs.tag }}
+  is_semver:
+    description: Whether the tag is a semantic version
+    value: ${{ steps.check_semver.outputs.is_semver }}
+
+inputs:
+  tag:
+    description: The image tag. If this is set it will return the tag as is.
+    required: false
+    default: ""
+
+runs:
+  using: "composite"
+  steps:
+    - name: "#️⃣ Get image tag (step)"
+      id: get_tag
+      shell: bash
+      run: |
+        if [[ -n "${{ inputs.tag }}" ]]; then
+          tag="${{ inputs.tag }}"
+        elif [[ "${{ github.ref_type }}" == "tag" ]]; then
+          if [[ "${{ github.ref_name }}" == infra-*-* ]]; then
+            env=$(echo ${{ github.ref_name }} | cut -d- -f2)
+            sha=$(echo ${{ github.sha }} | head -c7)
+            ts=$(date +%s)
+            tag=${env}-${sha}-${ts}
+          elif [[ "${{ github.ref_name }}" == v.docker.* ]]; then
+            version="${GITHUB_REF_NAME#v.docker.}"
+            tag="v${version}"
+          elif [[ "${{ github.ref_name }}" == build-* ]]; then
+            tag="${GITHUB_REF_NAME#build-}"
+          else
+            echo "Invalid git tag: ${{ github.ref_name }}"
+            exit 1
+          fi
+        elif [[ "${{ github.ref_name }}" == "main" ]]; then
+          tag="main"
+        else
+          echo "Invalid git ref: ${{ github.ref }}"
+          exit 1
+        fi
+        echo "tag=${tag}" >> "$GITHUB_OUTPUT"
+
+    - name: 🔍 Check for validity
+      id: check_validity
+      shell: bash
+      env:
+        tag: ${{ steps.get_tag.outputs.tag }}
+      run: |
+        if [[ "${tag}" =~ ^[a-z0-9]+([._-][a-z0-9]+)*$ ]]; then
+          echo "Tag is valid: ${tag}"
+        else
+          echo "Tag is not valid: ${tag}"
+          exit 1
+        fi
+
+    - name: 🆚 Check for semver
+      id: check_semver
+      shell: bash
+      env:
+        tag: ${{ steps.get_tag.outputs.tag }}
+      # Will match most semver formats except build metadata, i.e. v1.2.3+build.1
+      # Valid matches:
+      # v1.2.3
+      # v1.2.3-alpha
+      # v1.2.3-alpha.1
+      # v1.2.3-rc.1
+      # v1.2.3-beta-1
+      run: |
+        if [[ "${tag}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ ]]; then
+          echo "Tag is a semantic version: ${tag}"
+          is_semver=true
+        else
+          echo "Tag is not a semantic version: ${tag}"
+          is_semver=false
+        fi
+        echo "is_semver=${is_semver}" >> "$GITHUB_OUTPUT"

.github/copilot-instructions.md

@@ -0,0 +1 @@
+This is the repo for Trigger.dev, a background jobs platform written in TypeScript. Our webapp at apps/webapp is a Remix 2.1 app that uses Node.js v20. Our SDK is an isomorphic TypeScript SDK at packages/trigger-sdk. Always prefer using isomorphic code like fetch, ReadableStream, etc. instead of Node.js specific code. Our tests are all vitest. We use prisma in internal-packages/database for our database interactions using PostgreSQL. For TypeScript, we usually use types over interfaces. We use zod a lot in packages/core and in the webapp. Avoid enums. Use strict mode. No default exports, use function declarations.

.github/labeler.yml

@@ -0,0 +1,12 @@
+"📌 area: cli":
+  - any: ["cli/**/*"]
+
+"📌 area: t3-app":
+  - any: ["cli/template/**/*"]
+
+"📚 documentation":
+  - any: ["www/**/*"]
+  - any: ["**/*.md"]
+
+"📌 area: ci":
+  - any: [".github/**/*"]

.github/pull_request_template.md

@@ -0,0 +1,27 @@
+Closes #<issue>
+
+## ✅ Checklist
+
+- [ ] I have followed every step in the [contributing guide](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md)
+- [ ] The PR title follows the convention.
+- [ ] I ran and tested the code works
+
+---
+
+## Testing
+
+_[Describe the steps you took to test this change]_
+
+---
+
+## Changelog
+
+_[Short description of what has changed]_
+
+---
+
+## Screenshots
+
+_[Screenshots]_
+
+💯

.github/test/README.md

@@ -0,0 +1,70 @@
+# GitHub Action Tests
+
+This directory contains necessary files to allow local testing of GitHub Actions workflows, composite actions, etc. You will need to install [act](https://github.com/nektos/act) to perform tests.
+
+## Workflow tests
+
+Trigger specific workflow files by specifying their full path:
+
+```
+act -W .github/workflow/release.yml
+```
+
+You will likely need to override any custom runners we use, e.g. buildjet. For example:
+
+```
+override=catthehacker/ubuntu:act-latest
+
+act -W .github/workflow/release.yml \
+    -P buildjet-8vcpu-ubuntu-2204=$override
+
+# override multiple images at the same time
+act -W .github/workflow/release.yml \
+    -P buildjet-8vcpu-ubuntu-2204=$override \
+    -P buildjet-16vcpu-ubuntu-2204=$override
+```
+
+Trigger with specific event payloads to test pushing to branches or tags:
+
+```
+override=catthehacker/ubuntu:act-latest
+
+# simulate push to main
+act -W .github/workflow/publish.yml \
+    -P buildjet-8vcpu-ubuntu-2204=$override \
+    -P buildjet-16vcpu-ubuntu-2204=$override \
+    -e .github/events/push-tag-main.json
+
+# simulate a `build-` prefixed tag
+act -W .github/workflow/publish.yml \
+    -P buildjet-8vcpu-ubuntu-2204=$override \
+    -P buildjet-16vcpu-ubuntu-2204=$override \
+    -e .github/events/push-tag-buld.json
+```
+
+By default, `act` will send a push event. To trigger a different event:
+
+```
+# basic syntax
+act <EVENT> ...
+
+# simulate a pull request
+act pull_request
+
+# only trigger a specific workflow
+act pull_request -W .github/workflow/pr_checks.yml
+```
+
+## Composite action tests
+
+The composite (custom) action tests can be run by triggering the `test-actions` workflow:
+
+```
+act -W .github/test/test-actions.yml
+```
+
+## Helpful flags
+
+- `--pull=false` - perform fully offline tests if all images are already present
+- `-j <job_name>` - run the specified job only
+- `-l push` - list all workflows with push triggers

.github/test/events/push-main.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/heads/main"
+}

.github/test/events/push-tag-build.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/build-buildtag"
+}

.github/test/events/push-tag-docker-nonsemver.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/v.docker.nonsemver"
+}

.github/test/events/push-tag-docker.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/v.docker.1.2.3"
+}

.github/test/events/push-tag-infra-prod.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/infra-prod-anything"
+}

.github/test/events/push-tag-infra-test.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/infra-test-anything"
+}

.github/test/events/push-tag-semver.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/1.2.3"
+}

.github/test/events/push-tag.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/standard-tag"
+}

.github/test/test-actions.yml

@@ -0,0 +1,152 @@
+name: Test Actions
+
+on:
+  workflow_dispatch:
+
+jobs:
+  get-image-tag-none:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Log current ref
+        run: |
+          echo "ref: ${{ github.ref }}"
+          echo "ref_type: ${{ github.ref_type }}"
+          echo "ref_name: ${{ github.ref_name }}"
+
+      - name: Run without input tag
+        id: get_tag
+        # this step may fail depending on the current ref
+        continue-on-error: true
+        uses: ./.github/actions/get-image-tag
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-null:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Log current ref
+        run: |
+          echo "ref: ${{ github.ref }}"
+          echo "ref_type: ${{ github.ref_type }}"
+          echo "ref_name: ${{ github.ref_name }}"
+
+      - name: Run without input tag
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        # this step may fail depending on the current ref
+        continue-on-error: true
+        with:
+          # this should behave exactly as when no tag is provided
+          tag: null
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-override:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with tag override
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: "abc-123"
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-invalid-string:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with invalid string
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        # this step is expected to fail
+        continue-on-error: true
+        with:
+          # does not end with alphanumeric character
+          tag: "abc-123-"
+
+      - name: Fail job if previous step did not fail
+        if: steps.get_tag.outcome != 'failure'
+        run: exit 1
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-prerelease:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with prerelease semver
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: "v1.2.3-beta.4"
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-semver:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with basic semver
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: "v1.2.3"
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-invalid-semver:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with invalid semver
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        # this step is expected to fail
+        continue-on-error: true
+        with:
+          tag: "v1.2.3-"
+
+      - name: Fail job if previous step did not fail
+        if: steps.get_tag.outcome != 'failure'
+        run: exit 1
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"

.github/workflows/docs.yml

@@ -0,0 +1,42 @@
+name: 📚 Docs Checks
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "docs/**"
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - "docs/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  check-broken-links:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./docs
+    steps:
+      - name: 📥 Checkout repository
+        uses: actions/checkout@v4
+
+      - name: 📦 Cache npm
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.npm
+          key: |
+            ${{ runner.os }}-mintlify
+          restore-keys: |
+            ${{ runner.os }}-mintlify
+
+      - name: 🔗 Check for broken links
+        run: npx mintlify@4.0.222 broken-links

.github/workflows/e2e.yml

@@ -0,0 +1,53 @@
+name: "E2E"
+
+on:
+  workflow_call:
+    inputs:
+      package:
+        description: The identifier of the job to run
+        default: webapp
+        required: false
+        type: string
+
+jobs:
+  cli-v3:
+    name: "🧪 CLI v3 tests (${{ matrix.os }} - ${{ matrix.package-manager }})"
+    if: inputs.package == 'cli-v3' || inputs.package == ''
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [buildjet-8vcpu-ubuntu-2204, windows-latest]
+        package-manager: ["npm", "pnpm"]
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 8.15.5
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.11.1
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile --filter trigger.dev...
+
+      - name: 🔧 Build v3 cli monorepo dependencies
+        run: pnpm run build --filter trigger.dev^...
+
+      - name: 🔧 Build worker template files
+        run: pnpm --filter trigger.dev run build:workers
+
+      - name: Enable corepack
+        run: corepack enable
+
+      - name: Run E2E Tests
+        shell: bash
+        run: |
+          LOG=debug PM=${{ matrix.package-manager }} pnpm --filter trigger.dev run test:e2e

.github/workflows/pr_checks.yml

@@ -0,0 +1,61 @@
+name: 🤖 PR Checks
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths-ignore:
+      - "docs/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  typecheck:
+    uses: ./.github/workflows/typecheck.yml
+    secrets: inherit
+
+  units:
+    uses: ./.github/workflows/unit-tests.yml
+    secrets: inherit
+
+  e2e:
+    uses: ./.github/workflows/e2e.yml
+    with:
+      package: cli-v3
+    secrets: inherit
+
+  preview-release:
+    name: Preview Release
+    needs: [typecheck, units, e2e]
+    if: github.repository == 'triggerdotdev/trigger.dev'
+    runs-on: buildjet-8vcpu-ubuntu-2204
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 8.15.5
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.11.1
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 🏗️ Build
+        run: pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev"
+
+      - name: ⚡ Publish preview release
+        run: npx pkg-pr-new publish --no-template $(ls -d ./packages/*)

.github/workflows/publish-webapp.yml

@@ -0,0 +1,66 @@
+name: "🐳 Publish Webapp"
+
+on:
+  workflow_call:
+    inputs:
+      image_tag:
+        description: The image tag to publish
+        type: string
+        required: false
+        default: ""
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    env:
+      PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING: 1
+    outputs:
+      version: ${{ steps.get_tag.outputs.tag }}
+      short_sha: ${{ steps.get_commit.outputs.sha_short }}
+    steps:
+      - name: 🏭 Setup Depot CLI
+        uses: depot/setup-action@v1
+
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: "#️⃣ Get the image tag"
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: ${{ inputs.image_tag }}
+
+      - name: 🔢 Get the commit hash
+        id: get_commit
+        run: |
+          echo "sha_short=$(echo ${{ github.sha }} | cut -c1-7)" >> "$GITHUB_OUTPUT"
+
+      - name: 📛 Set the tags
+        id: set_tags
+        run: |
+          ref_without_tag=ghcr.io/triggerdotdev/trigger.dev
+          image_tags=$ref_without_tag:${{ steps.get_tag.outputs.tag }}
+
+          # if tag is a semver, also tag it as v3
+          if [[ "${{ steps.get_tag.outputs.is_semver }}" == true ]]; then
+            image_tags=$image_tags,$ref_without_tag:v3
+          fi
+
+          echo "image_tags=${image_tags}" >> "$GITHUB_OUTPUT"
+
+      - name: 🐙 Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 🐳 Build image and push to GitHub Container Registry
+        uses: depot/build-push-action@v1
+        with:
+          file: ./docker/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.set_tags.outputs.image_tags }}
+          push: true

.github/workflows/publish-worker.yml

@@ -0,0 +1,87 @@
+name: "⚒️ Publish Worker"
+
+on:
+  workflow_call:
+    inputs:
+      image_tag:
+        description: The image tag to publish
+        type: string
+        required: false
+        default: ""
+  push:
+    tags:
+      - "infra-dev-*"
+      - "infra-test-*"
+      - "infra-prod-*"
+
+permissions:
+  packages: write
+  contents: read
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        package: [coordinator, docker-provider, kubernetes-provider]
+    runs-on: buildjet-16vcpu-ubuntu-2204
+    env:
+      DOCKER_BUILDKIT: "1"
+    steps:
+      - name: ⬇️ Checkout git repo
+        uses: actions/checkout@v4
+
+      - name: 📦 Get image repo
+        id: get_repository
+        run: |
+          if [[ "${{ matrix.package }}" == *-provider ]]; then
+            provider_type=$(echo "${{ matrix.package }}" | cut -d- -f1)
+            repo=provider/${provider_type}
+          else
+            repo="${{ matrix.package }}"
+          fi
+          echo "repo=${repo}" >> "$GITHUB_OUTPUT"
+
+      - id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: ${{ inputs.image_tag }}
+
+      - name: 🐋 Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # ..to avoid rate limits when pulling images
+      - name: 🐳 Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: 🚢 Build Container Image
+        run: |
+          docker build -t infra_image -f ./apps/${{ matrix.package }}/Containerfile .
+
+      # ..to push image
+      - name: 🐙 Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 🐙 Push to GitHub Container Registry
+        run: |
+          docker tag infra_image "$REGISTRY/$REPOSITORY:$IMAGE_TAG"
+          docker push "$REGISTRY/$REPOSITORY:$IMAGE_TAG"
+        env:
+          REGISTRY: ghcr.io/triggerdotdev
+          REPOSITORY: ${{ steps.get_repository.outputs.repo }}
+          IMAGE_TAG: ${{ steps.get_tag.outputs.tag }}
+
+      - name: 🐙 Push 'v3' tag to GitHub Container Registry
+        if: steps.get_tag.outputs.is_semver == 'true'
+        run: |
+          docker tag infra_image "$REGISTRY/$REPOSITORY:v3"
+          docker push "$REGISTRY/$REPOSITORY:v3"
+        env:
+          REGISTRY: ghcr.io/triggerdotdev
+          REPOSITORY: ${{ steps.get_repository.outputs.repo }}

.github/workflows/publish.yml

@@ -1,14 +1,26 @@
 name: 🚀 Publish Trigger.dev Docker
 
 on:
+  workflow_call:
+    inputs:
+      image_tag:
+        description: The image tag to publish
+        required: true
+        type: string
   push:
     branches:
       - main
-      - improvements/*
     tags:
       - "v.docker.*"
+      - "build-*"
     paths:
+      - ".github/actions/**/*.yml"
       - ".github/workflows/publish.yml"
+      - ".github/workflows/typecheck.yml"
+      - ".github/workflows/unit-tests.yml"
+      - ".github/workflows/e2e.yml"
+      - ".github/workflows/publish-webapp.yml"
+      - ".github/workflows/publish-worker.yml"
       - "packages/**"
       - "!packages/**/*.md"
       - "!packages/**/*.eslintrc"
@@ -38,190 +50,23 @@ env:
 
 jobs:
   typecheck:
-    name: ʦ TypeScript
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    steps:
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
+    uses: ./.github/workflows/typecheck.yml
+    secrets: inherit
 
-      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
-        with:
-          version: 7.18
+  units:
+    uses: ./.github/workflows/unit-tests.yml
+    secrets: inherit
 
-      - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
-        with:
-          node-version: 18
-          cache: "pnpm"
+  publish-webapp:
+    needs: [typecheck, units]
+    uses: ./.github/workflows/publish-webapp.yml
+    secrets: inherit
+    with:
+      image_tag: ${{ inputs.image_tag }}
 
-      - name: 📥 Download deps
-        run: pnpm install --frozen-lockfile
-
-      - name: 📀 Generate Prisma Client
-        run: pnpm run generate
-
-      - name: 🔎 Type check
-        run: pnpm run typecheck --filter webapp
-
-  unitTests:
-    name: Unit Tests
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    steps:
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
-        with:
-          version: 7.18
-
-      - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
-        with:
-          node-version: 18
-          cache: "pnpm"
-
-      - name: 📥 Download deps
-        run: pnpm install --frozen-lockfile
-
-      - name: Run Unit Tests
-        run: |
-          pnpm run test
-
-  e2e:
-    name: e2e Tests
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    steps:
-      - name: 🐳 Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
-        with:
-          version: 7.18
-
-      - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
-        with:
-          node-version: 18
-          cache: "pnpm"
-
-      - name: 📥 Download deps
-        run: pnpm install --frozen-lockfile
-
-      - name: Install Playwright Browsers
-        run: npx playwright install --with-deps
-
-      - name: Run Playwright tests
-        run: |
-          # Setup environment variables
-          cp ./.env.example ./.env
-          cp ./examples/nextjs-test/.env.example ./examples/nextjs-test/.env.local
-
-          # Build packages
-          pnpm run build --filter @examples/nextjs-test^...
-          pnpm --filter @trigger.dev/database generate
-
-          # Move trigger-cli bin to correct place
-          pnpm install --frozen-lockfile
-
-          # Execute tests
-          pnpm run docker
-          pnpm run db:migrate
-          pnpm run db:seed
-          pnpm run test:e2e
-
-          # Cleanup
-          pnpm run docker:stop
-
-      - name: Upload Playwright report
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: playwright-report
-          path: playwright-report/
-          retention-days: 30
-
-  publish:
-    needs: [typecheck, unitTests, e2e]
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    outputs:
-      version: ${{ steps.get_version.outputs.version }}
-      short_sha: ${{ steps.get_commit.outputs.sha_short }}
-    steps:
-      - name: 🐳 Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
-
-      - name: 🆚 Get the version
-        id: get_version
-        run: |
-          IMAGE_TAG="${GITHUB_REF#refs/tags/}"
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-            if [[ $IMAGE_TAG == v.docker.* ]]; then
-              ORIGINAL_VERSION="${IMAGE_TAG#v.docker.}"
-              IMAGE_TAG="v${ORIGINAL_VERSION}"
-            fi
-            echo "IMAGE_TAG=${IMAGE_TAG}"
-          elif [[ $GITHUB_REF == refs/heads/improvements/* ]]; then
-            ORIGINAL_VERSION="${GITHUB_REF#refs/heads/improvements/}"
-            IMAGE_TAG="${ORIGINAL_VERSION}.rc"
-            echo "IMAGE_TAG=${IMAGE_TAG}"
-          elif [[ $GITHUB_REF == refs/heads/* ]]; then
-            IMAGE_TAG="${GITHUB_REF#refs/heads/}"
-            echo "IMAGE_TAG=${IMAGE_TAG}"
-          else
-            echo "Invalid reference: ${GITHUB_REF}"
-            exit 1
-          fi
-          echo "::set-output name=version::${IMAGE_TAG}"
-      - name: 🔢 Get the commit hash
-        id: get_commit
-        run: |
-          echo ::set-output name=sha_short::$(echo ${{ github.sha }} | cut -c1-7)
-
-      - name: 🐳 Build Docker Image
-        run: |
-          docker build -t release_build_image -f ./docker/Dockerfile .
-
-      - name: 🐙 Login to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: 🐙 Push to GitHub Container Registry
-        run: |
-          docker tag release_build_image $REGISTRY/$REPOSITORY:$IMAGE_TAG
-          docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
-        env:
-          REGISTRY: ghcr.io/triggerdotdev
-          REPOSITORY: trigger.dev
-          IMAGE_TAG: ${{ steps.get_version.outputs.version }}
-
-      - name: 🐙 Push 'latest' to GitHub Container Registry
-        if: startsWith(github.ref, 'refs/tags/')
-        run: |
-          docker tag release_build_image $REGISTRY/$REPOSITORY:latest
-          docker push $REGISTRY/$REPOSITORY:latest
-        env:
-          REGISTRY: ghcr.io/triggerdotdev
-          REPOSITORY: trigger.dev
+  publish-worker:
+    needs: [typecheck, units]
+    uses: ./.github/workflows/publish-worker.yml
+    secrets: inherit
+    with:
+      image_tag: ${{ inputs.image_tag }}

.github/workflows/release.yml

@@ -4,44 +4,40 @@ on:
   push:
     branches:
       - main
-    paths:
-      - ".github/workflows/release.yml"
-      - "packages/**"
-      - "!packages/**/*.md"
-      - ".changeset/**"
-      - "integrations/**"
-      - "!integrations/**/*.md"
-      - "pnpm-lock.yaml"
-      - "pnpm-workspace.yaml"
-      - "turbo.json"
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - ".github/CODEOWNERS"
+      - ".github/ISSUE_TEMPLATE/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   release:
     name: 🦋 Changesets Release
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    if: |
-      github.repository == 'triggerdotdev/trigger.dev'
+    runs-on: ubuntu-latest
+    if: github.repository == 'triggerdotdev/trigger.dev'
     outputs:
-      published_packages: ${{ steps.changesets.outputs.publishedPackages }}
       published: ${{ steps.changesets.outputs.published }}
+      published_packages: ${{ steps.changesets.outputs.publishedPackages }}
+      published_package_version: ${{ steps.get_version.outputs.package_version }}
     steps:
-      - name: 🛑 Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.11.0
-
       - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
         with:
-          version: 7.18
+          version: 8.15.5
 
       - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
+        uses: buildjet/setup-node@v4
         with:
-          node-version: 18
+          node-version: 20.11.1
           cache: "pnpm"
 
       - name: 📥 Download deps
@@ -50,8 +46,11 @@ jobs:
       - name: 📀 Generate Prisma Client
         run: pnpm run generate
 
+      - name: 🏗️ Build
+        run: pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev"
+
       - name: 🔎 Type check
-        run: pnpm run typecheck
+        run: pnpm run typecheck --filter "@trigger.dev/*" --filter "trigger.dev"
 
       - name: 🔐 Setup npm auth
         run: |
@@ -64,6 +63,7 @@ jobs:
         # PR is merged, the workflow will run again and this action will build +
         # publish to npm.
       - name: 🚀 PR / Publish
+        if: ${{ !env.ACT }}
         id: changesets
         uses: changesets/action@v1
         with:
@@ -71,7 +71,31 @@ jobs:
           commit: "chore: Update version for release"
           title: "chore: Update version for release"
           publish: pnpm run changeset:release
-          createGithubReleases: false
+          createGithubReleases: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+      # - name: 🚀 PR / Publish (mock)
+      #   if: ${{ env.ACT }}
+      #   id: changesets
+      #   run: |
+      #     echo "published=true" >> "$GITHUB_OUTPUT"
+      #     echo "publishedPackages=[{\"name\": \"@xx/xx\", \"version\": \"1.2.0\"}, {\"name\": \"@xx/xy\", \"version\": \"0.8.9\"}]" >> "$GITHUB_OUTPUT"
+
+      - name: 📦 Get package version
+        if: steps.changesets.outputs.published == 'true'
+        id: get_version
+        run: |
+          package_version=$(echo '${{ steps.changesets.outputs.publishedPackages }}' | jq -r '.[0].version')
+          echo "package_version=${package_version}" >> "$GITHUB_OUTPUT"
+
+  publish:
+    needs: release
+    uses: ./.github/workflows/publish.yml
+    secrets: inherit
+    # if: needs.release.outputs.published == 'true'
+    # disable automatic publishing for now
+    if: false
+    with:
+      image_tag: v${{ needs.release.outputs.published_package_version }}

.github/workflows/typecheck.yml

@@ -0,0 +1,37 @@
+name: "ʦ TypeScript"
+
+on:
+  workflow_call:
+
+jobs:
+  typecheck:
+    runs-on: buildjet-8vcpu-ubuntu-2204
+
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 8.15.5
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.11.1
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: 🔎 Type check
+        run: pnpm run typecheck
+
+      - name: 🔎 Check exports
+        run: pnpm run check-exports

.github/workflows/unit-tests.yml

@@ -0,0 +1,46 @@
+name: "🧪 Unit Tests"
+
+on:
+  workflow_call:
+
+jobs:
+  unitTests:
+    name: "🧪 Unit Tests"
+    runs-on: buildjet-16vcpu-ubuntu-2204
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 8.15.5
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.11.1
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: 🧪 Run Webapp Unit Tests
+        run: pnpm run test --filter webapp
+        env:
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
+          DIRECT_URL: postgresql://postgres:postgres@localhost:5432/postgres
+          SESSION_SECRET: "secret"
+          MAGIC_LINK_SECRET: "secret"
+          ENCRYPTION_KEY: "secret"
+      
+      - name: 🧪 Run Package Unit Tests
+        run: pnpm run test --filter "@trigger.dev/*"
+      
+      - name: 🧪 Run Internal Unit Tests
+        run: pnpm run test --filter "@internal/*"

.gitignore

@@ -12,7 +12,6 @@ coverage
 # next.js
 .next/
 out/
-build
 dist
 packages/**/dist
 
@@ -52,3 +51,10 @@ apps/**/public/build
 /test-results/
 /playwright-report/
 /playwright/.cache/
+
+.cosine
+.trigger
+.tshy*
+.yarn
+*.tsbuildinfo
+/packages/cli-v3/src/package.json

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "internal-packages/otlp-importer/protos"]
+	path = internal-packages/otlp-importer/protos
+	url = https://github.com/open-telemetry/opentelemetry-proto.git

.npmrc

@@ -1,2 +1,3 @@
 link-workspace-packages=false
-public-hoist-pattern[]=*prisma*
+public-hoist-pattern[]=*prisma*
+prefer-workspace-packages=true

.nvmrc

@@ -1 +1 @@
-v18.12.1
+v20.11.1

.prettierignore

@@ -7,5 +7,4 @@ tailwind.css
 **/.react-email/
 **/storybook-static/
 **/.changeset/
-**/build/
 **/dist/

.vscode/extensions.json

@@ -0,0 +1,4 @@
+{
+  "recommendations": ["bierner.comment-tagged-templates"],
+  "unwantedRecommendations": []
+}

.vscode/launch.json

@@ -9,16 +9,113 @@
       "request": "launch",
       "name": "Debug WebApp",
       "command": "pnpm run dev --filter webapp",
-      "envFile": "${workspaceFolder}/apps/webapp/.env",
+      "envFile": "${workspaceFolder}/.env",
       "cwd": "${workspaceFolder}",
       "sourceMaps": true
     },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug realtimeStreams.test.ts",
+      "command": "pnpm run test -t RealtimeStreams",
+      "envFile": "${workspaceFolder}/.env",
+      "cwd": "${workspaceFolder}/apps/webapp",
+      "sourceMaps": true
+    },
     {
       "type": "chrome",
       "request": "launch",
       "name": "Chrome webapp",
       "url": "http://localhost:3030",
       "webRoot": "${workspaceFolder}/apps/webapp/app"
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 init CLI",
+      "command": "pnpm exec trigger init",
+      "cwd": "${workspaceFolder}/references/init-shell",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 Dev CLI",
+      "command": "pnpm exec trigger dev",
+      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug Dev Next.js Realtime",
+      "command": "pnpm exec trigger dev",
+      "cwd": "${workspaceFolder}/references/nextjs-realtime",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug prisma-catalog deploy CLI",
+      "command": "pnpm exec trigger deploy --self-hosted --load-image",
+      "cwd": "${workspaceFolder}/references/prisma-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 Deploy CLI",
+      "command": "pnpm exec trigger deploy --self-hosted --load-image",
+      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 list-profiles CLI",
+      "command": "pnpm exec trigger list-profiles --log-level debug",
+      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 update CLI",
+      "command": "pnpm exec trigger update",
+      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 Management",
+      "command": "pnpm run management",
+      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node",
+      "request": "attach",
+      "name": "Attach to Trigger.dev CLI (v3)",
+      "port": 9229,
+      "restart": true,
+      "skipFiles": ["<node_internals>/**"]
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug CLI e2e tests",
+      "command": "MOD=otel-telemetry-loader pnpm run test:e2e",
+      "cwd": "${workspaceFolder}/packages/cli-v3",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "debug v3 hello-world dev",
+      "command": "pnpm exec trigger dev",
+      "cwd": "${workspaceFolder}/references/hello-world",
+      "sourceMaps": true
     }
   ]
 }

.vscode/settings.json

@@ -0,0 +1,9 @@
+{
+  "deno.enablePaths": ["references/deno-reference", "runtime_tests/tests/deno"],
+  "debug.toolBarLocation": "commandCenter",
+  "typescript.tsdk": "node_modules/typescript/lib",
+  "search.exclude": {
+    "**/node_modules/**": true,
+    "packages/cli-v3/e2e": true
+  }
+}

CHANGESETS.md

@@ -27,3 +27,31 @@ Please follow the best-practice of adding changesets in the same commit as the c
 3. Create version `pnpm run changeset:version`
 4. Release `pnpm run changeset:release`
 5. Switch back to normal mode by running `pnpm run changeset:normal`
+
+## Snapshot instructions
+
+!MAKE SURE TO UPDATE THE TAG IN THE INSTRUCTIONS BELOW!
+
+1. Add changesets as usual
+
+```sh
+pnpm run changeset:add
+```
+
+2. Create a snapshot version (replace "prerelease" with your tag)
+
+```sh
+pnpm exec changeset version --snapshot prerelease
+```
+
+3. Build the packages:
+
+```sh
+pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev"
+```
+
+4. Publish the snapshot (replace "dev" with your tag)
+
+```sh
+pnpm exec changeset publish --no-git-tag --snapshot --tag prerelease
+```

CODE_OF_CONDUCT.md

@@ -0,0 +1,43 @@
+# Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment include:
+
+- Using welcoming and inclusive language
+- Being respectful of differing viewpoints and experiences
+- Gracefully accepting constructive criticism
+- Focusing on what is best for the community
+- Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+- The use of sexualized language or imagery and unwelcome sexual attention or advances
+- Trolling, insulting/derogatory comments, and personal or political attacks
+- Public or private harassment
+- Publishing others' private information, such as a physical or electronic address, without explicit permission
+- Other conduct that could reasonably be considered inappropriate in a professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned with this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project email address, posting via an official social media account, or acting as an appointed representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team  All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the Contributor Covenant, version 2.0, available at [https://www.contributor-covenant.org/version/2/0/code_of_conduct.html](https://www.contributor-covenant.org/version/2/0/code_of_conduct.html).

CONTRIBUTING.md

@@ -10,38 +10,43 @@ Thank you for helping us make Trigger.dev even better! 🤩
 
 The development branch is `main`. This is the branch that all pull
 requests should be made against. The changes on the `main`
-branch are tagged into a release monthly.
+branch are tagged into a release periodically.
 
 ### Prerequisites
 
-- [Node.js](https://nodejs.org/en) version >=16.x
-- [pnpm package manager](https://pnpm.io/installation) version 7
+- [Node.js](https://nodejs.org/en) version 20.11.1
+- [pnpm package manager](https://pnpm.io/installation) version 8.15.5
 - [Docker](https://www.docker.com/get-started/)
+- [protobuf](https://github.com/protocolbuffers/protobuf)
 
 ### Setup
 
 1. Clone the repo into a public GitHub repository or [fork the repo](https://github.com/triggerdotdev/trigger.dev/fork). If you plan to distribute the code, keep the source code public to comply with the [Apache Licence 2.0](https://github.com/triggerdotdev/trigger.dev/blob/main/LICENSE).
 
    ```
-   git clone https://github.com/triggerdotdev/trigger.dev.git
+   git clone https://github.com/<github_username>/trigger.dev.git
    ```
 
    > If you are on windows, run the following command on gitbash with admin privileges:
-   > `git clone -c core.symlinks=true https://github.com/triggerdotdev/trigger.dev.git`
+   > `git clone -c core.symlinks=true https://github.com/<github_username>/trigger.dev.git`
 
 2. Navigate to the project folder
    ```
    cd trigger.dev
    ```
-3. Install the required packages using pnpm.
+3. Ensure you are on the correct version of Node.js (20.11.1). If you are using `nvm`, there is an `.nvmrc` file that will automatically select the correct version of Node.js when you navigate to the repository.
+
+4. Run `corepack enable` to use the correct version of pnpm (`8.15.5`) as specified in the root `package.json` file.
+
+5. Install the required packages using pnpm.
    ```
    pnpm i
    ```
-4. Create your `.env` file
+6. Create your `.env` file
    ```
    cp .env.example .env
    ```
-5. Open it and generate a new value for `ENCRYPTION_KEY`:
+7. Open it and generate a new value for `ENCRYPTION_KEY`:
 
    `ENCRYPTION_KEY` is used to two-way encrypt OAuth access tokens and so you'll probably want to actually generate a unique value, and it must be a random 16 byte hex string. You can generate one with the following command:
 
@@ -51,23 +56,23 @@ branch are tagged into a release monthly.
 
    Feel free to update `SESSION_SECRET` and `MAGIC_LINK_SECRET` as well using the same method.
 
-6. Start Docker. This starts the required services like Postgres. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)
+8. Start Docker. This starts the required services like Postgres & Redis. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)
+
    ```
    pnpm run docker
    ```
-7. Migrate the database
+
+   This will also start and run a local instance of [pgAdmin](https://www.pgadmin.org/) on [localhost:5480](http://localhost:5480), preconfigured with email `admin@example.com` and pwd `admin`. Then use `postgres` as the password to the Trigger.dev server.
+
+9. Migrate the database
    ```
    pnpm run db:migrate
    ```
-8. Build the app
-   ```
-   pnpm run build --filter webapp
-   ```
-9. Run the seed script
-   ```
-   pnpm run db:seed
-   ```
-10. Run the app. See the section below.
+10. Build the server app
+    ```
+    pnpm run build --filter webapp
+    ```
+11. Run the app. See the section below.
 
 ## Running
 
@@ -79,99 +84,96 @@ branch are tagged into a release monthly.
 
    It should run on port `3030`: [http://localhost:3030](http://localhost:3030/)
 
-2. Once the app is running click the magic link button and enter your email.
-3. Check your terminal, the magic link email should have printed out as following:
+2. Once the app is running click the magic link button and enter your email. You will automatically be logged in, since you are running locally. Create an Org and your first project in the dashboard.
 
-   ```sh
-   webapp:dev: Log in to Trigger.dev
-   webapp:dev:
-   webapp:dev: Click here to log in with this magic link
-   webapp:dev: [http://localhost:3030/magic?token=U2FsdGVkX18OvB0JxgaswTLCSbaRz%2FY82TN0EZWhSzFyZYwgG%2BIzKVTkeiaOtWfotPw7F8RwFzCHh53aBpMEu%2B%2B%2FItb%2FcJYh89MSjc3Pz92bevoEjqxSQ%2Ff%2BZbks09JOpqlBbYC3FzGWC8vuSVFBlxqLXxteSDLthZSUaC%2BS2LaA%2BJgp%2BLO7hgjAaC2lXbCHrM7MTgTdXOFt7i0Dvvuwz6%2BWY25RnfomZOPqDsyH0xz8Q2rzPTz0Xu53WSXrZ1hd]
-   webapp:dev:
-   webapp:dev: If you didn't try to log in, you can safely ignore this email.
-   ```
+## Manual testing using v3-catalog
 
-   Paste the magic link shown in your terminal into your browser to login.
+We use the `<root>/references/v3-catalog` subdirectory as a staging ground for testing changes to the SDK (`@trigger.dev/sdk` at `<root>/packages/trigger-sdk`), the Core package (`@trigger.dev/core` at `<root>packages/core`), the CLI (`trigger.dev` at `<root>/packages/cli-v3`) and the platform (The remix app at `<root>/apps/webapp`). The instructions below will get you started on using the `v3-catalog` for local development of Trigger.dev (v3).
 
-## Adding and running migrations
+### First-time setup
 
-1. Modify packages/database/prisma/schema.prisma file
-2. Change directory to the packages/database folder
-   ```sh
-   cd packages/database
-   ```
-3. Generate the Prisma client
+First, make sure you are running the webapp according to the instructions above. Then:
 
-   ```sh
-   pnpm run generate
-   ```
+1. In Postgres go to the "Organizations" table and on your org set the `v3Enabled` column to `true`.
 
-   The above updates the prisma client generated into node_modules/.prisma/client folder. This helps with typing of relevant prisma models. It ensures typescript
-   recognizes fields added or removed from a model and type-checks appropriately.
+2. Visit http://localhost:3030 in your browser and create a new V3 project called "v3-catalog". If you don't see an option for V3, you haven't set the `v3Enabled` flag to true.
 
-4. Create and apply the migrations
+3. In Postgres go to the "Projects" table and for the project you create change the `externalRef` to `yubjwjsfkxnylobaqvqz`.
 
-   ```
-   pnpm run db:migrate:dev
-   ```
-
-   This creates a migration file and executes the migrations against your database and applies changes to the database schema(s)
-
-5. Commit generated migrations as well as changes to the schema.prisma file
-6. If you're using VSCode you may need to restart the Typescript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`.
-
-## Testing CLI changes
-
-To test CLI changes, follow the steps below:
-
-1. Build the CLI and watch for changes
+4. Build the CLI
 
 ```sh
-cd packages/cli
-pnpm run dev
-```
-
-2. Open a new Terminal window and run the webapp locally and then create a new project in the dashboard. Copy out the dev API key.
-
-3. Create a new temporary Next.js app in examples directory
-
-```sh
-cd ./examples
-pnpm create next-app@latest test-cli --ts --no-eslint --tailwind --app --src-dir --import-alias "@/*"
-```
-
-4. Then once that's finished, add the `@trigger.dev/cli` to the `devDependencies` of the newly created Next.js app's `package.json` file, like so:
-
-```json
-{
-  // other package.json properties
-  "devDependencies": { "@trigger.dev/cli": "workspace:*" }
-}
-```
-
-5. Back in the terminal, navigate into the example, and initialize the CLI. When prompted, select `self-hosted` and enter `localhost:3030` if you are testing against the local instance of Trigger.dev, or you can just use the Trigger.dev cloud. When asked for an API key, use the key you copied earlier.
-
-```sh
-cd ./test-cli
+# Build the CLI
+pnpm run build --filter trigger.dev
+# Make it accessible to `pnpm exec`
 pnpm i
-pnpm exec trigger-cli init
 ```
 
-6. If you are just testing the `init` command, you can stop here. If you'd like to test the `dev` command, first start the Next.js app on port 3000:
+5. Change into the `<root>/references/v3-catalog` directory and authorize the CLI to the local server:
 
 ```sh
-pnpm run dev
+cd references/v3-catalog
+cp .env.example .env
+pnpm exec trigger login -a http://localhost:3030
 ```
 
-7. Open a new terminal window, and then run the `dev` command like so:
+This will open a new browser window and authorize the CLI against your local user account.
+
+You can optionally pass a `--profile` flag to the `login` command, which will allow you to use the CLI with separate accounts/servers. We suggest using a profile called `local` for your local development:
 
 ```sh
-pnpm exec trigger-cli dev
+cd references/v3-catalog
+pnpm exec trigger login -a http://localhost:3030 --profile local
+# later when you run the dev or deploy command:
+pnpm exec trigger dev --profile local
+pnpm exec trigger deploy --profile local
 ```
 
-8. Please remember to delete the temporary project you created after you've tested the changes, and before you raise a PR.
+### Running
 
-## Running end-to-end webapp tests
+The following steps should be followed any time you start working on a new feature you want to test in v3:
+
+1. Make sure the webapp is running on localhost:3030
+
+2. Open a terminal window and build the CLI and watch for changes
+
+```sh
+pnpm run dev --filter trigger.dev
+```
+
+2. Open a new terminal window, and anytime changes are made to the `@trigger.dev/core` package, you'll need to manually rebuild the CLI:
+
+```sh
+pnpm run build --filter trigger.dev
+```
+
+Note: You do not need to do the same for `@trigger.dev/sdk`, just core.
+
+3. Open another terminal window, and change into the `<root>/references/v3-catalog` directory.
+
+4. Run the `dev` command, which will register all the local tasks with the platform and allow you to start testing task execution:
+
+```sh
+# in <root>/references/v3-catalog
+pnpm exec trigger dev
+```
+
+If you want additional debug logging, you can use the `--log-level debug` flag:
+
+```sh
+# in <root>/references/v3-catalog
+pnpm exec trigger dev --log-level debug
+```
+
+5. If you make any changes in the CLI/Core/SDK, you'll need to `CTRL+C` to exit the `dev` command and restart it to pickup changes. Any changes to the files inside of the `v3-catalog/src/trigger` dir will automatically be rebuilt by the `dev` command.
+
+6. Navigate to the `v3-catalog` project in your local dashboard at localhost:3030 and you should see the list of tasks.
+
+7. Go to the "Test" page in the sidebar and select a task. Then enter a payload and click "Run test". You can tell what the payloads should be by looking at the relevant task file inside the `/references/v3-catalog/src/trigger` folder. Many of them accept an empty payload.
+
+8. Feel free to add additional files in `v3-catalog/src/trigger` to test out specific aspects of the system, or add in edge cases.
+
+## Running end-to-end webapp tests (deprecated)
 
 To run the end-to-end tests, follow the steps below:
 
@@ -179,14 +181,14 @@ To run the end-to-end tests, follow the steps below:
 
 ```sh
 cp ./.env.example ./.env
-cp ./examples/nextjs-test/.env.example ./examples/nextjs-test/.env.local
+cp ./references/nextjs-test/.env.example ./references/nextjs-test/.env.local
 ```
 
 2. Set up dependencies
 
 ```sh
 # Build packages
-pnpm run build --filter @examples/nextjs-test^...
+pnpm run build --filter @references/nextjs-test^...
 pnpm --filter @trigger.dev/database generate
 
 # Move trigger-cli bin to correct place
@@ -219,13 +221,33 @@ The end-to-end tests use a `setup` and `teardown` script to seed the database wi
 pnpm run db:studio
 ```
 
+## Adding and running migrations
+
+1. Modify internal-packages/database/prisma/schema.prisma file
+2. Change directory to the packages/database folder
+
+   ```sh
+   cd packages/database
+   ```
+
+3. Create and apply the migrations
+
+   ```
+   pnpm run db:migrate:dev
+   ```
+
+   This creates a migration file and executes the migrations against your database and applies changes to the database schema(s)
+
+4. Commit generated migrations as well as changes to the schema.prisma file
+5. If you're using VSCode you may need to restart the Typescript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`.
+
 ## Add sample jobs
 
-The [examples/jobs-starter](./examples/jobs-starter/) project defines simple jobs you can get started with.
+The [references/job-catalog](./references/job-catalog/) project defines simple jobs you can get started with.
 
-1. `cd` into `examples/jobs-starter`
-2. Create a `.env.local` file with the following content,
-   replacing `[TRIGGER_DEV_API_KEY]` with an actual key:
+1. `cd` into `references/job-catalog`
+2. Create a `.env` file with the following content,
+   replacing `<TRIGGER_DEV_API_KEY>` with an actual key:
 
 ```env
 TRIGGER_API_KEY=[TRIGGER_DEV_API_KEY]
@@ -235,12 +257,20 @@ TRIGGER_API_URL=http://localhost:3030
 `TRIGGER_API_URL` is used to configure the URL for your Trigger.dev instance,
 where the jobs will be registered.
 
-3. Run the `jobs-starter` app:
+3. Run one of the the `job-catalog` files:
 
 ```sh
-pnpm dev
+pnpm run events
 ```
 
+This will open up a local server using `express` on port 8080. Then in a new terminal window you can run the trigger-cli dev command:
+
+```sh
+pnpm run dev:trigger
+```
+
+See the [Job Catalog](./references/job-catalog/README.md) file for more.
+
 4. Navigate to your trigger.dev instance ([http://localhost:3030](http://localhost:3030/)), to see the jobs.
    You can use the test feature to trigger them.
 

README.md

@@ -5,81 +5,87 @@
   <img alt="Trigger.dev logo" src="https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/a45d1fa2-0ae8-4a39-4409-f4f934bfae00/public">
 </picture>
   
-### The open source background jobs framework
+### Open source background jobs with no timeouts
 
-[Discord](https://discord.gg/JtBAxBr2m3) | [Website](https://trigger.dev) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Docs](https://trigger.dev/docs)
+[Discord](https://trigger.dev/discord) | [Website](https://trigger.dev) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Docs](https://trigger.dev/docs)
 
 [![Twitter](https://img.shields.io/twitter/url/https/twitter.com/triggerdotdev.svg?style=social&label=Follow%20%40trigger.dev)](https://twitter.com/triggerdotdev)
-[![GitHub Repo stars](https://img.shields.io/github/stars/triggerdotdev/trigger.dev?style=social)](https://github.com/triggerdotdev/trigger.dev)
 
 </div>
 
-# About Trigger.dev
+## About Trigger.dev
 
-Create long-running jobs directly in your codebase with features like API integrations, webhooks, scheduling and delays.
+Trigger.dev is an open source platform and SDK which allows you to create long-running background jobs. Write normal async code, deploy, and never hit a timeout.
 
-## Long running Jobs on serverless
+### Key features:
 
-Reliably run jobs and don’t worry about function timeouts, we handle those for you.
-
-- Auto-resume after a function timeout
-- Auto-resume after a server outage
-- Add delays of up to a year
+- JavaScript and TypeScript SDK
+- Write reliable code by default
+- No timeouts
+- Retries (with exponential backoff)
+- Queues and concurrency controls
+- Schedules and crons
+- Full Observability; logs, live trace views, advanced filtering
+- Custom alerts, get notified by email, Slack or webhooks
+- No infrastructure to manage
+- Elastic (scaling)
+- Works with your existing tech stack
 
 ## In your codebase
 
-Create Jobs where they belong: in your codebase. Version control, localhost, test, review, and deploy like you're already used to.
+Create tasks where they belong: in your codebase. Version control, localhost, test and review like you're already used to.
 
-## Secure by design
+```ts
+import { task } from "@trigger.dev/sdk/v3";
 
-We only receive Triggers and the data you choose to send to us. You can even completely self-host the entire platform.
+//1. You need to export each task
+export const helloWorld = task({
+  //2. Use a unique id for each task
+  id: "hello-world",
+  //3. The run function is the main function of the task
+  run: async (payload: { message: string }) => {
+    //4. You can write code that runs for a long time here, there are no timeouts
+    console.log(payload.message);
+  },
+});
+```
 
-## Don't worry about deployment
+## Deployment
 
-Just use our SDK to write Jobs in your codebase. There's nothing extra to deploy and no CI to configure, your Jobs just connect to our cloud. Or you can always self-host.
+Use our SDK to write tasks in your codebase. There's no infrastructure to manage, your tasks automatically scale and connect to our cloud. Or you can always self-host.
+
+## Environments
+
+We support `Development`, `Staging`, and `Production` environments, allowing you to test your tasks before deploying them to production.
 
 ## Full visibility of every job run
 
-View every Task in every Run so you can tell exactly what happened.
+View every task in every run so you can tell exactly what happened. We provide a full trace view of every task run so you can see what happened at every step.
 
-![image](https://www.trigger.dev/build/_assets/web-app-2QFKXFLW.png)
-
-## Built-in integrations
-
-Easily integrate with hundreds of third-party APIs – including your own. Use API keys (which never leave your server) or let us handle OAuth for you. Install our integration packages and easily subscribe to webhooks and perform common tasks, or you can easily use your existing favorite Node.JS SDKs and get resumability and idempotency through our `runTask` function.
-
-## Our progress
-
-We’re building the most comprehensive and easy-to-use background jobs framework for developers.
-
-Click the links to join the discussions about our upcoming features.
-
-| Feature                                                                              | What it does                                        | Status |
-| ------------------------------------------------------------------------------------ | --------------------------------------------------- | ------ |
-| Integration kit                                                                      | Official Trigger.dev integrations or build your own | ✅     |
-| Self-hosting                                                                         | Host the platform yourself                          | ✅     |
-| Cloud                                                                                | Just write code, no deployment required             | ✅     |
-| Dashboard                                                                            | View every Task in every Run                        | ✅     |
-| Serverless                                                                           | Long-running Jobs on your serverless backend        | ✅     |
-| React hooks                                                                          | Easily update your UI with Job progress             | ✅     |
-| [Background tasks](https://github.com/triggerdotdev/trigger.dev/discussions/400)     | Offload long or intense Tasks to our infrastructure | 🛠️     |
-| [React frameworks](https://github.com/triggerdotdev/trigger.dev/discussions/411)     | Support for Remix, Astro, RedwoodJS & more          | 🛠️     |
-| [Long-running servers](https://github.com/triggerdotdev/trigger.dev/discussions/430) | Run Jobs on your long-running backend               | 🛠️     |
-| Polling Triggers                                                                     | Subscribe to changes without webhooks               | 🕝     |
-| Vercel integration                                                                   | Easy deploy and preview environment support         | 🕝     |
-| Streaming                                                                            | Receive data from your Jobs in realtime             | 🕝     |
-| 100+ integrations                                                                    | Comprehensive support for popular APIs              | 🕝     |
-| [Trigger.dev Connect](https://github.com/triggerdotdev/trigger.dev/discussions/441)  | Use integrations signed in as your users            | 🕝     |
-| File IO                                                                              | Create Tasks that have file outputs                 | 🕝     |
+![Trace view image](https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/7c1b347f-004c-4482-38a7-3f6fa9c00d00/public)
 
 # Getting started
 
-Visit our docs [here](https://trigger.dev/docs).
+The quickest way to get started is to create an account and project in our [web app](https://cloud.trigger.dev), and follow the instructions in the onboarding. Build and deploy your first task in minutes.
 
-## Self-host
+### Useful links:
 
-We provide an official trigger.dev docker image you can use to easily self-host the platform. We're working on more extensive guides but we currently provide a [Fly.io example repository](https://github.com/triggerdotdev/fly.io) with instructions in the README for deploying and using a self-hosted instance of Trigger.dev on Fly.io.
+- [Quick start](https://trigger.dev/docs/quick-start) - get up and running in minutes
+- [How it works](https://trigger.dev/docs/v3/how-it-works) - understand how Trigger.dev works under the hood
+- [Guides and examples](https://trigger.dev/docs/guides/introduction) - walk-through guides and code examples for popular frameworks and use cases
+
+## Self-hosting
+
+If you prefer to self-host Trigger.dev, you can follow our [self-hosting guide](https://trigger.dev/docs/v3/open-source-self-hosting#overview).
+
+We also have a dedicated self-hosting channel in our [Discord server](https://trigger.dev/discord) for support.
 
 ## Development
 
 To setup and develop locally or contribute to the open source project, follow our [development guide](./CONTRIBUTING.md).
+
+## Meet the Amazing People Behind This Project:
+
+<a href="https://github.com/triggerdotdev/trigger.dev/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=triggerdotdev/trigger.dev" />
+</a>

RELEASE.md

@@ -0,0 +1,13 @@
+## Guide on releasing a new version
+
+1. Merge in the changeset PR into main, making sure to cancel both the release and publish github actions from that merge.
+2. Pull the changes locally into main
+3. Run `pnpm i` which will update the pnpm lock file with the new versions
+4. create a commit with "Release 3.x.x" and push. This will build and release the packages
+5. Create a git tag on that release commit with v.docker.3.x.x and push the tag to origin. This will publish the `v3.x.x` docker image to GitHub Container Registry.
+6. Once the image is built and pushed, create a new GitHub release and select the tag you just created, along with the previous tag that was released.
+7. This will generate some release notes. Edit out the package changes and leave only the server changes.
+8. Name the release `@trigger.dev/docker@3.x.x`
+9. Include the package link (e.g. https://github.com/triggerdotdev/trigger.dev/pkgs/container/trigger.dev/278459584?tag=v3.x.x)
+10. Once the packages have been published, head over to the [v2-legacy repo](https://github.com/triggerdotdev/v2-legacy.trigger.dev) and follow the instructions in the README for creating a matching release.
+11. Before deploying to cloud, compare the differences in the previously created release and double check to see if there are any migrations with indexes created concurrently, and make sure to run those before deploying.

apps/coordinator/.env.example

@@ -0,0 +1,4 @@
+HTTP_SERVER_PORT=8020
+PLATFORM_ENABLED=true
+PLATFORM_WS_PORT=3030
+SECURE_CONNECTION=false

apps/coordinator/.gitignore

@@ -0,0 +1,3 @@
+dist/
+node_modules/
+.env

apps/coordinator/Containerfile

@@ -0,0 +1,60 @@
+# syntax=docker/dockerfile:labs
+
+FROM node:20-bookworm-slim@sha256:72f2f046a5f8468db28730b990b37de63ce93fd1a72a40f531d6aa82afdf0d46 AS node-20
+
+WORKDIR /app
+
+FROM node-20 AS pruner
+
+COPY --chown=node:node . .
+RUN npx -q turbo@1.10.9 prune --scope=coordinator --docker
+RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
+
+FROM node-20 AS base
+
+RUN apt-get update \
+  && apt-get install -y buildah ca-certificates dumb-init docker.io \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY --chown=node:node .gitignore .gitignore
+COPY --from=pruner --chown=node:node /app/out/json/ .
+COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
+COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
+
+FROM base AS dev-deps
+RUN corepack enable
+ENV NODE_ENV development
+
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
+
+FROM base AS builder
+RUN corepack enable
+
+COPY --from=pruner --chown=node:node /app/out/full/ .
+COPY --from=dev-deps --chown=node:node /app/ .
+COPY --chown=node:node turbo.json turbo.json
+
+RUN pnpm run -r --filter coordinator build:bundle
+
+FROM alpine AS cri-tools
+
+WORKDIR /cri-tools
+
+ARG CRICTL_VERSION=v1.29.0
+ARG CRICTL_CHECKSUM=sha256:d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb
+ADD --checksum=${CRICTL_CHECKSUM} \
+  https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz .
+RUN tar zxvf crictl-${CRICTL_VERSION}-linux-amd64.tar.gz
+
+FROM base AS runner
+
+RUN corepack enable
+ENV NODE_ENV production
+
+COPY --from=cri-tools --chown=node:node /cri-tools/crictl /usr/local/bin
+COPY --from=builder --chown=node:node /app/apps/coordinator/dist/index.mjs ./index.mjs
+
+EXPOSE 8000
+
+CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]

apps/coordinator/README.md

@@ -0,0 +1,3 @@
+# Coordinator
+
+Sits between the platform and tasks. Facilitates communication and checkpointing, amongst other things.

apps/coordinator/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "coordinator",
+  "private": true,
+  "version": "0.0.1",
+  "description": "",
+  "main": "dist/index.cjs",
+  "scripts": {
+    "build": "npm run build:bundle",
+    "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"",
+    "build:image": "docker build -f Containerfile . -t coordinator",
+    "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
+    "start": "tsx src/index.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@trigger.dev/core": "workspace:*",
+    "nanoid": "^5.0.6",
+    "prom-client": "^15.1.0",
+    "socket.io": "4.7.4",
+    "tinyexec": "^0.3.0"
+  },
+  "devDependencies": {
+    "@types/node": "^18",
+    "dotenv": "^16.4.2",
+    "esbuild": "^0.19.11",
+    "tsx": "^4.7.0",
+    "typescript": "^5.3.3"
+  }
+}

apps/coordinator/src/chaosMonkey.ts

@@ -0,0 +1,88 @@
+import { setTimeout as timeout } from "node:timers/promises";
+
+class ChaosMonkeyError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ChaosMonkeyError";
+  }
+}
+
+export class ChaosMonkey {
+  private chaosEventRate = 0.2;
+  private delayInSeconds = 45;
+
+  constructor(
+    private enabled = false,
+    private disableErrors = false,
+    private disableDelays = false
+  ) {
+    if (this.enabled) {
+      console.log("🍌 Chaos monkey enabled");
+    }
+  }
+
+  static Error = ChaosMonkeyError;
+
+  enable() {
+    this.enabled = true;
+    console.log("🍌 Chaos monkey enabled");
+  }
+
+  disable() {
+    this.enabled = false;
+    console.log("🍌 Chaos monkey disabled");
+  }
+
+  async call({
+    throwErrors = !this.disableErrors,
+    addDelays = !this.disableDelays,
+  }: {
+    throwErrors?: boolean;
+    addDelays?: boolean;
+  } = {}) {
+    if (!this.enabled) {
+      return;
+    }
+
+    const random = Math.random();
+
+    if (random > this.chaosEventRate) {
+      // Don't interfere with normal operation
+      return;
+    }
+
+    const chaosEvents: Array<() => Promise<any>> = [];
+
+    if (addDelays) {
+      chaosEvents.push(async () => {
+        console.log("🍌 Chaos monkey: Add delay");
+
+        await timeout(this.delayInSeconds * 1000);
+      });
+    }
+
+    if (throwErrors) {
+      chaosEvents.push(async () => {
+        console.log("🍌 Chaos monkey: Throw error");
+
+        throw new ChaosMonkey.Error("🍌 Chaos monkey: Throw error");
+      });
+    }
+
+    if (chaosEvents.length === 0) {
+      console.error("🍌 Chaos monkey: No events selected");
+      return;
+    }
+
+    const randomIndex = Math.floor(Math.random() * chaosEvents.length);
+
+    const chaosEvent = chaosEvents[randomIndex];
+
+    if (!chaosEvent) {
+      console.error("🍌 Chaos monkey: No event found");
+      return;
+    }
+
+    await chaosEvent();
+  }
+}

apps/coordinator/src/checkpointer.ts

@@ -0,0 +1,671 @@
+import { ExponentialBackoff } from "@trigger.dev/core/v3/apps";
+import { testDockerCheckpoint } from "@trigger.dev/core/v3/apps";
+import { nanoid } from "nanoid";
+import fs from "node:fs/promises";
+import { ChaosMonkey } from "./chaosMonkey";
+import { Buildah, Crictl, Exec } from "./exec";
+import { setTimeout } from "node:timers/promises";
+import { TempFileCleaner } from "./cleaner";
+import { numFromEnv, boolFromEnv } from "./util";
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+
+type CheckpointerInitializeReturn = {
+  canCheckpoint: boolean;
+  willSimulate: boolean;
+};
+
+type CheckpointAndPushOptions = {
+  runId: string;
+  leaveRunning?: boolean;
+  projectRef: string;
+  deploymentVersion: string;
+  shouldHeartbeat?: boolean;
+  attemptNumber?: number;
+};
+
+type CheckpointAndPushResult =
+  | { success: true; checkpoint: CheckpointData }
+  | {
+      success: false;
+      reason?: "CANCELED" | "DISABLED" | "ERROR" | "IN_PROGRESS" | "NO_SUPPORT" | "SKIP_RETRYING";
+    };
+
+type CheckpointData = {
+  location: string;
+  docker: boolean;
+};
+
+type CheckpointerOptions = {
+  dockerMode: boolean;
+  forceSimulate: boolean;
+  heartbeat: (runId: string) => void;
+  registryHost?: string;
+  registryNamespace?: string;
+  registryTlsVerify?: boolean;
+  disableCheckpointSupport?: boolean;
+  checkpointPath?: string;
+  simulateCheckpointFailure?: boolean;
+  simulateCheckpointFailureSeconds?: number;
+  simulatePushFailure?: boolean;
+  simulatePushFailureSeconds?: number;
+  chaosMonkey?: ChaosMonkey;
+};
+
+async function getFileSize(filePath: string): Promise<number> {
+  try {
+    const stats = await fs.stat(filePath);
+    return stats.size;
+  } catch (error) {
+    console.error("Error getting file size:", error);
+    return -1;
+  }
+}
+
+async function getParsedFileSize(filePath: string) {
+  const sizeInBytes = await getFileSize(filePath);
+
+  let message = `Size in bytes: ${sizeInBytes}`;
+
+  if (sizeInBytes > 1024 * 1024) {
+    const sizeInMB = (sizeInBytes / 1024 / 1024).toFixed(2);
+    message = `Size in MB (rounded): ${sizeInMB}`;
+  } else if (sizeInBytes > 1024) {
+    const sizeInKB = (sizeInBytes / 1024).toFixed(2);
+    message = `Size in KB (rounded): ${sizeInKB}`;
+  }
+
+  return {
+    path: filePath,
+    sizeInBytes,
+    message,
+  };
+}
+
+export class Checkpointer {
+  #initialized = false;
+  #canCheckpoint = false;
+  #dockerMode: boolean;
+
+  #logger = new SimpleStructuredLogger("checkpointer");
+  #abortControllers = new Map<string, AbortController>();
+  #failedCheckpoints = new Map<string, unknown>();
+  #waitingForRetry = new Set<string>();
+
+  private registryHost: string;
+  private registryNamespace: string;
+  private registryTlsVerify: boolean;
+
+  private disableCheckpointSupport: boolean;
+
+  private simulateCheckpointFailure: boolean;
+  private simulateCheckpointFailureSeconds: number;
+  private simulatePushFailure: boolean;
+  private simulatePushFailureSeconds: number;
+
+  private chaosMonkey: ChaosMonkey;
+  private tmpCleaner?: TempFileCleaner;
+
+  constructor(private opts: CheckpointerOptions) {
+    this.#dockerMode = opts.dockerMode;
+
+    this.registryHost = opts.registryHost ?? "localhost:5000";
+    this.registryNamespace = opts.registryNamespace ?? "trigger";
+    this.registryTlsVerify = opts.registryTlsVerify ?? true;
+
+    this.disableCheckpointSupport = opts.disableCheckpointSupport ?? false;
+
+    this.simulateCheckpointFailure = opts.simulateCheckpointFailure ?? false;
+    this.simulateCheckpointFailureSeconds = opts.simulateCheckpointFailureSeconds ?? 300;
+    this.simulatePushFailure = opts.simulatePushFailure ?? false;
+    this.simulatePushFailureSeconds = opts.simulatePushFailureSeconds ?? 300;
+
+    this.chaosMonkey = opts.chaosMonkey ?? new ChaosMonkey(!!process.env.CHAOS_MONKEY_ENABLED);
+    this.tmpCleaner = this.#createTmpCleaner();
+  }
+
+  async init(): Promise<CheckpointerInitializeReturn> {
+    if (this.#initialized) {
+      return this.#getInitReturn(this.#canCheckpoint);
+    }
+
+    this.#logger.log(`${this.#dockerMode ? "Docker" : "Kubernetes"} mode`);
+
+    if (this.#dockerMode) {
+      const testCheckpoint = await testDockerCheckpoint();
+
+      if (testCheckpoint.ok) {
+        return this.#getInitReturn(true);
+      }
+
+      this.#logger.error(testCheckpoint.message, { error: testCheckpoint.error });
+      return this.#getInitReturn(false);
+    }
+
+    const canLogin = await Buildah.canLogin(this.registryHost);
+
+    if (!canLogin) {
+      this.#logger.error(`No checkpoint support: Not logged in to registry ${this.registryHost}`);
+    }
+
+    return this.#getInitReturn(canLogin);
+  }
+
+  #getInitReturn(canCheckpoint: boolean): CheckpointerInitializeReturn {
+    this.#canCheckpoint = canCheckpoint;
+
+    if (canCheckpoint) {
+      if (!this.#initialized) {
+        this.#logger.log("Full checkpoint support!");
+      }
+    }
+
+    this.#initialized = true;
+
+    const willSimulate = this.#dockerMode && (!this.#canCheckpoint || this.opts.forceSimulate);
+
+    if (willSimulate) {
+      this.#logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", {
+        forceSimulate: this.opts.forceSimulate,
+      });
+    }
+
+    return {
+      canCheckpoint,
+      willSimulate,
+    };
+  }
+
+  #getImageRef(projectRef: string, deploymentVersion: string, shortCode: string) {
+    return `${this.registryHost}/${this.registryNamespace}/${projectRef}:${deploymentVersion}.prod-${shortCode}`;
+  }
+
+  #getExportLocation(projectRef: string, deploymentVersion: string, shortCode: string) {
+    const basename = `${projectRef}-${deploymentVersion}-${shortCode}`;
+
+    if (this.#dockerMode) {
+      return basename;
+    } else {
+      return Crictl.getExportLocation(basename);
+    }
+  }
+
+  async checkpointAndPush(opts: CheckpointAndPushOptions): Promise<CheckpointData | undefined> {
+    const start = performance.now();
+    this.#logger.log(`checkpointAndPush() start`, { start, opts });
+
+    let interval: NodeJS.Timer | undefined;
+
+    if (opts.shouldHeartbeat) {
+      interval = setInterval(() => {
+        this.#logger.log("Sending heartbeat", { runId: opts.runId });
+        this.opts.heartbeat(opts.runId);
+      }, 20_000);
+    }
+
+    try {
+      const result = await this.#checkpointAndPushWithBackoff(opts);
+
+      const end = performance.now();
+      this.#logger.log(`checkpointAndPush() end`, {
+        start,
+        end,
+        diff: end - start,
+        opts,
+        success: result.success,
+      });
+
+      if (!result.success) {
+        return;
+      }
+
+      return result.checkpoint;
+    } finally {
+      if (opts.shouldHeartbeat) {
+        clearInterval(interval);
+      }
+    }
+  }
+
+  isCheckpointing(runId: string) {
+    return this.#abortControllers.has(runId) || this.#waitingForRetry.has(runId);
+  }
+
+  cancelCheckpoint(runId: string): boolean {
+    // If the last checkpoint failed, pretend we canceled it
+    // This ensures tasks don't wait for external resume messages to continue
+    if (this.#hasFailedCheckpoint(runId)) {
+      this.#clearFailedCheckpoint(runId);
+      return true;
+    }
+
+    if (this.#waitingForRetry.has(runId)) {
+      this.#waitingForRetry.delete(runId);
+      return true;
+    }
+
+    const controller = this.#abortControllers.get(runId);
+
+    if (!controller) {
+      this.#logger.debug("Nothing to cancel", { runId });
+      return false;
+    }
+
+    if (controller.signal.aborted) {
+      this.#logger.debug("Controller already aborted", { runId });
+      return false;
+    }
+
+    controller.abort("cancelCheckpoint()");
+    this.#abortControllers.delete(runId);
+
+    return true;
+  }
+
+  async #checkpointAndPushWithBackoff({
+    runId,
+    leaveRunning = true, // This mirrors kubernetes behaviour more accurately
+    projectRef,
+    deploymentVersion,
+    attemptNumber,
+  }: CheckpointAndPushOptions): Promise<CheckpointAndPushResult> {
+    this.#logger.log("Checkpointing with backoff", {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+    });
+
+    const backoff = new ExponentialBackoff()
+      .type("EqualJitter")
+      .base(3)
+      .max(3 * 3600)
+      .maxElapsed(48 * 3600);
+
+    for await (const { delay, retry } of backoff) {
+      try {
+        if (retry > 0) {
+          this.#logger.error("Retrying checkpoint", {
+            runId,
+            retry,
+            delay,
+          });
+
+          this.#waitingForRetry.add(runId);
+          await setTimeout(delay.milliseconds);
+
+          if (!this.#waitingForRetry.has(runId)) {
+            this.#logger.log("Checkpoint canceled while waiting for retry", { runId });
+            return { success: false, reason: "CANCELED" };
+          } else {
+            this.#waitingForRetry.delete(runId);
+          }
+        }
+
+        const result = await this.#checkpointAndPush({
+          runId,
+          leaveRunning,
+          projectRef,
+          deploymentVersion,
+          attemptNumber,
+        });
+
+        if (result.success) {
+          return result;
+        }
+
+        if (result.reason === "CANCELED") {
+          this.#logger.log("Checkpoint canceled, won't retry", { runId });
+          // Don't fail the checkpoint, as it was canceled
+          return result;
+        }
+
+        if (result.reason === "IN_PROGRESS") {
+          this.#logger.log("Checkpoint already in progress, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        if (result.reason === "NO_SUPPORT") {
+          this.#logger.log("No checkpoint support, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        if (result.reason === "DISABLED") {
+          this.#logger.log("Checkpoint support disabled, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        if (result.reason === "SKIP_RETRYING") {
+          this.#logger.log("Skipping retrying", { runId });
+          return result;
+        }
+
+        continue;
+      } catch (error) {
+        this.#logger.error("Checkpoint error", {
+          retry,
+          runId,
+          delay,
+          error: error instanceof Error ? error.message : error,
+        });
+      }
+    }
+
+    this.#logger.error(`Checkpoint failed after exponential backoff`, {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+    });
+    this.#failCheckpoint(runId, "ERROR");
+
+    return { success: false, reason: "ERROR" };
+  }
+
+  async #checkpointAndPush({
+    runId,
+    leaveRunning = true, // This mirrors kubernetes behaviour more accurately
+    projectRef,
+    deploymentVersion,
+    attemptNumber,
+  }: CheckpointAndPushOptions): Promise<CheckpointAndPushResult> {
+    await this.init();
+
+    const options = {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+      attemptNumber,
+    };
+
+    if (!this.#dockerMode && !this.#canCheckpoint) {
+      this.#logger.error("No checkpoint support. Simulation requires docker.");
+      return { success: false, reason: "NO_SUPPORT" };
+    }
+
+    if (this.isCheckpointing(runId)) {
+      this.#logger.error("Checkpoint procedure already in progress", { options });
+      return { success: false, reason: "IN_PROGRESS" };
+    }
+
+    // This is a new checkpoint, clear any last failure for this run
+    this.#clearFailedCheckpoint(runId);
+
+    if (this.disableCheckpointSupport) {
+      this.#logger.error("Checkpoint support disabled", { options });
+      return { success: false, reason: "DISABLED" };
+    }
+
+    const controller = new AbortController();
+    this.#abortControllers.set(runId, controller);
+
+    const onAbort = () => {
+      this.#logger.error("Checkpoint aborted", { options });
+      controller.signal.removeEventListener("abort", onAbort);
+    };
+    controller.signal.addEventListener("abort", onAbort);
+
+    const shortCode = nanoid(8);
+    const imageRef = this.#getImageRef(projectRef, deploymentVersion, shortCode);
+    const exportLocation = this.#getExportLocation(projectRef, deploymentVersion, shortCode);
+
+    const buildah = new Buildah({ id: `${runId}-${shortCode}`, abortSignal: controller.signal });
+    const crictl = new Crictl({ id: `${runId}-${shortCode}`, abortSignal: controller.signal });
+
+    const cleanup = async () => {
+      const metadata = {
+        runId,
+        exportLocation,
+        imageRef,
+      };
+
+      if (this.#dockerMode) {
+        this.#logger.debug("Skipping cleanup in docker mode", metadata);
+        return;
+      }
+
+      this.#logger.log("Cleaning up", metadata);
+
+      try {
+        await buildah.cleanup();
+        await crictl.cleanup();
+      } catch (error) {
+        this.#logger.error("Error during cleanup", { ...metadata, error });
+      }
+
+      // Ensure only the current controller is removed
+      if (this.#abortControllers.get(runId) === controller) {
+        this.#abortControllers.delete(runId);
+      }
+      controller.signal.removeEventListener("abort", onAbort);
+    };
+
+    try {
+      await this.chaosMonkey.call();
+
+      this.#logger.log("Checkpointing:", { options });
+
+      const containterName = this.#getRunContainerName(runId);
+
+      // Create checkpoint (docker)
+      if (this.#dockerMode) {
+        await this.#createDockerCheckpoint(
+          controller.signal,
+          runId,
+          exportLocation,
+          leaveRunning,
+          attemptNumber
+        );
+
+        this.#logger.log("checkpoint created:", {
+          runId,
+          location: exportLocation,
+        });
+
+        return {
+          success: true,
+          checkpoint: {
+            location: exportLocation,
+            docker: true,
+          },
+        };
+      }
+
+      // Create checkpoint (CRI)
+      if (!this.#canCheckpoint) {
+        this.#logger.error("No checkpoint support in kubernetes mode.");
+        return { success: false, reason: "SKIP_RETRYING" };
+      }
+
+      const containerId = await crictl.ps(containterName, true);
+
+      if (!containerId.stdout) {
+        this.#logger.error("could not find container id", { options, containterName });
+        return { success: false, reason: "SKIP_RETRYING" };
+      }
+
+      const start = performance.now();
+
+      if (this.simulateCheckpointFailure) {
+        if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) {
+          this.#logger.error("Simulating checkpoint failure", { options });
+          throw new Error("SIMULATE_CHECKPOINT_FAILURE");
+        }
+      }
+
+      // Create checkpoint
+      await crictl.checkpoint(containerId.stdout, exportLocation);
+      const postCheckpoint = performance.now();
+
+      // Print checkpoint size
+      const size = await getParsedFileSize(exportLocation);
+      this.#logger.log("checkpoint archive created", { size, options });
+
+      // Create image from checkpoint
+      const workingContainer = await buildah.from("scratch");
+      const postFrom = performance.now();
+
+      await buildah.add(workingContainer.stdout, exportLocation, "/");
+      const postAdd = performance.now();
+
+      await buildah.config(workingContainer.stdout, [
+        `io.kubernetes.cri-o.annotations.checkpoint.name=${shortCode}`,
+      ]);
+      const postConfig = performance.now();
+
+      await buildah.commit(workingContainer.stdout, imageRef);
+      const postCommit = performance.now();
+
+      if (this.simulatePushFailure) {
+        if (performance.now() < this.simulatePushFailureSeconds * 1000) {
+          this.#logger.error("Simulating push failure", { options });
+          throw new Error("SIMULATE_PUSH_FAILURE");
+        }
+      }
+
+      // Push checkpoint image
+      await buildah.push(imageRef, this.registryTlsVerify);
+      const postPush = performance.now();
+
+      const perf = {
+        "crictl checkpoint": postCheckpoint - start,
+        "buildah from": postFrom - postCheckpoint,
+        "buildah add": postAdd - postFrom,
+        "buildah config": postConfig - postAdd,
+        "buildah commit": postCommit - postConfig,
+        "buildah push": postPush - postCommit,
+      };
+
+      this.#logger.log("Checkpointed and pushed image to:", { location: imageRef, perf });
+
+      return {
+        success: true,
+        checkpoint: {
+          location: imageRef,
+          docker: false,
+        },
+      };
+    } catch (error) {
+      if (error instanceof Exec.Result) {
+        if (error.aborted) {
+          this.#logger.error("Checkpoint canceled: Exec", { options });
+
+          return { success: false, reason: "CANCELED" };
+        } else {
+          this.#logger.error("Checkpoint command error", { options, error });
+
+          return { success: false, reason: "ERROR" };
+        }
+      }
+
+      this.#logger.error("Unhandled checkpoint error", { options, error });
+
+      return { success: false, reason: "ERROR" };
+    } finally {
+      await cleanup();
+
+      if (controller.signal.aborted) {
+        this.#logger.error("Checkpoint canceled: Cleanup", { options });
+
+        // Overrides any prior return value
+        return { success: false, reason: "CANCELED" };
+      }
+    }
+  }
+
+  async #createDockerCheckpoint(
+    abortSignal: AbortSignal,
+    runId: string,
+    exportLocation: string,
+    leaveRunning: boolean,
+    attemptNumber?: number
+  ) {
+    const containterNameWithAttempt = this.#getRunContainerName(runId, attemptNumber);
+    const exec = new Exec({ logger: this.#logger, abortSignal });
+
+    try {
+      if (this.opts.forceSimulate || !this.#canCheckpoint) {
+        this.#logger.log("Simulating checkpoint");
+
+        await exec.x("docker", ["pause", containterNameWithAttempt]);
+
+        return;
+      }
+
+      if (this.simulateCheckpointFailure) {
+        if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) {
+          this.#logger.error("Simulating checkpoint failure", {
+            runId,
+            exportLocation,
+            leaveRunning,
+            attemptNumber,
+          });
+
+          throw new Error("SIMULATE_CHECKPOINT_FAILURE");
+        }
+      }
+
+      const args = ["checkpoint", "create"];
+
+      if (leaveRunning) {
+        args.push("--leave-running");
+      }
+
+      args.push(containterNameWithAttempt, exportLocation);
+
+      await exec.x("docker", args);
+    } catch (error) {
+      this.#logger.error("Failed while creating docker checkpoint", { exportLocation });
+      throw error;
+    }
+  }
+
+  #failCheckpoint(runId: string, error: unknown) {
+    this.#failedCheckpoints.set(runId, error);
+  }
+
+  #clearFailedCheckpoint(runId: string) {
+    this.#failedCheckpoints.delete(runId);
+  }
+
+  #hasFailedCheckpoint(runId: string) {
+    return this.#failedCheckpoints.has(runId);
+  }
+
+  #getRunContainerName(suffix: string, attemptNumber?: number) {
+    return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
+  }
+
+  #createTmpCleaner() {
+    if (!boolFromEnv("TMP_CLEANER_ENABLED", false)) {
+      return;
+    }
+
+    const defaultPaths = [Buildah.tmpDir, Crictl.checkpointDir].filter(Boolean);
+    const pathsOverride = process.env.TMP_CLEANER_PATHS_OVERRIDE?.split(",").filter(Boolean) ?? [];
+    const paths = pathsOverride.length ? pathsOverride : defaultPaths;
+
+    if (paths.length === 0) {
+      this.#logger.error("TempFileCleaner enabled but no paths to clean", {
+        defaultPaths,
+        pathsOverride,
+        TMP_CLEANER_PATHS_OVERRIDE: process.env.TMP_CLEANER_PATHS_OVERRIDE,
+      });
+
+      return;
+    }
+    const cleaner = new TempFileCleaner({
+      paths,
+      maxAgeMinutes: numFromEnv("TMP_CLEANER_MAX_AGE_MINUTES", 60),
+      intervalSeconds: numFromEnv("TMP_CLEANER_INTERVAL_SECONDS", 300),
+      leadingEdge: boolFromEnv("TMP_CLEANER_LEADING_EDGE", false),
+    });
+
+    cleaner.start();
+
+    return cleaner;
+  }
+}

apps/coordinator/src/cleaner.ts

@@ -0,0 +1,106 @@
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import { Exec } from "./exec";
+import { setTimeout } from "timers/promises";
+
+interface TempFileCleanerOptions {
+  paths: string[];
+  maxAgeMinutes: number;
+  intervalSeconds: number;
+  leadingEdge?: boolean;
+}
+
+export class TempFileCleaner {
+  private enabled = false;
+
+  private logger: SimpleStructuredLogger;
+  private exec: Exec;
+
+  constructor(private opts: TempFileCleanerOptions) {
+    this.logger = new SimpleStructuredLogger("tmp-cleaner", undefined, { ...this.opts });
+    this.exec = new Exec({ logger: this.logger });
+  }
+
+  async start() {
+    this.logger.log("TempFileCleaner.start");
+    this.enabled = true;
+
+    if (!this.opts.leadingEdge) {
+      await this.wait();
+    }
+
+    while (this.enabled) {
+      try {
+        await this.clean();
+      } catch (error) {
+        this.logger.error("error during tick", { error });
+      }
+
+      await this.wait();
+    }
+  }
+
+  stop() {
+    this.logger.log("TempFileCleaner.stop");
+    this.enabled = false;
+  }
+
+  private wait() {
+    return setTimeout(this.opts.intervalSeconds * 1000);
+  }
+
+  private async clean() {
+    for (const path of this.opts.paths) {
+      try {
+        await this.cleanSingle(path);
+      } catch (error) {
+        this.logger.error("error while cleaning", { path, error });
+      }
+    }
+  }
+
+  private async cleanSingle(startingPoint: string) {
+    const maxAgeMinutes = this.opts.maxAgeMinutes;
+
+    const ignoreStartingPoint = ["!", "-path", startingPoint];
+    const onlyDirectDescendants = ["-maxdepth", "1"];
+    const onlyOldFiles = ["-mmin", `+${maxAgeMinutes}`];
+
+    const baseArgs = [
+      startingPoint,
+      ...ignoreStartingPoint,
+      ...onlyDirectDescendants,
+      ...onlyOldFiles,
+    ];
+
+    const duArgs = ["-exec", "du", "-ch", "{}", "+"];
+    const rmArgs = ["-exec", "rm", "-rf", "{}", "+"];
+
+    const du = this.x("find", [...baseArgs, ...duArgs]);
+    const duOutput = await du;
+
+    const duLines = duOutput.stdout.trim().split("\n");
+    const fileCount = duLines.length - 1; // last line is the total
+    const fileSize = duLines.at(-1)?.trim().split(/\s+/)[0];
+
+    if (fileCount === 0) {
+      this.logger.log("nothing to delete", { startingPoint, maxAgeMinutes });
+      return;
+    }
+
+    this.logger.log("deleting old files", { fileCount, fileSize, startingPoint, maxAgeMinutes });
+
+    const rm = this.x("find", [...baseArgs, ...rmArgs]);
+    const rmOutput = await rm;
+
+    if (rmOutput.stderr.length > 0) {
+      this.logger.error("delete unsuccessful", { rmOutput });
+      return;
+    }
+
+    this.logger.log("deleted old files", { fileCount, fileSize, startingPoint, maxAgeMinutes });
+  }
+
+  private get x() {
+    return this.exec.x.bind(this.exec);
+  }
+}

apps/coordinator/src/exec.ts

@@ -0,0 +1,293 @@
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import { randomUUID } from "crypto";
+import { homedir } from "os";
+import { type Result, x } from "tinyexec";
+
+class TinyResult {
+  pid?: number;
+  exitCode?: number;
+  aborted: boolean;
+  killed: boolean;
+
+  constructor(result: Result) {
+    this.pid = result.pid;
+    this.exitCode = result.exitCode;
+    this.aborted = result.aborted;
+    this.killed = result.killed;
+  }
+}
+
+interface ExecOptions {
+  logger?: SimpleStructuredLogger;
+  abortSignal?: AbortSignal;
+  logOutput?: boolean;
+  trimArgs?: boolean;
+  neverThrow?: boolean;
+}
+
+export class Exec {
+  private logger: SimpleStructuredLogger;
+  private abortSignal: AbortSignal | undefined;
+
+  private logOutput: boolean;
+  private trimArgs: boolean;
+  private neverThrow: boolean;
+
+  constructor(opts: ExecOptions) {
+    this.logger = opts.logger ?? new SimpleStructuredLogger("exec");
+    this.abortSignal = opts.abortSignal;
+
+    this.logOutput = opts.logOutput ?? true;
+    this.trimArgs = opts.trimArgs ?? true;
+    this.neverThrow = opts.neverThrow ?? false;
+  }
+
+  async x(
+    command: string,
+    args?: string[],
+    opts?: { neverThrow?: boolean; ignoreAbort?: boolean }
+  ) {
+    const argsTrimmed = this.trimArgs ? args?.map((arg) => arg.trim()) : args;
+
+    const commandWithFirstArg = `${command}${argsTrimmed?.length ? ` ${argsTrimmed[0]}` : ""}`;
+    this.logger.debug(`exec: ${commandWithFirstArg}`, { command, args, argsTrimmed });
+
+    const result = x(command, argsTrimmed, {
+      signal: opts?.ignoreAbort ? undefined : this.abortSignal,
+      // We don't use this as it doesn't cover killed and aborted processes
+      // throwOnError: true,
+    });
+
+    const output = await result;
+
+    const metadata = {
+      command,
+      argsRaw: args,
+      argsTrimmed,
+      globalOpts: {
+        trimArgs: this.trimArgs,
+        neverThrow: this.neverThrow,
+        hasAbortSignal: !!this.abortSignal,
+      },
+      localOpts: opts,
+      stdout: output.stdout,
+      stderr: output.stderr,
+      pid: result.pid,
+      exitCode: result.exitCode,
+      aborted: result.aborted,
+      killed: result.killed,
+    };
+
+    if (this.logOutput) {
+      this.logger.debug(`output: ${commandWithFirstArg}`, metadata);
+    }
+
+    if (this.neverThrow || opts?.neverThrow) {
+      return output;
+    }
+
+    if (result.aborted) {
+      this.logger.error(`aborted: ${commandWithFirstArg}`, metadata);
+      throw new TinyResult(result);
+    }
+
+    if (result.killed) {
+      this.logger.error(`killed: ${commandWithFirstArg}`, metadata);
+      throw new TinyResult(result);
+    }
+
+    if (result.exitCode !== 0) {
+      this.logger.error(`non-zero exit: ${commandWithFirstArg}`, metadata);
+      throw new TinyResult(result);
+    }
+
+    return output;
+  }
+
+  static Result = TinyResult;
+}
+
+interface BuildahOptions {
+  id?: string;
+  abortSignal?: AbortSignal;
+}
+
+export class Buildah {
+  private id: string;
+  private logger: SimpleStructuredLogger;
+  private exec: Exec;
+
+  private containers = new Set<string>();
+  private images = new Set<string>();
+
+  constructor(opts: BuildahOptions) {
+    this.id = opts.id ?? randomUUID();
+    this.logger = new SimpleStructuredLogger("buildah", undefined, { id: this.id });
+
+    this.exec = new Exec({
+      logger: this.logger,
+      abortSignal: opts.abortSignal,
+    });
+
+    this.logger.log("initiaized", { opts });
+  }
+
+  private get x() {
+    return this.exec.x.bind(this.exec);
+  }
+
+  async from(baseImage: string) {
+    const output = await this.x("buildah", ["from", baseImage]);
+    this.containers.add(output.stdout);
+    return output;
+  }
+
+  async add(container: string, src: string, dest: string) {
+    return await this.x("buildah", ["add", container, src, dest]);
+  }
+
+  async config(container: string, annotations: string[]) {
+    const args = ["config"];
+
+    for (const annotation of annotations) {
+      args.push(`--annotation=${annotation}`);
+    }
+
+    args.push(container);
+
+    return await this.x("buildah", args);
+  }
+
+  async commit(container: string, imageRef: string) {
+    const output = await this.x("buildah", ["commit", container, imageRef]);
+    this.images.add(output.stdout);
+    return output;
+  }
+
+  async push(imageRef: string, registryTlsVerify?: boolean) {
+    return await this.x("buildah", [
+      "push",
+      `--tls-verify=${String(!!registryTlsVerify)}`,
+      imageRef,
+    ]);
+  }
+
+  async cleanup() {
+    if (this.containers.size > 0) {
+      try {
+        const output = await this.x("buildah", ["rm", ...this.containers], { ignoreAbort: true });
+        this.containers.clear();
+
+        if (output.stderr.length > 0) {
+          this.logger.error("failed to remove some containers", { output });
+        }
+      } catch (error) {
+        this.logger.error("failed to clean up containers", { error, containers: this.containers });
+      }
+    } else {
+      this.logger.debug("no containers to clean up");
+    }
+
+    if (this.images.size > 0) {
+      try {
+        const output = await this.x("buildah", ["rmi", ...this.images], { ignoreAbort: true });
+        this.images.clear();
+
+        if (output.stderr.length > 0) {
+          this.logger.error("failed to remove some images", { output });
+        }
+      } catch (error) {
+        this.logger.error("failed to clean up images", { error, images: this.images });
+      }
+    } else {
+      this.logger.debug("no images to clean up");
+    }
+  }
+
+  static async canLogin(registryHost: string) {
+    try {
+      await x("buildah", ["login", "--get-login", registryHost], { throwOnError: true });
+      return true;
+    } catch (error) {
+      return false;
+    }
+  }
+
+  static get tmpDir() {
+    return process.env.TMPDIR ?? "/var/tmp";
+  }
+
+  static get storageRootDir() {
+    return process.getuid?.() === 0
+      ? "/var/lib/containers/storage"
+      : `${homedir()}/.local/share/containers/storage`;
+  }
+}
+
+interface CrictlOptions {
+  id?: string;
+  abortSignal?: AbortSignal;
+}
+
+export class Crictl {
+  private id: string;
+  private logger: SimpleStructuredLogger;
+  private exec: Exec;
+
+  private archives = new Set<string>();
+
+  constructor(opts: CrictlOptions) {
+    this.id = opts.id ?? randomUUID();
+    this.logger = new SimpleStructuredLogger("crictl", undefined, { id: this.id });
+
+    this.exec = new Exec({
+      logger: this.logger,
+      abortSignal: opts.abortSignal,
+    });
+
+    this.logger.log("initiaized", { opts });
+  }
+
+  private get x() {
+    return this.exec.x.bind(this.exec);
+  }
+
+  async ps(containerName: string, quiet?: boolean) {
+    return await this.x("crictl", ["ps", "--name", containerName, quiet ? "--quiet" : ""]);
+  }
+
+  async checkpoint(containerId: string, exportLocation: string) {
+    const output = await this.x("crictl", [
+      "checkpoint",
+      `--export=${exportLocation}`,
+      containerId,
+    ]);
+    this.archives.add(exportLocation);
+    return output;
+  }
+
+  async cleanup() {
+    if (this.archives.size > 0) {
+      try {
+        const output = await this.x("rm", ["-v", ...this.archives], { ignoreAbort: true });
+        this.archives.clear();
+
+        if (output.stderr.length > 0) {
+          this.logger.error("failed to remove some archives", { output });
+        }
+      } catch (error) {
+        this.logger.error("failed to clean up archives", { error, archives: this.archives });
+      }
+    } else {
+      this.logger.debug("no archives to clean up");
+    }
+  }
+
+  static getExportLocation(identifier: string) {
+    return `${this.checkpointDir}/${identifier}.tar`;
+  }
+
+  static get checkpointDir() {
+    return process.env.CRI_CHECKPOINT_DIR ?? "/checkpoints";
+  }
+}

apps/coordinator/src/util.ts

@@ -0,0 +1,19 @@
+export const boolFromEnv = (env: string, defaultValue: boolean): boolean => {
+  const value = process.env[env];
+
+  if (!value) {
+    return defaultValue;
+  }
+
+  return ["1", "true"].includes(value);
+};
+
+export const numFromEnv = (env: string, defaultValue: number): number => {
+  const value = process.env[env];
+
+  if (!value) {
+    return defaultValue;
+  }
+
+  return parseInt(value, 10);
+};

apps/coordinator/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "include": ["./src/**/*.ts"],
+  "exclude": ["node_modules"],
+  "compilerOptions": {
+    "target": "es2016",
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "resolveJsonModule": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "paths": {
+      "@trigger.dev/core/v3": ["../../packages/core/src/v3"],
+      "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"]
+    }
+  }
+}

apps/docker-provider/.env.example

@@ -0,0 +1,11 @@
+HTTP_SERVER_PORT=8050
+
+PLATFORM_WS_PORT=3030
+PLATFORM_SECRET=provider-secret
+SECURE_CONNECTION=false
+
+OTEL_EXPORTER_OTLP_ENDPOINT=http://0.0.0.0:3030/otel
+
+# Use this if you are on macOS
+# COORDINATOR_HOST="host.docker.internal"
+# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318"

apps/docker-provider/.gitignore

@@ -0,0 +1,3 @@
+dist/
+node_modules/
+.env

apps/docker-provider/Containerfile

@@ -0,0 +1,47 @@
+FROM node:20-alpine@sha256:7a91aa397f2e2dfbfcdad2e2d72599f374e0b0172be1d86eeb73f1d33f36a4b2 AS node-20-alpine
+
+WORKDIR /app
+
+FROM node-20-alpine AS pruner
+
+COPY --chown=node:node . .
+RUN npx -q turbo@1.10.9 prune --scope=docker-provider --docker
+RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
+
+FROM node-20-alpine AS base
+
+RUN apk add --no-cache dumb-init docker
+
+COPY --chown=node:node .gitignore .gitignore
+COPY --from=pruner --chown=node:node /app/out/json/ .
+COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
+COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
+
+FROM base AS dev-deps
+RUN corepack enable
+ENV NODE_ENV development
+
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
+
+FROM base AS builder
+RUN corepack enable
+
+COPY --from=pruner --chown=node:node /app/out/full/ .
+COPY --from=dev-deps --chown=node:node /app/ .
+COPY --chown=node:node turbo.json turbo.json
+
+RUN pnpm run -r --filter docker-provider build:bundle
+
+FROM base AS runner
+
+RUN corepack enable
+ENV NODE_ENV production
+
+COPY --from=builder --chown=node:node /app/apps/docker-provider/dist/index.mjs ./index.mjs
+
+EXPOSE 8000
+
+USER node
+
+CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]

apps/docker-provider/README.md

@@ -0,0 +1,3 @@
+# Docker provider
+
+The `docker-provider` allows the platform to be orchestrator-agnostic. The platform can perform actions such as `INDEX_TASKS` or `INVOKE_TASK` which the provider translates into Docker actions.

apps/docker-provider/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "docker-provider",
+  "private": true,
+  "version": "0.0.1",
+  "description": "",
+  "main": "dist/index.cjs",
+  "scripts": {
+    "build": "npm run build:bundle",
+    "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"",
+    "build:image": "docker build -f Containerfile . -t docker-provider",
+    "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
+    "start": "tsx src/index.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@trigger.dev/core": "workspace:*",
+    "execa": "^8.0.1"
+  },
+  "devDependencies": {
+    "@types/node": "^18.19.8",
+    "dotenv": "^16.4.2",
+    "esbuild": "^0.19.11",
+    "tsx": "^4.7.0",
+    "typescript": "^5.3.3"
+  }
+}

apps/docker-provider/src/index.ts

@@ -0,0 +1,291 @@
+import { $, type ExecaChildProcess, execa } from "execa";
+import {
+  ProviderShell,
+  TaskOperations,
+  TaskOperationsCreateOptions,
+  TaskOperationsIndexOptions,
+  TaskOperationsRestoreOptions,
+} from "@trigger.dev/core/v3/apps";
+import { SimpleLogger } from "@trigger.dev/core/v3/apps";
+import { isExecaChildProcess, testDockerCheckpoint } from "@trigger.dev/core/v3/apps";
+import { setTimeout } from "node:timers/promises";
+import { PostStartCauses, PreStopCauses } from "@trigger.dev/core/v3";
+
+const MACHINE_NAME = process.env.MACHINE_NAME || "local";
+const COORDINATOR_PORT = process.env.COORDINATOR_PORT || 8020;
+const COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
+const DOCKER_NETWORK = process.env.DOCKER_NETWORK || "host";
+
+const OTEL_EXPORTER_OTLP_ENDPOINT =
+  process.env.OTEL_EXPORTER_OTLP_ENDPOINT || "http://0.0.0.0:4318";
+
+const FORCE_CHECKPOINT_SIMULATION = ["1", "true"].includes(
+  process.env.FORCE_CHECKPOINT_SIMULATION ?? "false"
+);
+
+const logger = new SimpleLogger(`[${MACHINE_NAME}]`);
+
+type TaskOperationsInitReturn = {
+  canCheckpoint: boolean;
+  willSimulate: boolean;
+};
+
+class DockerTaskOperations implements TaskOperations {
+  #initialized = false;
+  #canCheckpoint = false;
+
+  constructor(private opts = { forceSimulate: false }) {}
+
+  async init(): Promise<TaskOperationsInitReturn> {
+    if (this.#initialized) {
+      return this.#getInitReturn(this.#canCheckpoint);
+    }
+
+    logger.log("Initializing task operations");
+
+    const testCheckpoint = await testDockerCheckpoint();
+
+    if (testCheckpoint.ok) {
+      return this.#getInitReturn(true);
+    }
+
+    logger.error(testCheckpoint.message, testCheckpoint.error);
+    return this.#getInitReturn(false);
+  }
+
+  #getInitReturn(canCheckpoint: boolean): TaskOperationsInitReturn {
+    this.#canCheckpoint = canCheckpoint;
+
+    if (canCheckpoint) {
+      if (!this.#initialized) {
+        logger.log("Full checkpoint support!");
+      }
+    }
+
+    this.#initialized = true;
+
+    const willSimulate = !canCheckpoint || this.opts.forceSimulate;
+
+    if (willSimulate) {
+      logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", {
+        forceSimulate: this.opts.forceSimulate,
+      });
+    }
+
+    return {
+      canCheckpoint,
+      willSimulate,
+    };
+  }
+
+  async index(opts: TaskOperationsIndexOptions) {
+    await this.init();
+
+    const containerName = this.#getIndexContainerName(opts.shortCode);
+
+    logger.log(`Indexing task ${opts.imageRef}`, {
+      host: COORDINATOR_HOST,
+      port: COORDINATOR_PORT,
+    });
+
+    logger.debug(
+      await execa("docker", [
+        "run",
+        `--network=${DOCKER_NETWORK}`,
+        "--rm",
+        `--env=INDEX_TASKS=true`,
+        `--env=TRIGGER_SECRET_KEY=${opts.apiKey}`,
+        `--env=TRIGGER_API_URL=${opts.apiUrl}`,
+        `--env=TRIGGER_ENV_ID=${opts.envId}`,
+        `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
+        `--env=POD_NAME=${containerName}`,
+        `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
+        `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
+        `--name=${containerName}`,
+        `${opts.imageRef}`,
+      ])
+    );
+  }
+
+  async create(opts: TaskOperationsCreateOptions) {
+    await this.init();
+
+    const containerName = this.#getRunContainerName(opts.runId, opts.nextAttemptNumber);
+
+    const runArgs = [
+      "run",
+      `--network=${DOCKER_NETWORK}`,
+      "--detach",
+      `--env=TRIGGER_ENV_ID=${opts.envId}`,
+      `--env=TRIGGER_RUN_ID=${opts.runId}`,
+      `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
+      `--env=POD_NAME=${containerName}`,
+      `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
+      `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
+      `--name=${containerName}`,
+    ];
+
+    if (process.env.ENFORCE_MACHINE_PRESETS) {
+      runArgs.push(`--cpus=${opts.machine.cpu}`, `--memory=${opts.machine.memory}G`);
+    }
+
+    runArgs.push(`${opts.image}`);
+
+    try {
+      logger.debug(await execa("docker", runArgs));
+    } catch (error) {
+      if (!isExecaChildProcess(error)) {
+        throw error;
+      }
+
+      logger.error("Create failed:", {
+        opts,
+        exitCode: error.exitCode,
+        escapedCommand: error.escapedCommand,
+        stdout: error.stdout,
+        stderr: error.stderr,
+      });
+    }
+  }
+
+  async restore(opts: TaskOperationsRestoreOptions) {
+    await this.init();
+
+    const containerName = this.#getRunContainerName(opts.runId, opts.attemptNumber);
+
+    if (!this.#canCheckpoint || this.opts.forceSimulate) {
+      logger.log("Simulating restore");
+
+      const unpause = logger.debug(await $`docker unpause ${containerName}`);
+
+      if (unpause.exitCode !== 0) {
+        throw new Error("docker unpause command failed");
+      }
+
+      await this.#sendPostStart(containerName);
+      return;
+    }
+
+    const { exitCode } = logger.debug(
+      await $`docker start --checkpoint=${opts.checkpointRef} ${containerName}`
+    );
+
+    if (exitCode !== 0) {
+      throw new Error("docker start command failed");
+    }
+
+    await this.#sendPostStart(containerName);
+  }
+
+  async delete(opts: { runId: string }) {
+    await this.init();
+
+    const containerName = this.#getRunContainerName(opts.runId);
+    await this.#sendPreStop(containerName);
+
+    logger.log("noop: delete");
+  }
+
+  async get(opts: { runId: string }) {
+    await this.init();
+
+    logger.log("noop: get");
+  }
+
+  #getIndexContainerName(suffix: string) {
+    return `task-index-${suffix}`;
+  }
+
+  #getRunContainerName(suffix: string, attemptNumber?: number) {
+    return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
+  }
+
+  async #sendPostStart(containerName: string): Promise<void> {
+    try {
+      const port = await this.#getHttpServerPort(containerName);
+      logger.debug(await this.#runLifecycleCommand(containerName, port, "postStart", "restore"));
+    } catch (error) {
+      logger.error("postStart error", { error });
+      throw new Error("postStart command failed");
+    }
+  }
+
+  async #sendPreStop(containerName: string): Promise<void> {
+    try {
+      const port = await this.#getHttpServerPort(containerName);
+      logger.debug(await this.#runLifecycleCommand(containerName, port, "preStop", "terminate"));
+    } catch (error) {
+      logger.error("preStop error", { error });
+      throw new Error("preStop command failed");
+    }
+  }
+
+  async #getHttpServerPort(containerName: string): Promise<number> {
+    // We first get the correct port, which is random during dev as we run with host networking and need to avoid clashes
+    // FIXME: Skip this in prod
+    const logs = logger.debug(await $`docker logs ${containerName}`);
+    const matches = logs.stdout.match(/http server listening on port (?<port>[0-9]+)/);
+
+    const port = Number(matches?.groups?.port);
+
+    if (!port) {
+      throw new Error("failed to extract port from logs");
+    }
+
+    return port;
+  }
+
+  async #runLifecycleCommand<THookType extends "postStart" | "preStop">(
+    containerName: string,
+    port: number,
+    type: THookType,
+    cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses,
+    retryCount = 0
+  ): Promise<ExecaChildProcess> {
+    try {
+      return await execa("docker", [
+        "exec",
+        containerName,
+        "busybox",
+        "wget",
+        "-q",
+        "-O-",
+        `127.0.0.1:${port}/${type}?cause=${cause}`,
+      ]);
+    } catch (error: any) {
+      if (type === "postStart" && retryCount < 6) {
+        logger.debug(`retriable ${type} error`, { retryCount, message: error?.message });
+        await setTimeout(exponentialBackoff(retryCount + 1, 2, 50, 1150, 50));
+
+        return this.#runLifecycleCommand(containerName, port, type, cause, retryCount + 1);
+      }
+
+      logger.error(`final ${type} error`, { message: error?.message });
+      throw new Error(`${type} command failed after ${retryCount - 1} retries`);
+    }
+  }
+}
+
+const provider = new ProviderShell({
+  tasks: new DockerTaskOperations({ forceSimulate: FORCE_CHECKPOINT_SIMULATION }),
+  type: "docker",
+});
+
+provider.listen();
+
+function exponentialBackoff(
+  retryCount: number,
+  exponential: number,
+  minDelay: number,
+  maxDelay: number,
+  jitter: number
+): number {
+  // Calculate the delay using the exponential backoff formula
+  const delay = Math.min(Math.pow(exponential, retryCount) * minDelay, maxDelay);
+
+  // Calculate the jitter
+  const jitterValue = Math.random() * jitter;
+
+  // Return the calculated delay with jitter
+  return delay + jitterValue;
+}

apps/docker-provider/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "es2018",
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "paths": {
+      "@trigger.dev/core/v3": ["../../packages/core/src/v3"],
+      "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"]
+    }
+  }
+}

apps/kubernetes-provider/.env.example

@@ -0,0 +1,9 @@
+HTTP_SERVER_PORT=8060
+
+PLATFORM_WS_PORT=3030
+PLATFORM_SECRET=provider-secret
+SECURE_CONNECTION=false
+
+# Use this if you are on macOS
+# COORDINATOR_HOST="host.docker.internal"
+# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318"

apps/kubernetes-provider/.gitignore

@@ -0,0 +1,3 @@
+dist/
+node_modules/
+.env

apps/kubernetes-provider/Containerfile

@@ -0,0 +1,47 @@
+FROM node:20-alpine@sha256:7a91aa397f2e2dfbfcdad2e2d72599f374e0b0172be1d86eeb73f1d33f36a4b2 AS node-20-alpine
+
+WORKDIR /app
+
+FROM node-20-alpine AS pruner
+
+COPY --chown=node:node . .
+RUN npx -q turbo@1.10.9 prune --scope=kubernetes-provider --docker
+RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
+
+FROM node-20-alpine AS base
+
+RUN apk add --no-cache dumb-init
+
+COPY --chown=node:node .gitignore .gitignore
+COPY --from=pruner --chown=node:node /app/out/json/ .
+COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
+COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
+
+FROM base AS dev-deps
+RUN corepack enable
+ENV NODE_ENV development
+
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
+
+FROM base AS builder
+RUN corepack enable
+
+COPY --from=pruner --chown=node:node /app/out/full/ .
+COPY --from=dev-deps --chown=node:node /app/ .
+COPY --chown=node:node turbo.json turbo.json
+
+RUN pnpm run -r --filter kubernetes-provider build:bundle
+
+FROM base AS runner
+
+RUN corepack enable
+ENV NODE_ENV production
+
+COPY --from=builder --chown=node:node /app/apps/kubernetes-provider/dist/index.mjs ./index.mjs
+
+EXPOSE 8000
+
+USER node
+
+CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]

apps/kubernetes-provider/README.md

@@ -0,0 +1,3 @@
+# Kubernetes provider
+
+The `kubernetes-provider` allows the platform to be orchestrator-agnostic. The platform can perform actions such as `INDEX_TASKS` or `INVOKE_TASK` which the provider translates into Kubernetes actions.

apps/kubernetes-provider/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "kubernetes-provider",
+  "private": true,
+  "version": "0.0.1",
+  "description": "",
+  "main": "dist/index.cjs",
+  "scripts": {
+    "build": "npm run build:bundle",
+    "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"",
+    "build:image": "docker build -f Containerfile . -t kubernetes-provider",
+    "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
+    "start": "tsx src/index.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@kubernetes/client-node": "^0.20.0",
+    "@trigger.dev/core": "workspace:*",
+    "p-queue": "^8.0.1"
+  },
+  "devDependencies": {
+    "dotenv": "^16.4.2",
+    "esbuild": "^0.19.11",
+    "tsx": "^4.7.0",
+    "typescript": "^5.3.3"
+  }
+}

apps/kubernetes-provider/src/index.ts

@@ -0,0 +1,717 @@
+import * as k8s from "@kubernetes/client-node";
+import {
+  EnvironmentType,
+  MachinePreset,
+  PostStartCauses,
+  PreStopCauses,
+} from "@trigger.dev/core/v3";
+import {
+  ProviderShell,
+  SimpleLogger,
+  TaskOperations,
+  TaskOperationsCreateOptions,
+  TaskOperationsIndexOptions,
+  TaskOperationsPrePullDeploymentOptions,
+  TaskOperationsRestoreOptions,
+} from "@trigger.dev/core/v3/apps";
+import { PodCleaner } from "./podCleaner";
+import { TaskMonitor } from "./taskMonitor";
+import { UptimeHeartbeat } from "./uptimeHeartbeat";
+
+const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
+const NODE_NAME = process.env.NODE_NAME || "local";
+const OTEL_EXPORTER_OTLP_ENDPOINT =
+  process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318";
+const COORDINATOR_HOST = process.env.COORDINATOR_HOST ?? undefined;
+const COORDINATOR_PORT = process.env.COORDINATOR_PORT ?? undefined;
+const KUBERNETES_NAMESPACE = process.env.KUBERNETES_NAMESPACE ?? "default";
+
+const POD_CLEANER_INTERVAL_SECONDS = Number(process.env.POD_CLEANER_INTERVAL_SECONDS || "300");
+
+const UPTIME_HEARTBEAT_URL = process.env.UPTIME_HEARTBEAT_URL;
+const UPTIME_INTERVAL_SECONDS = Number(process.env.UPTIME_INTERVAL_SECONDS || "60");
+const UPTIME_MAX_PENDING_RUNS = Number(process.env.UPTIME_MAX_PENDING_RUNS || "25");
+const UPTIME_MAX_PENDING_INDECES = Number(process.env.UPTIME_MAX_PENDING_INDECES || "10");
+const UPTIME_MAX_PENDING_ERRORS = Number(process.env.UPTIME_MAX_PENDING_ERRORS || "10");
+
+const POD_EPHEMERAL_STORAGE_SIZE_LIMIT = process.env.POD_EPHEMERAL_STORAGE_SIZE_LIMIT || "10Gi";
+const POD_EPHEMERAL_STORAGE_SIZE_REQUEST = process.env.POD_EPHEMERAL_STORAGE_SIZE_REQUEST || "2Gi";
+
+const logger = new SimpleLogger(`[${NODE_NAME}]`);
+logger.log(`running in ${RUNTIME_ENV} mode`);
+
+type Namespace = {
+  metadata: {
+    name: string;
+  };
+};
+
+type ResourceQuantities = {
+  [K in "cpu" | "memory" | "ephemeral-storage"]?: string;
+};
+
+class KubernetesTaskOperations implements TaskOperations {
+  #namespace: Namespace = {
+    metadata: {
+      name: "default",
+    },
+  };
+
+  #k8sApi: {
+    core: k8s.CoreV1Api;
+    batch: k8s.BatchV1Api;
+    apps: k8s.AppsV1Api;
+  };
+
+  constructor(opts: { namespace?: string } = {}) {
+    if (opts.namespace) {
+      this.#namespace.metadata.name = opts.namespace;
+    }
+
+    this.#k8sApi = this.#createK8sApi();
+  }
+
+  async init() {
+    // noop
+  }
+
+  async index(opts: TaskOperationsIndexOptions) {
+    await this.#createJob(
+      {
+        metadata: {
+          name: this.#getIndexContainerName(opts.shortCode),
+          namespace: this.#namespace.metadata.name,
+        },
+        spec: {
+          completions: 1,
+          backoffLimit: 0,
+          ttlSecondsAfterFinished: 300,
+          template: {
+            metadata: {
+              labels: {
+                ...this.#getSharedLabels(opts),
+                app: "task-index",
+                "app.kubernetes.io/part-of": "trigger-worker",
+                "app.kubernetes.io/component": "index",
+                deployment: opts.deploymentId,
+              },
+            },
+            spec: {
+              ...this.#defaultPodSpec,
+              containers: [
+                {
+                  name: this.#getIndexContainerName(opts.shortCode),
+                  image: opts.imageRef,
+                  ports: [
+                    {
+                      containerPort: 8000,
+                    },
+                  ],
+                  resources: {
+                    limits: {
+                      cpu: "1",
+                      memory: "2G",
+                      "ephemeral-storage": "2Gi",
+                    },
+                  },
+                  lifecycle: {
+                    preStop: {
+                      exec: {
+                        command: this.#getLifecycleCommand("preStop", "terminate"),
+                      },
+                    },
+                  },
+                  env: [
+                    ...this.#getSharedEnv(opts.envId),
+                    {
+                      name: "INDEX_TASKS",
+                      value: "true",
+                    },
+                    {
+                      name: "TRIGGER_SECRET_KEY",
+                      value: opts.apiKey,
+                    },
+                    {
+                      name: "TRIGGER_API_URL",
+                      value: opts.apiUrl,
+                    },
+                  ],
+                },
+              ],
+            },
+          },
+        },
+      },
+      this.#namespace
+    );
+  }
+
+  async create(opts: TaskOperationsCreateOptions) {
+    const containerName = this.#getRunContainerName(opts.runId, opts.nextAttemptNumber);
+
+    await this.#createPod(
+      {
+        metadata: {
+          name: containerName,
+          namespace: this.#namespace.metadata.name,
+          labels: {
+            ...this.#getSharedLabels(opts),
+            app: "task-run",
+            "app.kubernetes.io/part-of": "trigger-worker",
+            "app.kubernetes.io/component": "create",
+            run: opts.runId,
+          },
+        },
+        spec: {
+          ...this.#defaultPodSpec,
+          terminationGracePeriodSeconds: 60 * 60,
+          containers: [
+            {
+              name: containerName,
+              image: opts.image,
+              ports: [
+                {
+                  containerPort: 8000,
+                },
+              ],
+              resources: this.#getResourcesForMachine(opts.machine),
+              lifecycle: {
+                preStop: {
+                  exec: {
+                    command: this.#getLifecycleCommand("preStop", "terminate"),
+                  },
+                },
+              },
+              env: [
+                ...this.#getSharedEnv(opts.envId),
+                {
+                  name: "TRIGGER_RUN_ID",
+                  value: opts.runId,
+                },
+              ],
+              volumeMounts: [
+                {
+                  name: "taskinfo",
+                  mountPath: "/etc/taskinfo",
+                },
+              ],
+            },
+          ],
+          volumes: [
+            {
+              name: "taskinfo",
+              emptyDir: {},
+            },
+          ],
+        },
+      },
+      this.#namespace
+    );
+  }
+
+  async restore(opts: TaskOperationsRestoreOptions) {
+    await this.#createPod(
+      {
+        metadata: {
+          name: `${this.#getRunContainerName(opts.runId)}-${opts.checkpointId.slice(-8)}`,
+          namespace: this.#namespace.metadata.name,
+          labels: {
+            ...this.#getSharedLabels(opts),
+            app: "task-run",
+            "app.kubernetes.io/part-of": "trigger-worker",
+            "app.kubernetes.io/component": "restore",
+            run: opts.runId,
+            checkpoint: opts.checkpointId,
+          },
+        },
+        spec: {
+          ...this.#defaultPodSpec,
+          initContainers: [
+            {
+              name: "pull-base-image",
+              image: opts.imageRef,
+              command: ["sleep", "0"],
+            },
+            {
+              name: "populate-taskinfo",
+              image: "registry.digitalocean.com/trigger/busybox",
+              imagePullPolicy: "IfNotPresent",
+              command: ["/bin/sh", "-c"],
+              args: ["printenv COORDINATOR_HOST | tee /etc/taskinfo/coordinator-host"],
+              env: this.#coordinatorEnvVars,
+              volumeMounts: [
+                {
+                  name: "taskinfo",
+                  mountPath: "/etc/taskinfo",
+                },
+              ],
+            },
+          ],
+          containers: [
+            {
+              name: this.#getRunContainerName(opts.runId),
+              image: opts.checkpointRef,
+              ports: [
+                {
+                  containerPort: 8000,
+                },
+              ],
+              resources: this.#getResourcesForMachine(opts.machine),
+              lifecycle: {
+                postStart: {
+                  exec: {
+                    command: this.#getLifecycleCommand("postStart", "restore"),
+                  },
+                },
+                preStop: {
+                  exec: {
+                    command: this.#getLifecycleCommand("preStop", "terminate"),
+                  },
+                },
+              },
+              volumeMounts: [
+                {
+                  name: "taskinfo",
+                  mountPath: "/etc/taskinfo",
+                },
+              ],
+            },
+          ],
+          volumes: [
+            {
+              name: "taskinfo",
+              emptyDir: {},
+            },
+          ],
+        },
+      },
+      this.#namespace
+    );
+  }
+
+  async delete(opts: { runId: string }) {
+    await this.#deletePod({
+      runId: opts.runId,
+      namespace: this.#namespace,
+    });
+  }
+
+  async get(opts: { runId: string }) {
+    await this.#getPod(opts.runId, this.#namespace);
+  }
+
+  async prePullDeployment(opts: TaskOperationsPrePullDeploymentOptions) {
+    const metaName = this.#getPrePullContainerName(opts.shortCode);
+
+    const metaLabels = {
+      ...this.#getSharedLabels(opts),
+      app: "task-prepull",
+      "app.kubernetes.io/part-of": "trigger-worker",
+      "app.kubernetes.io/component": "prepull",
+      deployment: opts.deploymentId,
+      name: metaName,
+    } satisfies k8s.V1ObjectMeta["labels"];
+
+    await this.#createDaemonSet(
+      {
+        metadata: {
+          name: metaName,
+          namespace: this.#namespace.metadata.name,
+          labels: metaLabels,
+        },
+        spec: {
+          selector: {
+            matchLabels: {
+              name: metaName,
+            },
+          },
+          template: {
+            metadata: {
+              labels: metaLabels,
+            },
+            spec: {
+              ...this.#defaultPodSpec,
+              restartPolicy: "Always",
+              initContainers: [
+                {
+                  name: "prepull",
+                  image: opts.imageRef,
+                  command: ["/usr/bin/true"],
+                  resources: {
+                    limits: {
+                      cpu: "0.25",
+                      memory: "100Mi",
+                      "ephemeral-storage": "1Gi",
+                    },
+                  },
+                },
+              ],
+              containers: [
+                {
+                  name: "pause",
+                  image: "registry.k8s.io/pause:3.9",
+                  resources: {
+                    limits: {
+                      cpu: "1m",
+                      memory: "12Mi",
+                    },
+                  },
+                },
+              ],
+            },
+          },
+        },
+      },
+      this.#namespace
+    );
+  }
+
+  #envTypeToLabelValue(type: EnvironmentType) {
+    switch (type) {
+      case "PRODUCTION":
+        return "prod";
+      case "STAGING":
+        return "stg";
+      case "DEVELOPMENT":
+        return "dev";
+      case "PREVIEW":
+        return "preview";
+    }
+  }
+
+  get #defaultPodSpec(): Omit<k8s.V1PodSpec, "containers"> {
+    return {
+      restartPolicy: "Never",
+      automountServiceAccountToken: false,
+      imagePullSecrets: [
+        {
+          name: "registry-trigger",
+        },
+        {
+          name: "registry-trigger-failover",
+        },
+      ],
+      nodeSelector: {
+        nodetype: "worker",
+      },
+    };
+  }
+
+  get #defaultResourceRequests(): ResourceQuantities {
+    return {
+      "ephemeral-storage": POD_EPHEMERAL_STORAGE_SIZE_REQUEST,
+    };
+  }
+
+  get #defaultResourceLimits(): ResourceQuantities {
+    return {
+      "ephemeral-storage": POD_EPHEMERAL_STORAGE_SIZE_LIMIT,
+    };
+  }
+
+  get #coordinatorHostEnvVar(): k8s.V1EnvVar {
+    return COORDINATOR_HOST
+      ? {
+          name: "COORDINATOR_HOST",
+          value: COORDINATOR_HOST,
+        }
+      : {
+          name: "COORDINATOR_HOST",
+          valueFrom: {
+            fieldRef: {
+              fieldPath: "status.hostIP",
+            },
+          },
+        };
+  }
+
+  get #coordinatorPortEnvVar(): k8s.V1EnvVar | undefined {
+    if (COORDINATOR_PORT) {
+      return {
+        name: "COORDINATOR_PORT",
+        value: COORDINATOR_PORT,
+      };
+    }
+  }
+
+  get #coordinatorEnvVars(): k8s.V1EnvVar[] {
+    const envVars = [this.#coordinatorHostEnvVar];
+
+    if (this.#coordinatorPortEnvVar) {
+      envVars.push(this.#coordinatorPortEnvVar);
+    }
+
+    return envVars;
+  }
+
+  #getSharedEnv(envId: string): k8s.V1EnvVar[] {
+    return [
+      {
+        name: "TRIGGER_ENV_ID",
+        value: envId,
+      },
+      {
+        name: "DEBUG",
+        value: process.env.DEBUG ? "1" : "0",
+      },
+      {
+        name: "HTTP_SERVER_PORT",
+        value: "8000",
+      },
+      {
+        name: "OTEL_EXPORTER_OTLP_ENDPOINT",
+        value: OTEL_EXPORTER_OTLP_ENDPOINT,
+      },
+      {
+        name: "POD_NAME",
+        valueFrom: {
+          fieldRef: {
+            fieldPath: "metadata.name",
+          },
+        },
+      },
+      {
+        name: "MACHINE_NAME",
+        valueFrom: {
+          fieldRef: {
+            fieldPath: "spec.nodeName",
+          },
+        },
+      },
+      ...this.#coordinatorEnvVars,
+    ];
+  }
+
+  #getSharedLabels(
+    opts:
+      | TaskOperationsIndexOptions
+      | TaskOperationsCreateOptions
+      | TaskOperationsRestoreOptions
+      | TaskOperationsPrePullDeploymentOptions
+  ): Record<string, string> {
+    return {
+      env: opts.envId,
+      envtype: this.#envTypeToLabelValue(opts.envType),
+      org: opts.orgId,
+      project: opts.projectId,
+    };
+  }
+
+  #getResourceRequestsForMachine(preset: MachinePreset): ResourceQuantities {
+    return {
+      cpu: `${preset.cpu * 0.75}`,
+      memory: `${preset.memory}G`,
+    };
+  }
+
+  #getResourceLimitsForMachine(preset: MachinePreset): ResourceQuantities {
+    return {
+      cpu: `${preset.cpu}`,
+      memory: `${preset.memory}G`,
+    };
+  }
+
+  #getResourcesForMachine(preset: MachinePreset): k8s.V1ResourceRequirements {
+    return {
+      requests: {
+        ...this.#defaultResourceRequests,
+        ...this.#getResourceRequestsForMachine(preset),
+      },
+      limits: {
+        ...this.#defaultResourceLimits,
+        ...this.#getResourceLimitsForMachine(preset),
+      },
+    };
+  }
+
+  #getLifecycleCommand<THookType extends "postStart" | "preStop">(
+    type: THookType,
+    cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses
+  ) {
+    const retries = 5;
+
+    // This will retry sending the lifecycle hook up to `retries` times
+    // The sleep is required as this may start running before the HTTP server is up
+    const exec = [
+      "/bin/sh",
+      "-c",
+      `for i in $(seq ${retries}); do sleep 1; busybox wget -q -O- 127.0.0.1:8000/${type}?cause=${cause} && break; done`,
+    ];
+
+    logger.debug("getLifecycleCommand()", { exec });
+
+    return exec;
+  }
+
+  #getIndexContainerName(suffix: string) {
+    return `task-index-${suffix}`;
+  }
+
+  #getRunContainerName(suffix: string, attemptNumber?: number) {
+    return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
+  }
+
+  #getPrePullContainerName(suffix: string) {
+    return `task-prepull-${suffix}`;
+  }
+
+  #createK8sApi() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (RUNTIME_ENV === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (RUNTIME_ENV === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${RUNTIME_ENV}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      batch: kubeConfig.makeApiClient(k8s.BatchV1Api),
+      apps: kubeConfig.makeApiClient(k8s.AppsV1Api),
+    };
+  }
+
+  async #createPod(pod: k8s.V1Pod, namespace: Namespace) {
+    try {
+      const res = await this.#k8sApi.core.createNamespacedPod(namespace.metadata.name, pod);
+      logger.debug(res.body);
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
+    }
+  }
+
+  async #deletePod(opts: { runId: string; namespace: Namespace }) {
+    try {
+      const res = await this.#k8sApi.core.deleteNamespacedPod(
+        opts.runId,
+        opts.namespace.metadata.name
+      );
+      logger.debug(res.body);
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
+    }
+  }
+
+  async #getPod(runId: string, namespace: Namespace) {
+    try {
+      const res = await this.#k8sApi.core.readNamespacedPod(runId, namespace.metadata.name);
+      logger.debug(res.body);
+      return res.body;
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
+    }
+  }
+
+  async #createJob(job: k8s.V1Job, namespace: Namespace) {
+    try {
+      const res = await this.#k8sApi.batch.createNamespacedJob(namespace.metadata.name, job);
+      logger.debug(res.body);
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
+    }
+  }
+
+  async #createDaemonSet(daemonSet: k8s.V1DaemonSet, namespace: Namespace) {
+    try {
+      const res = await this.#k8sApi.apps.createNamespacedDaemonSet(
+        namespace.metadata.name,
+        daemonSet
+      );
+      logger.debug(res.body);
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
+    }
+  }
+
+  #throwUnlessRecord(candidate: unknown): asserts candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      throw candidate;
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    this.#throwUnlessRecord(err);
+
+    if ("body" in err && err.body) {
+      logger.error(err.body);
+      this.#throwUnlessRecord(err.body);
+
+      if (typeof err.body.message === "string") {
+        throw new Error(err.body?.message);
+      } else {
+        throw err.body;
+      }
+    } else {
+      logger.error(err);
+      throw err;
+    }
+  }
+}
+
+const provider = new ProviderShell({
+  tasks: new KubernetesTaskOperations({
+    namespace: KUBERNETES_NAMESPACE,
+  }),
+  type: "kubernetes",
+});
+
+provider.listen();
+
+const taskMonitor = new TaskMonitor({
+  runtimeEnv: RUNTIME_ENV,
+  namespace: KUBERNETES_NAMESPACE,
+  onIndexFailure: async (deploymentId, details) => {
+    logger.log("Indexing failed", { deploymentId, details });
+
+    try {
+      provider.platformSocket.send("INDEXING_FAILED", {
+        deploymentId,
+        error: {
+          name: `Crashed with exit code ${details.exitCode}`,
+          message: details.reason,
+          stack: details.logs,
+        },
+        overrideCompletion: details.overrideCompletion,
+      });
+    } catch (error) {
+      logger.error(error);
+    }
+  },
+  onRunFailure: async (runId, details) => {
+    logger.log("Run failed:", { runId, details });
+
+    try {
+      provider.platformSocket.send("WORKER_CRASHED", { runId, ...details });
+    } catch (error) {
+      logger.error(error);
+    }
+  },
+});
+
+taskMonitor.start();
+
+const podCleaner = new PodCleaner({
+  runtimeEnv: RUNTIME_ENV,
+  namespace: KUBERNETES_NAMESPACE,
+  intervalInSeconds: POD_CLEANER_INTERVAL_SECONDS,
+});
+
+podCleaner.start();
+
+if (UPTIME_HEARTBEAT_URL) {
+  const uptimeHeartbeat = new UptimeHeartbeat({
+    runtimeEnv: RUNTIME_ENV,
+    namespace: KUBERNETES_NAMESPACE,
+    intervalInSeconds: UPTIME_INTERVAL_SECONDS,
+    pingUrl: UPTIME_HEARTBEAT_URL,
+    maxPendingRuns: UPTIME_MAX_PENDING_RUNS,
+    maxPendingIndeces: UPTIME_MAX_PENDING_INDECES,
+    maxPendingErrors: UPTIME_MAX_PENDING_ERRORS,
+  });
+
+  uptimeHeartbeat.start();
+} else {
+  logger.log("Uptime heartbeat is disabled, set UPTIME_HEARTBEAT_URL to enable.");
+}

apps/kubernetes-provider/src/podCleaner.ts

@@ -0,0 +1,323 @@
+import * as k8s from "@kubernetes/client-node";
+import { SimpleLogger } from "@trigger.dev/core/v3/apps";
+
+type PodCleanerOptions = {
+  runtimeEnv: "local" | "kubernetes";
+  namespace?: string;
+  intervalInSeconds?: number;
+};
+
+export class PodCleaner {
+  private enabled = false;
+  private namespace = "default";
+  private intervalInSeconds = 300;
+
+  private logger = new SimpleLogger("[PodCleaner]");
+  private k8sClient: {
+    core: k8s.CoreV1Api;
+    apps: k8s.AppsV1Api;
+    kubeConfig: k8s.KubeConfig;
+  };
+
+  constructor(private opts: PodCleanerOptions) {
+    if (opts.namespace) {
+      this.namespace = opts.namespace;
+    }
+
+    if (opts.intervalInSeconds) {
+      this.intervalInSeconds = opts.intervalInSeconds;
+    }
+
+    this.k8sClient = this.#createK8sClient();
+  }
+
+  #createK8sClient() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (this.opts.runtimeEnv === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (this.opts.runtimeEnv === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      apps: kubeConfig.makeApiClient(k8s.AppsV1Api),
+      kubeConfig: kubeConfig,
+    };
+  }
+
+  #isRecord(candidate: unknown): candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  #logK8sError(err: unknown, debugOnly = false) {
+    if (debugOnly) {
+      this.logger.debug("K8s API Error", err);
+    } else {
+      this.logger.error("K8s API Error", err);
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
+      this.#logK8sError(err);
+      return;
+    }
+
+    this.#logK8sError(err, true);
+
+    if (typeof err.body.message === "string") {
+      this.#logK8sError({ message: err.body.message });
+      return;
+    }
+
+    this.#logK8sError({ body: err.body });
+  }
+
+  async #deletePods(opts: {
+    namespace: string;
+    dryRun?: boolean;
+    fieldSelector?: string;
+    labelSelector?: string;
+  }) {
+    return await this.k8sClient.core
+      .deleteCollectionNamespacedPod(
+        opts.namespace,
+        undefined, // pretty
+        undefined, // continue
+        opts.dryRun ? "All" : undefined,
+        opts.fieldSelector,
+        undefined, // gracePeriodSeconds
+        opts.labelSelector
+      )
+      .catch(this.#handleK8sError.bind(this));
+  }
+
+  async #deleteDaemonSets(opts: {
+    namespace: string;
+    dryRun?: boolean;
+    fieldSelector?: string;
+    labelSelector?: string;
+  }) {
+    return await this.k8sClient.apps
+      .deleteCollectionNamespacedDaemonSet(
+        opts.namespace,
+        undefined, // pretty
+        undefined, // continue
+        opts.dryRun ? "All" : undefined,
+        opts.fieldSelector,
+        undefined, // gracePeriodSeconds
+        opts.labelSelector
+      )
+      .catch(this.#handleK8sError.bind(this));
+  }
+
+  async #deleteCompletedRuns() {
+    this.logger.log("Deleting completed runs");
+
+    const start = Date.now();
+
+    const result = await this.#deletePods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=Succeeded",
+      labelSelector: "app=task-run",
+    });
+
+    const elapsedMs = Date.now() - start;
+
+    if (!result) {
+      this.logger.log("Deleting completed runs: No delete result", { elapsedMs });
+      return;
+    }
+
+    const total = (result.response as any)?.body?.items?.length ?? 0;
+
+    this.logger.log("Deleting completed runs: Done", { total, elapsedMs });
+  }
+
+  async #deleteFailedRuns() {
+    this.logger.log("Deleting failed runs");
+
+    const start = Date.now();
+
+    const result = await this.#deletePods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=Failed",
+      labelSelector: "app=task-run",
+    });
+
+    const elapsedMs = Date.now() - start;
+
+    if (!result) {
+      this.logger.log("Deleting failed runs: No delete result", { elapsedMs });
+      return;
+    }
+
+    const total = (result.response as any)?.body?.items?.length ?? 0;
+
+    this.logger.log("Deleting failed runs: Done", { total, elapsedMs });
+  }
+
+  async #deleteUnrecoverableRuns() {
+    await this.#deletePods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=?",
+      labelSelector: "app=task-run",
+    });
+  }
+
+  async #deleteCompletedPrePulls() {
+    this.logger.log("Deleting completed pre-pulls");
+
+    const start = Date.now();
+
+    const result = await this.#deleteDaemonSets({
+      namespace: this.namespace,
+      labelSelector: "app=task-prepull",
+    });
+
+    const elapsedMs = Date.now() - start;
+
+    if (!result) {
+      this.logger.log("Deleting completed pre-pulls: No delete result", { elapsedMs });
+      return;
+    }
+
+    const total = (result.response as any)?.body?.items?.length ?? 0;
+
+    this.logger.log("Deleting completed pre-pulls: Done", { total, elapsedMs });
+  }
+
+  async start() {
+    this.enabled = true;
+    this.logger.log("Starting");
+
+    const completedInterval = setInterval(async () => {
+      if (!this.enabled) {
+        clearInterval(completedInterval);
+        return;
+      }
+
+      try {
+        await this.#deleteCompletedRuns();
+      } catch (error) {
+        this.logger.error("Error deleting completed runs", error);
+      }
+    }, this.intervalInSeconds * 1000);
+
+    const failedInterval = setInterval(
+      async () => {
+        if (!this.enabled) {
+          clearInterval(failedInterval);
+          return;
+        }
+
+        try {
+          await this.#deleteFailedRuns();
+        } catch (error) {
+          this.logger.error("Error deleting completed runs", error);
+        }
+      },
+      // Use a longer interval for failed runs. This is only a backup in case the task monitor fails.
+      2 * this.intervalInSeconds * 1000
+    );
+
+    const completedPrePullInterval = setInterval(
+      async () => {
+        if (!this.enabled) {
+          clearInterval(completedPrePullInterval);
+          return;
+        }
+
+        try {
+          await this.#deleteCompletedPrePulls();
+        } catch (error) {
+          this.logger.error("Error deleting completed pre-pulls", error);
+        }
+      },
+      2 * this.intervalInSeconds * 1000
+    );
+
+    // this.#launchTests();
+  }
+
+  async stop() {
+    if (!this.enabled) {
+      return;
+    }
+
+    this.enabled = false;
+    this.logger.log("Shutting down..");
+  }
+
+  async #launchTests() {
+    const createPod = async (
+      container: k8s.V1Container,
+      name: string,
+      labels?: Record<string, string>
+    ) => {
+      this.logger.log("Creating pod:", name);
+
+      const pod = {
+        metadata: {
+          name,
+          labels,
+        },
+        spec: {
+          restartPolicy: "Never",
+          automountServiceAccountToken: false,
+          terminationGracePeriodSeconds: 1,
+          containers: [container],
+        },
+      } satisfies k8s.V1Pod;
+
+      await this.k8sClient.core
+        .createNamespacedPod(this.namespace, pod)
+        .catch(this.#handleK8sError.bind(this));
+    };
+
+    const createIdlePod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: ["-c", "sleep infinity"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createCompletedPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: ["-c", "true"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createFailedPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: ["-c", "false"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    await createIdlePod("test-idle-1", { app: "task-run" });
+    await createFailedPod("test-failed-1", { app: "task-run" });
+    await createCompletedPod("test-completed-1", { app: "task-run" });
+  }
+}

apps/kubernetes-provider/src/taskMonitor.ts

@@ -0,0 +1,459 @@
+import * as k8s from "@kubernetes/client-node";
+import { SimpleLogger } from "@trigger.dev/core/v3/apps";
+import { EXIT_CODE_ALREADY_HANDLED, EXIT_CODE_CHILD_NONZERO } from "@trigger.dev/core/v3/apps";
+import { setTimeout } from "timers/promises";
+import PQueue from "p-queue";
+import { TaskRunErrorCodes, type Prettify, type TaskRunInternalError } from "@trigger.dev/core/v3";
+
+type FailureDetails = Prettify<{
+  exitCode: number;
+  reason: string;
+  logs: string;
+  overrideCompletion: boolean;
+  errorCode: TaskRunInternalError["code"];
+}>;
+
+type IndexFailureHandler = (deploymentId: string, details: FailureDetails) => Promise<any>;
+
+type RunFailureHandler = (runId: string, details: FailureDetails) => Promise<any>;
+
+type TaskMonitorOptions = {
+  runtimeEnv: "local" | "kubernetes";
+  onIndexFailure?: IndexFailureHandler;
+  onRunFailure?: RunFailureHandler;
+  namespace?: string;
+};
+
+export class TaskMonitor {
+  #enabled = false;
+
+  #logger = new SimpleLogger("[TaskMonitor]");
+  #taskInformer: ReturnType<typeof k8s.makeInformer<k8s.V1Pod>>;
+  #processedPods = new Map<string, number>();
+  #queue = new PQueue({ concurrency: 10 });
+
+  #k8sClient: {
+    core: k8s.CoreV1Api;
+    kubeConfig: k8s.KubeConfig;
+  };
+
+  private namespace = "default";
+  private fieldSelector = "status.phase=Failed";
+  private labelSelector = "app in (task-index, task-run)";
+
+  constructor(private opts: TaskMonitorOptions) {
+    if (opts.namespace) {
+      this.namespace = opts.namespace;
+    }
+
+    this.#k8sClient = this.#createK8sClient();
+
+    this.#taskInformer = this.#createTaskInformer();
+    this.#taskInformer.on("connect", this.#onInformerConnected.bind(this));
+    this.#taskInformer.on("error", this.#onInformerError.bind(this));
+    this.#taskInformer.on("update", this.#enqueueOnPodUpdated.bind(this));
+  }
+
+  #createTaskInformer() {
+    const listTasks = () =>
+      this.#k8sClient.core.listNamespacedPod(
+        this.namespace,
+        undefined,
+        undefined,
+        undefined,
+        this.fieldSelector,
+        this.labelSelector
+      );
+
+    // Uses watch with local caching
+    // https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes
+    const informer = k8s.makeInformer(
+      this.#k8sClient.kubeConfig,
+      `/api/v1/namespaces/${this.namespace}/pods`,
+      listTasks,
+      this.labelSelector,
+      this.fieldSelector
+    );
+
+    return informer;
+  }
+
+  async #onInformerConnected() {
+    this.#logger.log("Connected");
+  }
+
+  async #onInformerError(error: any) {
+    this.#logger.error("Error:", error);
+
+    // Automatic reconnect
+    await setTimeout(2_000);
+    this.#taskInformer.start();
+  }
+
+  #enqueueOnPodUpdated(pod: k8s.V1Pod) {
+    this.#queue.add(async () => {
+      try {
+        // It would be better to only pass the cache key, but the pod may already be removed from the cache by the time we process it
+        await this.#onPodUpdated(pod);
+      } catch (error) {
+        this.#logger.error("Caught onPodUpdated() error:", error);
+      }
+    });
+  }
+
+  async #onPodUpdated(pod: k8s.V1Pod) {
+    this.#logger.debug(`Updated: ${pod.metadata?.name}`);
+    this.#logger.debug("Updated", JSON.stringify(pod, null, 2));
+
+    // We only care about failures
+    if (pod.status?.phase !== "Failed") {
+      return;
+    }
+
+    const podName = pod.metadata?.name;
+
+    if (!podName) {
+      this.#logger.error("Pod is nameless", { pod });
+      return;
+    }
+
+    const containerStatus = pod.status.containerStatuses?.[0];
+
+    if (!containerStatus?.state) {
+      this.#logger.error("Pod failed, but container status doesn't have state", {
+        status: pod.status,
+      });
+      return;
+    }
+
+    if (this.#processedPods.has(podName)) {
+      this.#logger.debug("Pod update already processed", {
+        podName,
+        timestamp: this.#processedPods.get(podName),
+      });
+      return;
+    }
+
+    this.#processedPods.set(podName, Date.now());
+
+    const podStatus = this.#getPodStatusSummary(pod.status);
+    const containerState = this.#getContainerStateSummary(containerStatus.state);
+    const exitCode = containerState.exitCode ?? -1;
+
+    if (exitCode === EXIT_CODE_ALREADY_HANDLED) {
+      this.#logger.debug("Ignoring pod failure, already handled by worker", {
+        podName,
+      });
+      return;
+    }
+
+    const rawLogs = await this.#getLogTail(podName);
+
+    this.#logger.log(`${podName} failed with:`, {
+      podStatus,
+      containerState,
+      rawLogs,
+    });
+
+    const rawReason = podStatus.reason ?? containerState.reason ?? "";
+    const message = podStatus.message ?? containerState.message ?? "";
+
+    let reason = rawReason || "Unknown error";
+    let logs = rawLogs || "";
+
+    /** This will only override existing task errors. It will not crash the run.  */
+    let onlyOverrideExistingError = exitCode === EXIT_CODE_CHILD_NONZERO;
+
+    let errorCode: TaskRunInternalError["code"] = TaskRunErrorCodes.POD_UNKNOWN_ERROR;
+
+    switch (rawReason) {
+      case "Error":
+        reason = "Unknown error.";
+        errorCode = TaskRunErrorCodes.POD_UNKNOWN_ERROR;
+        break;
+      case "Evicted":
+        if (message.startsWith("Pod ephemeral local storage usage")) {
+          reason = "Storage limit exceeded.";
+          errorCode = TaskRunErrorCodes.DISK_SPACE_EXCEEDED;
+        } else if (message) {
+          reason = `Evicted: ${message}`;
+          errorCode = TaskRunErrorCodes.POD_EVICTED;
+        } else {
+          reason = "Evicted for unknown reason.";
+          errorCode = TaskRunErrorCodes.POD_EVICTED;
+        }
+
+        if (logs.startsWith("failed to try resolving symlinks")) {
+          logs = "";
+        }
+        break;
+      case "OOMKilled":
+        reason =
+          "[TaskMonitor] Your task ran out of memory. Try increasing the machine specs. If this doesn't fix it there might be a memory leak.";
+        errorCode = TaskRunErrorCodes.TASK_PROCESS_OOM_KILLED;
+        break;
+      default:
+        break;
+    }
+
+    const failureInfo = {
+      exitCode,
+      reason,
+      logs,
+      overrideCompletion: onlyOverrideExistingError,
+      errorCode,
+    } satisfies FailureDetails;
+
+    const app = pod.metadata?.labels?.app;
+
+    switch (app) {
+      case "task-index":
+        const deploymentId = pod.metadata?.labels?.deployment;
+
+        if (!deploymentId) {
+          this.#logger.error("Index is missing ID", { pod });
+          return;
+        }
+
+        if (this.opts.onIndexFailure) {
+          await this.opts.onIndexFailure(deploymentId, failureInfo);
+        }
+        break;
+      case "task-run":
+        const runId = pod.metadata?.labels?.run;
+
+        if (!runId) {
+          this.#logger.error("Run is missing ID", { pod });
+          return;
+        }
+
+        if (this.opts.onRunFailure) {
+          await this.opts.onRunFailure(runId, failureInfo);
+        }
+        break;
+      default:
+        this.#logger.error("Pod has invalid app label", { pod });
+        return;
+    }
+
+    await this.#deletePod(podName);
+  }
+
+  async #getLogTail(podName: string) {
+    try {
+      const logs = await this.#k8sClient.core.readNamespacedPodLog(
+        podName,
+        this.namespace,
+        undefined,
+        undefined,
+        undefined,
+        1024, // limitBytes
+        undefined,
+        undefined,
+        undefined,
+        20 // tailLines
+      );
+
+      const responseBody = logs.body ?? "";
+
+      if (responseBody.startsWith("unable to retrieve container logs")) {
+        return "";
+      }
+
+      // Type is wrong, body may be undefined
+      return responseBody;
+    } catch (error) {
+      this.#logger.error("Log tail error:", error instanceof Error ? error.message : "unknown");
+      return "";
+    }
+  }
+
+  #getPodStatusSummary(status: k8s.V1PodStatus) {
+    return {
+      reason: status.reason,
+      message: status.message,
+    };
+  }
+
+  #getContainerStateSummary(state: k8s.V1ContainerState) {
+    return {
+      reason: state.terminated?.reason,
+      exitCode: state.terminated?.exitCode,
+      message: state.terminated?.message,
+    };
+  }
+
+  #createK8sClient() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (this.opts.runtimeEnv === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (this.opts.runtimeEnv === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      kubeConfig: kubeConfig,
+    };
+  }
+
+  #isRecord(candidate: unknown): candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  #logK8sError(err: unknown, debugOnly = false) {
+    if (debugOnly) {
+      this.#logger.debug("K8s API Error", err);
+    } else {
+      this.#logger.error("K8s API Error", err);
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
+      this.#logK8sError(err);
+      return;
+    }
+
+    this.#logK8sError(err, true);
+
+    if (typeof err.body.message === "string") {
+      this.#logK8sError({ message: err.body.message });
+      return;
+    }
+
+    this.#logK8sError({ body: err.body });
+  }
+
+  #printStats(includeMoreDetails = false) {
+    this.#logger.log("Stats:", {
+      cacheSize: this.#taskInformer.list().length,
+      totalProcessed: this.#processedPods.size,
+      ...(includeMoreDetails && {
+        processedPods: this.#processedPods,
+      }),
+    });
+  }
+
+  async #deletePod(name: string) {
+    this.#logger.debug("Deleting pod:", name);
+
+    await this.#k8sClient.core
+      .deleteNamespacedPod(name, this.namespace)
+      .catch(this.#handleK8sError.bind(this));
+  }
+
+  async start() {
+    this.#enabled = true;
+
+    const interval = setInterval(() => {
+      if (!this.#enabled) {
+        clearInterval(interval);
+        return;
+      }
+
+      this.#printStats();
+    }, 300_000);
+
+    await this.#taskInformer.start();
+
+    // this.#launchTests();
+  }
+
+  async stop() {
+    if (!this.#enabled) {
+      return;
+    }
+
+    this.#enabled = false;
+    this.#logger.log("Shutting down..");
+
+    await this.#taskInformer.stop();
+
+    this.#printStats(true);
+  }
+
+  async #launchTests() {
+    const createPod = async (
+      container: k8s.V1Container,
+      name: string,
+      labels?: Record<string, string>
+    ) => {
+      this.#logger.log("Creating pod:", name);
+
+      const pod = {
+        metadata: {
+          name,
+          labels,
+        },
+        spec: {
+          restartPolicy: "Never",
+          automountServiceAccountToken: false,
+          terminationGracePeriodSeconds: 1,
+          containers: [container],
+        },
+      } satisfies k8s.V1Pod;
+
+      await this.#k8sClient.core
+        .createNamespacedPod(this.namespace, pod)
+        .catch(this.#handleK8sError.bind(this));
+    };
+
+    const createOomPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "polinux/stress",
+        resources: {
+          limits: {
+            memory: "100Mi",
+          },
+        },
+        command: ["stress"],
+        args: ["--vm", "1", "--vm-bytes", "150M", "--vm-hang", "1"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createNonZeroExitPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: ["-c", "exit 1"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createOoDiskPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: [
+          "-c",
+          "echo creating huge-file..; head -c 1000m /dev/zero > huge-file; ls -lh huge-file; sleep infinity",
+        ],
+        resources: {
+          limits: {
+            "ephemeral-storage": "500Mi",
+          },
+        },
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    await createNonZeroExitPod("non-zero-exit-task", { app: "task-run", run: "123" });
+    await createOomPod("oom-task", { app: "task-index", deployment: "456" });
+    await createOoDiskPod("ood-task", { app: "task-run", run: "abc" });
+  }
+}

apps/kubernetes-provider/src/uptimeHeartbeat.ts

@@ -0,0 +1,272 @@
+import * as k8s from "@kubernetes/client-node";
+import { SimpleLogger } from "@trigger.dev/core/v3/apps";
+
+type UptimeHeartbeatOptions = {
+  runtimeEnv: "local" | "kubernetes";
+  pingUrl: string;
+  namespace?: string;
+  intervalInSeconds?: number;
+  maxPendingRuns?: number;
+  maxPendingIndeces?: number;
+  maxPendingErrors?: number;
+  leadingEdge?: boolean;
+};
+
+export class UptimeHeartbeat {
+  private enabled = false;
+  private namespace: string;
+
+  private intervalInSeconds: number;
+  private maxPendingRuns: number;
+  private maxPendingIndeces: number;
+  private maxPendingErrors: number;
+
+  private leadingEdge = true;
+
+  private logger = new SimpleLogger("[UptimeHeartbeat]");
+  private k8sClient: {
+    core: k8s.CoreV1Api;
+    kubeConfig: k8s.KubeConfig;
+  };
+
+  constructor(private opts: UptimeHeartbeatOptions) {
+    this.namespace = opts.namespace ?? "default";
+
+    this.intervalInSeconds = opts.intervalInSeconds ?? 60;
+    this.maxPendingRuns = opts.maxPendingRuns ?? 25;
+    this.maxPendingIndeces = opts.maxPendingIndeces ?? 10;
+    this.maxPendingErrors = opts.maxPendingErrors ?? 10;
+
+    this.k8sClient = this.#createK8sClient();
+  }
+
+  #createK8sClient() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (this.opts.runtimeEnv === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (this.opts.runtimeEnv === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      kubeConfig: kubeConfig,
+    };
+  }
+
+  #isRecord(candidate: unknown): candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  #logK8sError(err: unknown, debugOnly = false) {
+    if (debugOnly) {
+      this.logger.debug("K8s API Error", err);
+    } else {
+      this.logger.error("K8s API Error", err);
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
+      this.#logK8sError(err);
+      return;
+    }
+
+    this.#logK8sError(err, true);
+
+    if (typeof err.body.message === "string") {
+      this.#logK8sError({ message: err.body.message });
+      return;
+    }
+
+    this.#logK8sError({ body: err.body });
+  }
+
+  async #getPods(opts: {
+    namespace: string;
+    fieldSelector?: string;
+    labelSelector?: string;
+  }): Promise<Array<k8s.V1Pod> | undefined> {
+    const listReturn = await this.k8sClient.core
+      .listNamespacedPod(
+        opts.namespace,
+        undefined, // pretty
+        undefined, // allowWatchBookmarks
+        undefined, // _continue
+        opts.fieldSelector,
+        opts.labelSelector,
+        this.maxPendingRuns * 2, // limit
+        undefined, // resourceVersion
+        undefined, // resourceVersionMatch
+        undefined, // sendInitialEvents
+        this.intervalInSeconds, // timeoutSeconds,
+        undefined // watch
+      )
+      .catch(this.#handleK8sError.bind(this));
+
+    return listReturn?.body.items;
+  }
+
+  async #getPendingIndeces(): Promise<Array<k8s.V1Pod> | undefined> {
+    return await this.#getPods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=Pending",
+      labelSelector: "app=task-index",
+    });
+  }
+
+  async #getPendingTasks(): Promise<Array<k8s.V1Pod> | undefined> {
+    return await this.#getPods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=Pending",
+      labelSelector: "app=task-run",
+    });
+  }
+
+  #countPods(pods: Array<k8s.V1Pod>): number {
+    return pods.length;
+  }
+
+  #filterPendingPods(
+    pods: Array<k8s.V1Pod>,
+    waitingReason: "CreateContainerError" | "RunContainerError"
+  ): Array<k8s.V1Pod> {
+    return pods.filter((pod) => {
+      const containerStatus = pod.status?.containerStatuses?.[0];
+      return containerStatus?.state?.waiting?.reason === waitingReason;
+    });
+  }
+
+  async #sendPing() {
+    this.logger.log("Sending ping");
+
+    const start = Date.now();
+    const controller = new AbortController();
+
+    const timeoutMs = (this.intervalInSeconds * 1000) / 2;
+
+    const fetchTimeout = setTimeout(() => {
+      controller.abort();
+    }, timeoutMs);
+
+    try {
+      const response = await fetch(this.opts.pingUrl, {
+        signal: controller.signal,
+      });
+
+      if (!response.ok) {
+        this.logger.error("Failed to send ping, response not OK", {
+          status: response.status,
+        });
+        return;
+      }
+
+      const elapsedMs = Date.now() - start;
+      this.logger.log("Ping sent", { elapsedMs });
+    } catch (error) {
+      if (error instanceof DOMException && error.name === "AbortError") {
+        this.logger.log("Ping timeout", { timeoutSeconds: timeoutMs });
+        return;
+      }
+
+      this.logger.error("Failed to send ping", error);
+    } finally {
+      clearTimeout(fetchTimeout);
+    }
+  }
+
+  async #heartbeat() {
+    this.logger.log("Performing heartbeat");
+
+    const start = Date.now();
+
+    const pendingTasks = await this.#getPendingTasks();
+
+    if (!pendingTasks) {
+      this.logger.error("Failed to get pending tasks");
+      return;
+    }
+
+    const totalPendingTasks = this.#countPods(pendingTasks);
+
+    const pendingIndeces = await this.#getPendingIndeces();
+
+    if (!pendingIndeces) {
+      this.logger.error("Failed to get pending indeces");
+      return;
+    }
+
+    const totalPendingIndeces = this.#countPods(pendingIndeces);
+
+    const elapsedMs = Date.now() - start;
+
+    this.logger.log("Finished heartbeat checks", { elapsedMs });
+
+    if (totalPendingTasks > this.maxPendingRuns) {
+      this.logger.log("Too many pending tasks, skipping heartbeat", { totalPendingTasks });
+      return;
+    }
+
+    if (totalPendingIndeces > this.maxPendingIndeces) {
+      this.logger.log("Too many pending indeces, skipping heartbeat", { totalPendingIndeces });
+      return;
+    }
+
+    const totalCreateContainerErrors = this.#countPods(
+      this.#filterPendingPods(pendingTasks, "CreateContainerError")
+    );
+    const totalRunContainerErrors = this.#countPods(
+      this.#filterPendingPods(pendingTasks, "RunContainerError")
+    );
+
+    if (totalCreateContainerErrors + totalRunContainerErrors > this.maxPendingErrors) {
+      this.logger.log("Too many pending tasks with errors, skipping heartbeat", {
+        totalRunContainerErrors,
+        totalCreateContainerErrors,
+      });
+      return;
+    }
+
+    await this.#sendPing();
+
+    this.logger.log("Heartbeat done", { totalPendingTasks, elapsedMs });
+  }
+
+  async start() {
+    this.enabled = true;
+    this.logger.log("Starting");
+
+    if (this.leadingEdge) {
+      await this.#heartbeat();
+    }
+
+    const heartbeat = setInterval(async () => {
+      if (!this.enabled) {
+        clearInterval(heartbeat);
+        return;
+      }
+
+      try {
+        await this.#heartbeat();
+      } catch (error) {
+        this.logger.error("Error while heartbeating", error);
+      }
+    }, this.intervalInSeconds * 1000);
+  }
+
+  async stop() {
+    if (!this.enabled) {
+      return;
+    }
+
+    this.enabled = false;
+    this.logger.log("Shutting down..");
+  }
+}

apps/kubernetes-provider/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "es2016",
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "paths": {
+      "@trigger.dev/core/v3": ["../../packages/core/src/v3"],
+      "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"]
+    }
+  }
+}

apps/proxy/.dev.vars.example

@@ -0,0 +1,7 @@
+REWRITE_HOSTNAME=
+AWS_SQS_ACCESS_KEY_ID=
+AWS_SQS_SECRET_ACCESS_KEY=
+AWS_SQS_QUEUE_URL=
+AWS_SQS_REGION=
+#optional
+#REWRITE_PORT=

apps/proxy/.editorconfig

@@ -0,0 +1,13 @@
+# http://editorconfig.org
+root = true
+
+[*]
+indent_style = tab
+tab_width = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.yml]
+indent_style = space

apps/proxy/.gitignore

@@ -0,0 +1,172 @@
+# Logs
+
+logs
+_.log
+npm-debug.log_
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+
+report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
+
+# Runtime data
+
+pids
+_.pid
+_.seed
+\*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+
+lib-cov
+
+# Coverage directory used by tools like istanbul
+
+coverage
+\*.lcov
+
+# nyc test coverage
+
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+
+bower_components
+
+# node-waf configuration
+
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+
+build/Release
+
+# Dependency directories
+
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+
+web_modules/
+
+# TypeScript cache
+
+\*.tsbuildinfo
+
+# Optional npm cache directory
+
+.npm
+
+# Optional eslint cache
+
+.eslintcache
+
+# Optional stylelint cache
+
+.stylelintcache
+
+# Microbundle cache
+
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+
+.node_repl_history
+
+# Output of 'npm pack'
+
+\*.tgz
+
+# Yarn Integrity file
+
+.yarn-integrity
+
+# dotenv environment variable files
+
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+
+.cache
+.parcel-cache
+
+# Next.js build output
+
+.next
+out
+
+# Nuxt.js build / generate output
+
+.nuxt
+dist
+
+# Gatsby files
+
+.cache/
+
+# Comment in the public line in if your project uses Gatsby and not Next.js
+
+# https://nextjs.org/blog/next-9-1#public-directory-support
+
+# public
+
+# vuepress build output
+
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+
+.temp
+.cache
+
+# Docusaurus cache and generated files
+
+.docusaurus
+
+# Serverless directories
+
+.serverless/
+
+# FuseBox cache
+
+.fusebox/
+
+# DynamoDB Local files
+
+.dynamodb/
+
+# TernJS port file
+
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+
+.vscode-test
+
+# yarn v2
+
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.\*
+
+# wrangler project
+
+.dev.vars
+.wrangler/

apps/proxy/.prettierrc

@@ -0,0 +1,11 @@
+{
+	"semi": true,
+	"singleQuote": false,
+	"jsxSingleQuote": false,
+	"trailingComma": "es5",
+	"bracketSpacing": true,
+	"bracketSameLine": false,
+	"printWidth": 100,
+	"tabWidth": 2,
+	"useTabs": false
+}

apps/proxy/CHANGELOG.md

@@ -0,0 +1,72 @@
+# proxy
+
+## 0.0.11
+
+### Patch Changes
+
+- @trigger.dev/core@2.3.5
+
+## 0.0.10
+
+### Patch Changes
+
+- @trigger.dev/core@2.3.4
+
+## 0.0.9
+
+### Patch Changes
+
+- @trigger.dev/core@2.3.3
+
+## 0.0.8
+
+### Patch Changes
+
+- @trigger.dev/core@2.3.2
+
+## 0.0.7
+
+### Patch Changes
+
+- Updated dependencies [f3efcc0c]
+  - @trigger.dev/core@2.3.1
+
+## 0.0.6
+
+### Patch Changes
+
+- Updated dependencies [17f6f29d]
+  - @trigger.dev/core@2.3.0
+
+## 0.0.5
+
+### Patch Changes
+
+- @trigger.dev/core@2.2.11
+
+## 0.0.4
+
+### Patch Changes
+
+- @trigger.dev/core@2.2.10
+
+## 0.0.3
+
+### Patch Changes
+
+- Updated dependencies [6ebd435e]
+  - @trigger.dev/core@2.2.9
+
+## 0.0.2
+
+### Patch Changes
+
+- Updated dependencies [067e19fe]
+  - @trigger.dev/core@2.2.8
+
+## 0.0.1
+
+### Patch Changes
+
+- Updated dependencies [756024da]
+  - @trigger.dev/core@2.2.7

apps/proxy/README.md

@@ -0,0 +1,68 @@
+# Trigger.dev proxy
+
+This is an optional module that can be used to proxy and queue requests to the Trigger.dev API.
+
+## Why?
+
+The Trigger.dev API is designed to be fast and reliable. However, if you have a lot of traffic, you may want to use this proxy to queue requests to the API. It intercepts some requests to the API and adds them to an AWS SQS queue, then the webapp can be setup to process the queue.
+
+## Current features
+
+- Intercepts `sendEvent` requests and adds them to an AWS SQS queue. The webapp then reads from the queue and creates the events.
+
+## Setup
+
+### Create an AWS SQS queue
+
+In AWS you should create a new AWS SQS queue with appropriate security settings. You will need the queue URL for the next step.
+
+### Environment variables
+
+#### Cloudflare secrets
+
+Locally you should copy the `.dev.var.example` file to `.dev.var` and fill in the values.
+
+When deploying you should use `wrangler` (the Cloudflare CLI tool) to set secrets. Make sure you set the correct --env ("staging" or "prod")
+
+```bash
+wrangler secret put REWRITE_HOSTNAME --env staging
+wrangler secret put AWS_SQS_ACCESS_KEY_ID --env staging
+wrangler secret put AWS_SQS_SECRET_ACCESS_KEY --env staging
+wrangler secret put AWS_SQS_QUEUE_URL --env staging
+wrangler secret put AWS_SQS_REGION --env staging
+```
+
+You need to set your API CNAME entry to be proxied by Cloudflare. You can do this in the Cloudflare dashboard.
+
+#### Webapp
+
+These env vars also need setting in the webapp.
+
+```bash
+AWS_SQS_REGION
+AWS_SQS_ACCESS_KEY_ID
+AWS_SQS_SECRET_ACCESS_KEY
+AWS_SQS_QUEUE_URL
+AWS_SQS_BATCH_SIZE
+```
+
+## Deployment
+
+Staging:
+
+```bash
+npx wrangler@latest deploy --route "<your-api-subdomain>/*" --env staging
+```
+
+Prod:
+
+```bash
+npx wrangler@latest deploy --route "<your-api-subdomain>/*" --env prod
+```
+
+## Development
+
+Set the environment variables as described above.
+
+1. `pnpm install`
+2. `pnpm run dev --filter proxy`

apps/proxy/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "proxy",
+  "version": "0.0.11",
+  "private": true,
+  "scripts": {
+    "deploy": "wrangler deploy",
+    "dev": "wrangler dev",
+    "dry-run:staging": "wrangler deploy --dry-run --outdir=dist --env staging"
+  },
+  "devDependencies": {
+    "@cloudflare/workers-types": "^4.20240512.0",
+    "typescript": "^5.0.4",
+    "wrangler": "^3.57.1"
+  },
+  "dependencies": {
+    "@aws-sdk/client-sqs": "^3.445.0",
+    "@trigger.dev/core": "workspace:*",
+    "ulidx": "^2.2.1",
+    "zod": "3.23.8",
+    "zod-error": "1.5.0"
+  }
+}

apps/proxy/src/apikey.ts

@@ -0,0 +1,20 @@
+import { z } from "zod";
+
+const AuthorizationHeaderSchema = z.string().regex(/^Bearer .+$/);
+
+export function getApiKeyFromRequest(request: Request) {
+  const rawAuthorization = request.headers.get("Authorization");
+
+  const authorization = AuthorizationHeaderSchema.safeParse(rawAuthorization);
+  if (!authorization.success) {
+    return;
+  }
+
+  const apiKey = authorization.data.replace(/^Bearer /, "");
+  const type = isPrivateApiKey(apiKey) ? ("PRIVATE" as const) : ("PUBLIC" as const);
+  return { apiKey, type };
+}
+
+function isPrivateApiKey(key: string) {
+  return key.startsWith("tr_");
+}

apps/proxy/src/events/queueEvent.ts

@@ -0,0 +1,87 @@
+import { SQSClient, SendMessageCommand } from "@aws-sdk/client-sqs";
+import { ApiEventLog, SendEventBodySchema } from "@trigger.dev/core";
+import { generateErrorMessage } from "zod-error";
+import { Env } from "..";
+import { getApiKeyFromRequest } from "../apikey";
+import { json } from "../json";
+import { calculateDeliverAt } from "./utils";
+
+/** Adds the event to an AWS SQS queue, so it can be consumed from the main Trigger.dev API */
+export async function queueEvent(request: Request, env: Env): Promise<Response> {
+  //check there's a private API key
+  const apiKeyResult = getApiKeyFromRequest(request);
+  if (!apiKeyResult || apiKeyResult.type !== "PRIVATE") {
+    return json(
+      { error: "Invalid or Missing API key" },
+      {
+        status: 401,
+      }
+    );
+  }
+
+  //parse the request body
+  try {
+    const anyBody = await request.json();
+    const body = SendEventBodySchema.safeParse(anyBody);
+    if (!body.success) {
+      return json(
+        { error: generateErrorMessage(body.error.issues) },
+        {
+          status: 422,
+        }
+      );
+    }
+
+    // The AWS SDK tries to use crypto from off of the window,
+    // so we need to trick it into finding it where it expects it
+    globalThis.global = globalThis;
+
+    const client = new SQSClient({
+      region: env.AWS_SQS_REGION,
+      credentials: {
+        accessKeyId: env.AWS_SQS_ACCESS_KEY_ID,
+        secretAccessKey: env.AWS_SQS_SECRET_ACCESS_KEY,
+      },
+    });
+
+    const timestamp = body.data.event.timestamp ?? new Date();
+
+    //add the event to the queue
+    const send = new SendMessageCommand({
+      // use wrangler secrets to provide this global variable
+      QueueUrl: env.AWS_SQS_QUEUE_URL,
+      MessageBody: JSON.stringify({
+        event: { ...body.data.event, timestamp },
+        options: body.data.options,
+        apiKey: apiKeyResult.apiKey,
+      }),
+    });
+
+    const queuedEvent = await client.send(send);
+    console.log("Queued event", queuedEvent);
+
+    //respond with the event
+    const event: ApiEventLog = {
+      id: body.data.event.id,
+      name: body.data.event.name,
+      payload: body.data.event.payload,
+      context: body.data.event.context,
+      timestamp,
+      deliverAt: calculateDeliverAt(body.data.options),
+    };
+
+    return json(event, {
+      status: 200,
+    });
+  } catch (e) {
+    console.error("queueEvent error", e);
+    return json(
+      {
+        error: `Failed to send event: ${e instanceof Error ? e.message : JSON.stringify(e)}`,
+      },
+      {
+        status: 422,
+      }
+    );
+  }
+}

apps/proxy/src/events/queueEvents.ts

@@ -0,0 +1,112 @@
+import { SQSClient, SendMessageBatchCommand } from "@aws-sdk/client-sqs";
+import { ApiEventLog, SendBulkEventsBodySchema } from "@trigger.dev/core";
+import { generateErrorMessage } from "zod-error";
+import { Env } from "..";
+import { getApiKeyFromRequest } from "../apikey";
+import { json } from "../json";
+import { calculateDeliverAt } from "./utils";
+
+/** Adds the event to an AWS SQS queue, so it can be consumed from the main Trigger.dev API */
+export async function queueEvents(request: Request, env: Env): Promise<Response> {
+  //check there's a private API key
+  const apiKeyResult = getApiKeyFromRequest(request);
+  if (!apiKeyResult || apiKeyResult.type !== "PRIVATE") {
+    return json(
+      { error: "Invalid or Missing API key" },
+      {
+        status: 401,
+      }
+    );
+  }
+
+  //parse the request body
+  try {
+    const anyBody = await request.json();
+    const body = SendBulkEventsBodySchema.safeParse(anyBody);
+    if (!body.success) {
+      return json(
+        { error: generateErrorMessage(body.error.issues) },
+        {
+          status: 422,
+        }
+      );
+    }
+
+    // The AWS SDK tries to use crypto from off of the window,
+    // so we need to trick it into finding it where it expects it
+    globalThis.global = globalThis;
+
+    const client = new SQSClient({
+      region: env.AWS_SQS_REGION,
+      credentials: {
+        accessKeyId: env.AWS_SQS_ACCESS_KEY_ID,
+        secretAccessKey: env.AWS_SQS_SECRET_ACCESS_KEY,
+      },
+    });
+
+    const updatedEvents: ApiEventLog[] = body.data.events.map((event) => {
+      const timestamp = event.timestamp ?? new Date();
+      return {
+        ...event,
+        payload: event.payload,
+        timestamp,
+      };
+    });
+
+    //divide updatedEvents into multiple batches of 10 (max size SQS accepts)
+    const batches: ApiEventLog[][] = [];
+    let currentBatch: ApiEventLog[] = [];
+    for (let i = 0; i < updatedEvents.length; i++) {
+      currentBatch.push(updatedEvents[i]);
+      if (currentBatch.length === 10) {
+        batches.push(currentBatch);
+        currentBatch = [];
+      }
+    }
+    if (currentBatch.length > 0) {
+      batches.push(currentBatch);
+    }
+
+    //loop through the batches and send them
+    for (let i = 0; i < batches.length; i++) {
+      const batch = batches[i];
+      //add the event to the queue
+      const send = new SendMessageBatchCommand({
+        // use wrangler secrets to provide this global variable
+        QueueUrl: env.AWS_SQS_QUEUE_URL,
+        Entries: batch.map((event, index) => ({
+          Id: `event-${index}`,
+          MessageBody: JSON.stringify({
+            event,
+            options: body.data.options,
+            apiKey: apiKeyResult.apiKey,
+          }),
+        })),
+      });
+
+      const queuedEvent = await client.send(send);
+      console.log("Queued events", queuedEvent);
+    }
+
+    //respond with the events
+    const events: ApiEventLog[] = updatedEvents.map((event) => ({
+      ...event,
+      payload: event.payload,
+      deliverAt: calculateDeliverAt(body.data.options),
+    }));
+
+    return json(events, {
+      status: 200,
+    });
+  } catch (e) {
+    console.error("queueEvents error", e);
+    return json(
+      {
+        error: `Failed to send events: ${e instanceof Error ? e.message : JSON.stringify(e)}`,
+      },
+      {
+        status: 422,
+      }
+    );
+  }
+}

apps/proxy/src/events/utils.ts

@@ -0,0 +1,15 @@
+import { SendEventOptions } from "@trigger.dev/core";
+
+export function calculateDeliverAt(options?: SendEventOptions) {
+  // If deliverAt is a string and a valid date, convert it to a Date object
+  if (options?.deliverAt) {
+    return options?.deliverAt;
+  }
+
+  // deliverAfter is the number of seconds to wait before delivering the event
+  if (options?.deliverAfter) {
+    return new Date(Date.now() + options.deliverAfter * 1000);
+  }
+
+  return undefined;
+}

apps/proxy/src/index.ts

@@ -0,0 +1,53 @@
+import { queueEvent } from "./events/queueEvent";
+import { queueEvents } from "./events/queueEvents";
+import { applyRateLimit } from "./rateLimit";
+import { Ratelimit } from "./rateLimiter";
+
+export interface Env {
+  /** The hostname needs to be changed to allow requests to pass to the Trigger.dev platform */
+  REWRITE_HOSTNAME: string;
+  REWRITE_PORT?: string;
+  AWS_SQS_ACCESS_KEY_ID: string;
+  AWS_SQS_SECRET_ACCESS_KEY: string;
+  AWS_SQS_QUEUE_URL: string;
+  AWS_SQS_REGION: string;
+  //rate limiter
+  API_RATE_LIMITER: Ratelimit;
+}
+
+export default {
+  async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
+    if (!queueingIsEnabled(env)) {
+      console.log("Missing AWS credentials. Passing through to the origin.");
+      return fetch(request);
+    }
+
+    const url = new URL(request.url);
+    switch (url.pathname) {
+      case "/api/v1/events": {
+        if (request.method === "POST") {
+          return applyRateLimit(request, env, () => queueEvent(request, env));
+        }
+        break;
+      }
+      case "/api/v1/events/bulk": {
+        if (request.method === "POST") {
+          return applyRateLimit(request, env, () => queueEvents(request, env));
+        }
+        break;
+      }
+    }
+
+    //the same request but with the hostname (and port) changed
+    return fetch(request);
+  },
+};
+
+function queueingIsEnabled(env: Env) {
+  return (
+    env.AWS_SQS_ACCESS_KEY_ID &&
+    env.AWS_SQS_SECRET_ACCESS_KEY &&
+    env.AWS_SQS_QUEUE_URL &&
+    env.AWS_SQS_REGION
+  );
+}

apps/proxy/src/json.ts

@@ -0,0 +1,13 @@
+export function json(body: any, init?: ResponseInit) {
+  const headers = {
+    "content-type": "application/json",
+    ...(init?.headers ?? {}),
+  };
+
+  const responseInit: ResponseInit = {
+    ...(init ?? {}),
+    headers,
+  };
+
+  return new Response(JSON.stringify(body), responseInit);
+}

apps/proxy/src/rateLimit.ts

@@ -0,0 +1,46 @@
+import { Env } from "src";
+import { getApiKeyFromRequest } from "./apikey";
+import { json } from "./json";
+
+export async function applyRateLimit(
+  request: Request,
+  env: Env,
+  fn: () => Promise<Response>
+): Promise<Response> {
+  const apiKey = getApiKeyFromRequest(request);
+  if (apiKey) {
+    const result = await env.API_RATE_LIMITER.limit({ key: `apikey-${apiKey.apiKey}` });
+    const { success } = result;
+    console.log(`Rate limiter`, {
+      success,
+      key: `${apiKey.apiKey.substring(0, 12)}...`,
+    });
+    if (!success) {
+      //60s in the future
+      const reset = Date.now() + 60 * 1000;
+      const secondsUntilReset = Math.max(0, (reset - new Date().getTime()) / 1000);
+
+      return json(
+        {
+          title: "Rate Limit Exceeded",
+          status: 429,
+          type: "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429",
+          detail: `Rate limit exceeded. Retry in ${secondsUntilReset} seconds.`,
+          error: `Rate limit exceeded. Retry in ${secondsUntilReset} seconds.`,
+          reset,
+        },
+        {
+          status: 429,
+          headers: {
+            "x-ratelimit-reset": reset.toString(),
+          },
+        }
+      );
+    }
+  } else {
+    console.log(`Rate limiter: no API key for request`);
+  }
+
+  //call the original function
+  return fn();
+}

apps/proxy/src/rateLimiter.ts

@@ -0,0 +1,23 @@
+export interface Ratelimit {
+  /*
+   * The ratelimit function
+   * @param {RatelimitOptions} options
+   * @returns {Promise<RatelimitResponse>}
+   */
+  limit: (options: RatelimitOptions) => Promise<RatelimitResponse>;
+}
+
+export interface RatelimitOptions {
+  /*
+   * The key to identify the user, can be an IP address, user ID, etc.
+   */
+  key: string;
+}
+
+export interface RatelimitResponse {
+  /*
+   * The ratelimit success status
+   * @returns {boolean}
+   */
+  success: boolean;
+}

apps/proxy/tsconfig.json

@@ -0,0 +1,35 @@
+{
+  "compilerOptions": {
+    "target": "es2021" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
+    "lib": [
+      "es2021"
+    ] /* Specify a set of bundled library declaration files that describe the target runtime environment. */,
+    "jsx": "react" /* Specify what JSX code is generated. */,
+
+    "module": "es2022" /* Specify what module code is generated. */,
+    "moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */,
+
+    "types": [
+      "@cloudflare/workers-types"
+    ] /* Specify type package names to be included without being referenced in a source file. */,
+    "resolveJsonModule": true /* Enable importing .json files */,
+
+    "allowJs": true /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */,
+    "checkJs": false /* Enable error reporting in type-checked JavaScript files. */,
+
+    "noEmit": true /* Disable emitting files from a compilation. */,
+
+    "isolatedModules": true /* Ensure that each file can be safely transpiled without relying on other imports. */,
+    "allowSyntheticDefaultImports": true /* Allow 'import x from y' when a module doesn't have a default export. */,
+    "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
+
+    "strict": true /* Enable all strict type-checking options. */,
+
+    "skipLibCheck": true /* Skip type checking all .d.ts files. */,
+    "baseUrl": ".",
+    "paths": {
+      "@trigger.dev/core": ["../../packages/core/src/index"],
+      "@trigger.dev/core/*": ["../../packages/core/src/*"]
+    }
+  }
+}

apps/proxy/wrangler.toml

@@ -0,0 +1,33 @@
+name = "proxy"
+main = "src/index.ts"
+compatibility_date = "2024-05-13"
+compatibility_flags = [ "nodejs_compat" ]
+
+[env.staging]
+  # The rate limiting API is in open beta.
+  [[env.staging.unsafe.bindings]]
+  name = "API_RATE_LIMITER"
+  type = "ratelimit"
+  # An identifier you define, that is unique to your Cloudflare account.
+  # Must be an integer.
+  namespace_id = "1"
+
+  # Limit: the number of tokens allowed within a given period in a single
+  # Cloudflare location
+  # Period: the duration of the period, in seconds. Must be either 10 or 60
+  simple = { limit = 100, period = 60 }
+
+
+[env.prod]
+  # The rate limiting API is in open beta.
+  [[env.prod.unsafe.bindings]]
+  name = "API_RATE_LIMITER"
+  type = "ratelimit"
+  # An identifier you define, that is unique to your Cloudflare account.
+  # Must be an integer.
+  namespace_id = "2"
+
+  # Limit: the number of tokens allowed within a given period in a single
+  # Cloudflare location
+  # Period: the duration of the period, in seconds. Must be either 10 or 60
+  simple = { limit = 300, period = 60 }

apps/webapp/.eslintrc

@@ -1,13 +1,36 @@
 {
-  "extends": ["@remix-run/eslint-config", "@remix-run/eslint-config/node", "prettier"],
-  "rules": {
-    "@typescript-eslint/strict-boolean-expressions": [
-      "error",
-      {
-        "allowNullableBoolean": true,
-        "allowNullableString": true,
-        "allowNullableNumber": true
+  "plugins": [
+    "@trigger.dev/eslint-plugin",
+    "react-hooks",
+    "@typescript-eslint/eslint-plugin",
+    "import"
+  ],
+  "parser": "@typescript-eslint/parser",
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx"],
+      "rules": {
+        // Autofixes imports from "@trigger.dev/core" to fine grained modules
+        "@trigger.dev/no-trigger-core-import": "error",
+        // Normalize `import type {}` and `import { type }`
+        "@typescript-eslint/consistent-type-imports": [
+          "warn",
+          {
+            // the "type" annotation can get tangled and cause syntax errors
+            // during some autofixes, so easier to just turn it off
+            "prefer": "type-imports",
+            "disallowTypeAnnotations": true,
+            "fixStyle": "inline-type-imports"
+          }
+        ],
+        // no-trigger-core-import splits imports into multiple lines
+        // this one merges them back into a single line
+        // if they still import from the same module
+        "import/no-duplicates": ["warn", { "prefer-inline": true }],
+        // lots of undeclared vars, enable this rule if you want to clean them up
+        "turbo/no-undeclared-env-vars": "off"
       }
-    ]
-  }
+    }
+  ],
+  "ignorePatterns": ["seed.js", "seedCloud.ts", "populate.js"]
 }

apps/webapp/.gitignore

@@ -17,4 +17,5 @@ build-storybook.log
 .storybook-out
 storybook-static
 
-/prisma/seed.js
+/prisma/seed.js
+/prisma/populate.js

apps/webapp/.storybook/main.ts

@@ -1,49 +0,0 @@
-import type { StorybookConfig } from "@storybook/react-webpack5";
-import path from "path";
-
-const root = path.resolve(__dirname, "../app");
-
-const config: StorybookConfig = {
-  webpackFinal: async (config) => {
-    return {
-      ...config,
-      resolve: {
-        ...config.resolve,
-        alias: {
-          ...(config.resolve?.alias ?? {}),
-          "~": root,
-        },
-        extensions: [
-          ...(config.resolve?.extensions ?? []),
-          ...[".ts", ".tsx", ".js", ".jsx", ".mdx"],
-        ],
-      },
-    };
-  },
-  stories: ["../app/**/stories/*.mdx", "../app/**/stories/*.stories.@(js|jsx|ts|tsx)"],
-  addons: [
-    "@storybook/addon-links",
-    "@storybook/addon-essentials",
-    "@storybook/addon-interactions",
-    "storybook-addon-variants",
-    "storybook-addon-designs",
-    "@storybook/addon-docs",
-    {
-      name: "@storybook/addon-styling",
-      options: {
-        // Check out https://github.com/storybookjs/addon-styling/blob/main/docs/api.md
-        // For more details on this addon's options.
-        postCss: true,
-      },
-    },
-  ],
-  framework: {
-    name: "@storybook/react-webpack5",
-    options: {},
-  },
-  docs: {
-    autodocs: "tag",
-  },
-  staticDirs: [path.resolve("public")],
-};
-export default config;

apps/webapp/.storybook/preview.tsx

@@ -1,47 +0,0 @@
-import type { Preview } from "@storybook/react";
-import "../app/tailwind.css";
-import { unstable_createRemixStub } from "@remix-run/testing";
-import React from "react";
-import { LocaleContextProvider } from "../app/components/primitives/LocaleProvider";
-import { OperatingSystemContextProvider } from "../app/components/primitives/OperatingSystemProvider";
-
-const preview: Preview = {
-  parameters: {
-    actions: { argTypesRegex: "^on[A-Z].*" },
-    controls: {
-      matchers: {
-        color: /(background|color)$/i,
-        date: /Date$/,
-      },
-    },
-    backgrounds: {
-      default: "App background",
-      values: [
-        {
-          name: "App background",
-          value: "#0B1018",
-        },
-      ],
-    },
-  },
-  decorators: [
-    (Story) => {
-      const RemixStub = unstable_createRemixStub([
-        {
-          path: "/*",
-          element: <Story />,
-        },
-      ]);
-
-      return (
-        <OperatingSystemContextProvider platform="mac">
-          <LocaleContextProvider locales={window.navigator.languages as string[]}>
-            <RemixStub initialEntries={["/"]} />
-          </LocaleContextProvider>
-        </OperatingSystemContextProvider>
-      );
-    },
-  ],
-};
-
-export default preview;

apps/webapp/app/assets/icons/AISparkleIcon.tsx

@@ -0,0 +1,53 @@
+export function AISparkleIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M14.9806 0.803884C14.8871 0.33646 14.4767 0 14 0C13.5233 0 13.1129 0.33646 13.0194 0.803884L12.7809 1.99644C12.7017 2.3923 12.3923 2.70174 11.9964 2.78091L10.8039 3.01942C10.3365 3.1129 10 3.52332 10 4C10 4.47668 10.3365 4.8871 10.8039 4.98058L11.9964 5.21909C12.3923 5.29826 12.7017 5.6077 12.7809 6.00356L13.0194 7.19612C13.1129 7.66354 13.5233 8 14 8C14.4767 8 14.8871 7.66354 14.9806 7.19612L15.2191 6.00356C15.2983 5.6077 15.6077 5.29826 16.0036 5.21909L17.1961 4.98058C17.6635 4.8871 18 4.47668 18 4C18 3.52332 17.6635 3.1129 17.1961 3.01942L16.0036 2.78091C15.6077 2.70174 15.2983 2.3923 15.2191 1.99644L14.9806 0.803884Z"
+        fill="url(#paint0_linear_11402_36656)"
+      />
+      <path
+        d="M5.94868 4.68377C5.81257 4.27543 5.43043 4 5 4C4.56957 4 4.18743 4.27543 4.05132 4.68377L3.36754 6.73509C3.26801 7.03369 3.03369 7.26801 2.73509 7.36754L0.683772 8.05132C0.27543 8.18743 0 8.56957 0 9C0 9.43043 0.27543 9.81257 0.683772 9.94868L2.73509 10.6325C3.03369 10.732 3.26801 10.9663 3.36754 11.2649L4.05132 13.3162C4.18743 13.7246 4.56957 14 5 14C5.43043 14 5.81257 13.7246 5.94868 13.3162L6.63246 11.2649C6.73199 10.9663 6.96631 10.732 7.26491 10.6325L9.31623 9.94868C9.72457 9.81257 10 9.43043 10 9C10 8.56957 9.72457 8.18743 9.31623 8.05132L7.26491 7.36754C6.96631 7.26801 6.73199 7.03369 6.63246 6.73509L5.94868 4.68377Z"
+        fill="url(#paint1_linear_11402_36656)"
+      />
+      <path
+        d="M12.9487 12.6838C12.8126 12.2754 12.4304 12 12 12C11.5696 12 11.1874 12.2754 11.0513 12.6838L10.8675 13.2351C10.768 13.5337 10.5337 13.768 10.2351 13.8675L9.68377 14.0513C9.27543 14.1874 9 14.5696 9 15C9 15.4304 9.27543 15.8126 9.68377 15.9487L10.2351 16.1325C10.5337 16.232 10.768 16.4663 10.8675 16.7649L11.0513 17.3162C11.1874 17.7246 11.5696 18 12 18C12.4304 18 12.8126 17.7246 12.9487 17.3162L13.1325 16.7649C13.232 16.4663 13.4663 16.232 13.7649 16.1325L14.3162 15.9487C14.7246 15.8126 15 15.4304 15 15C15 14.5696 14.7246 14.1874 14.3162 14.0513L13.7649 13.8675C13.4663 13.768 13.232 13.5337 13.1325 13.2351L12.9487 12.6838Z"
+        fill="url(#paint2_linear_11402_36656)"
+      />
+      <defs>
+        <linearGradient
+          id="paint0_linear_11402_36656"
+          x1="9"
+          y1="0"
+          x2="9"
+          y2="18"
+          gradientUnits="userSpaceOnUse"
+        >
+          <stop stopColor="#E543FF" />
+          <stop offset="1" stopColor="#286399" />
+        </linearGradient>
+        <linearGradient
+          id="paint1_linear_11402_36656"
+          x1="9"
+          y1="0"
+          x2="9"
+          y2="18"
+          gradientUnits="userSpaceOnUse"
+        >
+          <stop stopColor="#E543FF" />
+          <stop offset="1" stopColor="#286399" />
+        </linearGradient>
+        <linearGradient
+          id="paint2_linear_11402_36656"
+          x1="9"
+          y1="0"
+          x2="9"
+          y2="18"
+          gradientUnits="userSpaceOnUse"
+        >
+          <stop stopColor="#E543FF" />
+          <stop offset="1" stopColor="#286399" />
+        </linearGradient>
+      </defs>
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/AttemptIcon.tsx

@@ -0,0 +1,29 @@
+export function AttemptIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="16"
+      height="16"
+      viewBox="0 0 16 16"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <g clipPath="url(#clip0_9964_113464)">
+        <path
+          fillRule="evenodd"
+          clipRule="evenodd"
+          d="M16 0H0V16H16V0ZM7.09906 4.4L4.53906 11.5H6.11906L6.63906 10H9.35906L9.87906 11.5H11.4591L8.89906 4.4H7.09906ZM7.99906 6L8.92906 8.73H7.06906L7.99906 6Z"
+          fill="currentColor"
+        />
+      </g>
+      <defs>
+        <clipPath id="clip0_9964_113464">
+          <path
+            d="M0 2C0 0.895431 0.895431 0 2 0H14C15.1046 0 16 0.895431 16 2V14C16 15.1046 15.1046 16 14 16H2C0.895431 16 0 15.1046 0 14V2Z"
+            fill="white"
+          />
+        </clipPath>
+      </defs>
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/EndpointIcon.tsx

@@ -0,0 +1,36 @@
+export function EndpointIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      viewBox="0 0 24 24"
+      fill="currentColor"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        d="M8.24336 5.11182C8.75159 4.60359 9.57559 4.60359 10.0838 5.11182L18.8882 13.9162C19.3964 14.4244 19.3964 15.2484 18.8882 15.7566L17.1469 17.4979C16.453 18.2196 15.6222 18.7956 14.7031 19.1922C13.7839 19.5889 12.7949 19.7982 11.7939 19.8081C10.7929 19.8179 9.79992 19.628 8.87318 19.2494C7.94643 18.8708 7.10449 18.3113 6.39662 17.6034C5.68875 16.8955 5.12916 16.0536 4.7506 15.1268C4.37205 14.2001 4.18212 13.2071 4.19194 12.2061C4.20176 11.2051 4.41112 10.2161 4.80779 9.29691C5.20445 8.37777 5.78044 7.54697 6.50206 6.85312L8.24336 5.11182Z"
+        fill="currentColor"
+      />
+      <path
+        d="M3.32398 20.676L7.11974 16.8803"
+        stroke="currentColor"
+        strokeWidth="2.16549"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M15.2535 3.32398L11.4578 7.11974"
+        stroke="currentColor"
+        strokeWidth="2.16549"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M20.676 8.74649L16.8803 12.5423"
+        stroke="currentColor"
+        strokeWidth="2.16549"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/ExitIcon.tsx

@@ -0,0 +1,14 @@
+export function ExitIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <line x1="3.5" y1="8" x2="11.5" y2="8" stroke="currentColor" strokeLinecap="round" />
+      <line x1="15.5" y1="1.5" x2="15.5" y2="14.5" stroke="currentColor" strokeLinecap="round" />
+      <path
+        d="M8.5 4.5L12 8L8.5 11.5"
+        stroke="currentColor"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/IntegrationIcon.tsx

@@ -0,0 +1,5 @@
+import { LogoIcon } from "~/components/LogoIcon";
+
+export function IntegrationIcon() {
+  return <LogoIcon className="h-3.5 w-3.5 flex-none pb-0.5" />;
+}