Release 3.3.1 (#1513)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.changeset/config.json

@@ -1,24 +1,19 @@
 {
   "$schema": "https://unpkg.com/@changesets/config@2.2.0/schema.json",
-  "changelog": "@changesets/cli/changelog",
-  "commit": false,
-  "fixed": [
-    [
-      "@trigger.dev/*"
-    ]
+  "changelog": [
+    "@remix-run/changelog-github",
+    {
+      "repo": "triggerdotdev/trigger.dev"
+    }
   ],
+  "commit": false,
+  "fixed": [["@trigger.dev/*", "trigger.dev"]],
   "linked": [],
   "access": "public",
   "baseBranch": "main",
   "updateInternalDependencies": "patch",
-  "ignore": [
-    "webapp",
-    "emails",
-    "proxy",
-    "yalt",
-    "@trigger.dev/database"
-  ],
+  "ignore": ["webapp", "proxy", "coordinator", "docker-provider", "kubernetes-provider"],
   "___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH": {
     "onlyUpdatePeerDependentsWhenOutOfRange": true
   }
-}
+}

.configs/tsconfig.base.json

@@ -0,0 +1,36 @@
+{
+  "compilerOptions": {
+    "target": "es2022",
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "moduleDetection": "force",
+    "verbatimModuleSyntax": false,
+    "jsx": "react",
+
+    "strict": true,
+    "alwaysStrict": true,
+    "strictPropertyInitialization": false,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noImplicitAny": true,
+    "noImplicitReturns": true,
+    "noImplicitThis": true,
+
+    "noFallthroughCasesInSwitch": true,
+    "resolveJsonModule": true,
+
+    "removeComments": false,
+    "esModuleInterop": true,
+    "emitDecoratorMetadata": false,
+    "experimentalDecorators": false,
+    "downlevelIteration": true,
+    "isolatedModules": true,
+    "noUncheckedIndexedAccess": true,
+
+    "pretty": true,
+    "customConditions": ["@triggerdotdev/source"]
+  }
+}

.dockerignore

@@ -37,6 +37,7 @@ packages/\*\*/dist
 .vercel
 .cache
 .output
+.trigger
 apps/\*\*/public/build
 
 cypress/screenshots

.env.example

@@ -10,16 +10,23 @@ DIRECT_URL=${DATABASE_URL}
 REMIX_APP_PORT=3030
 APP_ENV=development
 APP_ORIGIN=http://localhost:3030
+ELECTRIC_ORIGIN=http://localhost:3060
 NODE_ENV=development
+V3_ENABLED=true
 
-# Redis is used for concurrency control
-# REDIS_HOST="localhost"
-# REDIS_PORT="6379"
-# REDIS_TLS_DISABLED="true"
+# Redis is used for the v3 queuing and v2 concurrency control
+REDIS_HOST="localhost"
+REDIS_PORT="6379"
+REDIS_TLS_DISABLED="true"
+
+DEV_OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:3030/otel"
+DEV_OTEL_BATCH_PROCESSING_ENABLED="0"
 
 # OPTIONAL VARIABLES
 # This is used for validating emails that are allowed to log in. Every email that do not match this regex will be rejected.
 # WHITELISTED_EMAILS="authorized@yahoo\.com|authorized@gmail\.com"
+# Accounts with these emails will get global admin rights. This grants access to the admin UI.
+# ADMIN_EMAILS="admin@example\.com|another-admin@example\.com"
 # This is used for logging in via GitHub. You can leave these commented out if you don't want to use GitHub for authentication.
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
@@ -31,6 +38,9 @@ NODE_ENV=development
 # FROM_EMAIL=
 # REPLY_TO_EMAIL=
 
+# Remove the following line to enable logging telemetry traces to the console
+LOG_TELEMETRY="false"
+
 # CLOUD VARIABLES
 POSTHOG_PROJECT_KEY=
 PLAIN_API_KEY= 
@@ -42,4 +52,31 @@ CLOUD_LINEAR_CLIENT_ID=
 CLOUD_LINEAR_CLIENT_SECRET=
 CLOUD_SLACK_APP_HOST=
 CLOUD_SLACK_CLIENT_ID=
-CLOUD_SLACK_CLIENT_SECRET= 
+CLOUD_SLACK_CLIENT_SECRET= 
+
+# v3 variables
+PROVIDER_SECRET=provider-secret # generate the actual secret with `openssl rand -hex 32`
+COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl rand -hex 32`
+
+# Uncomment the following line to enable the registry proxy
+# ENABLE_REGISTRY_PROXY=true
+# DEPOT_TOKEN=<Depot org token>
+# DEPOT_PROJECT_ID=<Depot project id>
+# DEPLOY_REGISTRY_HOST=${APP_ORIGIN} # This is the host that the deploy CLI will use to push images to the registry
+# CONTAINER_REGISTRY_ORIGIN=<Container registry origin e.g. https://registry.digitalocean.com>
+# CONTAINER_REGISTRY_USERNAME=<Container registry username e.g. Digital ocean email address>
+# CONTAINER_REGISTRY_PASSWORD=<Container registry password e.g. Digital ocean PAT>
+# DEV_OTEL_EXPORTER_OTLP_ENDPOINT="http://0.0.0.0:4318"
+# These are needed for the object store (for handling large payloads/outputs)
+# OBJECT_STORE_BASE_URL="https://{bucket}.{accountId}.r2.cloudflarestorage.com"
+# OBJECT_STORE_ACCESS_KEY_ID=
+# OBJECT_STORE_SECRET_ACCESS_KEY=
+# CHECKPOINT_THRESHOLD_IN_MS=10000
+
+# These control the server-side internal telemetry
+# INTERNAL_OTEL_TRACE_EXPORTER_URL=<URL to send traces to>
+# INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_NAME=<Header name for the auth token>
+# INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE=<Auth token value>
+# INTERNAL_OTEL_TRACE_LOGGING_ENABLED=1
+# INTERNAL_OTEL_TRACE_SAMPING_RATE=20 # this means 1/20 traces or 5% of traces will be sampled (sampled = recorded)
+# INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED=0,

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: [triggerdotdev]

.github/ISSUE_TEMPLATE/instrumentation_request.yml

@@ -0,0 +1,21 @@
+name: OpenTelemetry Auto-Instrumentation Request
+description: Suggest an SDK that you'd like to be auto-instrumented in the Run log view
+title: "auto-instrumentation: "
+labels: ["🌟 enhancement"]
+body:
+  - type: textarea
+    attributes:
+      label: What API or SDK would you to have automatic spans for?
+      description: A clear description of which API or SDK you'd like, and links to it.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Is there an existing OpenTelemetry auto-instrumentation package?
+      description: You can search for existing ones – https://opentelemetry.io/ecosystem/registry/?component=instrumentation&language=js
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Additional information
+      description: Add any other information related to the feature here. If your feature request is related to any issues or discussions, link them here.

.github/actions/get-image-tag/action.yml

@@ -0,0 +1,84 @@
+name: "#️⃣ Get image tag (action)"
+
+description: This action gets the image tag from the commit ref or input (if provided)
+
+outputs:
+  tag:
+    description: The image tag
+    value: ${{ steps.get_tag.outputs.tag }}
+  is_semver:
+    description: Whether the tag is a semantic version
+    value: ${{ steps.check_semver.outputs.is_semver }}
+
+inputs:
+  tag:
+    description: The image tag. If this is set it will return the tag as is.
+    required: false
+    default: ""
+
+runs:
+  using: "composite"
+  steps:
+    - name: "#️⃣ Get image tag (step)"
+      id: get_tag
+      shell: bash
+      run: |
+        if [[ -n "${{ inputs.tag }}" ]]; then
+          tag="${{ inputs.tag }}"
+        elif [[ "${{ github.ref_type }}" == "tag" ]]; then
+          if [[ "${{ github.ref_name }}" == infra-*-* ]]; then
+            env=$(echo ${{ github.ref_name }} | cut -d- -f2)
+            sha=$(echo ${{ github.sha }} | head -c7)
+            ts=$(date +%s)
+            tag=${env}-${sha}-${ts}
+          elif [[ "${{ github.ref_name }}" == v.docker.* ]]; then
+            version="${GITHUB_REF_NAME#v.docker.}"
+            tag="v${version}"
+          elif [[ "${{ github.ref_name }}" == build-* ]]; then
+            tag="${GITHUB_REF_NAME#build-}"
+          else
+            echo "Invalid git tag: ${{ github.ref_name }}"
+            exit 1
+          fi
+        elif [[ "${{ github.ref_name }}" == "main" ]]; then
+          tag="main"
+        else
+          echo "Invalid git ref: ${{ github.ref }}"
+          exit 1
+        fi
+        echo "tag=${tag}" >> "$GITHUB_OUTPUT"
+
+    - name: 🔍 Check for validity
+      id: check_validity
+      shell: bash
+      env:
+        tag: ${{ steps.get_tag.outputs.tag }}
+      run: |
+        if [[ "${tag}" =~ ^[a-z0-9]+([._-][a-z0-9]+)*$ ]]; then
+          echo "Tag is valid: ${tag}"
+        else
+          echo "Tag is not valid: ${tag}"
+          exit 1
+        fi
+
+    - name: 🆚 Check for semver
+      id: check_semver
+      shell: bash
+      env:
+        tag: ${{ steps.get_tag.outputs.tag }}
+      # Will match most semver formats except build metadata, i.e. v1.2.3+build.1
+      # Valid matches:
+      # v1.2.3
+      # v1.2.3-alpha
+      # v1.2.3-alpha.1
+      # v1.2.3-rc.1
+      # v1.2.3-beta-1
+      run: |
+        if [[ "${tag}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ ]]; then
+          echo "Tag is a semantic version: ${tag}"
+          is_semver=true
+        else
+          echo "Tag is not a semantic version: ${tag}"
+          is_semver=false
+        fi
+        echo "is_semver=${is_semver}" >> "$GITHUB_OUTPUT"

.github/copilot-instructions.md

@@ -0,0 +1 @@
+This is the repo for Trigger.dev, a background jobs platform written in TypeScript. Our webapp at apps/webapp is a Remix 2.1 app that uses Node.js v20. Our SDK is an isomorphic TypeScript SDK at packages/trigger-sdk. Always prefer using isomorphic code like fetch, ReadableStream, etc. instead of Node.js specific code. Our tests are all vitest. We use prisma in internal-packages/database for our database interactions using PostgreSQL. For TypeScript, we usually use types over interfaces. We use zod a lot in packages/core and in the webapp. Avoid enums. Use strict mode. No default exports, use function declarations.

.github/test/README.md

@@ -0,0 +1,70 @@
+# GitHub Action Tests
+
+This directory contains necessary files to allow local testing of GitHub Actions workflows, composite actions, etc. You will need to install [act](https://github.com/nektos/act) to perform tests.
+
+## Workflow tests
+
+Trigger specific workflow files by specifying their full path:
+
+```
+act -W .github/workflow/release.yml
+```
+
+You will likely need to override any custom runners we use, e.g. buildjet. For example:
+
+```
+override=catthehacker/ubuntu:act-latest
+
+act -W .github/workflow/release.yml \
+    -P buildjet-8vcpu-ubuntu-2204=$override
+
+# override multiple images at the same time
+act -W .github/workflow/release.yml \
+    -P buildjet-8vcpu-ubuntu-2204=$override \
+    -P buildjet-16vcpu-ubuntu-2204=$override
+```
+
+Trigger with specific event payloads to test pushing to branches or tags:
+
+```
+override=catthehacker/ubuntu:act-latest
+
+# simulate push to main
+act -W .github/workflow/publish.yml \
+    -P buildjet-8vcpu-ubuntu-2204=$override \
+    -P buildjet-16vcpu-ubuntu-2204=$override \
+    -e .github/events/push-tag-main.json
+
+# simulate a `build-` prefixed tag
+act -W .github/workflow/publish.yml \
+    -P buildjet-8vcpu-ubuntu-2204=$override \
+    -P buildjet-16vcpu-ubuntu-2204=$override \
+    -e .github/events/push-tag-buld.json
+```
+
+By default, `act` will send a push event. To trigger a different event:
+
+```
+# basic syntax
+act <EVENT> ...
+
+# simulate a pull request
+act pull_request
+
+# only trigger a specific workflow
+act pull_request -W .github/workflow/pr_checks.yml
+```
+
+## Composite action tests
+
+The composite (custom) action tests can be run by triggering the `test-actions` workflow:
+
+```
+act -W .github/test/test-actions.yml
+```
+
+## Helpful flags
+
+- `--pull=false` - perform fully offline tests if all images are already present
+- `-j <job_name>` - run the specified job only
+- `-l push` - list all workflows with push triggers

.github/test/events/push-main.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/heads/main"
+}

.github/test/events/push-tag-build.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/build-buildtag"
+}

.github/test/events/push-tag-docker-nonsemver.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/v.docker.nonsemver"
+}

.github/test/events/push-tag-docker.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/v.docker.1.2.3"
+}

.github/test/events/push-tag-infra-prod.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/infra-prod-anything"
+}

.github/test/events/push-tag-infra-test.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/infra-test-anything"
+}

.github/test/events/push-tag-semver.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/1.2.3"
+}

.github/test/events/push-tag.json

@@ -0,0 +1,3 @@
+{
+  "ref": "refs/tags/standard-tag"
+}

.github/test/test-actions.yml

@@ -0,0 +1,152 @@
+name: Test Actions
+
+on:
+  workflow_dispatch:
+
+jobs:
+  get-image-tag-none:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Log current ref
+        run: |
+          echo "ref: ${{ github.ref }}"
+          echo "ref_type: ${{ github.ref_type }}"
+          echo "ref_name: ${{ github.ref_name }}"
+
+      - name: Run without input tag
+        id: get_tag
+        # this step may fail depending on the current ref
+        continue-on-error: true
+        uses: ./.github/actions/get-image-tag
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-null:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Log current ref
+        run: |
+          echo "ref: ${{ github.ref }}"
+          echo "ref_type: ${{ github.ref_type }}"
+          echo "ref_name: ${{ github.ref_name }}"
+
+      - name: Run without input tag
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        # this step may fail depending on the current ref
+        continue-on-error: true
+        with:
+          # this should behave exactly as when no tag is provided
+          tag: null
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-override:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with tag override
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: "abc-123"
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-invalid-string:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with invalid string
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        # this step is expected to fail
+        continue-on-error: true
+        with:
+          # does not end with alphanumeric character
+          tag: "abc-123-"
+
+      - name: Fail job if previous step did not fail
+        if: steps.get_tag.outcome != 'failure'
+        run: exit 1
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-prerelease:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with prerelease semver
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: "v1.2.3-beta.4"
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-semver:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with basic semver
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: "v1.2.3"
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"
+
+  get-image-tag-invalid-semver:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run with invalid semver
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        # this step is expected to fail
+        continue-on-error: true
+        with:
+          tag: "v1.2.3-"
+
+      - name: Fail job if previous step did not fail
+        if: steps.get_tag.outcome != 'failure'
+        run: exit 1
+
+      - name: Verify output
+        run: |
+          echo "${{ toJson(steps.get_tag) }}"

.github/workflows/docs.yml

@@ -0,0 +1,42 @@
+name: 📚 Docs Checks
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "docs/**"
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - "docs/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  check-broken-links:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./docs
+    steps:
+      - name: 📥 Checkout repository
+        uses: actions/checkout@v4
+
+      - name: 📦 Cache npm
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.npm
+          key: |
+            ${{ runner.os }}-mintlify
+          restore-keys: |
+            ${{ runner.os }}-mintlify
+
+      - name: 🔗 Check for broken links
+        run: npx mintlify@4.0.222 broken-links

.github/workflows/e2e.yml

@@ -1,66 +1,53 @@
-name: "🧪 E2E Tests"
+name: "E2E"
+
 on:
   workflow_call:
-jobs:
-  e2e:
-    name: "🧪 E2E Tests"
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    steps:
-      - name: 🐳 Login to Docker Hub
-        if: github.event_name == 'push'
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
+    inputs:
+      package:
+        description: The identifier of the job to run
+        default: webapp
+        required: false
+        type: string
 
+jobs:
+  cli-v3:
+    name: "🧪 CLI v3 tests (${{ matrix.os }} - ${{ matrix.package-manager }})"
+    if: inputs.package == 'cli-v3' || inputs.package == ''
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [buildjet-8vcpu-ubuntu-2204, windows-latest]
+        package-manager: ["npm", "pnpm"]
+    steps:
       - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
         with:
-          version: 7.18
+          version: 8.15.5
 
       - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
+        uses: buildjet/setup-node@v4
         with:
-          node-version: 18
-          cache: "pnpm"
+          node-version: 20.11.1
 
       - name: 📥 Download deps
-        run: pnpm install --frozen-lockfile
+        run: pnpm install --frozen-lockfile --filter trigger.dev...
 
-      - name: Install Playwright Browsers
-        run: npx playwright install --with-deps
+      - name: 🔧 Build v3 cli monorepo dependencies
+        run: pnpm run build --filter trigger.dev^...
 
-      - name: Run Playwright tests
+      - name: 🔧 Build worker template files
+        run: pnpm --filter trigger.dev run build:workers
+
+      - name: Enable corepack
+        run: corepack enable
+
+      - name: Run E2E Tests
+        shell: bash
         run: |
-          # Setup environment variables
-          cp ./.env.example ./.env
-          cp ./references/nextjs-test/.env.example ./references/nextjs-test/.env.local
-
-          # Build packages
-          pnpm run build --filter @references/nextjs-test^...
-          pnpm --filter @trigger.dev/database generate
-
-          # Move trigger-cli bin to correct place
-          pnpm install --frozen-lockfile
-
-          # Execute tests
-          pnpm run docker
-          pnpm run db:migrate
-          pnpm run db:seed
-          pnpm run test:e2e
-
-          # Cleanup
-          pnpm run docker:stop
-
-      - name: Upload Playwright report
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: playwright-report
-          path: playwright-report/
-          retention-days: 30
+          LOG=debug PM=${{ matrix.package-manager }} pnpm --filter trigger.dev run test:e2e

.github/workflows/pr_checks.yml

@@ -2,13 +2,9 @@ name: 🤖 PR Checks
 
 on:
   pull_request:
-    branches:
-      - main
+    types: [opened, synchronize, reopened]
     paths-ignore:
-      - "**.md"
-      - "**.mdx"
-      - ".github/CODEOWNERS"
-      - ".github/ISSUE_TEMPLATE/**"
+      - "docs/**"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -29,4 +25,37 @@ jobs:
 
   e2e:
     uses: ./.github/workflows/e2e.yml
+    with:
+      package: cli-v3
     secrets: inherit
+
+  preview-release:
+    name: Preview Release
+    needs: [typecheck, units, e2e]
+    if: github.repository == 'triggerdotdev/trigger.dev'
+    runs-on: buildjet-8vcpu-ubuntu-2204
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 8.15.5
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.11.1
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 🏗️ Build
+        run: pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev"
+
+      - name: ⚡ Publish preview release
+        run: npx pkg-pr-new publish --no-template $(ls -d ./packages/*)

.github/workflows/publish-docker.yml

@@ -1,84 +0,0 @@
-name: "🐳 Publish Docker"
-on:
-  workflow_call:
-jobs:
-  publish:
-    strategy:
-      fail-fast: true # when a job fails, all remaining ones will be cancelled
-      matrix:
-        runs-on: [buildjet-4vcpu-ubuntu-2204, buildjet-4vcpu-ubuntu-2204-arm]
-    name: ${{matrix.runs-on}}
-    runs-on: ${{matrix.runs-on}}
-    outputs:
-      version: ${{ steps.get_version.outputs.version }}
-      short_sha: ${{ steps.get_commit.outputs.sha_short }}
-    steps:
-      - name: 🐳 Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
-
-      - name: 🆚 Get the version
-        id: get_version
-        run: |
-          IMAGE_TAG="${GITHUB_REF#refs/tags/}"
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-            if [[ $IMAGE_TAG == v.docker.* ]]; then
-              ORIGINAL_VERSION="${IMAGE_TAG#v.docker.}"
-              IMAGE_TAG="v${ORIGINAL_VERSION}"
-            elif [[ $IMAGE_TAG == build-* ]]; then
-              IMAGE_TAG="${IMAGE_TAG#build-}"
-            fi
-            echo "IMAGE_TAG=${IMAGE_TAG}"
-          elif [[ $GITHUB_REF == refs/heads/main ]]; then
-            # Handle main branch specifically
-            IMAGE_TAG="main"
-            echo "IMAGE_TAG=${IMAGE_TAG}"
-          else
-            echo "Invalid reference: ${GITHUB_REF}"
-            exit 1
-          fi
-          if [[ ${{matrix.runs-on}} == *-arm ]]; then
-            IMAGE_TAG="${IMAGE_TAG}-arm"
-          fi
-          echo "::set-output name=version::${IMAGE_TAG}"
-      - name: 🔢 Get the commit hash
-        id: get_commit
-        run: |
-          echo ::set-output name=sha_short::$(echo ${{ github.sha }} | cut -c1-7)
-
-      - name: 🐳 Build Docker Image
-        run: |
-          docker build -t release_build_image -f ./docker/Dockerfile .
-
-      - name: 🐙 Login to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: 🐙 Push to GitHub Container Registry
-        run: |
-          docker tag release_build_image $REGISTRY/$REPOSITORY:$IMAGE_TAG
-          docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
-        env:
-          REGISTRY: ghcr.io/triggerdotdev
-          REPOSITORY: trigger.dev
-          IMAGE_TAG: ${{ steps.get_version.outputs.version }}
-
-      - name: 🐙 Push 'latest' to GitHub Container Registry
-        if: startsWith(github.ref, 'refs/tags/v.docker')
-        run: |
-          LATEST=latest
-          if [[ ${{matrix.runs-on}} == *-arm ]]; then
-            LATEST="${LATEST}-arm"
-          fi
-          docker tag release_build_image $REGISTRY/$REPOSITORY:$LATEST
-          docker push $REGISTRY/$REPOSITORY:$LATEST
-        env:
-          REGISTRY: ghcr.io/triggerdotdev
-          REPOSITORY: trigger.dev

.github/workflows/publish-webapp.yml

@@ -0,0 +1,66 @@
+name: "🐳 Publish Webapp"
+
+on:
+  workflow_call:
+    inputs:
+      image_tag:
+        description: The image tag to publish
+        type: string
+        required: false
+        default: ""
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    env:
+      PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING: 1
+    outputs:
+      version: ${{ steps.get_tag.outputs.tag }}
+      short_sha: ${{ steps.get_commit.outputs.sha_short }}
+    steps:
+      - name: 🏭 Setup Depot CLI
+        uses: depot/setup-action@v1
+
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: "#️⃣ Get the image tag"
+        id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: ${{ inputs.image_tag }}
+
+      - name: 🔢 Get the commit hash
+        id: get_commit
+        run: |
+          echo "sha_short=$(echo ${{ github.sha }} | cut -c1-7)" >> "$GITHUB_OUTPUT"
+
+      - name: 📛 Set the tags
+        id: set_tags
+        run: |
+          ref_without_tag=ghcr.io/triggerdotdev/trigger.dev
+          image_tags=$ref_without_tag:${{ steps.get_tag.outputs.tag }}
+
+          # if tag is a semver, also tag it as v3
+          if [[ "${{ steps.get_tag.outputs.is_semver }}" == true ]]; then
+            image_tags=$image_tags,$ref_without_tag:v3
+          fi
+
+          echo "image_tags=${image_tags}" >> "$GITHUB_OUTPUT"
+
+      - name: 🐙 Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 🐳 Build image and push to GitHub Container Registry
+        uses: depot/build-push-action@v1
+        with:
+          file: ./docker/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.set_tags.outputs.image_tags }}
+          push: true

.github/workflows/publish-worker.yml

@@ -0,0 +1,87 @@
+name: "⚒️ Publish Worker"
+
+on:
+  workflow_call:
+    inputs:
+      image_tag:
+        description: The image tag to publish
+        type: string
+        required: false
+        default: ""
+  push:
+    tags:
+      - "infra-dev-*"
+      - "infra-test-*"
+      - "infra-prod-*"
+
+permissions:
+  packages: write
+  contents: read
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        package: [coordinator, docker-provider, kubernetes-provider]
+    runs-on: buildjet-16vcpu-ubuntu-2204
+    env:
+      DOCKER_BUILDKIT: "1"
+    steps:
+      - name: ⬇️ Checkout git repo
+        uses: actions/checkout@v4
+
+      - name: 📦 Get image repo
+        id: get_repository
+        run: |
+          if [[ "${{ matrix.package }}" == *-provider ]]; then
+            provider_type=$(echo "${{ matrix.package }}" | cut -d- -f1)
+            repo=provider/${provider_type}
+          else
+            repo="${{ matrix.package }}"
+          fi
+          echo "repo=${repo}" >> "$GITHUB_OUTPUT"
+
+      - id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: ${{ inputs.image_tag }}
+
+      - name: 🐋 Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # ..to avoid rate limits when pulling images
+      - name: 🐳 Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: 🚢 Build Container Image
+        run: |
+          docker build -t infra_image -f ./apps/${{ matrix.package }}/Containerfile .
+
+      # ..to push image
+      - name: 🐙 Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 🐙 Push to GitHub Container Registry
+        run: |
+          docker tag infra_image "$REGISTRY/$REPOSITORY:$IMAGE_TAG"
+          docker push "$REGISTRY/$REPOSITORY:$IMAGE_TAG"
+        env:
+          REGISTRY: ghcr.io/triggerdotdev
+          REPOSITORY: ${{ steps.get_repository.outputs.repo }}
+          IMAGE_TAG: ${{ steps.get_tag.outputs.tag }}
+
+      - name: 🐙 Push 'v3' tag to GitHub Container Registry
+        if: steps.get_tag.outputs.is_semver == 'true'
+        run: |
+          docker tag infra_image "$REGISTRY/$REPOSITORY:v3"
+          docker push "$REGISTRY/$REPOSITORY:v3"
+        env:
+          REGISTRY: ghcr.io/triggerdotdev
+          REPOSITORY: ${{ steps.get_repository.outputs.repo }}

.github/workflows/publish.yml

@@ -1,6 +1,12 @@
 name: 🚀 Publish Trigger.dev Docker
 
 on:
+  workflow_call:
+    inputs:
+      image_tag:
+        description: The image tag to publish
+        required: true
+        type: string
   push:
     branches:
       - main
@@ -8,7 +14,13 @@ on:
       - "v.docker.*"
       - "build-*"
     paths:
+      - ".github/actions/**/*.yml"
       - ".github/workflows/publish.yml"
+      - ".github/workflows/typecheck.yml"
+      - ".github/workflows/unit-tests.yml"
+      - ".github/workflows/e2e.yml"
+      - ".github/workflows/publish-webapp.yml"
+      - ".github/workflows/publish-worker.yml"
       - "packages/**"
       - "!packages/**/*.md"
       - "!packages/**/*.eslintrc"
@@ -45,11 +57,16 @@ jobs:
     uses: ./.github/workflows/unit-tests.yml
     secrets: inherit
 
-  e2e:
-    uses: ./.github/workflows/e2e.yml
+  publish-webapp:
+    needs: [typecheck, units]
+    uses: ./.github/workflows/publish-webapp.yml
     secrets: inherit
+    with:
+      image_tag: ${{ inputs.image_tag }}
 
-  publish:
-    needs: [typecheck, units, e2e]
-    uses: ./.github/workflows/publish-docker.yml
+  publish-worker:
+    needs: [typecheck, units]
+    uses: ./.github/workflows/publish-worker.yml
     secrets: inherit
+    with:
+      image_tag: ${{ inputs.image_tag }}

.github/workflows/release.yml

@@ -5,38 +5,39 @@ on:
     branches:
       - main
     paths-ignore:
+      - "docs/**"
       - "**.md"
       - ".github/CODEOWNERS"
       - ".github/ISSUE_TEMPLATE/**"
-    
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   release:
     name: 🦋 Changesets Release
-    runs-on: buildjet-4vcpu-ubuntu-2204
-    if: |
-      github.repository == 'triggerdotdev/trigger.dev'
+    runs-on: ubuntu-latest
+    if: github.repository == 'triggerdotdev/trigger.dev'
     outputs:
-      published_packages: ${{ steps.changesets.outputs.publishedPackages }}
       published: ${{ steps.changesets.outputs.published }}
+      published_packages: ${{ steps.changesets.outputs.publishedPackages }}
+      published_package_version: ${{ steps.get_version.outputs.package_version }}
     steps:
-      - name: 🛑 Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.11.0
-
       - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
         with:
-          version: 7.18
+          version: 8.15.5
 
       - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
+        uses: buildjet/setup-node@v4
         with:
-          node-version: 18
+          node-version: 20.11.1
           cache: "pnpm"
 
       - name: 📥 Download deps
@@ -45,8 +46,11 @@ jobs:
       - name: 📀 Generate Prisma Client
         run: pnpm run generate
 
+      - name: 🏗️ Build
+        run: pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev"
+
       - name: 🔎 Type check
-        run: pnpm run typecheck
+        run: pnpm run typecheck --filter "@trigger.dev/*" --filter "trigger.dev"
 
       - name: 🔐 Setup npm auth
         run: |
@@ -59,6 +63,7 @@ jobs:
         # PR is merged, the workflow will run again and this action will build +
         # publish to npm.
       - name: 🚀 PR / Publish
+        if: ${{ !env.ACT }}
         id: changesets
         uses: changesets/action@v1
         with:
@@ -66,7 +71,31 @@ jobs:
           commit: "chore: Update version for release"
           title: "chore: Update version for release"
           publish: pnpm run changeset:release
-          createGithubReleases: false
+          createGithubReleases: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+      # - name: 🚀 PR / Publish (mock)
+      #   if: ${{ env.ACT }}
+      #   id: changesets
+      #   run: |
+      #     echo "published=true" >> "$GITHUB_OUTPUT"
+      #     echo "publishedPackages=[{\"name\": \"@xx/xx\", \"version\": \"1.2.0\"}, {\"name\": \"@xx/xy\", \"version\": \"0.8.9\"}]" >> "$GITHUB_OUTPUT"
+
+      - name: 📦 Get package version
+        if: steps.changesets.outputs.published == 'true'
+        id: get_version
+        run: |
+          package_version=$(echo '${{ steps.changesets.outputs.publishedPackages }}' | jq -r '.[0].version')
+          echo "package_version=${package_version}" >> "$GITHUB_OUTPUT"
+
+  publish:
+    needs: release
+    uses: ./.github/workflows/publish.yml
+    secrets: inherit
+    # if: needs.release.outputs.published == 'true'
+    # disable automatic publishing for now
+    if: false
+    with:
+      image_tag: v${{ needs.release.outputs.published_package_version }}

.github/workflows/typecheck.yml

@@ -1,25 +1,27 @@
 name: "ʦ TypeScript"
+
 on:
   workflow_call:
+
 jobs:
   typecheck:
-    runs-on: buildjet-4vcpu-ubuntu-2204
+    runs-on: buildjet-8vcpu-ubuntu-2204
 
     steps:
       - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
         with:
-          version: 7.18
+          version: 8.15.5
 
       - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
+        uses: buildjet/setup-node@v4
         with:
-          node-version: 18
+          node-version: 20.11.1
           cache: "pnpm"
 
       - name: 📥 Download deps
@@ -29,4 +31,7 @@ jobs:
         run: pnpm run generate
 
       - name: 🔎 Type check
-        run: pnpm run typecheck --filter webapp
+        run: pnpm run typecheck
+
+      - name: 🔎 Check exports
+        run: pnpm run check-exports

.github/workflows/unit-tests.yml

@@ -1,40 +1,46 @@
 name: "🧪 Unit Tests"
+
 on:
   workflow_call:
+
 jobs:
   unitTests:
     name: "🧪 Unit Tests"
-    runs-on: buildjet-4vcpu-ubuntu-2204
+    runs-on: buildjet-16vcpu-ubuntu-2204
     steps:
       - name: ⬇️ Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
         with:
-          version: 7.18
+          version: 8.15.5
 
       - name: ⎔ Setup node
-        uses: buildjet/setup-node@v3
+        uses: buildjet/setup-node@v4
         with:
-          node-version: 18
+          node-version: 20.11.1
           cache: "pnpm"
 
-      - name: ⎔ Setup Deno
-        uses: denoland/setup-deno@v1
-        with:
-          deno-version: v1.x
-
-      - name: ⎔ Setup bun
-        uses: oven-sh/setup-bun@v1
-        with:
-          bun-version: "1.0.15"
-
       - name: 📥 Download deps
         run: pnpm install --frozen-lockfile
 
-      - name: Run Unit Tests
-        run: |
-          pnpm run test
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: 🧪 Run Webapp Unit Tests
+        run: pnpm run test --filter webapp
+        env:
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
+          DIRECT_URL: postgresql://postgres:postgres@localhost:5432/postgres
+          SESSION_SECRET: "secret"
+          MAGIC_LINK_SECRET: "secret"
+          ENCRYPTION_KEY: "secret"
+      
+      - name: 🧪 Run Package Unit Tests
+        run: pnpm run test --filter "@trigger.dev/*"
+      
+      - name: 🧪 Run Internal Unit Tests
+        run: pnpm run test --filter "@internal/*"

.gitignore

@@ -12,7 +12,6 @@ coverage
 # next.js
 .next/
 out/
-build
 dist
 packages/**/dist
 
@@ -53,4 +52,9 @@ apps/**/public/build
 /playwright-report/
 /playwright/.cache/
 
-.cosine
+.cosine
+.trigger
+.tshy*
+.yarn
+*.tsbuildinfo
+/packages/cli-v3/src/package.json

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "internal-packages/otlp-importer/protos"]
+	path = internal-packages/otlp-importer/protos
+	url = https://github.com/open-telemetry/opentelemetry-proto.git

.npmrc

@@ -1,2 +1,3 @@
 link-workspace-packages=false
-public-hoist-pattern[]=*prisma*
+public-hoist-pattern[]=*prisma*
+prefer-workspace-packages=true

.nvmrc

@@ -1 +1 @@
-v18.18.0
+v20.11.1

.prettierignore

@@ -7,5 +7,4 @@ tailwind.css
 **/.react-email/
 **/storybook-static/
 **/.changeset/
-**/build/
 **/dist/

.vscode/extensions.json

@@ -1,9 +1,4 @@
 {
-	"recommendations": [
-        "astro-build.astro-vscode",
-        "denoland.vscode-deno"
-    ],
-	"unwantedRecommendations": [
-		
-	]
-}
+  "recommendations": ["bierner.comment-tagged-templates"],
+  "unwantedRecommendations": []
+}

.vscode/launch.json

@@ -9,10 +9,19 @@
       "request": "launch",
       "name": "Debug WebApp",
       "command": "pnpm run dev --filter webapp",
-      "envFile": "${workspaceFolder}/apps/webapp/.env",
+      "envFile": "${workspaceFolder}/.env",
       "cwd": "${workspaceFolder}",
       "sourceMaps": true
     },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug realtimeStreams.test.ts",
+      "command": "pnpm run test -t RealtimeStreams",
+      "envFile": "${workspaceFolder}/.env",
+      "cwd": "${workspaceFolder}/apps/webapp",
+      "sourceMaps": true
+    },
     {
       "type": "chrome",
       "request": "launch",
@@ -23,10 +32,89 @@
     {
       "type": "node-terminal",
       "request": "launch",
-      "name": "Debug BYO Auth",
-      "command": "pnpm run byo-auth",
-      "envFile": "${workspaceFolder}/references/job-catalog/.env",
-      "cwd": "${workspaceFolder}/references/job-catalog",
+      "name": "Debug V3 init CLI",
+      "command": "pnpm exec trigger init",
+      "cwd": "${workspaceFolder}/references/init-shell",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 Dev CLI",
+      "command": "pnpm exec trigger dev",
+      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug Dev Next.js Realtime",
+      "command": "pnpm exec trigger dev",
+      "cwd": "${workspaceFolder}/references/nextjs-realtime",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug prisma-catalog deploy CLI",
+      "command": "pnpm exec trigger deploy --self-hosted --load-image",
+      "cwd": "${workspaceFolder}/references/prisma-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 Deploy CLI",
+      "command": "pnpm exec trigger deploy --self-hosted --load-image",
+      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 list-profiles CLI",
+      "command": "pnpm exec trigger list-profiles --log-level debug",
+      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 update CLI",
+      "command": "pnpm exec trigger update",
+      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 Management",
+      "command": "pnpm run management",
+      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node",
+      "request": "attach",
+      "name": "Attach to Trigger.dev CLI (v3)",
+      "port": 9229,
+      "restart": true,
+      "skipFiles": ["<node_internals>/**"]
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug CLI e2e tests",
+      "command": "MOD=otel-telemetry-loader pnpm run test:e2e",
+      "cwd": "${workspaceFolder}/packages/cli-v3",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "debug v3 hello-world dev",
+      "command": "pnpm exec trigger dev",
+      "cwd": "${workspaceFolder}/references/hello-world",
       "sourceMaps": true
     }
   ]

.vscode/settings.json

@@ -1,4 +1,9 @@
 {
   "deno.enablePaths": ["references/deno-reference", "runtime_tests/tests/deno"],
-  "debug.toolBarLocation": "commandCenter"
+  "debug.toolBarLocation": "commandCenter",
+  "typescript.tsdk": "node_modules/typescript/lib",
+  "search.exclude": {
+    "**/node_modules/**": true,
+    "packages/cli-v3/e2e": true
+  }
 }

CHANGESETS.md

@@ -47,7 +47,7 @@ pnpm exec changeset version --snapshot prerelease
 3. Build the packages:
 
 ```sh
-pnpm run build --filter "@trigger.dev/*"
+pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev"
 ```
 
 4. Publish the snapshot (replace "dev" with your tag)

CONTRIBUTING.md

@@ -10,13 +10,14 @@ Thank you for helping us make Trigger.dev even better! 🤩
 
 The development branch is `main`. This is the branch that all pull
 requests should be made against. The changes on the `main`
-branch are tagged into a release monthly.
+branch are tagged into a release periodically.
 
 ### Prerequisites
 
-- [Node.js](https://nodejs.org/en) version >=16.x
-- [pnpm package manager](https://pnpm.io/installation) version 7
+- [Node.js](https://nodejs.org/en) version 20.11.1
+- [pnpm package manager](https://pnpm.io/installation) version 8.15.5
 - [Docker](https://www.docker.com/get-started/)
+- [protobuf](https://github.com/protocolbuffers/protobuf)
 
 ### Setup
 
@@ -33,15 +34,19 @@ branch are tagged into a release monthly.
    ```
    cd trigger.dev
    ```
-3. Install the required packages using pnpm.
+3. Ensure you are on the correct version of Node.js (20.11.1). If you are using `nvm`, there is an `.nvmrc` file that will automatically select the correct version of Node.js when you navigate to the repository.
+
+4. Run `corepack enable` to use the correct version of pnpm (`8.15.5`) as specified in the root `package.json` file.
+
+5. Install the required packages using pnpm.
    ```
    pnpm i
    ```
-4. Create your `.env` file
+6. Create your `.env` file
    ```
    cp .env.example .env
    ```
-5. Open it and generate a new value for `ENCRYPTION_KEY`:
+7. Open it and generate a new value for `ENCRYPTION_KEY`:
 
    `ENCRYPTION_KEY` is used to two-way encrypt OAuth access tokens and so you'll probably want to actually generate a unique value, and it must be a random 16 byte hex string. You can generate one with the following command:
 
@@ -51,7 +56,7 @@ branch are tagged into a release monthly.
 
    Feel free to update `SESSION_SECRET` and `MAGIC_LINK_SECRET` as well using the same method.
 
-6. Start Docker. This starts the required services like Postgres. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)
+8. Start Docker. This starts the required services like Postgres & Redis. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)
 
    ```
    pnpm run docker
@@ -59,19 +64,15 @@ branch are tagged into a release monthly.
 
    This will also start and run a local instance of [pgAdmin](https://www.pgadmin.org/) on [localhost:5480](http://localhost:5480), preconfigured with email `admin@example.com` and pwd `admin`. Then use `postgres` as the password to the Trigger.dev server.
 
-7. Migrate the database
+9. Migrate the database
    ```
    pnpm run db:migrate
    ```
-8. Build the app
-   ```
-   pnpm run build --filter webapp
-   ```
-9. Run the seed script
-   ```
-   pnpm run db:seed
-   ```
-10. Run the app. See the section below.
+10. Build the server app
+    ```
+    pnpm run build --filter webapp
+    ```
+11. Run the app. See the section below.
 
 ## Running
 
@@ -83,99 +84,96 @@ branch are tagged into a release monthly.
 
    It should run on port `3030`: [http://localhost:3030](http://localhost:3030/)
 
-2. Once the app is running click the magic link button and enter your email.
-3. Check your terminal, the magic link email should have printed out as following:
+2. Once the app is running click the magic link button and enter your email. You will automatically be logged in, since you are running locally. Create an Org and your first project in the dashboard.
 
-   ```sh
-   webapp:dev: Log in to Trigger.dev
-   webapp:dev:
-   webapp:dev: Click here to log in with this magic link
-   webapp:dev: [http://localhost:3030/magic?token=U2FsdGVkX18OvB0JxgaswTLCSbaRz%2FY82TN0EZWhSzFyZYwgG%2BIzKVTkeiaOtWfotPw7F8RwFzCHh53aBpMEu%2B%2B%2FItb%2FcJYh89MSjc3Pz92bevoEjqxSQ%2Ff%2BZbks09JOpqlBbYC3FzGWC8vuSVFBlxqLXxteSDLthZSUaC%2BS2LaA%2BJgp%2BLO7hgjAaC2lXbCHrM7MTgTdXOFt7i0Dvvuwz6%2BWY25RnfomZOPqDsyH0xz8Q2rzPTz0Xu53WSXrZ1hd]
-   webapp:dev:
-   webapp:dev: If you didn't try to log in, you can safely ignore this email.
-   ```
+## Manual testing using v3-catalog
 
-   Paste the magic link shown in your terminal into your browser to login.
+We use the `<root>/references/v3-catalog` subdirectory as a staging ground for testing changes to the SDK (`@trigger.dev/sdk` at `<root>/packages/trigger-sdk`), the Core package (`@trigger.dev/core` at `<root>packages/core`), the CLI (`trigger.dev` at `<root>/packages/cli-v3`) and the platform (The remix app at `<root>/apps/webapp`). The instructions below will get you started on using the `v3-catalog` for local development of Trigger.dev (v3).
 
-## Adding and running migrations
+### First-time setup
 
-1. Modify packages/database/prisma/schema.prisma file
-2. Change directory to the packages/database folder
-   ```sh
-   cd packages/database
-   ```
-3. Generate the Prisma client
+First, make sure you are running the webapp according to the instructions above. Then:
 
-   ```sh
-   pnpm run generate
-   ```
+1. In Postgres go to the "Organizations" table and on your org set the `v3Enabled` column to `true`.
 
-   The above updates the prisma client generated into node_modules/.prisma/client folder. This helps with typing of relevant prisma models. It ensures typescript
-   recognizes fields added or removed from a model and type-checks appropriately.
+2. Visit http://localhost:3030 in your browser and create a new V3 project called "v3-catalog". If you don't see an option for V3, you haven't set the `v3Enabled` flag to true.
 
-4. Create and apply the migrations
+3. In Postgres go to the "Projects" table and for the project you create change the `externalRef` to `yubjwjsfkxnylobaqvqz`.
 
-   ```
-   pnpm run db:migrate:dev
-   ```
-
-   This creates a migration file and executes the migrations against your database and applies changes to the database schema(s)
-
-5. Commit generated migrations as well as changes to the schema.prisma file
-6. If you're using VSCode you may need to restart the Typescript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`.
-
-## Testing CLI changes
-
-To test CLI changes, follow the steps below:
-
-1. Build the CLI and watch for changes
+4. Build the CLI
 
 ```sh
-cd packages/cli
-pnpm run dev
-```
-
-2. Open a new Terminal window and run the webapp locally and then create a new project in the dashboard. Copy out the dev API key.
-
-3. Create a new temporary Next.js app in references directory
-
-```sh
-cd ./references
-pnpm create next-app@latest test-cli --ts --no-eslint --tailwind --app --src-dir --import-alias "@/*"
-```
-
-4. Then once that's finished, add the `@trigger.dev/cli` to the `devDependencies` of the newly created Next.js app's `package.json` file, like so:
-
-```json
-{
-  // other package.json properties
-  "devDependencies": { "@trigger.dev/cli": "workspace:*" }
-}
-```
-
-5. Back in the terminal, navigate into the reference, and initialize the CLI. When prompted, select `self-hosted` and enter `localhost:3030` if you are testing against the local instance of Trigger.dev, or you can just use the Trigger.dev cloud. When asked for an API key, use the key you copied earlier.
-
-```sh
-cd ./test-cli
+# Build the CLI
+pnpm run build --filter trigger.dev
+# Make it accessible to `pnpm exec`
 pnpm i
-pnpm exec trigger-cli init
 ```
 
-6. If you are just testing the `init` command, you can stop here. If you'd like to test the `dev` command, first start the Next.js app on port 3000:
+5. Change into the `<root>/references/v3-catalog` directory and authorize the CLI to the local server:
 
 ```sh
-pnpm run dev
+cd references/v3-catalog
+cp .env.example .env
+pnpm exec trigger login -a http://localhost:3030
 ```
 
-7. Open a new terminal window, and then run the `dev` command like so:
+This will open a new browser window and authorize the CLI against your local user account.
+
+You can optionally pass a `--profile` flag to the `login` command, which will allow you to use the CLI with separate accounts/servers. We suggest using a profile called `local` for your local development:
 
 ```sh
-pnpm exec trigger-cli dev
+cd references/v3-catalog
+pnpm exec trigger login -a http://localhost:3030 --profile local
+# later when you run the dev or deploy command:
+pnpm exec trigger dev --profile local
+pnpm exec trigger deploy --profile local
 ```
 
-8. Please remember to delete the temporary project you created after you've tested the changes, and before you raise a PR.
+### Running
 
-## Running end-to-end webapp tests
+The following steps should be followed any time you start working on a new feature you want to test in v3:
+
+1. Make sure the webapp is running on localhost:3030
+
+2. Open a terminal window and build the CLI and watch for changes
+
+```sh
+pnpm run dev --filter trigger.dev
+```
+
+2. Open a new terminal window, and anytime changes are made to the `@trigger.dev/core` package, you'll need to manually rebuild the CLI:
+
+```sh
+pnpm run build --filter trigger.dev
+```
+
+Note: You do not need to do the same for `@trigger.dev/sdk`, just core.
+
+3. Open another terminal window, and change into the `<root>/references/v3-catalog` directory.
+
+4. Run the `dev` command, which will register all the local tasks with the platform and allow you to start testing task execution:
+
+```sh
+# in <root>/references/v3-catalog
+pnpm exec trigger dev
+```
+
+If you want additional debug logging, you can use the `--log-level debug` flag:
+
+```sh
+# in <root>/references/v3-catalog
+pnpm exec trigger dev --log-level debug
+```
+
+5. If you make any changes in the CLI/Core/SDK, you'll need to `CTRL+C` to exit the `dev` command and restart it to pickup changes. Any changes to the files inside of the `v3-catalog/src/trigger` dir will automatically be rebuilt by the `dev` command.
+
+6. Navigate to the `v3-catalog` project in your local dashboard at localhost:3030 and you should see the list of tasks.
+
+7. Go to the "Test" page in the sidebar and select a task. Then enter a payload and click "Run test". You can tell what the payloads should be by looking at the relevant task file inside the `/references/v3-catalog/src/trigger` folder. Many of them accept an empty payload.
+
+8. Feel free to add additional files in `v3-catalog/src/trigger` to test out specific aspects of the system, or add in edge cases.
+
+## Running end-to-end webapp tests (deprecated)
 
 To run the end-to-end tests, follow the steps below:
 
@@ -223,6 +221,26 @@ The end-to-end tests use a `setup` and `teardown` script to seed the database wi
 pnpm run db:studio
 ```
 
+## Adding and running migrations
+
+1. Modify internal-packages/database/prisma/schema.prisma file
+2. Change directory to the packages/database folder
+
+   ```sh
+   cd packages/database
+   ```
+
+3. Create and apply the migrations
+
+   ```
+   pnpm run db:migrate:dev
+   ```
+
+   This creates a migration file and executes the migrations against your database and applies changes to the database schema(s)
+
+4. Commit generated migrations as well as changes to the schema.prisma file
+5. If you're using VSCode you may need to restart the Typescript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`.
+
 ## Add sample jobs
 
 The [references/job-catalog](./references/job-catalog/) project defines simple jobs you can get started with.

README.md

@@ -5,86 +5,86 @@
   <img alt="Trigger.dev logo" src="https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/a45d1fa2-0ae8-4a39-4409-f4f934bfae00/public">
 </picture>
   
-### The open source background jobs framework
+### Open source background jobs with no timeouts
 
-[Discord](https://discord.gg/JtBAxBr2m3) | [Website](https://trigger.dev) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Docs](https://trigger.dev/docs)
+[Discord](https://trigger.dev/discord) | [Website](https://trigger.dev) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Docs](https://trigger.dev/docs)
 
 [![Twitter](https://img.shields.io/twitter/url/https/twitter.com/triggerdotdev.svg?style=social&label=Follow%20%40trigger.dev)](https://twitter.com/triggerdotdev)
-[![GitHub Repo stars](https://img.shields.io/github/stars/triggerdotdev/trigger.dev?style=social)](https://github.com/triggerdotdev/trigger.dev)
 
 </div>
 
-# About Trigger.dev
+## About Trigger.dev
 
-Create long-running jobs directly in your codebase with features like API integrations, webhooks, scheduling and delays.
+Trigger.dev is an open source platform and SDK which allows you to create long-running background jobs. Write normal async code, deploy, and never hit a timeout.
 
-## Long running Jobs on serverless
+### Key features:
 
-Reliably run jobs and don’t worry about function timeouts, we handle those for you.
-
-- Auto-resume after a function timeout
-- Auto-resume after a server outage
-- Add delays of up to a year
+- JavaScript and TypeScript SDK
+- Write reliable code by default
+- No timeouts
+- Retries (with exponential backoff)
+- Queues and concurrency controls
+- Schedules and crons
+- Full Observability; logs, live trace views, advanced filtering
+- Custom alerts, get notified by email, Slack or webhooks
+- No infrastructure to manage
+- Elastic (scaling)
+- Works with your existing tech stack
 
 ## In your codebase
 
-Create Jobs where they belong: in your codebase. Version control, localhost, test, review, and deploy like you're already used to.
+Create tasks where they belong: in your codebase. Version control, localhost, test and review like you're already used to.
 
-## Secure by design
+```ts
+import { task } from "@trigger.dev/sdk/v3";
 
-We only receive Triggers and the data you choose to send to us. You can even completely self-host the entire platform.
+//1. You need to export each task
+export const helloWorld = task({
+  //2. Use a unique id for each task
+  id: "hello-world",
+  //3. The run function is the main function of the task
+  run: async (payload: { message: string }) => {
+    //4. You can write code that runs for a long time here, there are no timeouts
+    console.log(payload.message);
+  },
+});
+```
 
-## Don't worry about deployment
+## Deployment
 
-Just use our SDK to write Jobs in your codebase. There's nothing extra to deploy and no CI to configure, your Jobs just connect to our cloud. Or you can always self-host.
+Use our SDK to write tasks in your codebase. There's no infrastructure to manage, your tasks automatically scale and connect to our cloud. Or you can always self-host.
+
+## Environments
+
+We support `Development`, `Staging`, and `Production` environments, allowing you to test your tasks before deploying them to production.
 
 ## Full visibility of every job run
 
-View every Task in every Run so you can tell exactly what happened.
+View every task in every run so you can tell exactly what happened. We provide a full trace view of every task run so you can see what happened at every step.
 
-![image](https://www.trigger.dev/build/_assets/web-app-2QFKXFLW.png)
-
-## Built-in integrations
-
-Easily integrate with hundreds of third-party APIs – including your own. Use API keys (which never leave your server) or let us handle OAuth for you. Install our integration packages and easily subscribe to webhooks and perform common tasks, or you can easily use your existing favorite Node.JS SDKs and get resumability and idempotency through our `runTask` function.
-
-## Our progress
-
-We’re building the most comprehensive and easy-to-use background jobs framework for developers.
-
-Click the links to join the discussions about our upcoming features.
-
-| Feature                                                                              | What it does                                        | Status |
-| ------------------------------------------------------------------------------------ | --------------------------------------------------- | ------ |
-| Integration kit                                                                      | Official Trigger.dev integrations or build your own | ✅     |
-| Self-hosting                                                                         | Host the platform yourself                          | ✅     |
-| Cloud                                                                                | Just write code, no deployment required             | ✅     |
-| Dashboard                                                                            | View every Task in every Run                        | ✅     |
-| Serverless                                                                           | Long-running Jobs on your serverless backend        | ✅     |
-| React hooks                                                                          | Easily update your UI with Job progress             | ✅     |
-| React frameworks                                                                     | Support for Remix, Astro, RedwoodJS & more          | ✅     |
-| [Background tasks](https://github.com/triggerdotdev/trigger.dev/discussions/400)     | Offload long or intense Tasks to our infrastructure | 🛠️     |
-| [Long-running servers](https://github.com/triggerdotdev/trigger.dev/discussions/430) | Run Jobs on your long-running backend               | 🛠️     |
-| Polling Triggers                                                                     | Subscribe to changes without webhooks               | 🕝     |
-| Vercel integration                                                                   | Easy deploy and preview environment support         | 🕝     |
-| Streaming                                                                            | Receive data from your Jobs in realtime             | 🕝     |
-| 100+ integrations                                                                    | Comprehensive support for popular APIs              | 🕝     |
-| [Trigger.dev Connect](https://github.com/triggerdotdev/trigger.dev/discussions/441)  | Use integrations signed in as your users            | 🕝     |
-| File IO                                                                              | Create Tasks that have file outputs                 | 🕝     |
+![Trace view image](https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/7c1b347f-004c-4482-38a7-3f6fa9c00d00/public)
 
 # Getting started
 
-Visit our docs [here](https://trigger.dev/docs).
+The quickest way to get started is to create an account and project in our [web app](https://cloud.trigger.dev), and follow the instructions in the onboarding. Build and deploy your first task in minutes.
 
-## Self-host
+### Useful links:
 
-We provide an official trigger.dev docker image you can use to easily self-host the platform. We're working on more extensive guides but we currently provide a [Fly.io example repository](https://github.com/triggerdotdev/fly.io) with instructions in the README for deploying and using a self-hosted instance of Trigger.dev on Fly.io.
+- [Quick start](https://trigger.dev/docs/quick-start) - get up and running in minutes
+- [How it works](https://trigger.dev/docs/v3/how-it-works) - understand how Trigger.dev works under the hood
+- [Guides and examples](https://trigger.dev/docs/guides/introduction) - walk-through guides and code examples for popular frameworks and use cases
+
+## Self-hosting
+
+If you prefer to self-host Trigger.dev, you can follow our [self-hosting guide](https://trigger.dev/docs/v3/open-source-self-hosting#overview).
+
+We also have a dedicated self-hosting channel in our [Discord server](https://trigger.dev/discord) for support.
 
 ## Development
 
 To setup and develop locally or contribute to the open source project, follow our [development guide](./CONTRIBUTING.md).
 
-## Meet the Amazing People Behind This Project 🚀
+## Meet the Amazing People Behind This Project:
 
 <a href="https://github.com/triggerdotdev/trigger.dev/graphs/contributors">
   <img src="https://contrib.rocks/image?repo=triggerdotdev/trigger.dev" />

RELEASE.md

@@ -0,0 +1,13 @@
+## Guide on releasing a new version
+
+1. Merge in the changeset PR into main, making sure to cancel both the release and publish github actions from that merge.
+2. Pull the changes locally into main
+3. Run `pnpm i` which will update the pnpm lock file with the new versions
+4. create a commit with "Release 3.x.x" and push. This will build and release the packages
+5. Create a git tag on that release commit with v.docker.3.x.x and push the tag to origin. This will publish the `v3.x.x` docker image to GitHub Container Registry.
+6. Once the image is built and pushed, create a new GitHub release and select the tag you just created, along with the previous tag that was released.
+7. This will generate some release notes. Edit out the package changes and leave only the server changes.
+8. Name the release `@trigger.dev/docker@3.x.x`
+9. Include the package link (e.g. https://github.com/triggerdotdev/trigger.dev/pkgs/container/trigger.dev/278459584?tag=v3.x.x)
+10. Once the packages have been published, head over to the [v2-legacy repo](https://github.com/triggerdotdev/v2-legacy.trigger.dev) and follow the instructions in the README for creating a matching release.
+11. Before deploying to cloud, compare the differences in the previously created release and double check to see if there are any migrations with indexes created concurrently, and make sure to run those before deploying.

apps/coordinator/.env.example

@@ -0,0 +1,4 @@
+HTTP_SERVER_PORT=8020
+PLATFORM_ENABLED=true
+PLATFORM_WS_PORT=3030
+SECURE_CONNECTION=false

apps/coordinator/.gitignore

@@ -0,0 +1,3 @@
+dist/
+node_modules/
+.env

apps/coordinator/Containerfile

@@ -0,0 +1,60 @@
+# syntax=docker/dockerfile:labs
+
+FROM node:20-bookworm-slim@sha256:72f2f046a5f8468db28730b990b37de63ce93fd1a72a40f531d6aa82afdf0d46 AS node-20
+
+WORKDIR /app
+
+FROM node-20 AS pruner
+
+COPY --chown=node:node . .
+RUN npx -q turbo@1.10.9 prune --scope=coordinator --docker
+RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
+
+FROM node-20 AS base
+
+RUN apt-get update \
+  && apt-get install -y buildah ca-certificates dumb-init docker.io \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY --chown=node:node .gitignore .gitignore
+COPY --from=pruner --chown=node:node /app/out/json/ .
+COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
+COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
+
+FROM base AS dev-deps
+RUN corepack enable
+ENV NODE_ENV development
+
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
+
+FROM base AS builder
+RUN corepack enable
+
+COPY --from=pruner --chown=node:node /app/out/full/ .
+COPY --from=dev-deps --chown=node:node /app/ .
+COPY --chown=node:node turbo.json turbo.json
+
+RUN pnpm run -r --filter coordinator build:bundle
+
+FROM alpine AS cri-tools
+
+WORKDIR /cri-tools
+
+ARG CRICTL_VERSION=v1.29.0
+ARG CRICTL_CHECKSUM=sha256:d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb
+ADD --checksum=${CRICTL_CHECKSUM} \
+  https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz .
+RUN tar zxvf crictl-${CRICTL_VERSION}-linux-amd64.tar.gz
+
+FROM base AS runner
+
+RUN corepack enable
+ENV NODE_ENV production
+
+COPY --from=cri-tools --chown=node:node /cri-tools/crictl /usr/local/bin
+COPY --from=builder --chown=node:node /app/apps/coordinator/dist/index.mjs ./index.mjs
+
+EXPOSE 8000
+
+CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]

apps/coordinator/README.md

@@ -0,0 +1,3 @@
+# Coordinator
+
+Sits between the platform and tasks. Facilitates communication and checkpointing, amongst other things.

apps/coordinator/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "coordinator",
+  "private": true,
+  "version": "0.0.1",
+  "description": "",
+  "main": "dist/index.cjs",
+  "scripts": {
+    "build": "npm run build:bundle",
+    "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"",
+    "build:image": "docker build -f Containerfile . -t coordinator",
+    "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
+    "start": "tsx src/index.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@trigger.dev/core": "workspace:*",
+    "nanoid": "^5.0.6",
+    "prom-client": "^15.1.0",
+    "socket.io": "4.7.4",
+    "tinyexec": "^0.3.0"
+  },
+  "devDependencies": {
+    "@types/node": "^18",
+    "dotenv": "^16.4.2",
+    "esbuild": "^0.19.11",
+    "tsx": "^4.7.0",
+    "typescript": "^5.3.3"
+  }
+}

apps/coordinator/src/chaosMonkey.ts

@@ -0,0 +1,88 @@
+import { setTimeout as timeout } from "node:timers/promises";
+
+class ChaosMonkeyError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ChaosMonkeyError";
+  }
+}
+
+export class ChaosMonkey {
+  private chaosEventRate = 0.2;
+  private delayInSeconds = 45;
+
+  constructor(
+    private enabled = false,
+    private disableErrors = false,
+    private disableDelays = false
+  ) {
+    if (this.enabled) {
+      console.log("🍌 Chaos monkey enabled");
+    }
+  }
+
+  static Error = ChaosMonkeyError;
+
+  enable() {
+    this.enabled = true;
+    console.log("🍌 Chaos monkey enabled");
+  }
+
+  disable() {
+    this.enabled = false;
+    console.log("🍌 Chaos monkey disabled");
+  }
+
+  async call({
+    throwErrors = !this.disableErrors,
+    addDelays = !this.disableDelays,
+  }: {
+    throwErrors?: boolean;
+    addDelays?: boolean;
+  } = {}) {
+    if (!this.enabled) {
+      return;
+    }
+
+    const random = Math.random();
+
+    if (random > this.chaosEventRate) {
+      // Don't interfere with normal operation
+      return;
+    }
+
+    const chaosEvents: Array<() => Promise<any>> = [];
+
+    if (addDelays) {
+      chaosEvents.push(async () => {
+        console.log("🍌 Chaos monkey: Add delay");
+
+        await timeout(this.delayInSeconds * 1000);
+      });
+    }
+
+    if (throwErrors) {
+      chaosEvents.push(async () => {
+        console.log("🍌 Chaos monkey: Throw error");
+
+        throw new ChaosMonkey.Error("🍌 Chaos monkey: Throw error");
+      });
+    }
+
+    if (chaosEvents.length === 0) {
+      console.error("🍌 Chaos monkey: No events selected");
+      return;
+    }
+
+    const randomIndex = Math.floor(Math.random() * chaosEvents.length);
+
+    const chaosEvent = chaosEvents[randomIndex];
+
+    if (!chaosEvent) {
+      console.error("🍌 Chaos monkey: No event found");
+      return;
+    }
+
+    await chaosEvent();
+  }
+}

apps/coordinator/src/checkpointer.ts

@@ -0,0 +1,671 @@
+import { ExponentialBackoff } from "@trigger.dev/core/v3/apps";
+import { testDockerCheckpoint } from "@trigger.dev/core/v3/apps";
+import { nanoid } from "nanoid";
+import fs from "node:fs/promises";
+import { ChaosMonkey } from "./chaosMonkey";
+import { Buildah, Crictl, Exec } from "./exec";
+import { setTimeout } from "node:timers/promises";
+import { TempFileCleaner } from "./cleaner";
+import { numFromEnv, boolFromEnv } from "./util";
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+
+type CheckpointerInitializeReturn = {
+  canCheckpoint: boolean;
+  willSimulate: boolean;
+};
+
+type CheckpointAndPushOptions = {
+  runId: string;
+  leaveRunning?: boolean;
+  projectRef: string;
+  deploymentVersion: string;
+  shouldHeartbeat?: boolean;
+  attemptNumber?: number;
+};
+
+type CheckpointAndPushResult =
+  | { success: true; checkpoint: CheckpointData }
+  | {
+      success: false;
+      reason?: "CANCELED" | "DISABLED" | "ERROR" | "IN_PROGRESS" | "NO_SUPPORT" | "SKIP_RETRYING";
+    };
+
+type CheckpointData = {
+  location: string;
+  docker: boolean;
+};
+
+type CheckpointerOptions = {
+  dockerMode: boolean;
+  forceSimulate: boolean;
+  heartbeat: (runId: string) => void;
+  registryHost?: string;
+  registryNamespace?: string;
+  registryTlsVerify?: boolean;
+  disableCheckpointSupport?: boolean;
+  checkpointPath?: string;
+  simulateCheckpointFailure?: boolean;
+  simulateCheckpointFailureSeconds?: number;
+  simulatePushFailure?: boolean;
+  simulatePushFailureSeconds?: number;
+  chaosMonkey?: ChaosMonkey;
+};
+
+async function getFileSize(filePath: string): Promise<number> {
+  try {
+    const stats = await fs.stat(filePath);
+    return stats.size;
+  } catch (error) {
+    console.error("Error getting file size:", error);
+    return -1;
+  }
+}
+
+async function getParsedFileSize(filePath: string) {
+  const sizeInBytes = await getFileSize(filePath);
+
+  let message = `Size in bytes: ${sizeInBytes}`;
+
+  if (sizeInBytes > 1024 * 1024) {
+    const sizeInMB = (sizeInBytes / 1024 / 1024).toFixed(2);
+    message = `Size in MB (rounded): ${sizeInMB}`;
+  } else if (sizeInBytes > 1024) {
+    const sizeInKB = (sizeInBytes / 1024).toFixed(2);
+    message = `Size in KB (rounded): ${sizeInKB}`;
+  }
+
+  return {
+    path: filePath,
+    sizeInBytes,
+    message,
+  };
+}
+
+export class Checkpointer {
+  #initialized = false;
+  #canCheckpoint = false;
+  #dockerMode: boolean;
+
+  #logger = new SimpleStructuredLogger("checkpointer");
+  #abortControllers = new Map<string, AbortController>();
+  #failedCheckpoints = new Map<string, unknown>();
+  #waitingForRetry = new Set<string>();
+
+  private registryHost: string;
+  private registryNamespace: string;
+  private registryTlsVerify: boolean;
+
+  private disableCheckpointSupport: boolean;
+
+  private simulateCheckpointFailure: boolean;
+  private simulateCheckpointFailureSeconds: number;
+  private simulatePushFailure: boolean;
+  private simulatePushFailureSeconds: number;
+
+  private chaosMonkey: ChaosMonkey;
+  private tmpCleaner?: TempFileCleaner;
+
+  constructor(private opts: CheckpointerOptions) {
+    this.#dockerMode = opts.dockerMode;
+
+    this.registryHost = opts.registryHost ?? "localhost:5000";
+    this.registryNamespace = opts.registryNamespace ?? "trigger";
+    this.registryTlsVerify = opts.registryTlsVerify ?? true;
+
+    this.disableCheckpointSupport = opts.disableCheckpointSupport ?? false;
+
+    this.simulateCheckpointFailure = opts.simulateCheckpointFailure ?? false;
+    this.simulateCheckpointFailureSeconds = opts.simulateCheckpointFailureSeconds ?? 300;
+    this.simulatePushFailure = opts.simulatePushFailure ?? false;
+    this.simulatePushFailureSeconds = opts.simulatePushFailureSeconds ?? 300;
+
+    this.chaosMonkey = opts.chaosMonkey ?? new ChaosMonkey(!!process.env.CHAOS_MONKEY_ENABLED);
+    this.tmpCleaner = this.#createTmpCleaner();
+  }
+
+  async init(): Promise<CheckpointerInitializeReturn> {
+    if (this.#initialized) {
+      return this.#getInitReturn(this.#canCheckpoint);
+    }
+
+    this.#logger.log(`${this.#dockerMode ? "Docker" : "Kubernetes"} mode`);
+
+    if (this.#dockerMode) {
+      const testCheckpoint = await testDockerCheckpoint();
+
+      if (testCheckpoint.ok) {
+        return this.#getInitReturn(true);
+      }
+
+      this.#logger.error(testCheckpoint.message, { error: testCheckpoint.error });
+      return this.#getInitReturn(false);
+    }
+
+    const canLogin = await Buildah.canLogin(this.registryHost);
+
+    if (!canLogin) {
+      this.#logger.error(`No checkpoint support: Not logged in to registry ${this.registryHost}`);
+    }
+
+    return this.#getInitReturn(canLogin);
+  }
+
+  #getInitReturn(canCheckpoint: boolean): CheckpointerInitializeReturn {
+    this.#canCheckpoint = canCheckpoint;
+
+    if (canCheckpoint) {
+      if (!this.#initialized) {
+        this.#logger.log("Full checkpoint support!");
+      }
+    }
+
+    this.#initialized = true;
+
+    const willSimulate = this.#dockerMode && (!this.#canCheckpoint || this.opts.forceSimulate);
+
+    if (willSimulate) {
+      this.#logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", {
+        forceSimulate: this.opts.forceSimulate,
+      });
+    }
+
+    return {
+      canCheckpoint,
+      willSimulate,
+    };
+  }
+
+  #getImageRef(projectRef: string, deploymentVersion: string, shortCode: string) {
+    return `${this.registryHost}/${this.registryNamespace}/${projectRef}:${deploymentVersion}.prod-${shortCode}`;
+  }
+
+  #getExportLocation(projectRef: string, deploymentVersion: string, shortCode: string) {
+    const basename = `${projectRef}-${deploymentVersion}-${shortCode}`;
+
+    if (this.#dockerMode) {
+      return basename;
+    } else {
+      return Crictl.getExportLocation(basename);
+    }
+  }
+
+  async checkpointAndPush(opts: CheckpointAndPushOptions): Promise<CheckpointData | undefined> {
+    const start = performance.now();
+    this.#logger.log(`checkpointAndPush() start`, { start, opts });
+
+    let interval: NodeJS.Timer | undefined;
+
+    if (opts.shouldHeartbeat) {
+      interval = setInterval(() => {
+        this.#logger.log("Sending heartbeat", { runId: opts.runId });
+        this.opts.heartbeat(opts.runId);
+      }, 20_000);
+    }
+
+    try {
+      const result = await this.#checkpointAndPushWithBackoff(opts);
+
+      const end = performance.now();
+      this.#logger.log(`checkpointAndPush() end`, {
+        start,
+        end,
+        diff: end - start,
+        opts,
+        success: result.success,
+      });
+
+      if (!result.success) {
+        return;
+      }
+
+      return result.checkpoint;
+    } finally {
+      if (opts.shouldHeartbeat) {
+        clearInterval(interval);
+      }
+    }
+  }
+
+  isCheckpointing(runId: string) {
+    return this.#abortControllers.has(runId) || this.#waitingForRetry.has(runId);
+  }
+
+  cancelCheckpoint(runId: string): boolean {
+    // If the last checkpoint failed, pretend we canceled it
+    // This ensures tasks don't wait for external resume messages to continue
+    if (this.#hasFailedCheckpoint(runId)) {
+      this.#clearFailedCheckpoint(runId);
+      return true;
+    }
+
+    if (this.#waitingForRetry.has(runId)) {
+      this.#waitingForRetry.delete(runId);
+      return true;
+    }
+
+    const controller = this.#abortControllers.get(runId);
+
+    if (!controller) {
+      this.#logger.debug("Nothing to cancel", { runId });
+      return false;
+    }
+
+    if (controller.signal.aborted) {
+      this.#logger.debug("Controller already aborted", { runId });
+      return false;
+    }
+
+    controller.abort("cancelCheckpoint()");
+    this.#abortControllers.delete(runId);
+
+    return true;
+  }
+
+  async #checkpointAndPushWithBackoff({
+    runId,
+    leaveRunning = true, // This mirrors kubernetes behaviour more accurately
+    projectRef,
+    deploymentVersion,
+    attemptNumber,
+  }: CheckpointAndPushOptions): Promise<CheckpointAndPushResult> {
+    this.#logger.log("Checkpointing with backoff", {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+    });
+
+    const backoff = new ExponentialBackoff()
+      .type("EqualJitter")
+      .base(3)
+      .max(3 * 3600)
+      .maxElapsed(48 * 3600);
+
+    for await (const { delay, retry } of backoff) {
+      try {
+        if (retry > 0) {
+          this.#logger.error("Retrying checkpoint", {
+            runId,
+            retry,
+            delay,
+          });
+
+          this.#waitingForRetry.add(runId);
+          await setTimeout(delay.milliseconds);
+
+          if (!this.#waitingForRetry.has(runId)) {
+            this.#logger.log("Checkpoint canceled while waiting for retry", { runId });
+            return { success: false, reason: "CANCELED" };
+          } else {
+            this.#waitingForRetry.delete(runId);
+          }
+        }
+
+        const result = await this.#checkpointAndPush({
+          runId,
+          leaveRunning,
+          projectRef,
+          deploymentVersion,
+          attemptNumber,
+        });
+
+        if (result.success) {
+          return result;
+        }
+
+        if (result.reason === "CANCELED") {
+          this.#logger.log("Checkpoint canceled, won't retry", { runId });
+          // Don't fail the checkpoint, as it was canceled
+          return result;
+        }
+
+        if (result.reason === "IN_PROGRESS") {
+          this.#logger.log("Checkpoint already in progress, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        if (result.reason === "NO_SUPPORT") {
+          this.#logger.log("No checkpoint support, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        if (result.reason === "DISABLED") {
+          this.#logger.log("Checkpoint support disabled, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        if (result.reason === "SKIP_RETRYING") {
+          this.#logger.log("Skipping retrying", { runId });
+          return result;
+        }
+
+        continue;
+      } catch (error) {
+        this.#logger.error("Checkpoint error", {
+          retry,
+          runId,
+          delay,
+          error: error instanceof Error ? error.message : error,
+        });
+      }
+    }
+
+    this.#logger.error(`Checkpoint failed after exponential backoff`, {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+    });
+    this.#failCheckpoint(runId, "ERROR");
+
+    return { success: false, reason: "ERROR" };
+  }
+
+  async #checkpointAndPush({
+    runId,
+    leaveRunning = true, // This mirrors kubernetes behaviour more accurately
+    projectRef,
+    deploymentVersion,
+    attemptNumber,
+  }: CheckpointAndPushOptions): Promise<CheckpointAndPushResult> {
+    await this.init();
+
+    const options = {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+      attemptNumber,
+    };
+
+    if (!this.#dockerMode && !this.#canCheckpoint) {
+      this.#logger.error("No checkpoint support. Simulation requires docker.");
+      return { success: false, reason: "NO_SUPPORT" };
+    }
+
+    if (this.isCheckpointing(runId)) {
+      this.#logger.error("Checkpoint procedure already in progress", { options });
+      return { success: false, reason: "IN_PROGRESS" };
+    }
+
+    // This is a new checkpoint, clear any last failure for this run
+    this.#clearFailedCheckpoint(runId);
+
+    if (this.disableCheckpointSupport) {
+      this.#logger.error("Checkpoint support disabled", { options });
+      return { success: false, reason: "DISABLED" };
+    }
+
+    const controller = new AbortController();
+    this.#abortControllers.set(runId, controller);
+
+    const onAbort = () => {
+      this.#logger.error("Checkpoint aborted", { options });
+      controller.signal.removeEventListener("abort", onAbort);
+    };
+    controller.signal.addEventListener("abort", onAbort);
+
+    const shortCode = nanoid(8);
+    const imageRef = this.#getImageRef(projectRef, deploymentVersion, shortCode);
+    const exportLocation = this.#getExportLocation(projectRef, deploymentVersion, shortCode);
+
+    const buildah = new Buildah({ id: `${runId}-${shortCode}`, abortSignal: controller.signal });
+    const crictl = new Crictl({ id: `${runId}-${shortCode}`, abortSignal: controller.signal });
+
+    const cleanup = async () => {
+      const metadata = {
+        runId,
+        exportLocation,
+        imageRef,
+      };
+
+      if (this.#dockerMode) {
+        this.#logger.debug("Skipping cleanup in docker mode", metadata);
+        return;
+      }
+
+      this.#logger.log("Cleaning up", metadata);
+
+      try {
+        await buildah.cleanup();
+        await crictl.cleanup();
+      } catch (error) {
+        this.#logger.error("Error during cleanup", { ...metadata, error });
+      }
+
+      // Ensure only the current controller is removed
+      if (this.#abortControllers.get(runId) === controller) {
+        this.#abortControllers.delete(runId);
+      }
+      controller.signal.removeEventListener("abort", onAbort);
+    };
+
+    try {
+      await this.chaosMonkey.call();
+
+      this.#logger.log("Checkpointing:", { options });
+
+      const containterName = this.#getRunContainerName(runId);
+
+      // Create checkpoint (docker)
+      if (this.#dockerMode) {
+        await this.#createDockerCheckpoint(
+          controller.signal,
+          runId,
+          exportLocation,
+          leaveRunning,
+          attemptNumber
+        );
+
+        this.#logger.log("checkpoint created:", {
+          runId,
+          location: exportLocation,
+        });
+
+        return {
+          success: true,
+          checkpoint: {
+            location: exportLocation,
+            docker: true,
+          },
+        };
+      }
+
+      // Create checkpoint (CRI)
+      if (!this.#canCheckpoint) {
+        this.#logger.error("No checkpoint support in kubernetes mode.");
+        return { success: false, reason: "SKIP_RETRYING" };
+      }
+
+      const containerId = await crictl.ps(containterName, true);
+
+      if (!containerId.stdout) {
+        this.#logger.error("could not find container id", { options, containterName });
+        return { success: false, reason: "SKIP_RETRYING" };
+      }
+
+      const start = performance.now();
+
+      if (this.simulateCheckpointFailure) {
+        if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) {
+          this.#logger.error("Simulating checkpoint failure", { options });
+          throw new Error("SIMULATE_CHECKPOINT_FAILURE");
+        }
+      }
+
+      // Create checkpoint
+      await crictl.checkpoint(containerId.stdout, exportLocation);
+      const postCheckpoint = performance.now();
+
+      // Print checkpoint size
+      const size = await getParsedFileSize(exportLocation);
+      this.#logger.log("checkpoint archive created", { size, options });
+
+      // Create image from checkpoint
+      const workingContainer = await buildah.from("scratch");
+      const postFrom = performance.now();
+
+      await buildah.add(workingContainer.stdout, exportLocation, "/");
+      const postAdd = performance.now();
+
+      await buildah.config(workingContainer.stdout, [
+        `io.kubernetes.cri-o.annotations.checkpoint.name=${shortCode}`,
+      ]);
+      const postConfig = performance.now();
+
+      await buildah.commit(workingContainer.stdout, imageRef);
+      const postCommit = performance.now();
+
+      if (this.simulatePushFailure) {
+        if (performance.now() < this.simulatePushFailureSeconds * 1000) {
+          this.#logger.error("Simulating push failure", { options });
+          throw new Error("SIMULATE_PUSH_FAILURE");
+        }
+      }
+
+      // Push checkpoint image
+      await buildah.push(imageRef, this.registryTlsVerify);
+      const postPush = performance.now();
+
+      const perf = {
+        "crictl checkpoint": postCheckpoint - start,
+        "buildah from": postFrom - postCheckpoint,
+        "buildah add": postAdd - postFrom,
+        "buildah config": postConfig - postAdd,
+        "buildah commit": postCommit - postConfig,
+        "buildah push": postPush - postCommit,
+      };
+
+      this.#logger.log("Checkpointed and pushed image to:", { location: imageRef, perf });
+
+      return {
+        success: true,
+        checkpoint: {
+          location: imageRef,
+          docker: false,
+        },
+      };
+    } catch (error) {
+      if (error instanceof Exec.Result) {
+        if (error.aborted) {
+          this.#logger.error("Checkpoint canceled: Exec", { options });
+
+          return { success: false, reason: "CANCELED" };
+        } else {
+          this.#logger.error("Checkpoint command error", { options, error });
+
+          return { success: false, reason: "ERROR" };
+        }
+      }
+
+      this.#logger.error("Unhandled checkpoint error", { options, error });
+
+      return { success: false, reason: "ERROR" };
+    } finally {
+      await cleanup();
+
+      if (controller.signal.aborted) {
+        this.#logger.error("Checkpoint canceled: Cleanup", { options });
+
+        // Overrides any prior return value
+        return { success: false, reason: "CANCELED" };
+      }
+    }
+  }
+
+  async #createDockerCheckpoint(
+    abortSignal: AbortSignal,
+    runId: string,
+    exportLocation: string,
+    leaveRunning: boolean,
+    attemptNumber?: number
+  ) {
+    const containterNameWithAttempt = this.#getRunContainerName(runId, attemptNumber);
+    const exec = new Exec({ logger: this.#logger, abortSignal });
+
+    try {
+      if (this.opts.forceSimulate || !this.#canCheckpoint) {
+        this.#logger.log("Simulating checkpoint");
+
+        await exec.x("docker", ["pause", containterNameWithAttempt]);
+
+        return;
+      }
+
+      if (this.simulateCheckpointFailure) {
+        if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) {
+          this.#logger.error("Simulating checkpoint failure", {
+            runId,
+            exportLocation,
+            leaveRunning,
+            attemptNumber,
+          });
+
+          throw new Error("SIMULATE_CHECKPOINT_FAILURE");
+        }
+      }
+
+      const args = ["checkpoint", "create"];
+
+      if (leaveRunning) {
+        args.push("--leave-running");
+      }
+
+      args.push(containterNameWithAttempt, exportLocation);
+
+      await exec.x("docker", args);
+    } catch (error) {
+      this.#logger.error("Failed while creating docker checkpoint", { exportLocation });
+      throw error;
+    }
+  }
+
+  #failCheckpoint(runId: string, error: unknown) {
+    this.#failedCheckpoints.set(runId, error);
+  }
+
+  #clearFailedCheckpoint(runId: string) {
+    this.#failedCheckpoints.delete(runId);
+  }
+
+  #hasFailedCheckpoint(runId: string) {
+    return this.#failedCheckpoints.has(runId);
+  }
+
+  #getRunContainerName(suffix: string, attemptNumber?: number) {
+    return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
+  }
+
+  #createTmpCleaner() {
+    if (!boolFromEnv("TMP_CLEANER_ENABLED", false)) {
+      return;
+    }
+
+    const defaultPaths = [Buildah.tmpDir, Crictl.checkpointDir].filter(Boolean);
+    const pathsOverride = process.env.TMP_CLEANER_PATHS_OVERRIDE?.split(",").filter(Boolean) ?? [];
+    const paths = pathsOverride.length ? pathsOverride : defaultPaths;
+
+    if (paths.length === 0) {
+      this.#logger.error("TempFileCleaner enabled but no paths to clean", {
+        defaultPaths,
+        pathsOverride,
+        TMP_CLEANER_PATHS_OVERRIDE: process.env.TMP_CLEANER_PATHS_OVERRIDE,
+      });
+
+      return;
+    }
+    const cleaner = new TempFileCleaner({
+      paths,
+      maxAgeMinutes: numFromEnv("TMP_CLEANER_MAX_AGE_MINUTES", 60),
+      intervalSeconds: numFromEnv("TMP_CLEANER_INTERVAL_SECONDS", 300),
+      leadingEdge: boolFromEnv("TMP_CLEANER_LEADING_EDGE", false),
+    });
+
+    cleaner.start();
+
+    return cleaner;
+  }
+}

apps/coordinator/src/cleaner.ts

@@ -0,0 +1,106 @@
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import { Exec } from "./exec";
+import { setTimeout } from "timers/promises";
+
+interface TempFileCleanerOptions {
+  paths: string[];
+  maxAgeMinutes: number;
+  intervalSeconds: number;
+  leadingEdge?: boolean;
+}
+
+export class TempFileCleaner {
+  private enabled = false;
+
+  private logger: SimpleStructuredLogger;
+  private exec: Exec;
+
+  constructor(private opts: TempFileCleanerOptions) {
+    this.logger = new SimpleStructuredLogger("tmp-cleaner", undefined, { ...this.opts });
+    this.exec = new Exec({ logger: this.logger });
+  }
+
+  async start() {
+    this.logger.log("TempFileCleaner.start");
+    this.enabled = true;
+
+    if (!this.opts.leadingEdge) {
+      await this.wait();
+    }
+
+    while (this.enabled) {
+      try {
+        await this.clean();
+      } catch (error) {
+        this.logger.error("error during tick", { error });
+      }
+
+      await this.wait();
+    }
+  }
+
+  stop() {
+    this.logger.log("TempFileCleaner.stop");
+    this.enabled = false;
+  }
+
+  private wait() {
+    return setTimeout(this.opts.intervalSeconds * 1000);
+  }
+
+  private async clean() {
+    for (const path of this.opts.paths) {
+      try {
+        await this.cleanSingle(path);
+      } catch (error) {
+        this.logger.error("error while cleaning", { path, error });
+      }
+    }
+  }
+
+  private async cleanSingle(startingPoint: string) {
+    const maxAgeMinutes = this.opts.maxAgeMinutes;
+
+    const ignoreStartingPoint = ["!", "-path", startingPoint];
+    const onlyDirectDescendants = ["-maxdepth", "1"];
+    const onlyOldFiles = ["-mmin", `+${maxAgeMinutes}`];
+
+    const baseArgs = [
+      startingPoint,
+      ...ignoreStartingPoint,
+      ...onlyDirectDescendants,
+      ...onlyOldFiles,
+    ];
+
+    const duArgs = ["-exec", "du", "-ch", "{}", "+"];
+    const rmArgs = ["-exec", "rm", "-rf", "{}", "+"];
+
+    const du = this.x("find", [...baseArgs, ...duArgs]);
+    const duOutput = await du;
+
+    const duLines = duOutput.stdout.trim().split("\n");
+    const fileCount = duLines.length - 1; // last line is the total
+    const fileSize = duLines.at(-1)?.trim().split(/\s+/)[0];
+
+    if (fileCount === 0) {
+      this.logger.log("nothing to delete", { startingPoint, maxAgeMinutes });
+      return;
+    }
+
+    this.logger.log("deleting old files", { fileCount, fileSize, startingPoint, maxAgeMinutes });
+
+    const rm = this.x("find", [...baseArgs, ...rmArgs]);
+    const rmOutput = await rm;
+
+    if (rmOutput.stderr.length > 0) {
+      this.logger.error("delete unsuccessful", { rmOutput });
+      return;
+    }
+
+    this.logger.log("deleted old files", { fileCount, fileSize, startingPoint, maxAgeMinutes });
+  }
+
+  private get x() {
+    return this.exec.x.bind(this.exec);
+  }
+}

apps/coordinator/src/exec.ts

@@ -0,0 +1,293 @@
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+import { randomUUID } from "crypto";
+import { homedir } from "os";
+import { type Result, x } from "tinyexec";
+
+class TinyResult {
+  pid?: number;
+  exitCode?: number;
+  aborted: boolean;
+  killed: boolean;
+
+  constructor(result: Result) {
+    this.pid = result.pid;
+    this.exitCode = result.exitCode;
+    this.aborted = result.aborted;
+    this.killed = result.killed;
+  }
+}
+
+interface ExecOptions {
+  logger?: SimpleStructuredLogger;
+  abortSignal?: AbortSignal;
+  logOutput?: boolean;
+  trimArgs?: boolean;
+  neverThrow?: boolean;
+}
+
+export class Exec {
+  private logger: SimpleStructuredLogger;
+  private abortSignal: AbortSignal | undefined;
+
+  private logOutput: boolean;
+  private trimArgs: boolean;
+  private neverThrow: boolean;
+
+  constructor(opts: ExecOptions) {
+    this.logger = opts.logger ?? new SimpleStructuredLogger("exec");
+    this.abortSignal = opts.abortSignal;
+
+    this.logOutput = opts.logOutput ?? true;
+    this.trimArgs = opts.trimArgs ?? true;
+    this.neverThrow = opts.neverThrow ?? false;
+  }
+
+  async x(
+    command: string,
+    args?: string[],
+    opts?: { neverThrow?: boolean; ignoreAbort?: boolean }
+  ) {
+    const argsTrimmed = this.trimArgs ? args?.map((arg) => arg.trim()) : args;
+
+    const commandWithFirstArg = `${command}${argsTrimmed?.length ? ` ${argsTrimmed[0]}` : ""}`;
+    this.logger.debug(`exec: ${commandWithFirstArg}`, { command, args, argsTrimmed });
+
+    const result = x(command, argsTrimmed, {
+      signal: opts?.ignoreAbort ? undefined : this.abortSignal,
+      // We don't use this as it doesn't cover killed and aborted processes
+      // throwOnError: true,
+    });
+
+    const output = await result;
+
+    const metadata = {
+      command,
+      argsRaw: args,
+      argsTrimmed,
+      globalOpts: {
+        trimArgs: this.trimArgs,
+        neverThrow: this.neverThrow,
+        hasAbortSignal: !!this.abortSignal,
+      },
+      localOpts: opts,
+      stdout: output.stdout,
+      stderr: output.stderr,
+      pid: result.pid,
+      exitCode: result.exitCode,
+      aborted: result.aborted,
+      killed: result.killed,
+    };
+
+    if (this.logOutput) {
+      this.logger.debug(`output: ${commandWithFirstArg}`, metadata);
+    }
+
+    if (this.neverThrow || opts?.neverThrow) {
+      return output;
+    }
+
+    if (result.aborted) {
+      this.logger.error(`aborted: ${commandWithFirstArg}`, metadata);
+      throw new TinyResult(result);
+    }
+
+    if (result.killed) {
+      this.logger.error(`killed: ${commandWithFirstArg}`, metadata);
+      throw new TinyResult(result);
+    }
+
+    if (result.exitCode !== 0) {
+      this.logger.error(`non-zero exit: ${commandWithFirstArg}`, metadata);
+      throw new TinyResult(result);
+    }
+
+    return output;
+  }
+
+  static Result = TinyResult;
+}
+
+interface BuildahOptions {
+  id?: string;
+  abortSignal?: AbortSignal;
+}
+
+export class Buildah {
+  private id: string;
+  private logger: SimpleStructuredLogger;
+  private exec: Exec;
+
+  private containers = new Set<string>();
+  private images = new Set<string>();
+
+  constructor(opts: BuildahOptions) {
+    this.id = opts.id ?? randomUUID();
+    this.logger = new SimpleStructuredLogger("buildah", undefined, { id: this.id });
+
+    this.exec = new Exec({
+      logger: this.logger,
+      abortSignal: opts.abortSignal,
+    });
+
+    this.logger.log("initiaized", { opts });
+  }
+
+  private get x() {
+    return this.exec.x.bind(this.exec);
+  }
+
+  async from(baseImage: string) {
+    const output = await this.x("buildah", ["from", baseImage]);
+    this.containers.add(output.stdout);
+    return output;
+  }
+
+  async add(container: string, src: string, dest: string) {
+    return await this.x("buildah", ["add", container, src, dest]);
+  }
+
+  async config(container: string, annotations: string[]) {
+    const args = ["config"];
+
+    for (const annotation of annotations) {
+      args.push(`--annotation=${annotation}`);
+    }
+
+    args.push(container);
+
+    return await this.x("buildah", args);
+  }
+
+  async commit(container: string, imageRef: string) {
+    const output = await this.x("buildah", ["commit", container, imageRef]);
+    this.images.add(output.stdout);
+    return output;
+  }
+
+  async push(imageRef: string, registryTlsVerify?: boolean) {
+    return await this.x("buildah", [
+      "push",
+      `--tls-verify=${String(!!registryTlsVerify)}`,
+      imageRef,
+    ]);
+  }
+
+  async cleanup() {
+    if (this.containers.size > 0) {
+      try {
+        const output = await this.x("buildah", ["rm", ...this.containers], { ignoreAbort: true });
+        this.containers.clear();
+
+        if (output.stderr.length > 0) {
+          this.logger.error("failed to remove some containers", { output });
+        }
+      } catch (error) {
+        this.logger.error("failed to clean up containers", { error, containers: this.containers });
+      }
+    } else {
+      this.logger.debug("no containers to clean up");
+    }
+
+    if (this.images.size > 0) {
+      try {
+        const output = await this.x("buildah", ["rmi", ...this.images], { ignoreAbort: true });
+        this.images.clear();
+
+        if (output.stderr.length > 0) {
+          this.logger.error("failed to remove some images", { output });
+        }
+      } catch (error) {
+        this.logger.error("failed to clean up images", { error, images: this.images });
+      }
+    } else {
+      this.logger.debug("no images to clean up");
+    }
+  }
+
+  static async canLogin(registryHost: string) {
+    try {
+      await x("buildah", ["login", "--get-login", registryHost], { throwOnError: true });
+      return true;
+    } catch (error) {
+      return false;
+    }
+  }
+
+  static get tmpDir() {
+    return process.env.TMPDIR ?? "/var/tmp";
+  }
+
+  static get storageRootDir() {
+    return process.getuid?.() === 0
+      ? "/var/lib/containers/storage"
+      : `${homedir()}/.local/share/containers/storage`;
+  }
+}
+
+interface CrictlOptions {
+  id?: string;
+  abortSignal?: AbortSignal;
+}
+
+export class Crictl {
+  private id: string;
+  private logger: SimpleStructuredLogger;
+  private exec: Exec;
+
+  private archives = new Set<string>();
+
+  constructor(opts: CrictlOptions) {
+    this.id = opts.id ?? randomUUID();
+    this.logger = new SimpleStructuredLogger("crictl", undefined, { id: this.id });
+
+    this.exec = new Exec({
+      logger: this.logger,
+      abortSignal: opts.abortSignal,
+    });
+
+    this.logger.log("initiaized", { opts });
+  }
+
+  private get x() {
+    return this.exec.x.bind(this.exec);
+  }
+
+  async ps(containerName: string, quiet?: boolean) {
+    return await this.x("crictl", ["ps", "--name", containerName, quiet ? "--quiet" : ""]);
+  }
+
+  async checkpoint(containerId: string, exportLocation: string) {
+    const output = await this.x("crictl", [
+      "checkpoint",
+      `--export=${exportLocation}`,
+      containerId,
+    ]);
+    this.archives.add(exportLocation);
+    return output;
+  }
+
+  async cleanup() {
+    if (this.archives.size > 0) {
+      try {
+        const output = await this.x("rm", ["-v", ...this.archives], { ignoreAbort: true });
+        this.archives.clear();
+
+        if (output.stderr.length > 0) {
+          this.logger.error("failed to remove some archives", { output });
+        }
+      } catch (error) {
+        this.logger.error("failed to clean up archives", { error, archives: this.archives });
+      }
+    } else {
+      this.logger.debug("no archives to clean up");
+    }
+  }
+
+  static getExportLocation(identifier: string) {
+    return `${this.checkpointDir}/${identifier}.tar`;
+  }
+
+  static get checkpointDir() {
+    return process.env.CRI_CHECKPOINT_DIR ?? "/checkpoints";
+  }
+}

apps/coordinator/src/util.ts

@@ -0,0 +1,19 @@
+export const boolFromEnv = (env: string, defaultValue: boolean): boolean => {
+  const value = process.env[env];
+
+  if (!value) {
+    return defaultValue;
+  }
+
+  return ["1", "true"].includes(value);
+};
+
+export const numFromEnv = (env: string, defaultValue: number): number => {
+  const value = process.env[env];
+
+  if (!value) {
+    return defaultValue;
+  }
+
+  return parseInt(value, 10);
+};

apps/coordinator/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "include": ["./src/**/*.ts"],
+  "exclude": ["node_modules"],
+  "compilerOptions": {
+    "target": "es2016",
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "resolveJsonModule": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "paths": {
+      "@trigger.dev/core/v3": ["../../packages/core/src/v3"],
+      "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"]
+    }
+  }
+}

apps/docker-provider/.env.example

@@ -0,0 +1,11 @@
+HTTP_SERVER_PORT=8050
+
+PLATFORM_WS_PORT=3030
+PLATFORM_SECRET=provider-secret
+SECURE_CONNECTION=false
+
+OTEL_EXPORTER_OTLP_ENDPOINT=http://0.0.0.0:3030/otel
+
+# Use this if you are on macOS
+# COORDINATOR_HOST="host.docker.internal"
+# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318"

apps/docker-provider/.gitignore

@@ -0,0 +1,3 @@
+dist/
+node_modules/
+.env

apps/docker-provider/Containerfile

@@ -0,0 +1,47 @@
+FROM node:20-alpine@sha256:7a91aa397f2e2dfbfcdad2e2d72599f374e0b0172be1d86eeb73f1d33f36a4b2 AS node-20-alpine
+
+WORKDIR /app
+
+FROM node-20-alpine AS pruner
+
+COPY --chown=node:node . .
+RUN npx -q turbo@1.10.9 prune --scope=docker-provider --docker
+RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
+
+FROM node-20-alpine AS base
+
+RUN apk add --no-cache dumb-init docker
+
+COPY --chown=node:node .gitignore .gitignore
+COPY --from=pruner --chown=node:node /app/out/json/ .
+COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
+COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
+
+FROM base AS dev-deps
+RUN corepack enable
+ENV NODE_ENV development
+
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
+
+FROM base AS builder
+RUN corepack enable
+
+COPY --from=pruner --chown=node:node /app/out/full/ .
+COPY --from=dev-deps --chown=node:node /app/ .
+COPY --chown=node:node turbo.json turbo.json
+
+RUN pnpm run -r --filter docker-provider build:bundle
+
+FROM base AS runner
+
+RUN corepack enable
+ENV NODE_ENV production
+
+COPY --from=builder --chown=node:node /app/apps/docker-provider/dist/index.mjs ./index.mjs
+
+EXPOSE 8000
+
+USER node
+
+CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]

apps/docker-provider/README.md

@@ -0,0 +1,3 @@
+# Docker provider
+
+The `docker-provider` allows the platform to be orchestrator-agnostic. The platform can perform actions such as `INDEX_TASKS` or `INVOKE_TASK` which the provider translates into Docker actions.

apps/docker-provider/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "docker-provider",
+  "private": true,
+  "version": "0.0.1",
+  "description": "",
+  "main": "dist/index.cjs",
+  "scripts": {
+    "build": "npm run build:bundle",
+    "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"",
+    "build:image": "docker build -f Containerfile . -t docker-provider",
+    "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
+    "start": "tsx src/index.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@trigger.dev/core": "workspace:*",
+    "execa": "^8.0.1"
+  },
+  "devDependencies": {
+    "@types/node": "^18.19.8",
+    "dotenv": "^16.4.2",
+    "esbuild": "^0.19.11",
+    "tsx": "^4.7.0",
+    "typescript": "^5.3.3"
+  }
+}

apps/docker-provider/src/index.ts

@@ -0,0 +1,291 @@
+import { $, type ExecaChildProcess, execa } from "execa";
+import {
+  ProviderShell,
+  TaskOperations,
+  TaskOperationsCreateOptions,
+  TaskOperationsIndexOptions,
+  TaskOperationsRestoreOptions,
+} from "@trigger.dev/core/v3/apps";
+import { SimpleLogger } from "@trigger.dev/core/v3/apps";
+import { isExecaChildProcess, testDockerCheckpoint } from "@trigger.dev/core/v3/apps";
+import { setTimeout } from "node:timers/promises";
+import { PostStartCauses, PreStopCauses } from "@trigger.dev/core/v3";
+
+const MACHINE_NAME = process.env.MACHINE_NAME || "local";
+const COORDINATOR_PORT = process.env.COORDINATOR_PORT || 8020;
+const COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
+const DOCKER_NETWORK = process.env.DOCKER_NETWORK || "host";
+
+const OTEL_EXPORTER_OTLP_ENDPOINT =
+  process.env.OTEL_EXPORTER_OTLP_ENDPOINT || "http://0.0.0.0:4318";
+
+const FORCE_CHECKPOINT_SIMULATION = ["1", "true"].includes(
+  process.env.FORCE_CHECKPOINT_SIMULATION ?? "false"
+);
+
+const logger = new SimpleLogger(`[${MACHINE_NAME}]`);
+
+type TaskOperationsInitReturn = {
+  canCheckpoint: boolean;
+  willSimulate: boolean;
+};
+
+class DockerTaskOperations implements TaskOperations {
+  #initialized = false;
+  #canCheckpoint = false;
+
+  constructor(private opts = { forceSimulate: false }) {}
+
+  async init(): Promise<TaskOperationsInitReturn> {
+    if (this.#initialized) {
+      return this.#getInitReturn(this.#canCheckpoint);
+    }
+
+    logger.log("Initializing task operations");
+
+    const testCheckpoint = await testDockerCheckpoint();
+
+    if (testCheckpoint.ok) {
+      return this.#getInitReturn(true);
+    }
+
+    logger.error(testCheckpoint.message, testCheckpoint.error);
+    return this.#getInitReturn(false);
+  }
+
+  #getInitReturn(canCheckpoint: boolean): TaskOperationsInitReturn {
+    this.#canCheckpoint = canCheckpoint;
+
+    if (canCheckpoint) {
+      if (!this.#initialized) {
+        logger.log("Full checkpoint support!");
+      }
+    }
+
+    this.#initialized = true;
+
+    const willSimulate = !canCheckpoint || this.opts.forceSimulate;
+
+    if (willSimulate) {
+      logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", {
+        forceSimulate: this.opts.forceSimulate,
+      });
+    }
+
+    return {
+      canCheckpoint,
+      willSimulate,
+    };
+  }
+
+  async index(opts: TaskOperationsIndexOptions) {
+    await this.init();
+
+    const containerName = this.#getIndexContainerName(opts.shortCode);
+
+    logger.log(`Indexing task ${opts.imageRef}`, {
+      host: COORDINATOR_HOST,
+      port: COORDINATOR_PORT,
+    });
+
+    logger.debug(
+      await execa("docker", [
+        "run",
+        `--network=${DOCKER_NETWORK}`,
+        "--rm",
+        `--env=INDEX_TASKS=true`,
+        `--env=TRIGGER_SECRET_KEY=${opts.apiKey}`,
+        `--env=TRIGGER_API_URL=${opts.apiUrl}`,
+        `--env=TRIGGER_ENV_ID=${opts.envId}`,
+        `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
+        `--env=POD_NAME=${containerName}`,
+        `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
+        `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
+        `--name=${containerName}`,
+        `${opts.imageRef}`,
+      ])
+    );
+  }
+
+  async create(opts: TaskOperationsCreateOptions) {
+    await this.init();
+
+    const containerName = this.#getRunContainerName(opts.runId, opts.nextAttemptNumber);
+
+    const runArgs = [
+      "run",
+      `--network=${DOCKER_NETWORK}`,
+      "--detach",
+      `--env=TRIGGER_ENV_ID=${opts.envId}`,
+      `--env=TRIGGER_RUN_ID=${opts.runId}`,
+      `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
+      `--env=POD_NAME=${containerName}`,
+      `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
+      `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
+      `--name=${containerName}`,
+    ];
+
+    if (process.env.ENFORCE_MACHINE_PRESETS) {
+      runArgs.push(`--cpus=${opts.machine.cpu}`, `--memory=${opts.machine.memory}G`);
+    }
+
+    runArgs.push(`${opts.image}`);
+
+    try {
+      logger.debug(await execa("docker", runArgs));
+    } catch (error) {
+      if (!isExecaChildProcess(error)) {
+        throw error;
+      }
+
+      logger.error("Create failed:", {
+        opts,
+        exitCode: error.exitCode,
+        escapedCommand: error.escapedCommand,
+        stdout: error.stdout,
+        stderr: error.stderr,
+      });
+    }
+  }
+
+  async restore(opts: TaskOperationsRestoreOptions) {
+    await this.init();
+
+    const containerName = this.#getRunContainerName(opts.runId, opts.attemptNumber);
+
+    if (!this.#canCheckpoint || this.opts.forceSimulate) {
+      logger.log("Simulating restore");
+
+      const unpause = logger.debug(await $`docker unpause ${containerName}`);
+
+      if (unpause.exitCode !== 0) {
+        throw new Error("docker unpause command failed");
+      }
+
+      await this.#sendPostStart(containerName);
+      return;
+    }
+
+    const { exitCode } = logger.debug(
+      await $`docker start --checkpoint=${opts.checkpointRef} ${containerName}`
+    );
+
+    if (exitCode !== 0) {
+      throw new Error("docker start command failed");
+    }
+
+    await this.#sendPostStart(containerName);
+  }
+
+  async delete(opts: { runId: string }) {
+    await this.init();
+
+    const containerName = this.#getRunContainerName(opts.runId);
+    await this.#sendPreStop(containerName);
+
+    logger.log("noop: delete");
+  }
+
+  async get(opts: { runId: string }) {
+    await this.init();
+
+    logger.log("noop: get");
+  }
+
+  #getIndexContainerName(suffix: string) {
+    return `task-index-${suffix}`;
+  }
+
+  #getRunContainerName(suffix: string, attemptNumber?: number) {
+    return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
+  }
+
+  async #sendPostStart(containerName: string): Promise<void> {
+    try {
+      const port = await this.#getHttpServerPort(containerName);
+      logger.debug(await this.#runLifecycleCommand(containerName, port, "postStart", "restore"));
+    } catch (error) {
+      logger.error("postStart error", { error });
+      throw new Error("postStart command failed");
+    }
+  }
+
+  async #sendPreStop(containerName: string): Promise<void> {
+    try {
+      const port = await this.#getHttpServerPort(containerName);
+      logger.debug(await this.#runLifecycleCommand(containerName, port, "preStop", "terminate"));
+    } catch (error) {
+      logger.error("preStop error", { error });
+      throw new Error("preStop command failed");
+    }
+  }
+
+  async #getHttpServerPort(containerName: string): Promise<number> {
+    // We first get the correct port, which is random during dev as we run with host networking and need to avoid clashes
+    // FIXME: Skip this in prod
+    const logs = logger.debug(await $`docker logs ${containerName}`);
+    const matches = logs.stdout.match(/http server listening on port (?<port>[0-9]+)/);
+
+    const port = Number(matches?.groups?.port);
+
+    if (!port) {
+      throw new Error("failed to extract port from logs");
+    }
+
+    return port;
+  }
+
+  async #runLifecycleCommand<THookType extends "postStart" | "preStop">(
+    containerName: string,
+    port: number,
+    type: THookType,
+    cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses,
+    retryCount = 0
+  ): Promise<ExecaChildProcess> {
+    try {
+      return await execa("docker", [
+        "exec",
+        containerName,
+        "busybox",
+        "wget",
+        "-q",
+        "-O-",
+        `127.0.0.1:${port}/${type}?cause=${cause}`,
+      ]);
+    } catch (error: any) {
+      if (type === "postStart" && retryCount < 6) {
+        logger.debug(`retriable ${type} error`, { retryCount, message: error?.message });
+        await setTimeout(exponentialBackoff(retryCount + 1, 2, 50, 1150, 50));
+
+        return this.#runLifecycleCommand(containerName, port, type, cause, retryCount + 1);
+      }
+
+      logger.error(`final ${type} error`, { message: error?.message });
+      throw new Error(`${type} command failed after ${retryCount - 1} retries`);
+    }
+  }
+}
+
+const provider = new ProviderShell({
+  tasks: new DockerTaskOperations({ forceSimulate: FORCE_CHECKPOINT_SIMULATION }),
+  type: "docker",
+});
+
+provider.listen();
+
+function exponentialBackoff(
+  retryCount: number,
+  exponential: number,
+  minDelay: number,
+  maxDelay: number,
+  jitter: number
+): number {
+  // Calculate the delay using the exponential backoff formula
+  const delay = Math.min(Math.pow(exponential, retryCount) * minDelay, maxDelay);
+
+  // Calculate the jitter
+  const jitterValue = Math.random() * jitter;
+
+  // Return the calculated delay with jitter
+  return delay + jitterValue;
+}

apps/docker-provider/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "es2018",
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "paths": {
+      "@trigger.dev/core/v3": ["../../packages/core/src/v3"],
+      "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"]
+    }
+  }
+}

apps/kubernetes-provider/.env.example

@@ -0,0 +1,9 @@
+HTTP_SERVER_PORT=8060
+
+PLATFORM_WS_PORT=3030
+PLATFORM_SECRET=provider-secret
+SECURE_CONNECTION=false
+
+# Use this if you are on macOS
+# COORDINATOR_HOST="host.docker.internal"
+# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318"

apps/kubernetes-provider/.gitignore

@@ -0,0 +1,3 @@
+dist/
+node_modules/
+.env

apps/kubernetes-provider/Containerfile

@@ -0,0 +1,47 @@
+FROM node:20-alpine@sha256:7a91aa397f2e2dfbfcdad2e2d72599f374e0b0172be1d86eeb73f1d33f36a4b2 AS node-20-alpine
+
+WORKDIR /app
+
+FROM node-20-alpine AS pruner
+
+COPY --chown=node:node . .
+RUN npx -q turbo@1.10.9 prune --scope=kubernetes-provider --docker
+RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
+
+FROM node-20-alpine AS base
+
+RUN apk add --no-cache dumb-init
+
+COPY --chown=node:node .gitignore .gitignore
+COPY --from=pruner --chown=node:node /app/out/json/ .
+COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
+COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
+
+FROM base AS dev-deps
+RUN corepack enable
+ENV NODE_ENV development
+
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
+
+FROM base AS builder
+RUN corepack enable
+
+COPY --from=pruner --chown=node:node /app/out/full/ .
+COPY --from=dev-deps --chown=node:node /app/ .
+COPY --chown=node:node turbo.json turbo.json
+
+RUN pnpm run -r --filter kubernetes-provider build:bundle
+
+FROM base AS runner
+
+RUN corepack enable
+ENV NODE_ENV production
+
+COPY --from=builder --chown=node:node /app/apps/kubernetes-provider/dist/index.mjs ./index.mjs
+
+EXPOSE 8000
+
+USER node
+
+CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]

apps/kubernetes-provider/README.md

@@ -0,0 +1,3 @@
+# Kubernetes provider
+
+The `kubernetes-provider` allows the platform to be orchestrator-agnostic. The platform can perform actions such as `INDEX_TASKS` or `INVOKE_TASK` which the provider translates into Kubernetes actions.

apps/kubernetes-provider/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "kubernetes-provider",
+  "private": true,
+  "version": "0.0.1",
+  "description": "",
+  "main": "dist/index.cjs",
+  "scripts": {
+    "build": "npm run build:bundle",
+    "build:bundle": "esbuild src/index.ts --bundle --outfile=dist/index.mjs --platform=node --format=esm --target=esnext --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\"",
+    "build:image": "docker build -f Containerfile . -t kubernetes-provider",
+    "dev": "tsx --no-warnings=ExperimentalWarning --require dotenv/config --watch src/index.ts",
+    "start": "tsx src/index.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@kubernetes/client-node": "^0.20.0",
+    "@trigger.dev/core": "workspace:*",
+    "p-queue": "^8.0.1"
+  },
+  "devDependencies": {
+    "dotenv": "^16.4.2",
+    "esbuild": "^0.19.11",
+    "tsx": "^4.7.0",
+    "typescript": "^5.3.3"
+  }
+}

apps/kubernetes-provider/src/index.ts

@@ -0,0 +1,717 @@
+import * as k8s from "@kubernetes/client-node";
+import {
+  EnvironmentType,
+  MachinePreset,
+  PostStartCauses,
+  PreStopCauses,
+} from "@trigger.dev/core/v3";
+import {
+  ProviderShell,
+  SimpleLogger,
+  TaskOperations,
+  TaskOperationsCreateOptions,
+  TaskOperationsIndexOptions,
+  TaskOperationsPrePullDeploymentOptions,
+  TaskOperationsRestoreOptions,
+} from "@trigger.dev/core/v3/apps";
+import { PodCleaner } from "./podCleaner";
+import { TaskMonitor } from "./taskMonitor";
+import { UptimeHeartbeat } from "./uptimeHeartbeat";
+
+const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
+const NODE_NAME = process.env.NODE_NAME || "local";
+const OTEL_EXPORTER_OTLP_ENDPOINT =
+  process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318";
+const COORDINATOR_HOST = process.env.COORDINATOR_HOST ?? undefined;
+const COORDINATOR_PORT = process.env.COORDINATOR_PORT ?? undefined;
+const KUBERNETES_NAMESPACE = process.env.KUBERNETES_NAMESPACE ?? "default";
+
+const POD_CLEANER_INTERVAL_SECONDS = Number(process.env.POD_CLEANER_INTERVAL_SECONDS || "300");
+
+const UPTIME_HEARTBEAT_URL = process.env.UPTIME_HEARTBEAT_URL;
+const UPTIME_INTERVAL_SECONDS = Number(process.env.UPTIME_INTERVAL_SECONDS || "60");
+const UPTIME_MAX_PENDING_RUNS = Number(process.env.UPTIME_MAX_PENDING_RUNS || "25");
+const UPTIME_MAX_PENDING_INDECES = Number(process.env.UPTIME_MAX_PENDING_INDECES || "10");
+const UPTIME_MAX_PENDING_ERRORS = Number(process.env.UPTIME_MAX_PENDING_ERRORS || "10");
+
+const POD_EPHEMERAL_STORAGE_SIZE_LIMIT = process.env.POD_EPHEMERAL_STORAGE_SIZE_LIMIT || "10Gi";
+const POD_EPHEMERAL_STORAGE_SIZE_REQUEST = process.env.POD_EPHEMERAL_STORAGE_SIZE_REQUEST || "2Gi";
+
+const logger = new SimpleLogger(`[${NODE_NAME}]`);
+logger.log(`running in ${RUNTIME_ENV} mode`);
+
+type Namespace = {
+  metadata: {
+    name: string;
+  };
+};
+
+type ResourceQuantities = {
+  [K in "cpu" | "memory" | "ephemeral-storage"]?: string;
+};
+
+class KubernetesTaskOperations implements TaskOperations {
+  #namespace: Namespace = {
+    metadata: {
+      name: "default",
+    },
+  };
+
+  #k8sApi: {
+    core: k8s.CoreV1Api;
+    batch: k8s.BatchV1Api;
+    apps: k8s.AppsV1Api;
+  };
+
+  constructor(opts: { namespace?: string } = {}) {
+    if (opts.namespace) {
+      this.#namespace.metadata.name = opts.namespace;
+    }
+
+    this.#k8sApi = this.#createK8sApi();
+  }
+
+  async init() {
+    // noop
+  }
+
+  async index(opts: TaskOperationsIndexOptions) {
+    await this.#createJob(
+      {
+        metadata: {
+          name: this.#getIndexContainerName(opts.shortCode),
+          namespace: this.#namespace.metadata.name,
+        },
+        spec: {
+          completions: 1,
+          backoffLimit: 0,
+          ttlSecondsAfterFinished: 300,
+          template: {
+            metadata: {
+              labels: {
+                ...this.#getSharedLabels(opts),
+                app: "task-index",
+                "app.kubernetes.io/part-of": "trigger-worker",
+                "app.kubernetes.io/component": "index",
+                deployment: opts.deploymentId,
+              },
+            },
+            spec: {
+              ...this.#defaultPodSpec,
+              containers: [
+                {
+                  name: this.#getIndexContainerName(opts.shortCode),
+                  image: opts.imageRef,
+                  ports: [
+                    {
+                      containerPort: 8000,
+                    },
+                  ],
+                  resources: {
+                    limits: {
+                      cpu: "1",
+                      memory: "2G",
+                      "ephemeral-storage": "2Gi",
+                    },
+                  },
+                  lifecycle: {
+                    preStop: {
+                      exec: {
+                        command: this.#getLifecycleCommand("preStop", "terminate"),
+                      },
+                    },
+                  },
+                  env: [
+                    ...this.#getSharedEnv(opts.envId),
+                    {
+                      name: "INDEX_TASKS",
+                      value: "true",
+                    },
+                    {
+                      name: "TRIGGER_SECRET_KEY",
+                      value: opts.apiKey,
+                    },
+                    {
+                      name: "TRIGGER_API_URL",
+                      value: opts.apiUrl,
+                    },
+                  ],
+                },
+              ],
+            },
+          },
+        },
+      },
+      this.#namespace
+    );
+  }
+
+  async create(opts: TaskOperationsCreateOptions) {
+    const containerName = this.#getRunContainerName(opts.runId, opts.nextAttemptNumber);
+
+    await this.#createPod(
+      {
+        metadata: {
+          name: containerName,
+          namespace: this.#namespace.metadata.name,
+          labels: {
+            ...this.#getSharedLabels(opts),
+            app: "task-run",
+            "app.kubernetes.io/part-of": "trigger-worker",
+            "app.kubernetes.io/component": "create",
+            run: opts.runId,
+          },
+        },
+        spec: {
+          ...this.#defaultPodSpec,
+          terminationGracePeriodSeconds: 60 * 60,
+          containers: [
+            {
+              name: containerName,
+              image: opts.image,
+              ports: [
+                {
+                  containerPort: 8000,
+                },
+              ],
+              resources: this.#getResourcesForMachine(opts.machine),
+              lifecycle: {
+                preStop: {
+                  exec: {
+                    command: this.#getLifecycleCommand("preStop", "terminate"),
+                  },
+                },
+              },
+              env: [
+                ...this.#getSharedEnv(opts.envId),
+                {
+                  name: "TRIGGER_RUN_ID",
+                  value: opts.runId,
+                },
+              ],
+              volumeMounts: [
+                {
+                  name: "taskinfo",
+                  mountPath: "/etc/taskinfo",
+                },
+              ],
+            },
+          ],
+          volumes: [
+            {
+              name: "taskinfo",
+              emptyDir: {},
+            },
+          ],
+        },
+      },
+      this.#namespace
+    );
+  }
+
+  async restore(opts: TaskOperationsRestoreOptions) {
+    await this.#createPod(
+      {
+        metadata: {
+          name: `${this.#getRunContainerName(opts.runId)}-${opts.checkpointId.slice(-8)}`,
+          namespace: this.#namespace.metadata.name,
+          labels: {
+            ...this.#getSharedLabels(opts),
+            app: "task-run",
+            "app.kubernetes.io/part-of": "trigger-worker",
+            "app.kubernetes.io/component": "restore",
+            run: opts.runId,
+            checkpoint: opts.checkpointId,
+          },
+        },
+        spec: {
+          ...this.#defaultPodSpec,
+          initContainers: [
+            {
+              name: "pull-base-image",
+              image: opts.imageRef,
+              command: ["sleep", "0"],
+            },
+            {
+              name: "populate-taskinfo",
+              image: "registry.digitalocean.com/trigger/busybox",
+              imagePullPolicy: "IfNotPresent",
+              command: ["/bin/sh", "-c"],
+              args: ["printenv COORDINATOR_HOST | tee /etc/taskinfo/coordinator-host"],
+              env: this.#coordinatorEnvVars,
+              volumeMounts: [
+                {
+                  name: "taskinfo",
+                  mountPath: "/etc/taskinfo",
+                },
+              ],
+            },
+          ],
+          containers: [
+            {
+              name: this.#getRunContainerName(opts.runId),
+              image: opts.checkpointRef,
+              ports: [
+                {
+                  containerPort: 8000,
+                },
+              ],
+              resources: this.#getResourcesForMachine(opts.machine),
+              lifecycle: {
+                postStart: {
+                  exec: {
+                    command: this.#getLifecycleCommand("postStart", "restore"),
+                  },
+                },
+                preStop: {
+                  exec: {
+                    command: this.#getLifecycleCommand("preStop", "terminate"),
+                  },
+                },
+              },
+              volumeMounts: [
+                {
+                  name: "taskinfo",
+                  mountPath: "/etc/taskinfo",
+                },
+              ],
+            },
+          ],
+          volumes: [
+            {
+              name: "taskinfo",
+              emptyDir: {},
+            },
+          ],
+        },
+      },
+      this.#namespace
+    );
+  }
+
+  async delete(opts: { runId: string }) {
+    await this.#deletePod({
+      runId: opts.runId,
+      namespace: this.#namespace,
+    });
+  }
+
+  async get(opts: { runId: string }) {
+    await this.#getPod(opts.runId, this.#namespace);
+  }
+
+  async prePullDeployment(opts: TaskOperationsPrePullDeploymentOptions) {
+    const metaName = this.#getPrePullContainerName(opts.shortCode);
+
+    const metaLabels = {
+      ...this.#getSharedLabels(opts),
+      app: "task-prepull",
+      "app.kubernetes.io/part-of": "trigger-worker",
+      "app.kubernetes.io/component": "prepull",
+      deployment: opts.deploymentId,
+      name: metaName,
+    } satisfies k8s.V1ObjectMeta["labels"];
+
+    await this.#createDaemonSet(
+      {
+        metadata: {
+          name: metaName,
+          namespace: this.#namespace.metadata.name,
+          labels: metaLabels,
+        },
+        spec: {
+          selector: {
+            matchLabels: {
+              name: metaName,
+            },
+          },
+          template: {
+            metadata: {
+              labels: metaLabels,
+            },
+            spec: {
+              ...this.#defaultPodSpec,
+              restartPolicy: "Always",
+              initContainers: [
+                {
+                  name: "prepull",
+                  image: opts.imageRef,
+                  command: ["/usr/bin/true"],
+                  resources: {
+                    limits: {
+                      cpu: "0.25",
+                      memory: "100Mi",
+                      "ephemeral-storage": "1Gi",
+                    },
+                  },
+                },
+              ],
+              containers: [
+                {
+                  name: "pause",
+                  image: "registry.k8s.io/pause:3.9",
+                  resources: {
+                    limits: {
+                      cpu: "1m",
+                      memory: "12Mi",
+                    },
+                  },
+                },
+              ],
+            },
+          },
+        },
+      },
+      this.#namespace
+    );
+  }
+
+  #envTypeToLabelValue(type: EnvironmentType) {
+    switch (type) {
+      case "PRODUCTION":
+        return "prod";
+      case "STAGING":
+        return "stg";
+      case "DEVELOPMENT":
+        return "dev";
+      case "PREVIEW":
+        return "preview";
+    }
+  }
+
+  get #defaultPodSpec(): Omit<k8s.V1PodSpec, "containers"> {
+    return {
+      restartPolicy: "Never",
+      automountServiceAccountToken: false,
+      imagePullSecrets: [
+        {
+          name: "registry-trigger",
+        },
+        {
+          name: "registry-trigger-failover",
+        },
+      ],
+      nodeSelector: {
+        nodetype: "worker",
+      },
+    };
+  }
+
+  get #defaultResourceRequests(): ResourceQuantities {
+    return {
+      "ephemeral-storage": POD_EPHEMERAL_STORAGE_SIZE_REQUEST,
+    };
+  }
+
+  get #defaultResourceLimits(): ResourceQuantities {
+    return {
+      "ephemeral-storage": POD_EPHEMERAL_STORAGE_SIZE_LIMIT,
+    };
+  }
+
+  get #coordinatorHostEnvVar(): k8s.V1EnvVar {
+    return COORDINATOR_HOST
+      ? {
+          name: "COORDINATOR_HOST",
+          value: COORDINATOR_HOST,
+        }
+      : {
+          name: "COORDINATOR_HOST",
+          valueFrom: {
+            fieldRef: {
+              fieldPath: "status.hostIP",
+            },
+          },
+        };
+  }
+
+  get #coordinatorPortEnvVar(): k8s.V1EnvVar | undefined {
+    if (COORDINATOR_PORT) {
+      return {
+        name: "COORDINATOR_PORT",
+        value: COORDINATOR_PORT,
+      };
+    }
+  }
+
+  get #coordinatorEnvVars(): k8s.V1EnvVar[] {
+    const envVars = [this.#coordinatorHostEnvVar];
+
+    if (this.#coordinatorPortEnvVar) {
+      envVars.push(this.#coordinatorPortEnvVar);
+    }
+
+    return envVars;
+  }
+
+  #getSharedEnv(envId: string): k8s.V1EnvVar[] {
+    return [
+      {
+        name: "TRIGGER_ENV_ID",
+        value: envId,
+      },
+      {
+        name: "DEBUG",
+        value: process.env.DEBUG ? "1" : "0",
+      },
+      {
+        name: "HTTP_SERVER_PORT",
+        value: "8000",
+      },
+      {
+        name: "OTEL_EXPORTER_OTLP_ENDPOINT",
+        value: OTEL_EXPORTER_OTLP_ENDPOINT,
+      },
+      {
+        name: "POD_NAME",
+        valueFrom: {
+          fieldRef: {
+            fieldPath: "metadata.name",
+          },
+        },
+      },
+      {
+        name: "MACHINE_NAME",
+        valueFrom: {
+          fieldRef: {
+            fieldPath: "spec.nodeName",
+          },
+        },
+      },
+      ...this.#coordinatorEnvVars,
+    ];
+  }
+
+  #getSharedLabels(
+    opts:
+      | TaskOperationsIndexOptions
+      | TaskOperationsCreateOptions
+      | TaskOperationsRestoreOptions
+      | TaskOperationsPrePullDeploymentOptions
+  ): Record<string, string> {
+    return {
+      env: opts.envId,
+      envtype: this.#envTypeToLabelValue(opts.envType),
+      org: opts.orgId,
+      project: opts.projectId,
+    };
+  }
+
+  #getResourceRequestsForMachine(preset: MachinePreset): ResourceQuantities {
+    return {
+      cpu: `${preset.cpu * 0.75}`,
+      memory: `${preset.memory}G`,
+    };
+  }
+
+  #getResourceLimitsForMachine(preset: MachinePreset): ResourceQuantities {
+    return {
+      cpu: `${preset.cpu}`,
+      memory: `${preset.memory}G`,
+    };
+  }
+
+  #getResourcesForMachine(preset: MachinePreset): k8s.V1ResourceRequirements {
+    return {
+      requests: {
+        ...this.#defaultResourceRequests,
+        ...this.#getResourceRequestsForMachine(preset),
+      },
+      limits: {
+        ...this.#defaultResourceLimits,
+        ...this.#getResourceLimitsForMachine(preset),
+      },
+    };
+  }
+
+  #getLifecycleCommand<THookType extends "postStart" | "preStop">(
+    type: THookType,
+    cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses
+  ) {
+    const retries = 5;
+
+    // This will retry sending the lifecycle hook up to `retries` times
+    // The sleep is required as this may start running before the HTTP server is up
+    const exec = [
+      "/bin/sh",
+      "-c",
+      `for i in $(seq ${retries}); do sleep 1; busybox wget -q -O- 127.0.0.1:8000/${type}?cause=${cause} && break; done`,
+    ];
+
+    logger.debug("getLifecycleCommand()", { exec });
+
+    return exec;
+  }
+
+  #getIndexContainerName(suffix: string) {
+    return `task-index-${suffix}`;
+  }
+
+  #getRunContainerName(suffix: string, attemptNumber?: number) {
+    return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
+  }
+
+  #getPrePullContainerName(suffix: string) {
+    return `task-prepull-${suffix}`;
+  }
+
+  #createK8sApi() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (RUNTIME_ENV === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (RUNTIME_ENV === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${RUNTIME_ENV}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      batch: kubeConfig.makeApiClient(k8s.BatchV1Api),
+      apps: kubeConfig.makeApiClient(k8s.AppsV1Api),
+    };
+  }
+
+  async #createPod(pod: k8s.V1Pod, namespace: Namespace) {
+    try {
+      const res = await this.#k8sApi.core.createNamespacedPod(namespace.metadata.name, pod);
+      logger.debug(res.body);
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
+    }
+  }
+
+  async #deletePod(opts: { runId: string; namespace: Namespace }) {
+    try {
+      const res = await this.#k8sApi.core.deleteNamespacedPod(
+        opts.runId,
+        opts.namespace.metadata.name
+      );
+      logger.debug(res.body);
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
+    }
+  }
+
+  async #getPod(runId: string, namespace: Namespace) {
+    try {
+      const res = await this.#k8sApi.core.readNamespacedPod(runId, namespace.metadata.name);
+      logger.debug(res.body);
+      return res.body;
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
+    }
+  }
+
+  async #createJob(job: k8s.V1Job, namespace: Namespace) {
+    try {
+      const res = await this.#k8sApi.batch.createNamespacedJob(namespace.metadata.name, job);
+      logger.debug(res.body);
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
+    }
+  }
+
+  async #createDaemonSet(daemonSet: k8s.V1DaemonSet, namespace: Namespace) {
+    try {
+      const res = await this.#k8sApi.apps.createNamespacedDaemonSet(
+        namespace.metadata.name,
+        daemonSet
+      );
+      logger.debug(res.body);
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
+    }
+  }
+
+  #throwUnlessRecord(candidate: unknown): asserts candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      throw candidate;
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    this.#throwUnlessRecord(err);
+
+    if ("body" in err && err.body) {
+      logger.error(err.body);
+      this.#throwUnlessRecord(err.body);
+
+      if (typeof err.body.message === "string") {
+        throw new Error(err.body?.message);
+      } else {
+        throw err.body;
+      }
+    } else {
+      logger.error(err);
+      throw err;
+    }
+  }
+}
+
+const provider = new ProviderShell({
+  tasks: new KubernetesTaskOperations({
+    namespace: KUBERNETES_NAMESPACE,
+  }),
+  type: "kubernetes",
+});
+
+provider.listen();
+
+const taskMonitor = new TaskMonitor({
+  runtimeEnv: RUNTIME_ENV,
+  namespace: KUBERNETES_NAMESPACE,
+  onIndexFailure: async (deploymentId, details) => {
+    logger.log("Indexing failed", { deploymentId, details });
+
+    try {
+      provider.platformSocket.send("INDEXING_FAILED", {
+        deploymentId,
+        error: {
+          name: `Crashed with exit code ${details.exitCode}`,
+          message: details.reason,
+          stack: details.logs,
+        },
+        overrideCompletion: details.overrideCompletion,
+      });
+    } catch (error) {
+      logger.error(error);
+    }
+  },
+  onRunFailure: async (runId, details) => {
+    logger.log("Run failed:", { runId, details });
+
+    try {
+      provider.platformSocket.send("WORKER_CRASHED", { runId, ...details });
+    } catch (error) {
+      logger.error(error);
+    }
+  },
+});
+
+taskMonitor.start();
+
+const podCleaner = new PodCleaner({
+  runtimeEnv: RUNTIME_ENV,
+  namespace: KUBERNETES_NAMESPACE,
+  intervalInSeconds: POD_CLEANER_INTERVAL_SECONDS,
+});
+
+podCleaner.start();
+
+if (UPTIME_HEARTBEAT_URL) {
+  const uptimeHeartbeat = new UptimeHeartbeat({
+    runtimeEnv: RUNTIME_ENV,
+    namespace: KUBERNETES_NAMESPACE,
+    intervalInSeconds: UPTIME_INTERVAL_SECONDS,
+    pingUrl: UPTIME_HEARTBEAT_URL,
+    maxPendingRuns: UPTIME_MAX_PENDING_RUNS,
+    maxPendingIndeces: UPTIME_MAX_PENDING_INDECES,
+    maxPendingErrors: UPTIME_MAX_PENDING_ERRORS,
+  });
+
+  uptimeHeartbeat.start();
+} else {
+  logger.log("Uptime heartbeat is disabled, set UPTIME_HEARTBEAT_URL to enable.");
+}

apps/kubernetes-provider/src/podCleaner.ts

@@ -0,0 +1,323 @@
+import * as k8s from "@kubernetes/client-node";
+import { SimpleLogger } from "@trigger.dev/core/v3/apps";
+
+type PodCleanerOptions = {
+  runtimeEnv: "local" | "kubernetes";
+  namespace?: string;
+  intervalInSeconds?: number;
+};
+
+export class PodCleaner {
+  private enabled = false;
+  private namespace = "default";
+  private intervalInSeconds = 300;
+
+  private logger = new SimpleLogger("[PodCleaner]");
+  private k8sClient: {
+    core: k8s.CoreV1Api;
+    apps: k8s.AppsV1Api;
+    kubeConfig: k8s.KubeConfig;
+  };
+
+  constructor(private opts: PodCleanerOptions) {
+    if (opts.namespace) {
+      this.namespace = opts.namespace;
+    }
+
+    if (opts.intervalInSeconds) {
+      this.intervalInSeconds = opts.intervalInSeconds;
+    }
+
+    this.k8sClient = this.#createK8sClient();
+  }
+
+  #createK8sClient() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (this.opts.runtimeEnv === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (this.opts.runtimeEnv === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      apps: kubeConfig.makeApiClient(k8s.AppsV1Api),
+      kubeConfig: kubeConfig,
+    };
+  }
+
+  #isRecord(candidate: unknown): candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  #logK8sError(err: unknown, debugOnly = false) {
+    if (debugOnly) {
+      this.logger.debug("K8s API Error", err);
+    } else {
+      this.logger.error("K8s API Error", err);
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
+      this.#logK8sError(err);
+      return;
+    }
+
+    this.#logK8sError(err, true);
+
+    if (typeof err.body.message === "string") {
+      this.#logK8sError({ message: err.body.message });
+      return;
+    }
+
+    this.#logK8sError({ body: err.body });
+  }
+
+  async #deletePods(opts: {
+    namespace: string;
+    dryRun?: boolean;
+    fieldSelector?: string;
+    labelSelector?: string;
+  }) {
+    return await this.k8sClient.core
+      .deleteCollectionNamespacedPod(
+        opts.namespace,
+        undefined, // pretty
+        undefined, // continue
+        opts.dryRun ? "All" : undefined,
+        opts.fieldSelector,
+        undefined, // gracePeriodSeconds
+        opts.labelSelector
+      )
+      .catch(this.#handleK8sError.bind(this));
+  }
+
+  async #deleteDaemonSets(opts: {
+    namespace: string;
+    dryRun?: boolean;
+    fieldSelector?: string;
+    labelSelector?: string;
+  }) {
+    return await this.k8sClient.apps
+      .deleteCollectionNamespacedDaemonSet(
+        opts.namespace,
+        undefined, // pretty
+        undefined, // continue
+        opts.dryRun ? "All" : undefined,
+        opts.fieldSelector,
+        undefined, // gracePeriodSeconds
+        opts.labelSelector
+      )
+      .catch(this.#handleK8sError.bind(this));
+  }
+
+  async #deleteCompletedRuns() {
+    this.logger.log("Deleting completed runs");
+
+    const start = Date.now();
+
+    const result = await this.#deletePods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=Succeeded",
+      labelSelector: "app=task-run",
+    });
+
+    const elapsedMs = Date.now() - start;
+
+    if (!result) {
+      this.logger.log("Deleting completed runs: No delete result", { elapsedMs });
+      return;
+    }
+
+    const total = (result.response as any)?.body?.items?.length ?? 0;
+
+    this.logger.log("Deleting completed runs: Done", { total, elapsedMs });
+  }
+
+  async #deleteFailedRuns() {
+    this.logger.log("Deleting failed runs");
+
+    const start = Date.now();
+
+    const result = await this.#deletePods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=Failed",
+      labelSelector: "app=task-run",
+    });
+
+    const elapsedMs = Date.now() - start;
+
+    if (!result) {
+      this.logger.log("Deleting failed runs: No delete result", { elapsedMs });
+      return;
+    }
+
+    const total = (result.response as any)?.body?.items?.length ?? 0;
+
+    this.logger.log("Deleting failed runs: Done", { total, elapsedMs });
+  }
+
+  async #deleteUnrecoverableRuns() {
+    await this.#deletePods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=?",
+      labelSelector: "app=task-run",
+    });
+  }
+
+  async #deleteCompletedPrePulls() {
+    this.logger.log("Deleting completed pre-pulls");
+
+    const start = Date.now();
+
+    const result = await this.#deleteDaemonSets({
+      namespace: this.namespace,
+      labelSelector: "app=task-prepull",
+    });
+
+    const elapsedMs = Date.now() - start;
+
+    if (!result) {
+      this.logger.log("Deleting completed pre-pulls: No delete result", { elapsedMs });
+      return;
+    }
+
+    const total = (result.response as any)?.body?.items?.length ?? 0;
+
+    this.logger.log("Deleting completed pre-pulls: Done", { total, elapsedMs });
+  }
+
+  async start() {
+    this.enabled = true;
+    this.logger.log("Starting");
+
+    const completedInterval = setInterval(async () => {
+      if (!this.enabled) {
+        clearInterval(completedInterval);
+        return;
+      }
+
+      try {
+        await this.#deleteCompletedRuns();
+      } catch (error) {
+        this.logger.error("Error deleting completed runs", error);
+      }
+    }, this.intervalInSeconds * 1000);
+
+    const failedInterval = setInterval(
+      async () => {
+        if (!this.enabled) {
+          clearInterval(failedInterval);
+          return;
+        }
+
+        try {
+          await this.#deleteFailedRuns();
+        } catch (error) {
+          this.logger.error("Error deleting completed runs", error);
+        }
+      },
+      // Use a longer interval for failed runs. This is only a backup in case the task monitor fails.
+      2 * this.intervalInSeconds * 1000
+    );
+
+    const completedPrePullInterval = setInterval(
+      async () => {
+        if (!this.enabled) {
+          clearInterval(completedPrePullInterval);
+          return;
+        }
+
+        try {
+          await this.#deleteCompletedPrePulls();
+        } catch (error) {
+          this.logger.error("Error deleting completed pre-pulls", error);
+        }
+      },
+      2 * this.intervalInSeconds * 1000
+    );
+
+    // this.#launchTests();
+  }
+
+  async stop() {
+    if (!this.enabled) {
+      return;
+    }
+
+    this.enabled = false;
+    this.logger.log("Shutting down..");
+  }
+
+  async #launchTests() {
+    const createPod = async (
+      container: k8s.V1Container,
+      name: string,
+      labels?: Record<string, string>
+    ) => {
+      this.logger.log("Creating pod:", name);
+
+      const pod = {
+        metadata: {
+          name,
+          labels,
+        },
+        spec: {
+          restartPolicy: "Never",
+          automountServiceAccountToken: false,
+          terminationGracePeriodSeconds: 1,
+          containers: [container],
+        },
+      } satisfies k8s.V1Pod;
+
+      await this.k8sClient.core
+        .createNamespacedPod(this.namespace, pod)
+        .catch(this.#handleK8sError.bind(this));
+    };
+
+    const createIdlePod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: ["-c", "sleep infinity"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createCompletedPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: ["-c", "true"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createFailedPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: ["-c", "false"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    await createIdlePod("test-idle-1", { app: "task-run" });
+    await createFailedPod("test-failed-1", { app: "task-run" });
+    await createCompletedPod("test-completed-1", { app: "task-run" });
+  }
+}

apps/kubernetes-provider/src/taskMonitor.ts

@@ -0,0 +1,459 @@
+import * as k8s from "@kubernetes/client-node";
+import { SimpleLogger } from "@trigger.dev/core/v3/apps";
+import { EXIT_CODE_ALREADY_HANDLED, EXIT_CODE_CHILD_NONZERO } from "@trigger.dev/core/v3/apps";
+import { setTimeout } from "timers/promises";
+import PQueue from "p-queue";
+import { TaskRunErrorCodes, type Prettify, type TaskRunInternalError } from "@trigger.dev/core/v3";
+
+type FailureDetails = Prettify<{
+  exitCode: number;
+  reason: string;
+  logs: string;
+  overrideCompletion: boolean;
+  errorCode: TaskRunInternalError["code"];
+}>;
+
+type IndexFailureHandler = (deploymentId: string, details: FailureDetails) => Promise<any>;
+
+type RunFailureHandler = (runId: string, details: FailureDetails) => Promise<any>;
+
+type TaskMonitorOptions = {
+  runtimeEnv: "local" | "kubernetes";
+  onIndexFailure?: IndexFailureHandler;
+  onRunFailure?: RunFailureHandler;
+  namespace?: string;
+};
+
+export class TaskMonitor {
+  #enabled = false;
+
+  #logger = new SimpleLogger("[TaskMonitor]");
+  #taskInformer: ReturnType<typeof k8s.makeInformer<k8s.V1Pod>>;
+  #processedPods = new Map<string, number>();
+  #queue = new PQueue({ concurrency: 10 });
+
+  #k8sClient: {
+    core: k8s.CoreV1Api;
+    kubeConfig: k8s.KubeConfig;
+  };
+
+  private namespace = "default";
+  private fieldSelector = "status.phase=Failed";
+  private labelSelector = "app in (task-index, task-run)";
+
+  constructor(private opts: TaskMonitorOptions) {
+    if (opts.namespace) {
+      this.namespace = opts.namespace;
+    }
+
+    this.#k8sClient = this.#createK8sClient();
+
+    this.#taskInformer = this.#createTaskInformer();
+    this.#taskInformer.on("connect", this.#onInformerConnected.bind(this));
+    this.#taskInformer.on("error", this.#onInformerError.bind(this));
+    this.#taskInformer.on("update", this.#enqueueOnPodUpdated.bind(this));
+  }
+
+  #createTaskInformer() {
+    const listTasks = () =>
+      this.#k8sClient.core.listNamespacedPod(
+        this.namespace,
+        undefined,
+        undefined,
+        undefined,
+        this.fieldSelector,
+        this.labelSelector
+      );
+
+    // Uses watch with local caching
+    // https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes
+    const informer = k8s.makeInformer(
+      this.#k8sClient.kubeConfig,
+      `/api/v1/namespaces/${this.namespace}/pods`,
+      listTasks,
+      this.labelSelector,
+      this.fieldSelector
+    );
+
+    return informer;
+  }
+
+  async #onInformerConnected() {
+    this.#logger.log("Connected");
+  }
+
+  async #onInformerError(error: any) {
+    this.#logger.error("Error:", error);
+
+    // Automatic reconnect
+    await setTimeout(2_000);
+    this.#taskInformer.start();
+  }
+
+  #enqueueOnPodUpdated(pod: k8s.V1Pod) {
+    this.#queue.add(async () => {
+      try {
+        // It would be better to only pass the cache key, but the pod may already be removed from the cache by the time we process it
+        await this.#onPodUpdated(pod);
+      } catch (error) {
+        this.#logger.error("Caught onPodUpdated() error:", error);
+      }
+    });
+  }
+
+  async #onPodUpdated(pod: k8s.V1Pod) {
+    this.#logger.debug(`Updated: ${pod.metadata?.name}`);
+    this.#logger.debug("Updated", JSON.stringify(pod, null, 2));
+
+    // We only care about failures
+    if (pod.status?.phase !== "Failed") {
+      return;
+    }
+
+    const podName = pod.metadata?.name;
+
+    if (!podName) {
+      this.#logger.error("Pod is nameless", { pod });
+      return;
+    }
+
+    const containerStatus = pod.status.containerStatuses?.[0];
+
+    if (!containerStatus?.state) {
+      this.#logger.error("Pod failed, but container status doesn't have state", {
+        status: pod.status,
+      });
+      return;
+    }
+
+    if (this.#processedPods.has(podName)) {
+      this.#logger.debug("Pod update already processed", {
+        podName,
+        timestamp: this.#processedPods.get(podName),
+      });
+      return;
+    }
+
+    this.#processedPods.set(podName, Date.now());
+
+    const podStatus = this.#getPodStatusSummary(pod.status);
+    const containerState = this.#getContainerStateSummary(containerStatus.state);
+    const exitCode = containerState.exitCode ?? -1;
+
+    if (exitCode === EXIT_CODE_ALREADY_HANDLED) {
+      this.#logger.debug("Ignoring pod failure, already handled by worker", {
+        podName,
+      });
+      return;
+    }
+
+    const rawLogs = await this.#getLogTail(podName);
+
+    this.#logger.log(`${podName} failed with:`, {
+      podStatus,
+      containerState,
+      rawLogs,
+    });
+
+    const rawReason = podStatus.reason ?? containerState.reason ?? "";
+    const message = podStatus.message ?? containerState.message ?? "";
+
+    let reason = rawReason || "Unknown error";
+    let logs = rawLogs || "";
+
+    /** This will only override existing task errors. It will not crash the run.  */
+    let onlyOverrideExistingError = exitCode === EXIT_CODE_CHILD_NONZERO;
+
+    let errorCode: TaskRunInternalError["code"] = TaskRunErrorCodes.POD_UNKNOWN_ERROR;
+
+    switch (rawReason) {
+      case "Error":
+        reason = "Unknown error.";
+        errorCode = TaskRunErrorCodes.POD_UNKNOWN_ERROR;
+        break;
+      case "Evicted":
+        if (message.startsWith("Pod ephemeral local storage usage")) {
+          reason = "Storage limit exceeded.";
+          errorCode = TaskRunErrorCodes.DISK_SPACE_EXCEEDED;
+        } else if (message) {
+          reason = `Evicted: ${message}`;
+          errorCode = TaskRunErrorCodes.POD_EVICTED;
+        } else {
+          reason = "Evicted for unknown reason.";
+          errorCode = TaskRunErrorCodes.POD_EVICTED;
+        }
+
+        if (logs.startsWith("failed to try resolving symlinks")) {
+          logs = "";
+        }
+        break;
+      case "OOMKilled":
+        reason =
+          "[TaskMonitor] Your task ran out of memory. Try increasing the machine specs. If this doesn't fix it there might be a memory leak.";
+        errorCode = TaskRunErrorCodes.TASK_PROCESS_OOM_KILLED;
+        break;
+      default:
+        break;
+    }
+
+    const failureInfo = {
+      exitCode,
+      reason,
+      logs,
+      overrideCompletion: onlyOverrideExistingError,
+      errorCode,
+    } satisfies FailureDetails;
+
+    const app = pod.metadata?.labels?.app;
+
+    switch (app) {
+      case "task-index":
+        const deploymentId = pod.metadata?.labels?.deployment;
+
+        if (!deploymentId) {
+          this.#logger.error("Index is missing ID", { pod });
+          return;
+        }
+
+        if (this.opts.onIndexFailure) {
+          await this.opts.onIndexFailure(deploymentId, failureInfo);
+        }
+        break;
+      case "task-run":
+        const runId = pod.metadata?.labels?.run;
+
+        if (!runId) {
+          this.#logger.error("Run is missing ID", { pod });
+          return;
+        }
+
+        if (this.opts.onRunFailure) {
+          await this.opts.onRunFailure(runId, failureInfo);
+        }
+        break;
+      default:
+        this.#logger.error("Pod has invalid app label", { pod });
+        return;
+    }
+
+    await this.#deletePod(podName);
+  }
+
+  async #getLogTail(podName: string) {
+    try {
+      const logs = await this.#k8sClient.core.readNamespacedPodLog(
+        podName,
+        this.namespace,
+        undefined,
+        undefined,
+        undefined,
+        1024, // limitBytes
+        undefined,
+        undefined,
+        undefined,
+        20 // tailLines
+      );
+
+      const responseBody = logs.body ?? "";
+
+      if (responseBody.startsWith("unable to retrieve container logs")) {
+        return "";
+      }
+
+      // Type is wrong, body may be undefined
+      return responseBody;
+    } catch (error) {
+      this.#logger.error("Log tail error:", error instanceof Error ? error.message : "unknown");
+      return "";
+    }
+  }
+
+  #getPodStatusSummary(status: k8s.V1PodStatus) {
+    return {
+      reason: status.reason,
+      message: status.message,
+    };
+  }
+
+  #getContainerStateSummary(state: k8s.V1ContainerState) {
+    return {
+      reason: state.terminated?.reason,
+      exitCode: state.terminated?.exitCode,
+      message: state.terminated?.message,
+    };
+  }
+
+  #createK8sClient() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (this.opts.runtimeEnv === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (this.opts.runtimeEnv === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      kubeConfig: kubeConfig,
+    };
+  }
+
+  #isRecord(candidate: unknown): candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  #logK8sError(err: unknown, debugOnly = false) {
+    if (debugOnly) {
+      this.#logger.debug("K8s API Error", err);
+    } else {
+      this.#logger.error("K8s API Error", err);
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
+      this.#logK8sError(err);
+      return;
+    }
+
+    this.#logK8sError(err, true);
+
+    if (typeof err.body.message === "string") {
+      this.#logK8sError({ message: err.body.message });
+      return;
+    }
+
+    this.#logK8sError({ body: err.body });
+  }
+
+  #printStats(includeMoreDetails = false) {
+    this.#logger.log("Stats:", {
+      cacheSize: this.#taskInformer.list().length,
+      totalProcessed: this.#processedPods.size,
+      ...(includeMoreDetails && {
+        processedPods: this.#processedPods,
+      }),
+    });
+  }
+
+  async #deletePod(name: string) {
+    this.#logger.debug("Deleting pod:", name);
+
+    await this.#k8sClient.core
+      .deleteNamespacedPod(name, this.namespace)
+      .catch(this.#handleK8sError.bind(this));
+  }
+
+  async start() {
+    this.#enabled = true;
+
+    const interval = setInterval(() => {
+      if (!this.#enabled) {
+        clearInterval(interval);
+        return;
+      }
+
+      this.#printStats();
+    }, 300_000);
+
+    await this.#taskInformer.start();
+
+    // this.#launchTests();
+  }
+
+  async stop() {
+    if (!this.#enabled) {
+      return;
+    }
+
+    this.#enabled = false;
+    this.#logger.log("Shutting down..");
+
+    await this.#taskInformer.stop();
+
+    this.#printStats(true);
+  }
+
+  async #launchTests() {
+    const createPod = async (
+      container: k8s.V1Container,
+      name: string,
+      labels?: Record<string, string>
+    ) => {
+      this.#logger.log("Creating pod:", name);
+
+      const pod = {
+        metadata: {
+          name,
+          labels,
+        },
+        spec: {
+          restartPolicy: "Never",
+          automountServiceAccountToken: false,
+          terminationGracePeriodSeconds: 1,
+          containers: [container],
+        },
+      } satisfies k8s.V1Pod;
+
+      await this.#k8sClient.core
+        .createNamespacedPod(this.namespace, pod)
+        .catch(this.#handleK8sError.bind(this));
+    };
+
+    const createOomPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "polinux/stress",
+        resources: {
+          limits: {
+            memory: "100Mi",
+          },
+        },
+        command: ["stress"],
+        args: ["--vm", "1", "--vm-bytes", "150M", "--vm-hang", "1"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createNonZeroExitPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: ["-c", "exit 1"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createOoDiskPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: [
+          "-c",
+          "echo creating huge-file..; head -c 1000m /dev/zero > huge-file; ls -lh huge-file; sleep infinity",
+        ],
+        resources: {
+          limits: {
+            "ephemeral-storage": "500Mi",
+          },
+        },
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    await createNonZeroExitPod("non-zero-exit-task", { app: "task-run", run: "123" });
+    await createOomPod("oom-task", { app: "task-index", deployment: "456" });
+    await createOoDiskPod("ood-task", { app: "task-run", run: "abc" });
+  }
+}

apps/kubernetes-provider/src/uptimeHeartbeat.ts

@@ -0,0 +1,272 @@
+import * as k8s from "@kubernetes/client-node";
+import { SimpleLogger } from "@trigger.dev/core/v3/apps";
+
+type UptimeHeartbeatOptions = {
+  runtimeEnv: "local" | "kubernetes";
+  pingUrl: string;
+  namespace?: string;
+  intervalInSeconds?: number;
+  maxPendingRuns?: number;
+  maxPendingIndeces?: number;
+  maxPendingErrors?: number;
+  leadingEdge?: boolean;
+};
+
+export class UptimeHeartbeat {
+  private enabled = false;
+  private namespace: string;
+
+  private intervalInSeconds: number;
+  private maxPendingRuns: number;
+  private maxPendingIndeces: number;
+  private maxPendingErrors: number;
+
+  private leadingEdge = true;
+
+  private logger = new SimpleLogger("[UptimeHeartbeat]");
+  private k8sClient: {
+    core: k8s.CoreV1Api;
+    kubeConfig: k8s.KubeConfig;
+  };
+
+  constructor(private opts: UptimeHeartbeatOptions) {
+    this.namespace = opts.namespace ?? "default";
+
+    this.intervalInSeconds = opts.intervalInSeconds ?? 60;
+    this.maxPendingRuns = opts.maxPendingRuns ?? 25;
+    this.maxPendingIndeces = opts.maxPendingIndeces ?? 10;
+    this.maxPendingErrors = opts.maxPendingErrors ?? 10;
+
+    this.k8sClient = this.#createK8sClient();
+  }
+
+  #createK8sClient() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (this.opts.runtimeEnv === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (this.opts.runtimeEnv === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      kubeConfig: kubeConfig,
+    };
+  }
+
+  #isRecord(candidate: unknown): candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  #logK8sError(err: unknown, debugOnly = false) {
+    if (debugOnly) {
+      this.logger.debug("K8s API Error", err);
+    } else {
+      this.logger.error("K8s API Error", err);
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
+      this.#logK8sError(err);
+      return;
+    }
+
+    this.#logK8sError(err, true);
+
+    if (typeof err.body.message === "string") {
+      this.#logK8sError({ message: err.body.message });
+      return;
+    }
+
+    this.#logK8sError({ body: err.body });
+  }
+
+  async #getPods(opts: {
+    namespace: string;
+    fieldSelector?: string;
+    labelSelector?: string;
+  }): Promise<Array<k8s.V1Pod> | undefined> {
+    const listReturn = await this.k8sClient.core
+      .listNamespacedPod(
+        opts.namespace,
+        undefined, // pretty
+        undefined, // allowWatchBookmarks
+        undefined, // _continue
+        opts.fieldSelector,
+        opts.labelSelector,
+        this.maxPendingRuns * 2, // limit
+        undefined, // resourceVersion
+        undefined, // resourceVersionMatch
+        undefined, // sendInitialEvents
+        this.intervalInSeconds, // timeoutSeconds,
+        undefined // watch
+      )
+      .catch(this.#handleK8sError.bind(this));
+
+    return listReturn?.body.items;
+  }
+
+  async #getPendingIndeces(): Promise<Array<k8s.V1Pod> | undefined> {
+    return await this.#getPods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=Pending",
+      labelSelector: "app=task-index",
+    });
+  }
+
+  async #getPendingTasks(): Promise<Array<k8s.V1Pod> | undefined> {
+    return await this.#getPods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=Pending",
+      labelSelector: "app=task-run",
+    });
+  }
+
+  #countPods(pods: Array<k8s.V1Pod>): number {
+    return pods.length;
+  }
+
+  #filterPendingPods(
+    pods: Array<k8s.V1Pod>,
+    waitingReason: "CreateContainerError" | "RunContainerError"
+  ): Array<k8s.V1Pod> {
+    return pods.filter((pod) => {
+      const containerStatus = pod.status?.containerStatuses?.[0];
+      return containerStatus?.state?.waiting?.reason === waitingReason;
+    });
+  }
+
+  async #sendPing() {
+    this.logger.log("Sending ping");
+
+    const start = Date.now();
+    const controller = new AbortController();
+
+    const timeoutMs = (this.intervalInSeconds * 1000) / 2;
+
+    const fetchTimeout = setTimeout(() => {
+      controller.abort();
+    }, timeoutMs);
+
+    try {
+      const response = await fetch(this.opts.pingUrl, {
+        signal: controller.signal,
+      });
+
+      if (!response.ok) {
+        this.logger.error("Failed to send ping, response not OK", {
+          status: response.status,
+        });
+        return;
+      }
+
+      const elapsedMs = Date.now() - start;
+      this.logger.log("Ping sent", { elapsedMs });
+    } catch (error) {
+      if (error instanceof DOMException && error.name === "AbortError") {
+        this.logger.log("Ping timeout", { timeoutSeconds: timeoutMs });
+        return;
+      }
+
+      this.logger.error("Failed to send ping", error);
+    } finally {
+      clearTimeout(fetchTimeout);
+    }
+  }
+
+  async #heartbeat() {
+    this.logger.log("Performing heartbeat");
+
+    const start = Date.now();
+
+    const pendingTasks = await this.#getPendingTasks();
+
+    if (!pendingTasks) {
+      this.logger.error("Failed to get pending tasks");
+      return;
+    }
+
+    const totalPendingTasks = this.#countPods(pendingTasks);
+
+    const pendingIndeces = await this.#getPendingIndeces();
+
+    if (!pendingIndeces) {
+      this.logger.error("Failed to get pending indeces");
+      return;
+    }
+
+    const totalPendingIndeces = this.#countPods(pendingIndeces);
+
+    const elapsedMs = Date.now() - start;
+
+    this.logger.log("Finished heartbeat checks", { elapsedMs });
+
+    if (totalPendingTasks > this.maxPendingRuns) {
+      this.logger.log("Too many pending tasks, skipping heartbeat", { totalPendingTasks });
+      return;
+    }
+
+    if (totalPendingIndeces > this.maxPendingIndeces) {
+      this.logger.log("Too many pending indeces, skipping heartbeat", { totalPendingIndeces });
+      return;
+    }
+
+    const totalCreateContainerErrors = this.#countPods(
+      this.#filterPendingPods(pendingTasks, "CreateContainerError")
+    );
+    const totalRunContainerErrors = this.#countPods(
+      this.#filterPendingPods(pendingTasks, "RunContainerError")
+    );
+
+    if (totalCreateContainerErrors + totalRunContainerErrors > this.maxPendingErrors) {
+      this.logger.log("Too many pending tasks with errors, skipping heartbeat", {
+        totalRunContainerErrors,
+        totalCreateContainerErrors,
+      });
+      return;
+    }
+
+    await this.#sendPing();
+
+    this.logger.log("Heartbeat done", { totalPendingTasks, elapsedMs });
+  }
+
+  async start() {
+    this.enabled = true;
+    this.logger.log("Starting");
+
+    if (this.leadingEdge) {
+      await this.#heartbeat();
+    }
+
+    const heartbeat = setInterval(async () => {
+      if (!this.enabled) {
+        clearInterval(heartbeat);
+        return;
+      }
+
+      try {
+        await this.#heartbeat();
+      } catch (error) {
+        this.logger.error("Error while heartbeating", error);
+      }
+    }, this.intervalInSeconds * 1000);
+  }
+
+  async stop() {
+    if (!this.enabled) {
+      return;
+    }
+
+    this.enabled = false;
+    this.logger.log("Shutting down..");
+  }
+}

apps/kubernetes-provider/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "es2016",
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "paths": {
+      "@trigger.dev/core/v3": ["../../packages/core/src/v3"],
+      "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"]
+    }
+  }
+}

apps/proxy/package.json

@@ -4,18 +4,19 @@
   "private": true,
   "scripts": {
     "deploy": "wrangler deploy",
-    "dev": "wrangler dev"
+    "dev": "wrangler dev",
+    "dry-run:staging": "wrangler deploy --dry-run --outdir=dist --env staging"
   },
   "devDependencies": {
-    "@cloudflare/workers-types": "^4.20230419.0",
+    "@cloudflare/workers-types": "^4.20240512.0",
     "typescript": "^5.0.4",
-    "wrangler": "^3.0.0"
+    "wrangler": "^3.57.1"
   },
   "dependencies": {
     "@aws-sdk/client-sqs": "^3.445.0",
     "@trigger.dev/core": "workspace:*",
     "ulidx": "^2.2.1",
-    "zod": "3.22.3",
+    "zod": "3.23.8",
     "zod-error": "1.5.0"
   }
-}
+}

apps/proxy/src/index.ts

@@ -1,5 +1,7 @@
 import { queueEvent } from "./events/queueEvent";
 import { queueEvents } from "./events/queueEvents";
+import { applyRateLimit } from "./rateLimit";
+import { Ratelimit } from "./rateLimiter";
 
 export interface Env {
   /** The hostname needs to be changed to allow requests to pass to the Trigger.dev platform */
@@ -9,54 +11,38 @@ export interface Env {
   AWS_SQS_SECRET_ACCESS_KEY: string;
   AWS_SQS_QUEUE_URL: string;
   AWS_SQS_REGION: string;
+  //rate limiter
+  API_RATE_LIMITER: Ratelimit;
 }
 
 export default {
   async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
-    if (!env.REWRITE_HOSTNAME) throw new Error("Missing REWRITE_HOSTNAME");
-    console.log("url", request.url);
-
     if (!queueingIsEnabled(env)) {
       console.log("Missing AWS credentials. Passing through to the origin.");
-      return redirectToOrigin(request, env);
+      return fetch(request);
     }
 
     const url = new URL(request.url);
     switch (url.pathname) {
       case "/api/v1/events": {
         if (request.method === "POST") {
-          return queueEvent(request, env);
+          return applyRateLimit(request, env, () => queueEvent(request, env));
         }
         break;
       }
       case "/api/v1/events/bulk": {
         if (request.method === "POST") {
-          return queueEvents(request, env);
+          return applyRateLimit(request, env, () => queueEvents(request, env));
         }
         break;
       }
     }
 
     //the same request but with the hostname (and port) changed
-    return redirectToOrigin(request, env);
+    return fetch(request);
   },
 };
 
-function redirectToOrigin(request: Request, env: Env) {
-  const newUrl = new URL(request.url);
-  newUrl.hostname = env.REWRITE_HOSTNAME;
-  newUrl.port = env.REWRITE_PORT || newUrl.port;
-
-  const requestInit: RequestInit = {
-    method: request.method,
-    headers: request.headers,
-    body: request.body,
-  };
-
-  console.log("rewritten url", newUrl.toString());
-  return fetch(newUrl.toString(), requestInit);
-}
-
 function queueingIsEnabled(env: Env) {
   return (
     env.AWS_SQS_ACCESS_KEY_ID &&

apps/proxy/src/rateLimit.ts

@@ -0,0 +1,46 @@
+import { Env } from "src";
+import { getApiKeyFromRequest } from "./apikey";
+import { json } from "./json";
+
+export async function applyRateLimit(
+  request: Request,
+  env: Env,
+  fn: () => Promise<Response>
+): Promise<Response> {
+  const apiKey = getApiKeyFromRequest(request);
+  if (apiKey) {
+    const result = await env.API_RATE_LIMITER.limit({ key: `apikey-${apiKey.apiKey}` });
+    const { success } = result;
+    console.log(`Rate limiter`, {
+      success,
+      key: `${apiKey.apiKey.substring(0, 12)}...`,
+    });
+    if (!success) {
+      //60s in the future
+      const reset = Date.now() + 60 * 1000;
+      const secondsUntilReset = Math.max(0, (reset - new Date().getTime()) / 1000);
+
+      return json(
+        {
+          title: "Rate Limit Exceeded",
+          status: 429,
+          type: "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429",
+          detail: `Rate limit exceeded. Retry in ${secondsUntilReset} seconds.`,
+          error: `Rate limit exceeded. Retry in ${secondsUntilReset} seconds.`,
+          reset,
+        },
+        {
+          status: 429,
+          headers: {
+            "x-ratelimit-reset": reset.toString(),
+          },
+        }
+      );
+    }
+  } else {
+    console.log(`Rate limiter: no API key for request`);
+  }
+
+  //call the original function
+  return fn();
+}

apps/proxy/src/rateLimiter.ts

@@ -0,0 +1,23 @@
+export interface Ratelimit {
+  /*
+   * The ratelimit function
+   * @param {RatelimitOptions} options
+   * @returns {Promise<RatelimitResponse>}
+   */
+  limit: (options: RatelimitOptions) => Promise<RatelimitResponse>;
+}
+
+export interface RatelimitOptions {
+  /*
+   * The key to identify the user, can be an IP address, user ID, etc.
+   */
+  key: string;
+}
+
+export interface RatelimitResponse {
+  /*
+   * The ratelimit success status
+   * @returns {boolean}
+   */
+  success: boolean;
+}

apps/proxy/wrangler.toml

@@ -1,7 +1,33 @@
 name = "proxy"
 main = "src/index.ts"
-compatibility_date = "2023-10-30"
+compatibility_date = "2024-05-13"
 compatibility_flags = [ "nodejs_compat" ]
 
 [env.staging]
-[env.prod]
+  # The rate limiting API is in open beta.
+  [[env.staging.unsafe.bindings]]
+  name = "API_RATE_LIMITER"
+  type = "ratelimit"
+  # An identifier you define, that is unique to your Cloudflare account.
+  # Must be an integer.
+  namespace_id = "1"
+
+  # Limit: the number of tokens allowed within a given period in a single
+  # Cloudflare location
+  # Period: the duration of the period, in seconds. Must be either 10 or 60
+  simple = { limit = 100, period = 60 }
+
+
+[env.prod]
+  # The rate limiting API is in open beta.
+  [[env.prod.unsafe.bindings]]
+  name = "API_RATE_LIMITER"
+  type = "ratelimit"
+  # An identifier you define, that is unique to your Cloudflare account.
+  # Must be an integer.
+  namespace_id = "2"
+
+  # Limit: the number of tokens allowed within a given period in a single
+  # Cloudflare location
+  # Period: the duration of the period, in seconds. Must be either 10 or 60
+  simple = { limit = 300, period = 60 }

apps/webapp/.eslintrc

@@ -1,13 +1,36 @@
 {
-  "extends": ["@remix-run/eslint-config", "@remix-run/eslint-config/node", "prettier"],
-  "rules": {
-    "@typescript-eslint/strict-boolean-expressions": [
-      "error",
-      {
-        "allowNullableBoolean": true,
-        "allowNullableString": true,
-        "allowNullableNumber": true
+  "plugins": [
+    "@trigger.dev/eslint-plugin",
+    "react-hooks",
+    "@typescript-eslint/eslint-plugin",
+    "import"
+  ],
+  "parser": "@typescript-eslint/parser",
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx"],
+      "rules": {
+        // Autofixes imports from "@trigger.dev/core" to fine grained modules
+        "@trigger.dev/no-trigger-core-import": "error",
+        // Normalize `import type {}` and `import { type }`
+        "@typescript-eslint/consistent-type-imports": [
+          "warn",
+          {
+            // the "type" annotation can get tangled and cause syntax errors
+            // during some autofixes, so easier to just turn it off
+            "prefer": "type-imports",
+            "disallowTypeAnnotations": true,
+            "fixStyle": "inline-type-imports"
+          }
+        ],
+        // no-trigger-core-import splits imports into multiple lines
+        // this one merges them back into a single line
+        // if they still import from the same module
+        "import/no-duplicates": ["warn", { "prefer-inline": true }],
+        // lots of undeclared vars, enable this rule if you want to clean them up
+        "turbo/no-undeclared-env-vars": "off"
       }
-    ]
-  }
+    }
+  ],
+  "ignorePatterns": ["seed.js", "seedCloud.ts", "populate.js"]
 }

apps/webapp/.gitignore

@@ -17,4 +17,5 @@ build-storybook.log
 .storybook-out
 storybook-static
 
-/prisma/seed.js
+/prisma/seed.js
+/prisma/populate.js

apps/webapp/.storybook/main.ts

@@ -1,49 +0,0 @@
-import type { StorybookConfig } from "@storybook/react-webpack5";
-import path from "path";
-
-const root = path.resolve(__dirname, "../app");
-
-const config: StorybookConfig = {
-  webpackFinal: async (config) => {
-    return {
-      ...config,
-      resolve: {
-        ...config.resolve,
-        alias: {
-          ...(config.resolve?.alias ?? {}),
-          "~": root,
-        },
-        extensions: [
-          ...(config.resolve?.extensions ?? []),
-          ...[".ts", ".tsx", ".js", ".jsx", ".mdx"],
-        ],
-      },
-    };
-  },
-  stories: ["../app/**/stories/*.mdx", "../app/**/stories/*.stories.@(js|jsx|ts|tsx)"],
-  addons: [
-    "@storybook/addon-links",
-    "@storybook/addon-essentials",
-    "@storybook/addon-interactions",
-    "storybook-addon-variants",
-    "storybook-addon-designs",
-    "@storybook/addon-docs",
-    {
-      name: "@storybook/addon-styling",
-      options: {
-        // Check out https://github.com/storybookjs/addon-styling/blob/main/docs/api.md
-        // For more details on this addon's options.
-        postCss: true,
-      },
-    },
-  ],
-  framework: {
-    name: "@storybook/react-webpack5",
-    options: {},
-  },
-  docs: {
-    autodocs: "tag",
-  },
-  staticDirs: [path.resolve("public")],
-};
-export default config;

apps/webapp/.storybook/preview.tsx

@@ -1,50 +0,0 @@
-import type { Preview } from "@storybook/react";
-import "../app/tailwind.css";
-import { createRemixStub } from "@remix-run/testing";
-import React from "react";
-import { LocaleContextProvider } from "../app/components/primitives/LocaleProvider";
-import { OperatingSystemContextProvider } from "../app/components/primitives/OperatingSystemProvider";
-
-const preview: Preview = {
-  parameters: {
-    actions: { argTypesRegex: "^on[A-Z].*" },
-    controls: {
-      matchers: {
-        color: /(background|color)$/i,
-        date: /Date$/,
-      },
-    },
-    backgrounds: {
-      default: "App background",
-      values: [
-        {
-          name: "App background",
-          value: "#0B1018",
-        },
-      ],
-    },
-    layout: "fullscreen",
-  },
-  decorators: [
-    (Story) => {
-      const RemixStub = createRemixStub([
-        {
-          path: "/*",
-          action: () => ({ redirect: "/" }),
-          loader: () => ({ redirect: "/" }),
-          Component: Story,
-        },
-      ]);
-
-      return (
-        <OperatingSystemContextProvider platform="mac">
-          <LocaleContextProvider locales={window.navigator.languages as string[]}>
-            <RemixStub initialEntries={["/"]} />
-          </LocaleContextProvider>
-        </OperatingSystemContextProvider>
-      );
-    },
-  ],
-};
-
-export default preview;

apps/webapp/app/assets/icons/AISparkleIcon.tsx

@@ -0,0 +1,53 @@
+export function AISparkleIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M14.9806 0.803884C14.8871 0.33646 14.4767 0 14 0C13.5233 0 13.1129 0.33646 13.0194 0.803884L12.7809 1.99644C12.7017 2.3923 12.3923 2.70174 11.9964 2.78091L10.8039 3.01942C10.3365 3.1129 10 3.52332 10 4C10 4.47668 10.3365 4.8871 10.8039 4.98058L11.9964 5.21909C12.3923 5.29826 12.7017 5.6077 12.7809 6.00356L13.0194 7.19612C13.1129 7.66354 13.5233 8 14 8C14.4767 8 14.8871 7.66354 14.9806 7.19612L15.2191 6.00356C15.2983 5.6077 15.6077 5.29826 16.0036 5.21909L17.1961 4.98058C17.6635 4.8871 18 4.47668 18 4C18 3.52332 17.6635 3.1129 17.1961 3.01942L16.0036 2.78091C15.6077 2.70174 15.2983 2.3923 15.2191 1.99644L14.9806 0.803884Z"
+        fill="url(#paint0_linear_11402_36656)"
+      />
+      <path
+        d="M5.94868 4.68377C5.81257 4.27543 5.43043 4 5 4C4.56957 4 4.18743 4.27543 4.05132 4.68377L3.36754 6.73509C3.26801 7.03369 3.03369 7.26801 2.73509 7.36754L0.683772 8.05132C0.27543 8.18743 0 8.56957 0 9C0 9.43043 0.27543 9.81257 0.683772 9.94868L2.73509 10.6325C3.03369 10.732 3.26801 10.9663 3.36754 11.2649L4.05132 13.3162C4.18743 13.7246 4.56957 14 5 14C5.43043 14 5.81257 13.7246 5.94868 13.3162L6.63246 11.2649C6.73199 10.9663 6.96631 10.732 7.26491 10.6325L9.31623 9.94868C9.72457 9.81257 10 9.43043 10 9C10 8.56957 9.72457 8.18743 9.31623 8.05132L7.26491 7.36754C6.96631 7.26801 6.73199 7.03369 6.63246 6.73509L5.94868 4.68377Z"
+        fill="url(#paint1_linear_11402_36656)"
+      />
+      <path
+        d="M12.9487 12.6838C12.8126 12.2754 12.4304 12 12 12C11.5696 12 11.1874 12.2754 11.0513 12.6838L10.8675 13.2351C10.768 13.5337 10.5337 13.768 10.2351 13.8675L9.68377 14.0513C9.27543 14.1874 9 14.5696 9 15C9 15.4304 9.27543 15.8126 9.68377 15.9487L10.2351 16.1325C10.5337 16.232 10.768 16.4663 10.8675 16.7649L11.0513 17.3162C11.1874 17.7246 11.5696 18 12 18C12.4304 18 12.8126 17.7246 12.9487 17.3162L13.1325 16.7649C13.232 16.4663 13.4663 16.232 13.7649 16.1325L14.3162 15.9487C14.7246 15.8126 15 15.4304 15 15C15 14.5696 14.7246 14.1874 14.3162 14.0513L13.7649 13.8675C13.4663 13.768 13.232 13.5337 13.1325 13.2351L12.9487 12.6838Z"
+        fill="url(#paint2_linear_11402_36656)"
+      />
+      <defs>
+        <linearGradient
+          id="paint0_linear_11402_36656"
+          x1="9"
+          y1="0"
+          x2="9"
+          y2="18"
+          gradientUnits="userSpaceOnUse"
+        >
+          <stop stopColor="#E543FF" />
+          <stop offset="1" stopColor="#286399" />
+        </linearGradient>
+        <linearGradient
+          id="paint1_linear_11402_36656"
+          x1="9"
+          y1="0"
+          x2="9"
+          y2="18"
+          gradientUnits="userSpaceOnUse"
+        >
+          <stop stopColor="#E543FF" />
+          <stop offset="1" stopColor="#286399" />
+        </linearGradient>
+        <linearGradient
+          id="paint2_linear_11402_36656"
+          x1="9"
+          y1="0"
+          x2="9"
+          y2="18"
+          gradientUnits="userSpaceOnUse"
+        >
+          <stop stopColor="#E543FF" />
+          <stop offset="1" stopColor="#286399" />
+        </linearGradient>
+      </defs>
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/AttemptIcon.tsx

@@ -0,0 +1,29 @@
+export function AttemptIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="16"
+      height="16"
+      viewBox="0 0 16 16"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <g clipPath="url(#clip0_9964_113464)">
+        <path
+          fillRule="evenodd"
+          clipRule="evenodd"
+          d="M16 0H0V16H16V0ZM7.09906 4.4L4.53906 11.5H6.11906L6.63906 10H9.35906L9.87906 11.5H11.4591L8.89906 4.4H7.09906ZM7.99906 6L8.92906 8.73H7.06906L7.99906 6Z"
+          fill="currentColor"
+        />
+      </g>
+      <defs>
+        <clipPath id="clip0_9964_113464">
+          <path
+            d="M0 2C0 0.895431 0.895431 0 2 0H14C15.1046 0 16 0.895431 16 2V14C16 15.1046 15.1046 16 14 16H2C0.895431 16 0 15.1046 0 14V2Z"
+            fill="white"
+          />
+        </clipPath>
+      </defs>
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/ExitIcon.tsx

@@ -0,0 +1,14 @@
+export function ExitIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <line x1="3.5" y1="8" x2="11.5" y2="8" stroke="currentColor" strokeLinecap="round" />
+      <line x1="15.5" y1="1.5" x2="15.5" y2="14.5" stroke="currentColor" strokeLinecap="round" />
+      <path
+        d="M8.5 4.5L12 8L8.5 11.5"
+        stroke="currentColor"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/ShowParentIcon.tsx

@@ -0,0 +1,25 @@
+export function ShowParentIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        fill="currentColor"
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M3.125 16H3.625H4.125V15H3.125H2C1.89733 15 1.8 14.9848 1.70919 14.9573L1.19765 14.8024L1.04267 14.2908C1.01515 14.2 1 14.1027 1 14V12.875V11.875H0V12.375V12.875V14C0 14.202 0.0299409 14.397 0.0856265 14.5808C0.135047 14.7439 0.204744 14.8982 0.291786 15.0407C0.363386 15.158 0.446722 15.2673 0.540161 15.3671C0.570065 15.399 0.601004 15.4299 0.632924 15.4598C0.732663 15.5533 0.841982 15.6366 0.95925 15.7082C1.10181 15.7953 1.25612 15.865 1.41923 15.9144C1.60303 15.9701 1.79802 16 2 16H3.125ZM10.125 15V16H9.625H9.125H6.875H6.375H5.875V15H6.875H9.125H10.125ZM15 5.875H16V6.375V6.875V9.125V9.625V10.125H15V9.125V6.875V5.875ZM5.875 1V0H6.375H6.875H9.125H9.625H10.125V1H9.125H6.875H5.875ZM1 10.125H0V9.625V9.125V6.875V6.375V5.875H1V6.875V9.125V10.125ZM0 3.625V4.125H1V3.125V2C1 1.89733 1.01515 1.8 1.04267 1.70919L1.19765 1.19765L1.70919 1.04267C1.8 1.01515 1.89733 1 2 1H3.125H4.125V0H3.625H3.125H2C1.79802 0 1.60303 0.0299409 1.41923 0.0856264C1.25612 0.135047 1.10181 0.204744 0.95925 0.291786C0.841983 0.363386 0.732663 0.446722 0.632924 0.540161C0.601004 0.570065 0.570065 0.601004 0.540161 0.632924C0.446722 0.732663 0.363386 0.841982 0.291786 0.95925C0.204744 1.10181 0.135047 1.25612 0.0856265 1.41923C0.0299409 1.60303 0 1.79802 0 2V3.125V3.625ZM12.375 0H11.875V1H12.875H14C14.1027 1 14.2 1.01515 14.2908 1.04267L14.8024 1.19765L14.9573 1.70919C14.9848 1.8 15 1.89733 15 2V3.125V4.125H16V3.625V3.125V2C16 1.79802 15.9701 1.60303 15.9144 1.41923C15.865 1.25612 15.7953 1.10181 15.7082 0.95925C15.6366 0.841982 15.5533 0.732663 15.4598 0.632924C15.4299 0.601004 15.399 0.570065 15.3671 0.540161C15.2673 0.446722 15.158 0.363386 15.0407 0.291786C14.8982 0.204744 14.7439 0.135047 14.5808 0.0856265C14.397 0.0299409 14.202 0 14 0H12.875H12.375ZM16 12.375V11.875H15V12.875V14C15 14.1027 14.9848 14.2 14.9573 14.2908L14.8024 14.8024L14.2908 14.9573C14.2 14.9848 14.1027 15 14 15H12.875H11.875V16H12.375H12.875H14C14.202 16 14.397 15.9701 14.5808 15.9144C14.7439 15.865 14.8982 15.7953 15.0407 15.7082C15.158 15.6366 15.2673 15.5533 15.3671 15.4598C15.399 15.4299 15.4299 15.399 15.4598 15.3671C15.5533 15.2673 15.6366 15.158 15.7082 15.0407C15.7953 14.8982 15.865 14.7439 15.9144 14.5808C15.9701 14.397 16 14.202 16 14V12.875V12.375ZM8.75 6.26758L10.4523 7.96991C10.7452 8.26281 11.2201 8.26281 11.513 7.96991C11.8059 7.67702 11.8059 7.20215 11.513 6.90925L8.59099 3.98725C8.43431 3.83057 8.22556 3.7577 8.02045 3.76865C7.81604 3.75834 7.60821 3.83124 7.45209 3.98736L4.53009 6.90937C4.23719 7.20226 4.23719 7.67713 4.53009 7.97003C4.82298 8.26292 5.29785 8.26292 5.59075 7.97003L7.25 6.31077L7.25 11.25C7.25 11.6642 7.58579 12 8 12C8.41421 12 8.75 11.6642 8.75 11.25L8.75 6.26758Z"
+      />
+    </svg>
+  );
+}
+
+export function ShowParentIconSelected({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        fill="currentColor"
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M2 0C0.895431 0 0 0.895431 0 2V14C0 15.1046 0.895431 16 2 16H14C15.1046 16 16 15.1046 16 14V2C16 0.895431 15.1046 0 14 0H2ZM8.75 6.26758L10.4523 7.96991C10.7452 8.26281 11.2201 8.26281 11.513 7.96991C11.8059 7.67702 11.8059 7.20215 11.513 6.90925L8.59099 3.98725C8.43344 3.8297 8.22324 3.7569 8.01703 3.76884C7.8138 3.75958 7.60752 3.83254 7.45234 3.98773L4.53033 6.90973C4.23744 7.20263 4.23744 7.6775 4.53033 7.97039C4.82322 8.26329 5.2981 8.26329 5.59099 7.97039L7.25 6.31138L7.25 11.25C7.25 11.6642 7.58579 12 8 12C8.41421 12 8.75 11.6642 8.75 11.25L8.75 6.26758Z"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/SideMenuRightClosed.tsx

@@ -0,0 +1,15 @@
+export function SideMenuRightClosedIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="20"
+      height="20"
+      viewBox="0 0 20 20"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <rect x="12" y="4" width="1" height="12" fill="currentColor" />
+      <rect x="2.5" y="3.5" width="15" height="13" rx="2.5" stroke="currentColor" />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/TaskIcon.tsx

@@ -0,0 +1,29 @@
+export function TaskIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="16"
+      height="16"
+      viewBox="0 0 16 16"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <g clipPath="url(#clip0_9221_99732)">
+        <path
+          fillRule="evenodd"
+          clipRule="evenodd"
+          d="M16 0H0V16H16V0ZM10.8901 5.73995V4.44995H5.11011V5.73995H7.23011V11.55H8.77011V5.73995H10.8901Z"
+          fill="currentColor"
+        />
+      </g>
+      <defs>
+        <clipPath id="clip0_9221_99732">
+          <path
+            d="M0 2C0 0.895431 0.895431 0 2 0H14C15.1046 0 16 0.895431 16 2V14C16 15.1046 15.1046 16 14 16H2C0.895431 16 0 15.1046 0 14V2Z"
+            fill="white"
+          />
+        </clipPath>
+      </defs>
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/TimedOutIcon.tsx

@@ -0,0 +1,19 @@
+export function TimedOutIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="20"
+      height="20"
+      viewBox="0 0 20 20"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M9 2H8V3H9V4.07089C5.60771 4.55612 3 7.47353 3 11C3 14.866 6.13401 18 10 18C13.866 18 17 14.866 17 11C17 7.47353 14.3923 4.55612 11 4.07089V3H12V2H11H9ZM13.7603 3.36575C14.7218 2.40422 16.2807 2.40422 17.2422 3.36575C18.2038 4.32727 18.2038 5.8862 17.2422 6.84772L17.1462 6.94375C17.1251 6.96488 17.0908 6.96488 17.0697 6.94375L13.6642 3.53831C13.6431 3.51717 13.6431 3.4829 13.6642 3.46177L13.7603 3.36575ZM6.28876 3.58524C6.33584 3.53816 6.33584 3.46184 6.28876 3.41476L6.23971 3.36571C5.27819 2.40419 3.71925 2.40419 2.75773 3.36571C1.79621 4.32723 1.79621 5.88616 2.75773 6.84769L2.80678 6.89674C2.85386 6.94382 2.93019 6.94381 2.97726 6.89674L6.28876 3.58524ZM14.5858 17L16 15.5858L16.7071 16.2929C17.0976 16.6834 17.0976 17.3166 16.7071 17.7071C16.3166 18.0976 15.6834 18.0976 15.2929 17.7071L14.5858 17ZM5.42297 17L4.00875 15.5858L3.30165 16.2929C2.91112 16.6834 2.91112 17.3166 3.30165 17.7071C3.69217 18.0977 4.32534 18.0977 4.71586 17.7071L5.42297 17ZM6.29289 7.29289C6.68342 6.90237 7.31658 6.90237 7.70711 7.29289L10 9.58579L12.2929 7.29289C12.6834 6.90237 13.3166 6.90237 13.7071 7.29289C14.0976 7.68342 14.0976 8.31658 13.7071 8.70711L11.4142 11L13.7071 13.2929C14.0976 13.6834 14.0976 14.3166 13.7071 14.7071C13.3166 15.0976 12.6834 15.0976 12.2929 14.7071L10 12.4142L7.70711 14.7071C7.31658 15.0976 6.68342 15.0976 6.29289 14.7071C5.90237 14.3166 5.90237 13.6834 6.29289 13.2929L8.58579 11L6.29289 8.70711C5.90237 8.31658 5.90237 7.68342 6.29289 7.29289Z"
+        fill="currentColor"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/v3.svg

@@ -0,0 +1,4 @@
+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect x="0.861435" y="0.861435" width="30.2771" height="30.2771" rx="15.1386" stroke="#D7D9DD" stroke-width="1.72287"/>
+<path d="M14.9669 12.642L11.6417 21.7732H9.8155L6.49036 12.642H8.04094L10.7286 20.2571L13.4163 12.642H14.9669ZM20.534 14.5544C21.6884 14.5716 23.0666 14.0375 23.0666 12.4525C23.0666 11.3843 22.1018 10.7124 20.534 10.7124C19.2591 10.7124 18.3632 11.3326 18.2082 12.3146L16.6576 12.2113C16.8815 10.4884 18.4838 9.26516 20.534 9.26516C22.946 9.26516 24.6345 10.5745 24.6345 12.4525C24.6345 13.8135 23.8592 14.7267 22.3775 15.1574C24.0831 15.657 25.0479 16.8286 25.0479 18.4136C25.0479 20.55 23.1872 22.0489 20.534 22.0489C18.0876 22.0489 16.313 20.6361 16.2096 18.6203L17.743 18.517C17.8808 20.0159 19.3108 20.6016 20.534 20.6016C21.9812 20.6016 23.4974 19.947 23.4974 18.293C23.4974 16.6563 21.9812 15.9499 20.534 15.9844L19.5865 16.0016V14.5371L20.534 14.5544Z" fill="white"/>
+</svg>

apps/webapp/app/assets/logos/ATAndTLogo.tsx

@@ -0,0 +1,21 @@
+export function ATAndTLogo({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="66"
+      height="33"
+      viewBox="0 0 66 33"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        d="M57.0405 21.156C57.0064 21.1581 56.9723 21.1529 56.9403 21.1408C56.9084 21.1287 56.8794 21.1099 56.8552 21.0858C56.8311 21.0616 56.8123 21.0326 56.8002 21.0007C56.7881 20.9687 56.7829 20.9346 56.785 20.9005V13.4102H54.2407C54.2068 13.4114 54.173 13.4056 54.1415 13.3932C54.1099 13.3808 54.0813 13.362 54.0573 13.338C54.0333 13.3141 54.0145 13.2854 54.0021 13.2538C53.9897 13.2223 53.984 13.1885 53.9852 13.1546V12.1226C53.9839 12.0887 53.9896 12.0549 54.002 12.0233C54.0143 11.9916 54.0331 11.9629 54.0571 11.9389C54.0811 11.9148 54.1098 11.896 54.1414 11.8836C54.173 11.8711 54.2068 11.8653 54.2407 11.8666H61.0127C61.0468 11.8646 61.0809 11.8698 61.1129 11.882C61.1448 11.8942 61.1738 11.913 61.1979 11.9372C61.222 11.9614 61.2407 11.9904 61.2527 12.0224C61.2647 12.0544 61.2698 12.0886 61.2677 12.1226V13.1557C61.2689 13.1896 61.2632 13.2233 61.2508 13.2548C61.2384 13.2863 61.2197 13.315 61.1958 13.3389C61.1718 13.3629 61.1432 13.3817 61.1118 13.3941C61.0803 13.4066 61.0465 13.4124 61.0127 13.4112H58.4684V20.9005C58.4696 20.9344 58.4639 20.9681 58.4515 20.9997C58.4391 21.0313 58.4203 21.0599 58.3963 21.0839C58.3723 21.1079 58.3437 21.1266 58.3121 21.1391C58.2805 21.1515 58.2468 21.1572 58.2129 21.156H57.0394M37.5459 17.4163L36.2102 13.5845L34.8629 17.4163H37.5459ZM40.5081 20.8528C40.5662 21.0041 40.4735 21.1555 40.3107 21.1555H39.1087C39.0327 21.1609 38.9572 21.1394 38.8955 21.0947C38.8338 21.0499 38.7899 20.9848 38.7715 20.9109L38.0861 18.9369H34.3343L33.6489 20.9109C33.6305 20.9847 33.5867 21.0498 33.5251 21.0945C33.4635 21.1392 33.3881 21.1608 33.3122 21.1555H32.1744C32.0231 21.1555 31.9183 21.0041 31.977 20.8528L35.1245 12.0985C35.1826 11.9357 35.2873 11.866 35.4612 11.866H37.0184C37.1928 11.866 37.3091 11.9351 37.3672 12.0985L40.5147 20.8528M49.5381 19.9014C50.2811 19.9014 50.7812 19.5422 51.1875 18.9265L49.3067 16.9058C48.5862 17.3127 48.1212 17.7185 48.1212 18.5316C48.1212 19.3322 48.7716 19.9025 49.5387 19.9025M50.0613 13.1031C49.4581 13.1031 49.1088 13.4869 49.1088 13.9969C49.1088 14.3917 49.3172 14.7399 49.7942 15.2509C50.6189 14.7739 50.9677 14.4843 50.9677 13.9733C50.9677 13.4962 50.6661 13.096 50.0618 13.096M55.2025 20.8303C55.3533 20.9931 55.2606 21.1555 55.0742 21.1555H53.5937C53.511 21.1627 53.4278 21.1483 53.3524 21.1136C53.277 21.0788 53.212 21.025 53.1638 20.9575L52.2864 19.9826C51.6942 20.7722 50.869 21.3644 49.4981 21.3644C47.8021 21.3644 46.4658 20.3428 46.4658 18.5898C46.4658 17.2425 47.1863 16.5225 48.2786 15.9194C47.744 15.3041 47.5 14.6538 47.5 14.0852C47.5 12.6447 48.5106 11.6582 50.0316 11.6582C51.5889 11.6582 52.5408 12.5761 52.5408 13.9338C52.5408 15.0952 51.7046 15.7444 50.8218 16.2325L52.123 17.6379L52.8545 16.3602C52.9477 16.2094 53.0519 16.1519 53.2383 16.1519H54.3646C54.5511 16.1519 54.6553 16.2802 54.5401 16.477L53.2449 18.706L55.208 20.8314M43.6161 21.1566C43.65 21.1579 43.6838 21.1521 43.7155 21.1398C43.7471 21.1274 43.7758 21.1086 43.7998 21.0846C43.8239 21.0607 43.8427 21.032 43.8552 21.0004C43.8676 20.9688 43.8734 20.935 43.8721 20.901V13.4107H46.4164C46.4503 13.4119 46.4841 13.4062 46.5157 13.3938C46.5472 13.3813 46.5759 13.3626 46.5999 13.3386C46.6238 13.3146 46.6426 13.2859 46.655 13.2544C46.6674 13.2228 46.6732 13.1891 46.672 13.1552V12.1226C46.6733 12.0887 46.6676 12.0549 46.6552 12.0233C46.6428 11.9916 46.624 11.9629 46.6001 11.9389C46.5761 11.9148 46.5474 11.896 46.5158 11.8836C46.4842 11.8711 46.4504 11.8653 46.4164 11.8666H39.6406C39.6067 11.8653 39.5728 11.8711 39.5412 11.8836C39.5097 11.896 39.481 11.9148 39.457 11.9389C39.433 11.9629 39.4142 11.9916 39.4019 12.0233C39.3895 12.0549 39.3838 12.0887 39.3851 12.1226V13.1557C39.3838 13.1896 39.3896 13.2234 39.402 13.2549C39.4144 13.2865 39.4332 13.3152 39.4572 13.3391C39.4812 13.3631 39.5098 13.3819 39.5414 13.3943C39.5729 13.4067 39.6067 13.4125 39.6406 13.4112H42.1849V20.9005C42.1837 20.9344 42.1894 20.9681 42.2018 20.9997C42.2143 21.0313 42.233 21.0599 42.257 21.0839C42.281 21.1079 42.3096 21.1266 42.3412 21.1391C42.3728 21.1515 42.4065 21.1572 42.4404 21.156L43.6161 21.1566Z"
+        fill="currentColor"
+      />
+      <path
+        d="M9.2351 25.6737C11.2728 27.2567 13.7799 28.1157 16.3602 28.1149C19.296 28.1149 21.9725 27.0248 24.0151 25.2361C24.0397 25.2142 24.0277 25.1999 24.003 25.2142C23.0862 25.8261 20.4739 27.1624 16.3602 27.1624C12.7851 27.1624 10.5259 26.3646 9.2499 25.6529C9.22523 25.6408 9.217 25.6583 9.23455 25.6748M17.1487 26.2686C20.0083 26.2686 23.1503 25.4889 25.0295 23.9464C25.5438 23.5258 26.033 22.9665 26.4716 22.2148C26.74 21.7483 26.9742 21.263 27.1724 20.7628C27.1812 20.7381 27.1669 20.726 27.1477 20.754C25.4002 23.3306 20.339 24.9296 15.1188 24.9296C11.4252 24.9296 7.45135 23.7485 5.89571 21.4931C5.88035 21.4723 5.865 21.4811 5.87377 21.5052C7.3181 24.5863 11.7152 26.2686 17.1476 26.2686M14.0232 21.1581C8.07591 21.1581 5.2717 18.389 4.76338 16.4983C4.7579 16.4709 4.73926 16.4764 4.73926 16.5016C4.73926 17.1377 4.80287 17.9591 4.91253 18.5041C4.96463 18.7695 5.1867 19.1857 5.49706 19.5186C6.9392 21.0166 10.5286 23.1212 16.7468 23.1212C25.2187 23.1212 27.1554 20.2994 27.5508 19.3705C27.8337 18.7125 27.9801 17.5073 27.9801 16.5C27.9803 16.29 27.9751 16.0801 27.9648 15.8705C27.9648 15.8392 27.9467 15.8376 27.9406 15.8672C27.5173 18.1373 20.2792 21.157 14.0249 21.157M5.85458 11.5172C5.49236 12.2876 5.21385 13.0946 5.02385 13.9244C4.96901 14.1766 4.99533 14.2989 5.07868 14.4876C5.79152 15.9999 9.39686 18.4192 17.8068 18.4192C22.9376 18.4192 26.9235 17.158 27.5689 14.8582C27.6878 14.4349 27.6939 13.9875 27.5414 13.3854C27.3769 12.712 27.0512 11.9268 26.7803 11.3757C26.7716 11.3576 26.7557 11.3604 26.7584 11.3812C26.8588 14.3982 18.4456 16.3426 14.2003 16.3426C9.60194 16.3426 5.76685 14.5111 5.76685 12.1971C5.76685 11.9751 5.81291 11.7585 5.87652 11.521C5.882 11.4991 5.86445 11.4964 5.85458 11.5156M24.0348 7.81261C24.0859 7.89265 24.1114 7.9864 24.1077 8.0813C24.1077 9.37209 20.158 11.6548 13.8702 11.6548C9.25045 11.6548 8.38572 9.94127 8.38572 8.85117C8.38572 8.46733 8.53487 8.0632 8.86442 7.65798C8.88252 7.63385 8.86716 7.62508 8.84633 7.64262C8.24498 8.15186 7.6971 8.72105 7.21118 9.34138C6.97978 9.63365 6.83666 9.89246 6.83666 10.0476C6.83666 12.3068 12.4999 13.9441 17.7958 13.9441C23.4437 13.9441 25.9562 12.1017 25.9562 10.4896C25.9562 9.91111 25.7369 9.57388 25.1556 8.91861C24.7816 8.49255 24.428 8.15094 24.0589 7.80438C24.0408 7.78958 24.0282 7.80164 24.0408 7.81974M22.3108 6.52949C20.5693 5.48545 18.547 4.8916 16.3668 4.8916C14.1713 4.8916 12.0887 5.50574 10.3351 6.57775C9.81086 6.90017 9.51585 7.15899 9.51585 7.49128C9.51585 8.47117 11.8057 9.52453 15.8673 9.52453C19.8866 9.52453 23.005 8.37082 23.005 7.25988C23.005 6.99503 22.7731 6.80915 22.3042 6.52949"
+        fill="currentColor"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/logos/CalComLogo.tsx

@@ -0,0 +1,50 @@
+export function CalComLogo({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="64"
+      height="15"
+      viewBox="0 0 64 15"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <g clipPath="url(#clip0_9632_116886)">
+        <path
+          d="M6.38629 13.7111C2.75645 13.7111 0.0224609 10.8382 0.0224609 7.29138C0.0224609 3.73276 2.61685 0.836182 6.38629 0.836182C8.38733 0.836182 9.77174 1.45096 10.8537 2.85787L9.1086 4.31207C8.37568 3.53177 7.49149 3.14162 6.38629 3.14162C3.93149 3.14162 2.58194 5.02143 2.58194 7.29138C2.58194 9.56139 4.05946 11.4057 6.38629 11.4057C7.47984 11.4057 8.41061 11.0156 9.14353 10.2353L10.8654 11.7486C9.82995 13.0963 8.41061 13.7111 6.38629 13.7111Z"
+          fill="currentColor"
+        />
+        <path
+          d="M18.3806 4.26438H20.7307V13.4861H18.3806V12.1383C17.892 13.096 17.0776 13.7344 15.5187 13.7344C13.029 13.7344 11.0396 11.5709 11.0396 8.91074C11.0396 6.2506 13.029 4.08704 15.5187 4.08704C17.066 4.08704 17.892 4.72547 18.3806 5.68311V4.26438ZM18.4504 8.91074C18.4504 7.46837 17.4615 6.27425 15.9026 6.27425C14.4018 6.27425 13.4245 7.48014 13.4245 8.91074C13.4245 10.3058 14.4018 11.5472 15.9026 11.5472C17.4499 11.5472 18.4504 10.3413 18.4504 8.91074Z"
+          fill="currentColor"
+        />
+        <path d="M22.395 0.540283H24.7451V13.4744H22.395V0.540283Z" fill="currentColor" />
+        <path
+          d="M25.791 12.2569C25.791 11.5003 26.396 10.8618 27.222 10.8618C28.048 10.8618 28.6298 11.5003 28.6298 12.2569C28.6298 13.0372 28.0364 13.6756 27.222 13.6756C26.4076 13.6756 25.791 13.0372 25.791 12.2569Z"
+          fill="currentColor"
+        />
+        <path
+          d="M37.6222 11.9965C36.7496 13.0605 35.4234 13.7344 33.8527 13.7344C31.0489 13.7344 28.9897 11.5709 28.9897 8.91074C28.9897 6.2506 31.0489 4.08704 33.8527 4.08704C35.3651 4.08704 36.6798 4.72547 37.5524 5.7304L35.7375 7.25553C35.2837 6.68804 34.6904 6.26242 33.8527 6.26242C32.352 6.26242 31.3747 7.46837 31.3747 8.8989C31.3747 10.3294 32.352 11.5354 33.8527 11.5354C34.7602 11.5354 35.3884 11.0625 35.8538 10.424L37.6222 11.9965Z"
+          fill="currentColor"
+        />
+        <path
+          d="M37.8208 8.91074C37.8208 6.2506 39.88 4.08704 42.6838 4.08704C45.4876 4.08704 47.5468 6.2506 47.5468 8.91074C47.5468 11.5709 45.4876 13.7344 42.6838 13.7344C39.88 13.7225 37.8208 11.5709 37.8208 8.91074ZM45.1618 8.91074C45.1618 7.46837 44.1846 6.27425 42.6838 6.27425C41.183 6.26242 40.2058 7.46837 40.2058 8.91074C40.2058 10.3413 41.183 11.5472 42.6838 11.5472C44.1846 11.5472 45.1618 10.3413 45.1618 8.91074Z"
+          fill="currentColor"
+        />
+        <path
+          d="M63.4389 7.84684V13.4745H61.089V8.42614C61.089 6.83009 60.3444 6.14437 59.2276 6.14437C58.1805 6.14437 57.4359 6.66457 57.4359 8.42614V13.4745H55.0859V8.42614C55.0859 6.83009 54.3296 6.14437 53.2244 6.14437C52.1774 6.14437 51.2582 6.66457 51.2582 8.42614V13.4745H48.9082V4.25273H51.2582V5.52958C51.7469 4.53647 52.6311 4.03992 53.9923 4.03992C55.2837 4.03992 56.3656 4.67834 56.9589 5.75421C57.5523 4.6547 58.4248 4.03992 60.007 4.03992C61.9383 4.05174 63.4389 5.51776 63.4389 7.84684Z"
+          fill="currentColor"
+        />
+      </g>
+      <defs>
+        <clipPath id="clip0_9632_116886">
+          <rect
+            width="63.9027"
+            height="13.9194"
+            fill="currentColor"
+            transform="translate(0.0224609 0.540283)"
+          />
+        </clipPath>
+      </defs>
+    </svg>
+  );
+}

apps/webapp/app/assets/logos/ShopifyLogo.tsx

@@ -0,0 +1,39 @@
+export function ShopifyLogo({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="85"
+      height="25"
+      viewBox="0 0 85 25"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <g clipPath="url(#clip0_9632_116870)">
+        <path
+          fillRule="evenodd"
+          clipRule="evenodd"
+          d="M10.6613 1.00225C10.5632 0.945637 10.4859 0.900988 10.3639 0.900988C8.19769 0.900988 7.03127 3.73373 6.44806 5.40004L8.28101 4.90015C8.7809 2.5673 9.78069 1.4842 10.7805 1.06762C10.7358 1.0453 10.6972 1.02297 10.6613 1.00225ZM11.6136 3.73373C11.6136 2.81725 11.447 2.23404 11.2804 1.73415C10.5305 2.06741 9.69738 2.81725 9.19748 4.56688L11.6136 3.90036V3.73373ZM12.3635 3.56709L13.4466 3.23383C13.1966 2.40067 12.6967 1.65083 12.1135 1.4842C12.2802 1.98409 12.3635 2.5673 12.3635 3.31715V3.56709ZM18.6955 4.40025C18.7788 4.40025 18.8621 4.48357 18.8621 4.56688C18.8621 4.73352 21.3616 21.7299 21.3616 21.7299L14.1964 23.3129L0.949219 20.8135C0.949219 20.8135 2.61553 7.64957 2.69885 7.23299C2.70036 7.22241 2.70184 7.212 2.7033 7.20175C2.78239 6.64673 2.79425 6.56344 3.44869 6.39983C3.64023 6.33599 3.82633 6.27214 4.00838 6.20968C4.53904 6.02763 5.03532 5.85737 5.53159 5.7333C6.03148 3.81704 7.44785 0.0678302 10.4472 0.151146C10.8638 0.151146 11.3637 0.317778 11.697 0.817672H11.7803C13.03 0.817672 13.7799 1.90078 14.1964 3.0672C14.613 2.90057 14.863 2.81725 14.863 2.81725C14.8796 2.81725 14.9029 2.81392 14.9309 2.80992C15.0429 2.79392 15.2295 2.76726 15.3629 2.90057C15.5295 2.98388 16.7792 4.23362 16.7792 4.23362C16.7792 4.23362 18.5289 4.40025 18.6955 4.40025ZM10.8622 11.0656L11.7786 8.39946C11.7786 8.39946 11.1954 7.98288 9.77906 8.06619C6.69638 8.06619 5.11338 10.1491 5.11338 12.3153C5.11338 13.764 5.92592 14.4263 6.63951 15.008C7.19816 15.4633 7.69617 15.8693 7.69617 16.5644C7.69617 16.981 7.44622 17.4809 6.7797 17.4809C5.77991 17.4809 4.61348 16.4811 4.61348 16.4811L4.03027 18.4807C4.03027 18.4807 5.19669 19.897 7.44622 19.897C9.36249 19.897 10.7789 18.4807 10.7789 16.2311C10.7789 14.5056 9.60415 13.6457 8.68583 12.9735C8.09656 12.5421 7.61285 12.188 7.61285 11.7321C7.61285 11.4821 7.69617 10.649 9.11254 10.649C10.1123 10.649 10.8622 11.0656 10.8622 11.0656Z"
+          fill="currentColor"
+        />
+        <path
+          d="M18.6111 4.40038C18.5277 4.40038 16.7781 4.23375 16.7781 4.23375C16.7781 4.23375 15.5284 2.98401 15.3617 2.9007C15.2784 2.81738 15.2784 2.81738 15.1951 2.81738L14.1953 23.3131L21.3605 21.7301C21.3605 21.7301 18.861 4.73365 18.861 4.56701C18.861 4.4837 18.6944 4.40038 18.6111 4.40038Z"
+          fill="currentColor"
+        />
+        <path
+          d="M29.7751 13.3151C29.0253 12.8985 28.692 12.5653 28.692 12.1487C28.692 11.5655 29.1919 11.2322 30.0251 11.2322C30.9416 11.2322 31.7747 11.6488 31.7747 11.6488L32.4412 9.64922C32.4412 9.64922 31.858 9.14932 30.1084 9.14932C27.6089 9.14932 25.9426 10.5657 25.9426 12.5653C25.9426 13.7317 26.7758 14.5648 27.7756 15.1481C28.6087 15.648 28.942 15.9812 28.942 16.4811C28.942 16.981 28.5254 17.4809 27.6922 17.4809C26.5258 17.4809 25.3594 16.8977 25.3594 16.8977L24.6929 18.8973C24.6929 18.8973 25.6927 19.5638 27.4423 19.5638C29.9418 19.5638 31.7747 18.3141 31.7747 16.0645C31.7747 14.8148 30.8582 13.8983 29.7751 13.3151ZM39.8563 9.06601C38.6066 9.06601 37.6068 9.64922 36.857 10.5657L37.9401 4.90022H35.1073L32.3579 19.3138H35.1907L36.1071 14.3982C36.4404 12.5653 37.4402 11.3988 38.3567 11.3988C39.0232 11.3988 39.2731 11.8154 39.2731 12.482C39.2731 12.8985 39.2731 13.3151 39.1898 13.7317L38.1067 19.3972H40.9395L42.0226 13.5651C42.1059 12.9818 42.1892 12.232 42.1892 11.7321C42.1892 10.0658 41.356 9.06601 39.8563 9.06601ZM47.1048 17.3143C46.105 17.3143 45.7718 16.4811 45.7718 15.4813C45.7718 13.8983 46.6049 11.2322 48.1046 11.2322C49.1044 11.2322 49.4377 12.0654 49.4377 12.8985C49.4377 14.7315 48.6045 17.3143 47.1048 17.3143ZM48.5212 9.06601C45.1052 9.06601 42.8557 12.1487 42.8557 15.5646C42.8557 17.7308 44.1888 19.4805 46.7716 19.4805C50.1042 19.4805 52.3537 16.4811 52.3537 12.9818C52.3537 10.9823 51.1873 9.06601 48.5212 9.06601ZM56.8528 17.3976C56.1029 17.3976 55.6863 16.981 55.6863 16.981L56.1862 14.3149C56.5195 12.5653 57.436 11.3988 58.4358 11.3988C59.2689 11.3988 59.6022 12.232 59.6022 12.9818C59.5189 14.8148 58.4358 17.3976 56.8528 17.3976ZM59.6022 9.06601C57.6859 9.06601 56.6028 10.7323 56.6028 10.7323L56.7695 9.23264H54.27C54.1867 10.2324 53.9367 11.8154 53.6868 12.9818L51.6872 23.313H54.5199L55.2698 19.1472H55.3531C55.3531 19.1472 55.9363 19.4805 57.0194 19.4805C60.352 19.4805 62.5182 16.0645 62.5182 12.6486C62.5182 10.8156 61.6851 9.06601 59.6022 9.06601ZM66.5174 5.06685C65.6009 5.06685 64.9344 5.81669 64.9344 6.73316C64.9344 7.56632 65.4343 8.14953 66.2675 8.14953C67.1839 8.14953 67.9338 7.56632 67.9338 6.48322C67.9338 5.65006 67.3506 5.06685 66.5174 5.06685ZM62.6016 19.3138H65.4343L67.3506 9.31595H64.5178L62.6016 19.3138ZM74.5157 9.31595H72.5161L72.5995 8.81606C72.7661 7.81627 73.3493 6.98311 74.2658 6.98311C74.7657 6.98311 75.1822 7.14974 75.1822 7.14974L75.7655 4.90022C75.7655 4.90022 75.2656 4.65027 74.2658 4.65027C73.266 4.65027 72.2662 4.90022 71.5163 5.56674C70.5166 6.3999 70.1 7.56632 69.85 8.73274L69.7667 9.23264H68.4337L68.0171 11.3988H69.3501L67.8505 19.3138H70.6832L72.1829 11.3988H74.0991L74.5157 9.31595ZM81.2643 9.31595C81.2643 9.31595 79.5147 13.7317 78.6815 16.1478C78.5982 15.398 78.015 9.31595 78.015 9.31595H75.0156L76.6819 18.4807C76.6819 18.6473 76.6819 18.814 76.5986 18.9806C76.2654 19.6471 75.6821 20.2303 75.0989 20.6469C74.599 21.0635 73.9325 21.3134 73.5159 21.4801L74.2658 23.8962C74.849 23.8129 76.0154 23.313 77.0152 22.3965C78.2649 21.2301 79.5147 19.3138 80.6811 16.8144L84.097 9.39927L81.2643 9.31595Z"
+          fill="currentColor"
+        />
+      </g>
+      <defs>
+        <clipPath id="clip0_9632_116870">
+          <rect
+            width="83.3158"
+            height="23.8283"
+            fill="currentColor"
+            transform="translate(0.949219 0.585938)"
+          />
+        </clipPath>
+      </defs>
+    </svg>
+  );
+}

apps/webapp/app/assets/logos/VerizonLogo.tsx

@@ -0,0 +1,33 @@
+export function VerizonLogo({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="82"
+      height="19"
+      viewBox="0 0 82 19"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <g clipPath="url(#clip0_9632_116867)">
+        <path
+          d="M79.373 0.50708L74.9504 9.96817L73.2874 6.3833H71.4888L74.2236 12.2472H75.6773L81.1593 0.50708H79.373Z"
+          fill="currentColor"
+        />
+        <path
+          d="M60.4008 18.0743H63.961V11.5575C63.961 10.0669 64.8234 9.01976 66.1046 9.01976C67.3365 9.01976 67.9771 9.8821 67.9771 11.1263V18.0743H71.5373V10.4488C71.5373 7.86176 69.9851 6.03853 67.472 6.03853C65.8705 6.03853 64.7495 6.7284 63.9118 7.98495H63.8379V6.38347H60.3885L60.4008 18.0743ZM53.4036 6.06317C49.6832 6.06317 47.1947 8.73642 47.1947 12.2597C47.1947 15.7583 49.6832 18.4562 53.4036 18.4562C57.1239 18.4562 59.6124 15.7583 59.6124 12.2597C59.6247 8.73642 57.1239 6.06317 53.4036 6.06317ZM53.3789 15.8199C51.6912 15.8199 50.7919 14.4032 50.7919 12.2597C50.7919 10.0915 51.6789 8.69946 53.3789 8.69946C55.0666 8.69946 56.0029 10.0915 56.0029 12.2597C56.0152 14.4032 55.079 15.8199 53.3789 15.8199ZM37.0438 18.0743H47.0469V15.2656H41.3801V15.1916L46.8005 9.11831V6.37115H37.0438V9.19223H42.5258V9.26614L37.0438 15.3887V18.0743ZM32.4734 18.0743H36.0582V6.38347H32.4734V18.0743ZM24.158 18.0743H27.7305V12.7032C27.7305 10.264 29.1965 9.17991 31.4262 9.48789H31.5002V6.40811C31.3154 6.33419 31.1059 6.32187 30.7733 6.32187C29.3813 6.32187 28.445 6.96247 27.6443 8.32989H27.5704V6.38347H24.158V18.0743ZM17.6042 15.8322C16.0273 15.8322 15.0664 14.8097 14.8817 13.2329H23.3079C23.3326 10.7198 22.6673 8.71178 21.2753 7.45523C20.2898 6.54362 18.9962 6.03853 17.3455 6.03853C13.8222 6.03853 11.383 8.71178 11.383 12.2104C11.383 15.7337 13.7113 18.4193 17.5795 18.4193C19.0455 18.4193 20.2035 18.025 21.1644 17.3721C22.1869 16.6699 22.9261 15.6351 23.1232 14.6126H19.6984C19.3658 15.3764 18.6513 15.8322 17.6042 15.8322ZM17.4194 8.58859C18.676 8.58859 19.5383 9.52484 19.6861 10.8923H14.9063C15.1896 9.51253 15.9411 8.58859 17.4194 8.58859ZM4.26258 18.0743H8.00759L12.0729 6.38347H8.48804L6.209 13.8981H6.15972L3.88068 6.38347H0.197266L4.26258 18.0743ZM32.4734 2.2196H36.0582V5.39794H32.4734V2.2196Z"
+          fill="currentColor"
+        />
+      </g>
+      <defs>
+        <clipPath id="clip0_9632_116867">
+          <rect
+            width="81.0845"
+            height="17.9859"
+            fill="white"
+            transform="translate(0.074707 0.50708)"
+          />
+        </clipPath>
+      </defs>
+    </svg>
+  );
+}

apps/webapp/app/components/ActiveBadge.tsx

@@ -17,13 +17,13 @@ export function ActiveBadge({ active, className, badgeSize = "normal" }: ActiveB
   switch (active) {
     case true:
       return (
-        <span className={cn(variant[badgeSize], "bg-slate-800 text-green-500", className)}>
+        <span className={cn(variant[badgeSize], "bg-charcoal-800 text-green-500", className)}>
           Active
         </span>
       );
     case false:
       return (
-        <span className={cn(variant[badgeSize], "bg-slate-800 text-dimmed", className)}>
+        <span className={cn(variant[badgeSize], "bg-charcoal-800 text-text-dimmed", className)}>
           Disabled
         </span>
       );
@@ -38,7 +38,7 @@ export function MissingIntegrationBadge({
   badgeSize?: keyof typeof variant;
 }) {
   return (
-    <span className={cn(variant[badgeSize], "bg-rose-600 text-white", className)}>
+    <span className={cn(variant[badgeSize], "bg-error text-white", className)}>
       Missing Integration
     </span>
   );
@@ -52,6 +52,8 @@ export function NewBadge({
   badgeSize?: keyof typeof variant;
 }) {
   return (
-    <span className={cn(variant[badgeSize], "bg-green-600 text-background", className)}>New!</span>
+    <span className={cn(variant[badgeSize], "bg-green-600 text-background-dimmed", className)}>
+      New!
+    </span>
   );
 }

apps/webapp/app/components/BlankstateInstructions.tsx

@@ -0,0 +1,23 @@
+import { cn } from "~/utils/cn";
+import { Header2 } from "./primitives/Headers";
+
+export function BlankstateInstructions({
+  children,
+  className,
+  title,
+}: {
+  children: React.ReactNode;
+  className?: string;
+  title?: string;
+}) {
+  return (
+    <div className={cn("flex flex-col gap-2", className)}>
+      {title && (
+        <div className="flex h-10 items-center border-b border-grid-bright">
+          <Header2>{title}</Header2>
+        </div>
+      )}
+      {children}
+    </div>
+  );
+}

apps/webapp/app/components/ComingSoon.tsx

@@ -13,10 +13,10 @@ type ComingSoonProps = {
 export function ComingSoon({ title, description, icon }: ComingSoonProps) {
   return (
     <MainCenteredContainer>
-      <div className="flex flex-col justify-center rounded border border-slate-800 bg-slate-850">
-        <div className="flex items-center gap-2 border-b border-slate-750 px-4 py-4">
+      <div className="flex flex-col justify-center rounded border border-charcoal-800 bg-charcoal-850">
+        <div className="flex items-center gap-2 border-b border-charcoal-750 px-4 py-4">
           {typeof icon === "string" ? (
-            <NamedIconInBox name={icon} className={"h-10 w-10 bg-midnight-800"} />
+            <NamedIconInBox name={icon} className={"bg-midnight-800 h-10 w-10"} />
           ) : (
             icon
           )}

apps/webapp/app/components/DefinitionTooltip.tsx

@@ -15,11 +15,11 @@ export function DefinitionTip({
     <TooltipProvider>
       <Tooltip disableHoverableContent>
         <TooltipTrigger>
-          <span className="underline decoration-slate-600 decoration-dashed underline-offset-4 transition hover:decoration-slate-500">
+          <span className="cursor-default underline decoration-charcoal-500 decoration-dashed underline-offset-4 transition hover:decoration-charcoal-400">
             {children}
           </span>
         </TooltipTrigger>
-        <TooltipContent align="end" side="right" variant="dark" className="w-[16rem] min-w-[16rem]">
+        <TooltipContent align="end" side="right" className="w-[16rem] min-w-[16rem]">
           <Header3 className="mb-1">{title}</Header3>
           {typeof content === "string" ? (
             <Paragraph variant="small">{content}</Paragraph>

apps/webapp/app/components/ErrorDisplay.tsx

@@ -1,7 +1,11 @@
+import { HomeIcon } from "@heroicons/react/20/solid";
 import { isRouteErrorResponse, useRouteError } from "@remix-run/react";
-import { LinkButton } from "./primitives/Buttons";
-import { Header1, Header3 } from "./primitives/Headers";
+import { motion } from "framer-motion";
 import { friendlyErrorDisplay } from "~/utils/httpErrors";
+import { LinkButton } from "./primitives/Buttons";
+import { Header1 } from "./primitives/Headers";
+import { Paragraph } from "./primitives/Paragraph";
+import Spline from "@splinetool/react-spline";
 
 type ErrorDisplayOptions = {
   button?: {
@@ -39,12 +43,27 @@ type DisplayOptionsProps = {
 
 export function ErrorDisplay({ title, message, button }: DisplayOptionsProps) {
   return (
-    <div className="p-4">
-      <Header1 className="mb-4 border-b border-slate-800 pb-4">{title}</Header1>
-      {message && <Header3>{message}</Header3>}
-      <LinkButton to={button ? button.to : "/"} variant="primary/medium" className="mt-8">
-        {button ? button.title : "Home"}
-      </LinkButton>
+    <div className="relative flex min-h-screen flex-col items-center justify-center bg-[#16181C]">
+      <div className="z-10 mt-[30vh] flex flex-col items-center gap-8">
+        <Header1>{title}</Header1>
+        {message && <Paragraph>{message}</Paragraph>}
+        <LinkButton
+          to={button ? button.to : "/"}
+          shortcut={{ modifiers: ["meta"], key: "g" }}
+          variant="primary/medium"
+          LeadingIcon={HomeIcon}
+        >
+          {button ? button.title : "Go to homepage"}
+        </LinkButton>
+      </div>
+      <motion.div
+        className="pointer-events-none absolute inset-0 overflow-hidden"
+        initial={{ opacity: 0 }}
+        animate={{ opacity: 1 }}
+        transition={{ delay: 0.5, duration: 2, ease: "easeOut" }}
+      >
+        <Spline scene="https://prod.spline.design/wRly8TZN-e0Twb8W/scene.splinecode" />
+      </motion.div>
     </div>
   );
 }

apps/webapp/app/components/Feedback.tsx

@@ -1,28 +1,24 @@
 import { conform, useForm } from "@conform-to/react";
 import { parse } from "@conform-to/zod";
-import { ChevronRightIcon } from "@heroicons/react/24/solid";
+import { InformationCircleIcon } from "@heroicons/react/20/solid";
+import { EnvelopeIcon } from "@heroicons/react/24/solid";
 import { Form, useActionData, useLocation, useNavigation } from "@remix-run/react";
-import { DiscordIcon } from "@trigger.dev/companyicons";
-import { ReactNode, useState } from "react";
-import { FeedbackType, feedbackTypeLabel, schema } from "~/routes/resources.feedback";
+import { type ReactNode, useEffect, useState } from "react";
+import { type FeedbackType, feedbackTypeLabel, schema } from "~/routes/resources.feedback";
 import { Button } from "./primitives/Buttons";
+import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "./primitives/Dialog";
 import { Fieldset } from "./primitives/Fieldset";
 import { FormButtons } from "./primitives/FormButtons";
 import { FormError } from "./primitives/FormError";
-import { Header2 } from "./primitives/Headers";
+import { Icon } from "./primitives/Icon";
+import { InfoPanel } from "./primitives/InfoPanel";
 import { InputGroup } from "./primitives/InputGroup";
 import { Label } from "./primitives/Label";
 import { Paragraph } from "./primitives/Paragraph";
-import {
-  Select,
-  SelectContent,
-  SelectGroup,
-  SelectItem,
-  SelectTrigger,
-  SelectValue,
-} from "./primitives/Select";
-import { Sheet, SheetBody, SheetContent, SheetHeader, SheetTrigger } from "./primitives/Sheet";
+import { Select, SelectItem } from "./primitives/Select";
 import { TextArea } from "./primitives/TextArea";
+import { TextLink } from "./primitives/TextLink";
+import { DialogClose } from "@radix-ui/react-dialog";
 
 type FeedbackProps = {
   button: ReactNode;
@@ -34,52 +30,89 @@ export function Feedback({ button, defaultValue = "bug" }: FeedbackProps) {
   const location = useLocation();
   const lastSubmission = useActionData();
   const navigation = useNavigation();
+  const [type, setType] = useState<FeedbackType>(defaultValue);
 
   const [form, { path, feedbackType, message }] = useForm({
     id: "accept-invite",
-    // TODO: type this
     lastSubmission: lastSubmission as any,
     onValidate({ formData }) {
       return parse(formData, { schema });
     },
+    shouldRevalidate: "onInput",
   });
 
-  if (
-    open &&
-    navigation.formAction === "/resources/feedback" &&
-    form.error === undefined &&
-    form.errors.length === 0
-  ) {
-    setOpen(false);
-  }
+  useEffect(() => {
+    if (
+      navigation.formAction === "/resources/feedback" &&
+      navigation.state === "loading" &&
+      form.error === undefined &&
+      form.errors.length === 0
+    ) {
+      setOpen(false);
+    }
+  }, [navigation, form]);
 
   return (
-    <Sheet open={open} onOpenChange={setOpen}>
-      <SheetTrigger asChild={true}>{button}</SheetTrigger>
-      <SheetContent>
-        <SheetHeader className="justify-between">How can we help?</SheetHeader>
-        <SheetBody className="flex h-full flex-col justify-between">
-          <DiscordBanner />
-          <hr className="mb-3" />
-          <Header2 className="mb-4">Send us a message</Header2>
-          <Form method="post" action="/resources/feedback" {...form.props}>
-            <Fieldset className="max-w-full">
+    <Dialog open={open} onOpenChange={setOpen}>
+      <DialogTrigger asChild>{button}</DialogTrigger>
+      <DialogContent>
+        <DialogHeader>Contact us</DialogHeader>
+        <div className="mt-2 flex flex-col gap-4">
+          <div className="flex items-center gap-4">
+            <Icon icon={EnvelopeIcon} className="size-10 min-w-[2.5rem] text-blue-500" />
+            <Paragraph variant="base/bright">
+              How can we help? We read every message and will respond as quickly as we can.
+            </Paragraph>
+          </div>
+          <hr className="border-charcoal-800" />
+          <Form method="post" action="/resources/feedback" {...form.props} className="w-full">
+            <Fieldset className="max-w-full gap-y-3">
               <input value={location.pathname} {...conform.input(path, { type: "hidden" })} />
               <InputGroup className="max-w-full">
-                <SelectGroup>
-                  <Select {...conform.input(feedbackType)} defaultValue={defaultValue}>
-                    <SelectTrigger size="medium" width="full">
-                      <SelectValue placeholder="Type" />
-                    </SelectTrigger>
-                    <SelectContent>
-                      {Object.entries(feedbackTypeLabel).map(([key, value]) => (
-                        <SelectItem key={key} value={key}>
-                          {value}
-                        </SelectItem>
-                      ))}
-                    </SelectContent>
-                  </Select>
-                </SelectGroup>
+                {type === "feature" && (
+                  <InfoPanel
+                    icon={InformationCircleIcon}
+                    iconClassName="text-blue-500"
+                    panelClassName="w-full mb-2"
+                  >
+                    <Paragraph variant="small">
+                      All our feature requests are public and voted on by the community. The best
+                      way to submit your feature request is to{" "}
+                      <TextLink to="https://feedback.trigger.dev">
+                        post it to our feedback forum
+                      </TextLink>
+                      .
+                    </Paragraph>
+                  </InfoPanel>
+                )}
+                {type === "help" && (
+                  <InfoPanel
+                    icon={InformationCircleIcon}
+                    iconClassName="text-blue-500"
+                    panelClassName="w-full mb-2"
+                  >
+                    <Paragraph variant="small">
+                      The quickest way to get answers from the Trigger.dev team and community is to{" "}
+                      <TextLink to="https://trigger.dev/discord">ask in our Discord</TextLink>.
+                    </Paragraph>
+                  </InfoPanel>
+                )}
+                <Select
+                  {...conform.select(feedbackType)}
+                  variant="tertiary/medium"
+                  value={type}
+                  defaultValue={type}
+                  setValue={(v) => setType(v as FeedbackType)}
+                  placeholder="Select type"
+                  text={(value) => feedbackTypeLabel[value as FeedbackType]}
+                  dropdownIcon
+                >
+                  {Object.entries(feedbackTypeLabel).map(([name, title]) => (
+                    <SelectItem key={name} value={name}>
+                      {title}
+                    </SelectItem>
+                  ))}
+                </Select>
                 <FormError id={feedbackType.errorId}>{feedbackType.error}</FormError>
               </InputGroup>
               <InputGroup className="max-w-full">
@@ -88,46 +121,22 @@ export function Feedback({ button, defaultValue = "bug" }: FeedbackProps) {
                 <FormError id={message.errorId}>{message.error}</FormError>
               </InputGroup>
               <FormError>{form.error}</FormError>
-              <div className="flex w-full items-center justify-between">
-                <Paragraph variant="small" className="w-full">
-                  We read every message and respond quickly.
-                </Paragraph>
-                <FormButtons
-                  className="m-0 w-max"
-                  confirmButton={
-                    <Button type="submit" variant="primary/medium">
-                      Send
-                    </Button>
-                  }
-                />
-              </div>
+              <FormButtons
+                confirmButton={
+                  <Button type="submit" variant="primary/medium">
+                    Send message
+                  </Button>
+                }
+                cancelButton={
+                  <DialogClose asChild>
+                    <Button variant="tertiary/medium">Cancel</Button>
+                  </DialogClose>
+                }
+              />
             </Fieldset>
           </Form>
-        </SheetBody>
-      </SheetContent>
-    </Sheet>
-  );
-}
-
-function DiscordBanner() {
-  return (
-    <a
-      href="https://trigger.dev/discord"
-      target="_blank"
-      className="group mb-4 flex w-full items-center justify-between rounded-md border border-slate-600 bg-gradient-to-br from-blue-400/30 to-indigo-400/50 p-4 transition hover:border-indigo-400"
-    >
-      <div className="flex flex-col gap-y-2">
-        <DiscordIcon className="h-8 w-8" />
-        <h2 className="font-title text-2xl text-bright transition group-hover:text-white">
-          Join the Trigger.dev
-          <br />
-          Discord community
-        </h2>
-        <Paragraph variant="small/bright">
-          Get help or answer questions from the Trigger.dev community.
-        </Paragraph>
-      </div>
-      <ChevronRightIcon className="h-5 w-5 text-slate-400 transition group-hover:translate-x-1 group-hover:text-indigo-400" />
-    </a>
+        </div>
+      </DialogContent>
+    </Dialog>
   );
 }

apps/webapp/app/components/ListPagination.tsx

@@ -5,8 +5,8 @@ import { cn } from "~/utils/cn";
 
 type List = {
   pagination: {
-    next: string | undefined;
-    previous: string | undefined;
+    next?: string | undefined;
+    previous?: string | undefined;
   };
 };
 
@@ -25,11 +25,13 @@ function NextButton({ cursor }: { cursor?: string }) {
   return (
     <LinkButton
       to={path ?? "#"}
-      variant={"tertiary/small"}
+      variant={"minimal/small"}
       TrailingIcon="chevron-right"
+      trailingIconClassName="text-text-dimmed"
       className={cn(
         "flex items-center",
-        !path && "cursor-default opacity-50 group-hover:bg-transparent group-hover:text-slate-800"
+        !path &&
+          "cursor-not-allowed opacity-50 group-hover:bg-transparent group-hover:text-text-dimmed"
       )}
       onClick={(e) => !path && e.preventDefault()}
     >
@@ -44,11 +46,13 @@ function PreviousButton({ cursor }: { cursor?: string }) {
   return (
     <LinkButton
       to={path ?? "#"}
-      variant={"tertiary/small"}
+      variant={"minimal/small"}
       LeadingIcon="chevron-left"
+      leadingIconClassName="text-text-dimmed"
       className={cn(
         "flex items-center",
-        !path && "cursor-default opacity-50 group-hover:bg-transparent group-hover:text-slate-800"
+        !path &&
+          "cursor-not-allowed opacity-50 group-hover:bg-transparent group-hover:text-text-dimmed"
       )}
       onClick={(e) => !path && e.preventDefault()}
     >