Release v3.0.0

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.changeset/chilled-hornets-move.md

@@ -1,5 +0,0 @@
----
-"trigger.dev": patch
----
-
-Vastly improved dev command output

.changeset/clean-pianos-listen.md

@@ -1,8 +0,0 @@
----
-"@trigger.dev/sdk": patch
-"@trigger.dev/core-apps": patch
-"trigger.dev": patch
-"@trigger.dev/core": patch
----
-
-add machine config and secure zod connection

.changeset/config.json

@@ -4,7 +4,8 @@
   "commit": false,
   "fixed": [
     [
-      "@trigger.dev/*"
+      "@trigger.dev/*",
+      "trigger.dev"
     ]
   ],
   "linked": [],
@@ -15,11 +16,11 @@
     "webapp",
     "emails",
     "proxy",
-    "yalt",
     "@trigger.dev/database",
     "coordinator",
     "docker-provider",
-    "kubernetes-provider"
+    "kubernetes-provider",
+    "@trigger.dev/otlp-importer"
   ],
   "___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH": {
     "onlyUpdatePeerDependentsWhenOutOfRange": true

.changeset/lemon-jobs-repair.md

@@ -1,5 +0,0 @@
----
-"@trigger.dev/nestjs": patch
----
-
-fix: [nestjs integration] fastify HTTP adapter detection now works correctly for response headers

.changeset/light-bulldogs-press.md

@@ -1,5 +0,0 @@
----
-"trigger.dev": patch
----
-
-Changed "Worker" to "Version" in the dev command key

.changeset/pre.json

@@ -1,57 +0,0 @@
-{
-  "mode": "pre",
-  "tag": "beta",
-  "initialVersions": {
-    "coordinator": "0.0.1",
-    "docker-provider": "0.0.1",
-    "kubernetes-provider": "0.0.1",
-    "proxy": "0.0.11",
-    "webapp": "1.0.0",
-    "yalt": "0.0.1",
-    "@trigger.dev/airtable": "2.3.18",
-    "@trigger.dev/github": "2.3.18",
-    "@trigger.dev/linear": "2.3.18",
-    "@trigger.dev/openai": "2.3.18",
-    "@trigger.dev/plain": "2.3.18",
-    "@trigger.dev/replicate": "2.3.18",
-    "@trigger.dev/resend": "2.3.18",
-    "@trigger.dev/sendgrid": "2.3.18",
-    "@trigger.dev/shopify": "2.3.18",
-    "@trigger.dev/slack": "2.3.18",
-    "@trigger.dev/stripe": "2.3.18",
-    "@trigger.dev/supabase": "2.3.18",
-    "@trigger.dev/typeform": "2.3.18",
-    "@trigger.dev/astro": "2.3.18",
-    "@trigger.dev/cli": "2.3.18",
-    "trigger.dev": "2.3.18",
-    "@trigger.dev/core": "2.3.18",
-    "@trigger.dev/core-apps": "0.0.0",
-    "@trigger.dev/core-backend": "2.3.18",
-    "@trigger.dev/database": "0.0.1",
-    "emails": "1.0.0",
-    "@trigger.dev/eslint-plugin": "2.3.18",
-    "@trigger.dev/express": "2.3.18",
-    "@trigger.dev/hono": "2.3.18",
-    "@trigger.dev/integration-kit": "2.3.18",
-    "@trigger.dev/nestjs": "2.3.18",
-    "@trigger.dev/nextjs": "2.3.18",
-    "@trigger.dev/otlp-importer": "2.3.11",
-    "@trigger.dev/react": "2.3.18",
-    "@trigger.dev/remix": "2.3.18",
-    "@trigger.dev/sveltekit": "2.3.18",
-    "@trigger.dev/testing": "2.3.18",
-    "@trigger.dev/sdk": "2.3.18",
-    "@trigger.dev/yalt": "2.3.18"
-  },
-  "changesets": [
-    "chilled-hornets-move",
-    "clean-pianos-listen",
-    "lemon-jobs-repair",
-    "light-bulldogs-press",
-    "real-planets-stare",
-    "strange-ghosts-matter",
-    "stupid-bulldogs-applaud",
-    "sweet-lizards-press",
-    "tall-bees-wave"
-  ]
-}

.changeset/real-planets-stare.md

@@ -1,7 +0,0 @@
----
-"@trigger.dev/sdk": patch
-"trigger.dev": patch
-"@trigger.dev/core": patch
----
-
-Fixed incorrect span timings around checkpoints by implementing a precise wall clock that resets after restores

.changeset/strange-ghosts-matter.md

@@ -1,5 +0,0 @@
----
-"trigger.dev": patch
----
-
-Added DEBUG to the ignored env vars

.changeset/stupid-bulldogs-applaud.md

@@ -1,5 +0,0 @@
----
-"trigger.dev": patch
----
-
-Fixed an issue where the trigger.dev package was not being built before publishing to npm

.changeset/sweet-lizards-press.md

@@ -1,8 +0,0 @@
----
-"trigger.dev": major
-"@trigger.dev/core": major
-"@trigger.dev/otlp-importer": major
-"@trigger.dev/sdk": major
----
-
-Updates to support Trigger.dev v3

.changeset/tall-bees-wave.md

@@ -1,7 +0,0 @@
----
-"@trigger.dev/sdk": patch
-"trigger.dev": patch
-"@trigger.dev/core": patch
----
-
-Dynamically import superjson and fix some bundling issues

.configs/tsconfig.base.json

@@ -0,0 +1,36 @@
+{
+  "compilerOptions": {
+    "target": "es2022",
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "moduleDetection": "force",
+    "verbatimModuleSyntax": false,
+    "jsx": "react",
+
+    "strict": true,
+    "alwaysStrict": true,
+    "strictPropertyInitialization": false,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noImplicitAny": true,
+    "noImplicitReturns": true,
+    "noImplicitThis": true,
+
+    "noFallthroughCasesInSwitch": true,
+    "resolveJsonModule": true,
+
+    "removeComments": false,
+    "esModuleInterop": true,
+    "emitDecoratorMetadata": false,
+    "experimentalDecorators": false,
+    "downlevelIteration": true,
+    "isolatedModules": true,
+    "noUncheckedIndexedAccess": true,
+
+    "pretty": true,
+    "customConditions": ["@triggerdotdev/source"]
+  }
+}

.dockerignore

@@ -37,6 +37,7 @@ packages/\*\*/dist
 .vercel
 .cache
 .output
+.trigger
 apps/\*\*/public/build
 
 cypress/screenshots

.env.example

@@ -10,16 +10,23 @@ DIRECT_URL=${DATABASE_URL}
 REMIX_APP_PORT=3030
 APP_ENV=development
 APP_ORIGIN=http://localhost:3030
+ELECTRIC_ORIGIN=http://localhost:3060
 NODE_ENV=development
+V3_ENABLED=true
 
-# Redis is used for concurrency control
-# REDIS_HOST="localhost"
-# REDIS_PORT="6379"
-# REDIS_TLS_DISABLED="true"
+# Redis is used for the v3 queuing and v2 concurrency control
+REDIS_HOST="localhost"
+REDIS_PORT="6379"
+REDIS_TLS_DISABLED="true"
+
+DEV_OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:3030/otel"
+DEV_OTEL_BATCH_PROCESSING_ENABLED="0"
 
 # OPTIONAL VARIABLES
 # This is used for validating emails that are allowed to log in. Every email that do not match this regex will be rejected.
 # WHITELISTED_EMAILS="authorized@yahoo\.com|authorized@gmail\.com"
+# Accounts with these emails will get global admin rights. This grants access to the admin UI.
+# ADMIN_EMAILS="admin@example\.com|another-admin@example\.com"
 # This is used for logging in via GitHub. You can leave these commented out if you don't want to use GitHub for authentication.
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
@@ -64,4 +71,12 @@ COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl
 # OBJECT_STORE_BASE_URL="https://{bucket}.{accountId}.r2.cloudflarestorage.com"
 # OBJECT_STORE_ACCESS_KEY_ID=
 # OBJECT_STORE_SECRET_ACCESS_KEY=
-# RUNTIME_WAIT_THRESHOLD_IN_MS=10000
+# CHECKPOINT_THRESHOLD_IN_MS=10000
+
+# These control the server-side internal telemetry
+# INTERNAL_OTEL_TRACE_EXPORTER_URL=<URL to send traces to>
+# INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_NAME=<Header name for the auth token>
+# INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE=<Auth token value>
+# INTERNAL_OTEL_TRACE_LOGGING_ENABLED=1
+# INTERNAL_OTEL_TRACE_SAMPING_RATE=20 # this means 1/20 traces or 5% of traces will be sampled (sampled = recorded)
+# INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED=0,

.github/workflows/e2e.yml

@@ -1,25 +1,30 @@
-name: "🧪 E2E Tests"
+name: "E2E"
 on:
   workflow_call:
+    inputs:
+      package:
+        description: The identifier of the job to run
+        default: webapp
+        required: false
+        type: string
 jobs:
-  e2e:
-    name: "🧪 E2E Tests"
-    runs-on: buildjet-16vcpu-ubuntu-2204
+  cli-v3:
+    name: "🧪 CLI v3 tests (${{ matrix.os }} - ${{ matrix.package-manager }})"
+    if: inputs.package == 'cli-v3' || inputs.package == ''
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [buildjet-8vcpu-ubuntu-2204, windows-latest]
+        package-manager: ["npm", "pnpm"]
     steps:
-      - name: 🐳 Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME || vars.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN || vars.DOCKERHUB_TOKEN }}
-
       - name: ⬇️ Checkout repo
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-          submodules: recursive
 
       - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
         with:
           version: 8.15.5
 
@@ -27,48 +32,22 @@ jobs:
         uses: buildjet/setup-node@v3
         with:
           node-version: 20.11.1
-          cache: "pnpm"
-
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v3
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: 📥 Download deps
-        run: pnpm install --frozen-lockfile
+        run: pnpm install --frozen-lockfile --filter trigger.dev...
 
-      - name: Install Playwright Browsers
-        run: npx playwright install --with-deps
+      - name: 🔧 Build v3 cli monorepo dependencies
+        run: pnpm run build --filter trigger.dev^...
 
-      - name: Run Playwright tests
+      - name: 🔧 Build worker template files
+        run: pnpm --filter trigger.dev run build:workers
+
+      - name: Enable corepack
+        run: corepack enable
+
+      - name: Run E2E Tests
+        shell: bash
         run: |
-          # Setup environment variables
-          cp ./.env.example ./.env
-          cp ./references/nextjs-test/.env.example ./references/nextjs-test/.env.local
+          LOG=debug PM=${{ matrix.package-manager }} pnpm --filter trigger.dev run test:e2e
 
-          # Build packages
-          pnpm run build --filter @references/nextjs-test^...
-          cd apps/webapp && pnpm run build:server
-          cd ../..
-          pnpm --filter @trigger.dev/database generate
-          pnpm --filter @trigger.dev/otlp-importer generate
 
-          # Move trigger-cli bin to correct place
-          pnpm install --frozen-lockfile
-
-          # Execute tests
-          pnpm run docker
-          pnpm run db:migrate
-          pnpm run db:seed
-          pnpm run test:e2e
-
-          # Cleanup
-          pnpm run docker:stop
-
-      - name: Upload Playwright report
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: playwright-report
-          path: playwright-report/
-          retention-days: 30

.github/workflows/pr_checks.yml

@@ -2,13 +2,7 @@ name: 🤖 PR Checks
 
 on:
   pull_request:
-    branches:
-      - main
-    paths-ignore:
-      - "**.md"
-      - "**.mdx"
-      - ".github/CODEOWNERS"
-      - ".github/ISSUE_TEMPLATE/**"
+    types: [opened, synchronize, reopened]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -29,4 +23,38 @@ jobs:
 
   e2e:
     uses: ./.github/workflows/e2e.yml
+    with:
+      package: cli-v3
     secrets: inherit
+
+  preview-release:
+    name: Preview Release
+    needs: [typecheck, units, e2e]
+    if: |
+      github.repository == 'triggerdotdev/trigger.dev'
+    runs-on: buildjet-8vcpu-ubuntu-2204
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 8.15.5
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v3
+        with:
+          node-version: 20.11.1
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 🏗️ Build
+        run: pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev"
+
+      - name: ⚡ publish preview release
+        run: npx pkg-pr-new publish --no-template $(ls -d ./packages/*)

.github/workflows/publish-docker.yml

@@ -4,13 +4,15 @@ on:
 jobs:
   publish:
     runs-on: ubuntu-latest
+    env:
+      PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING: 1
     outputs:
       version: ${{ steps.get_version.outputs.version }}
       short_sha: ${{ steps.get_commit.outputs.sha_short }}
     steps:
       - name: Setup Depot CLI
         uses: depot/setup-action@v1
-        
+
       - name: ⬇️ Checkout repo
         uses: actions/checkout@v3
         with:
@@ -37,11 +39,25 @@ jobs:
             exit 1
           fi
           echo "::set-output name=version::${IMAGE_TAG}"
+
       - name: 🔢 Get the commit hash
         id: get_commit
         run: |
           echo ::set-output name=sha_short::$(echo ${{ github.sha }} | cut -c1-7)
 
+      - name: 📛 Set the tags
+        id: set_tags
+        run: |
+          ref_without_tag=ghcr.io/triggerdotdev/trigger.dev
+          image_tags=$ref_without_tag:${{ steps.get_version.outputs.version }}
+
+          # if it's a versioned tag, also tag it as v3
+          if [[ "${{ github.ref_name }}" == v.docker.* ]]; then
+            image_tags=$image_tags,$ref_without_tag:v3
+          fi
+          
+          echo "IMAGE_TAGS=${image_tags}" >> "$GITHUB_OUTPUT"
+
       - name: 🐙 Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:
@@ -54,6 +70,5 @@ jobs:
         with:
           file: ./docker/Dockerfile
           platforms: linux/amd64,linux/arm64
-          tags: |
-            ghcr.io/triggerdotdev/trigger.dev:${{ steps.get_version.outputs.version }}
+          tags: ${{ steps.set_tags.outputs.IMAGE_TAGS }}
           push: true

.github/workflows/publish-infra.yml

@@ -1,6 +1,7 @@
 name: "🚢 Publish Infra Images"
 
 on:
+  workflow_call:
   push:
     tags:
       - "infra-dev-*"
@@ -29,9 +30,6 @@ permissions:
   packages: write
   contents: read
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-
 env:
   AWS_REGION: us-east-1
 
@@ -39,7 +37,7 @@ jobs:
   build:
     strategy:
       matrix:
-        package: [coordinator, kubernetes-provider]
+        package: [coordinator, docker-provider, kubernetes-provider]
     runs-on: buildjet-16vcpu-ubuntu-2204
     env:
       DOCKER_BUILDKIT: "1"
@@ -48,20 +46,40 @@ jobs:
 
       - name: Generate image reference
         id: prep
-        # WARNING: This step expects the workflow to have been triggered by a specific tag format of: infra-${env}-*
         run: |
-          env=$(echo ${{ github.ref_name }} | cut -d- -f2)
-          sha=${GITHUB_SHA::7}
-          ts=$(date +%s)
+          # set image repo
           if [[ "${{ matrix.package }}" == *-provider ]]; then
-            provider_type=$(echo ${{ matrix.package }} | cut -d- -f1)
+            provider_type=$(echo "${{ matrix.package }}" | cut -d- -f1)
             repository=provider/${provider_type}
           else
-            repository=${{ matrix.package }}
+            repository="${{ matrix.package }}"
           fi
-          echo "IMAGE_TAG=${env}-${sha}-${ts}" >> "$GITHUB_OUTPUT"
           echo "REPOSITORY=${repository}" >> "$GITHUB_OUTPUT"
 
+          # set image tag
+          if [[ "${{ github.ref_type }}" == "tag" ]]; then
+            if [[ "${{ github.ref_name }}" == infra-*-* ]]; then
+              env=$(echo ${{ github.ref_name }} | cut -d- -f2)
+              sha=$(echo ${{ github.sha }} | head -c7)
+              ts=$(date +%s)
+              image_tag=${env}-${sha}-${ts}
+            elif [[ "${{ github.ref_name }}" == v.docker.* ]]; then
+              version="${GITHUB_REF_NAME#v.docker.}"
+              image_tag="v${version}"
+            elif [[ "${{ github.ref_name }}" == build-* ]]; then
+              image_tag="${GITHUB_REF_NAME#build-}"
+            else
+              echo "Invalid tag: ${{ github.ref_name }}"
+              exit 1
+            fi
+          elif [[ "${{ github.ref_name }}" == "main" ]]; then
+            image_tag="main"
+          else
+            echo "Invalid reference: ${{ github.ref }}"
+            exit 1
+          fi
+          echo "IMAGE_TAG=${image_tag}" >> "$GITHUB_OUTPUT"
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
@@ -92,3 +110,12 @@ jobs:
           REGISTRY: ghcr.io/triggerdotdev
           REPOSITORY: ${{ steps.prep.outputs.REPOSITORY }}
           IMAGE_TAG: ${{ steps.prep.outputs.IMAGE_TAG }}
+
+      - name: 🐙 Push 'v3' tag to GitHub Container Registry
+        if: startsWith(github.ref_name, 'v.docker.')
+        run: |
+          docker tag infra_image $REGISTRY/$REPOSITORY:v3
+          docker push $REGISTRY/$REPOSITORY:v3
+        env:
+          REGISTRY: ghcr.io/triggerdotdev
+          REPOSITORY: ${{ steps.prep.outputs.REPOSITORY }}

.github/workflows/publish.yml

@@ -49,11 +49,12 @@ jobs:
     uses: ./.github/workflows/unit-tests.yml
     secrets: inherit
 
-  e2e:
-    uses: ./.github/workflows/e2e.yml
-    secrets: inherit
-
   publish:
-    needs: [typecheck, units, e2e]
+    needs: [typecheck, units]
     uses: ./.github/workflows/publish-docker.yml
     secrets: inherit
+
+  publish-infra:
+    needs: [typecheck, units]
+    uses: ./.github/workflows/publish-infra.yml
+    secrets: inherit

.github/workflows/release.yml

@@ -6,6 +6,7 @@ on:
       - main
     paths-ignore:
       - "**.md"
+      - "**.mdx"
       - ".github/CODEOWNERS"
       - ".github/ISSUE_TEMPLATE/**"
 
@@ -28,7 +29,7 @@ jobs:
           fetch-depth: 0
 
       - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
         with:
           version: 8.15.5
 
@@ -38,17 +39,15 @@ jobs:
           node-version: 20.11.1
           cache: "pnpm"
 
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v3
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-
       - name: 📥 Download deps
         run: pnpm install --frozen-lockfile
 
       - name: 📀 Generate Prisma Client
         run: pnpm run generate
 
+      - name: 🏗️ Build
+        run: pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev"
+
       - name: 🔎 Type check
         run: pnpm run typecheck --filter "@trigger.dev/*" --filter "trigger.dev"
 

.github/workflows/typecheck.yml

@@ -12,7 +12,7 @@ jobs:
           fetch-depth: 0
 
       - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
         with:
           version: 8.15.5
 
@@ -22,11 +22,6 @@ jobs:
           node-version: 20.11.1
           cache: "pnpm"
 
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v3
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-
       - name: 📥 Download deps
         run: pnpm install --frozen-lockfile
 
@@ -34,4 +29,7 @@ jobs:
         run: pnpm run generate
 
       - name: 🔎 Type check
-        run: pnpm run typecheck --filter webapp
+        run: pnpm run typecheck
+      
+      - name: 🔎 Check exports
+        run: pnpm run check-exports

.github/workflows/unit-tests.yml

@@ -12,7 +12,7 @@ jobs:
           fetch-depth: 0
 
       - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
         with:
           version: 8.15.5
 
@@ -22,21 +22,6 @@ jobs:
           node-version: 20.11.1
           cache: "pnpm"
 
-      - name: Install Protoc
-        uses: arduino/setup-protoc@v3
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: ⎔ Setup Deno
-        uses: denoland/setup-deno@v1
-        with:
-          deno-version: v1.x
-
-      - name: ⎔ Setup bun
-        uses: oven-sh/setup-bun@v1
-        with:
-          bun-version: "1.0.15"
-
       - name: 📥 Download deps
         run: pnpm install --frozen-lockfile
 

.gitignore

@@ -12,7 +12,6 @@ coverage
 # next.js
 .next/
 out/
-build
 dist
 packages/**/dist
 
@@ -54,4 +53,8 @@ apps/**/public/build
 /playwright/.cache/
 
 .cosine
-.trigger/
+.trigger
+.tshy*
+.yarn
+*.tsbuildinfo
+/packages/cli-v3/src/package.json

.prettierignore

@@ -7,5 +7,4 @@ tailwind.css
 **/.react-email/
 **/storybook-static/
 **/.changeset/
-**/build/
 **/dist/

.vscode/extensions.json

@@ -1,9 +1,8 @@
 {
 	"recommendations": [
-        "astro-build.astro-vscode",
         "denoland.vscode-deno"
     ],
 	"unwantedRecommendations": [
 		
 	]
-}
+}

.vscode/launch.json

@@ -23,17 +23,16 @@
     {
       "type": "node-terminal",
       "request": "launch",
-      "name": "Debug BYO Auth",
-      "command": "pnpm run byo-auth",
-      "envFile": "${workspaceFolder}/references/job-catalog/.env",
-      "cwd": "${workspaceFolder}/references/job-catalog",
+      "name": "Debug V3 init CLI",
+      "command": "pnpm exec triggerdev init",
+      "cwd": "${workspaceFolder}/references/init-shell",
       "sourceMaps": true
     },
     {
       "type": "node-terminal",
       "request": "launch",
       "name": "Debug V3 Dev CLI",
-      "command": "pnpm exec trigger.dev dev --log-level debug",
+      "command": "pnpm exec triggerdev dev",
       "cwd": "${workspaceFolder}/references/v3-catalog",
       "sourceMaps": true
     },
@@ -41,7 +40,31 @@
       "type": "node-terminal",
       "request": "launch",
       "name": "Debug V3 Deploy CLI",
-      "command": "pnpm exec trigger.dev deploy",
+      "command": "pnpm exec triggerdev deploy --self-hosted --load-image",
+      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 list-profiles CLI",
+      "command": "pnpm exec triggerdev list-profiles --log-level debug",
+      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 update CLI",
+      "command": "pnpm exec triggerdev update",
+      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 Management",
+      "command": "pnpm run management",
       "cwd": "${workspaceFolder}/references/v3-catalog",
       "sourceMaps": true
     },
@@ -52,6 +75,22 @@
       "port": 9229,
       "restart": true,
       "skipFiles": ["<node_internals>/**"]
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug CLI e2e tests",
+      "command": "MOD=otel-telemetry-loader pnpm run test:e2e",
+      "cwd": "${workspaceFolder}/packages/cli-v3",
+      "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "debug v3 hello-world dev",
+      "command": "pnpm exec triggerdev dev",
+      "cwd": "${workspaceFolder}/references/hello-world",
+      "sourceMaps": true
     }
   ]
 }

.vscode/settings.json

@@ -1,4 +1,9 @@
 {
   "deno.enablePaths": ["references/deno-reference", "runtime_tests/tests/deno"],
-  "debug.toolBarLocation": "commandCenter"
+  "debug.toolBarLocation": "commandCenter",
+  "typescript.tsdk": "node_modules/typescript/lib",
+  "search.exclude": {
+    "**/node_modules/**": true,
+    "packages/cli-v3/e2e": true
+  }
 }

CONTRIBUTING.md

@@ -10,13 +10,14 @@ Thank you for helping us make Trigger.dev even better! 🤩
 
 The development branch is `main`. This is the branch that all pull
 requests should be made against. The changes on the `main`
-branch are tagged into a release monthly.
+branch are tagged into a release periodically.
 
 ### Prerequisites
 
-- [Node.js](https://nodejs.org/en) version >=16.x
-- [pnpm package manager](https://pnpm.io/installation) version 7
+- [Node.js](https://nodejs.org/en) version 20.11.1
+- [pnpm package manager](https://pnpm.io/installation) version 8.15.5
 - [Docker](https://www.docker.com/get-started/)
+- [protobuf](https://github.com/protocolbuffers/protobuf)
 
 ### Setup
 
@@ -33,15 +34,19 @@ branch are tagged into a release monthly.
    ```
    cd trigger.dev
    ```
-3. Install the required packages using pnpm.
+3. Ensure you are on the correct version of Node.js (20.11.1). If you are using `nvm`, there is an `.nvmrc` file that will automatically select the correct version of Node.js when you navigate to the repository.
+
+4. Run `corepack enable` to use the correct version of pnpm (`8.15.5`) as specified in the root `package.json` file.
+
+5. Install the required packages using pnpm.
    ```
    pnpm i
    ```
-4. Create your `.env` file
+6. Create your `.env` file
    ```
    cp .env.example .env
    ```
-5. Open it and generate a new value for `ENCRYPTION_KEY`:
+7. Open it and generate a new value for `ENCRYPTION_KEY`:
 
    `ENCRYPTION_KEY` is used to two-way encrypt OAuth access tokens and so you'll probably want to actually generate a unique value, and it must be a random 16 byte hex string. You can generate one with the following command:
 
@@ -51,7 +56,7 @@ branch are tagged into a release monthly.
 
    Feel free to update `SESSION_SECRET` and `MAGIC_LINK_SECRET` as well using the same method.
 
-6. Start Docker. This starts the required services like Postgres. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)
+8. Start Docker. This starts the required services like Postgres & Redis. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)
 
    ```
    pnpm run docker
@@ -59,19 +64,15 @@ branch are tagged into a release monthly.
 
    This will also start and run a local instance of [pgAdmin](https://www.pgadmin.org/) on [localhost:5480](http://localhost:5480), preconfigured with email `admin@example.com` and pwd `admin`. Then use `postgres` as the password to the Trigger.dev server.
 
-7. Migrate the database
+9. Migrate the database
    ```
    pnpm run db:migrate
    ```
-8. Build the app
-   ```
-   pnpm run build --filter webapp
-   ```
-9. Run the seed script
-   ```
-   pnpm run db:seed
-   ```
-10. Run the app. See the section below.
+10. Build the server app
+    ```
+    pnpm run build --filter webapp
+    ```
+11. Run the app. See the section below.
 
 ## Running
 
@@ -83,99 +84,96 @@ branch are tagged into a release monthly.
 
    It should run on port `3030`: [http://localhost:3030](http://localhost:3030/)
 
-2. Once the app is running click the magic link button and enter your email.
-3. Check your terminal, the magic link email should have printed out as following:
+2. Once the app is running click the magic link button and enter your email. You will automatically be logged in, since you are running locally. Create an Org and your first project in the dashboard.
 
-   ```sh
-   webapp:dev: Log in to Trigger.dev
-   webapp:dev:
-   webapp:dev: Click here to log in with this magic link
-   webapp:dev: [http://localhost:3030/magic?token=U2FsdGVkX18OvB0JxgaswTLCSbaRz%2FY82TN0EZWhSzFyZYwgG%2BIzKVTkeiaOtWfotPw7F8RwFzCHh53aBpMEu%2B%2B%2FItb%2FcJYh89MSjc3Pz92bevoEjqxSQ%2Ff%2BZbks09JOpqlBbYC3FzGWC8vuSVFBlxqLXxteSDLthZSUaC%2BS2LaA%2BJgp%2BLO7hgjAaC2lXbCHrM7MTgTdXOFt7i0Dvvuwz6%2BWY25RnfomZOPqDsyH0xz8Q2rzPTz0Xu53WSXrZ1hd]
-   webapp:dev:
-   webapp:dev: If you didn't try to log in, you can safely ignore this email.
-   ```
+## Manual testing using v3-catalog
 
-   Paste the magic link shown in your terminal into your browser to login.
+We use the `<root>/references/v3-catalog` subdirectory as a staging ground for testing changes to the SDK (`@trigger.dev/sdk` at `<root>/packages/trigger-sdk`), the Core package (`@trigger.dev/core` at `<root>packages/core`), the CLI (`trigger.dev` at `<root>/packages/cli-v3`) and the platform (The remix app at `<root>/apps/webapp`). The instructions below will get you started on using the `v3-catalog` for local development of Trigger.dev (v3).
 
-## Adding and running migrations
+### First-time setup
 
-1. Modify packages/database/prisma/schema.prisma file
-2. Change directory to the packages/database folder
-   ```sh
-   cd packages/database
-   ```
-3. Generate the Prisma client
+First, make sure you are running the webapp according to the instructions above. Then:
 
-   ```sh
-   pnpm run generate
-   ```
+1. In Postgres go to the "Organizations" table and on your org set the `v3Enabled` column to `true`.
 
-   The above updates the prisma client generated into node_modules/.prisma/client folder. This helps with typing of relevant prisma models. It ensures typescript
-   recognizes fields added or removed from a model and type-checks appropriately.
+2. Visit http://localhost:3030 in your browser and create a new V3 project called "v3-catalog". If you don't see an option for V3, you haven't set the `v3Enabled` flag to true.
 
-4. Create and apply the migrations
+3. In Postgres go to the "Projects" table and for the project you create change the `externalRef` to `yubjwjsfkxnylobaqvqz`.
 
-   ```
-   pnpm run db:migrate:dev
-   ```
-
-   This creates a migration file and executes the migrations against your database and applies changes to the database schema(s)
-
-5. Commit generated migrations as well as changes to the schema.prisma file
-6. If you're using VSCode you may need to restart the Typescript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`.
-
-## Testing CLI changes
-
-To test CLI changes, follow the steps below:
-
-1. Build the CLI and watch for changes
+4. Build the CLI
 
 ```sh
-cd packages/cli
-pnpm run dev
-```
-
-2. Open a new Terminal window and run the webapp locally and then create a new project in the dashboard. Copy out the dev API key.
-
-3. Create a new temporary Next.js app in references directory
-
-```sh
-cd ./references
-pnpm create next-app@latest test-cli --ts --no-eslint --tailwind --app --src-dir --import-alias "@/*"
-```
-
-4. Then once that's finished, add the `@trigger.dev/cli` to the `devDependencies` of the newly created Next.js app's `package.json` file, like so:
-
-```json
-{
-  // other package.json properties
-  "devDependencies": { "@trigger.dev/cli": "workspace:*" }
-}
-```
-
-5. Back in the terminal, navigate into the reference, and initialize the CLI. When prompted, select `self-hosted` and enter `localhost:3030` if you are testing against the local instance of Trigger.dev, or you can just use the Trigger.dev cloud. When asked for an API key, use the key you copied earlier.
-
-```sh
-cd ./test-cli
+# Build the CLI
+pnpm run build --filter trigger.dev
+# Make it accessible to `pnpm exec`
 pnpm i
-pnpm exec trigger-cli init
 ```
 
-6. If you are just testing the `init` command, you can stop here. If you'd like to test the `dev` command, first start the Next.js app on port 3000:
+5. Change into the `<root>/references/v3-catalog` directory and authorize the CLI to the local server:
 
 ```sh
-pnpm run dev
+cd references/v3-catalog
+cp .env.example .env
+pnpm exec triggerdev login -a http://localhost:3030
 ```
 
-7. Open a new terminal window, and then run the `dev` command like so:
+This will open a new browser window and authorize the CLI against your local user account.
+
+You can optionally pass a `--profile` flag to the `login` command, which will allow you to use the CLI with separate accounts/servers. We suggest using a profile called `local` for your local development:
 
 ```sh
-pnpm exec trigger-cli dev
+cd references/v3-catalog
+pnpm exec triggerdev login -a http://localhost:3030 --profile local
+# later when you run the dev or deploy command:
+pnpm exec triggerdev dev --profile local
+pnpm exec triggerdev deploy --profile local
 ```
 
-8. Please remember to delete the temporary project you created after you've tested the changes, and before you raise a PR.
+### Running
 
-## Running end-to-end webapp tests
+The following steps should be followed any time you start working on a new feature you want to test in v3:
+
+1. Make sure the webapp is running on localhost:3030
+
+2. Open a terminal window and build the CLI and watch for changes
+
+```sh
+pnpm run dev --filter trigger.dev
+```
+
+2. Open a new terminal window, and anytime changes are made to the `@trigger.dev/core` package, you'll need to manually rebuild the CLI:
+
+```sh
+pnpm run build --filter trigger.dev
+```
+
+Note: You do not need to do the same for `@trigger.dev/sdk`, just core.
+
+3. Open another terminal window, and change into the `<root>/references/v3-catalog` directory.
+
+4. Run the `dev` command, which will register all the local tasks with the platform and allow you to start testing task execution:
+
+```sh
+# in <root>/references/v3-catalog
+pnpm exec triggerdev dev
+```
+
+If you want additional debug logging, you can use the `--log-level debug` flag:
+
+```sh
+# in <root>/references/v3-catalog
+pnpm exec triggerdev dev --log-level debug
+```
+
+5. If you make any changes in the CLI/Core/SDK, you'll need to `CTRL+C` to exit the `dev` command and restart it to pickup changes. Any changes to the files inside of the `v3-catalog/src/trigger` dir will automatically be rebuilt by the `dev` command.
+
+6. Navigate to the `v3-catalog` project in your local dashboard at localhost:3030 and you should see the list of tasks.
+
+7. Go to the "Test" page in the sidebar and select a task. Then enter a payload and click "Run test". You can tell what the payloads should be by looking at the relevant task file inside the `/references/v3-catalog/src/trigger` folder. Many of them accept an empty payload.
+
+8. Feel free to add additional files in `v3-catalog/src/trigger` to test out specific aspects of the system, or add in edge cases.
+
+## Running end-to-end webapp tests (deprecated)
 
 To run the end-to-end tests, follow the steps below:
 
@@ -223,6 +221,26 @@ The end-to-end tests use a `setup` and `teardown` script to seed the database wi
 pnpm run db:studio
 ```
 
+## Adding and running migrations
+
+1. Modify packages/database/prisma/schema.prisma file
+2. Change directory to the packages/database folder
+
+   ```sh
+   cd packages/database
+   ```
+
+3. Create and apply the migrations
+
+   ```
+   pnpm run db:migrate:dev
+   ```
+
+   This creates a migration file and executes the migrations against your database and applies changes to the database schema(s)
+
+4. Commit generated migrations as well as changes to the schema.prisma file
+5. If you're using VSCode you may need to restart the Typescript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`.
+
 ## Add sample jobs
 
 The [references/job-catalog](./references/job-catalog/) project defines simple jobs you can get started with.

README.md

@@ -5,86 +5,71 @@
   <img alt="Trigger.dev logo" src="https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/a45d1fa2-0ae8-4a39-4409-f4f934bfae00/public">
 </picture>
   
-### The open source background jobs framework
+### Open source background jobs with no timeouts
 
-[Discord](https://discord.gg/JtBAxBr2m3) | [Website](https://trigger.dev) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Docs](https://trigger.dev/docs)
+[Discord](https://trigger.dev/discord) | [Website](https://trigger.dev) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Docs](https://trigger.dev/docs)
 
 [![Twitter](https://img.shields.io/twitter/url/https/twitter.com/triggerdotdev.svg?style=social&label=Follow%20%40trigger.dev)](https://twitter.com/triggerdotdev)
-[![GitHub Repo stars](https://img.shields.io/github/stars/triggerdotdev/trigger.dev?style=social)](https://github.com/triggerdotdev/trigger.dev)
 
 </div>
 
-# About Trigger.dev
+## About Trigger.dev
 
-Create long-running jobs directly in your codebase with features like API integrations, webhooks, scheduling and delays.
+Trigger.dev is an open source platform and SDK which allows you to create long-running background jobs with no timeouts. Write normal async code, deploy, and never hit a timeout.
 
-## Long running Jobs on serverless
+#### Features:
 
-Reliably run jobs and don’t worry about function timeouts, we handle those for you.
-
-- Auto-resume after a function timeout
-- Auto-resume after a server outage
-- Add delays of up to a year
+- JavaScript and TypeScript SDK
+- Write reliable code by default
+- No infrastructure to manage
+- Works with your existing tech stack
 
 ## In your codebase
 
-Create Jobs where they belong: in your codebase. Version control, localhost, test, review, and deploy like you're already used to.
+Create tasks where they belong: in your codebase. Version control, localhost, test and review like you're already used to.
 
-## Secure by design
+```ts
+import { task } from "@trigger.dev/sdk/v3";
 
-We only receive Triggers and the data you choose to send to us. You can even completely self-host the entire platform.
+//1. You need to export each task
+export const helloWorld = task({
+  //2. Use a unique id for each task
+  id: "hello-world",
+  //3. The run function is the main function of the task
+  run: async (payload: { message: string }) => {
+    //4. You can write code that runs for a long time here, there are no timeouts
+    console.log(payload.message);
+  },
+});
+```
 
-## Don't worry about deployment
+## Deployment
 
-Just use our SDK to write Jobs in your codebase. There's nothing extra to deploy and no CI to configure, your Jobs just connect to our cloud. Or you can always self-host.
+Use our SDK to write tasks in your codebase. There's no infrastructure to manage, your tasks automatically scale and connect to our cloud. Or you can always [self-host](https://trigger.dev/docs/v3/open-source-self-hosting#overview).
+
+## Environments
+
+We support `Development`, `Staging`, and `Production` environments, allowing you to test your tasks before deploying them to production.
 
 ## Full visibility of every job run
 
-View every Task in every Run so you can tell exactly what happened.
+View every task in every run so you can tell exactly what happened. We provide a full trace view of every task run so you can see what happened at every step.
 
-![image](https://www.trigger.dev/build/_assets/web-app-2QFKXFLW.png)
-
-## Built-in integrations
-
-Easily integrate with hundreds of third-party APIs – including your own. Use API keys (which never leave your server) or let us handle OAuth for you. Install our integration packages and easily subscribe to webhooks and perform common tasks, or you can easily use your existing favorite Node.JS SDKs and get resumability and idempotency through our `runTask` function.
-
-## Our progress
-
-We’re building the most comprehensive and easy-to-use background jobs framework for developers.
-
-Click the links to join the discussions about our upcoming features.
-
-| Feature                                                                              | What it does                                        | Status |
-| ------------------------------------------------------------------------------------ | --------------------------------------------------- | ------ |
-| Integration kit                                                                      | Official Trigger.dev integrations or build your own | ✅     |
-| Self-hosting                                                                         | Host the platform yourself                          | ✅     |
-| Cloud                                                                                | Just write code, no deployment required             | ✅     |
-| Dashboard                                                                            | View every Task in every Run                        | ✅     |
-| Serverless                                                                           | Long-running Jobs on your serverless backend        | ✅     |
-| React hooks                                                                          | Easily update your UI with Job progress             | ✅     |
-| React frameworks                                                                     | Support for Remix, Astro, RedwoodJS & more          | ✅     |
-| [Background tasks](https://github.com/triggerdotdev/trigger.dev/discussions/400)     | Offload long or intense Tasks to our infrastructure | 🛠️     |
-| [Long-running servers](https://github.com/triggerdotdev/trigger.dev/discussions/430) | Run Jobs on your long-running backend               | 🛠️     |
-| Polling Triggers                                                                     | Subscribe to changes without webhooks               | 🕝     |
-| Vercel integration                                                                   | Easy deploy and preview environment support         | 🕝     |
-| Streaming                                                                            | Receive data from your Jobs in realtime             | 🕝     |
-| 100+ integrations                                                                    | Comprehensive support for popular APIs              | 🕝     |
-| [Trigger.dev Connect](https://github.com/triggerdotdev/trigger.dev/discussions/441)  | Use integrations signed in as your users            | 🕝     |
-| File IO                                                                              | Create Tasks that have file outputs                 | 🕝     |
+![Trace view image](https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/7c1b347f-004c-4482-38a7-3f6fa9c00d00/public)
 
 # Getting started
 
-Visit our docs [here](https://trigger.dev/docs).
+Visit our docs [here](https://trigger.dev/docs/v3/introduction) for a full guide on how to get started with Trigger.dev.
 
 ## Self-host
 
-We provide an official trigger.dev docker image you can use to easily self-host the platform. We're working on more extensive guides but we currently provide a [Fly.io example repository](https://github.com/triggerdotdev/fly.io) with instructions in the README for deploying and using a self-hosted instance of Trigger.dev on Fly.io.
+If you prefer to self-host, you can follow our [self-hosting guide](https://trigger.dev/docs/v3/open-source-self-hosting#overview).
 
 ## Development
 
 To setup and develop locally or contribute to the open source project, follow our [development guide](./CONTRIBUTING.md).
 
-## Meet the Amazing People Behind This Project 🚀
+## Meet the Amazing People Behind This Project:
 
 <a href="https://github.com/triggerdotdev/trigger.dev/graphs/contributors">
   <img src="https://contrib.rocks/image?repo=triggerdotdev/trigger.dev" />

apps/coordinator/Containerfile

@@ -1,19 +1,19 @@
 # syntax=docker/dockerfile:labs
 
-FROM node:18-bullseye-slim@sha256:a4edd54dcfdcacc8a4100fee71498e8671d99556a1acf5614539214a70092426 AS node-18
+FROM node:20-bookworm-slim@sha256:72f2f046a5f8468db28730b990b37de63ce93fd1a72a40f531d6aa82afdf0d46 AS node-20
 
 WORKDIR /app
 
-FROM node-18 AS pruner
+FROM node-20 AS pruner
 
 COPY --chown=node:node . .
 RUN npx -q turbo@1.10.9 prune --scope=coordinator --docker
 RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
 
-FROM node-18 AS base
+FROM node-20 AS base
 
 RUN apt-get update \
-  && apt-get install -y buildah ca-certificates dumb-init \
+  && apt-get install -y buildah ca-certificates dumb-init docker.io \
   && rm -rf /var/lib/apt/lists/*
 
 COPY --chown=node:node .gitignore .gitignore

apps/coordinator/package.json

@@ -17,12 +17,10 @@
   "license": "MIT",
   "dependencies": {
     "@trigger.dev/core": "workspace:*",
-    "@trigger.dev/core-apps": "workspace:*",
     "execa": "^8.0.1",
     "nanoid": "^5.0.6",
     "prom-client": "^15.1.0",
-    "socket.io": "^4.7.4",
-    "socket.io-client": "^4.7.4"
+    "socket.io": "4.7.4"
   },
   "devDependencies": {
     "@types/node": "^18",

apps/coordinator/src/chaosMonkey.ts

@@ -0,0 +1,99 @@
+import type { Execa$ } from "execa";
+import { setTimeout as timeout } from "node:timers/promises";
+
+class ChaosMonkeyError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ChaosMonkeyError";
+  }
+}
+
+export class ChaosMonkey {
+  private chaosEventRate = 0.2;
+  private delayInSeconds = 45;
+
+  constructor(
+    private enabled = false,
+    private disableErrors = false,
+    private disableDelays = false
+  ) {
+    if (this.enabled) {
+      console.log("🍌 Chaos monkey enabled");
+    }
+  }
+
+  static Error = ChaosMonkeyError;
+
+  enable() {
+    this.enabled = true;
+    console.log("🍌 Chaos monkey enabled");
+  }
+
+  disable() {
+    this.enabled = false;
+    console.log("🍌 Chaos monkey disabled");
+  }
+
+  async call({
+    $,
+    throwErrors = !this.disableErrors,
+    addDelays = !this.disableDelays,
+  }: {
+    $?: Execa$<string>;
+    throwErrors?: boolean;
+    addDelays?: boolean;
+  } = {}) {
+    if (!this.enabled) {
+      return;
+    }
+
+    const random = Math.random();
+
+    if (random > this.chaosEventRate) {
+      // Don't interfere with normal operation
+      return;
+    }
+
+    const chaosEvents: Array<() => Promise<any>> = [];
+
+    if (addDelays) {
+      chaosEvents.push(async () => {
+        console.log("🍌 Chaos monkey: Add delay");
+
+        if ($) {
+          await $`sleep ${this.delayInSeconds}`;
+        } else {
+          await timeout(this.delayInSeconds * 1000);
+        }
+      });
+    }
+
+    if (throwErrors) {
+      chaosEvents.push(async () => {
+        console.log("🍌 Chaos monkey: Throw error");
+
+        if ($) {
+          await $`false`;
+        } else {
+          throw new ChaosMonkey.Error("🍌 Chaos monkey: Throw error");
+        }
+      });
+    }
+
+    if (chaosEvents.length === 0) {
+      console.error("🍌 Chaos monkey: No events selected");
+      return;
+    }
+
+    const randomIndex = Math.floor(Math.random() * chaosEvents.length);
+
+    const chaosEvent = chaosEvents[randomIndex];
+
+    if (!chaosEvent) {
+      console.error("🍌 Chaos monkey: No event found");
+      return;
+    }
+
+    await chaosEvent();
+  }
+}

apps/coordinator/src/checkpointer.ts

@@ -0,0 +1,593 @@
+import { ExponentialBackoff } from "@trigger.dev/core/v3/apps";
+import { testDockerCheckpoint, isExecaChildProcess } from "@trigger.dev/core/v3/apps";
+import { SimpleLogger } from "@trigger.dev/core/v3/apps";
+import { $ } from "execa";
+import { nanoid } from "nanoid";
+import fs from "node:fs/promises";
+import { ChaosMonkey } from "./chaosMonkey";
+
+type CheckpointerInitializeReturn = {
+  canCheckpoint: boolean;
+  willSimulate: boolean;
+};
+
+type CheckpointAndPushOptions = {
+  runId: string;
+  leaveRunning?: boolean;
+  projectRef: string;
+  deploymentVersion: string;
+  shouldHeartbeat?: boolean;
+  attemptNumber?: number;
+};
+
+type CheckpointAndPushResult =
+  | { success: true; checkpoint: CheckpointData }
+  | {
+      success: false;
+      reason?: "CANCELED" | "DISABLED" | "ERROR" | "IN_PROGRESS" | "NO_SUPPORT" | "SKIP_RETRYING";
+    };
+
+type CheckpointData = {
+  location: string;
+  docker: boolean;
+};
+
+type CheckpointerOptions = {
+  dockerMode: boolean;
+  forceSimulate: boolean;
+  heartbeat: (runId: string) => void;
+  registryHost?: string;
+  registryNamespace?: string;
+  registryTlsVerify?: boolean;
+  disableCheckpointSupport?: boolean;
+  checkpointPath?: string;
+  simulateCheckpointFailure?: boolean;
+  simulateCheckpointFailureSeconds?: number;
+  simulatePushFailure?: boolean;
+  simulatePushFailureSeconds?: number;
+  chaosMonkey?: ChaosMonkey;
+};
+
+async function getFileSize(filePath: string): Promise<number> {
+  try {
+    const stats = await fs.stat(filePath);
+    return stats.size;
+  } catch (error) {
+    console.error("Error getting file size:", error);
+    return -1;
+  }
+}
+
+async function getParsedFileSize(filePath: string) {
+  const sizeInBytes = await getFileSize(filePath);
+
+  let message = `Size in bytes: ${sizeInBytes}`;
+
+  if (sizeInBytes > 1024 * 1024) {
+    const sizeInMB = (sizeInBytes / 1024 / 1024).toFixed(2);
+    message = `Size in MB (rounded): ${sizeInMB}`;
+  } else if (sizeInBytes > 1024) {
+    const sizeInKB = (sizeInBytes / 1024).toFixed(2);
+    message = `Size in KB (rounded): ${sizeInKB}`;
+  }
+
+  return {
+    path: filePath,
+    sizeInBytes,
+    message,
+  };
+}
+
+export class Checkpointer {
+  #initialized = false;
+  #canCheckpoint = false;
+  #dockerMode: boolean;
+
+  #logger = new SimpleLogger("[checkptr]");
+  #abortControllers = new Map<string, AbortController>();
+  #failedCheckpoints = new Map<string, unknown>();
+  #waitingForRetry = new Set<string>();
+
+  private registryHost: string;
+  private registryNamespace: string;
+  private registryTlsVerify: boolean;
+
+  private disableCheckpointSupport: boolean;
+  private checkpointPath: string;
+
+  private simulateCheckpointFailure: boolean;
+  private simulateCheckpointFailureSeconds: number;
+  private simulatePushFailure: boolean;
+  private simulatePushFailureSeconds: number;
+
+  private chaosMonkey: ChaosMonkey;
+
+  constructor(private opts: CheckpointerOptions) {
+    this.#dockerMode = opts.dockerMode;
+
+    this.registryHost = opts.registryHost ?? "localhost:5000";
+    this.registryNamespace = opts.registryNamespace ?? "trigger";
+    this.registryTlsVerify = opts.registryTlsVerify ?? true;
+
+    this.disableCheckpointSupport = opts.disableCheckpointSupport ?? false;
+    this.checkpointPath = opts.checkpointPath ?? "/checkpoints";
+
+    this.simulateCheckpointFailure = opts.simulateCheckpointFailure ?? false;
+    this.simulateCheckpointFailureSeconds = opts.simulateCheckpointFailureSeconds ?? 300;
+    this.simulatePushFailure = opts.simulatePushFailure ?? false;
+    this.simulatePushFailureSeconds = opts.simulatePushFailureSeconds ?? 300;
+
+    this.chaosMonkey = opts.chaosMonkey ?? new ChaosMonkey(!!process.env.CHAOS_MONKEY_ENABLED);
+  }
+
+  async init(): Promise<CheckpointerInitializeReturn> {
+    if (this.#initialized) {
+      return this.#getInitReturn(this.#canCheckpoint);
+    }
+
+    this.#logger.log(`${this.#dockerMode ? "Docker" : "Kubernetes"} mode`);
+
+    if (this.#dockerMode) {
+      const testCheckpoint = await testDockerCheckpoint();
+
+      if (testCheckpoint.ok) {
+        return this.#getInitReturn(true);
+      }
+
+      this.#logger.error(testCheckpoint.message, testCheckpoint.error ?? "");
+      return this.#getInitReturn(false);
+    } else {
+      try {
+        await $`buildah login --get-login ${this.registryHost}`;
+      } catch (error) {
+        this.#logger.error(`No checkpoint support: Not logged in to registry ${this.registryHost}`);
+        return this.#getInitReturn(false);
+      }
+    }
+
+    return this.#getInitReturn(true);
+  }
+
+  #getInitReturn(canCheckpoint: boolean): CheckpointerInitializeReturn {
+    this.#canCheckpoint = canCheckpoint;
+
+    if (canCheckpoint) {
+      if (!this.#initialized) {
+        this.#logger.log("Full checkpoint support!");
+      }
+    }
+
+    this.#initialized = true;
+
+    const willSimulate = this.#dockerMode && (!this.#canCheckpoint || this.opts.forceSimulate);
+
+    if (willSimulate) {
+      this.#logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", {
+        forceSimulate: this.opts.forceSimulate,
+      });
+    }
+
+    return {
+      canCheckpoint,
+      willSimulate,
+    };
+  }
+
+  #getImageRef(projectRef: string, deploymentVersion: string, shortCode: string) {
+    return `${this.registryHost}/${this.registryNamespace}/${projectRef}:${deploymentVersion}.prod-${shortCode}`;
+  }
+
+  #getExportLocation(projectRef: string, deploymentVersion: string, shortCode: string) {
+    const basename = `${projectRef}-${deploymentVersion}-${shortCode}`;
+
+    if (this.#dockerMode) {
+      return basename;
+    } else {
+      return `${this.checkpointPath}/${basename}.tar`;
+    }
+  }
+
+  async checkpointAndPush(opts: CheckpointAndPushOptions): Promise<CheckpointData | undefined> {
+    const start = performance.now();
+    this.#logger.log(`checkpointAndPush() start`, { start, opts });
+
+    let interval: NodeJS.Timer | undefined;
+
+    if (opts.shouldHeartbeat) {
+      interval = setInterval(() => {
+        this.#logger.log("Sending heartbeat", { runId: opts.runId });
+        this.opts.heartbeat(opts.runId);
+      }, 20_000);
+    }
+
+    try {
+      const result = await this.#checkpointAndPushWithBackoff(opts);
+
+      const end = performance.now();
+      this.#logger.log(`checkpointAndPush() end`, {
+        start,
+        end,
+        diff: end - start,
+        opts,
+        success: result.success,
+      });
+
+      if (!result.success) {
+        return;
+      }
+
+      return result.checkpoint;
+    } finally {
+      if (opts.shouldHeartbeat) {
+        clearInterval(interval);
+      }
+    }
+  }
+
+  isCheckpointing(runId: string) {
+    return this.#abortControllers.has(runId) || this.#waitingForRetry.has(runId);
+  }
+
+  cancelCheckpoint(runId: string): boolean {
+    // If the last checkpoint failed, pretend we canceled it
+    // This ensures tasks don't wait for external resume messages to continue
+    if (this.#hasFailedCheckpoint(runId)) {
+      this.#clearFailedCheckpoint(runId);
+      return true;
+    }
+
+    if (this.#waitingForRetry.has(runId)) {
+      this.#waitingForRetry.delete(runId);
+      return true;
+    }
+
+    const controller = this.#abortControllers.get(runId);
+
+    if (!controller) {
+      this.#logger.debug("Nothing to cancel", { runId });
+      return false;
+    }
+
+    controller.abort("cancelCheckpointing()");
+    this.#abortControllers.delete(runId);
+
+    return true;
+  }
+
+  async #checkpointAndPushWithBackoff({
+    runId,
+    leaveRunning = true, // This mirrors kubernetes behaviour more accurately
+    projectRef,
+    deploymentVersion,
+    attemptNumber,
+  }: CheckpointAndPushOptions): Promise<CheckpointAndPushResult> {
+    this.#logger.log("Checkpointing with backoff", {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+    });
+
+    const backoff = new ExponentialBackoff()
+      .type("EqualJitter")
+      .base(3)
+      .max(3 * 3600)
+      .maxElapsed(48 * 3600);
+
+    for await (const { delay, retry } of backoff) {
+      try {
+        if (retry > 0) {
+          this.#logger.error("Retrying checkpoint", {
+            runId,
+            retry,
+            delay,
+          });
+
+          this.#waitingForRetry.add(runId);
+          await new Promise((resolve) => setTimeout(resolve, delay.milliseconds));
+
+          if (!this.#waitingForRetry.has(runId)) {
+            this.#logger.log("Checkpoint canceled while waiting for retry", { runId });
+            return { success: false, reason: "CANCELED" };
+          } else {
+            this.#waitingForRetry.delete(runId);
+          }
+        }
+
+        const result = await this.#checkpointAndPush({
+          runId,
+          leaveRunning,
+          projectRef,
+          deploymentVersion,
+          attemptNumber,
+        });
+
+        if (result.success) {
+          return result;
+        }
+
+        if (result.reason === "CANCELED") {
+          this.#logger.log("Checkpoint canceled, won't retry", { runId });
+          // Don't fail the checkpoint, as it was canceled
+          return result;
+        }
+
+        if (result.reason === "IN_PROGRESS") {
+          this.#logger.log("Checkpoint already in progress, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        if (result.reason === "NO_SUPPORT") {
+          this.#logger.log("No checkpoint support, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        if (result.reason === "DISABLED") {
+          this.#logger.log("Checkpoint support disabled, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        if (result.reason === "SKIP_RETRYING") {
+          this.#logger.log("Skipping retrying", { runId });
+          return result;
+        }
+
+        continue;
+      } catch (error) {
+        this.#logger.error("Checkpoint error", {
+          retry,
+          runId,
+          delay,
+          error: error instanceof Error ? error.message : error,
+        });
+      }
+    }
+
+    this.#logger.error(`Checkpoint failed after exponential backoff`, {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+    });
+    this.#failCheckpoint(runId, "ERROR");
+
+    return { success: false, reason: "ERROR" };
+  }
+
+  async #checkpointAndPush({
+    runId,
+    leaveRunning = true, // This mirrors kubernetes behaviour more accurately
+    projectRef,
+    deploymentVersion,
+    attemptNumber,
+  }: CheckpointAndPushOptions): Promise<CheckpointAndPushResult> {
+    await this.init();
+
+    const options = {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+      attemptNumber,
+    };
+
+    if (!this.#dockerMode && !this.#canCheckpoint) {
+      this.#logger.error("No checkpoint support. Simulation requires docker.");
+      return { success: false, reason: "NO_SUPPORT" };
+    }
+
+    if (this.isCheckpointing(runId)) {
+      this.#logger.error("Checkpoint procedure already in progress", { options });
+      return { success: false, reason: "IN_PROGRESS" };
+    }
+
+    // This is a new checkpoint, clear any last failure for this run
+    this.#clearFailedCheckpoint(runId);
+
+    if (this.disableCheckpointSupport) {
+      this.#logger.error("Checkpoint support disabled", { options });
+      return { success: false, reason: "DISABLED" };
+    }
+
+    const controller = new AbortController();
+    this.#abortControllers.set(runId, controller);
+
+    const $$ = $({ signal: controller.signal });
+
+    const shortCode = nanoid(8);
+    const imageRef = this.#getImageRef(projectRef, deploymentVersion, shortCode);
+    const exportLocation = this.#getExportLocation(projectRef, deploymentVersion, shortCode);
+
+    const cleanup = async () => {
+      if (this.#dockerMode) {
+        return;
+      }
+
+      try {
+        await $`rm ${exportLocation}`;
+        this.#logger.log("Deleted checkpoint archive", { exportLocation });
+
+        await $`buildah rmi ${imageRef}`;
+        this.#logger.log("Deleted checkpoint image", { imageRef });
+      } catch (error) {
+        this.#logger.error("Failure during checkpoint cleanup", { exportLocation, error });
+      }
+    };
+
+    try {
+      await this.chaosMonkey.call({ $: $$ });
+
+      this.#logger.log("Checkpointing:", { options });
+
+      const containterName = this.#getRunContainerName(runId);
+      const containterNameWithAttempt = this.#getRunContainerName(runId, attemptNumber);
+
+      // Create checkpoint (docker)
+      if (this.#dockerMode) {
+        try {
+          if (this.opts.forceSimulate || !this.#canCheckpoint) {
+            this.#logger.log("Simulating checkpoint");
+            this.#logger.debug(await $$`docker pause ${containterNameWithAttempt}`);
+          } else {
+            if (this.simulateCheckpointFailure) {
+              if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) {
+                this.#logger.error("Simulating checkpoint failure", { options });
+                throw new Error("SIMULATE_CHECKPOINT_FAILURE");
+              }
+            }
+
+            if (leaveRunning) {
+              this.#logger.debug(
+                await $$`docker checkpoint create --leave-running ${containterNameWithAttempt} ${exportLocation}`
+              );
+            } else {
+              this.#logger.debug(
+                await $$`docker checkpoint create ${containterNameWithAttempt} ${exportLocation}`
+              );
+            }
+          }
+        } catch (error) {
+          this.#logger.error("Failed while creating docker checkpoint", { exportLocation });
+          throw error;
+        }
+
+        this.#logger.log("checkpoint created:", {
+          runId,
+          location: exportLocation,
+        });
+
+        return {
+          success: true,
+          checkpoint: {
+            location: exportLocation,
+            docker: true,
+          },
+        };
+      }
+
+      // Create checkpoint (CRI)
+      if (!this.#canCheckpoint) {
+        this.#logger.error("No checkpoint support in kubernetes mode.");
+        return { success: false, reason: "SKIP_RETRYING" };
+      }
+
+      const containerId = this.#logger.debug(
+        // @ts-expect-error
+        await $$`crictl ps`
+          .pipeStdout($$({ stdin: "pipe" })`grep ${containterName}`)
+          .pipeStdout($$({ stdin: "pipe" })`cut -f1 ${"-d "}`)
+      );
+
+      if (!containerId.stdout) {
+        this.#logger.error("could not find container id", { options, containterName });
+        return { success: false, reason: "SKIP_RETRYING" };
+      }
+
+      const start = performance.now();
+
+      if (this.simulateCheckpointFailure) {
+        if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) {
+          this.#logger.error("Simulating checkpoint failure", { options });
+          throw new Error("SIMULATE_CHECKPOINT_FAILURE");
+        }
+      }
+
+      // Create checkpoint
+      this.#logger.debug(await $$`crictl checkpoint --export=${exportLocation} ${containerId}`);
+      const postCheckpoint = performance.now();
+
+      // Print checkpoint size
+      const size = await getParsedFileSize(exportLocation);
+      this.#logger.log("checkpoint archive created", { size, options });
+
+      // Create image from checkpoint
+      const container = this.#logger.debug(await $$`buildah from scratch`);
+      const postFrom = performance.now();
+
+      this.#logger.debug(await $$`buildah add ${container} ${exportLocation} /`);
+      const postAdd = performance.now();
+
+      this.#logger.debug(
+        await $$`buildah config --annotation=io.kubernetes.cri-o.annotations.checkpoint.name=counter ${container}`
+      );
+      const postConfig = performance.now();
+
+      this.#logger.debug(await $$`buildah commit ${container} ${imageRef}`);
+      const postCommit = performance.now();
+
+      this.#logger.debug(await $$`buildah rm ${container}`);
+      const postRm = performance.now();
+
+      if (this.simulatePushFailure) {
+        if (performance.now() < this.simulatePushFailureSeconds * 1000) {
+          this.#logger.error("Simulating push failure", { options });
+          throw new Error("SIMULATE_PUSH_FAILURE");
+        }
+      }
+
+      // Push checkpoint image
+      this.#logger.debug(
+        await $$`buildah push --tls-verify=${String(this.registryTlsVerify)} ${imageRef}`
+      );
+      const postPush = performance.now();
+
+      const perf = {
+        "crictl checkpoint": postCheckpoint - start,
+        "buildah from": postFrom - postCheckpoint,
+        "buildah add": postAdd - postFrom,
+        "buildah config": postConfig - postAdd,
+        "buildah commit": postCommit - postConfig,
+        "buildah rm": postRm - postCommit,
+        "buildah push": postPush - postRm,
+      };
+
+      this.#logger.log("Checkpointed and pushed image to:", { location: imageRef, perf });
+
+      return {
+        success: true,
+        checkpoint: {
+          location: imageRef,
+          docker: false,
+        },
+      };
+    } catch (error) {
+      if (isExecaChildProcess(error)) {
+        if (error.isCanceled) {
+          this.#logger.error("Checkpoint canceled", { options, error });
+
+          return { success: false, reason: "CANCELED" };
+        }
+
+        this.#logger.error("Checkpoint command error", { options, error });
+
+        return { success: false, reason: "ERROR" };
+      }
+
+      this.#logger.error("Unhandled checkpoint error", { options, error });
+
+      return { success: false, reason: "ERROR" };
+    } finally {
+      this.#abortControllers.delete(runId);
+      await cleanup();
+    }
+  }
+
+  #failCheckpoint(runId: string, error: unknown) {
+    this.#failedCheckpoints.set(runId, error);
+  }
+
+  #clearFailedCheckpoint(runId: string) {
+    this.#failedCheckpoints.delete(runId);
+  }
+
+  #hasFailedCheckpoint(runId: string) {
+    return this.#failedCheckpoints.has(runId);
+  }
+
+  #getRunContainerName(suffix: string, attemptNumber?: number) {
+    return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
+  }
+}

apps/coordinator/tsconfig.json

@@ -11,9 +11,7 @@
     "skipLibCheck": true,
     "paths": {
       "@trigger.dev/core/v3": ["../../packages/core/src/v3"],
-      "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"],
-      "@trigger.dev/core-apps": ["../../packages/core-apps/src"],
-      "@trigger.dev/core-apps/*": ["../../packages/core-apps/src/*"]
+      "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"]
     }
   }
 }

apps/docker-provider/.env.example

@@ -4,6 +4,8 @@ PLATFORM_WS_PORT=3030
 PLATFORM_SECRET=provider-secret
 SECURE_CONNECTION=false
 
+OTEL_EXPORTER_OTLP_ENDPOINT=http://0.0.0.0:3030/otel
+
 # Use this if you are on macOS
 # COORDINATOR_HOST="host.docker.internal"
 # OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318"

apps/docker-provider/Containerfile

@@ -1,16 +1,47 @@
-# syntax=docker/dockerfile:labs
-
-FROM node:18-slim AS base
-
-RUN apt-get update \
-  && apt-get install -y dumb-init
-
-FROM base
+FROM node:20-alpine@sha256:7a91aa397f2e2dfbfcdad2e2d72599f374e0b0172be1d86eeb73f1d33f36a4b2 AS node-20-alpine
 
 WORKDIR /app
 
-COPY --chown=node dist/index.mjs /app/
+FROM node-20-alpine AS pruner
+
+COPY --chown=node:node . .
+RUN npx -q turbo@1.10.9 prune --scope=docker-provider --docker
+RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
+
+FROM node-20-alpine AS base
+
+RUN apk add --no-cache dumb-init docker
+
+COPY --chown=node:node .gitignore .gitignore
+COPY --from=pruner --chown=node:node /app/out/json/ .
+COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
+COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
+
+FROM base AS dev-deps
+RUN corepack enable
+ENV NODE_ENV development
+
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
+
+FROM base AS builder
+RUN corepack enable
+
+COPY --from=pruner --chown=node:node /app/out/full/ .
+COPY --from=dev-deps --chown=node:node /app/ .
+COPY --chown=node:node turbo.json turbo.json
+
+RUN pnpm run -r --filter docker-provider build:bundle
+
+FROM base AS runner
+
+RUN corepack enable
+ENV NODE_ENV production
+
+COPY --from=builder --chown=node:node /app/apps/docker-provider/dist/index.mjs ./index.mjs
 
 EXPOSE 8000
 
-ENTRYPOINT [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "/app/index.mjs" ]
+USER node
+
+CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]

apps/docker-provider/package.json

@@ -17,9 +17,7 @@
   "license": "MIT",
   "dependencies": {
     "@trigger.dev/core": "workspace:*",
-    "@trigger.dev/core-apps": "workspace:*",
-    "execa": "^8.0.1",
-    "socket.io-client": "^4.7.4"
+    "execa": "^8.0.1"
   },
   "devDependencies": {
     "@types/node": "^18.19.8",

apps/docker-provider/src/index.ts

@@ -1,87 +1,85 @@
 import { $, type ExecaChildProcess, execa } from "execa";
 import {
-  SimpleLogger,
-  TaskOperations,
   ProviderShell,
-  TaskOperationsRestoreOptions,
+  TaskOperations,
   TaskOperationsCreateOptions,
   TaskOperationsIndexOptions,
-} from "@trigger.dev/core-apps";
+  TaskOperationsRestoreOptions,
+} from "@trigger.dev/core/v3/apps";
+import { SimpleLogger } from "@trigger.dev/core/v3/apps";
+import { isExecaChildProcess, testDockerCheckpoint } from "@trigger.dev/core/v3/apps";
 import { setTimeout } from "node:timers/promises";
 import { PostStartCauses, PreStopCauses } from "@trigger.dev/core/v3";
 
 const MACHINE_NAME = process.env.MACHINE_NAME || "local";
 const COORDINATOR_PORT = process.env.COORDINATOR_PORT || 8020;
 const COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
+const DOCKER_NETWORK = process.env.DOCKER_NETWORK || "host";
+
 const OTEL_EXPORTER_OTLP_ENDPOINT =
   process.env.OTEL_EXPORTER_OTLP_ENDPOINT || "http://0.0.0.0:4318";
 
+const FORCE_CHECKPOINT_SIMULATION = ["1", "true"].includes(
+  process.env.FORCE_CHECKPOINT_SIMULATION ?? "true"
+);
+
 const logger = new SimpleLogger(`[${MACHINE_NAME}]`);
 
-type InitializeReturn = {
+type TaskOperationsInitReturn = {
   canCheckpoint: boolean;
   willSimulate: boolean;
 };
 
-function isExecaChildProcess(maybeExeca: unknown): maybeExeca is Awaited<ExecaChildProcess> {
-  return typeof maybeExeca === "object" && maybeExeca !== null && "escapedCommand" in maybeExeca;
-}
-
 class DockerTaskOperations implements TaskOperations {
   #initialized = false;
   #canCheckpoint = false;
 
   constructor(private opts = { forceSimulate: false }) {}
 
-  async #initialize(): Promise<InitializeReturn> {
+  async init(): Promise<TaskOperationsInitReturn> {
     if (this.#initialized) {
-      return this.#getInitializeReturn();
+      return this.#getInitReturn(this.#canCheckpoint);
     }
 
     logger.log("Initializing task operations");
 
-    if (this.opts.forceSimulate) {
-      logger.log("Forced simulation enabled. Will simulate regardless of checkpoint support.");
+    const testCheckpoint = await testDockerCheckpoint();
+
+    if (testCheckpoint.ok) {
+      return this.#getInitReturn(true);
     }
 
-    try {
-      await $`criu --version`;
-    } catch (error) {
-      logger.error("No checkpoint support: Missing CRIU binary. Will simulate instead.");
-      this.#canCheckpoint = false;
-      this.#initialized = true;
-
-      return this.#getInitializeReturn();
-    }
-
-    try {
-      await $`docker checkpoint`;
-    } catch (error) {
-      logger.error("No checkpoint support: Docker needs to have experimental features enabled");
-      logger.error("Will simulate instead");
-      this.#canCheckpoint = false;
-      this.#initialized = true;
-
-      return this.#getInitializeReturn();
-    }
-
-    logger.log("Full checkpoint support!");
-
-    this.#initialized = true;
-    this.#canCheckpoint = true;
-
-    return this.#getInitializeReturn();
+    logger.error(testCheckpoint.message, testCheckpoint.error);
+    return this.#getInitReturn(false);
   }
 
-  #getInitializeReturn(): InitializeReturn {
+  #getInitReturn(canCheckpoint: boolean): TaskOperationsInitReturn {
+    this.#canCheckpoint = canCheckpoint;
+
+    if (canCheckpoint) {
+      if (!this.#initialized) {
+        logger.log("Full checkpoint support!");
+      }
+    }
+
+    this.#initialized = true;
+
+    const willSimulate = !canCheckpoint || this.opts.forceSimulate;
+
+    if (willSimulate) {
+      logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", {
+        forceSimulate: this.opts.forceSimulate,
+      });
+    }
+
     return {
-      canCheckpoint: this.#canCheckpoint,
-      willSimulate: !this.#canCheckpoint || this.opts.forceSimulate,
+      canCheckpoint,
+      willSimulate,
     };
   }
 
   async index(opts: TaskOperationsIndexOptions) {
-    await this.#initialize();
+    await this.init();
 
     const containerName = this.#getIndexContainerName(opts.shortCode);
 
@@ -90,60 +88,51 @@ class DockerTaskOperations implements TaskOperations {
       port: COORDINATOR_PORT,
     });
 
-    try {
-      logger.debug(
-        await execa("docker", [
-          "run",
-          "--network=host",
-          "--rm",
-          `--env=INDEX_TASKS=true`,
-          `--env=TRIGGER_SECRET_KEY=${opts.apiKey}`,
-          `--env=TRIGGER_API_URL=${opts.apiUrl}`,
-          `--env=TRIGGER_ENV_ID=${opts.envId}`,
-          `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
-          `--env=POD_NAME=${containerName}`,
-          `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
-          `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
-          `--name=${containerName}`,
-          `${opts.imageRef}`,
-        ])
-      );
-    } catch (error: any) {
-      if (!isExecaChildProcess(error)) {
-        throw error;
-      }
-
-      logger.error("Index failed:", {
-        opts,
-        exitCode: error.exitCode,
-        escapedCommand: error.escapedCommand,
-        stdout: error.stdout,
-        stderr: error.stderr,
-      });
-    }
+    logger.debug(
+      await execa("docker", [
+        "run",
+        `--network=${DOCKER_NETWORK}`,
+        "--rm",
+        `--env=INDEX_TASKS=true`,
+        `--env=TRIGGER_SECRET_KEY=${opts.apiKey}`,
+        `--env=TRIGGER_API_URL=${opts.apiUrl}`,
+        `--env=TRIGGER_ENV_ID=${opts.envId}`,
+        `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
+        `--env=POD_NAME=${containerName}`,
+        `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
+        `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
+        `--name=${containerName}`,
+        `${opts.imageRef}`,
+      ])
+    );
   }
 
   async create(opts: TaskOperationsCreateOptions) {
-    await this.#initialize();
+    await this.init();
 
-    const containerName = this.#getRunContainerName(opts.runId);
+    const containerName = this.#getRunContainerName(opts.runId, opts.nextAttemptNumber);
+
+    const runArgs = [
+      "run",
+      `--network=${DOCKER_NETWORK}`,
+      "--detach",
+      `--env=TRIGGER_ENV_ID=${opts.envId}`,
+      `--env=TRIGGER_RUN_ID=${opts.runId}`,
+      `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
+      `--env=POD_NAME=${containerName}`,
+      `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
+      `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
+      `--name=${containerName}`,
+    ];
+
+    if (process.env.ENFORCE_MACHINE_PRESETS) {
+      runArgs.push(`--cpus=${opts.machine.cpu}`, `--memory=${opts.machine.memory}G`);
+    }
+
+    runArgs.push(`${opts.image}`);
 
     try {
-      logger.debug(
-        await execa("docker", [
-          "run",
-          "--network=host",
-          "--detach",
-          `--env=TRIGGER_ENV_ID=${opts.envId}`,
-          `--env=TRIGGER_RUN_ID=${opts.runId}`,
-          `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
-          `--env=POD_NAME=${containerName}`,
-          `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
-          `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
-          `--name=${containerName}`,
-          `${opts.image}`,
-        ])
-      );
+      logger.debug(await execa("docker", runArgs));
     } catch (error) {
       if (!isExecaChildProcess(error)) {
         throw error;
@@ -160,9 +149,9 @@ class DockerTaskOperations implements TaskOperations {
   }
 
   async restore(opts: TaskOperationsRestoreOptions) {
-    await this.#initialize();
+    await this.init();
 
-    const containerName = this.#getRunContainerName(opts.runId);
+    const containerName = this.#getRunContainerName(opts.runId, opts.attemptNumber);
 
     if (!this.#canCheckpoint || this.opts.forceSimulate) {
       logger.log("Simulating restore");
@@ -189,7 +178,7 @@ class DockerTaskOperations implements TaskOperations {
   }
 
   async delete(opts: { runId: string }) {
-    await this.#initialize();
+    await this.init();
 
     const containerName = this.#getRunContainerName(opts.runId);
     await this.#sendPreStop(containerName);
@@ -198,7 +187,7 @@ class DockerTaskOperations implements TaskOperations {
   }
 
   async get(opts: { runId: string }) {
-    await this.#initialize();
+    await this.init();
 
     logger.log("noop: get");
   }
@@ -207,8 +196,8 @@ class DockerTaskOperations implements TaskOperations {
     return `task-index-${suffix}`;
   }
 
-  #getRunContainerName(suffix: string) {
-    return `task-run-${suffix}`;
+  #getRunContainerName(suffix: string, attemptNumber?: number) {
+    return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
   }
 
   async #sendPostStart(containerName: string): Promise<void> {
@@ -257,6 +246,7 @@ class DockerTaskOperations implements TaskOperations {
       return await execa("docker", [
         "exec",
         containerName,
+        "busybox",
         "wget",
         "-q",
         "-O-",
@@ -277,7 +267,7 @@ class DockerTaskOperations implements TaskOperations {
 }
 
 const provider = new ProviderShell({
-  tasks: new DockerTaskOperations({ forceSimulate: true }),
+  tasks: new DockerTaskOperations({ forceSimulate: FORCE_CHECKPOINT_SIMULATION }),
   type: "docker",
 });
 

apps/docker-provider/tsconfig.json

@@ -1,6 +1,6 @@
 {
   "compilerOptions": {
-    "target": "es2016",
+    "target": "es2018",
     "module": "commonjs",
     "esModuleInterop": true,
     "forceConsistentCasingInFileNames": true,
@@ -9,9 +9,7 @@
     "skipLibCheck": true,
     "paths": {
       "@trigger.dev/core/v3": ["../../packages/core/src/v3"],
-      "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"],
-      "@trigger.dev/core-apps": ["../../packages/core-apps/src"],
-      "@trigger.dev/core-apps/*": ["../../packages/core-apps/src/*"]
+      "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"]
     }
   }
 }

apps/kubernetes-provider/Containerfile

@@ -1,14 +1,14 @@
-FROM node:18-alpine@sha256:ca9f6cb0466f9638e59e0c249d335a07c867cd50c429b5c7830dda1bed584649 AS node-18-alpine
+FROM node:20-alpine@sha256:7a91aa397f2e2dfbfcdad2e2d72599f374e0b0172be1d86eeb73f1d33f36a4b2 AS node-20-alpine
 
 WORKDIR /app
 
-FROM node-18-alpine AS pruner
+FROM node-20-alpine AS pruner
 
 COPY --chown=node:node . .
 RUN npx -q turbo@1.10.9 prune --scope=kubernetes-provider --docker
 RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
 
-FROM node-18-alpine AS base
+FROM node-20-alpine AS base
 
 RUN apk add --no-cache dumb-init
 

apps/kubernetes-provider/package.json

@@ -18,8 +18,7 @@
   "dependencies": {
     "@kubernetes/client-node": "^0.20.0",
     "@trigger.dev/core": "workspace:*",
-    "@trigger.dev/core-apps": "workspace:*",
-    "socket.io-client": "^4.7.4"
+    "p-queue": "^8.0.1"
   },
   "devDependencies": {
     "dotenv": "^16.4.2",

apps/kubernetes-provider/src/index.ts

@@ -1,20 +1,36 @@
 import * as k8s from "@kubernetes/client-node";
 import {
   ProviderShell,
-  SimpleLogger,
   TaskOperations,
   TaskOperationsCreateOptions,
   TaskOperationsIndexOptions,
+  TaskOperationsPrePullDeploymentOptions,
   TaskOperationsRestoreOptions,
-} from "@trigger.dev/core-apps";
-import { Machine, PostStartCauses, PreStopCauses, EnvironmentType } from "@trigger.dev/core/v3";
-import { randomUUID } from "crypto";
+} from "@trigger.dev/core/v3/apps";
+import { SimpleLogger } from "@trigger.dev/core/v3/apps";
+import {
+  MachinePreset,
+  PostStartCauses,
+  PreStopCauses,
+  EnvironmentType,
+} from "@trigger.dev/core/v3";
+import { TaskMonitor } from "./taskMonitor";
+import { PodCleaner } from "./podCleaner";
+import { UptimeHeartbeat } from "./uptimeHeartbeat";
 
 const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
 const NODE_NAME = process.env.NODE_NAME || "local";
 const OTEL_EXPORTER_OTLP_ENDPOINT =
   process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318";
 
+const POD_CLEANER_INTERVAL_SECONDS = Number(process.env.POD_CLEANER_INTERVAL_SECONDS || "300");
+
+const UPTIME_HEARTBEAT_URL = process.env.UPTIME_HEARTBEAT_URL;
+const UPTIME_INTERVAL_SECONDS = Number(process.env.UPTIME_INTERVAL_SECONDS || "60");
+const UPTIME_MAX_PENDING_RUNS = Number(process.env.UPTIME_MAX_PENDING_RUNS || "25");
+const UPTIME_MAX_PENDING_INDECES = Number(process.env.UPTIME_MAX_PENDING_INDECES || "10");
+const UPTIME_MAX_PENDING_ERRORS = Number(process.env.UPTIME_MAX_PENDING_ERRORS || "10");
+
 const logger = new SimpleLogger(`[${NODE_NAME}]`);
 logger.log(`running in ${RUNTIME_ENV} mode`);
 
@@ -24,11 +40,16 @@ type Namespace = {
   };
 };
 
+type ResourceQuantities = {
+  [K in "cpu" | "memory" | "ephemeral-storage"]?: string;
+};
+
 class KubernetesTaskOperations implements TaskOperations {
   #namespace: Namespace;
   #k8sApi: {
     core: k8s.CoreV1Api;
     batch: k8s.BatchV1Api;
+    apps: k8s.AppsV1Api;
   };
 
   constructor(namespace = "default") {
@@ -41,6 +62,10 @@ class KubernetesTaskOperations implements TaskOperations {
     this.#k8sApi = this.#createK8sApi();
   }
 
+  async init() {
+    // noop
+  }
+
   async index(opts: TaskOperationsIndexOptions) {
     await this.#createJob(
       {
@@ -55,25 +80,15 @@ class KubernetesTaskOperations implements TaskOperations {
           template: {
             metadata: {
               labels: {
+                ...this.#getSharedLabels(opts),
                 app: "task-index",
                 "app.kubernetes.io/part-of": "trigger-worker",
                 "app.kubernetes.io/component": "index",
-                env: opts.envId,
-                envtype: this.#envTypeToLabelValue(opts.envType),
-                org: opts.orgId,
-                project: opts.projectId,
+                deployment: opts.deploymentId,
               },
             },
             spec: {
-              restartPolicy: "Never",
-              imagePullSecrets: [
-                {
-                  name: "registry-trigger",
-                },
-              ],
-              nodeSelector: {
-                nodetype: "worker",
-              },
+              ...this.#defaultPodSpec,
               containers: [
                 {
                   name: this.#getIndexContainerName(opts.shortCode),
@@ -83,12 +98,13 @@ class KubernetesTaskOperations implements TaskOperations {
                       containerPort: 8000,
                     },
                   ],
-                  // resources: {
-                  //   limits: {
-                  //     cpu: "100m",
-                  //     memory: "50Mi",
-                  //   },
-                  // },
+                  resources: {
+                    limits: {
+                      cpu: "1",
+                      memory: "2G",
+                      "ephemeral-storage": "2Gi",
+                    },
+                  },
                   lifecycle: {
                     preStop: {
                       exec: {
@@ -97,10 +113,7 @@ class KubernetesTaskOperations implements TaskOperations {
                     },
                   },
                   env: [
-                    {
-                      name: "DEBUG",
-                      value: "true",
-                    },
+                    ...this.#getSharedEnv(opts.envId),
                     {
                       name: "INDEX_TASKS",
                       value: "true",
@@ -113,42 +126,6 @@ class KubernetesTaskOperations implements TaskOperations {
                       name: "TRIGGER_API_URL",
                       value: opts.apiUrl,
                     },
-                    {
-                      name: "TRIGGER_ENV_ID",
-                      value: opts.envId,
-                    },
-                    {
-                      name: "OTEL_EXPORTER_OTLP_ENDPOINT",
-                      value: OTEL_EXPORTER_OTLP_ENDPOINT,
-                    },
-                    {
-                      name: "HTTP_SERVER_PORT",
-                      value: "8000",
-                    },
-                    {
-                      name: "POD_NAME",
-                      valueFrom: {
-                        fieldRef: {
-                          fieldPath: "metadata.name",
-                        },
-                      },
-                    },
-                    {
-                      name: "COORDINATOR_HOST",
-                      valueFrom: {
-                        fieldRef: {
-                          fieldPath: "status.hostIP",
-                        },
-                      },
-                    },
-                    {
-                      name: "MACHINE_NAME",
-                      valueFrom: {
-                        fieldRef: {
-                          fieldPath: "spec.nodeName",
-                        },
-                      },
-                    },
                   ],
                 },
               ],
@@ -161,50 +138,35 @@ class KubernetesTaskOperations implements TaskOperations {
   }
 
   async create(opts: TaskOperationsCreateOptions) {
+    const containerName = this.#getRunContainerName(opts.runId, opts.nextAttemptNumber);
+
     await this.#createPod(
       {
         metadata: {
-          name: this.#getRunContainerName(opts.runId),
+          name: containerName,
           namespace: this.#namespace.metadata.name,
           labels: {
+            ...this.#getSharedLabels(opts),
             app: "task-run",
             "app.kubernetes.io/part-of": "trigger-worker",
             "app.kubernetes.io/component": "create",
-            env: opts.envId,
-            envtype: this.#envTypeToLabelValue(opts.envType),
-            org: opts.orgId,
-            project: opts.projectId,
             run: opts.runId,
           },
         },
         spec: {
-          restartPolicy: "Never",
-          imagePullSecrets: [
-            {
-              name: "registry-trigger",
-            },
-          ],
-          nodeSelector: {
-            nodetype: "worker",
-          },
+          ...this.#defaultPodSpec,
+          terminationGracePeriodSeconds: 60 * 60,
           containers: [
             {
-              name: this.#getRunContainerName(opts.runId),
+              name: containerName,
               image: opts.image,
               ports: [
                 {
                   containerPort: 8000,
                 },
               ],
-              resources: {
-                limits: this.#getResourcesFromMachineConfig(opts.machine),
-              },
+              resources: this.#getResourcesForMachine(opts.machine),
               lifecycle: {
-                postStart: {
-                  exec: {
-                    command: this.#getLifecycleCommand("postStart", "create"),
-                  },
-                },
                 preStop: {
                   exec: {
                     command: this.#getLifecycleCommand("preStop", "terminate"),
@@ -212,54 +174,11 @@ class KubernetesTaskOperations implements TaskOperations {
                 },
               },
               env: [
-                {
-                  name: "DEBUG",
-                  value: "true",
-                },
-                {
-                  name: "HTTP_SERVER_PORT",
-                  value: "8000",
-                },
-                {
-                  name: "TRIGGER_ENV_ID",
-                  value: opts.envId,
-                },
+                ...this.#getSharedEnv(opts.envId),
                 {
                   name: "TRIGGER_RUN_ID",
                   value: opts.runId,
                 },
-                {
-                  name: "TRIGGER_WORKER_VERSION",
-                  value: opts.version,
-                },
-                {
-                  name: "OTEL_EXPORTER_OTLP_ENDPOINT",
-                  value: OTEL_EXPORTER_OTLP_ENDPOINT,
-                },
-                {
-                  name: "POD_NAME",
-                  valueFrom: {
-                    fieldRef: {
-                      fieldPath: "metadata.name",
-                    },
-                  },
-                },
-                {
-                  name: "COORDINATOR_HOST",
-                  valueFrom: {
-                    fieldRef: {
-                      fieldPath: "status.hostIP",
-                    },
-                  },
-                },
-                {
-                  name: "NODE_NAME",
-                  valueFrom: {
-                    fieldRef: {
-                      fieldPath: "spec.nodeName",
-                    },
-                  },
-                },
               ],
               volumeMounts: [
                 {
@@ -285,30 +204,19 @@ class KubernetesTaskOperations implements TaskOperations {
     await this.#createPod(
       {
         metadata: {
-          name: `${this.#getRunContainerName(opts.runId)}-${randomUUID().slice(0, 8)}`,
+          name: `${this.#getRunContainerName(opts.runId)}-${opts.checkpointId.slice(-8)}`,
           namespace: this.#namespace.metadata.name,
           labels: {
+            ...this.#getSharedLabels(opts),
             app: "task-run",
             "app.kubernetes.io/part-of": "trigger-worker",
             "app.kubernetes.io/component": "restore",
-            env: opts.envId,
-            envtype: this.#envTypeToLabelValue(opts.envType),
-            org: opts.orgId,
-            project: opts.projectId,
             run: opts.runId,
             checkpoint: opts.checkpointId,
           },
         },
         spec: {
-          restartPolicy: "Never",
-          imagePullSecrets: [
-            {
-              name: "registry-trigger",
-            },
-          ],
-          nodeSelector: {
-            nodetype: "worker",
-          },
+          ...this.#defaultPodSpec,
           initContainers: [
             {
               name: "pull-base-image",
@@ -317,7 +225,8 @@ class KubernetesTaskOperations implements TaskOperations {
             },
             {
               name: "populate-taskinfo",
-              image: "busybox",
+              image: "registry.digitalocean.com/trigger/busybox",
+              imagePullPolicy: "IfNotPresent",
               command: ["/bin/sh", "-c"],
               args: ["printenv COORDINATOR_HOST | tee /etc/taskinfo/coordinator-host"],
               env: [
@@ -347,9 +256,7 @@ class KubernetesTaskOperations implements TaskOperations {
                   containerPort: 8000,
                 },
               ],
-              resources: {
-                limits: this.#getResourcesFromMachineConfig(opts.machine),
-              },
+              resources: this.#getResourcesForMachine(opts.machine),
               lifecycle: {
                 postStart: {
                   exec: {
@@ -393,6 +300,72 @@ class KubernetesTaskOperations implements TaskOperations {
     await this.#getPod(opts.runId, this.#namespace);
   }
 
+  async prePullDeployment(opts: TaskOperationsPrePullDeploymentOptions) {
+    const metaName = this.#getPrePullContainerName(opts.shortCode);
+
+    const metaLabels = {
+      ...this.#getSharedLabels(opts),
+      app: "task-prepull",
+      "app.kubernetes.io/part-of": "trigger-worker",
+      "app.kubernetes.io/component": "prepull",
+      deployment: opts.deploymentId,
+      name: metaName,
+    } satisfies k8s.V1ObjectMeta["labels"];
+
+    await this.#createDaemonSet(
+      {
+        metadata: {
+          name: metaName,
+          namespace: this.#namespace.metadata.name,
+          labels: metaLabels,
+        },
+        spec: {
+          selector: {
+            matchLabels: {
+              name: metaName,
+            },
+          },
+          template: {
+            metadata: {
+              labels: metaLabels,
+            },
+            spec: {
+              ...this.#defaultPodSpec,
+              restartPolicy: "Always",
+              initContainers: [
+                {
+                  name: "prepull",
+                  image: opts.imageRef,
+                  command: ["/usr/bin/true"],
+                  resources: {
+                    limits: {
+                      cpu: "0.25",
+                      memory: "100Mi",
+                      "ephemeral-storage": "1Gi",
+                    },
+                  },
+                },
+              ],
+              containers: [
+                {
+                  name: "pause",
+                  image: "registry.k8s.io/pause:3.9",
+                  resources: {
+                    limits: {
+                      cpu: "1m",
+                      memory: "12Mi",
+                    },
+                  },
+                },
+              ],
+            },
+          },
+        },
+      },
+      this.#namespace
+    );
+  }
+
   #envTypeToLabelValue(type: EnvironmentType) {
     switch (type) {
       case "PRODUCTION":
@@ -406,10 +379,120 @@ class KubernetesTaskOperations implements TaskOperations {
     }
   }
 
-  #getResourcesFromMachineConfig(config: Machine) {
+  get #defaultPodSpec(): Omit<k8s.V1PodSpec, "containers"> {
     return {
-      cpu: `${config.cpu}`,
-      memory: `${config.memory}G`,
+      restartPolicy: "Never",
+      automountServiceAccountToken: false,
+      imagePullSecrets: [
+        {
+          name: "registry-trigger",
+        },
+        {
+          name: "registry-trigger-failover",
+        },
+      ],
+      nodeSelector: {
+        nodetype: "worker",
+      },
+    };
+  }
+
+  get #defaultResourceRequests(): ResourceQuantities {
+    return {
+      "ephemeral-storage": "2Gi",
+    };
+  }
+
+  get #defaultResourceLimits(): ResourceQuantities {
+    return {
+      "ephemeral-storage": "10Gi",
+    };
+  }
+
+  #getSharedEnv(envId: string): k8s.V1EnvVar[] {
+    return [
+      {
+        name: "TRIGGER_ENV_ID",
+        value: envId,
+      },
+      {
+        name: "DEBUG",
+        value: process.env.DEBUG ? "1" : "0",
+      },
+      {
+        name: "HTTP_SERVER_PORT",
+        value: "8000",
+      },
+      {
+        name: "OTEL_EXPORTER_OTLP_ENDPOINT",
+        value: OTEL_EXPORTER_OTLP_ENDPOINT,
+      },
+      {
+        name: "POD_NAME",
+        valueFrom: {
+          fieldRef: {
+            fieldPath: "metadata.name",
+          },
+        },
+      },
+      {
+        name: "COORDINATOR_HOST",
+        valueFrom: {
+          fieldRef: {
+            fieldPath: "status.hostIP",
+          },
+        },
+      },
+      {
+        name: "MACHINE_NAME",
+        valueFrom: {
+          fieldRef: {
+            fieldPath: "spec.nodeName",
+          },
+        },
+      },
+    ];
+  }
+
+  #getSharedLabels(
+    opts:
+      | TaskOperationsIndexOptions
+      | TaskOperationsCreateOptions
+      | TaskOperationsRestoreOptions
+      | TaskOperationsPrePullDeploymentOptions
+  ): Record<string, string> {
+    return {
+      env: opts.envId,
+      envtype: this.#envTypeToLabelValue(opts.envType),
+      org: opts.orgId,
+      project: opts.projectId,
+    };
+  }
+
+  #getResourceRequestsForMachine(preset: MachinePreset): ResourceQuantities {
+    return {
+      cpu: `${preset.cpu * 0.75}`,
+      memory: `${preset.memory}G`,
+    };
+  }
+
+  #getResourceLimitsForMachine(preset: MachinePreset): ResourceQuantities {
+    return {
+      cpu: `${preset.cpu}`,
+      memory: `${preset.memory}G`,
+    };
+  }
+
+  #getResourcesForMachine(preset: MachinePreset): k8s.V1ResourceRequirements {
+    return {
+      requests: {
+        ...this.#defaultResourceRequests,
+        ...this.#getResourceRequestsForMachine(preset),
+      },
+      limits: {
+        ...this.#defaultResourceLimits,
+        ...this.#getResourceLimitsForMachine(preset),
+      },
     };
   }
 
@@ -417,15 +500,31 @@ class KubernetesTaskOperations implements TaskOperations {
     type: THookType,
     cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses
   ) {
-    return ["/bin/sh", "-c", `sleep 1; wget -q -O- 127.0.0.1:8000/${type}?cause=${cause}`];
+    const retries = 5;
+
+    // This will retry sending the lifecycle hook up to `retries` times
+    // The sleep is required as this may start running before the HTTP server is up
+    const exec = [
+      "/bin/sh",
+      "-c",
+      `for i in $(seq ${retries}); do sleep 1; busybox wget -q -O- 127.0.0.1:8000/${type}?cause=${cause} && break; done`,
+    ];
+
+    logger.debug("getLifecycleCommand()", { exec });
+
+    return exec;
   }
 
   #getIndexContainerName(suffix: string) {
     return `task-index-${suffix}`;
   }
 
-  #getRunContainerName(suffix: string) {
-    return `task-run-${suffix}`;
+  #getRunContainerName(suffix: string, attemptNumber?: number) {
+    return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
+  }
+
+  #getPrePullContainerName(suffix: string) {
+    return `task-prepull-${suffix}`;
   }
 
   #createK8sApi() {
@@ -442,6 +541,7 @@ class KubernetesTaskOperations implements TaskOperations {
     return {
       core: kubeConfig.makeApiClient(k8s.CoreV1Api),
       batch: kubeConfig.makeApiClient(k8s.BatchV1Api),
+      apps: kubeConfig.makeApiClient(k8s.AppsV1Api),
     };
   }
 
@@ -485,6 +585,18 @@ class KubernetesTaskOperations implements TaskOperations {
     }
   }
 
+  async #createDaemonSet(daemonSet: k8s.V1DaemonSet, namespace: Namespace) {
+    try {
+      const res = await this.#k8sApi.apps.createNamespacedDaemonSet(
+        namespace.metadata.name,
+        daemonSet
+      );
+      logger.debug(res.body);
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
+    }
+  }
+
   #throwUnlessRecord(candidate: unknown): asserts candidate is Record<string, unknown> {
     if (typeof candidate !== "object" || candidate === null) {
       throw candidate;
@@ -516,3 +628,59 @@ const provider = new ProviderShell({
 });
 
 provider.listen();
+
+const taskMonitor = new TaskMonitor({
+  runtimeEnv: RUNTIME_ENV,
+  onIndexFailure: async (deploymentId, details) => {
+    logger.log("Indexing failed", { deploymentId, details });
+
+    try {
+      provider.platformSocket.send("INDEXING_FAILED", {
+        deploymentId,
+        error: {
+          name: `Crashed with exit code ${details.exitCode}`,
+          message: details.reason,
+          stack: details.logs,
+        },
+        overrideCompletion: details.overrideCompletion,
+      });
+    } catch (error) {
+      logger.error(error);
+    }
+  },
+  onRunFailure: async (runId, details) => {
+    logger.log("Run failed:", { runId, details });
+
+    try {
+      provider.platformSocket.send("WORKER_CRASHED", { runId, ...details });
+    } catch (error) {
+      logger.error(error);
+    }
+  },
+});
+
+taskMonitor.start();
+
+const podCleaner = new PodCleaner({
+  runtimeEnv: RUNTIME_ENV,
+  namespace: "default",
+  intervalInSeconds: POD_CLEANER_INTERVAL_SECONDS,
+});
+
+podCleaner.start();
+
+if (UPTIME_HEARTBEAT_URL) {
+  const uptimeHeartbeat = new UptimeHeartbeat({
+    runtimeEnv: RUNTIME_ENV,
+    namespace: "default",
+    intervalInSeconds: UPTIME_INTERVAL_SECONDS,
+    pingUrl: UPTIME_HEARTBEAT_URL,
+    maxPendingRuns: UPTIME_MAX_PENDING_RUNS,
+    maxPendingIndeces: UPTIME_MAX_PENDING_INDECES,
+    maxPendingErrors: UPTIME_MAX_PENDING_ERRORS,
+  });
+
+  uptimeHeartbeat.start();
+} else {
+  logger.log("Uptime heartbeat is disabled, set UPTIME_HEARTBEAT_URL to enable.");
+}

apps/kubernetes-provider/src/podCleaner.ts

@@ -0,0 +1,323 @@
+import * as k8s from "@kubernetes/client-node";
+import { SimpleLogger } from "@trigger.dev/core/v3/apps";
+
+type PodCleanerOptions = {
+  runtimeEnv: "local" | "kubernetes";
+  namespace?: string;
+  intervalInSeconds?: number;
+};
+
+export class PodCleaner {
+  private enabled = false;
+  private namespace = "default";
+  private intervalInSeconds = 300;
+
+  private logger = new SimpleLogger("[PodCleaner]");
+  private k8sClient: {
+    core: k8s.CoreV1Api;
+    apps: k8s.AppsV1Api;
+    kubeConfig: k8s.KubeConfig;
+  };
+
+  constructor(private opts: PodCleanerOptions) {
+    if (opts.namespace) {
+      this.namespace = opts.namespace;
+    }
+
+    if (opts.intervalInSeconds) {
+      this.intervalInSeconds = opts.intervalInSeconds;
+    }
+
+    this.k8sClient = this.#createK8sClient();
+  }
+
+  #createK8sClient() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (this.opts.runtimeEnv === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (this.opts.runtimeEnv === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      apps: kubeConfig.makeApiClient(k8s.AppsV1Api),
+      kubeConfig: kubeConfig,
+    };
+  }
+
+  #isRecord(candidate: unknown): candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  #logK8sError(err: unknown, debugOnly = false) {
+    if (debugOnly) {
+      this.logger.debug("K8s API Error", err);
+    } else {
+      this.logger.error("K8s API Error", err);
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
+      this.#logK8sError(err);
+      return;
+    }
+
+    this.#logK8sError(err, true);
+
+    if (typeof err.body.message === "string") {
+      this.#logK8sError({ message: err.body.message });
+      return;
+    }
+
+    this.#logK8sError({ body: err.body });
+  }
+
+  async #deletePods(opts: {
+    namespace: string;
+    dryRun?: boolean;
+    fieldSelector?: string;
+    labelSelector?: string;
+  }) {
+    return await this.k8sClient.core
+      .deleteCollectionNamespacedPod(
+        opts.namespace,
+        undefined, // pretty
+        undefined, // continue
+        opts.dryRun ? "All" : undefined,
+        opts.fieldSelector,
+        undefined, // gracePeriodSeconds
+        opts.labelSelector
+      )
+      .catch(this.#handleK8sError.bind(this));
+  }
+
+  async #deleteDaemonSets(opts: {
+    namespace: string;
+    dryRun?: boolean;
+    fieldSelector?: string;
+    labelSelector?: string;
+  }) {
+    return await this.k8sClient.apps
+      .deleteCollectionNamespacedDaemonSet(
+        opts.namespace,
+        undefined, // pretty
+        undefined, // continue
+        opts.dryRun ? "All" : undefined,
+        opts.fieldSelector,
+        undefined, // gracePeriodSeconds
+        opts.labelSelector
+      )
+      .catch(this.#handleK8sError.bind(this));
+  }
+
+  async #deleteCompletedRuns() {
+    this.logger.log("Deleting completed runs");
+
+    const start = Date.now();
+
+    const result = await this.#deletePods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=Succeeded",
+      labelSelector: "app=task-run",
+    });
+
+    const elapsedMs = Date.now() - start;
+
+    if (!result) {
+      this.logger.log("Deleting completed runs: No delete result", { elapsedMs });
+      return;
+    }
+
+    const total = (result.response as any)?.body?.items?.length ?? 0;
+
+    this.logger.log("Deleting completed runs: Done", { total, elapsedMs });
+  }
+
+  async #deleteFailedRuns() {
+    this.logger.log("Deleting failed runs");
+
+    const start = Date.now();
+
+    const result = await this.#deletePods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=Failed",
+      labelSelector: "app=task-run",
+    });
+
+    const elapsedMs = Date.now() - start;
+
+    if (!result) {
+      this.logger.log("Deleting failed runs: No delete result", { elapsedMs });
+      return;
+    }
+
+    const total = (result.response as any)?.body?.items?.length ?? 0;
+
+    this.logger.log("Deleting failed runs: Done", { total, elapsedMs });
+  }
+
+  async #deleteUnrecoverableRuns() {
+    await this.#deletePods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=?",
+      labelSelector: "app=task-run",
+    });
+  }
+
+  async #deleteCompletedPrePulls() {
+    this.logger.log("Deleting completed pre-pulls");
+
+    const start = Date.now();
+
+    const result = await this.#deleteDaemonSets({
+      namespace: this.namespace,
+      labelSelector: "app=task-prepull",
+    });
+
+    const elapsedMs = Date.now() - start;
+
+    if (!result) {
+      this.logger.log("Deleting completed pre-pulls: No delete result", { elapsedMs });
+      return;
+    }
+
+    const total = (result.response as any)?.body?.items?.length ?? 0;
+
+    this.logger.log("Deleting completed pre-pulls: Done", { total, elapsedMs });
+  }
+
+  async start() {
+    this.enabled = true;
+    this.logger.log("Starting");
+
+    const completedInterval = setInterval(async () => {
+      if (!this.enabled) {
+        clearInterval(completedInterval);
+        return;
+      }
+
+      try {
+        await this.#deleteCompletedRuns();
+      } catch (error) {
+        this.logger.error("Error deleting completed runs", error);
+      }
+    }, this.intervalInSeconds * 1000);
+
+    const failedInterval = setInterval(
+      async () => {
+        if (!this.enabled) {
+          clearInterval(failedInterval);
+          return;
+        }
+
+        try {
+          await this.#deleteFailedRuns();
+        } catch (error) {
+          this.logger.error("Error deleting completed runs", error);
+        }
+      },
+      // Use a longer interval for failed runs. This is only a backup in case the task monitor fails.
+      2 * this.intervalInSeconds * 1000
+    );
+
+    const completedPrePullInterval = setInterval(
+      async () => {
+        if (!this.enabled) {
+          clearInterval(completedPrePullInterval);
+          return;
+        }
+
+        try {
+          await this.#deleteCompletedPrePulls();
+        } catch (error) {
+          this.logger.error("Error deleting completed pre-pulls", error);
+        }
+      },
+      2 * this.intervalInSeconds * 1000
+    );
+
+    // this.#launchTests();
+  }
+
+  async stop() {
+    if (!this.enabled) {
+      return;
+    }
+
+    this.enabled = false;
+    this.logger.log("Shutting down..");
+  }
+
+  async #launchTests() {
+    const createPod = async (
+      container: k8s.V1Container,
+      name: string,
+      labels?: Record<string, string>
+    ) => {
+      this.logger.log("Creating pod:", name);
+
+      const pod = {
+        metadata: {
+          name,
+          labels,
+        },
+        spec: {
+          restartPolicy: "Never",
+          automountServiceAccountToken: false,
+          terminationGracePeriodSeconds: 1,
+          containers: [container],
+        },
+      } satisfies k8s.V1Pod;
+
+      await this.k8sClient.core
+        .createNamespacedPod(this.namespace, pod)
+        .catch(this.#handleK8sError.bind(this));
+    };
+
+    const createIdlePod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: ["-c", "sleep infinity"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createCompletedPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: ["-c", "true"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createFailedPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: ["-c", "false"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    await createIdlePod("test-idle-1", { app: "task-run" });
+    await createFailedPod("test-failed-1", { app: "task-run" });
+    await createCompletedPod("test-completed-1", { app: "task-run" });
+  }
+}

apps/kubernetes-provider/src/taskMonitor.ts

@@ -0,0 +1,450 @@
+import * as k8s from "@kubernetes/client-node";
+import { SimpleLogger } from "@trigger.dev/core/v3/apps";
+import { EXIT_CODE_ALREADY_HANDLED, EXIT_CODE_CHILD_NONZERO } from "@trigger.dev/core/v3/apps";
+import { setTimeout } from "timers/promises";
+import PQueue from "p-queue";
+import type { Prettify } from "@trigger.dev/core/v3";
+
+type FailureDetails = Prettify<{
+  exitCode: number;
+  reason: string;
+  logs: string;
+  overrideCompletion: boolean;
+}>;
+
+type IndexFailureHandler = (deploymentId: string, details: FailureDetails) => Promise<any>;
+
+type RunFailureHandler = (runId: string, details: FailureDetails) => Promise<any>;
+
+type TaskMonitorOptions = {
+  runtimeEnv: "local" | "kubernetes";
+  onIndexFailure?: IndexFailureHandler;
+  onRunFailure?: RunFailureHandler;
+  namespace?: string;
+};
+
+export class TaskMonitor {
+  #enabled = false;
+
+  #logger = new SimpleLogger("[TaskMonitor]");
+  #taskInformer: ReturnType<typeof k8s.makeInformer<k8s.V1Pod>>;
+  #processedPods = new Map<string, number>();
+  #queue = new PQueue({ concurrency: 10 });
+
+  #k8sClient: {
+    core: k8s.CoreV1Api;
+    kubeConfig: k8s.KubeConfig;
+  };
+
+  private namespace = "default";
+  private fieldSelector = "status.phase=Failed";
+  private labelSelector = "app in (task-index, task-run)";
+
+  constructor(private opts: TaskMonitorOptions) {
+    if (opts.namespace) {
+      this.namespace = opts.namespace;
+    }
+
+    this.#k8sClient = this.#createK8sClient();
+
+    this.#taskInformer = this.#createTaskInformer();
+    this.#taskInformer.on("connect", this.#onInformerConnected.bind(this));
+    this.#taskInformer.on("error", this.#onInformerError.bind(this));
+    this.#taskInformer.on("update", this.#enqueueOnPodUpdated.bind(this));
+  }
+
+  #createTaskInformer() {
+    const listTasks = () =>
+      this.#k8sClient.core.listNamespacedPod(
+        this.namespace,
+        undefined,
+        undefined,
+        undefined,
+        this.fieldSelector,
+        this.labelSelector
+      );
+
+    // Uses watch with local caching
+    // https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes
+    const informer = k8s.makeInformer(
+      this.#k8sClient.kubeConfig,
+      `/api/v1/namespaces/${this.namespace}/pods`,
+      listTasks,
+      this.labelSelector,
+      this.fieldSelector
+    );
+
+    return informer;
+  }
+
+  async #onInformerConnected() {
+    this.#logger.log("Connected");
+  }
+
+  async #onInformerError(error: any) {
+    this.#logger.error("Error:", error);
+
+    // Automatic reconnect
+    await setTimeout(2_000);
+    this.#taskInformer.start();
+  }
+
+  #enqueueOnPodUpdated(pod: k8s.V1Pod) {
+    this.#queue.add(async () => {
+      try {
+        // It would be better to only pass the cache key, but the pod may already be removed from the cache by the time we process it
+        await this.#onPodUpdated(pod);
+      } catch (error) {
+        this.#logger.error("Caught onPodUpdated() error:", error);
+      }
+    });
+  }
+
+  async #onPodUpdated(pod: k8s.V1Pod) {
+    this.#logger.debug(`Updated: ${pod.metadata?.name}`);
+    this.#logger.debug("Updated", JSON.stringify(pod, null, 2));
+
+    // We only care about failures
+    if (pod.status?.phase !== "Failed") {
+      return;
+    }
+
+    const podName = pod.metadata?.name;
+
+    if (!podName) {
+      this.#logger.error("Pod is nameless", { pod });
+      return;
+    }
+
+    const containerStatus = pod.status.containerStatuses?.[0];
+
+    if (!containerStatus?.state) {
+      this.#logger.error("Pod failed, but container status doesn't have state", {
+        status: pod.status,
+      });
+      return;
+    }
+
+    if (this.#processedPods.has(podName)) {
+      this.#logger.debug("Pod update already processed", {
+        podName,
+        timestamp: this.#processedPods.get(podName),
+      });
+      return;
+    }
+
+    this.#processedPods.set(podName, Date.now());
+
+    const podStatus = this.#getPodStatusSummary(pod.status);
+    const containerState = this.#getContainerStateSummary(containerStatus.state);
+    const exitCode = containerState.exitCode ?? -1;
+
+    if (exitCode === EXIT_CODE_ALREADY_HANDLED) {
+      this.#logger.debug("Ignoring pod failure, already handled by worker", {
+        podName,
+      });
+      return;
+    }
+
+    const rawLogs = await this.#getLogTail(podName);
+
+    this.#logger.log(`${podName} failed with:`, {
+      podStatus,
+      containerState,
+      rawLogs,
+    });
+
+    const rawReason = podStatus.reason ?? containerState.reason ?? "";
+    const message = podStatus.message ?? containerState.message ?? "";
+
+    let reason = rawReason || "Unknown error";
+    let logs = rawLogs || "";
+    let overrideCompletion = false;
+
+    switch (rawReason) {
+      case "Error":
+        reason = "Unknown error.";
+        break;
+      case "Evicted":
+        if (message.startsWith("Pod ephemeral local storage usage")) {
+          reason = "Storage limit exceeded.";
+        } else if (message) {
+          reason = `Evicted: ${message}`;
+        } else {
+          reason = "Evicted for unknown reason.";
+        }
+
+        if (logs.startsWith("failed to try resolving symlinks")) {
+          logs = "";
+        }
+        break;
+      case "OOMKilled":
+        overrideCompletion = true;
+        reason = `${
+          exitCode === EXIT_CODE_CHILD_NONZERO ? "Child process" : "Parent process"
+        } ran out of memory! Try choosing a machine preset with more memory for this task.`;
+        break;
+      default:
+        break;
+    }
+
+    const failureInfo = {
+      exitCode,
+      reason,
+      logs,
+      overrideCompletion,
+    } satisfies FailureDetails;
+
+    const app = pod.metadata?.labels?.app;
+
+    switch (app) {
+      case "task-index":
+        const deploymentId = pod.metadata?.labels?.deployment;
+
+        if (!deploymentId) {
+          this.#logger.error("Index is missing ID", { pod });
+          return;
+        }
+
+        if (this.opts.onIndexFailure) {
+          await this.opts.onIndexFailure(deploymentId, failureInfo);
+        }
+        break;
+      case "task-run":
+        const runId = pod.metadata?.labels?.run;
+
+        if (!runId) {
+          this.#logger.error("Run is missing ID", { pod });
+          return;
+        }
+
+        if (this.opts.onRunFailure) {
+          await this.opts.onRunFailure(runId, failureInfo);
+        }
+        break;
+      default:
+        this.#logger.error("Pod has invalid app label", { pod });
+        return;
+    }
+
+    await this.#deletePod(podName);
+  }
+
+  async #getLogTail(podName: string) {
+    try {
+      const logs = await this.#k8sClient.core.readNamespacedPodLog(
+        podName,
+        this.namespace,
+        undefined,
+        undefined,
+        undefined,
+        1024, // limitBytes
+        undefined,
+        undefined,
+        undefined,
+        20 // tailLines
+      );
+
+      const responseBody = logs.body ?? "";
+
+      if (responseBody.startsWith("unable to retrieve container logs")) {
+        return "";
+      }
+
+      // Type is wrong, body may be undefined
+      return responseBody;
+    } catch (error) {
+      this.#logger.error("Log tail error:", error instanceof Error ? error.message : "unknown");
+      return "";
+    }
+  }
+
+  #getPodStatusSummary(status: k8s.V1PodStatus) {
+    return {
+      reason: status.reason,
+      message: status.message,
+    };
+  }
+
+  #getContainerStateSummary(state: k8s.V1ContainerState) {
+    return {
+      reason: state.terminated?.reason,
+      exitCode: state.terminated?.exitCode,
+      message: state.terminated?.message,
+    };
+  }
+
+  #createK8sClient() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (this.opts.runtimeEnv === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (this.opts.runtimeEnv === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      kubeConfig: kubeConfig,
+    };
+  }
+
+  #isRecord(candidate: unknown): candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  #logK8sError(err: unknown, debugOnly = false) {
+    if (debugOnly) {
+      this.#logger.debug("K8s API Error", err);
+    } else {
+      this.#logger.error("K8s API Error", err);
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
+      this.#logK8sError(err);
+      return;
+    }
+
+    this.#logK8sError(err, true);
+
+    if (typeof err.body.message === "string") {
+      this.#logK8sError({ message: err.body.message });
+      return;
+    }
+
+    this.#logK8sError({ body: err.body });
+  }
+
+  #printStats(includeMoreDetails = false) {
+    this.#logger.log("Stats:", {
+      cacheSize: this.#taskInformer.list().length,
+      totalProcessed: this.#processedPods.size,
+      ...(includeMoreDetails && {
+        processedPods: this.#processedPods,
+      }),
+    });
+  }
+
+  async #deletePod(name: string) {
+    this.#logger.debug("Deleting pod:", name);
+
+    await this.#k8sClient.core
+      .deleteNamespacedPod(name, this.namespace)
+      .catch(this.#handleK8sError.bind(this));
+  }
+
+  async start() {
+    this.#enabled = true;
+
+    const interval = setInterval(() => {
+      if (!this.#enabled) {
+        clearInterval(interval);
+        return;
+      }
+
+      this.#printStats();
+    }, 300_000);
+
+    await this.#taskInformer.start();
+
+    // this.#launchTests();
+  }
+
+  async stop() {
+    if (!this.#enabled) {
+      return;
+    }
+
+    this.#enabled = false;
+    this.#logger.log("Shutting down..");
+
+    await this.#taskInformer.stop();
+
+    this.#printStats(true);
+  }
+
+  async #launchTests() {
+    const createPod = async (
+      container: k8s.V1Container,
+      name: string,
+      labels?: Record<string, string>
+    ) => {
+      this.#logger.log("Creating pod:", name);
+
+      const pod = {
+        metadata: {
+          name,
+          labels,
+        },
+        spec: {
+          restartPolicy: "Never",
+          automountServiceAccountToken: false,
+          terminationGracePeriodSeconds: 1,
+          containers: [container],
+        },
+      } satisfies k8s.V1Pod;
+
+      await this.#k8sClient.core
+        .createNamespacedPod(this.namespace, pod)
+        .catch(this.#handleK8sError.bind(this));
+    };
+
+    const createOomPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "polinux/stress",
+        resources: {
+          limits: {
+            memory: "100Mi",
+          },
+        },
+        command: ["stress"],
+        args: ["--vm", "1", "--vm-bytes", "150M", "--vm-hang", "1"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createNonZeroExitPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: ["-c", "exit 1"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createOoDiskPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: [
+          "-c",
+          "echo creating huge-file..; head -c 1000m /dev/zero > huge-file; ls -lh huge-file; sleep infinity",
+        ],
+        resources: {
+          limits: {
+            "ephemeral-storage": "500Mi",
+          },
+        },
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    await createNonZeroExitPod("non-zero-exit-task", { app: "task-run", run: "123" });
+    await createOomPod("oom-task", { app: "task-index", deployment: "456" });
+    await createOoDiskPod("ood-task", { app: "task-run", run: "abc" });
+  }
+}

apps/kubernetes-provider/src/uptimeHeartbeat.ts

@@ -0,0 +1,272 @@
+import * as k8s from "@kubernetes/client-node";
+import { SimpleLogger } from "@trigger.dev/core/v3/apps";
+
+type UptimeHeartbeatOptions = {
+  runtimeEnv: "local" | "kubernetes";
+  pingUrl: string;
+  namespace?: string;
+  intervalInSeconds?: number;
+  maxPendingRuns?: number;
+  maxPendingIndeces?: number;
+  maxPendingErrors?: number;
+  leadingEdge?: boolean;
+};
+
+export class UptimeHeartbeat {
+  private enabled = false;
+  private namespace: string;
+
+  private intervalInSeconds: number;
+  private maxPendingRuns: number;
+  private maxPendingIndeces: number;
+  private maxPendingErrors: number;
+
+  private leadingEdge = true;
+
+  private logger = new SimpleLogger("[UptimeHeartbeat]");
+  private k8sClient: {
+    core: k8s.CoreV1Api;
+    kubeConfig: k8s.KubeConfig;
+  };
+
+  constructor(private opts: UptimeHeartbeatOptions) {
+    this.namespace = opts.namespace ?? "default";
+
+    this.intervalInSeconds = opts.intervalInSeconds ?? 60;
+    this.maxPendingRuns = opts.maxPendingRuns ?? 25;
+    this.maxPendingIndeces = opts.maxPendingIndeces ?? 10;
+    this.maxPendingErrors = opts.maxPendingErrors ?? 10;
+
+    this.k8sClient = this.#createK8sClient();
+  }
+
+  #createK8sClient() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (this.opts.runtimeEnv === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (this.opts.runtimeEnv === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      kubeConfig: kubeConfig,
+    };
+  }
+
+  #isRecord(candidate: unknown): candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  #logK8sError(err: unknown, debugOnly = false) {
+    if (debugOnly) {
+      this.logger.debug("K8s API Error", err);
+    } else {
+      this.logger.error("K8s API Error", err);
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
+      this.#logK8sError(err);
+      return;
+    }
+
+    this.#logK8sError(err, true);
+
+    if (typeof err.body.message === "string") {
+      this.#logK8sError({ message: err.body.message });
+      return;
+    }
+
+    this.#logK8sError({ body: err.body });
+  }
+
+  async #getPods(opts: {
+    namespace: string;
+    fieldSelector?: string;
+    labelSelector?: string;
+  }): Promise<Array<k8s.V1Pod> | undefined> {
+    const listReturn = await this.k8sClient.core
+      .listNamespacedPod(
+        opts.namespace,
+        undefined, // pretty
+        undefined, // allowWatchBookmarks
+        undefined, // _continue
+        opts.fieldSelector,
+        opts.labelSelector,
+        this.maxPendingRuns * 2, // limit
+        undefined, // resourceVersion
+        undefined, // resourceVersionMatch
+        undefined, // sendInitialEvents
+        this.intervalInSeconds, // timeoutSeconds,
+        undefined // watch
+      )
+      .catch(this.#handleK8sError.bind(this));
+
+    return listReturn?.body.items;
+  }
+
+  async #getPendingIndeces(): Promise<Array<k8s.V1Pod> | undefined> {
+    return await this.#getPods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=Pending",
+      labelSelector: "app=task-index",
+    });
+  }
+
+  async #getPendingTasks(): Promise<Array<k8s.V1Pod> | undefined> {
+    return await this.#getPods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=Pending",
+      labelSelector: "app=task-run",
+    });
+  }
+
+  #countPods(pods: Array<k8s.V1Pod>): number {
+    return pods.length;
+  }
+
+  #filterPendingPods(
+    pods: Array<k8s.V1Pod>,
+    waitingReason: "CreateContainerError" | "RunContainerError"
+  ): Array<k8s.V1Pod> {
+    return pods.filter((pod) => {
+      const containerStatus = pod.status?.containerStatuses?.[0];
+      return containerStatus?.state?.waiting?.reason === waitingReason;
+    });
+  }
+
+  async #sendPing() {
+    this.logger.log("Sending ping");
+
+    const start = Date.now();
+    const controller = new AbortController();
+
+    const timeoutMs = (this.intervalInSeconds * 1000) / 2;
+
+    const fetchTimeout = setTimeout(() => {
+      controller.abort();
+    }, timeoutMs);
+
+    try {
+      const response = await fetch(this.opts.pingUrl, {
+        signal: controller.signal,
+      });
+
+      if (!response.ok) {
+        this.logger.error("Failed to send ping, response not OK", {
+          status: response.status,
+        });
+        return;
+      }
+
+      const elapsedMs = Date.now() - start;
+      this.logger.log("Ping sent", { elapsedMs });
+    } catch (error) {
+      if (error instanceof DOMException && error.name === "AbortError") {
+        this.logger.log("Ping timeout", { timeoutSeconds: timeoutMs });
+        return;
+      }
+
+      this.logger.error("Failed to send ping", error);
+    } finally {
+      clearTimeout(fetchTimeout);
+    }
+  }
+
+  async #heartbeat() {
+    this.logger.log("Performing heartbeat");
+
+    const start = Date.now();
+
+    const pendingTasks = await this.#getPendingTasks();
+
+    if (!pendingTasks) {
+      this.logger.error("Failed to get pending tasks");
+      return;
+    }
+
+    const totalPendingTasks = this.#countPods(pendingTasks);
+
+    const pendingIndeces = await this.#getPendingIndeces();
+
+    if (!pendingIndeces) {
+      this.logger.error("Failed to get pending indeces");
+      return;
+    }
+
+    const totalPendingIndeces = this.#countPods(pendingIndeces);
+
+    const elapsedMs = Date.now() - start;
+
+    this.logger.log("Finished heartbeat checks", { elapsedMs });
+
+    if (totalPendingTasks > this.maxPendingRuns) {
+      this.logger.log("Too many pending tasks, skipping heartbeat", { totalPendingTasks });
+      return;
+    }
+
+    if (totalPendingIndeces > this.maxPendingIndeces) {
+      this.logger.log("Too many pending indeces, skipping heartbeat", { totalPendingIndeces });
+      return;
+    }
+
+    const totalCreateContainerErrors = this.#countPods(
+      this.#filterPendingPods(pendingTasks, "CreateContainerError")
+    );
+    const totalRunContainerErrors = this.#countPods(
+      this.#filterPendingPods(pendingTasks, "RunContainerError")
+    );
+
+    if (totalCreateContainerErrors + totalRunContainerErrors > this.maxPendingErrors) {
+      this.logger.log("Too many pending tasks with errors, skipping heartbeat", {
+        totalRunContainerErrors,
+        totalCreateContainerErrors,
+      });
+      return;
+    }
+
+    await this.#sendPing();
+
+    this.logger.log("Heartbeat done", { totalPendingTasks, elapsedMs });
+  }
+
+  async start() {
+    this.enabled = true;
+    this.logger.log("Starting");
+
+    if (this.leadingEdge) {
+      await this.#heartbeat();
+    }
+
+    const heartbeat = setInterval(async () => {
+      if (!this.enabled) {
+        clearInterval(heartbeat);
+        return;
+      }
+
+      try {
+        await this.#heartbeat();
+      } catch (error) {
+        this.logger.error("Error while heartbeating", error);
+      }
+    }, this.intervalInSeconds * 1000);
+  }
+
+  async stop() {
+    if (!this.enabled) {
+      return;
+    }
+
+    this.enabled = false;
+    this.logger.log("Shutting down..");
+  }
+}

apps/kubernetes-provider/tsconfig.json

@@ -9,9 +9,7 @@
     "skipLibCheck": true,
     "paths": {
       "@trigger.dev/core/v3": ["../../packages/core/src/v3"],
-      "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"],
-      "@trigger.dev/core-apps": ["../../packages/core-apps/src"],
-      "@trigger.dev/core-apps/*": ["../../packages/core-apps/src/*"]
+      "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"]
     }
   }
 }

apps/proxy/package.json

@@ -7,9 +7,9 @@
     "dev": "wrangler dev"
   },
   "devDependencies": {
-    "@cloudflare/workers-types": "^4.20230419.0",
+    "@cloudflare/workers-types": "^4.20240512.0",
     "typescript": "^5.0.4",
-    "wrangler": "^3.0.0"
+    "wrangler": "^3.57.1"
   },
   "dependencies": {
     "@aws-sdk/client-sqs": "^3.445.0",
@@ -18,4 +18,4 @@
     "zod": "3.22.3",
     "zod-error": "1.5.0"
   }
-}
+}

apps/proxy/src/index.ts

@@ -1,5 +1,7 @@
 import { queueEvent } from "./events/queueEvent";
 import { queueEvents } from "./events/queueEvents";
+import { applyRateLimit } from "./rateLimit";
+import { Ratelimit } from "./rateLimiter";
 
 export interface Env {
   /** The hostname needs to be changed to allow requests to pass to the Trigger.dev platform */
@@ -9,6 +11,8 @@ export interface Env {
   AWS_SQS_SECRET_ACCESS_KEY: string;
   AWS_SQS_QUEUE_URL: string;
   AWS_SQS_REGION: string;
+  //rate limiter
+  API_RATE_LIMITER: Ratelimit;
 }
 
 export default {
@@ -25,13 +29,13 @@ export default {
     switch (url.pathname) {
       case "/api/v1/events": {
         if (request.method === "POST") {
-          return queueEvent(request, env);
+          return applyRateLimit(request, env, () => queueEvent(request, env));
         }
         break;
       }
       case "/api/v1/events/bulk": {
         if (request.method === "POST") {
-          return queueEvents(request, env);
+          return applyRateLimit(request, env, () => queueEvents(request, env));
         }
         break;
       }

apps/proxy/src/rateLimit.ts

@@ -0,0 +1,46 @@
+import { Env } from "src";
+import { getApiKeyFromRequest } from "./apikey";
+import { json } from "./json";
+
+export async function applyRateLimit(
+  request: Request,
+  env: Env,
+  fn: () => Promise<Response>
+): Promise<Response> {
+  const apiKey = getApiKeyFromRequest(request);
+  if (apiKey) {
+    const result = await env.API_RATE_LIMITER.limit({ key: `apikey-${apiKey.apiKey}` });
+    const { success } = result;
+    console.log(`Rate limiter`, {
+      success,
+      key: `${apiKey.apiKey.substring(0, 12)}...`,
+    });
+    if (!success) {
+      //60s in the future
+      const reset = Date.now() + 60 * 1000;
+      const secondsUntilReset = Math.max(0, (reset - new Date().getTime()) / 1000);
+
+      return json(
+        {
+          title: "Rate Limit Exceeded",
+          status: 429,
+          type: "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429",
+          detail: `Rate limit exceeded. Retry in ${secondsUntilReset} seconds.`,
+          error: `Rate limit exceeded. Retry in ${secondsUntilReset} seconds.`,
+          reset,
+        },
+        {
+          status: 429,
+          headers: {
+            "x-ratelimit-reset": reset.toString(),
+          },
+        }
+      );
+    }
+  } else {
+    console.log(`Rate limiter: no API key for request`);
+  }
+
+  //call the original function
+  return fn();
+}

apps/proxy/src/rateLimiter.ts

@@ -0,0 +1,23 @@
+export interface Ratelimit {
+  /*
+   * The ratelimit function
+   * @param {RatelimitOptions} options
+   * @returns {Promise<RatelimitResponse>}
+   */
+  limit: (options: RatelimitOptions) => Promise<RatelimitResponse>;
+}
+
+export interface RatelimitOptions {
+  /*
+   * The key to identify the user, can be an IP address, user ID, etc.
+   */
+  key: string;
+}
+
+export interface RatelimitResponse {
+  /*
+   * The ratelimit success status
+   * @returns {boolean}
+   */
+  success: boolean;
+}

apps/proxy/wrangler.toml

@@ -1,7 +1,33 @@
 name = "proxy"
 main = "src/index.ts"
-compatibility_date = "2023-10-30"
+compatibility_date = "2024-05-13"
 compatibility_flags = [ "nodejs_compat" ]
 
 [env.staging]
-[env.prod]
+  # The rate limiting API is in open beta.
+  [[env.staging.unsafe.bindings]]
+  name = "API_RATE_LIMITER"
+  type = "ratelimit"
+  # An identifier you define, that is unique to your Cloudflare account.
+  # Must be an integer.
+  namespace_id = "1"
+
+  # Limit: the number of tokens allowed within a given period in a single
+  # Cloudflare location
+  # Period: the duration of the period, in seconds. Must be either 10 or 60
+  simple = { limit = 100, period = 60 }
+
+
+[env.prod]
+  # The rate limiting API is in open beta.
+  [[env.prod.unsafe.bindings]]
+  name = "API_RATE_LIMITER"
+  type = "ratelimit"
+  # An identifier you define, that is unique to your Cloudflare account.
+  # Must be an integer.
+  namespace_id = "2"
+
+  # Limit: the number of tokens allowed within a given period in a single
+  # Cloudflare location
+  # Period: the duration of the period, in seconds. Must be either 10 or 60
+  simple = { limit = 300, period = 60 }

apps/webapp/.eslintrc

@@ -1,13 +1,36 @@
 {
-  "extends": ["@remix-run/eslint-config", "@remix-run/eslint-config/node", "prettier"],
-  "rules": {
-    "@typescript-eslint/strict-boolean-expressions": [
-      "error",
-      {
-        "allowNullableBoolean": true,
-        "allowNullableString": true,
-        "allowNullableNumber": true
+  "plugins": [
+    "@trigger.dev/eslint-plugin",
+    "react-hooks",
+    "@typescript-eslint/eslint-plugin",
+    "import"
+  ],
+  "parser": "@typescript-eslint/parser",
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx"],
+      "rules": {
+        // Autofixes imports from "@trigger.dev/core" to fine grained modules
+        "@trigger.dev/no-trigger-core-import": "error",
+        // Normalize `import type {}` and `import { type }`
+        "@typescript-eslint/consistent-type-imports": [
+          "warn",
+          {
+            // the "type" annotation can get tangled and cause syntax errors
+            // during some autofixes, so easier to just turn it off
+            "prefer": "type-imports",
+            "disallowTypeAnnotations": true,
+            "fixStyle": "inline-type-imports"
+          }
+        ],
+        // no-trigger-core-import splits imports into multiple lines
+        // this one merges them back into a single line
+        // if they still import from the same module
+        "import/no-duplicates": ["warn", { "prefer-inline": true }],
+        // lots of undeclared vars, enable this rule if you want to clean them up
+        "turbo/no-undeclared-env-vars": "off"
       }
-    ]
-  }
+    }
+  ],
+  "ignorePatterns": ["seed.js", "seedCloud.ts", "populate.js"]
 }

apps/webapp/.gitignore

@@ -17,4 +17,5 @@ build-storybook.log
 .storybook-out
 storybook-static
 
-/prisma/seed.js
+/prisma/seed.js
+/prisma/populate.js

apps/webapp/app/assets/icons/AISparkleIcon.tsx

@@ -0,0 +1,53 @@
+export function AISparkleIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M14.9806 0.803884C14.8871 0.33646 14.4767 0 14 0C13.5233 0 13.1129 0.33646 13.0194 0.803884L12.7809 1.99644C12.7017 2.3923 12.3923 2.70174 11.9964 2.78091L10.8039 3.01942C10.3365 3.1129 10 3.52332 10 4C10 4.47668 10.3365 4.8871 10.8039 4.98058L11.9964 5.21909C12.3923 5.29826 12.7017 5.6077 12.7809 6.00356L13.0194 7.19612C13.1129 7.66354 13.5233 8 14 8C14.4767 8 14.8871 7.66354 14.9806 7.19612L15.2191 6.00356C15.2983 5.6077 15.6077 5.29826 16.0036 5.21909L17.1961 4.98058C17.6635 4.8871 18 4.47668 18 4C18 3.52332 17.6635 3.1129 17.1961 3.01942L16.0036 2.78091C15.6077 2.70174 15.2983 2.3923 15.2191 1.99644L14.9806 0.803884Z"
+        fill="url(#paint0_linear_11402_36656)"
+      />
+      <path
+        d="M5.94868 4.68377C5.81257 4.27543 5.43043 4 5 4C4.56957 4 4.18743 4.27543 4.05132 4.68377L3.36754 6.73509C3.26801 7.03369 3.03369 7.26801 2.73509 7.36754L0.683772 8.05132C0.27543 8.18743 0 8.56957 0 9C0 9.43043 0.27543 9.81257 0.683772 9.94868L2.73509 10.6325C3.03369 10.732 3.26801 10.9663 3.36754 11.2649L4.05132 13.3162C4.18743 13.7246 4.56957 14 5 14C5.43043 14 5.81257 13.7246 5.94868 13.3162L6.63246 11.2649C6.73199 10.9663 6.96631 10.732 7.26491 10.6325L9.31623 9.94868C9.72457 9.81257 10 9.43043 10 9C10 8.56957 9.72457 8.18743 9.31623 8.05132L7.26491 7.36754C6.96631 7.26801 6.73199 7.03369 6.63246 6.73509L5.94868 4.68377Z"
+        fill="url(#paint1_linear_11402_36656)"
+      />
+      <path
+        d="M12.9487 12.6838C12.8126 12.2754 12.4304 12 12 12C11.5696 12 11.1874 12.2754 11.0513 12.6838L10.8675 13.2351C10.768 13.5337 10.5337 13.768 10.2351 13.8675L9.68377 14.0513C9.27543 14.1874 9 14.5696 9 15C9 15.4304 9.27543 15.8126 9.68377 15.9487L10.2351 16.1325C10.5337 16.232 10.768 16.4663 10.8675 16.7649L11.0513 17.3162C11.1874 17.7246 11.5696 18 12 18C12.4304 18 12.8126 17.7246 12.9487 17.3162L13.1325 16.7649C13.232 16.4663 13.4663 16.232 13.7649 16.1325L14.3162 15.9487C14.7246 15.8126 15 15.4304 15 15C15 14.5696 14.7246 14.1874 14.3162 14.0513L13.7649 13.8675C13.4663 13.768 13.232 13.5337 13.1325 13.2351L12.9487 12.6838Z"
+        fill="url(#paint2_linear_11402_36656)"
+      />
+      <defs>
+        <linearGradient
+          id="paint0_linear_11402_36656"
+          x1="9"
+          y1="0"
+          x2="9"
+          y2="18"
+          gradientUnits="userSpaceOnUse"
+        >
+          <stop stopColor="#E543FF" />
+          <stop offset="1" stopColor="#286399" />
+        </linearGradient>
+        <linearGradient
+          id="paint1_linear_11402_36656"
+          x1="9"
+          y1="0"
+          x2="9"
+          y2="18"
+          gradientUnits="userSpaceOnUse"
+        >
+          <stop stopColor="#E543FF" />
+          <stop offset="1" stopColor="#286399" />
+        </linearGradient>
+        <linearGradient
+          id="paint2_linear_11402_36656"
+          x1="9"
+          y1="0"
+          x2="9"
+          y2="18"
+          gradientUnits="userSpaceOnUse"
+        >
+          <stop stopColor="#E543FF" />
+          <stop offset="1" stopColor="#286399" />
+        </linearGradient>
+      </defs>
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/v3.svg

@@ -0,0 +1,4 @@
+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect x="0.861435" y="0.861435" width="30.2771" height="30.2771" rx="15.1386" stroke="#D7D9DD" stroke-width="1.72287"/>
+<path d="M14.9669 12.642L11.6417 21.7732H9.8155L6.49036 12.642H8.04094L10.7286 20.2571L13.4163 12.642H14.9669ZM20.534 14.5544C21.6884 14.5716 23.0666 14.0375 23.0666 12.4525C23.0666 11.3843 22.1018 10.7124 20.534 10.7124C19.2591 10.7124 18.3632 11.3326 18.2082 12.3146L16.6576 12.2113C16.8815 10.4884 18.4838 9.26516 20.534 9.26516C22.946 9.26516 24.6345 10.5745 24.6345 12.4525C24.6345 13.8135 23.8592 14.7267 22.3775 15.1574C24.0831 15.657 25.0479 16.8286 25.0479 18.4136C25.0479 20.55 23.1872 22.0489 20.534 22.0489C18.0876 22.0489 16.313 20.6361 16.2096 18.6203L17.743 18.517C17.8808 20.0159 19.3108 20.6016 20.534 20.6016C21.9812 20.6016 23.4974 19.947 23.4974 18.293C23.4974 16.6563 21.9812 15.9499 20.534 15.9844L19.5865 16.0016V14.5371L20.534 14.5544Z" fill="white"/>
+</svg>

apps/webapp/app/components/DefinitionTooltip.tsx

@@ -15,11 +15,11 @@ export function DefinitionTip({
     <TooltipProvider>
       <Tooltip disableHoverableContent>
         <TooltipTrigger>
-          <span className="underline decoration-charcoal-600 decoration-dashed underline-offset-4 transition hover:decoration-charcoal-500">
+          <span className="cursor-default underline decoration-charcoal-500 decoration-dashed underline-offset-4 transition hover:decoration-charcoal-400">
             {children}
           </span>
         </TooltipTrigger>
-        <TooltipContent align="end" side="right" variant="dark" className="w-[16rem] min-w-[16rem]">
+        <TooltipContent align="end" side="right" className="w-[16rem] min-w-[16rem]">
           <Header3 className="mb-1">{title}</Header3>
           {typeof content === "string" ? (
             <Paragraph variant="small">{content}</Paragraph>

apps/webapp/app/components/Feedback.tsx

@@ -1,32 +1,31 @@
 import { conform, useForm } from "@conform-to/react";
 import { parse } from "@conform-to/zod";
-import { ChevronRightIcon } from "@heroicons/react/24/solid";
+import { BookOpenIcon } from "@heroicons/react/20/solid";
+import {
+  CalendarDaysIcon,
+  ChevronRightIcon,
+  EnvelopeIcon,
+  LifebuoyIcon,
+  LightBulbIcon,
+} from "@heroicons/react/24/solid";
 import { Form, useActionData, useLocation, useNavigation } from "@remix-run/react";
-import { DiscordIcon, GitHubLightIcon } from "@trigger.dev/companyicons";
-import { ReactNode, useState } from "react";
-import { FeedbackType, feedbackTypeLabel, schema } from "~/routes/resources.feedback";
+import { DiscordIcon } from "@trigger.dev/companyicons";
+import { ActivityIcon } from "lucide-react";
+import { type ReactNode, useState } from "react";
+import { type FeedbackType, feedbackTypeLabel, schema } from "~/routes/resources.feedback";
+import { cn } from "~/utils/cn";
+import { docsTroubleshootingPath } from "~/utils/pathBuilder";
 import { Button, LinkButton } from "./primitives/Buttons";
 import { Fieldset } from "./primitives/Fieldset";
 import { FormButtons } from "./primitives/FormButtons";
 import { FormError } from "./primitives/FormError";
-import { Header1, Header2 } from "./primitives/Headers";
+import { Header1 } from "./primitives/Headers";
 import { InputGroup } from "./primitives/InputGroup";
 import { Label } from "./primitives/Label";
 import { Paragraph } from "./primitives/Paragraph";
-import {
-  Select,
-  SelectContent,
-  SelectGroup,
-  SelectItem,
-  SelectTrigger,
-  SelectValue,
-} from "./primitives/Select";
+import { Select, SelectItem } from "./primitives/Select";
 import { Sheet, SheetBody, SheetContent, SheetTrigger } from "./primitives/Sheet";
 import { TextArea } from "./primitives/TextArea";
-import { cn } from "~/utils/cn";
-import { BookOpenIcon } from "@heroicons/react/20/solid";
-import { ActivityIcon, HeartPulseIcon } from "lucide-react";
-import { docsPath } from "~/utils/pathBuilder";
 
 type FeedbackProps = {
   button: ReactNode;
@@ -62,130 +61,137 @@ export function Feedback({ button, defaultValue = "bug" }: FeedbackProps) {
       <SheetTrigger asChild={true}>{button}</SheetTrigger>
       <SheetContent className="@container">
         <SheetBody className="flex h-full flex-col justify-between">
-          <Header2 className="mb-2.5 text-xl">Get help from the community</Header2>
-          <Paragraph className="mb-4">
-            The quickest way to get help and feedback or to provide advice to others is to join our
-            Discord.
-          </Paragraph>
-          <div className="flex flex-col gap-x-4 @[30rem]:flex-row">
-            <DiscordBanner />
-            <GitHubDiscussionsBanner />
-          </div>
-          <hr className="mb-4" />
-          <Header2 className="mb-2.5 text-xl">Send us an email</Header2>
-          <Paragraph className="mb-4">We read every message and respond quickly.</Paragraph>
-          <Form method="post" action="/resources/feedback" {...form.props}>
-            <Fieldset className="max-w-full gap-y-3">
-              <input value={location.pathname} {...conform.input(path, { type: "hidden" })} />
-              <InputGroup className="max-w-full">
-                <SelectGroup>
-                  <Select {...conform.input(feedbackType)} defaultValue={defaultValue}>
-                    <SelectTrigger size="medium" width="full">
-                      <SelectValue placeholder="Type" />
-                    </SelectTrigger>
-                    <SelectContent>
-                      {Object.entries(feedbackTypeLabel).map(([key, value]) => (
-                        <SelectItem key={key} value={key}>
-                          {value}
-                        </SelectItem>
-                      ))}
-                    </SelectContent>
+          <LinkBanner
+            title="Join our Discord community"
+            icon={<DiscordIcon className="size-9" />}
+            to="https://trigger.dev/discord"
+            className="hover:border-text-link"
+          >
+            <Paragraph>The quickest way to get answers from the Trigger.dev community.</Paragraph>
+          </LinkBanner>
+          <LinkBanner
+            title="Book a 15 min chat with the founders"
+            icon={<CalendarDaysIcon className="size-9 text-green-500" />}
+            to="https://cal.com/team/triggerdotdev/founders-call"
+            className="hover:border-green-500"
+          >
+            <Paragraph>Have a question or want to chat? Book a time to talk with us.</Paragraph>
+          </LinkBanner>
+          <LinkBanner
+            title="Suggest a feature"
+            icon={<LightBulbIcon className="size-9 text-sun-500" />}
+            to="https://feedback.trigger.dev/"
+            className="hover:border-sun-400"
+          >
+            <Paragraph>Have an idea for a new feature or improvement? Let us know!</Paragraph>
+          </LinkBanner>
+          <LinkBanner
+            title="Troubleshooting"
+            icon={<LifebuoyIcon className="size-9 text-rose-500" />}
+          >
+            <Paragraph>
+              If you're having trouble, check out our troubleshooting guide or the Trigger.dev
+              Status page.
+            </Paragraph>
+            <div className="flex flex-wrap gap-2">
+              <LinkButton
+                to={docsTroubleshootingPath("")}
+                variant="tertiary/medium"
+                LeadingIcon={BookOpenIcon}
+              >
+                Troubleshooting Docs
+              </LinkButton>
+              <LinkButton
+                to={"https://status.trigger.dev/"}
+                variant="tertiary/medium"
+                LeadingIcon={ActivityIcon}
+              >
+                Trigger.dev Status
+              </LinkButton>
+            </div>
+          </LinkBanner>
+          <LinkBanner
+            title="Send us an email"
+            icon={<EnvelopeIcon className="size-9 text-blue-500" />}
+          >
+            <Paragraph>We read every message and respond quickly.</Paragraph>
+            <Form method="post" action="/resources/feedback" {...form.props} className="w-full">
+              <Fieldset className="max-w-full gap-y-3">
+                <input value={location.pathname} {...conform.input(path, { type: "hidden" })} />
+                <InputGroup className="max-w-full">
+                  <Select
+                    {...conform.select(feedbackType)}
+                    variant="tertiary/medium"
+                    defaultValue={defaultValue}
+                    placeholder="Select type"
+                    text={(value) => feedbackTypeLabel[value]}
+                    dropdownIcon
+                  >
+                    {Object.entries(feedbackTypeLabel).map(([name, title]) => (
+                      <SelectItem key={name} value={name}>
+                        {title}
+                      </SelectItem>
+                    ))}
                   </Select>
-                </SelectGroup>
-                <FormError id={feedbackType.errorId}>{feedbackType.error}</FormError>
-              </InputGroup>
-              <InputGroup className="max-w-full">
-                <Label>Message</Label>
-                <TextArea {...conform.textarea(message)} />
-                <FormError id={message.errorId}>{message.error}</FormError>
-              </InputGroup>
-              <FormError>{form.error}</FormError>
-              <div className="flex w-full justify-end">
-                <FormButtons
-                  className="m-0 w-max"
-                  confirmButton={
-                    <Button type="submit" variant="tertiary/medium">
-                      Send message
-                    </Button>
-                  }
-                />
-              </div>
-            </Fieldset>
-          </Form>
-          <hr className="my-4" />
-          <Header2 className="mb-2.5 text-xl">Troubleshooting</Header2>
-          <Paragraph className="mb-4">
-            If you're having trouble, check out our documentation or the Trigger.dev Status page.
-          </Paragraph>
-          <div className="flex flex-wrap gap-2">
-            <LinkButton to={docsPath("")} variant="tertiary/medium" LeadingIcon={BookOpenIcon}>
-              Docs
-            </LinkButton>
-            <LinkButton
-              to={docsPath("v3/introduction")}
-              variant="tertiary/medium"
-              LeadingIcon={BookOpenIcon}
-            >
-              v3 Docs (Developer preview)
-            </LinkButton>
-            <LinkButton
-              to={"https://trigger.openstatus.dev/"}
-              variant="tertiary/medium"
-              LeadingIcon={ActivityIcon}
-            >
-              Trigger.dev Status
-            </LinkButton>
-          </div>
+                  <FormError id={feedbackType.errorId}>{feedbackType.error}</FormError>
+                </InputGroup>
+                <InputGroup className="max-w-full">
+                  <Label>Message</Label>
+                  <TextArea {...conform.textarea(message)} />
+                  <FormError id={message.errorId}>{message.error}</FormError>
+                </InputGroup>
+                <FormError>{form.error}</FormError>
+                <div className="flex w-full justify-end">
+                  <FormButtons
+                    className="m-0 w-max"
+                    confirmButton={
+                      <Button type="submit" variant="tertiary/medium">
+                        Send message
+                      </Button>
+                    }
+                  />
+                </div>
+              </Fieldset>
+            </Form>
+          </LinkBanner>
         </SheetBody>
       </SheetContent>
     </Sheet>
   );
 }
 
-function DiscordBanner({ className }: { className?: string }) {
+function LinkBanner({
+  className,
+  icon,
+  title,
+  children,
+  to,
+}: {
+  className?: string;
+  icon?: ReactNode;
+  title?: string;
+  children?: ReactNode;
+  to?: string;
+}) {
   return (
     <a
-      href="https://trigger.dev/discord"
+      href={to}
       target="_blank"
       className={cn(
-        "group mb-4 flex w-full items-center justify-between rounded-md border border-charcoal-600 p-4 transition hover:border-text-link",
+        "group/banner mb-4 flex w-full items-center justify-between rounded-md border border-grid-bright bg-charcoal-750 p-4 transition",
         className
       )}
     >
-      <div className="flex flex-col gap-y-2">
-        <DiscordIcon className="h-8 w-8" />
-        <Header1 className="text-2xl font-semibold text-text-bright transition group-hover:text-white">
-          Join our Discord community
-        </Header1>
-        <Paragraph variant="small" className="mb-4">
-          Get help or answer questions from the Trigger.dev community.
-        </Paragraph>
+      <div className="flex w-full items-start gap-4">
+        <span>{icon}</span>
+        <div className="flex w-full flex-col gap-2">
+          <Header1 className="text-2xl font-semibold text-text-bright">{title}</Header1>
+          {children}
+        </div>
       </div>
-      <ChevronRightIcon className="size-5 text-charcoal-500 transition group-hover:translate-x-1 group-hover:text-text-link" />
-    </a>
-  );
-}
-
-function GitHubDiscussionsBanner({ className }: { className?: string }) {
-  return (
-    <a
-      href="https://github.com/triggerdotdev/trigger.dev/discussions"
-      target="_blank"
-      className={cn(
-        "group mb-4 flex w-full items-center justify-between rounded-md border border-charcoal-600 p-4 transition hover:border-text-dimmed",
-        className
+      {to && (
+        <ChevronRightIcon className="size-5 text-charcoal-500 transition group-hover:translate-x-1 group-hover/banner:text-text-bright" />
       )}
-    >
-      <div className="flex flex-col gap-y-2">
-        <GitHubLightIcon className="mb-1 h-7 w-7" />
-        <Header1 className="text-2xl font-semibold text-text-bright transition group-hover:text-white">
-          View our GitHub Discussions
-        </Header1>
-        <Paragraph variant="small">
-          Post your questions, feedback, and feature requests on GitHub.
-        </Paragraph>
-      </div>
-      <ChevronRightIcon className="size-5 text-charcoal-500 transition group-hover:translate-x-1 group-hover:text-text-bright" />
     </a>
   );
 }

apps/webapp/app/components/LoginPageLayout.tsx

@@ -2,7 +2,6 @@ import { useEffect, useState } from "react";
 import { ATAndTLogo } from "~/assets/logos/ATAndTLogo";
 import { AppsmithLogo } from "~/assets/logos/AppsmithLogo";
 import { CalComLogo } from "~/assets/logos/CalComLogo";
-import { OktaLogo } from "~/assets/logos/OktaLogo";
 import { ShopifyLogo } from "~/assets/logos/ShopifyLogo";
 import { VerizonLogo } from "~/assets/logos/VerizonLogo";
 import { LogoType } from "./LogoType";
@@ -75,7 +74,6 @@ export function LoginPageLayout({ children }: { children: React.ReactNode }) {
           <div className="flex w-full flex-wrap items-center justify-center gap-x-6 gap-y-3 text-charcoal-500 xl:justify-between xl:gap-0">
             <VerizonLogo />
             <ShopifyLogo />
-            <OktaLogo />
             <ATAndTLogo />
             <AppsmithLogo />
             <CalComLogo />

apps/webapp/app/components/SetupCommands.tsx

@@ -8,6 +8,7 @@ import {
 } from "./primitives/ClientTabs";
 import { ClipboardField } from "./primitives/ClipboardField";
 import { Paragraph } from "./primitives/Paragraph";
+import { useAppOrigin } from "~/hooks/useAppOrigin";
 
 export function InitCommand({ appOrigin, apiKey }: { appOrigin: string; apiKey: string }) {
   return (
@@ -131,11 +132,40 @@ export function TriggerDevStep({ extra }: { extra?: string }) {
 }
 
 // Trigger.dev version 3 setup commands
-const v3PackageTag = "beta";
+const v3PackageTag = "latest";
+
+function getApiUrlArg() {
+  const appOrigin = useAppOrigin();
+
+  let apiUrl: string | undefined = undefined;
+
+  switch (appOrigin) {
+    case "https://cloud.trigger.dev":
+      // don't display the arg, use the CLI default
+      break;
+    case "https://test-cloud.trigger.dev":
+      apiUrl = "https://test-api.trigger.dev";
+      break;
+    case "https://internal.trigger.dev":
+      apiUrl = "https://internal-api.trigger.dev";
+      break;
+    default:
+      apiUrl = appOrigin;
+      break;
+  }
+
+  return apiUrl ? `-a ${apiUrl}` : undefined;
+}
 
 export function InitCommandV3() {
   const project = useProject();
   const projectRef = project.ref;
+
+  const apiUrlArg = getApiUrlArg();
+
+  const initCommandParts = [`trigger.dev@${v3PackageTag}`, "init", `-p ${projectRef}`, apiUrlArg];
+  const initCommand = initCommandParts.filter(Boolean).join(" ");
+
   return (
     <ClientTabs defaultValue="npm">
       <ClientTabsList>
@@ -148,7 +178,7 @@ export function InitCommandV3() {
           variant="primary/medium"
           iconButton
           className="mb-4"
-          value={`npx trigger.dev@${v3PackageTag} init -p ${projectRef}`}
+          value={`npx ${initCommand}`}
         />
       </ClientTabsContent>
       <ClientTabsContent value={"pnpm"}>
@@ -156,7 +186,7 @@ export function InitCommandV3() {
           variant="primary/medium"
           iconButton
           className="mb-4"
-          value={`pnpm dlx trigger.dev@${v3PackageTag} init -p ${projectRef}`}
+          value={`pnpm dlx ${initCommand}`}
         />
       </ClientTabsContent>
       <ClientTabsContent value={"yarn"}>
@@ -164,7 +194,7 @@ export function InitCommandV3() {
           variant="primary/medium"
           iconButton
           className="mb-4"
-          value={`yarn dlx trigger.dev@${v3PackageTag} init -p ${projectRef}`}
+          value={`yarn dlx ${initCommand}`}
         />
       </ClientTabsContent>
     </ClientTabs>
@@ -206,3 +236,39 @@ export function TriggerDevStepV3() {
     </ClientTabs>
   );
 }
+
+export function TriggerLoginStepV3() {
+  return (
+    <ClientTabs defaultValue="npm">
+      <ClientTabsList>
+        <ClientTabsTrigger value={"npm"}>npm</ClientTabsTrigger>
+        <ClientTabsTrigger value={"pnpm"}>pnpm</ClientTabsTrigger>
+        <ClientTabsTrigger value={"yarn"}>yarn</ClientTabsTrigger>
+      </ClientTabsList>
+      <ClientTabsContent value={"npm"}>
+        <ClipboardField
+          variant="primary/medium"
+          iconButton
+          className="mb-4"
+          value={`npx trigger.dev@${v3PackageTag} login`}
+        />
+      </ClientTabsContent>
+      <ClientTabsContent value={"pnpm"}>
+        <ClipboardField
+          variant="primary/medium"
+          iconButton
+          className="mb-4"
+          value={`pnpm dlx trigger.dev@${v3PackageTag} login`}
+        />
+      </ClientTabsContent>
+      <ClientTabsContent value={"yarn"}>
+        <ClipboardField
+          variant="primary/medium"
+          iconButton
+          className="mb-4"
+          value={`yarn dlx trigger.dev@${v3PackageTag} login`}
+        />
+      </ClientTabsContent>
+    </ClientTabs>
+  );
+}

apps/webapp/app/components/admin/debugTooltip.tsx

@@ -0,0 +1,69 @@
+import { ShieldCheckIcon } from "@heroicons/react/20/solid";
+import * as Property from "~/components/primitives/PropertyTable";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger,
+} from "~/components/primitives/Tooltip";
+import { useIsImpersonating, useOptionalOrganization } from "~/hooks/useOrganizations";
+import { useOptionalProject } from "~/hooks/useProject";
+import { useHasAdminAccess, useUser } from "~/hooks/useUser";
+
+export function AdminDebugTooltip({ children }: { children?: React.ReactNode }) {
+  const hasAdminAccess = useHasAdminAccess();
+  const isImpersonating = useIsImpersonating();
+
+  if (!hasAdminAccess && !isImpersonating) {
+    return null;
+  }
+
+  return (
+    <TooltipProvider>
+      <Tooltip>
+        <TooltipTrigger>
+          <ShieldCheckIcon className="size-5" />
+        </TooltipTrigger>
+        <TooltipContent className="max-h-[90vh] overflow-y-auto">
+          <Content>{children}</Content>
+        </TooltipContent>
+      </Tooltip>
+    </TooltipProvider>
+  );
+}
+
+function Content({ children }: { children: React.ReactNode }) {
+  const organization = useOptionalOrganization();
+  const project = useOptionalProject();
+  const user = useUser();
+
+  return (
+    <div className="flex flex-col gap-2 divide-y divide-slate-700">
+      <Property.Table>
+        <Property.Item>
+          <Property.Label>User ID</Property.Label>
+          <Property.Value>{user.id}</Property.Value>
+        </Property.Item>
+        {organization && (
+          <Property.Item>
+            <Property.Label>Org ID</Property.Label>
+            <Property.Value>{organization.id}</Property.Value>
+          </Property.Item>
+        )}
+        {project && (
+          <>
+            <Property.Item>
+              <Property.Label>Project ID</Property.Label>
+              <Property.Value>{project.id}</Property.Value>
+            </Property.Item>
+            <Property.Item>
+              <Property.Label>Project ref</Property.Label>
+              <Property.Value>{project.ref}</Property.Value>
+            </Property.Item>
+          </>
+        )}
+      </Property.Table>
+      <div className="pt-2">{children}</div>
+    </div>
+  );
+}

apps/webapp/app/components/billing/UpgradePrompt.tsx

@@ -1,61 +0,0 @@
-import { ExclamationCircleIcon } from "@heroicons/react/20/solid";
-import { ArrowUpCircleIcon } from "@heroicons/react/24/outline";
-import tileBgPath from "~/assets/images/error-banner-tile@2x.png";
-import { MatchedOrganization } from "~/hooks/useOrganizations";
-import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route";
-import { formatNumberCompact } from "~/utils/numberFormatter";
-import { LinkButton } from "../primitives/Buttons";
-import { Icon } from "../primitives/Icon";
-import { Paragraph } from "../primitives/Paragraph";
-
-type UpgradePromptProps = {
-  runsEnabled: boolean;
-  runCountCap: number;
-  planPath: string;
-};
-
-export function UpgradePrompt({ runsEnabled, runCountCap, planPath }: UpgradePromptProps) {
-  return (
-    <div
-      className="flex h-10 items-center justify-between border border-error bg-repeat py-0 pl-3 pr-2"
-      style={{ backgroundImage: `url(${tileBgPath})`, backgroundSize: "8px 8px" }}
-    >
-      <div className="flex items-center gap-2">
-        <Icon icon={ExclamationCircleIcon} className="h-5 w-5 text-error" />
-        <Paragraph variant="small" className="text-error">
-          {runsEnabled
-            ? `You have exceeded the monthly ${formatNumberCompact(runCountCap)} runs
-          limit`
-            : `No runs are executing because you have exceeded the free limit`}
-        </Paragraph>
-      </div>
-      <LinkButton
-        variant={"primary/small"}
-        LeadingIcon={ArrowUpCircleIcon}
-        leadingIconClassName="px-0"
-        to={planPath}
-      >
-        Upgrade
-      </LinkButton>
-    </div>
-  );
-}
-
-export function useShowUpgradePrompt(organization?: MatchedOrganization) {
-  const currentPlan = useCurrentPlan();
-  const shouldShow =
-    organization !== undefined &&
-    currentPlan !== undefined &&
-    currentPlan.usage.exceededRunCount &&
-    currentPlan.usage.runCountCap !== undefined;
-
-  if (!shouldShow) {
-    return { shouldShow };
-  }
-
-  return {
-    shouldShow,
-    runCountCap: currentPlan.usage.runCountCap!,
-    runsEnabled: organization.runsEnabled,
-  };
-}

apps/webapp/app/components/billing/v2/ConcurrentRunsChart.tsx

@@ -8,7 +8,7 @@ import {
   XAxis,
   YAxis,
 } from "recharts";
-import { Paragraph } from "../primitives/Paragraph";
+import { Paragraph } from "../../primitives/Paragraph";
 
 const tooltipStyle = {
   display: "flex",

apps/webapp/app/components/billing/v2/DailyRunsChat.tsx

@@ -1,5 +1,5 @@
 import { Label, Line, LineChart, ResponsiveContainer, Tooltip, XAxis, YAxis } from "recharts";
-import { Paragraph } from "../primitives/Paragraph";
+import { Paragraph } from "../../primitives/Paragraph";
 
 const tooltipStyle = {
   display: "flex",

apps/webapp/app/components/billing/v2/FreePlanUsage.tsx

@@ -1,6 +1,6 @@
 import { ArrowUpCircleIcon } from "@heroicons/react/24/outline";
 import { motion, useMotionValue, useTransform } from "framer-motion";
-import { Paragraph } from "../primitives/Paragraph";
+import { Paragraph } from "../../primitives/Paragraph";
 import { Link } from "@remix-run/react";
 import { cn } from "~/utils/cn";
 
@@ -28,7 +28,7 @@ export function FreePlanUsage({ to, percentage }: { to: string; percentage: numb
           <Paragraph className="text-2sm text-text-bright">Free Plan</Paragraph>
         </div>
         <Link to={to} className="text-2sm text-text-link">
-          Learn more
+          Upgrade
         </Link>
       </div>
       <div className="relative mt-3 h-1 rounded-full bg-background-dimmed">

apps/webapp/app/components/billing/v2/PricingCalculator.tsx

@@ -1,9 +1,9 @@
 import * as Slider from "@radix-ui/react-slider";
-import { Plans, estimate } from "@trigger.dev/billing";
+import { Plans, estimate } from "@trigger.dev/platform/v2";
 import { useCallback, useState } from "react";
-import { DefinitionTip } from "../DefinitionTooltip";
-import { Header2 } from "../primitives/Headers";
-import { Paragraph } from "../primitives/Paragraph";
+import { DefinitionTip } from "../../DefinitionTooltip";
+import { Header2 } from "../../primitives/Headers";
+import { Paragraph } from "../../primitives/Paragraph";
 import { formatCurrency, formatNumberCompact } from "~/utils/numberFormatter";
 import { cn } from "~/utils/cn";
 

apps/webapp/app/components/billing/v2/PricingTiers.tsx

@@ -2,17 +2,17 @@ import { useForm } from "@conform-to/react";
 import { parse } from "@conform-to/zod";
 import { CheckIcon, XMarkIcon } from "@heroicons/react/24/solid";
 import { Form, useActionData, useNavigation } from "@remix-run/react";
-import { ActiveSubscription, Plan, Plans, SetPlanBodySchema } from "@trigger.dev/billing";
+import { ActiveSubscription, Plan, Plans, SetPlanBodySchema } from "@trigger.dev/platform/v2";
 import { useState } from "react";
 import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route";
 import { cn } from "~/utils/cn";
 import { formatNumberCompact } from "~/utils/numberFormatter";
-import { DefinitionTip } from "../DefinitionTooltip";
-import { Feedback } from "../Feedback";
-import { Button, LinkButton } from "../primitives/Buttons";
-import SegmentedControl from "../primitives/SegmentedControl";
+import { DefinitionTip } from "../../DefinitionTooltip";
+import { Feedback } from "../../Feedback";
+import { Button, LinkButton } from "../../primitives/Buttons";
+import SegmentedControl from "../../primitives/SegmentedControl";
 import { RunsVolumeDiscountTable } from "./RunsVolumeDiscountTable";
-import { Spinner } from "../primitives/Spinner";
+import { Spinner } from "../../primitives/Spinner";
 
 const pricingDefinitions = {
   concurrentRuns: {
@@ -290,6 +290,7 @@ export function TierPro({
           options={concurrencyTiers.map((c) => ({ label: `Up to ${c.upto}`, value: c.code }))}
           fullWidth
           value={concurrentBracketCode}
+          variant="primary"
           onChange={(v) => setConcurrentBracketCode(v)}
         />
         <div className="py-6">

apps/webapp/app/components/billing/v2/RunsVolumeDiscountTable.tsx

@@ -1,6 +1,6 @@
-import { RunPriceBracket } from "@trigger.dev/billing";
-import { Header2 } from "../primitives/Headers";
-import { Paragraph } from "../primitives/Paragraph";
+import { RunPriceBracket } from "@trigger.dev/platform/v2";
+import { Header2 } from "../../primitives/Headers";
+import { Paragraph } from "../../primitives/Paragraph";
 import { formatNumberCompact } from "~/utils/numberFormatter";
 
 export function RunsVolumeDiscountTable({

apps/webapp/app/components/billing/v2/UsageBar.tsx

@@ -1,7 +1,7 @@
 import { cn } from "~/utils/cn";
 import { formatNumberCompact } from "~/utils/numberFormatter";
-import { Paragraph } from "../primitives/Paragraph";
-import { SimpleTooltip } from "../primitives/Tooltip";
+import { Paragraph } from "../../primitives/Paragraph";
+import { SimpleTooltip } from "../../primitives/Tooltip";
 import { motion } from "framer-motion";
 
 type UsageBarProps = {

apps/webapp/app/components/billing/v3/UpgradePrompt.tsx

@@ -0,0 +1,53 @@
+import { ExclamationCircleIcon } from "@heroicons/react/20/solid";
+import tileBgPath from "~/assets/images/error-banner-tile@2x.png";
+import { MatchedOrganization, useOrganization } from "~/hooks/useOrganizations";
+import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route";
+import { v3BillingPath } from "~/utils/pathBuilder";
+import { LinkButton } from "../../primitives/Buttons";
+import { Icon } from "../../primitives/Icon";
+import { Paragraph } from "../../primitives/Paragraph";
+import { DateTime } from "~/components/primitives/DateTime";
+
+export function UpgradePrompt() {
+  const organization = useOrganization();
+  const plan = useCurrentPlan();
+
+  if (!plan || !plan.v3Usage.hasExceededFreeTier) {
+    return null;
+  }
+
+  const nextMonth = new Date();
+  nextMonth.setUTCMonth(nextMonth.getMonth() + 1);
+  nextMonth.setUTCDate(1);
+  nextMonth.setUTCHours(0, 0, 0, 0);
+
+  return (
+    <div
+      className="flex h-10 items-center justify-between border border-error bg-repeat py-0 pl-3 pr-2"
+      style={{ backgroundImage: `url(${tileBgPath})`, backgroundSize: "8px 8px" }}
+    >
+      <div className="flex items-center gap-2">
+        <Icon icon={ExclamationCircleIcon} className="h-5 w-5 text-error" />
+        <Paragraph variant="small" className="text-error">
+          You have exceeded the monthly $
+          {(plan.v3Subscription?.plan?.limits.includedUsage ?? 500) / 100} free credits. No runs
+          will execute in Prod until{" "}
+          <DateTime date={nextMonth} includeTime={false} timeZone="utc" />, or you upgrade.
+        </Paragraph>
+      </div>
+      <LinkButton
+        variant={"primary/small"}
+        leadingIconClassName="px-0"
+        to={v3BillingPath(organization)}
+      >
+        Upgrade
+      </LinkButton>
+    </div>
+  );
+}
+
+export function useShowUpgradePrompt(organization?: MatchedOrganization) {
+  const currentPlan = useCurrentPlan();
+  const shouldShow = currentPlan?.v3Usage.hasExceededFreeTier === true;
+  return { shouldShow };
+}

apps/webapp/app/components/billing/v3/UsageBar.tsx

@@ -0,0 +1,171 @@
+import { cn } from "~/utils/cn";
+import { formatCurrency } from "~/utils/numberFormatter";
+import { Paragraph } from "../../primitives/Paragraph";
+import { SimpleTooltip } from "../../primitives/Tooltip";
+import { motion } from "framer-motion";
+
+type UsageBarProps = {
+  current: number;
+  billingLimit?: number;
+  tierLimit?: number;
+  projectedUsage?: number;
+  isPaying: boolean;
+};
+
+const startFactor = 4;
+
+export function UsageBar({
+  current,
+  billingLimit,
+  tierLimit,
+  projectedUsage,
+  isPaying,
+}: UsageBarProps) {
+  const getLargestNumber = Math.max(
+    current,
+    tierLimit ?? -Infinity,
+    projectedUsage ?? -Infinity,
+    billingLimit ?? -Infinity,
+    5
+  );
+  //creates a maximum range for the progress bar, add 10% to the largest number so the bar doesn't reach the end
+  const maxRange = Math.round(getLargestNumber * 1.1);
+  const tierRunLimitPercentage = tierLimit ? Math.round((tierLimit / maxRange) * 100) : 0;
+  const projectedRunsPercentage = projectedUsage
+    ? Math.round((projectedUsage / maxRange) * 100)
+    : 0;
+  const billingLimitPercentage =
+    billingLimit !== undefined ? Math.round((billingLimit / maxRange) * 100) : 0;
+  const usagePercentage = Math.round((current / maxRange) * 100);
+
+  //cap the usagePercentage to the freeRunLimitPercentage
+  const usageCappedToLimitPercentage = Math.min(usagePercentage, tierRunLimitPercentage);
+
+  return (
+    <div className="h-fit w-full py-12">
+      <div className="relative h-3 w-full rounded-sm bg-background-bright">
+        {billingLimit !== undefined && (
+          <motion.div
+            initial={{ width: billingLimitPercentage / startFactor + "%" }}
+            animate={{ width: billingLimitPercentage + "%" }}
+            transition={{ duration: 1.5, type: "spring" }}
+            style={{ width: `${billingLimitPercentage}%` }}
+            className="absolute h-3 rounded-l-sm"
+          >
+            <Legend
+              text="Billing limit:"
+              value={formatCurrency(billingLimit, false)}
+              position="bottomRow2"
+              percentage={billingLimitPercentage}
+            />
+          </motion.div>
+        )}
+        <motion.div
+          initial={{ width: usagePercentage / startFactor + "%" }}
+          animate={{ width: usagePercentage + "%" }}
+          transition={{ duration: 1.5, type: "spring" }}
+          style={{ width: `${usagePercentage}%` }}
+          className={cn(
+            "absolute h-3 rounded-l-sm",
+            tierLimit && current > tierLimit ? "bg-green-700" : "bg-green-600"
+          )}
+        >
+          <Legend
+            text="Used:"
+            value={formatCurrency(current, false)}
+            position="topRow1"
+            percentage={usagePercentage}
+          />
+        </motion.div>
+        {tierLimit !== undefined && (
+          <motion.div
+            initial={{ width: tierRunLimitPercentage / startFactor + "%" }}
+            animate={{ width: tierRunLimitPercentage + "%" }}
+            transition={{ duration: 1.5, type: "spring" }}
+            style={{ width: `${tierRunLimitPercentage}%` }}
+            className="absolute h-3 rounded-l-sm bg-green-900/50"
+          >
+            <Legend
+              text={isPaying ? `Included usage:` : `Tier limit:`}
+              value={formatCurrency(tierLimit, false)}
+              position="bottomRow1"
+              percentage={tierRunLimitPercentage}
+            />
+          </motion.div>
+        )}
+        {projectedUsage !== undefined && projectedUsage !== 0 && (
+          <motion.div
+            initial={{ width: projectedRunsPercentage / startFactor + "%" }}
+            animate={{ width: projectedRunsPercentage + "%" }}
+            transition={{ duration: 1.5, type: "spring" }}
+            style={{ width: `${projectedRunsPercentage}%` }}
+            className="absolute h-3 rounded-l-sm"
+          >
+            <Legend
+              text="Projected:"
+              value={formatCurrency(projectedUsage, false)}
+              position="topRow2"
+              percentage={projectedRunsPercentage}
+            />
+          </motion.div>
+        )}
+
+        <motion.div
+          initial={{ width: usageCappedToLimitPercentage / startFactor + "%" }}
+          animate={{ width: usageCappedToLimitPercentage + "%" }}
+          transition={{ duration: 1.5, type: "spring" }}
+          style={{ width: `${usageCappedToLimitPercentage}%` }}
+          className="absolute h-3 rounded-l-sm bg-green-600"
+        />
+      </div>
+    </div>
+  );
+}
+
+const positions = {
+  topRow1: "bottom-0 h-9",
+  topRow2: "bottom-0 h-14",
+  bottomRow1: "top-0 h-9 items-end",
+  bottomRow2: "top-0 h-14 items-end",
+};
+
+type LegendProps = {
+  text: string;
+  value: number | string;
+  percentage: number;
+  position: keyof typeof positions;
+  tooltipContent?: string;
+};
+
+function Legend({ text, value, position, percentage, tooltipContent }: LegendProps) {
+  const flipLegendPositionValue = 80;
+  const flipLegendPosition = percentage > flipLegendPositionValue ? true : false;
+  return (
+    <div
+      className={cn(
+        "absolute left-full z-10 flex border-charcoal-500",
+        positions[position],
+        flipLegendPosition === true ? "-translate-x-full border-r" : "border-l"
+      )}
+    >
+      {tooltipContent ? (
+        <SimpleTooltip
+          button={
+            <Paragraph className="mr-px h-fit whitespace-nowrap bg-background-bright px-1.5 text-xs text-text-bright">
+              {text}
+              <span className="ml-1 text-text-dimmed">{value}</span>
+            </Paragraph>
+          }
+          side="top"
+          content={tooltipContent}
+          className="z-50 h-fit"
+        />
+      ) : (
+        <Paragraph className="mr-px h-fit whitespace-nowrap bg-background-bright px-1.5 text-xs text-text-bright">
+          {text}
+          <span className="ml-1 text-text-dimmed">{value}</span>
+        </Paragraph>
+      )}
+    </div>
+  );
+}

apps/webapp/app/components/code/CodeBlock.tsx

@@ -1,14 +1,23 @@
 import { Clipboard, ClipboardCheck } from "lucide-react";
 import type { Language, PrismTheme } from "prism-react-renderer";
-import Highlight, { defaultProps } from "prism-react-renderer";
-import { forwardRef, useCallback, useState } from "react";
+import { Highlight, Prism } from "prism-react-renderer";
+import { forwardRef, ReactNode, useCallback, useState } from "react";
 import { cn } from "~/utils/cn";
-import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "../primitives/Tooltip";
 import { Paragraph } from "../primitives/Paragraph";
+import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "../primitives/Tooltip";
 
 //This is a fork of https://github.com/mantinedev/mantine/blob/master/src/mantine-prism/src/Prism/Prism.tsx
 //it didn't support highlighting lines by dimming the rest of the code, or animations on the highlighting
 
+async function setup() {
+  (typeof global !== "undefined" ? global : window).Prism = Prism;
+  //@ts-ignore
+  await import("prismjs/components/prism-json");
+  //@ts-ignore
+  await import("prismjs/components/prism-typescript");
+}
+setup();
+
 type CodeBlockProps = {
   /** Code which will be highlighted */
   code: string;
@@ -49,103 +58,116 @@ const extraLinesWhenClipping = 0.35;
 
 const defaultTheme: PrismTheme = {
   plain: {
-    color: "#9CDCFE",
+    color: "#9C9AF2",
     backgroundColor: "rgba(0, 0, 0, 0)",
   },
   styles: [
     {
-      types: ["prolog"],
+      types: ["comment", "prolog", "doctype", "cdata"],
       style: {
-        color: "rgb(0, 0, 128)",
+        color: "#5F6570",
       },
     },
     {
-      types: ["comment"],
-      style: {
-        color: "rgb(106, 153, 85)",
-      },
-    },
-    {
-      types: ["builtin", "changed", "keyword", "interpolation-punctuation"],
-      style: {
-        color: "rgb(86, 156, 214)",
-      },
-    },
-    {
-      types: ["number", "inserted"],
-      style: {
-        color: "rgb(181, 206, 168)",
-      },
-    },
-    {
-      types: ["constant"],
-      style: {
-        color: "rgb(100, 102, 149)",
-      },
-    },
-    {
-      types: ["attr-name", "variable"],
-      style: {
-        color: "rgb(156, 220, 254)",
-      },
-    },
-    {
-      types: ["deleted", "string", "attr-value", "template-punctuation"],
-      style: {
-        color: "rgb(206, 145, 120)",
-      },
-    },
-    {
-      types: ["selector"],
-      style: {
-        color: "rgb(215, 186, 125)",
-      },
-    },
-    {
-      // Fix tag color
-      types: ["tag"],
-      style: {
-        color: "rgb(78, 201, 176)",
-      },
-    },
-    {
-      // Fix tag color for HTML
-      types: ["tag"],
-      languages: ["markup"],
-      style: {
-        color: "rgb(86, 156, 214)",
-      },
-    },
-    {
-      types: ["punctuation", "operator"],
-      style: {
-        color: "rgb(212, 212, 212)",
-      },
-    },
-    {
-      // Fix punctuation color for HTML
       types: ["punctuation"],
-      languages: ["markup"],
       style: {
-        color: "#808080",
+        color: "#878C99",
       },
     },
     {
-      types: ["function"],
+      types: ["property", "tag", "boolean", "number", "constant", "symbol", "deleted"],
       style: {
-        color: "rgb(220, 220, 170)",
+        color: "#9B99FF",
       },
     },
     {
-      types: ["class-name"],
+      types: ["selector", "attr-name", "string", "char", "builtin", "inserted"],
       style: {
-        color: "rgb(78, 201, 176)",
+        color: "#AFEC73",
+      },
+    },
+    {
+      types: ["operator", "entity", "url"],
+      style: {
+        color: "#D4D4D4",
+      },
+    },
+    {
+      types: ["variable"],
+      style: {
+        color: "#CCCBFF",
+      },
+    },
+    {
+      types: ["atrule", "attr-value", "keyword"],
+      style: {
+        color: "#E888F8",
+      },
+    },
+    {
+      types: ["function", "class-name"],
+      style: {
+        color: "#D9F07C",
+      },
+    },
+    {
+      types: ["regex"],
+      style: {
+        color: "#d16969",
+      },
+    },
+    {
+      types: ["important", "bold"],
+      style: {
+        fontWeight: "bold",
+      },
+    },
+    {
+      types: ["italic"],
+      style: {
+        fontStyle: "italic",
+      },
+    },
+    {
+      types: ["namespace"],
+      style: {
+        opacity: 0.7,
+      },
+    },
+    {
+      types: ["deleted"],
+      style: {
+        color: "#F85149",
+      },
+    },
+    {
+      types: ["boolean"],
+      style: {
+        color: "#9B99FF",
       },
     },
     {
       types: ["char"],
       style: {
-        color: "rgb(209, 105, 105)",
+        color: "#b5cea8",
+      },
+    },
+    {
+      types: ["tag"],
+      style: {
+        color: "#D7BA7D",
+      },
+    },
+    {
+      types: ["keyword.operator"],
+      style: {
+        color: "#8271ED",
+      },
+    },
+    {
+      types: ["meta.template.expression"],
+      style: {
+        color: "#d4d4d4",
       },
     },
   ],
@@ -219,16 +241,12 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
                 onMouseEnter={() => setMouseOver(true)}
                 onMouseLeave={() => setMouseOver(false)}
                 className={cn(
-                  "absolute  right-3 z-50 transition-colors duration-100 hover:cursor-pointer",
-                  showChrome ? "top-10" : "top-3",
+                  "absolute right-3 z-50 transition-colors duration-100 hover:cursor-pointer",
+                  showChrome ? "top-10" : "top-2.5",
                   copied ? "text-emerald-500" : "text-charcoal-500 hover:text-charcoal-300"
                 )}
               >
-                {copied ? (
-                  <ClipboardCheck className="h-5 w-5" />
-                ) : (
-                  <Clipboard className="h-5 w-5" />
-                )}
+                {copied ? <ClipboardCheck className="size-4" /> : <Clipboard className="size-4" />}
               </TooltipTrigger>
               <TooltipContent side="left" className="text-xs">
                 {copied ? "Copied" : "Copy"}
@@ -238,7 +256,7 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
         )}
 
         {shouldHighlight ? (
-          <Highlight {...defaultProps} theme={theme} code={code} language={language}>
+          <Highlight theme={theme} code={code} language={language}>
             {({
               className: inheritedClassName,
               style: inheritedStyle,
@@ -283,7 +301,7 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
 
                       return (
                         <div
-                          key={lineProps.key}
+                          key={lineNumber}
                           {...lineProps}
                           className={cn(
                             "flex w-full justify-start transition-opacity duration-500",
@@ -312,7 +330,7 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
                               const tokenProps = getTokenProps({ token, key });
                               return (
                                 <span
-                                  key={tokenProps.key}
+                                  key={key}
                                   {...tokenProps}
                                   style={{
                                     color: tokenProps?.style?.color as string,
@@ -367,10 +385,10 @@ function Chrome({ title }: { title?: string }) {
   );
 }
 
-export function TitleRow({ title }: { title: string }) {
+export function TitleRow({ title }: { title: ReactNode }) {
   return (
-    <div className="flex items-center justify-between px-4">
-      <Paragraph variant="base/bright" className="w-full border-b border-grid-dimmed py-2.5">
+    <div className="flex items-center justify-between px-3">
+      <Paragraph variant="small/bright" className="w-full border-b border-grid-dimmed py-2">
         {title}
       </Paragraph>
     </div>

apps/webapp/app/components/code/InlineCode.tsx

@@ -1,7 +1,7 @@
 import { cn } from "~/utils/cn";
 
 const inlineCode =
-  "px-1 py-0.5 rounded border border-charcoal-700 bg-charcoal-800 text-charcoal-200 font-mono";
+  "px-1 py-0.5 rounded border border-charcoal-700 bg-charcoal-800 text-charcoal-200 font-mono text-wrap";
 
 const variants = {
   "extra-extra-small": "text-xxs",

apps/webapp/app/components/code/JSONEditor.tsx

@@ -107,39 +107,51 @@ export function JSONEditor(opts: JSONEditorProps) {
     }, 1500);
   }, [view]);
 
+  const showButtons = showClearButton || showCopyButton;
+
   return (
-    <div className={cn(opts.className, "grid grid-rows-[2.5rem_1fr]")}>
-      <div className="mx-3 flex items-center justify-end gap-2 border-b border-grid-dimmed">
-        {showClearButton && (
-          <Button
-            type="button"
-            variant="minimal/small"
-            TrailingIcon={TrashIcon}
-            onClick={(event) => {
-              event.preventDefault();
-              event.stopPropagation();
-              clear();
-            }}
-          >
-            Clear
-          </Button>
-        )}
-        {showCopyButton && (
-          <Button
-            type="button"
-            variant="minimal/small"
-            TrailingIcon={copied ? CheckIcon : ClipboardIcon}
-            trailingIconClassName={copied ? "text-green-500 group-hover:text-green-500" : undefined}
-            onClick={(event) => {
-              event.preventDefault();
-              event.stopPropagation();
-              copy();
-            }}
-          >
-            Copy
-          </Button>
-        )}
-      </div>
+    <div
+      className={cn(
+        opts.className,
+        "grid",
+        showButtons ? "grid-rows-[2.5rem_1fr]" : "grid-rows-[1fr]"
+      )}
+    >
+      {showButtons && (
+        <div className="mx-3 flex items-center justify-end gap-2 border-b border-grid-dimmed">
+          {showClearButton && (
+            <Button
+              type="button"
+              variant="minimal/small"
+              TrailingIcon={TrashIcon}
+              onClick={(event) => {
+                event.preventDefault();
+                event.stopPropagation();
+                clear();
+              }}
+            >
+              Clear
+            </Button>
+          )}
+          {showCopyButton && (
+            <Button
+              type="button"
+              variant="minimal/small"
+              TrailingIcon={copied ? CheckIcon : ClipboardIcon}
+              trailingIconClassName={
+                copied ? "text-green-500 group-hover:text-green-500" : undefined
+              }
+              onClick={(event) => {
+                event.preventDefault();
+                event.stopPropagation();
+                copy();
+              }}
+            >
+              Copy
+            </Button>
+          )}
+        </div>
+      )}
       <div
         className="w-full overflow-auto"
         ref={editor}

apps/webapp/app/components/environments/EnvironmentLabel.tsx

@@ -1,5 +1,7 @@
-import { RuntimeEnvironment } from "~/models/runtimeEnvironment.server";
+import type { RuntimeEnvironment } from "~/models/runtimeEnvironment.server";
 import { cn } from "~/utils/cn";
+import { sortEnvironments } from "~/utils/environmentSort";
+import { SimpleTooltip } from "../primitives/Tooltip";
 
 type Environment = Pick<RuntimeEnvironment, "type">;
 const variants = {
@@ -7,6 +9,30 @@ const variants = {
   large: "h-6 text-xs px-1.5 rounded",
 };
 
+export function EnvironmentTypeLabel({
+  environment,
+  size = "small",
+  className,
+}: {
+  environment: Environment;
+  size?: keyof typeof variants;
+  className?: string;
+}) {
+  return (
+    <span
+      className={cn(
+        "text-midnight-900 inline-flex items-center justify-center whitespace-nowrap border font-medium uppercase tracking-wider",
+        environmentBorderClassName(environment),
+        environmentTextClassName(environment),
+        variants[size],
+        className
+      )}
+    >
+      {environmentTypeTitle(environment)}
+    </span>
+  );
+}
+
 export function EnvironmentLabel({
   environment,
   size = "small",
@@ -33,6 +59,74 @@ export function EnvironmentLabel({
   );
 }
 
+type EnvironmentWithUsername = Environment & { userName?: string };
+
+export function EnvironmentLabels({
+  environments,
+  size = "small",
+  className,
+}: {
+  environments: EnvironmentWithUsername[];
+  size?: keyof typeof variants;
+  className?: string;
+}) {
+  const devEnvironments = sortEnvironments(
+    environments.filter((env) => env.type === "DEVELOPMENT")
+  );
+  const firstDevEnvironment = devEnvironments[0];
+  const otherDevEnvironments = devEnvironments.slice(1);
+  const otherEnvironments = environments.filter((env) => env.type !== "DEVELOPMENT");
+
+  return (
+    <div className={cn("flex items-baseline gap-2", className)}>
+      {firstDevEnvironment && (
+        <EnvironmentLabel
+          environment={firstDevEnvironment}
+          userName={firstDevEnvironment.userName}
+          size={size}
+        />
+      )}
+      {otherDevEnvironments.length > 0 ? (
+        <SimpleTooltip
+          disableHoverableContent
+          button={
+            <span
+              className={cn(
+                "inline-flex items-center justify-center whitespace-nowrap border font-medium uppercase tracking-wider",
+                environmentBorderClassName({ type: "DEVELOPMENT" }),
+                environmentTextClassName({ type: "DEVELOPMENT" }),
+                variants[size]
+              )}
+            >
+              +{otherDevEnvironments.length}
+            </span>
+          }
+          content={
+            <div className="flex gap-1 py-1">
+              {otherDevEnvironments.map((environment, index) => (
+                <EnvironmentLabel
+                  key={index}
+                  environment={environment}
+                  userName={environment.userName}
+                  size={size}
+                />
+              ))}
+            </div>
+          }
+        />
+      ) : null}
+      {otherEnvironments.map((environment, index) => (
+        <EnvironmentLabel
+          key={index}
+          environment={environment}
+          userName={environment.userName}
+          size={size}
+        />
+      ))}
+    </div>
+  );
+}
+
 export function environmentTitle(environment: Environment, username?: string) {
   switch (environment.type) {
     case "PRODUCTION":
@@ -46,6 +140,19 @@ export function environmentTitle(environment: Environment, username?: string) {
   }
 }
 
+export function environmentTypeTitle(environment: Environment) {
+  switch (environment.type) {
+    case "PRODUCTION":
+      return "Prod";
+    case "STAGING":
+      return "Staging";
+    case "DEVELOPMENT":
+      return "Dev";
+    case "PREVIEW":
+      return "Preview";
+  }
+}
+
 export function environmentColorClassName(environment: Environment) {
   switch (environment.type) {
     case "PRODUCTION":

apps/webapp/app/components/events/EventsFilters.tsx

@@ -9,7 +9,7 @@ import {
   SelectItem,
   SelectTrigger,
   SelectValue,
-} from "../primitives/Select";
+} from "../primitives/SimpleSelect";
 import { EventListSearchSchema } from "./EventStatuses";
 import { environmentKeys, FilterableEnvironment } from "~/components/runs/RunStatuses";
 import { TimeFrameFilter } from "../runs/TimeFrameFilter";

apps/webapp/app/components/integrations/ConnectToOAuthForm.tsx

@@ -12,7 +12,7 @@ import { ApiAuthenticationMethodOAuth2, Integration, Scope } from "~/services/ex
 import { cn } from "~/utils/cn";
 import { CodeBlock } from "../code/CodeBlock";
 import { Button } from "../primitives/Buttons";
-import { Checkbox } from "../primitives/Checkbox";
+import { CheckboxWithLabel } from "../primitives/Checkbox";
 import { Fieldset } from "../primitives/Fieldset";
 import { FormError } from "../primitives/FormError";
 import { Header2, Header3 } from "../primitives/Headers";
@@ -123,7 +123,7 @@ export function ConnectToOAuthForm({
           <Paragraph variant="small" className="mb-2">
             To use your own OAuth app, check the option below and insert the details.
           </Paragraph>
-          <Checkbox
+          <CheckboxWithLabel
             id="hasCustomClient"
             label="Use my OAuth App"
             variant="simple/small"
@@ -200,7 +200,7 @@ export function ConnectToOAuthForm({
               )}
               {authMethod.scopes.map((s) => {
                 return (
-                  <Checkbox
+                  <CheckboxWithLabel
                     key={s.name}
                     id={s.name}
                     value={s.name}

apps/webapp/app/components/integrations/UpdateOAuthForm.tsx

@@ -10,7 +10,7 @@ import { ApiAuthenticationMethodOAuth2, Integration, Scope } from "~/services/ex
 import { cn } from "~/utils/cn";
 import { CodeBlock } from "../code/CodeBlock";
 import { Button } from "../primitives/Buttons";
-import { Checkbox } from "../primitives/Checkbox";
+import { CheckboxWithLabel } from "../primitives/Checkbox";
 import { Fieldset } from "../primitives/Fieldset";
 import { FormError } from "../primitives/FormError";
 import { Header2, Header3 } from "../primitives/Headers";
@@ -113,7 +113,7 @@ export function UpdateOAuthForm({
           <Paragraph variant="small" className="mb-2">
             To use your own OAuth app, check the option below and insert the details.
           </Paragraph>
-          <Checkbox
+          <CheckboxWithLabel
             id="hasCustomClient"
             label="Use my OAuth App"
             variant="simple/small"
@@ -189,7 +189,7 @@ export function UpdateOAuthForm({
               )}
               {authMethod.scopes.map((s) => {
                 return (
-                  <Checkbox
+                  <CheckboxWithLabel
                     key={s.name}
                     id={s.name}
                     value={s.name}

apps/webapp/app/components/layout/AppLayout.tsx

@@ -1,6 +1,6 @@
 import { useOptionalOrganization } from "~/hooks/useOrganizations";
 import { cn } from "~/utils/cn";
-import { useShowUpgradePrompt } from "../billing/UpgradePrompt";
+import { useShowUpgradePrompt } from "../billing/v3/UpgradePrompt";
 
 /** This container is used to surround the entire app, it correctly places the nav bar */
 export function AppContainer({ children }: { children: React.ReactNode }) {
@@ -41,7 +41,7 @@ export function PageBody({
     <div
       className={cn(
         scrollable
-          ? "overflow-y-auto p-4 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600"
+          ? "overflow-y-auto p-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600"
           : "overflow-hidden",
         className
       )}
@@ -60,7 +60,9 @@ export function MainCenteredContainer({
 }) {
   return (
     <div className="h-full w-full overflow-y-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600">
-      <div className={cn("mx-auto mt-[25vh] max-w-xs overflow-y-auto", className)}>{children}</div>
+      <div className={cn("mx-auto mt-6 max-w-xs overflow-y-auto p-1 md:mt-[22vh]", className)}>
+        {children}
+      </div>
     </div>
   );
 }

apps/webapp/app/components/navigation/SideMenu.tsx

@@ -1,14 +1,18 @@
 import {
   AcademicCapIcon,
-  ArrowRightIcon,
   ArrowRightOnRectangleIcon,
   BeakerIcon,
+  BellAlertIcon,
   ChartBarIcon,
+  ClockIcon,
+  CreditCardIcon,
   CursorArrowRaysIcon,
   IdentificationIcon,
   KeyIcon,
+  RectangleStackIcon,
   ServerStackIcon,
   ShieldCheckIcon,
+  SignalIcon,
 } from "@heroicons/react/20/solid";
 import { UserGroupIcon, UserPlusIcon } from "@heroicons/react/24/solid";
 import { useNavigation } from "@remix-run/react";
@@ -16,9 +20,9 @@ import { DiscordIcon, SlackIcon } from "@trigger.dev/companyicons";
 import { Fragment, useEffect, useRef, useState } from "react";
 import { TaskIcon } from "~/assets/icons/TaskIcon";
 import { useFeatures } from "~/hooks/useFeatures";
-import { MatchedOrganization } from "~/hooks/useOrganizations";
-import { MatchedProject } from "~/hooks/useProject";
-import { User } from "~/models/user.server";
+import { type MatchedOrganization } from "~/hooks/useOrganizations";
+import { type MatchedProject } from "~/hooks/useProject";
+import { type User } from "~/models/user.server";
 import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route";
 import { cn } from "~/utils/cn";
 import {
@@ -42,21 +46,26 @@ import {
   projectSetupPath,
   projectTriggersPath,
   v3ApiKeysPath,
+  v3BillingPath,
+  v3ConcurrencyPath,
   v3DeploymentsPath,
   v3EnvironmentVariablesPath,
+  v3ProjectAlertsPath,
   v3ProjectPath,
   v3ProjectSettingsPath,
   v3RunsPath,
+  v3SchedulesPath,
   v3TestPath,
+  v3UsagePath,
 } from "~/utils/pathBuilder";
 import { Feedback } from "../Feedback";
 import { ImpersonationBanner } from "../ImpersonationBanner";
 import { LogoIcon } from "../LogoIcon";
 import { StepContentContainer } from "../StepContentContainer";
 import { UserProfilePhoto } from "../UserProfilePhoto";
-import { FreePlanUsage } from "../billing/FreePlanUsage";
+import { FreePlanUsage } from "../billing/v2/FreePlanUsage";
 import { Badge } from "../primitives/Badge";
-import { Button } from "../primitives/Buttons";
+import { Button, LinkButton } from "../primitives/Buttons";
 import { ClipboardField } from "../primitives/ClipboardField";
 import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "../primitives/Dialog";
 import { Icon } from "../primitives/Icon";
@@ -70,20 +79,12 @@ import {
   PopoverSectionHeader,
 } from "../primitives/Popover";
 import { StepNumber } from "../primitives/StepNumber";
+import { TextLink } from "../primitives/TextLink";
 import { SideMenuHeader } from "./SideMenuHeader";
-import { MenuCount, SideMenuItem } from "./SideMenuItem";
+import { SideMenuItem } from "./SideMenuItem";
 
 type SideMenuUser = Pick<User, "email" | "admin"> & { isImpersonating: boolean };
-type SideMenuProject = Pick<
-  MatchedProject,
-  | "id"
-  | "name"
-  | "slug"
-  | "hasInactiveExternalTriggers"
-  | "jobCount"
-  | "httpEndpointCount"
-  | "version"
->;
+type SideMenuProject = Pick<MatchedProject, "id" | "name" | "slug" | "version">;
 
 type SideMenuProps = {
   user: SideMenuUser;
@@ -95,8 +96,11 @@ type SideMenuProps = {
 export function SideMenu({ user, project, organization, organizations }: SideMenuProps) {
   const borderRef = useRef<HTMLDivElement>(null);
   const [showHeaderDivider, setShowHeaderDivider] = useState(false);
-  const { isManagedCloud } = useFeatures();
   const currentPlan = useCurrentPlan();
+  const { isManagedCloud } = useFeatures();
+
+  const isV3Project = project.version === "V3";
+  const isFreeV3User = currentPlan?.v3Subscription?.isPaying === false;
 
   useEffect(() => {
     const handleScroll = () => {
@@ -155,7 +159,6 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
                 icon="integration"
                 to={organizationIntegrationsPath(organization)}
                 data-action="integrations"
-                hasWarning={organization.hasUnconfiguredIntegrations}
               />
             )}
             <SideMenuItem
@@ -168,16 +171,41 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
               name="Team"
               icon={UserGroupIcon}
               to={organizationTeamPath(organization)}
-              iconColor="text-sky-500"
+              iconColor="text-amber-500"
               data-action="team"
             />
-            <SideMenuItem
-              name={isManagedCloud ? "Usage & Billing" : "Usage"}
-              icon={ChartBarIcon}
-              to={organizationBillingPath(organization)}
-              iconColor="text-green-600"
-              data-action="usage & billing"
-            />
+            {organization.projects.some((proj) => proj.version === "V3") && isManagedCloud && (
+              <>
+                <SideMenuItem
+                  name="Usage"
+                  icon={ChartBarIcon}
+                  to={v3UsagePath(organization)}
+                  iconColor="text-green-600"
+                  data-action="usage"
+                />
+                <SideMenuItem
+                  name="Billing"
+                  icon={CreditCardIcon}
+                  to={v3BillingPath(organization)}
+                  iconColor="text-blue-600"
+                  data-action="billing"
+                  badge={
+                    currentPlan?.v3Subscription?.isPaying
+                      ? currentPlan?.v3Subscription?.plan?.title
+                      : undefined
+                  }
+                />
+              </>
+            )}
+            {organization.projects.some((proj) => proj.version === "V2") && (
+              <SideMenuItem
+                name="Usage (v2)"
+                icon={ChartBarIcon}
+                to={organizationBillingPath(organization)}
+                iconColor="text-green-600"
+                data-action="usage & billing"
+              />
+            )}
             <SideMenuItem
               name="Organization settings"
               icon="settings"
@@ -187,8 +215,31 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
             />
           </div>
         </div>
+        <div className="m-2">
+          {project.version === "V2" && (
+            <div className="flex flex-col gap-3 rounded border border-success/50 bg-success/10 p-3">
+              <Paragraph variant="small/bright">
+                This is a v2 project. V2 will be deprecated on January 31, 2025.{" "}
+                <TextLink
+                  className="text-text-bright underline decoration-text-dimmed underline-offset-2 transition hover:text-text-bright hover:decoration-text-bright"
+                  to="https://trigger.dev/blog/v2-end-of-life-announcement"
+                >
+                  Learn more
+                </TextLink>
+                .
+              </Paragraph>
+              <LinkButton
+                variant="primary/medium"
+                to="https://trigger.dev/docs/v3/upgrading-from-v2"
+                fullWidth
+              >
+                Upgrade to v3
+              </LinkButton>
+            </div>
+          )}
+        </div>
         <div className="flex flex-col gap-1 border-t border-grid-bright p-1">
-          {currentPlan?.subscription?.isPaying === true && (
+          {currentPlan?.v3Subscription?.plan?.limits.support === "slack" && (
             <Dialog>
               <DialogTrigger asChild>
                 <Button
@@ -213,31 +264,22 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
                   </div>
                   <hr className="border-charcoal-800" />
                   <div>
-                    <StepNumber stepNumber="1" title="Create a new Slack channel" />
+                    <StepNumber stepNumber="1" title="Email us" />
                     <StepContentContainer>
                       <Paragraph>
-                        In your Slack app, create a new channel from the main menu by going to File{" "}
-                        <ArrowRightIcon className="inline h-4 w-4 text-text-dimmed" /> New Channel
-                      </Paragraph>
-                    </StepContentContainer>
-                    <StepNumber stepNumber="2" title="Setup your channel" />
-                    <StepContentContainer>
-                      <Paragraph>
-                        Name your channel, set its visibility and click 'Create'.
-                      </Paragraph>
-                    </StepContentContainer>
-                    <StepNumber stepNumber="3" title="Invite Trigger.dev" />
-                    <StepContentContainer>
-                      <Paragraph>
-                        Invite this email address to your channel:{" "}
+                        Send us an email to this address from your Trigger.dev account email
+                        address:
                         <ClipboardField
                           variant="primary/medium"
-                          value="james@trigger.dev"
+                          value="priority-support@trigger.dev"
                           className="my-2"
                         />
                       </Paragraph>
+                    </StepContentContainer>
+                    <StepNumber stepNumber="2" title="Look out for an invite from Slack" />
+                    <StepContentContainer>
                       <Paragraph>
-                        As soon as we can, we'll accept your invitation and say hello!
+                        As soon as we can, we'll setup a Slack Connect channel and say hello!
                       </Paragraph>
                     </StepContentContainer>
                   </div>
@@ -252,23 +294,13 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
             data-action="join our discord"
             target="_blank"
           />
-          {project.version === "V2" ? (
-            <SideMenuItem
-              name="Documentation"
-              icon="docs"
-              to="https://trigger.dev/docs"
-              data-action="documentation"
-              target="_blank"
-            />
-          ) : (
-            <SideMenuItem
-              name="Documentation (v3)"
-              icon="docs"
-              to="https://trigger.dev/docs/v3"
-              data-action="documentation"
-              target="_blank"
-            />
-          )}
+          <SideMenuItem
+            name="Documentation"
+            icon="docs"
+            to="https://trigger.dev/docs"
+            data-action="documentation"
+            target="_blank"
+          />
           <SideMenuItem
             name="Changelog"
             icon="star"
@@ -276,41 +308,31 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
             data-action="changelog"
             target="_blank"
           />
-          {project.version === "V2" ? (
-            <Feedback
-              button={
-                <Button
-                  variant="small-menu-item"
-                  LeadingIcon="log"
-                  data-action="help & feedback"
-                  fullWidth
-                  textAlignLeft
-                >
-                  Help & Feedback
-                </Button>
-              }
-            />
-          ) : (
-            <Feedback
-              defaultValue="developer preview"
-              button={
-                <Button
-                  variant="small-menu-item"
-                  LeadingIcon="log"
-                  leadingIconClassName="text-primary"
-                  data-action="help & feedback"
-                  fullWidth
-                  textAlignLeft
-                >
-                  <span className="text-primary">Give feedback on v3</span>
-                </Button>
-              }
-            />
-          )}
-          {currentPlan && !currentPlan.subscription?.isPaying && currentPlan.usage.runCountCap && (
+          <SideMenuItem
+            name="Status"
+            icon={SignalIcon}
+            to="https://status.trigger.dev/"
+            data-action="status"
+            target="_blank"
+          />
+          <Feedback
+            button={
+              <Button
+                variant="small-menu-item"
+                LeadingIcon="log"
+                leadingIconClassName="text-primary"
+                data-action="help & feedback"
+                fullWidth
+                textAlignLeft
+              >
+                <span className="text-primary">Help & Feedback</span>
+              </Button>
+            }
+          />
+          {isV3Project && isFreeV3User && (
             <FreePlanUsage
-              to={organizationBillingPath(organization)}
-              percentage={currentPlan.usage.currentRunCount / currentPlan.usage.runCountCap}
+              to={v3BillingPath(organization)}
+              percentage={currentPlan.v3Usage.usagePercentage}
             />
           )}
         </div>
@@ -362,10 +384,10 @@ function ProjectSelector({
                       title={
                         <div className="flex w-full items-center justify-between text-text-bright">
                           <span className="grow truncate text-left">{p.name}</span>
-                          {p.version === "V2" ? (
-                            <MenuCount count={p.jobCount} />
-                          ) : (
-                            <Badge variant="v3">v3</Badge>
+                          {p.version === "V2" && (
+                            <Badge variant="small" className="normal-case">
+                              v2
+                            </Badge>
                           )}
                         </div>
                       }
@@ -384,7 +406,7 @@ function ProjectSelector({
             </div>
           </Fragment>
         ))}
-        <div className="border-t border-charcoal-800 p-1">
+        <div className="border-t border-charcoal-700 p-1">
           <PopoverMenuItem to={newOrganizationPath()} title="New Organization" icon="plus" />
         </div>
       </PopoverContent>
@@ -463,7 +485,7 @@ function V2ProjectSideMenu({
 }) {
   return (
     <>
-      <SideMenuHeader title={"Project"}>
+      <SideMenuHeader title={"Project (v2)"}>
         <PopoverMenuItem
           to={projectSetupPath(organization, project)}
           title="Framework setup"
@@ -474,7 +496,6 @@ function V2ProjectSideMenu({
         name="Jobs"
         icon="job"
         iconColor="text-indigo-500"
-        count={project.jobCount}
         to={projectPath(organization, project)}
         data-action="jobs"
       />
@@ -490,7 +511,6 @@ function V2ProjectSideMenu({
         iconColor="text-amber-500"
         to={projectTriggersPath(organization, project)}
         data-action="triggers"
-        hasWarning={project.hasInactiveExternalTriggers}
       />
       <SideMenuItem
         name="Events"
@@ -502,7 +522,6 @@ function V2ProjectSideMenu({
         name="HTTP endpoints"
         icon="http-endpoint"
         iconColor="text-pink-500"
-        count={project.httpEndpointCount}
         to={projectHttpEndpointsPath(organization, project)}
         data-action="httpendpoints"
       />
@@ -531,14 +550,15 @@ function V3ProjectSideMenu({
   project: SideMenuProject;
   organization: MatchedOrganization;
 }) {
+  const { alertsEnabled } = useFeatures();
+
   return (
     <>
-      <SideMenuHeader title={"Project (v3)"} />
+      <SideMenuHeader title={"Project"} />
       <SideMenuItem
         name="Tasks"
         icon={TaskIcon}
         iconColor="text-blue-500"
-        count={project.jobCount}
         to={v3ProjectPath(organization, project)}
         data-action="tasks"
       />
@@ -555,6 +575,13 @@ function V3ProjectSideMenu({
         to={v3TestPath(organization, project)}
         data-action="test"
       />
+      <SideMenuItem
+        name="Schedules"
+        icon={ClockIcon}
+        iconColor="text-sun-500"
+        to={v3SchedulesPath(organization, project)}
+        data-action="schedules"
+      />
       <SideMenuItem
         name="API keys"
         icon={KeyIcon}
@@ -569,6 +596,7 @@ function V3ProjectSideMenu({
         to={v3EnvironmentVariablesPath(organization, project)}
         data-action="environment variables"
       />
+
       <SideMenuItem
         name="Deployments"
         icon={ServerStackIcon}
@@ -576,6 +604,22 @@ function V3ProjectSideMenu({
         to={v3DeploymentsPath(organization, project)}
         data-action="deployments"
       />
+      {alertsEnabled && (
+        <SideMenuItem
+          name="Alerts"
+          icon={BellAlertIcon}
+          iconColor="text-red-500"
+          to={v3ProjectAlertsPath(organization, project)}
+          data-action="alerts"
+        />
+      )}
+      <SideMenuItem
+        name="Concurrency limits"
+        icon={RectangleStackIcon}
+        iconColor="text-indigo-500"
+        to={v3ConcurrencyPath(organization, project)}
+        data-action="concurrency"
+      />
       <SideMenuItem
         name="Project settings"
         icon="settings"

apps/webapp/app/components/navigation/SideMenuItem.tsx

@@ -1,8 +1,8 @@
-import { AnchorHTMLAttributes } from "react";
+import { type AnchorHTMLAttributes } from "react";
 import { usePathName } from "~/hooks/usePathName";
 import { cn } from "~/utils/cn";
 import { LinkButton } from "../primitives/Buttons";
-import { IconNames } from "../primitives/NamedIcon";
+import { type IconNames } from "../primitives/NamedIcon";
 import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "../primitives/Tooltip";
 import { Icon } from "../primitives/Icon";
 import { IconExclamationCircle } from "@tabler/icons-react";
@@ -13,7 +13,7 @@ export function SideMenuItem({
   name,
   to,
   hasWarning,
-  count,
+  badge,
   target,
   subItem = false,
 }: {
@@ -22,7 +22,7 @@ export function SideMenuItem({
   name: string;
   to: string;
   hasWarning?: string | boolean;
-  count?: number;
+  badge?: string;
   target?: AnchorHTMLAttributes<HTMLAnchorElement>["target"];
   subItem?: boolean;
 }) {
@@ -47,20 +47,20 @@ export function SideMenuItem({
       <div className="flex w-full items-center justify-between">
         {name}
         <div className="flex items-center gap-1">
-          {count !== undefined && count > 0 && <MenuCount count={count} />}
+          {badge !== undefined && <MenuCount count={badge} />}
           {typeof hasWarning === "string" ? (
             <TooltipProvider>
               <Tooltip>
                 <TooltipTrigger>
-                  <Icon icon={IconExclamationCircle} className="h-5 w-5 text-rose-500" />
+                  <Icon icon={IconExclamationCircle} className="h-5 w-5 text-error" />
                 </TooltipTrigger>
-                <TooltipContent className="flex items-center gap-1 border border-rose-500 bg-rose-500/20 backdrop-blur-xl">
+                <TooltipContent className="flex items-center gap-1 border border-error bg-error/20 backdrop-blur-xl">
                   {hasWarning}
                 </TooltipContent>
               </Tooltip>
             </TooltipProvider>
           ) : (
-            hasWarning && <Icon icon={IconExclamationCircle} className="h-5 w-5 text-rose-500" />
+            hasWarning && <Icon icon={IconExclamationCircle} className="h-5 w-5 text-error" />
           )}
         </div>
       </div>
@@ -68,8 +68,10 @@ export function SideMenuItem({
   );
 }
 
-export function MenuCount({ count }: { count: number | string }) {
+function MenuCount({ count }: { count: number | string }) {
   return (
-    <div className="rounded-full bg-charcoal-900 px-2 py-1 text-xxs text-text-dimmed">{count}</div>
+    <div className="rounded-full bg-charcoal-900 px-2 py-1 text-xxs uppercase tracking-wider text-text-dimmed">
+      {count}
+    </div>
   );
 }

apps/webapp/app/components/primitives/Alert.tsx

@@ -8,12 +8,8 @@ const Alert = AlertDialogPrimitive.Root;
 
 const AlertTrigger = AlertDialogPrimitive.Trigger;
 
-const AlertPortal = ({
-  className,
-  children,
-  ...props
-}: AlertDialogPrimitive.AlertDialogPortalProps) => (
-  <AlertDialogPrimitive.Portal className={cn(className)} {...props}>
+const AlertPortal = ({ children, ...props }: AlertDialogPrimitive.AlertDialogPortalProps) => (
+  <AlertDialogPrimitive.Portal {...props}>
     <div className="fixed inset-0 z-50 flex items-end justify-center sm:items-center">
       {children}
     </div>

apps/webapp/app/components/primitives/AppliedFilter.tsx

@@ -0,0 +1,56 @@
+import { XMarkIcon } from "@heroicons/react/20/solid";
+import { ReactNode } from "react";
+import { cn } from "~/utils/cn";
+
+const variants = {
+  "tertiary/small": {
+    box: "h-6 bg-tertiary rounded pl-1.5 gap-1.5 text-xs divide-x divide-black/15 group-hover:bg-charcoal-600",
+    clear: "size-6 text-text-dimmed hover:text-text-bright transition-colors",
+  },
+  "minimal/small": {
+    box: "h-6 hover:bg-tertiary rounded pl-1.5 gap-1.5 text-xs",
+    clear: "size-6 text-text-dimmed hover:text-text-bright transition-colors",
+  },
+};
+
+type Variant = keyof typeof variants;
+
+type AppliedFilterProps = {
+  label: ReactNode;
+  value: ReactNode;
+  removable?: boolean;
+  onRemove?: () => void;
+  variant?: Variant;
+  className?: string;
+};
+
+export function AppliedFilter({
+  label,
+  value,
+  removable = true,
+  onRemove,
+  variant = "tertiary/small",
+  className,
+}: AppliedFilterProps) {
+  const variantClassName = variants[variant];
+  return (
+    <div className={cn("flex items-center transition", variantClassName.box, className)}>
+      <div className="flex items-center gap-0.5">
+        <div className="text-text-dimmed">
+          <span>{label}</span>:
+        </div>
+        <div className="text-text-bright">
+          <div>{value}</div>
+        </div>
+      </div>
+      {removable && (
+        <button
+          className={cn("group flex size-6 items-center justify-center", variantClassName.clear)}
+          onClick={onRemove}
+        >
+          <XMarkIcon className="size-3.5" />
+        </button>
+      )}
+    </div>
+  );
+}

apps/webapp/app/components/primitives/Badge.tsx

@@ -8,7 +8,6 @@ const variants = {
     "grid place-items-center rounded-full px-[0.4rem] h-4 tracking-wider text-xxs bg-background-dimmed text-text-dimmed uppercase whitespace-nowrap",
   outline:
     "grid place-items-center rounded-sm px-1.5 h-5 tracking-wider text-xxs border border-dimmed text-text-dimmed uppercase whitespace-nowrap",
-  v3: "grid place-items-center rounded-full px-[0.4rem] h-5 tracking-wider text-xxs bg-charcoal-750 text-primary whitespace-nowrap",
   "outline-rounded":
     "grid place-items-center rounded-full px-1 h-4 tracking-wider text-xxs border border-blue-500 text-blue-500 uppercase whitespace-nowrap",
 };

apps/webapp/app/components/primitives/Buttons.tsx

@@ -21,7 +21,7 @@ const sizes = {
     shortcut: "-ml-0.5 -mr-1.5 rounded justify-self-center",
   },
   large: {
-    button: "h-10 px-2 text-sm font-medium",
+    button: "h-10 px-2 text-base font-medium",
     icon: "h-5",
     iconSpacing: "gap-x-0.5",
     shortcutVariant: "medium" as const,
@@ -48,11 +48,11 @@ const theme = {
       "border-black/40 text-charcoal-900 group-hover:border-black/60 group-hover:text-charcoal-900",
   },
   secondary: {
-    textColor: "text-primary group-hover:text-apple-200 transition group-disabled:text-primary",
+    textColor: "text-secondary group-hover:text-secondary transition group-disabled:text-secondary",
     button:
-      "bg-transparent border border-primary group-hover:border-apple-200 group-hover:bg-apple-950 group-disabled:opacity-30 group-disabled:border-primary group-disabled:bg-transparent group-disabled:pointer-events-none",
+      "bg-transparent border border-secondary group-hover:border-secondary group-hover:bg-secondary/10 group-disabled:opacity-30 group-disabled:border-secondary group-disabled:bg-transparent group-disabled:pointer-events-none",
     shortcut:
-      "border-primary/30 text-apple-200 group-hover:text-text-bright/80 group-hover:border-dimmed/60",
+      "border-secondary/30 text-secondary group-hover:text-text-bright/80 group-hover:border-dimmed/60",
   },
   tertiary: {
     textColor: "text-text-bright transition group-disabled:text-text-dimmed/80",
@@ -245,12 +245,12 @@ export function ButtonContent(props: ButtonContentPropsType) {
 
 type ButtonPropsType = Pick<
   JSX.IntrinsicElements["button"],
-  "type" | "disabled" | "onClick" | "name" | "value" | "form"
+  "type" | "disabled" | "onClick" | "name" | "value" | "form" | "autoFocus"
 > &
   React.ComponentProps<typeof ButtonContent>;
 
 export const Button = forwardRef<HTMLButtonElement, ButtonPropsType>(
-  ({ type, disabled, onClick, ...props }, ref) => {
+  ({ type, disabled, autoFocus, onClick, ...props }, ref) => {
     const innerRef = useRef<HTMLButtonElement>(null);
     useImperativeHandle(ref, () => innerRef.current as HTMLButtonElement);
 
@@ -276,6 +276,7 @@ export const Button = forwardRef<HTMLButtonElement, ButtonPropsType>(
         value={props.value}
         ref={innerRef}
         form={props.form}
+        autoFocus={autoFocus}
       >
         <ButtonContent {...props} />
       </button>
@@ -286,8 +287,7 @@ export const Button = forwardRef<HTMLButtonElement, ButtonPropsType>(
 type LinkPropsType = Pick<
   LinkProps,
   "to" | "target" | "onClick" | "onMouseDown" | "onMouseEnter" | "onMouseLeave" | "download"
-> &
-  React.ComponentProps<typeof ButtonContent>;
+> & { disabled?: boolean } & React.ComponentProps<typeof ButtonContent>;
 export const LinkButton = ({
   to,
   onClick,
@@ -295,6 +295,7 @@ export const LinkButton = ({
   onMouseEnter,
   onMouseLeave,
   download,
+  disabled = false,
   ...props
 }: LinkPropsType) => {
   const innerRef = useRef<HTMLAnchorElement>(null);
@@ -309,6 +310,19 @@ export const LinkButton = ({
     });
   }
 
+  if (disabled) {
+    return (
+      <div
+        className={cn(
+          "group pointer-events-none cursor-default opacity-40 outline-none",
+          props.fullWidth ? "w-full" : ""
+        )}
+      >
+        <ButtonContent {...props} />
+      </div>
+    );
+  }
+
   if (to.toString().startsWith("http") || to.toString().startsWith("/resources")) {
     return (
       <ExtLink

apps/webapp/app/components/primitives/Chart.tsx

@@ -0,0 +1,326 @@
+"use client";
+
+import * as React from "react";
+import * as RechartsPrimitive from "recharts";
+import { cn } from "~/utils/cn";
+
+// Format: { THEME_NAME: CSS_SELECTOR }
+const THEMES = { light: "", dark: ".dark" } as const;
+
+export type ChartConfig = {
+  [k in string]: {
+    label?: React.ReactNode;
+    icon?: React.ComponentType;
+  } & (
+    | { color?: string; theme?: never }
+    | { color?: never; theme: Record<keyof typeof THEMES, string> }
+  );
+};
+
+type ChartContextProps = {
+  config: ChartConfig;
+};
+
+const ChartContext = React.createContext<ChartContextProps | null>(null);
+
+function useChart() {
+  const context = React.useContext(ChartContext);
+
+  if (!context) {
+    throw new Error("useChart must be used within a <ChartContainer />");
+  }
+
+  return context;
+}
+
+const ChartContainer = React.forwardRef<
+  HTMLDivElement,
+  React.ComponentProps<"div"> & {
+    config: ChartConfig;
+    children: React.ComponentProps<typeof RechartsPrimitive.ResponsiveContainer>["children"];
+  }
+>(({ id, className, children, config, ...props }, ref) => {
+  const uniqueId = React.useId();
+  const chartId = `chart-${id || uniqueId.replace(/:/g, "")}`;
+
+  return (
+    <ChartContext.Provider value={{ config }}>
+      <div
+        data-chart={chartId}
+        ref={ref}
+        className={cn(
+          "flex aspect-video justify-center text-xs [&_.recharts-cartesian-axis-tick_text]:fill-text-dimmed/50 [&_.recharts-cartesian-grid_line]:stroke-grid-bright/50 [&_.recharts-curve.recharts-tooltip-cursor]:stroke-grid-dimmed [&_.recharts-dot[stroke='#fff']]:stroke-transparent [&_.recharts-layer]:outline-none [&_.recharts-polar-grid_[stroke='#ccc']]:stroke-grid-dimmed [&_.recharts-radial-bar-background-sector]:fill-grid-dimmed [&_.recharts-rectangle.recharts-tooltip-cursor]:fill-grid-dimmed [&_.recharts-reference-line-line]:stroke-grid-dimmed [&_.recharts-sector[stroke='#fff']]:stroke-transparent [&_.recharts-sector]:outline-none [&_.recharts-surface]:outline-none",
+          className
+        )}
+        {...props}
+      >
+        <ChartStyle id={chartId} config={config} />
+        <RechartsPrimitive.ResponsiveContainer className="!h-auto">
+          {children}
+        </RechartsPrimitive.ResponsiveContainer>
+      </div>
+    </ChartContext.Provider>
+  );
+});
+ChartContainer.displayName = "Chart";
+
+const ChartStyle = ({ id, config }: { id: string; config: ChartConfig }) => {
+  const colorConfig = Object.entries(config).filter(([_, config]) => config.theme || config.color);
+
+  if (!colorConfig.length) {
+    return null;
+  }
+
+  return (
+    <style
+      dangerouslySetInnerHTML={{
+        __html: Object.entries(THEMES).map(
+          ([theme, prefix]) => `
+${prefix} [data-chart=${id}] {
+${colorConfig
+  .map(([key, itemConfig]) => {
+    const color = itemConfig.theme?.[theme as keyof typeof itemConfig.theme] || itemConfig.color;
+    return color ? `  --color-${key}: ${color};` : null;
+  })
+  .join("\n")}
+}
+`
+        ),
+      }}
+    />
+  );
+};
+
+const ChartTooltip = RechartsPrimitive.Tooltip;
+
+const ChartTooltipContent = React.forwardRef<
+  HTMLDivElement,
+  React.ComponentProps<typeof RechartsPrimitive.Tooltip> &
+    React.ComponentProps<"div"> & {
+      hideLabel?: boolean;
+      hideIndicator?: boolean;
+      indicator?: "line" | "dot" | "dashed";
+      nameKey?: string;
+      labelKey?: string;
+    }
+>(
+  (
+    {
+      active,
+      payload,
+      className,
+      indicator = "dot",
+      hideLabel = false,
+      hideIndicator = false,
+      label,
+      labelFormatter,
+      labelClassName,
+      formatter,
+      color,
+      nameKey,
+      labelKey,
+    },
+    ref
+  ) => {
+    const { config } = useChart();
+
+    const tooltipLabel = React.useMemo(() => {
+      if (hideLabel || !payload?.length) {
+        return null;
+      }
+
+      const [item] = payload;
+      const key = `${labelKey || item.dataKey || item.name || "value"}`;
+      const itemConfig = getPayloadConfigFromPayload(config, item, key);
+      const value =
+        !labelKey && typeof label === "string"
+          ? config[label as keyof typeof config]?.label || label
+          : itemConfig?.label;
+
+      if (labelFormatter) {
+        return (
+          <div className={cn("font-medium", labelClassName)}>{labelFormatter(value, payload)}</div>
+        );
+      }
+
+      if (!value) {
+        return null;
+      }
+
+      return <div className={cn("font-medium", labelClassName)}>{value}</div>;
+    }, [label, labelFormatter, payload, hideLabel, labelClassName, config, labelKey]);
+
+    if (!active || !payload?.length) {
+      return null;
+    }
+
+    const nestLabel = payload.length === 1 && indicator !== "dot";
+
+    return (
+      <div
+        ref={ref}
+        className={cn(
+          "grid min-w-[8rem] items-start gap-1.5 rounded-lg border border-grid-bright/50 bg-background-dimmed px-2.5 py-1.5 text-xs shadow-xl",
+          className
+        )}
+      >
+        {!nestLabel ? tooltipLabel : null}
+        <div className="grid gap-1.5">
+          {payload.map((item, index) => {
+            const key = `${nameKey || item.name || item.dataKey || "value"}`;
+            const itemConfig = getPayloadConfigFromPayload(config, item, key);
+            const indicatorColor = color || item.payload.fill || item.color;
+
+            return (
+              <div
+                key={item.dataKey}
+                className={cn(
+                  "flex w-full items-stretch gap-2 [&>svg]:h-2.5 [&>svg]:w-2.5 [&>svg]:text-text-dimmed",
+                  indicator === "dot" && "items-center"
+                )}
+              >
+                {formatter && item.value && item.name ? (
+                  formatter(item.value, item.name, item, index, item.payload)
+                ) : (
+                  <>
+                    {itemConfig?.icon ? (
+                      <itemConfig.icon />
+                    ) : (
+                      !hideIndicator && (
+                        <div
+                          className={cn(
+                            "shrink-0 rounded-[2px] border-[--color-border] bg-[--color-bg]",
+                            {
+                              "h-2.5 w-2.5": indicator === "dot",
+                              "w-1": indicator === "line",
+                              "w-0 border-[1.5px] border-dashed bg-transparent":
+                                indicator === "dashed",
+                              "my-0.5": nestLabel && indicator === "dashed",
+                            }
+                          )}
+                          style={
+                            {
+                              "--color-bg": indicatorColor,
+                              "--color-border": indicatorColor,
+                            } as React.CSSProperties
+                          }
+                        />
+                      )
+                    )}
+                    <div
+                      className={cn(
+                        "flex flex-1 justify-between gap-1.5 leading-none",
+                        nestLabel ? "items-end" : "items-center"
+                      )}
+                    >
+                      <div className="grid gap-1.5">
+                        {nestLabel ? tooltipLabel : null}
+                        <span className="text-text-dimmed">{itemConfig?.label || item.name}</span>
+                      </div>
+                      {item.value && (
+                        <span className="font-mono font-medium tabular-nums text-text-bright">
+                          {item.value.toLocaleString()}
+                        </span>
+                      )}
+                    </div>
+                  </>
+                )}
+              </div>
+            );
+          })}
+        </div>
+      </div>
+    );
+  }
+);
+ChartTooltipContent.displayName = "ChartTooltip";
+
+const ChartLegend = RechartsPrimitive.Legend;
+
+const ChartLegendContent = React.forwardRef<
+  HTMLDivElement,
+  React.ComponentProps<"div"> &
+    Pick<RechartsPrimitive.LegendProps, "payload" | "verticalAlign"> & {
+      hideIcon?: boolean;
+      nameKey?: string;
+    }
+>(({ className, hideIcon = false, payload, verticalAlign = "bottom", nameKey }, ref) => {
+  const { config } = useChart();
+
+  if (!payload?.length) {
+    return null;
+  }
+
+  return (
+    <div
+      ref={ref}
+      className={cn(
+        "flex items-center justify-center gap-4",
+        verticalAlign === "top" ? "pb-3" : "pt-3",
+        className
+      )}
+    >
+      {payload.map((item) => {
+        const key = `${nameKey || item.dataKey || "value"}`;
+        const itemConfig = getPayloadConfigFromPayload(config, item, key);
+
+        return (
+          <div
+            key={item.value}
+            className={cn(
+              "flex items-center gap-1.5 [&>svg]:h-3 [&>svg]:w-3 [&>svg]:text-text-dimmed"
+            )}
+          >
+            {itemConfig?.icon && !hideIcon ? (
+              <itemConfig.icon />
+            ) : (
+              <div
+                className="h-2 w-2 shrink-0 rounded-[2px]"
+                style={{
+                  backgroundColor: item.color,
+                }}
+              />
+            )}
+            {itemConfig?.label}
+          </div>
+        );
+      })}
+    </div>
+  );
+});
+ChartLegendContent.displayName = "ChartLegend";
+
+// Helper to extract item config from a payload.
+function getPayloadConfigFromPayload(config: ChartConfig, payload: unknown, key: string) {
+  if (typeof payload !== "object" || payload === null) {
+    return undefined;
+  }
+
+  const payloadPayload =
+    "payload" in payload && typeof payload.payload === "object" && payload.payload !== null
+      ? payload.payload
+      : undefined;
+
+  let configLabelKey: string = key;
+
+  if (key in payload && typeof payload[key as keyof typeof payload] === "string") {
+    configLabelKey = payload[key as keyof typeof payload] as string;
+  } else if (
+    payloadPayload &&
+    key in payloadPayload &&
+    typeof payloadPayload[key as keyof typeof payloadPayload] === "string"
+  ) {
+    configLabelKey = payloadPayload[key as keyof typeof payloadPayload] as string;
+  }
+
+  return configLabelKey in config ? config[configLabelKey] : config[key as keyof typeof config];
+}
+
+export {
+  ChartContainer,
+  ChartTooltip,
+  ChartTooltipContent,
+  ChartLegend,
+  ChartLegendContent,
+  ChartStyle,
+};

apps/webapp/app/components/primitives/Checkbox.tsx

@@ -1,5 +1,5 @@
 import * as React from "react";
-import { useEffect, useState } from "react";
+import { forwardRef, useEffect, useState } from "react";
 import { cn } from "~/utils/cn";
 import { Badge } from "./Badge";
 import { Paragraph } from "./Paragraph";
@@ -23,7 +23,7 @@ const variants = {
   },
   "button/small": {
     button:
-      "flex items-center w-fit h-8 pl-2 pr-3 rounded border border-charcoal-800 hover:bg-charcoal-850 hover:border-charcoal-750 transition",
+      "flex items-center w-fit h-8 pl-2 pr-3 rounded border border-charcoal-600 hover:bg-charcoal-850 hover:border-charcoal-500 transition",
     label: "text-sm text-text-bright select-none",
     description: "text-text-dimmed",
     inputPosition: "mt-0",
@@ -32,7 +32,7 @@ const variants = {
   },
   button: {
     button:
-      "w-fit py-2 pl-3 pr-4 rounded border border-charcoal-800 hover:bg-charcoal-850 hover:border-charcoal-750 transition",
+      "w-fit py-2 pl-3 pr-4 rounded border border-charcoal-600 hover:bg-charcoal-850 hover:border-charcoal-500 transition",
     label: "text-text-bright select-none",
     description: "text-text-dimmed",
     inputPosition: "mt-1",
@@ -53,18 +53,18 @@ export type CheckboxProps = Omit<
   React.InputHTMLAttributes<HTMLInputElement>,
   "checked" | "onChange"
 > & {
-  id: string;
+  id?: string;
   name?: string;
   value?: string;
   variant?: keyof typeof variants;
-  label?: string;
+  label: React.ReactNode;
   description?: string;
   badges?: string[];
   className?: string;
   onChange?: (isChecked: boolean) => void;
 };
 
-export const Checkbox = React.forwardRef<HTMLInputElement, CheckboxProps>(
+export const CheckboxWithLabel = React.forwardRef<HTMLInputElement, CheckboxProps>(
   (
     {
       id,
@@ -137,7 +137,7 @@ export const Checkbox = React.forwardRef<HTMLInputElement, CheckboxProps>(
           className={cn(
             inputPositionClasses,
             props.readOnly || disabled ? "cursor-default" : "cursor-pointer",
-            "read-only:border-charcoal-650 disabled:border-charcoal-650 rounded-sm border border-charcoal-700 bg-transparent transition checked:!bg-indigo-500 read-only:!bg-charcoal-700 group-hover:bg-charcoal-900 group-hover:checked:bg-indigo-500 group-focus:ring-1 focus:ring-indigo-500 focus:ring-offset-0 focus:ring-offset-transparent focus-visible:outline-none  focus-visible:ring-indigo-500 disabled:!bg-charcoal-700"
+            "read-only:border-charcoal-650 disabled:border-charcoal-650 rounded-sm border border-charcoal-600 bg-transparent transition checked:!bg-indigo-500 read-only:!bg-charcoal-700 group-hover:bg-charcoal-900 group-hover:checked:bg-indigo-500 group-focus:ring-1 focus:ring-indigo-500 focus:ring-offset-0 focus:ring-offset-transparent focus-visible:outline-none  focus-visible:ring-indigo-500 disabled:!bg-charcoal-700"
           )}
           id={id}
           ref={ref}
@@ -172,3 +172,21 @@ export const Checkbox = React.forwardRef<HTMLInputElement, CheckboxProps>(
     );
   }
 );
+
+type SimpleCheckboxProps = Omit<React.ComponentProps<"input">, "type">;
+
+export const Checkbox = forwardRef<HTMLInputElement, SimpleCheckboxProps>(
+  ({ className, ...props }: SimpleCheckboxProps, ref) => {
+    return (
+      <input
+        type="checkbox"
+        className={cn(
+          props.readOnly || props.disabled ? "cursor-default" : "cursor-pointer",
+          "read-only:border-charcoal-650 disabled:border-charcoal-650 rounded-sm border border-charcoal-600 bg-transparent transition checked:!bg-indigo-500 read-only:!bg-charcoal-700 group-hover:bg-charcoal-900 group-hover:checked:bg-indigo-500 group-focus:ring-1 focus:ring-indigo-500 focus:ring-offset-0 focus:ring-offset-transparent focus-visible:outline-none  focus-visible:ring-indigo-500 disabled:!bg-charcoal-700"
+        )}
+        {...props}
+        ref={ref}
+      />
+    );
+  }
+);

apps/webapp/app/components/primitives/DateField.tsx

@@ -1,3 +1,4 @@
+import { BellAlertIcon } from "@heroicons/react/20/solid";
 import { CalendarDateTime, createCalendar } from "@internationalized/date";
 import { useDateField, useDateSegment } from "@react-aria/datepicker";
 import type { DateFieldState, DateSegment } from "@react-stately/datepicker";
@@ -5,11 +6,25 @@ import { useDateFieldState } from "@react-stately/datepicker";
 import { Granularity } from "@react-types/datepicker";
 import { useEffect, useRef, useState } from "react";
 import { cn } from "~/utils/cn";
-import { useLocales } from "./LocaleProvider";
 import { Button } from "./Buttons";
 
+const variants = {
+  small: {
+    fieldStyles: "h-5 text-sm rounded-sm px-0.5",
+    nowButtonVariant: "tertiary/small" as const,
+    clearButtonVariant: "minimal/small" as const,
+  },
+  medium: {
+    fieldStyles: "h-7 text-base rounded px-1",
+    nowButtonVariant: "tertiary/medium" as const,
+    clearButtonVariant: "minimal/medium" as const,
+  },
+};
+
+type Variant = keyof typeof variants;
+
 type DateFieldProps = {
-  label?: string;
+  label: string;
   defaultValue?: Date;
   minValue?: Date;
   maxValue?: Date;
@@ -20,6 +35,7 @@ type DateFieldProps = {
   showNowButton?: boolean;
   showClearButton?: boolean;
   onValueChange?: (value: Date | undefined) => void;
+  variant?: Variant;
 };
 
 export function DateField({
@@ -34,6 +50,7 @@ export function DateField({
   showGuide = false,
   showNowButton = false,
   showClearButton = false,
+  variant = "small",
 }: DateFieldProps) {
   const [value, setValue] = useState<undefined | CalendarDateTime>(
     utcDateToCalendarDate(defaultValue)
@@ -90,48 +107,50 @@ export function DateField({
 
   return (
     <div className={`flex flex-col items-start ${className || ""}`}>
-      <span {...labelProps} className="mb-1 ml-0.5 text-xs text-charcoal-300">
-        {label}
-      </span>
-      <div className="flex flex-row items-center gap-1">
+      <div className="flex flex-row items-center gap-1" aria-label={label}>
         <div
           {...fieldProps}
           ref={ref}
           className={cn(
-            "flex rounded-sm border border-charcoal-800 bg-charcoal-750 p-0.5 px-1.5 transition-colors focus-within:border-charcoal-500 hover:border-charcoal-700 focus-within:hover:border-charcoal-500",
+            "flex rounded-sm border bg-charcoal-700 p-0.5 transition focus-within:border-charcoal-600 hover:border-charcoal-600",
             fieldClassName
           )}
         >
-          <DateSegment segment={yearSegment} state={state} />
-          <DateSegment segment={literalSegment("/")} state={state} />
-          <DateSegment segment={monthSegment} state={state} />
-          <DateSegment segment={literalSegment("/")} state={state} />
-          <DateSegment segment={daySegment} state={state} />
-          <DateSegment segment={literalSegment(", ")} state={state} />
-          <DateSegment segment={hourSegment} state={state} />
-          <DateSegment segment={literalSegment(":")} state={state} />
-          <DateSegment segment={minuteSegment} state={state} />
-          <DateSegment segment={literalSegment(":")} state={state} />
-          <DateSegment segment={secondSegment} state={state} />
-          <DateSegment segment={literalSegment(" ")} state={state} />
-          <DateSegment segment={dayPeriodSegment} state={state} />
+          <DateSegment segment={yearSegment} state={state} variant={variant} />
+          <DateSegment segment={literalSegment("/")} state={state} variant={variant} />
+          <DateSegment segment={monthSegment} state={state} variant={variant} />
+          <DateSegment segment={literalSegment("/")} state={state} variant={variant} />
+          <DateSegment segment={daySegment} state={state} variant={variant} />
+          <DateSegment segment={literalSegment(", ")} state={state} variant={variant} />
+          <DateSegment segment={hourSegment} state={state} variant={variant} />
+          <DateSegment segment={literalSegment(":")} state={state} variant={variant} />
+          <DateSegment segment={minuteSegment} state={state} variant={variant} />
+          <DateSegment segment={literalSegment(":")} state={state} variant={variant} />
+          <DateSegment segment={secondSegment} state={state} variant={variant} />
+          <DateSegment segment={literalSegment(" ")} state={state} variant={variant} />
+          <DateSegment segment={dayPeriodSegment} state={state} variant={variant} />
         </div>
         {showNowButton && (
           <Button
-            variant="tertiary/small"
+            type="button"
+            variant={variants[variant].nowButtonVariant}
+            LeadingIcon={BellAlertIcon}
+            leadingIconClassName="text-text-dimmed group-hover:text-text-bright"
             onClick={() => {
               const now = new Date();
               setValue(utcDateToCalendarDate(new Date()));
               onValueChange?.(now);
             }}
           >
-            Now
+            <span className="text-text-dimmed transition group-hover:text-text-bright">Now</span>
           </Button>
         )}
         {showClearButton && (
           <Button
-            variant="tertiary/small"
+            type="button"
+            variant={variants[variant].clearButtonVariant}
             LeadingIcon={"close"}
+            leadingIconClassName="-mr-2"
             onClick={() => {
               setValue(undefined);
               onValueChange?.(undefined);
@@ -142,7 +161,9 @@ export function DateField({
               state.clearSegment("minute");
               state.clearSegment("second");
             }}
-          />
+          >
+            Clear
+          </Button>
         )}
       </div>
       {showGuide && (
@@ -172,11 +193,13 @@ function utcDateToCalendarDate(date?: Date) {
 type DateSegmentProps = {
   segment: DateSegment;
   state: DateFieldState;
+  variant: Variant;
 };
 
-function DateSegment({ segment, state }: DateSegmentProps) {
+function DateSegment({ segment, state, variant }: DateSegmentProps) {
   const ref = useRef<null | HTMLDivElement>(null);
   const { segmentProps } = useDateSegment(segment, state, ref);
+  const sizeVariant = variants[variant];
 
   return (
     <div
@@ -186,23 +209,27 @@ function DateSegment({ segment, state }: DateSegmentProps) {
         ...segmentProps.style,
         minWidth: minWidthForSegment(segment),
       }}
-      className={`group box-content rounded-sm px-0.5 text-right text-sm tabular-nums outline-none focus:bg-indigo-500 focus:text-white ${
+      className={cn(
+        "group box-content text-center tabular-nums outline-none focus:bg-charcoal-600 focus:text-text-bright",
+        sizeVariant.fieldStyles,
         !segment.isEditable ? "text-charcoal-500" : "text-text-bright"
-      }`}
+      )}
     >
       {/* Always reserve space for the placeholder, to prevent layout shift when editing. */}
       <span
         aria-hidden="true"
-        className="block text-center italic text-charcoal-500 group-focus:text-white"
+        className="flex h-full items-center justify-center text-center text-charcoal-500 group-focus:text-text-bright"
         style={{
           visibility: segment.isPlaceholder ? undefined : "hidden",
-          height: segment.isPlaceholder ? "" : 0,
+          height: segment.isPlaceholder ? undefined : 0,
           pointerEvents: "none",
         }}
       >
         {segment.placeholder}
       </span>
-      {segment.isPlaceholder ? "" : segment.text}
+      <span className="flex h-full items-center justify-center">
+        {segment.isPlaceholder ? "" : segment.text}
+      </span>
     </div>
   );
 }
@@ -231,7 +258,7 @@ function DateSegmentGuide({ segment }: { segment: DateSegment }) {
       style={{
         minWidth: minWidthForSegment(segment),
       }}
-      className={`group box-content rounded-sm px-0.5 text-right text-sm tabular-nums outline-none ${
+      className={`group box-content rounded-sm px-0.5 text-right text-sm tabular-nums text-rose-500 outline-none ${
         !segment.isEditable ? "text-charcoal-500" : "text-text-bright"
       }`}
     >

apps/webapp/app/components/primitives/DateTime.tsx

@@ -6,13 +6,15 @@ type DateTimeProps = {
   timeZone?: string;
   includeSeconds?: boolean;
   includeTime?: boolean;
+  showTimezone?: boolean;
 };
 
 export const DateTime = ({
   date,
-  timeZone = "UTC",
+  timeZone,
   includeSeconds = true,
   includeTime = true,
+  showTimezone = false,
 }: DateTimeProps) => {
   const locales = useLocales();
 
@@ -20,7 +22,7 @@ export const DateTime = ({
 
   const initialFormattedDateTime = formatDateTime(
     realDate,
-    timeZone,
+    timeZone ?? "UTC",
     locales,
     includeSeconds,
     includeTime
@@ -32,11 +34,22 @@ export const DateTime = ({
     const resolvedOptions = Intl.DateTimeFormat().resolvedOptions();
 
     setFormattedDateTime(
-      formatDateTime(realDate, resolvedOptions.timeZone, locales, includeSeconds, includeTime)
+      formatDateTime(
+        realDate,
+        timeZone ?? resolvedOptions.timeZone,
+        locales,
+        includeSeconds,
+        includeTime
+      )
     );
   }, [locales, includeSeconds, realDate]);
 
-  return <Fragment>{formattedDateTime.replace(/\s/g, String.fromCharCode(32))}</Fragment>;
+  return (
+    <Fragment>
+      {formattedDateTime.replace(/\s/g, String.fromCharCode(32))}
+      {showTimezone ? ` (${timeZone ?? "UTC"})` : null}
+    </Fragment>
+  );
 };
 
 export function formatDateTime(

apps/webapp/app/components/primitives/DetailCell.tsx

@@ -49,12 +49,7 @@ export function DetailCell({
   const variation = variations[variant];
 
   return (
-    <div
-      className={cn(
-        "group flex h-11 w-full items-center gap-3 rounded-md p-1 pr-3 transition hover:bg-charcoal-900",
-        className
-      )}
-    >
+    <div className={cn("group flex h-11 w-full items-center gap-3 rounded-md p-1 pr-3", className)}>
       <IconInBox
         icon={leadingIcon}
         className={cn("flex-none transition group-hover:border-charcoal-750", leadingIconClassName)}
@@ -62,20 +57,14 @@ export function DetailCell({
       <div className="flex flex-1 flex-col">
         <Paragraph
           variant={variation.label.variant}
-          className={cn(
-            "flex-1 text-left transition group-hover:text-text-bright",
-            variation.label.className
-          )}
+          className={cn("flex-1 text-left", variation.label.className)}
         >
           {label}
         </Paragraph>
         {description && (
           <Paragraph
             variant={variation.description.variant}
-            className={cn(
-              "flex-1 text-left text-text-dimmed transition group-hover:text-text-bright",
-              variation.description.className
-            )}
+            className={cn("flex-1 text-left text-text-dimmed", variation.description.className)}
           >
             {description}
           </Paragraph>

apps/webapp/app/components/primitives/Dialog.tsx

@@ -10,8 +10,8 @@ const Dialog = DialogPrimitive.Root;
 
 const DialogTrigger = DialogPrimitive.Trigger;
 
-const DialogPortal = ({ className, children, ...props }: DialogPrimitive.DialogPortalProps) => (
-  <DialogPrimitive.Portal className={cn(className)} {...props}>
+const DialogPortal = ({ children, ...props }: DialogPrimitive.DialogPortalProps) => (
+  <DialogPrimitive.Portal {...props}>
     <div className="fixed inset-0 z-50 flex items-start justify-center sm:items-center">
       {children}
     </div>
@@ -43,7 +43,7 @@ const DialogContent = React.forwardRef<
     <DialogPrimitive.Content
       ref={ref}
       className={cn(
-        "fixed z-50 grid w-full gap-4 rounded-b-lg border bg-background-dimmed px-4 pb-6 pt-3.5 shadow-lg animate-in data-[state=open]:fade-in-90 data-[state=open]:slide-in-from-bottom-10 sm:max-w-lg sm:rounded-lg sm:zoom-in-90 data-[state=open]:sm:slide-in-from-bottom-0",
+        "fixed z-50 grid w-full gap-4 rounded-b-lg border bg-background-dimmed px-4 pb-4 pt-2.5 shadow-lg animate-in data-[state=open]:fade-in-90 data-[state=open]:slide-in-from-bottom-10 sm:max-w-lg sm:rounded-lg sm:zoom-in-90 data-[state=open]:sm:slide-in-from-bottom-0",
         className
       )}
       {...props}
@@ -74,7 +74,7 @@ DialogContent.displayName = DialogPrimitive.Content.displayName;
 
 const DialogHeader = ({ className, ...props }: React.HTMLAttributes<HTMLDivElement>) => (
   <div
-    className={cn("flex flex-col text-left font-medium text-text-dimmed", className)}
+    className={cn("flex flex-col text-left font-medium text-text-bright", className)}
     {...props}
   />
 );

apps/webapp/app/components/primitives/Headers.tsx

@@ -2,7 +2,7 @@ import { cn } from "~/utils/cn";
 
 const headerVariants = {
   header1: {
-    text: "font-sans text-2xl leading-5 md:leading-6 lg:leading-7 font-semibold",
+    text: "font-sans text-2xl leading-5 md:leading-6 lg:leading-7 font-medium",
     spacing: "mb-2",
   },
   header2: {

apps/webapp/app/components/primitives/Hint.tsx

@@ -1,5 +1,9 @@
 import { Paragraph } from "./Paragraph";
 
-export function Hint({ children }: { children: React.ReactNode }) {
-  return <Paragraph variant="extra-small">{children}</Paragraph>;
+export function Hint({ children, className }: { children: React.ReactNode; className?: string }) {
+  return (
+    <Paragraph variant="extra-small" className={className}>
+      {children}
+    </Paragraph>
+  );
 }

apps/webapp/app/components/primitives/InfoPanel.tsx

@@ -0,0 +1,71 @@
+import { LinkButton } from "./Buttons";
+import { Header3 } from "./Headers";
+import { Paragraph } from "./Paragraph";
+import { cn } from "~/utils/cn";
+
+const variants = {
+  info: {
+    panelStyle: "border-grid-bright bg-background-bright",
+  },
+  upgrade: {
+    panelStyle: "border-indigo-400/20 bg-indigo-800/10",
+  },
+};
+
+type InfoPanelVariant = keyof typeof variants;
+
+type Props = {
+  title?: string;
+  children: React.ReactNode;
+  to?: string;
+  buttonLabel?: string;
+  icon: React.ComponentType<any>;
+  iconClassName?: string;
+  variant?: InfoPanelVariant;
+  panelClassName?: string;
+};
+
+export function InfoPanel({
+  title,
+  children,
+  to,
+  buttonLabel,
+  icon,
+  iconClassName,
+  variant = "info",
+  panelClassName = "max-w-sm",
+}: Props) {
+  const Icon = icon;
+  const variantStyle = variants[variant];
+
+  return (
+    <div
+      className={cn(
+        variantStyle.panelStyle,
+        title ? "flex-col" : "",
+        "flex h-fit items-start gap-3 rounded-md border p-4",
+        panelClassName
+      )}
+    >
+      <div className={cn("flex items-center gap-2", to ? "w-full justify-between" : "")}>
+        <Icon className={cn("size-5", iconClassName)} />
+
+        {to && (
+          <LinkButton to={to} variant="secondary/small">
+            {buttonLabel}
+          </LinkButton>
+        )}
+      </div>
+      <div className="flex flex-col gap-1">
+        {title && <Header3 className="text-text-bright">{title}</Header3>}
+        {typeof children === "string" ? (
+          <Paragraph variant={"small"} className="text-text-dimmed">
+            {children}
+          </Paragraph>
+        ) : (
+          children
+        )}
+      </div>
+    </div>
+  );
+}

apps/webapp/app/components/primitives/Label.tsx

@@ -1,29 +1,41 @@
 import * as React from "react";
 import { cn } from "~/utils/cn";
+import { InfoIconTooltip, SimpleTooltip } from "./Tooltip";
 
-const labelVariants = {
+const variants = {
   small: {
-    text: "font-sans text-sm font-normal",
+    text: "font-sans text-sm font-normal text-text-bright leading-tight flex items-center gap-1",
   },
   medium: {
-    text: "font-sans text-sm leading-5 font-medium",
+    text: "font-sans text-sm text-text-bright leading-tight flex items-center gap-1",
   },
   large: {
-    text: "font-sans text-base leading-6 font-medium",
+    text: "font-sans text-base font-medium text-text-bright leading-tight flex items-center gap-1",
   },
 };
 
 type LabelProps = React.AllHTMLAttributes<HTMLLabelElement> & {
   className?: string;
   children: React.ReactNode;
-  variant?: keyof typeof labelVariants;
+  variant?: keyof typeof variants;
+  required?: boolean;
+  tooltip?: React.ReactNode;
 };
 
-export function Label({ className, children, variant = "medium", ...props }: LabelProps) {
-  const variation = labelVariants[variant];
+export function Label({
+  className,
+  children,
+  variant = "medium",
+  required = true,
+  tooltip,
+  ...props
+}: LabelProps) {
+  const variation = variants[variant];
   return (
     <label className={cn(variation.text, className)} {...props}>
       {children}
+      {tooltip ? <InfoIconTooltip content={tooltip} /> : null}
+      {!required && <span className="text-text-dimmed"> (optional)</span>}
     </label>
   );
 }

apps/webapp/app/components/primitives/PageHeader.tsx

@@ -1,10 +1,8 @@
 import { ArrowUpRightIcon } from "@heroicons/react/20/solid";
 import { Link, useNavigation } from "@remix-run/react";
 import { useOptionalOrganization } from "~/hooks/useOrganizations";
-import { useOptionalProject } from "~/hooks/useProject";
 import { cn } from "~/utils/cn";
-import { plansPath } from "~/utils/pathBuilder";
-import { UpgradePrompt, useShowUpgradePrompt } from "../billing/UpgradePrompt";
+import { UpgradePrompt, useShowUpgradePrompt } from "../billing/v3/UpgradePrompt";
 import { BreadcrumbIcon } from "./BreadcrumbIcon";
 import { LinkButton } from "./Buttons";
 import { Header2 } from "./Headers";
@@ -12,9 +10,11 @@ import { LoadingBarDivider } from "./LoadingBarDivider";
 import { NamedIcon } from "./NamedIcon";
 import { Paragraph } from "./Paragraph";
 import { Tabs, TabsProps } from "./Tabs";
+import { ReactNode } from "react";
 
 type WithChildren = {
   children: React.ReactNode;
+  className?: string;
 };
 
 export function NavBar({ children }: WithChildren) {
@@ -22,27 +22,20 @@ export function NavBar({ children }: WithChildren) {
   const showUpgradePrompt = useShowUpgradePrompt(organization);
   const navigation = useNavigation();
   const isLoading = navigation.state === "loading" || navigation.state === "submitting";
-  const project = useOptionalProject();
 
   return (
     <div>
       <div className="grid h-10 w-full grid-rows-[auto_1px] bg-background-bright">
-        <div className="flex w-full items-center justify-between pl-3 pr-1">{children}</div>
+        <div className="flex w-full items-center justify-between pl-3 pr-2">{children}</div>
         <LoadingBarDivider isLoading={isLoading} />
       </div>
-      {showUpgradePrompt.shouldShow && organization && (
-        <UpgradePrompt
-          runsEnabled={showUpgradePrompt.runsEnabled}
-          runCountCap={showUpgradePrompt.runCountCap}
-          planPath={plansPath(organization)}
-        />
-      )}
+      {showUpgradePrompt.shouldShow && organization && <UpgradePrompt />}
     </div>
   );
 }
 
 type PageTitleProps = {
-  title: string;
+  title: ReactNode;
   backButton?: {
     to: string;
     text: string;
@@ -69,11 +62,11 @@ export function PageTitle({ title, backButton }: PageTitleProps) {
 }
 
 export function PageAccessories({ children }: WithChildren) {
-  return <div className="flex items-center gap-3">{children}</div>;
+  return <div className="flex items-center gap-2">{children}</div>;
 }
 
-export function PageInfoRow({ children }: WithChildren) {
-  return <div className="flex w-full items-center gap-2">{children}</div>;
+export function PageInfoRow({ children, className }: WithChildren) {
+  return <div className={cn("flex w-full items-center gap-2", className)}>{children}</div>;
 }
 
 export function PageInfoGroup({

apps/webapp/app/components/primitives/Pagination.tsx

@@ -1,16 +1,18 @@
 import { ChevronRightIcon } from "@heroicons/react/24/outline";
 import { ChevronLeftIcon } from "@heroicons/react/24/solid";
 import { Link, useLocation } from "@remix-run/react";
-import { LinkDisabled } from "./LinkWithDisabled";
 import { cn } from "~/utils/cn";
-import { ButtonContent, LinkButton } from "./Buttons";
+import { ButtonContent } from "./Buttons";
+import { LinkDisabled } from "./LinkWithDisabled";
 
 export function PaginationControls({
   currentPage,
   totalPages,
+  showPageNumbers = true,
 }: {
   currentPage: number;
   totalPages: number;
+  showPageNumbers?: boolean;
 }) {
   const location = useLocation();
   if (totalPages <= 1) {
@@ -30,9 +32,11 @@ export function PaginationControls({
         </ButtonContent>
       </LinkDisabled>
 
-      {calculatePageLinks(currentPage, totalPages).map((page, i) => (
-        <PageLinkComponent page={page} key={i} location={location} />
-      ))}
+      {showPageNumbers
+        ? calculatePageLinks(currentPage, totalPages).map((page, i) => (
+            <PageLinkComponent page={page} key={i} location={location} />
+          ))
+        : null}
 
       <LinkDisabled
         to={pageUrl(location, currentPage + 1)}

apps/webapp/app/components/primitives/Popover.tsx

@@ -4,8 +4,8 @@ import { ChevronDownIcon, EllipsisVerticalIcon } from "@heroicons/react/24/solid
 import * as PopoverPrimitive from "@radix-ui/react-popover";
 import * as React from "react";
 import { cn } from "~/utils/cn";
-import { ButtonContentPropsType, LinkButton } from "./Buttons";
-import { Paragraph, ParagraphVariant } from "./Paragraph";
+import { type ButtonContentPropsType, LinkButton } from "./Buttons";
+import { Paragraph, type ParagraphVariant } from "./Paragraph";
 
 const Popover = PopoverPrimitive.Root;
 const PopoverTrigger = PopoverPrimitive.Trigger;
@@ -41,7 +41,7 @@ function PopoverSectionHeader({
   variant?: ParagraphVariant;
 }) {
   return (
-    <Paragraph variant={variant} className="bg-charcoal-900 px-2.5 py-1.5">
+    <Paragraph variant={variant} className="bg-charcoal-750 px-2.5 py-1.5">
       {title}
     </Paragraph>
   );
@@ -71,7 +71,10 @@ function PopoverMenuItem({
       fullWidth
       textAlignLeft
       TrailingIcon={isSelected ? "check" : undefined}
-      className={isSelected ? "bg-charcoal-750 group-hover:bg-charcoal-800" : undefined}
+      className={cn(
+        "group-hover:bg-charcoal-700",
+        isSelected ? "bg-charcoal-750 group-hover:bg-charcoal-600/50" : undefined
+      )}
     >
       {title}
     </LinkButton>

apps/webapp/app/components/primitives/PropertyTable.tsx

@@ -1,40 +1,31 @@
-import { ReactNode } from "react";
+import { type ReactNode } from "react";
 import { Paragraph } from "./Paragraph";
 import { cn } from "~/utils/cn";
 
-export function PropertyTable({
-  children,
-  className,
-}: {
+type ChildrenClassName = {
   children: ReactNode;
   className?: string;
-}) {
-  return (
-    <div className={cn("grid grid-cols-[auto,1fr] items-center gap-x-4 gap-y-2", className)}>
-      {children}
-    </div>
-  );
-}
-
-export type PropertyProps = {
-  label: ReactNode;
-  labelClassName?: string;
-  children: ReactNode;
 };
 
-export function Property({ label, labelClassName, children }: PropertyProps) {
-  return (
-    <>
-      <div className={labelClassName}>
-        {typeof label === "string" ? <Paragraph variant="small">{label}</Paragraph> : label}
-      </div>
-      <div>
-        {typeof children === "string" ? (
-          <Paragraph variant="small/bright">{children}</Paragraph>
-        ) : (
-          children
-        )}
-      </div>
-    </>
-  );
+function PropertyTable({ children, className }: { children: ReactNode; className?: string }) {
+  return <div className={cn("flex flex-col gap-y-3", className)}>{children}</div>;
 }
+
+function PropertyItem({ children, className }: ChildrenClassName) {
+  return <div className={cn("flex flex-col gap-0 text-sm", className)}>{children}</div>;
+}
+
+function PropertyLabel({ children, className }: ChildrenClassName) {
+  return <div className={cn("font-medium text-text-bright", className)}>{children}</div>;
+}
+
+function PropertyValue({ children, className }: ChildrenClassName) {
+  return <div className={cn("text-text-dimmed", className)}>{children}</div>;
+}
+
+export {
+  PropertyTable as Table,
+  PropertyItem as Item,
+  PropertyLabel as Label,
+  PropertyValue as Value,
+};