Release 3.0.0-beta.53

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.changeset/angry-trees-drop.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Increase span attribute value length limit to 2048

.changeset/beige-pears-explode.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Add an e2e suite to test compiling with v3 CLI.

.changeset/brown-boats-bathe.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/core-apps": patch
+"@trigger.dev/core": patch
+---
+
+Pre-pull deployment images for faster startups

.changeset/brown-spies-burn.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+cli v3: increase otel force flush timeout to 30s from 500ms

.changeset/curly-monkeys-tell.md

@@ -0,0 +1,7 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+- Prevent uncaught exceptions when handling WebSocket messages
+- Improve CLI dev command WebSocket debug and error logging

.changeset/fast-colts-relax.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Increase dev worker timeout

.changeset/fast-melons-listen.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Added version to ctx.run

.changeset/few-poems-vanish.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Fix trigger functions for custom queues

.changeset/fifty-lions-think.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Improve prisma errors for missing postinstall

.changeset/flat-onions-punch.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Fix return type of runs.retrieve, and allow passing the type of the task to runs.retrieve

.changeset/friendly-walls-repair.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Add sox and audiowaveform binaries to worker images

.changeset/gorgeous-cycles-guess.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+try/catch opening the login URL

.changeset/green-pens-battle.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix automatic opening of login URL on linux-server systems with missing xdg-open

.changeset/hot-buckets-behave.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Support triggering tasks with non-URL friendly characters in the ID

.changeset/hot-wasps-sin.md

@@ -0,0 +1,7 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+"@trigger.dev/sdk": patch
+---
+
+v3: Usage tracking

.changeset/hungry-sloths-promise.md

@@ -0,0 +1,12 @@
+---
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Fix issues that could result in unreezable state run crashes. Details:
+- Never checkpoint between attempts
+- Some messages and socket data now include attempt numbers
+- Remove attempt completion replays
+- Additional prod entry point logging
+- Fail runs that receive deprecated (pre-lazy attempt) execute messages

.changeset/itchy-chairs-itch.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Fix for calling trigger and passing a custom queue

.changeset/large-seahorses-cheat.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Extract common trigger code into internal functions and add a tasks.batchTriggerAndWait function

.changeset/lemon-sloths-hide.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+v3: recover from server rate limiting errors in a more reliable way

.changeset/long-hounds-wave.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+v3: update @depot/cli to latest 0.0.1-cli.2.71.0

.changeset/lucky-items-film.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Automatically bundle internal packages that use file, link or workspace protocl

.changeset/mighty-eggs-grab.md

@@ -0,0 +1,30 @@
+---
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Tasks should now be much more robust and resilient to reconnects during crucial operations and other failure scenarios.
+
+Task runs now have to signal checkpointable state prior to ALL checkpoints. This ensures flushing always happens.
+
+All important socket.io RPCs will now be retried with backoff. Actions relying on checkpoints will be replayed if we haven't been checkpointed and restored as expected, e.g. after reconnect.
+
+Other changes:
+
+- Fix retry check in shared queue
+- Fix env var sync spinner
+- Heartbeat between retries
+- Fix retry prep
+- Fix prod worker no tasks detection
+- Fail runs above `MAX_TASK_RUN_ATTEMPTS`
+- Additional debug logs in all places
+- Prevent crashes due to failed socket schema parsing
+- Remove core-apps barrel
+- Upgrade socket.io-client to fix an ACK memleak
+- Additional index failure logs
+- Prevent message loss during reconnect
+- Prevent burst of heartbeats on reconnect
+- Prevent crash on failed cleanup
+- Handle at-least-once lazy execute message delivery
+- Handle uncaught entry point exceptions

.changeset/mighty-parrots-sin.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+v3: Remove aggressive otel flush timeouts in dev/prod

.changeset/modern-stingrays-end.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+v3: Trigger delayed runs and reschedule them

.changeset/nervous-baboons-sin.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Added config option extraCACerts to ProjectConfig type. This copies the ca file along with additionalFiles and sets NODE_EXTRA_CA_CERTS environment variable in built image as well as running the task.

.changeset/nervous-planets-sparkle.md

@@ -0,0 +1,9 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+- Improve non-zero exit code error messages
+- Detect OOM conditions within worker child processes
+- Internal errors can have optional stack traces
+- Docker provider can be set to enforce machine presets

.changeset/nervous-seas-shave.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+v3: Export AbortTaskRunError from @trigger.dev/sdk/v3

.changeset/odd-beds-wonder.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+You can now add tags to runs and list runs using them

.changeset/plenty-ducks-beam.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Output stderr logs on dev worker failure

.changeset/polite-pears-grow.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Use global setTimeout to ensure cross-runtime support

.changeset/polite-pots-walk.md

@@ -0,0 +1,7 @@
+---
+"trigger.dev": patch
+---
+
+Add e2e fixtures corresponding to past issues
+Implement e2e suite parallelism
+Enhance log level for specific e2e suite messages

.changeset/pre.json

@@ -46,10 +46,14 @@
   "changesets": [
     "afraid-sheep-joke",
     "angry-eagles-trade",
+    "angry-trees-drop",
+    "beige-pears-explode",
     "beige-pens-dance",
     "big-tomatoes-deliver",
     "blue-pumas-whisper",
     "breezy-gorillas-mate",
+    "brown-boats-bathe",
+    "brown-spies-burn",
     "chilled-hornets-move",
     "clean-pianos-listen",
     "clever-apes-collect",
@@ -58,73 +62,121 @@
     "cool-comics-burn",
     "cool-glasses-bake",
     "cuddly-feet-approve",
+    "curly-monkeys-tell",
     "dry-walls-check",
     "dull-mangos-press",
     "eight-pumas-float",
     "eleven-paws-join",
     "famous-boats-tease",
+    "fast-colts-relax",
+    "fast-melons-listen",
+    "few-poems-vanish",
     "few-students-share",
+    "fifty-lions-think",
     "five-toes-destroy",
+    "flat-onions-punch",
+    "friendly-walls-repair",
     "funny-swans-destroy",
+    "gorgeous-cycles-guess",
     "gorgeous-gorillas-compete",
     "green-bags-wink",
+    "green-pens-battle",
+    "hot-buckets-behave",
     "hot-fishes-retire",
+    "hot-wasps-sin",
+    "hungry-sloths-promise",
+    "itchy-chairs-itch",
     "khaki-apricots-design",
     "khaki-poems-lay",
+    "large-seahorses-cheat",
     "late-icons-lie",
     "late-steaks-behave",
     "lazy-files-lay",
     "lemon-jobs-repair",
+    "lemon-sloths-hide",
     "light-bulldogs-press",
     "light-dragons-complain",
     "little-crabs-cross",
     "long-fireants-search",
+    "long-hounds-wave",
     "loud-actors-remember",
     "lovely-drinks-flash",
+    "lucky-items-film",
     "many-ligers-pump",
     "mighty-camels-joke",
+    "mighty-eggs-grab",
     "mighty-flowers-train",
+    "mighty-parrots-sin",
+    "modern-stingrays-end",
     "nasty-jars-pump",
+    "nervous-baboons-sin",
+    "nervous-planets-sparkle",
+    "nervous-seas-shave",
     "new-pants-beg",
     "new-rivers-tell",
     "nice-bulldogs-turn",
     "ninety-pets-travel",
+    "odd-beds-wonder",
     "odd-poets-own",
     "pink-pumas-rhyme",
+    "plenty-ducks-beam",
     "polite-ducks-switch",
+    "polite-pears-grow",
+    "polite-pots-walk",
     "polite-rockets-matter",
     "poor-flowers-cross",
     "purple-garlics-shop",
+    "purple-spiders-care",
     "rare-lamps-promise",
     "rare-roses-float",
     "real-planets-stare",
     "rich-kangaroos-unite",
     "rotten-beers-refuse",
     "rotten-dryers-exercise",
+    "rude-toys-compare",
     "selfish-ducks-sort",
+    "serious-hats-rest",
     "shaggy-spoons-taste",
+    "shaggy-weeks-live",
     "sharp-emus-compare",
     "sharp-zebras-serve",
     "shiny-coats-cry",
+    "silly-buses-obey",
+    "silly-forks-kiss",
     "silly-suits-switch",
+    "silver-doors-juggle",
     "six-ligers-exist",
+    "six-rats-hunt",
     "sixty-insects-watch",
     "slow-buses-own",
+    "slow-kiwis-hide",
+    "slow-sloths-retire",
     "smart-needles-move",
     "smart-olives-eat",
+    "sour-pugs-teach",
+    "spicy-frogs-remain",
     "spicy-lamps-smoke",
+    "spicy-terms-bow",
     "strange-ghosts-matter",
     "strange-sheep-pull",
     "strong-lemons-add",
     "strong-owls-know",
+    "strong-phones-smoke",
+    "stupid-adults-sniff",
     "stupid-bulldogs-applaud",
+    "sweet-ducks-remember",
     "sweet-lizards-press",
     "swift-dragons-peel",
     "tall-bees-wave",
+    "tall-masks-repeat",
+    "tame-apricots-clap",
     "tame-guests-know",
     "tender-moose-tell",
     "tender-oranges-rhyme",
+    "tender-turkeys-compete",
+    "thick-carrots-sneeze",
     "thin-parents-heal",
+    "thirty-hotels-raise",
     "thirty-islands-kiss",
     "tidy-balloons-suffer",
     "tidy-dryers-sleep",
@@ -134,8 +186,13 @@
     "tricky-bulldogs-heal",
     "tricky-keys-attack",
     "tricky-ladybugs-unite",
+    "twelve-knives-notice",
     "two-pumas-wait",
+    "violet-clocks-notice",
     "warm-olives-provide",
-    "warm-planes-taste"
+    "warm-planes-taste",
+    "yellow-roses-arrive",
+    "young-jars-wait",
+    "young-snails-sell"
   ]
 }

.changeset/purple-spiders-care.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Await file watcher cleanup in dev

.changeset/rude-toys-compare.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Capture and display stderr on index failures

.changeset/serious-hats-rest.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+v3: [prod] force flush timeout should be 1s

.changeset/shaggy-weeks-live.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+v3: sanitize errors with null unicode characters in some places

.changeset/silly-buses-obey.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Add callback to checkpoint created message

.changeset/silly-forks-kiss.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+v3: Copy over more of the project's package.json keys into the deployed package.json (support for custom config like zenstack)

.changeset/silver-doors-juggle.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Make deduplicationKey required when creating/updating a schedule

.changeset/six-rats-hunt.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Improved ESM module require error detection logic

.changeset/slow-kiwis-hide.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+v3: vercel edge runtime support

.changeset/slow-sloths-retire.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/core-apps": patch
+"@trigger.dev/core": patch
+---
+
+- Fix uncaught provider exception
+- Remove unused provider messages

.changeset/sour-pugs-teach.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+v3: fix otel flushing causing CLEANUP ack timeout errors by always setting a forceFlushTimeoutMillis value

.changeset/spicy-frogs-remain.md

@@ -0,0 +1,10 @@
+---
+"trigger.dev": patch
+---
+
+- Prevent downgrades during update check and advise to upgrade CLI
+- Detect bun and use npm instead
+- During init, fail early and advise if not a TypeScript project
+- During init, allow specifying custom package manager args
+- Add links to dev worker started message
+- Fix links in unsupported terminals

.changeset/spicy-terms-bow.md

@@ -0,0 +1,9 @@
+---
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+---
+
+- Fix init command SDK pinning
+- Show --api-url / -a flag where needed
+- CLI now also respects `TRIGGER_TELEMETRY_DISABLED`
+- Dedicated docker checkpoint test function

.changeset/strong-phones-smoke.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+fix: allow command login to read api url from cli args

.changeset/stupid-adults-sniff.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Management SDK overhaul and adding the runs.list API

.changeset/sweet-ducks-remember.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+v3: Adding SDK functions for triggering tasks in a typesafe way, without importing task file

.changeset/tall-masks-repeat.md

@@ -0,0 +1,9 @@
+---
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+- Fix artifact detection logs
+- Fix OOM detection and error messages
+- Add test link to cli deployment completion

.changeset/tame-apricots-clap.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+v3: postInstall config option now replaces the postinstall script found in package.json

.changeset/tender-turkeys-compete.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Added timezone support to schedules

.changeset/thick-carrots-sneeze.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/core": patch
+"@trigger.dev/sdk": patch
+---
+
+v3: Include presigned urls for downloading large payloads and outputs when using runs.retrieve

.changeset/thirty-hotels-raise.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Added declarative cron schedules

.changeset/twelve-knives-notice.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+v3: fix missing init output in task run function when no middleware is defined

.changeset/violet-clocks-notice.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix jsonc-parser import

.changeset/yellow-roses-arrive.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Add more package exports that can be used from the web app

.changeset/young-jars-wait.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix various e2e issues for 'resolve-legacy-peer-deps' fixture, installation of fixture deps and lockfile-based test skipping'

.changeset/young-snails-sell.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Increase cleanup IPC timeout

.env.example

@@ -11,7 +11,6 @@ REMIX_APP_PORT=3030
 APP_ENV=development
 APP_ORIGIN=http://localhost:3030
 NODE_ENV=development
-CLOUD_ENV=development
 V3_ENABLED=true
 
 # Redis is used for the v3 queuing and v2 concurrency control
@@ -25,6 +24,8 @@ DEV_OTEL_BATCH_PROCESSING_ENABLED="0"
 # OPTIONAL VARIABLES
 # This is used for validating emails that are allowed to log in. Every email that do not match this regex will be rejected.
 # WHITELISTED_EMAILS="authorized@yahoo\.com|authorized@gmail\.com"
+# Accounts with these emails will get global admin rights. This grants access to the admin UI.
+# ADMIN_EMAILS="admin@example\.com|another-admin@example\.com"
 # This is used for logging in via GitHub. You can leave these commented out if you don't want to use GitHub for authentication.
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
@@ -69,7 +70,7 @@ COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl
 # OBJECT_STORE_BASE_URL="https://{bucket}.{accountId}.r2.cloudflarestorage.com"
 # OBJECT_STORE_ACCESS_KEY_ID=
 # OBJECT_STORE_SECRET_ACCESS_KEY=
-# RUNTIME_WAIT_THRESHOLD_IN_MS=10000
+# CHECKPOINT_THRESHOLD_IN_MS=10000
 
 # These control the server-side internal telemetry
 # INTERNAL_OTEL_TRACE_EXPORTER_URL=<URL to send traces to>
@@ -77,4 +78,4 @@ COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl
 # INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE=<Auth token value>
 # INTERNAL_OTEL_TRACE_LOGGING_ENABLED=1
 # INTERNAL_OTEL_TRACE_SAMPING_RATE=20 # this means 1/20 traces or 5% of traces will be sampled (sampled = recorded)
-# INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED=0,
+# INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED=0,

.github/workflows/e2e.yml

@@ -1,9 +1,56 @@
-name: "🧪 E2E Tests"
+name: "E2E"
 on:
   workflow_call:
+    inputs:
+      package:
+        description: The identifier of the job to run
+        default: webapp
+        required: false
+        type: string
 jobs:
-  e2e:
-    name: "🧪 E2E Tests"
+  cli-v3:
+    name: "🧪 CLI v3 tests"
+    if: inputs.package == 'cli-v3' || inputs.package == ''
+    runs-on: buildjet-8vcpu-ubuntu-2204
+    strategy:
+      fail-fast: false
+      matrix:
+        package-manager: ["npm", "pnpm", "yarn"]
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 8.15.5
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v3
+        with:
+          node-version: 20.11.1
+          cache: "pnpm"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile --filter trigger.dev...
+
+      - name: 🔧 Build v3 cli monorepo dependencies
+        run: pnpm run build --filter trigger.dev^...
+
+      - name: 🔧 Build worker template files
+        run: pnpm --filter trigger.dev run build:workers
+
+      - name: Enable corepack
+        run: corepack enable
+
+      - name: Run E2E Tests
+        run: |
+          PM=${{ matrix.package-manager }} pnpm --filter trigger.dev run test:e2e
+  webapp:
+    name: "🧪 Webapp tests"
+    if: inputs.package == 'webapp' || inputs.package == ''
     runs-on: buildjet-16vcpu-ubuntu-2204
     steps:
       - name: 🐳 Login to Docker Hub
@@ -19,7 +66,7 @@ jobs:
           submodules: recursive
 
       - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
         with:
           version: 8.15.5
 

.github/workflows/pr_checks.yml

@@ -29,4 +29,6 @@ jobs:
 
   # e2e:
   #   uses: ./.github/workflows/e2e.yml
+  #   with:
+  #     package: webapp
   #   secrets: inherit

.github/workflows/publish-docker.yml

@@ -39,11 +39,25 @@ jobs:
             exit 1
           fi
           echo "::set-output name=version::${IMAGE_TAG}"
+
       - name: 🔢 Get the commit hash
         id: get_commit
         run: |
           echo ::set-output name=sha_short::$(echo ${{ github.sha }} | cut -c1-7)
 
+      - name: 📛 Set the tags
+        id: set_tags
+        run: |
+          ref_without_tag=ghcr.io/triggerdotdev/trigger.dev
+          image_tags=$ref_without_tag:${{ steps.get_version.outputs.version }}
+
+          # if it's a versioned tag, also tag it as v3
+          if [[ "${{ github.ref_name }}" == v.docker.* ]]; then
+            image_tags=$image_tags,$ref_without_tag:v3
+          fi
+          
+          echo "IMAGE_TAGS=${image_tags}" >> "$GITHUB_OUTPUT"
+
       - name: 🐙 Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:
@@ -56,6 +70,5 @@ jobs:
         with:
           file: ./docker/Dockerfile
           platforms: linux/amd64,linux/arm64
-          tags: |
-            ghcr.io/triggerdotdev/trigger.dev:${{ steps.get_version.outputs.version }}
+          tags: ${{ steps.set_tags.outputs.IMAGE_TAGS }}
           push: true

.github/workflows/publish-infra.yml

@@ -1,6 +1,7 @@
 name: "🚢 Publish Infra Images"
 
 on:
+  workflow_call:
   push:
     tags:
       - "infra-dev-*"
@@ -29,9 +30,6 @@ permissions:
   packages: write
   contents: read
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-
 env:
   AWS_REGION: us-east-1
 
@@ -39,7 +37,7 @@ jobs:
   build:
     strategy:
       matrix:
-        package: [coordinator, kubernetes-provider]
+        package: [coordinator, docker-provider, kubernetes-provider]
     runs-on: buildjet-16vcpu-ubuntu-2204
     env:
       DOCKER_BUILDKIT: "1"
@@ -48,20 +46,40 @@ jobs:
 
       - name: Generate image reference
         id: prep
-        # WARNING: This step expects the workflow to have been triggered by a specific tag format of: infra-${env}-*
         run: |
-          env=$(echo ${{ github.ref_name }} | cut -d- -f2)
-          sha=${GITHUB_SHA::7}
-          ts=$(date +%s)
+          # set image repo
           if [[ "${{ matrix.package }}" == *-provider ]]; then
-            provider_type=$(echo ${{ matrix.package }} | cut -d- -f1)
+            provider_type=$(echo "${{ matrix.package }}" | cut -d- -f1)
             repository=provider/${provider_type}
           else
-            repository=${{ matrix.package }}
+            repository="${{ matrix.package }}"
           fi
-          echo "IMAGE_TAG=${env}-${sha}-${ts}" >> "$GITHUB_OUTPUT"
           echo "REPOSITORY=${repository}" >> "$GITHUB_OUTPUT"
 
+          # set image tag
+          if [[ "${{ github.ref_type }}" == "tag" ]]; then
+            if [[ "${{ github.ref_name }}" == infra-*-* ]]; then
+              env=$(echo ${{ github.ref_name }} | cut -d- -f2)
+              sha=$(echo ${{ github.sha }} | head -c7)
+              ts=$(date +%s)
+              image_tag=${env}-${sha}-${ts}
+            elif [[ "${{ github.ref_name }}" == v.docker.* ]]; then
+              version="${GITHUB_REF_NAME#v.docker.}"
+              image_tag="v${version}"
+            elif [[ "${{ github.ref_name }}" == build-* ]]; then
+              image_tag="${GITHUB_REF_NAME#build-}"
+            else
+              echo "Invalid tag: ${{ github.ref_name }}"
+              exit 1
+            fi
+          elif [[ "${{ github.ref_name }}" == "main" ]]; then
+            image_tag="main"
+          else
+            echo "Invalid reference: ${{ github.ref }}"
+            exit 1
+          fi
+          echo "IMAGE_TAG=${image_tag}" >> "$GITHUB_OUTPUT"
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
@@ -92,3 +110,12 @@ jobs:
           REGISTRY: ghcr.io/triggerdotdev
           REPOSITORY: ${{ steps.prep.outputs.REPOSITORY }}
           IMAGE_TAG: ${{ steps.prep.outputs.IMAGE_TAG }}
+
+      - name: 🐙 Push 'v3' tag to GitHub Container Registry
+        if: startsWith(github.ref_name, 'v.docker.')
+        run: |
+          docker tag infra_image $REGISTRY/$REPOSITORY:v3
+          docker push $REGISTRY/$REPOSITORY:v3
+        env:
+          REGISTRY: ghcr.io/triggerdotdev
+          REPOSITORY: ${{ steps.prep.outputs.REPOSITORY }}

.github/workflows/publish.yml

@@ -49,11 +49,18 @@ jobs:
     uses: ./.github/workflows/unit-tests.yml
     secrets: inherit
 
-  # e2e:
-  #   uses: ./.github/workflows/e2e.yml
-  #   secrets: inherit
+  e2e:
+    uses: ./.github/workflows/e2e.yml
+    with:
+      package: cli-v3
+    secrets: inherit
 
   publish:
     needs: [typecheck, units]
     uses: ./.github/workflows/publish-docker.yml
     secrets: inherit
+
+  publish-infra:
+    needs: [typecheck, units]
+    uses: ./.github/workflows/publish-infra.yml
+    secrets: inherit

.github/workflows/release.yml

@@ -28,7 +28,7 @@ jobs:
           fetch-depth: 0
 
       - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
         with:
           version: 8.15.5
 

.github/workflows/typecheck.yml

@@ -12,7 +12,7 @@ jobs:
           fetch-depth: 0
 
       - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
         with:
           version: 8.15.5
 

.github/workflows/unit-tests.yml

@@ -12,7 +12,7 @@ jobs:
           fetch-depth: 0
 
       - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
         with:
           version: 8.15.5
 

README.md

@@ -5,64 +5,71 @@
   <img alt="Trigger.dev logo" src="https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/a45d1fa2-0ae8-4a39-4409-f4f934bfae00/public">
 </picture>
   
-### The open source background jobs platform
+### Open source background jobs with no timeouts
 
 [Discord](https://trigger.dev/discord) | [Website](https://trigger.dev) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Docs](https://trigger.dev/docs)
 
 [![Twitter](https://img.shields.io/twitter/url/https/twitter.com/triggerdotdev.svg?style=social&label=Follow%20%40trigger.dev)](https://twitter.com/triggerdotdev)
-[![GitHub Repo stars](https://img.shields.io/github/stars/triggerdotdev/trigger.dev?style=social)](https://github.com/triggerdotdev/trigger.dev)
 
 </div>
 
-> The Trigger.dev v3 developer preview is now open. For more information and to get early access, check out our [developer preview launch post](https://trigger.dev/blog/v3-developer-preview-launch/).
-
 ## About Trigger.dev
 
-Create long-running jobs directly in your codebase with features like API integrations, webhooks, scheduling and delays.
+Trigger.dev is an open source platform and SDK which allows you to create long-running background jobs with no timeouts. Write normal async code, deploy, and never hit a timeout.
 
-## Long running Jobs on serverless
+#### Features:
 
-Reliably run jobs and don’t worry about function timeouts, we handle those for you.
-
-- Auto-resume after a function timeout
-- Auto-resume after a server outage
-- Add delays of up to a year
+- JavaScript and TypeScript SDK
+- Write reliable code by default
+- No infrastructure to manage
+- Works with your existing tech stack
 
 ## In your codebase
 
-Create Jobs where they belong: in your codebase. Version control, localhost, test, review, and deploy like you're already used to.
+Create tasks where they belong: in your codebase. Version control, localhost, test and review like you're already used to.
 
-## Secure by design
+```ts
+import { task } from "@trigger.dev/sdk/v3";
 
-We only receive Triggers and the data you choose to send to us. You can even completely self-host the entire platform.
+//1. You need to export each task
+export const helloWorld = task({
+  //2. Use a unique id for each task
+  id: "hello-world",
+  //3. The run function is the main function of the task
+  run: async (payload: { message: string }) => {
+    //4. You can write code that runs for a long time here, there are no timeouts
+    console.log(payload.message);
+  },
+});
+```
 
-## Don't worry about deployment
+## Deployment
 
-Just use our SDK to write Jobs in your codebase. There's nothing extra to deploy and no CI to configure, your Jobs just connect to our cloud. Or you can always self-host.
+Use our SDK to write tasks in your codebase. There's no infrastructure to manage, your tasks automatically scale and connect to our cloud. Or you can always [self-host](https://trigger.dev/docs/v3/open-source-self-hosting#overview).
+
+## Environments
+
+We support `Development`, `Staging`, and `Production` environments, allowing you to test your tasks before deploying them to production.
 
 ## Full visibility of every job run
 
-View every Task in every Run so you can tell exactly what happened.
+View every task in every run so you can tell exactly what happened. We provide a full trace view of every task run so you can see what happened at every step.
 
-![image](https://www.trigger.dev/build/_assets/web-app-2QFKXFLW.png)
-
-## Built-in integrations
-
-Easily integrate with hundreds of third-party APIs – including your own. Use API keys (which never leave your server) or let us handle OAuth for you. Install our integration packages and easily subscribe to webhooks and perform common tasks, or you can easily use your existing favorite Node.JS SDKs and get resumability and idempotency through our `runTask` function.
+![Trace view image](https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/7c1b347f-004c-4482-38a7-3f6fa9c00d00/public)
 
 # Getting started
 
-Visit our docs [here](https://trigger.dev/docs).
+Visit our docs [here](https://trigger.dev/docs/v3/introduction) for a full guide on how to get started with Trigger.dev.
 
 ## Self-host
 
-We provide an official trigger.dev docker image you can use to easily self-host the platform. We're working on more extensive guides but we currently provide a [Fly.io example repository](https://github.com/triggerdotdev/fly.io) with instructions in the README for deploying and using a self-hosted instance of Trigger.dev on Fly.io.
+If you prefer to self-host, you can follow our [self-hosting guide](https://trigger.dev/docs/v3/open-source-self-hosting#overview).
 
 ## Development
 
 To setup and develop locally or contribute to the open source project, follow our [development guide](./CONTRIBUTING.md).
 
-## Meet the Amazing People Behind This Project 🚀
+## Meet the Amazing People Behind This Project:
 
 <a href="https://github.com/triggerdotdev/trigger.dev/graphs/contributors">
   <img src="https://contrib.rocks/image?repo=triggerdotdev/trigger.dev" />

apps/coordinator/Containerfile

@@ -1,19 +1,19 @@
 # syntax=docker/dockerfile:labs
 
-FROM node:18-bullseye-slim@sha256:a4edd54dcfdcacc8a4100fee71498e8671d99556a1acf5614539214a70092426 AS node-18
+FROM node:20-bookworm-slim@sha256:72f2f046a5f8468db28730b990b37de63ce93fd1a72a40f531d6aa82afdf0d46 AS node-20
 
 WORKDIR /app
 
-FROM node-18 AS pruner
+FROM node-20 AS pruner
 
 COPY --chown=node:node . .
 RUN npx -q turbo@1.10.9 prune --scope=coordinator --docker
 RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
 
-FROM node-18 AS base
+FROM node-20 AS base
 
 RUN apt-get update \
-  && apt-get install -y buildah ca-certificates dumb-init \
+  && apt-get install -y buildah ca-certificates dumb-init docker.io \
   && rm -rf /var/lib/apt/lists/*
 
 COPY --chown=node:node .gitignore .gitignore

apps/coordinator/package.json

@@ -21,8 +21,7 @@
     "execa": "^8.0.1",
     "nanoid": "^5.0.6",
     "prom-client": "^15.1.0",
-    "socket.io": "4.7.4",
-    "socket.io-client": "4.7.4"
+    "socket.io": "4.7.4"
   },
   "devDependencies": {
     "@types/node": "^18",

apps/coordinator/src/chaosMonkey.ts

@@ -0,0 +1,99 @@
+import type { Execa$ } from "execa";
+import { setTimeout as timeout } from "node:timers/promises";
+
+class ChaosMonkeyError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ChaosMonkeyError";
+  }
+}
+
+export class ChaosMonkey {
+  private chaosEventRate = 0.2;
+  private delayInSeconds = 45;
+
+  constructor(
+    private enabled = false,
+    private disableErrors = false,
+    private disableDelays = false
+  ) {
+    if (this.enabled) {
+      console.log("🍌 Chaos monkey enabled");
+    }
+  }
+
+  static Error = ChaosMonkeyError;
+
+  enable() {
+    this.enabled = true;
+    console.log("🍌 Chaos monkey enabled");
+  }
+
+  disable() {
+    this.enabled = false;
+    console.log("🍌 Chaos monkey disabled");
+  }
+
+  async call({
+    $,
+    throwErrors = !this.disableErrors,
+    addDelays = !this.disableDelays,
+  }: {
+    $?: Execa$<string>;
+    throwErrors?: boolean;
+    addDelays?: boolean;
+  } = {}) {
+    if (!this.enabled) {
+      return;
+    }
+
+    const random = Math.random();
+
+    if (random > this.chaosEventRate) {
+      // Don't interfere with normal operation
+      return;
+    }
+
+    const chaosEvents: Array<() => Promise<any>> = [];
+
+    if (addDelays) {
+      chaosEvents.push(async () => {
+        console.log("🍌 Chaos monkey: Add delay");
+
+        if ($) {
+          await $`sleep ${this.delayInSeconds}`;
+        } else {
+          await timeout(this.delayInSeconds * 1000);
+        }
+      });
+    }
+
+    if (throwErrors) {
+      chaosEvents.push(async () => {
+        console.log("🍌 Chaos monkey: Throw error");
+
+        if ($) {
+          await $`false`;
+        } else {
+          throw new ChaosMonkey.Error("🍌 Chaos monkey: Throw error");
+        }
+      });
+    }
+
+    if (chaosEvents.length === 0) {
+      console.error("🍌 Chaos monkey: No events selected");
+      return;
+    }
+
+    const randomIndex = Math.floor(Math.random() * chaosEvents.length);
+
+    const chaosEvent = chaosEvents[randomIndex];
+
+    if (!chaosEvent) {
+      console.error("🍌 Chaos monkey: No event found");
+      return;
+    }
+
+    await chaosEvent();
+  }
+}

apps/coordinator/src/checkpointer.ts

@@ -0,0 +1,593 @@
+import { ExponentialBackoff } from "@trigger.dev/core-apps/backoff";
+import { isExecaChildProcess, testDockerCheckpoint } from "@trigger.dev/core-apps/checkpoints";
+import { SimpleLogger } from "@trigger.dev/core-apps/logger";
+import { $ } from "execa";
+import { nanoid } from "nanoid";
+import fs from "node:fs/promises";
+import { ChaosMonkey } from "./chaosMonkey";
+
+type CheckpointerInitializeReturn = {
+  canCheckpoint: boolean;
+  willSimulate: boolean;
+};
+
+type CheckpointAndPushOptions = {
+  runId: string;
+  leaveRunning?: boolean;
+  projectRef: string;
+  deploymentVersion: string;
+  shouldHeartbeat?: boolean;
+  attemptNumber?: number;
+};
+
+type CheckpointAndPushResult =
+  | { success: true; checkpoint: CheckpointData }
+  | {
+      success: false;
+      reason?: "CANCELED" | "DISABLED" | "ERROR" | "IN_PROGRESS" | "NO_SUPPORT" | "SKIP_RETRYING";
+    };
+
+type CheckpointData = {
+  location: string;
+  docker: boolean;
+};
+
+type CheckpointerOptions = {
+  dockerMode: boolean;
+  forceSimulate: boolean;
+  heartbeat: (runId: string) => void;
+  registryHost?: string;
+  registryNamespace?: string;
+  registryTlsVerify?: boolean;
+  disableCheckpointSupport?: boolean;
+  checkpointPath?: string;
+  simulateCheckpointFailure?: boolean;
+  simulateCheckpointFailureSeconds?: number;
+  simulatePushFailure?: boolean;
+  simulatePushFailureSeconds?: number;
+  chaosMonkey?: ChaosMonkey;
+};
+
+async function getFileSize(filePath: string): Promise<number> {
+  try {
+    const stats = await fs.stat(filePath);
+    return stats.size;
+  } catch (error) {
+    console.error("Error getting file size:", error);
+    return -1;
+  }
+}
+
+async function getParsedFileSize(filePath: string) {
+  const sizeInBytes = await getFileSize(filePath);
+
+  let message = `Size in bytes: ${sizeInBytes}`;
+
+  if (sizeInBytes > 1024 * 1024) {
+    const sizeInMB = (sizeInBytes / 1024 / 1024).toFixed(2);
+    message = `Size in MB (rounded): ${sizeInMB}`;
+  } else if (sizeInBytes > 1024) {
+    const sizeInKB = (sizeInBytes / 1024).toFixed(2);
+    message = `Size in KB (rounded): ${sizeInKB}`;
+  }
+
+  return {
+    path: filePath,
+    sizeInBytes,
+    message,
+  };
+}
+
+export class Checkpointer {
+  #initialized = false;
+  #canCheckpoint = false;
+  #dockerMode: boolean;
+
+  #logger = new SimpleLogger("[checkptr]");
+  #abortControllers = new Map<string, AbortController>();
+  #failedCheckpoints = new Map<string, unknown>();
+  #waitingForRetry = new Set<string>();
+
+  private registryHost: string;
+  private registryNamespace: string;
+  private registryTlsVerify: boolean;
+
+  private disableCheckpointSupport: boolean;
+  private checkpointPath: string;
+
+  private simulateCheckpointFailure: boolean;
+  private simulateCheckpointFailureSeconds: number;
+  private simulatePushFailure: boolean;
+  private simulatePushFailureSeconds: number;
+
+  private chaosMonkey: ChaosMonkey;
+
+  constructor(private opts: CheckpointerOptions) {
+    this.#dockerMode = opts.dockerMode;
+
+    this.registryHost = opts.registryHost ?? "localhost:5000";
+    this.registryNamespace = opts.registryNamespace ?? "trigger";
+    this.registryTlsVerify = opts.registryTlsVerify ?? true;
+
+    this.disableCheckpointSupport = opts.disableCheckpointSupport ?? false;
+    this.checkpointPath = opts.checkpointPath ?? "/checkpoints";
+
+    this.simulateCheckpointFailure = opts.simulateCheckpointFailure ?? false;
+    this.simulateCheckpointFailureSeconds = opts.simulateCheckpointFailureSeconds ?? 300;
+    this.simulatePushFailure = opts.simulatePushFailure ?? false;
+    this.simulatePushFailureSeconds = opts.simulatePushFailureSeconds ?? 300;
+
+    this.chaosMonkey = opts.chaosMonkey ?? new ChaosMonkey(!!process.env.CHAOS_MONKEY_ENABLED);
+  }
+
+  async init(): Promise<CheckpointerInitializeReturn> {
+    if (this.#initialized) {
+      return this.#getInitReturn(this.#canCheckpoint);
+    }
+
+    this.#logger.log(`${this.#dockerMode ? "Docker" : "Kubernetes"} mode`);
+
+    if (this.#dockerMode) {
+      const testCheckpoint = await testDockerCheckpoint();
+
+      if (testCheckpoint.ok) {
+        return this.#getInitReturn(true);
+      }
+
+      this.#logger.error(testCheckpoint.message, testCheckpoint.error ?? "");
+      return this.#getInitReturn(false);
+    } else {
+      try {
+        await $`buildah login --get-login ${this.registryHost}`;
+      } catch (error) {
+        this.#logger.error(`No checkpoint support: Not logged in to registry ${this.registryHost}`);
+        return this.#getInitReturn(false);
+      }
+    }
+
+    return this.#getInitReturn(true);
+  }
+
+  #getInitReturn(canCheckpoint: boolean): CheckpointerInitializeReturn {
+    this.#canCheckpoint = canCheckpoint;
+
+    if (canCheckpoint) {
+      if (!this.#initialized) {
+        this.#logger.log("Full checkpoint support!");
+      }
+    }
+
+    this.#initialized = true;
+
+    const willSimulate = this.#dockerMode && (!this.#canCheckpoint || this.opts.forceSimulate);
+
+    if (willSimulate) {
+      this.#logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", {
+        forceSimulate: this.opts.forceSimulate,
+      });
+    }
+
+    return {
+      canCheckpoint,
+      willSimulate,
+    };
+  }
+
+  #getImageRef(projectRef: string, deploymentVersion: string, shortCode: string) {
+    return `${this.registryHost}/${this.registryNamespace}/${projectRef}:${deploymentVersion}.prod-${shortCode}`;
+  }
+
+  #getExportLocation(projectRef: string, deploymentVersion: string, shortCode: string) {
+    const basename = `${projectRef}-${deploymentVersion}-${shortCode}`;
+
+    if (this.#dockerMode) {
+      return basename;
+    } else {
+      return `${this.checkpointPath}/${basename}.tar`;
+    }
+  }
+
+  async checkpointAndPush(opts: CheckpointAndPushOptions): Promise<CheckpointData | undefined> {
+    const start = performance.now();
+    this.#logger.log(`checkpointAndPush() start`, { start, opts });
+
+    let interval: NodeJS.Timer | undefined;
+
+    if (opts.shouldHeartbeat) {
+      interval = setInterval(() => {
+        this.#logger.log("Sending heartbeat", { runId: opts.runId });
+        this.opts.heartbeat(opts.runId);
+      }, 20_000);
+    }
+
+    try {
+      const result = await this.#checkpointAndPushWithBackoff(opts);
+
+      const end = performance.now();
+      this.#logger.log(`checkpointAndPush() end`, {
+        start,
+        end,
+        diff: end - start,
+        opts,
+        success: result.success,
+      });
+
+      if (!result.success) {
+        return;
+      }
+
+      return result.checkpoint;
+    } finally {
+      if (opts.shouldHeartbeat) {
+        clearInterval(interval);
+      }
+    }
+  }
+
+  isCheckpointing(runId: string) {
+    return this.#abortControllers.has(runId) || this.#waitingForRetry.has(runId);
+  }
+
+  cancelCheckpoint(runId: string): boolean {
+    // If the last checkpoint failed, pretend we canceled it
+    // This ensures tasks don't wait for external resume messages to continue
+    if (this.#hasFailedCheckpoint(runId)) {
+      this.#clearFailedCheckpoint(runId);
+      return true;
+    }
+
+    if (this.#waitingForRetry.has(runId)) {
+      this.#waitingForRetry.delete(runId);
+      return true;
+    }
+
+    const controller = this.#abortControllers.get(runId);
+
+    if (!controller) {
+      this.#logger.debug("Nothing to cancel", { runId });
+      return false;
+    }
+
+    controller.abort("cancelCheckpointing()");
+    this.#abortControllers.delete(runId);
+
+    return true;
+  }
+
+  async #checkpointAndPushWithBackoff({
+    runId,
+    leaveRunning = true, // This mirrors kubernetes behaviour more accurately
+    projectRef,
+    deploymentVersion,
+    attemptNumber,
+  }: CheckpointAndPushOptions): Promise<CheckpointAndPushResult> {
+    this.#logger.log("Checkpointing with backoff", {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+    });
+
+    const backoff = new ExponentialBackoff()
+      .type("EqualJitter")
+      .base(3)
+      .max(3 * 3600)
+      .maxElapsed(48 * 3600);
+
+    for await (const { delay, retry } of backoff) {
+      try {
+        if (retry > 0) {
+          this.#logger.error("Retrying checkpoint", {
+            runId,
+            retry,
+            delay,
+          });
+
+          this.#waitingForRetry.add(runId);
+          await new Promise((resolve) => setTimeout(resolve, delay.milliseconds));
+
+          if (!this.#waitingForRetry.has(runId)) {
+            this.#logger.log("Checkpoint canceled while waiting for retry", { runId });
+            return { success: false, reason: "CANCELED" };
+          } else {
+            this.#waitingForRetry.delete(runId);
+          }
+        }
+
+        const result = await this.#checkpointAndPush({
+          runId,
+          leaveRunning,
+          projectRef,
+          deploymentVersion,
+          attemptNumber,
+        });
+
+        if (result.success) {
+          return result;
+        }
+
+        if (result.reason === "CANCELED") {
+          this.#logger.log("Checkpoint canceled, won't retry", { runId });
+          // Don't fail the checkpoint, as it was canceled
+          return result;
+        }
+
+        if (result.reason === "IN_PROGRESS") {
+          this.#logger.log("Checkpoint already in progress, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        if (result.reason === "NO_SUPPORT") {
+          this.#logger.log("No checkpoint support, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        if (result.reason === "DISABLED") {
+          this.#logger.log("Checkpoint support disabled, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        if (result.reason === "SKIP_RETRYING") {
+          this.#logger.log("Skipping retrying", { runId });
+          return result;
+        }
+
+        continue;
+      } catch (error) {
+        this.#logger.error("Checkpoint error", {
+          retry,
+          runId,
+          delay,
+          error: error instanceof Error ? error.message : error,
+        });
+      }
+    }
+
+    this.#logger.error(`Checkpoint failed after exponential backoff`, {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+    });
+    this.#failCheckpoint(runId, "ERROR");
+
+    return { success: false, reason: "ERROR" };
+  }
+
+  async #checkpointAndPush({
+    runId,
+    leaveRunning = true, // This mirrors kubernetes behaviour more accurately
+    projectRef,
+    deploymentVersion,
+    attemptNumber,
+  }: CheckpointAndPushOptions): Promise<CheckpointAndPushResult> {
+    await this.init();
+
+    const options = {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+      attemptNumber,
+    };
+
+    if (!this.#dockerMode && !this.#canCheckpoint) {
+      this.#logger.error("No checkpoint support. Simulation requires docker.");
+      return { success: false, reason: "NO_SUPPORT" };
+    }
+
+    if (this.isCheckpointing(runId)) {
+      this.#logger.error("Checkpoint procedure already in progress", { options });
+      return { success: false, reason: "IN_PROGRESS" };
+    }
+
+    // This is a new checkpoint, clear any last failure for this run
+    this.#clearFailedCheckpoint(runId);
+
+    if (this.disableCheckpointSupport) {
+      this.#logger.error("Checkpoint support disabled", { options });
+      return { success: false, reason: "DISABLED" };
+    }
+
+    const controller = new AbortController();
+    this.#abortControllers.set(runId, controller);
+
+    const $$ = $({ signal: controller.signal });
+
+    const shortCode = nanoid(8);
+    const imageRef = this.#getImageRef(projectRef, deploymentVersion, shortCode);
+    const exportLocation = this.#getExportLocation(projectRef, deploymentVersion, shortCode);
+
+    const cleanup = async () => {
+      if (this.#dockerMode) {
+        return;
+      }
+
+      try {
+        await $`rm ${exportLocation}`;
+        this.#logger.log("Deleted checkpoint archive", { exportLocation });
+
+        await $`buildah rmi ${imageRef}`;
+        this.#logger.log("Deleted checkpoint image", { imageRef });
+      } catch (error) {
+        this.#logger.error("Failure during checkpoint cleanup", { exportLocation, error });
+      }
+    };
+
+    try {
+      await this.chaosMonkey.call({ $: $$ });
+
+      this.#logger.log("Checkpointing:", { options });
+
+      const containterName = this.#getRunContainerName(runId);
+      const containterNameWithAttempt = this.#getRunContainerName(runId, attemptNumber);
+
+      // Create checkpoint (docker)
+      if (this.#dockerMode) {
+        try {
+          if (this.opts.forceSimulate || !this.#canCheckpoint) {
+            this.#logger.log("Simulating checkpoint");
+            this.#logger.debug(await $$`docker pause ${containterNameWithAttempt}`);
+          } else {
+            if (this.simulateCheckpointFailure) {
+              if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) {
+                this.#logger.error("Simulating checkpoint failure", { options });
+                throw new Error("SIMULATE_CHECKPOINT_FAILURE");
+              }
+            }
+
+            if (leaveRunning) {
+              this.#logger.debug(
+                await $$`docker checkpoint create --leave-running ${containterNameWithAttempt} ${exportLocation}`
+              );
+            } else {
+              this.#logger.debug(
+                await $$`docker checkpoint create ${containterNameWithAttempt} ${exportLocation}`
+              );
+            }
+          }
+        } catch (error) {
+          this.#logger.error("Failed while creating docker checkpoint", { exportLocation });
+          throw error;
+        }
+
+        this.#logger.log("checkpoint created:", {
+          runId,
+          location: exportLocation,
+        });
+
+        return {
+          success: true,
+          checkpoint: {
+            location: exportLocation,
+            docker: true,
+          },
+        };
+      }
+
+      // Create checkpoint (CRI)
+      if (!this.#canCheckpoint) {
+        this.#logger.error("No checkpoint support in kubernetes mode.");
+        return { success: false, reason: "SKIP_RETRYING" };
+      }
+
+      const containerId = this.#logger.debug(
+        // @ts-expect-error
+        await $$`crictl ps`
+          .pipeStdout($$({ stdin: "pipe" })`grep ${containterName}`)
+          .pipeStdout($$({ stdin: "pipe" })`cut -f1 ${"-d "}`)
+      );
+
+      if (!containerId.stdout) {
+        this.#logger.error("could not find container id", { options, containterName });
+        return { success: false, reason: "SKIP_RETRYING" };
+      }
+
+      const start = performance.now();
+
+      if (this.simulateCheckpointFailure) {
+        if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) {
+          this.#logger.error("Simulating checkpoint failure", { options });
+          throw new Error("SIMULATE_CHECKPOINT_FAILURE");
+        }
+      }
+
+      // Create checkpoint
+      this.#logger.debug(await $$`crictl checkpoint --export=${exportLocation} ${containerId}`);
+      const postCheckpoint = performance.now();
+
+      // Print checkpoint size
+      const size = await getParsedFileSize(exportLocation);
+      this.#logger.log("checkpoint archive created", { size, options });
+
+      // Create image from checkpoint
+      const container = this.#logger.debug(await $$`buildah from scratch`);
+      const postFrom = performance.now();
+
+      this.#logger.debug(await $$`buildah add ${container} ${exportLocation} /`);
+      const postAdd = performance.now();
+
+      this.#logger.debug(
+        await $$`buildah config --annotation=io.kubernetes.cri-o.annotations.checkpoint.name=counter ${container}`
+      );
+      const postConfig = performance.now();
+
+      this.#logger.debug(await $$`buildah commit ${container} ${imageRef}`);
+      const postCommit = performance.now();
+
+      this.#logger.debug(await $$`buildah rm ${container}`);
+      const postRm = performance.now();
+
+      if (this.simulatePushFailure) {
+        if (performance.now() < this.simulatePushFailureSeconds * 1000) {
+          this.#logger.error("Simulating push failure", { options });
+          throw new Error("SIMULATE_PUSH_FAILURE");
+        }
+      }
+
+      // Push checkpoint image
+      this.#logger.debug(
+        await $$`buildah push --tls-verify=${String(this.registryTlsVerify)} ${imageRef}`
+      );
+      const postPush = performance.now();
+
+      const perf = {
+        "crictl checkpoint": postCheckpoint - start,
+        "buildah from": postFrom - postCheckpoint,
+        "buildah add": postAdd - postFrom,
+        "buildah config": postConfig - postAdd,
+        "buildah commit": postCommit - postConfig,
+        "buildah rm": postRm - postCommit,
+        "buildah push": postPush - postRm,
+      };
+
+      this.#logger.log("Checkpointed and pushed image to:", { location: imageRef, perf });
+
+      return {
+        success: true,
+        checkpoint: {
+          location: imageRef,
+          docker: false,
+        },
+      };
+    } catch (error) {
+      if (isExecaChildProcess(error)) {
+        if (error.isCanceled) {
+          this.#logger.error("Checkpoint canceled", { options, error });
+
+          return { success: false, reason: "CANCELED" };
+        }
+
+        this.#logger.error("Checkpoint command error", { options, error });
+
+        return { success: false, reason: "ERROR" };
+      }
+
+      this.#logger.error("Unhandled checkpoint error", { options, error });
+
+      return { success: false, reason: "ERROR" };
+    } finally {
+      this.#abortControllers.delete(runId);
+      await cleanup();
+    }
+  }
+
+  #failCheckpoint(runId: string, error: unknown) {
+    this.#failedCheckpoints.set(runId, error);
+  }
+
+  #clearFailedCheckpoint(runId: string) {
+    this.#failedCheckpoints.delete(runId);
+  }
+
+  #hasFailedCheckpoint(runId: string) {
+    return this.#failedCheckpoints.has(runId);
+  }
+
+  #getRunContainerName(suffix: string, attemptNumber?: number) {
+    return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
+  }
+}

apps/docker-provider/Containerfile

@@ -1,16 +1,47 @@
-# syntax=docker/dockerfile:labs
-
-FROM node:18-slim AS base
-
-RUN apt-get update \
-  && apt-get install -y dumb-init
-
-FROM base
+FROM node:20-alpine@sha256:7a91aa397f2e2dfbfcdad2e2d72599f374e0b0172be1d86eeb73f1d33f36a4b2 AS node-20-alpine
 
 WORKDIR /app
 
-COPY --chown=node dist/index.mjs /app/
+FROM node-20-alpine AS pruner
+
+COPY --chown=node:node . .
+RUN npx -q turbo@1.10.9 prune --scope=docker-provider --docker
+RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
+
+FROM node-20-alpine AS base
+
+RUN apk add --no-cache dumb-init docker
+
+COPY --chown=node:node .gitignore .gitignore
+COPY --from=pruner --chown=node:node /app/out/json/ .
+COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
+COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
+
+FROM base AS dev-deps
+RUN corepack enable
+ENV NODE_ENV development
+
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
+
+FROM base AS builder
+RUN corepack enable
+
+COPY --from=pruner --chown=node:node /app/out/full/ .
+COPY --from=dev-deps --chown=node:node /app/ .
+COPY --chown=node:node turbo.json turbo.json
+
+RUN pnpm run -r --filter docker-provider build:bundle
+
+FROM base AS runner
+
+RUN corepack enable
+ENV NODE_ENV production
+
+COPY --from=builder --chown=node:node /app/apps/docker-provider/dist/index.mjs ./index.mjs
 
 EXPOSE 8000
 
-ENTRYPOINT [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "/app/index.mjs" ]
+USER node
+
+CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]

apps/docker-provider/package.json

@@ -18,8 +18,7 @@
   "dependencies": {
     "@trigger.dev/core": "workspace:*",
     "@trigger.dev/core-apps": "workspace:*",
-    "execa": "^8.0.1",
-    "socket.io-client": "^4.7.4"
+    "execa": "^8.0.1"
   },
   "devDependencies": {
     "@types/node": "^18.19.8",

apps/docker-provider/src/index.ts

@@ -1,12 +1,13 @@
 import { $, type ExecaChildProcess, execa } from "execa";
 import {
-  SimpleLogger,
-  TaskOperations,
   ProviderShell,
-  TaskOperationsRestoreOptions,
+  TaskOperations,
   TaskOperationsCreateOptions,
   TaskOperationsIndexOptions,
-} from "@trigger.dev/core-apps";
+  TaskOperationsRestoreOptions,
+} from "@trigger.dev/core-apps/provider";
+import { SimpleLogger } from "@trigger.dev/core-apps/logger";
+import { isExecaChildProcess, testDockerCheckpoint } from "@trigger.dev/core-apps/checkpoints";
 import { setTimeout } from "node:timers/promises";
 import { PostStartCauses, PreStopCauses } from "@trigger.dev/core/v3";
 
@@ -23,70 +24,61 @@ const FORCE_CHECKPOINT_SIMULATION = ["1", "true"].includes(
 
 const logger = new SimpleLogger(`[${MACHINE_NAME}]`);
 
-type InitializeReturn = {
+type TaskOperationsInitReturn = {
   canCheckpoint: boolean;
   willSimulate: boolean;
 };
 
-function isExecaChildProcess(maybeExeca: unknown): maybeExeca is Awaited<ExecaChildProcess> {
-  return typeof maybeExeca === "object" && maybeExeca !== null && "escapedCommand" in maybeExeca;
-}
-
 class DockerTaskOperations implements TaskOperations {
   #initialized = false;
   #canCheckpoint = false;
 
   constructor(private opts = { forceSimulate: false }) {}
 
-  async #initialize(): Promise<InitializeReturn> {
+  async init(): Promise<TaskOperationsInitReturn> {
     if (this.#initialized) {
-      return this.#getInitializeReturn();
+      return this.#getInitReturn(this.#canCheckpoint);
     }
 
     logger.log("Initializing task operations");
 
-    if (this.opts.forceSimulate) {
-      logger.log("Forced simulation enabled. Will simulate regardless of checkpoint support.");
+    const testCheckpoint = await testDockerCheckpoint();
+
+    if (testCheckpoint.ok) {
+      return this.#getInitReturn(true);
     }
 
-    try {
-      await $`criu --version`;
-    } catch (error) {
-      logger.error("No checkpoint support: Missing CRIU binary. Will simulate instead.");
-      this.#canCheckpoint = false;
-      this.#initialized = true;
-
-      return this.#getInitializeReturn();
-    }
-
-    try {
-      await $`docker checkpoint`;
-    } catch (error) {
-      logger.error("No checkpoint support: Docker needs to have experimental features enabled");
-      logger.error("Will simulate instead");
-      this.#canCheckpoint = false;
-      this.#initialized = true;
-
-      return this.#getInitializeReturn();
-    }
-
-    logger.log("Full checkpoint support!");
-
-    this.#initialized = true;
-    this.#canCheckpoint = true;
-
-    return this.#getInitializeReturn();
+    logger.error(testCheckpoint.message, testCheckpoint.error);
+    return this.#getInitReturn(false);
   }
 
-  #getInitializeReturn(): InitializeReturn {
+  #getInitReturn(canCheckpoint: boolean): TaskOperationsInitReturn {
+    this.#canCheckpoint = canCheckpoint;
+
+    if (canCheckpoint) {
+      if (!this.#initialized) {
+        logger.log("Full checkpoint support!");
+      }
+    }
+
+    this.#initialized = true;
+
+    const willSimulate = !canCheckpoint || this.opts.forceSimulate;
+
+    if (willSimulate) {
+      logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", {
+        forceSimulate: this.opts.forceSimulate,
+      });
+    }
+
     return {
-      canCheckpoint: this.#canCheckpoint,
-      willSimulate: !this.#canCheckpoint || this.opts.forceSimulate,
+      canCheckpoint,
+      willSimulate,
     };
   }
 
   async index(opts: TaskOperationsIndexOptions) {
-    await this.#initialize();
+    await this.init();
 
     const containerName = this.#getIndexContainerName(opts.shortCode);
 
@@ -95,60 +87,51 @@ class DockerTaskOperations implements TaskOperations {
       port: COORDINATOR_PORT,
     });
 
-    try {
-      logger.debug(
-        await execa("docker", [
-          "run",
-          "--network=host",
-          "--rm",
-          `--env=INDEX_TASKS=true`,
-          `--env=TRIGGER_SECRET_KEY=${opts.apiKey}`,
-          `--env=TRIGGER_API_URL=${opts.apiUrl}`,
-          `--env=TRIGGER_ENV_ID=${opts.envId}`,
-          `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
-          `--env=POD_NAME=${containerName}`,
-          `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
-          `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
-          `--name=${containerName}`,
-          `${opts.imageRef}`,
-        ])
-      );
-    } catch (error: any) {
-      if (!isExecaChildProcess(error)) {
-        throw error;
-      }
-
-      logger.error("Index failed:", {
-        opts,
-        exitCode: error.exitCode,
-        escapedCommand: error.escapedCommand,
-        stdout: error.stdout,
-        stderr: error.stderr,
-      });
-    }
+    logger.debug(
+      await execa("docker", [
+        "run",
+        "--network=host",
+        "--rm",
+        `--env=INDEX_TASKS=true`,
+        `--env=TRIGGER_SECRET_KEY=${opts.apiKey}`,
+        `--env=TRIGGER_API_URL=${opts.apiUrl}`,
+        `--env=TRIGGER_ENV_ID=${opts.envId}`,
+        `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
+        `--env=POD_NAME=${containerName}`,
+        `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
+        `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
+        `--name=${containerName}`,
+        `${opts.imageRef}`,
+      ])
+    );
   }
 
   async create(opts: TaskOperationsCreateOptions) {
-    await this.#initialize();
+    await this.init();
 
-    const containerName = this.#getRunContainerName(opts.runId);
+    const containerName = this.#getRunContainerName(opts.runId, opts.nextAttemptNumber);
+
+    const runArgs = [
+      "run",
+      "--network=host",
+      "--detach",
+      `--env=TRIGGER_ENV_ID=${opts.envId}`,
+      `--env=TRIGGER_RUN_ID=${opts.runId}`,
+      `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
+      `--env=POD_NAME=${containerName}`,
+      `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
+      `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
+      `--name=${containerName}`,
+    ];
+
+    if (process.env.ENFORCE_MACHINE_PRESETS) {
+      runArgs.push(`--cpus=${opts.machine.cpu}`, `--memory=${opts.machine.memory}G`);
+    }
+
+    runArgs.push(`${opts.image}`);
 
     try {
-      logger.debug(
-        await execa("docker", [
-          "run",
-          "--network=host",
-          "--detach",
-          `--env=TRIGGER_ENV_ID=${opts.envId}`,
-          `--env=TRIGGER_RUN_ID=${opts.runId}`,
-          `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
-          `--env=POD_NAME=${containerName}`,
-          `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
-          `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
-          `--name=${containerName}`,
-          `${opts.image}`,
-        ])
-      );
+      logger.debug(await execa("docker", runArgs));
     } catch (error) {
       if (!isExecaChildProcess(error)) {
         throw error;
@@ -165,9 +148,9 @@ class DockerTaskOperations implements TaskOperations {
   }
 
   async restore(opts: TaskOperationsRestoreOptions) {
-    await this.#initialize();
+    await this.init();
 
-    const containerName = this.#getRunContainerName(opts.runId);
+    const containerName = this.#getRunContainerName(opts.runId, opts.attemptNumber);
 
     if (!this.#canCheckpoint || this.opts.forceSimulate) {
       logger.log("Simulating restore");
@@ -194,7 +177,7 @@ class DockerTaskOperations implements TaskOperations {
   }
 
   async delete(opts: { runId: string }) {
-    await this.#initialize();
+    await this.init();
 
     const containerName = this.#getRunContainerName(opts.runId);
     await this.#sendPreStop(containerName);
@@ -203,7 +186,7 @@ class DockerTaskOperations implements TaskOperations {
   }
 
   async get(opts: { runId: string }) {
-    await this.#initialize();
+    await this.init();
 
     logger.log("noop: get");
   }
@@ -212,8 +195,8 @@ class DockerTaskOperations implements TaskOperations {
     return `task-index-${suffix}`;
   }
 
-  #getRunContainerName(suffix: string) {
-    return `task-run-${suffix}`;
+  #getRunContainerName(suffix: string, attemptNumber?: number) {
+    return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
   }
 
   async #sendPostStart(containerName: string): Promise<void> {

apps/docker-provider/tsconfig.json

@@ -1,6 +1,6 @@
 {
   "compilerOptions": {
-    "target": "es2016",
+    "target": "es2018",
     "module": "commonjs",
     "esModuleInterop": true,
     "forceConsistentCasingInFileNames": true,

apps/kubernetes-provider/Containerfile

@@ -1,14 +1,14 @@
-FROM node:18-alpine@sha256:ca9f6cb0466f9638e59e0c249d335a07c867cd50c429b5c7830dda1bed584649 AS node-18-alpine
+FROM node:20-alpine@sha256:7a91aa397f2e2dfbfcdad2e2d72599f374e0b0172be1d86eeb73f1d33f36a4b2 AS node-20-alpine
 
 WORKDIR /app
 
-FROM node-18-alpine AS pruner
+FROM node-20-alpine AS pruner
 
 COPY --chown=node:node . .
 RUN npx -q turbo@1.10.9 prune --scope=kubernetes-provider --docker
 RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
 
-FROM node-18-alpine AS base
+FROM node-20-alpine AS base
 
 RUN apk add --no-cache dumb-init
 

apps/kubernetes-provider/package.json

@@ -19,8 +19,7 @@
     "@kubernetes/client-node": "^0.20.0",
     "@trigger.dev/core": "workspace:*",
     "@trigger.dev/core-apps": "workspace:*",
-    "p-queue": "^8.0.1",
-    "socket.io-client": "^4.7.4"
+    "p-queue": "^8.0.1"
   },
   "devDependencies": {
     "dotenv": "^16.4.2",

apps/kubernetes-provider/src/index.ts

@@ -1,22 +1,37 @@
 import * as k8s from "@kubernetes/client-node";
 import {
   ProviderShell,
-  SimpleLogger,
   TaskOperations,
   TaskOperationsCreateOptions,
   TaskOperationsIndexOptions,
+  TaskOperationsPrePullDeploymentOptions,
   TaskOperationsRestoreOptions,
-} from "@trigger.dev/core-apps";
-import { Machine, PostStartCauses, PreStopCauses, EnvironmentType } from "@trigger.dev/core/v3";
+} from "@trigger.dev/core-apps/provider";
+import { SimpleLogger } from "@trigger.dev/core-apps/logger";
+import {
+  MachinePreset,
+  PostStartCauses,
+  PreStopCauses,
+  EnvironmentType,
+} from "@trigger.dev/core/v3";
 import { randomUUID } from "crypto";
 import { TaskMonitor } from "./taskMonitor";
 import { PodCleaner } from "./podCleaner";
+import { UptimeHeartbeat } from "./uptimeHeartbeat";
 
 const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
 const NODE_NAME = process.env.NODE_NAME || "local";
 const OTEL_EXPORTER_OTLP_ENDPOINT =
   process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318";
 
+const POD_CLEANER_INTERVAL_SECONDS = Number(process.env.POD_CLEANER_INTERVAL_SECONDS || "300");
+
+const UPTIME_HEARTBEAT_URL = process.env.UPTIME_HEARTBEAT_URL;
+const UPTIME_INTERVAL_SECONDS = Number(process.env.UPTIME_INTERVAL_SECONDS || "60");
+const UPTIME_MAX_PENDING_RUNS = Number(process.env.UPTIME_MAX_PENDING_RUNS || "25");
+const UPTIME_MAX_PENDING_INDECES = Number(process.env.UPTIME_MAX_PENDING_INDECES || "10");
+const UPTIME_MAX_PENDING_ERRORS = Number(process.env.UPTIME_MAX_PENDING_ERRORS || "10");
+
 const logger = new SimpleLogger(`[${NODE_NAME}]`);
 logger.log(`running in ${RUNTIME_ENV} mode`);
 
@@ -35,6 +50,7 @@ class KubernetesTaskOperations implements TaskOperations {
   #k8sApi: {
     core: k8s.CoreV1Api;
     batch: k8s.BatchV1Api;
+    apps: k8s.AppsV1Api;
   };
 
   constructor(namespace = "default") {
@@ -47,6 +63,10 @@ class KubernetesTaskOperations implements TaskOperations {
     this.#k8sApi = this.#createK8sApi();
   }
 
+  async init() {
+    // noop
+  }
+
   async index(opts: TaskOperationsIndexOptions) {
     await this.#createJob(
       {
@@ -82,7 +102,7 @@ class KubernetesTaskOperations implements TaskOperations {
                   resources: {
                     limits: {
                       cpu: "1",
-                      memory: "1G",
+                      memory: "2G",
                       "ephemeral-storage": "2Gi",
                     },
                   },
@@ -119,10 +139,12 @@ class KubernetesTaskOperations implements TaskOperations {
   }
 
   async create(opts: TaskOperationsCreateOptions) {
+    const containerName = this.#getRunContainerName(opts.runId, opts.nextAttemptNumber);
+
     await this.#createPod(
       {
         metadata: {
-          name: this.#getRunContainerName(opts.runId),
+          name: containerName,
           namespace: this.#namespace.metadata.name,
           labels: {
             ...this.#getSharedLabels(opts),
@@ -137,7 +159,7 @@ class KubernetesTaskOperations implements TaskOperations {
           terminationGracePeriodSeconds: 60 * 60,
           containers: [
             {
-              name: this.#getRunContainerName(opts.runId),
+              name: containerName,
               image: opts.image,
               ports: [
                 {
@@ -191,7 +213,7 @@ class KubernetesTaskOperations implements TaskOperations {
     await this.#createPod(
       {
         metadata: {
-          name: `${this.#getRunContainerName(opts.runId)}-${randomUUID().slice(0, 8)}`,
+          name: `${this.#getRunContainerName(opts.runId)}-${opts.checkpointId.slice(-8)}`,
           namespace: this.#namespace.metadata.name,
           labels: {
             ...this.#getSharedLabels(opts),
@@ -295,6 +317,72 @@ class KubernetesTaskOperations implements TaskOperations {
     await this.#getPod(opts.runId, this.#namespace);
   }
 
+  async prePullDeployment(opts: TaskOperationsPrePullDeploymentOptions) {
+    const metaName = this.#getPrePullContainerName(opts.shortCode);
+
+    const metaLabels = {
+      ...this.#getSharedLabels(opts),
+      app: "task-prepull",
+      "app.kubernetes.io/part-of": "trigger-worker",
+      "app.kubernetes.io/component": "prepull",
+      deployment: opts.deploymentId,
+      name: metaName,
+    } satisfies k8s.V1ObjectMeta["labels"];
+
+    await this.#createDaemonSet(
+      {
+        metadata: {
+          name: metaName,
+          namespace: this.#namespace.metadata.name,
+          labels: metaLabels,
+        },
+        spec: {
+          selector: {
+            matchLabels: {
+              name: metaName,
+            },
+          },
+          template: {
+            metadata: {
+              labels: metaLabels,
+            },
+            spec: {
+              ...this.#defaultPodSpec,
+              restartPolicy: "Always",
+              initContainers: [
+                {
+                  name: "prepull",
+                  image: opts.imageRef,
+                  command: ["/usr/bin/true"],
+                  resources: {
+                    limits: {
+                      cpu: "0.25",
+                      memory: "100Mi",
+                      "ephemeral-storage": "1Gi",
+                    },
+                  },
+                },
+              ],
+              containers: [
+                {
+                  name: "pause",
+                  image: "registry.k8s.io/pause:3.9",
+                  resources: {
+                    limits: {
+                      cpu: "1m",
+                      memory: "12Mi",
+                    },
+                  },
+                },
+              ],
+            },
+          },
+        },
+      },
+      this.#namespace
+    );
+  }
+
   #envTypeToLabelValue(type: EnvironmentType) {
     switch (type) {
       case "PRODUCTION":
@@ -316,6 +404,9 @@ class KubernetesTaskOperations implements TaskOperations {
         {
           name: "registry-trigger",
         },
+        {
+          name: "registry-trigger-failover",
+        },
       ],
       nodeSelector: {
         nodetype: "worker",
@@ -381,7 +472,11 @@ class KubernetesTaskOperations implements TaskOperations {
   }
 
   #getSharedLabels(
-    opts: TaskOperationsIndexOptions | TaskOperationsCreateOptions | TaskOperationsRestoreOptions
+    opts:
+      | TaskOperationsIndexOptions
+      | TaskOperationsCreateOptions
+      | TaskOperationsRestoreOptions
+      | TaskOperationsPrePullDeploymentOptions
   ): Record<string, string> {
     return {
       env: opts.envId,
@@ -391,10 +486,10 @@ class KubernetesTaskOperations implements TaskOperations {
     };
   }
 
-  #getResourcesFromMachineConfig(config: Machine): ComputeResources {
+  #getResourcesFromMachineConfig(preset: MachinePreset): ComputeResources {
     return {
-      cpu: `${config.cpu}`,
-      memory: `${config.memory}G`,
+      cpu: `${preset.cpu}`,
+      memory: `${preset.memory}G`,
     };
   }
 
@@ -421,8 +516,12 @@ class KubernetesTaskOperations implements TaskOperations {
     return `task-index-${suffix}`;
   }
 
-  #getRunContainerName(suffix: string) {
-    return `task-run-${suffix}`;
+  #getRunContainerName(suffix: string, attemptNumber?: number) {
+    return `task-run-${suffix}${attemptNumber && attemptNumber > 1 ? `-att${attemptNumber}` : ""}`;
+  }
+
+  #getPrePullContainerName(suffix: string) {
+    return `task-prepull-${suffix}`;
   }
 
   #createK8sApi() {
@@ -439,6 +538,7 @@ class KubernetesTaskOperations implements TaskOperations {
     return {
       core: kubeConfig.makeApiClient(k8s.CoreV1Api),
       batch: kubeConfig.makeApiClient(k8s.BatchV1Api),
+      apps: kubeConfig.makeApiClient(k8s.AppsV1Api),
     };
   }
 
@@ -482,6 +582,18 @@ class KubernetesTaskOperations implements TaskOperations {
     }
   }
 
+  async #createDaemonSet(daemonSet: k8s.V1DaemonSet, namespace: Namespace) {
+    try {
+      const res = await this.#k8sApi.apps.createNamespacedDaemonSet(
+        namespace.metadata.name,
+        daemonSet
+      );
+      logger.debug(res.body);
+    } catch (err: unknown) {
+      this.#handleK8sError(err);
+    }
+  }
+
   #throwUnlessRecord(candidate: unknown): asserts candidate is Record<string, unknown> {
     if (typeof candidate !== "object" || candidate === null) {
       throw candidate;
@@ -516,27 +628,28 @@ provider.listen();
 
 const taskMonitor = new TaskMonitor({
   runtimeEnv: RUNTIME_ENV,
-  onIndexFailure: async (deploymentId, failureInfo) => {
-    logger.log("Indexing failed", { deploymentId, failureInfo });
+  onIndexFailure: async (deploymentId, details) => {
+    logger.log("Indexing failed", { deploymentId, details });
 
     try {
       provider.platformSocket.send("INDEXING_FAILED", {
         deploymentId,
         error: {
-          name: `Crashed with exit code ${failureInfo.exitCode}`,
-          message: failureInfo.reason,
-          stack: failureInfo.logs,
+          name: `Crashed with exit code ${details.exitCode}`,
+          message: details.reason,
+          stack: details.logs,
         },
+        overrideCompletion: details.overrideCompletion,
       });
     } catch (error) {
       logger.error(error);
     }
   },
-  onRunFailure: async (runId, failureInfo) => {
-    logger.log("Run failed:", { runId, failureInfo });
+  onRunFailure: async (runId, details) => {
+    logger.log("Run failed:", { runId, details });
 
     try {
-      provider.platformSocket.send("WORKER_CRASHED", { runId, ...failureInfo });
+      provider.platformSocket.send("WORKER_CRASHED", { runId, ...details });
     } catch (error) {
       logger.error(error);
     }
@@ -548,7 +661,23 @@ taskMonitor.start();
 const podCleaner = new PodCleaner({
   runtimeEnv: RUNTIME_ENV,
   namespace: "default",
-  intervalInSeconds: 300,
+  intervalInSeconds: POD_CLEANER_INTERVAL_SECONDS,
 });
 
 podCleaner.start();
+
+if (UPTIME_HEARTBEAT_URL) {
+  const uptimeHeartbeat = new UptimeHeartbeat({
+    runtimeEnv: RUNTIME_ENV,
+    namespace: "default",
+    intervalInSeconds: UPTIME_INTERVAL_SECONDS,
+    pingUrl: UPTIME_HEARTBEAT_URL,
+    maxPendingRuns: UPTIME_MAX_PENDING_RUNS,
+    maxPendingIndeces: UPTIME_MAX_PENDING_INDECES,
+    maxPendingErrors: UPTIME_MAX_PENDING_ERRORS,
+  });
+
+  uptimeHeartbeat.start();
+} else {
+  logger.log("Uptime heartbeat is disabled, set UPTIME_HEARTBEAT_URL to enable.");
+}

apps/kubernetes-provider/src/podCleaner.ts

@@ -1,5 +1,5 @@
 import * as k8s from "@kubernetes/client-node";
-import { SimpleLogger } from "@trigger.dev/core-apps";
+import { SimpleLogger } from "@trigger.dev/core-apps/logger";
 
 type PodCleanerOptions = {
   runtimeEnv: "local" | "kubernetes";
@@ -15,6 +15,7 @@ export class PodCleaner {
   private logger = new SimpleLogger("[PodCleaner]");
   private k8sClient: {
     core: k8s.CoreV1Api;
+    apps: k8s.AppsV1Api;
     kubeConfig: k8s.KubeConfig;
   };
 
@@ -43,6 +44,7 @@ export class PodCleaner {
 
     return {
       core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      apps: kubeConfig.makeApiClient(k8s.AppsV1Api),
       kubeConfig: kubeConfig,
     };
   }
@@ -98,6 +100,25 @@ export class PodCleaner {
       .catch(this.#handleK8sError.bind(this));
   }
 
+  async #deleteDaemonSets(opts: {
+    namespace: string;
+    dryRun?: boolean;
+    fieldSelector?: string;
+    labelSelector?: string;
+  }) {
+    return await this.k8sClient.apps
+      .deleteCollectionNamespacedDaemonSet(
+        opts.namespace,
+        undefined, // pretty
+        undefined, // continue
+        opts.dryRun ? "All" : undefined,
+        opts.fieldSelector,
+        undefined, // gracePeriodSeconds
+        opts.labelSelector
+      )
+      .catch(this.#handleK8sError.bind(this));
+  }
+
   async #deleteCompletedRuns() {
     this.logger.log("Deleting completed runs");
 
@@ -152,6 +173,28 @@ export class PodCleaner {
     });
   }
 
+  async #deleteCompletedPrePulls() {
+    this.logger.log("Deleting completed pre-pulls");
+
+    const start = Date.now();
+
+    const result = await this.#deleteDaemonSets({
+      namespace: this.namespace,
+      labelSelector: "app=task-prepull",
+    });
+
+    const elapsedMs = Date.now() - start;
+
+    if (!result) {
+      this.logger.log("Deleting completed pre-pulls: No delete result", { elapsedMs });
+      return;
+    }
+
+    const total = (result.response as any)?.body?.items?.length ?? 0;
+
+    this.logger.log("Deleting completed pre-pulls: Done", { total, elapsedMs });
+  }
+
   async start() {
     this.enabled = true;
     this.logger.log("Starting");
@@ -186,6 +229,22 @@ export class PodCleaner {
       2 * this.intervalInSeconds * 1000
     );
 
+    const completedPrePullInterval = setInterval(
+      async () => {
+        if (!this.enabled) {
+          clearInterval(completedPrePullInterval);
+          return;
+        }
+
+        try {
+          await this.#deleteCompletedPrePulls();
+        } catch (error) {
+          this.logger.error("Error deleting completed pre-pulls", error);
+        }
+      },
+      2 * this.intervalInSeconds * 1000
+    );
+
     // this.#launchTests();
   }
 

apps/kubernetes-provider/src/taskMonitor.ts

@@ -1,25 +1,20 @@
 import * as k8s from "@kubernetes/client-node";
-import { SimpleLogger } from "@trigger.dev/core-apps";
+import { SimpleLogger } from "@trigger.dev/core-apps/logger";
+import { EXIT_CODE_ALREADY_HANDLED, EXIT_CODE_CHILD_NONZERO } from "@trigger.dev/core-apps/process";
 import { setTimeout } from "timers/promises";
 import PQueue from "p-queue";
+import type { Prettify } from "@trigger.dev/core/v3";
 
-type IndexFailureHandler = (
-  deploymentId: string,
-  failureInfo: {
-    exitCode: number;
-    reason: string;
-    logs: string;
-  }
-) => Promise<any>;
+type FailureDetails = Prettify<{
+  exitCode: number;
+  reason: string;
+  logs: string;
+  overrideCompletion: boolean;
+}>;
 
-type RunFailureHandler = (
-  runId: string,
-  failureInfo: {
-    exitCode: number;
-    reason: string;
-    logs: string;
-  }
-) => Promise<any>;
+type IndexFailureHandler = (deploymentId: string, details: FailureDetails) => Promise<any>;
+
+type RunFailureHandler = (runId: string, details: FailureDetails) => Promise<any>;
 
 type TaskMonitorOptions = {
   runtimeEnv: "local" | "kubernetes";
@@ -144,8 +139,10 @@ export class TaskMonitor {
     const containerState = this.#getContainerStateSummary(containerStatus.state);
     const exitCode = containerState.exitCode ?? -1;
 
-    // We use this special exit code to signal any errors were already handled elsewhere
-    if (exitCode === 111) {
+    if (exitCode === EXIT_CODE_ALREADY_HANDLED) {
+      this.#logger.debug("Ignoring pod failure, already handled by worker", {
+        podName,
+      });
       return;
     }
 
@@ -162,6 +159,7 @@ export class TaskMonitor {
 
     let reason = rawReason || "Unknown error";
     let logs = rawLogs || "";
+    let overrideCompletion = false;
 
     switch (rawReason) {
       case "Error":
@@ -181,7 +179,10 @@ export class TaskMonitor {
         }
         break;
       case "OOMKilled":
-        reason = "Out of memory! Try increasing the memory on this task.";
+        overrideCompletion = true;
+        reason = `${
+          exitCode === EXIT_CODE_CHILD_NONZERO ? "Child process" : "Parent process"
+        } ran out of memory! Try choosing a machine preset with more memory for this task.`;
         break;
       default:
         break;
@@ -191,7 +192,8 @@ export class TaskMonitor {
       exitCode,
       reason,
       logs,
-    };
+      overrideCompletion,
+    } satisfies FailureDetails;
 
     const app = pod.metadata?.labels?.app;
 

apps/kubernetes-provider/src/uptimeHeartbeat.ts

@@ -0,0 +1,272 @@
+import * as k8s from "@kubernetes/client-node";
+import { SimpleLogger } from "@trigger.dev/core-apps/logger";
+
+type UptimeHeartbeatOptions = {
+  runtimeEnv: "local" | "kubernetes";
+  pingUrl: string;
+  namespace?: string;
+  intervalInSeconds?: number;
+  maxPendingRuns?: number;
+  maxPendingIndeces?: number;
+  maxPendingErrors?: number;
+  leadingEdge?: boolean;
+};
+
+export class UptimeHeartbeat {
+  private enabled = false;
+  private namespace: string;
+
+  private intervalInSeconds: number;
+  private maxPendingRuns: number;
+  private maxPendingIndeces: number;
+  private maxPendingErrors: number;
+
+  private leadingEdge = true;
+
+  private logger = new SimpleLogger("[UptimeHeartbeat]");
+  private k8sClient: {
+    core: k8s.CoreV1Api;
+    kubeConfig: k8s.KubeConfig;
+  };
+
+  constructor(private opts: UptimeHeartbeatOptions) {
+    this.namespace = opts.namespace ?? "default";
+
+    this.intervalInSeconds = opts.intervalInSeconds ?? 60;
+    this.maxPendingRuns = opts.maxPendingRuns ?? 25;
+    this.maxPendingIndeces = opts.maxPendingIndeces ?? 10;
+    this.maxPendingErrors = opts.maxPendingErrors ?? 10;
+
+    this.k8sClient = this.#createK8sClient();
+  }
+
+  #createK8sClient() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (this.opts.runtimeEnv === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (this.opts.runtimeEnv === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      kubeConfig: kubeConfig,
+    };
+  }
+
+  #isRecord(candidate: unknown): candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  #logK8sError(err: unknown, debugOnly = false) {
+    if (debugOnly) {
+      this.logger.debug("K8s API Error", err);
+    } else {
+      this.logger.error("K8s API Error", err);
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
+      this.#logK8sError(err);
+      return;
+    }
+
+    this.#logK8sError(err, true);
+
+    if (typeof err.body.message === "string") {
+      this.#logK8sError({ message: err.body.message });
+      return;
+    }
+
+    this.#logK8sError({ body: err.body });
+  }
+
+  async #getPods(opts: {
+    namespace: string;
+    fieldSelector?: string;
+    labelSelector?: string;
+  }): Promise<Array<k8s.V1Pod> | undefined> {
+    const listReturn = await this.k8sClient.core
+      .listNamespacedPod(
+        opts.namespace,
+        undefined, // pretty
+        undefined, // allowWatchBookmarks
+        undefined, // _continue
+        opts.fieldSelector,
+        opts.labelSelector,
+        this.maxPendingRuns * 2, // limit
+        undefined, // resourceVersion
+        undefined, // resourceVersionMatch
+        undefined, // sendInitialEvents
+        this.intervalInSeconds, // timeoutSeconds,
+        undefined // watch
+      )
+      .catch(this.#handleK8sError.bind(this));
+
+    return listReturn?.body.items;
+  }
+
+  async #getPendingIndeces(): Promise<Array<k8s.V1Pod> | undefined> {
+    return await this.#getPods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=Pending",
+      labelSelector: "app=task-index",
+    });
+  }
+
+  async #getPendingTasks(): Promise<Array<k8s.V1Pod> | undefined> {
+    return await this.#getPods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=Pending",
+      labelSelector: "app=task-run",
+    });
+  }
+
+  #countPods(pods: Array<k8s.V1Pod>): number {
+    return pods.length;
+  }
+
+  #filterPendingPods(
+    pods: Array<k8s.V1Pod>,
+    waitingReason: "CreateContainerError" | "RunContainerError"
+  ): Array<k8s.V1Pod> {
+    return pods.filter((pod) => {
+      const containerStatus = pod.status?.containerStatuses?.[0];
+      return containerStatus?.state?.waiting?.reason === waitingReason;
+    });
+  }
+
+  async #sendPing() {
+    this.logger.log("Sending ping");
+
+    const start = Date.now();
+    const controller = new AbortController();
+
+    const timeoutMs = (this.intervalInSeconds * 1000) / 2;
+
+    const fetchTimeout = setTimeout(() => {
+      controller.abort();
+    }, timeoutMs);
+
+    try {
+      const response = await fetch(this.opts.pingUrl, {
+        signal: controller.signal,
+      });
+
+      if (!response.ok) {
+        this.logger.error("Failed to send ping, response not OK", {
+          status: response.status,
+        });
+        return;
+      }
+
+      const elapsedMs = Date.now() - start;
+      this.logger.log("Ping sent", { elapsedMs });
+    } catch (error) {
+      if (error instanceof DOMException && error.name === "AbortError") {
+        this.logger.log("Ping timeout", { timeoutSeconds: timeoutMs });
+        return;
+      }
+
+      this.logger.error("Failed to send ping", error);
+    } finally {
+      clearTimeout(fetchTimeout);
+    }
+  }
+
+  async #heartbeat() {
+    this.logger.log("Performing heartbeat");
+
+    const start = Date.now();
+
+    const pendingTasks = await this.#getPendingTasks();
+
+    if (!pendingTasks) {
+      this.logger.error("Failed to get pending tasks");
+      return;
+    }
+
+    const totalPendingTasks = this.#countPods(pendingTasks);
+
+    const pendingIndeces = await this.#getPendingIndeces();
+
+    if (!pendingIndeces) {
+      this.logger.error("Failed to get pending indeces");
+      return;
+    }
+
+    const totalPendingIndeces = this.#countPods(pendingIndeces);
+
+    const elapsedMs = Date.now() - start;
+
+    this.logger.log("Finished heartbeat checks", { elapsedMs });
+
+    if (totalPendingTasks > this.maxPendingRuns) {
+      this.logger.log("Too many pending tasks, skipping heartbeat", { totalPendingTasks });
+      return;
+    }
+
+    if (totalPendingIndeces > this.maxPendingIndeces) {
+      this.logger.log("Too many pending indeces, skipping heartbeat", { totalPendingIndeces });
+      return;
+    }
+
+    const totalCreateContainerErrors = this.#countPods(
+      this.#filterPendingPods(pendingTasks, "CreateContainerError")
+    );
+    const totalRunContainerErrors = this.#countPods(
+      this.#filterPendingPods(pendingTasks, "RunContainerError")
+    );
+
+    if (totalCreateContainerErrors + totalRunContainerErrors > this.maxPendingErrors) {
+      this.logger.log("Too many pending tasks with errors, skipping heartbeat", {
+        totalRunContainerErrors,
+        totalCreateContainerErrors,
+      });
+      return;
+    }
+
+    await this.#sendPing();
+
+    this.logger.log("Heartbeat done", { totalPendingTasks, elapsedMs });
+  }
+
+  async start() {
+    this.enabled = true;
+    this.logger.log("Starting");
+
+    if (this.leadingEdge) {
+      await this.#heartbeat();
+    }
+
+    const heartbeat = setInterval(async () => {
+      if (!this.enabled) {
+        clearInterval(heartbeat);
+        return;
+      }
+
+      try {
+        await this.#heartbeat();
+      } catch (error) {
+        this.logger.error("Error while heartbeating", error);
+      }
+    }, this.intervalInSeconds * 1000);
+  }
+
+  async stop() {
+    if (!this.enabled) {
+      return;
+    }
+
+    this.enabled = false;
+    this.logger.log("Shutting down..");
+  }
+}

apps/webapp/.eslintrc

@@ -1,13 +1,36 @@
 {
-  "extends": ["@remix-run/eslint-config", "@remix-run/eslint-config/node", "prettier"],
-  "rules": {
-    "@typescript-eslint/strict-boolean-expressions": [
-      "error",
-      {
-        "allowNullableBoolean": true,
-        "allowNullableString": true,
-        "allowNullableNumber": true
+  "plugins": [
+    "@trigger.dev/eslint-plugin",
+    "react-hooks",
+    "@typescript-eslint/eslint-plugin",
+    "import"
+  ],
+  "parser": "@typescript-eslint/parser",
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx"],
+      "rules": {
+        // Autofixes imports from "@trigger.dev/core" to fine grained modules
+        "@trigger.dev/no-trigger-core-import": "error",
+        // Normalize `import type {}` and `import { type }`
+        "@typescript-eslint/consistent-type-imports": [
+          "warn",
+          {
+            // the "type" annotation can get tangled and cause syntax errors
+            // during some autofixes, so easier to just turn it off
+            "prefer": "type-imports",
+            "disallowTypeAnnotations": true,
+            "fixStyle": "inline-type-imports"
+          }
+        ],
+        // no-trigger-core-import splits imports into multiple lines
+        // this one merges them back into a single line
+        // if they still import from the same module
+        "import/no-duplicates": ["warn", { "prefer-inline": true }],
+        // lots of undeclared vars, enable this rule if you want to clean them up
+        "turbo/no-undeclared-env-vars": "off"
       }
-    ]
-  }
+    }
+  ],
+  "ignorePatterns": ["seed.js", "seedCloud.ts", "populate.js"]
 }

apps/webapp/app/components/DefinitionTooltip.tsx

@@ -15,11 +15,11 @@ export function DefinitionTip({
     <TooltipProvider>
       <Tooltip disableHoverableContent>
         <TooltipTrigger>
-          <span className="underline decoration-charcoal-600 decoration-dashed underline-offset-4 transition hover:decoration-charcoal-500">
+          <span className="cursor-default underline decoration-charcoal-500 decoration-dashed underline-offset-4 transition hover:decoration-charcoal-400">
             {children}
           </span>
         </TooltipTrigger>
-        <TooltipContent align="end" side="right" variant="dark" className="w-[16rem] min-w-[16rem]">
+        <TooltipContent align="end" side="right" className="w-[16rem] min-w-[16rem]">
           <Header3 className="mb-1">{title}</Header3>
           {typeof content === "string" ? (
             <Paragraph variant="small">{content}</Paragraph>

apps/webapp/app/components/Feedback.tsx

@@ -1,19 +1,25 @@
 import { conform, useForm } from "@conform-to/react";
 import { parse } from "@conform-to/zod";
 import { BookOpenIcon } from "@heroicons/react/20/solid";
-import { ChevronRightIcon } from "@heroicons/react/24/solid";
+import {
+  CalendarDaysIcon,
+  ChevronRightIcon,
+  EnvelopeIcon,
+  LifebuoyIcon,
+  LightBulbIcon,
+} from "@heroicons/react/24/solid";
 import { Form, useActionData, useLocation, useNavigation } from "@remix-run/react";
-import { DiscordIcon, GitHubLightIcon } from "@trigger.dev/companyicons";
+import { DiscordIcon } from "@trigger.dev/companyicons";
 import { ActivityIcon } from "lucide-react";
-import { ReactNode, useState } from "react";
-import { FeedbackType, feedbackTypeLabel, schema } from "~/routes/resources.feedback";
+import { type ReactNode, useState } from "react";
+import { type FeedbackType, feedbackTypeLabel, schema } from "~/routes/resources.feedback";
 import { cn } from "~/utils/cn";
-import { docsPath } from "~/utils/pathBuilder";
+import { docsTroubleshootingPath } from "~/utils/pathBuilder";
 import { Button, LinkButton } from "./primitives/Buttons";
 import { Fieldset } from "./primitives/Fieldset";
 import { FormButtons } from "./primitives/FormButtons";
 import { FormError } from "./primitives/FormError";
-import { Header1, Header2 } from "./primitives/Headers";
+import { Header1 } from "./primitives/Headers";
 import { InputGroup } from "./primitives/InputGroup";
 import { Label } from "./primitives/Label";
 import { Paragraph } from "./primitives/Paragraph";
@@ -55,123 +61,137 @@ export function Feedback({ button, defaultValue = "bug" }: FeedbackProps) {
       <SheetTrigger asChild={true}>{button}</SheetTrigger>
       <SheetContent className="@container">
         <SheetBody className="flex h-full flex-col justify-between">
-          <Header2 className="mb-2.5 text-xl">Get help from the community</Header2>
-          <Paragraph className="mb-4">
-            The quickest way to get help and feedback or to provide advice to others is to join our
-            Discord.
-          </Paragraph>
-          <div className="flex flex-col gap-x-4 @[30rem]:flex-row">
-            <DiscordBanner />
-            <GitHubDiscussionsBanner />
-          </div>
-          <hr className="mb-4" />
-          <Header2 className="mb-2.5 text-xl">Send us an email</Header2>
-          <Paragraph className="mb-4">We read every message and respond quickly.</Paragraph>
-          <Form method="post" action="/resources/feedback" {...form.props}>
-            <Fieldset className="max-w-full gap-y-3">
-              <input value={location.pathname} {...conform.input(path, { type: "hidden" })} />
-              <InputGroup className="max-w-full">
-                <Select
-                  {...conform.select(feedbackType)}
-                  variant="tertiary/medium"
-                  defaultValue={defaultValue}
-                  placeholder="Select type"
-                  text={(value) => feedbackTypeLabel[value]}
-                  dropdownIcon
-                >
-                  {Object.entries(feedbackTypeLabel).map(([name, title]) => (
-                    <SelectItem key={name} value={name}>
-                      {title}
-                    </SelectItem>
-                  ))}
-                </Select>
-                <FormError id={feedbackType.errorId}>{feedbackType.error}</FormError>
-              </InputGroup>
-              <InputGroup className="max-w-full">
-                <Label>Message</Label>
-                <TextArea {...conform.textarea(message)} />
-                <FormError id={message.errorId}>{message.error}</FormError>
-              </InputGroup>
-              <FormError>{form.error}</FormError>
-              <div className="flex w-full justify-end">
-                <FormButtons
-                  className="m-0 w-max"
-                  confirmButton={
-                    <Button type="submit" variant="tertiary/medium">
-                      Send message
-                    </Button>
-                  }
-                />
-              </div>
-            </Fieldset>
-          </Form>
-          <hr className="my-4" />
-          <Header2 className="mb-2.5 text-xl">Troubleshooting</Header2>
-          <Paragraph className="mb-4">
-            If you're having trouble, check out our documentation or the Trigger.dev Status page.
-          </Paragraph>
-          <div className="flex flex-wrap gap-2">
-            <LinkButton to={docsPath("")} variant="tertiary/medium" LeadingIcon={BookOpenIcon}>
-              Docs
-            </LinkButton>
-            <LinkButton
-              to={"https://status.trigger.dev/"}
-              variant="tertiary/medium"
-              LeadingIcon={ActivityIcon}
-            >
-              Trigger.dev Status
-            </LinkButton>
-          </div>
+          <LinkBanner
+            title="Join our Discord community"
+            icon={<DiscordIcon className="size-9" />}
+            to="https://trigger.dev/discord"
+            className="hover:border-text-link"
+          >
+            <Paragraph>The quickest way to get answers from the Trigger.dev community.</Paragraph>
+          </LinkBanner>
+          <LinkBanner
+            title="Book a 15 min chat with the founders"
+            icon={<CalendarDaysIcon className="size-9 text-green-500" />}
+            to="https://cal.com/team/triggerdotdev/founders-call"
+            className="hover:border-green-500"
+          >
+            <Paragraph>Have a question or want to chat? Book a time to talk with us.</Paragraph>
+          </LinkBanner>
+          <LinkBanner
+            title="Suggest a feature"
+            icon={<LightBulbIcon className="size-9 text-sun-500" />}
+            to="https://feedback.trigger.dev/"
+            className="hover:border-sun-400"
+          >
+            <Paragraph>Have an idea for a new feature or improvement? Let us know!</Paragraph>
+          </LinkBanner>
+          <LinkBanner
+            title="Troubleshooting"
+            icon={<LifebuoyIcon className="size-9 text-rose-500" />}
+          >
+            <Paragraph>
+              If you're having trouble, check out our troubleshooting guide or the Trigger.dev
+              Status page.
+            </Paragraph>
+            <div className="flex flex-wrap gap-2">
+              <LinkButton
+                to={docsTroubleshootingPath("")}
+                variant="tertiary/medium"
+                LeadingIcon={BookOpenIcon}
+              >
+                Troubleshooting Docs
+              </LinkButton>
+              <LinkButton
+                to={"https://status.trigger.dev/"}
+                variant="tertiary/medium"
+                LeadingIcon={ActivityIcon}
+              >
+                Trigger.dev Status
+              </LinkButton>
+            </div>
+          </LinkBanner>
+          <LinkBanner
+            title="Send us an email"
+            icon={<EnvelopeIcon className="size-9 text-blue-500" />}
+          >
+            <Paragraph>We read every message and respond quickly.</Paragraph>
+            <Form method="post" action="/resources/feedback" {...form.props} className="w-full">
+              <Fieldset className="max-w-full gap-y-3">
+                <input value={location.pathname} {...conform.input(path, { type: "hidden" })} />
+                <InputGroup className="max-w-full">
+                  <Select
+                    {...conform.select(feedbackType)}
+                    variant="tertiary/medium"
+                    defaultValue={defaultValue}
+                    placeholder="Select type"
+                    text={(value) => feedbackTypeLabel[value]}
+                    dropdownIcon
+                  >
+                    {Object.entries(feedbackTypeLabel).map(([name, title]) => (
+                      <SelectItem key={name} value={name}>
+                        {title}
+                      </SelectItem>
+                    ))}
+                  </Select>
+                  <FormError id={feedbackType.errorId}>{feedbackType.error}</FormError>
+                </InputGroup>
+                <InputGroup className="max-w-full">
+                  <Label>Message</Label>
+                  <TextArea {...conform.textarea(message)} />
+                  <FormError id={message.errorId}>{message.error}</FormError>
+                </InputGroup>
+                <FormError>{form.error}</FormError>
+                <div className="flex w-full justify-end">
+                  <FormButtons
+                    className="m-0 w-max"
+                    confirmButton={
+                      <Button type="submit" variant="tertiary/medium">
+                        Send message
+                      </Button>
+                    }
+                  />
+                </div>
+              </Fieldset>
+            </Form>
+          </LinkBanner>
         </SheetBody>
       </SheetContent>
     </Sheet>
   );
 }
 
-function DiscordBanner({ className }: { className?: string }) {
+function LinkBanner({
+  className,
+  icon,
+  title,
+  children,
+  to,
+}: {
+  className?: string;
+  icon?: ReactNode;
+  title?: string;
+  children?: ReactNode;
+  to?: string;
+}) {
   return (
     <a
-      href="https://trigger.dev/discord"
+      href={to}
       target="_blank"
       className={cn(
-        "group mb-4 flex w-full items-center justify-between rounded-md border border-charcoal-600 p-4 transition hover:border-text-link",
+        "group/banner mb-4 flex w-full items-center justify-between rounded-md border border-grid-bright bg-charcoal-750 p-4 transition",
         className
       )}
     >
-      <div className="flex flex-col gap-y-2">
-        <DiscordIcon className="h-8 w-8" />
-        <Header1 className="text-2xl font-semibold text-text-bright transition group-hover:text-white">
-          Join our Discord community
-        </Header1>
-        <Paragraph variant="small" className="mb-4">
-          Get help or answer questions from the Trigger.dev community.
-        </Paragraph>
+      <div className="flex w-full items-start gap-4">
+        <span>{icon}</span>
+        <div className="flex w-full flex-col gap-2">
+          <Header1 className="text-2xl font-semibold text-text-bright">{title}</Header1>
+          {children}
+        </div>
       </div>
-      <ChevronRightIcon className="size-5 text-charcoal-500 transition group-hover:translate-x-1 group-hover:text-text-link" />
-    </a>
-  );
-}
-
-function GitHubDiscussionsBanner({ className }: { className?: string }) {
-  return (
-    <a
-      href="https://github.com/triggerdotdev/trigger.dev/discussions"
-      target="_blank"
-      className={cn(
-        "group mb-4 flex w-full items-center justify-between rounded-md border border-charcoal-600 p-4 transition hover:border-text-dimmed",
-        className
+      {to && (
+        <ChevronRightIcon className="size-5 text-charcoal-500 transition group-hover:translate-x-1 group-hover/banner:text-text-bright" />
       )}
-    >
-      <div className="flex flex-col gap-y-2">
-        <GitHubLightIcon className="mb-1 h-7 w-7" />
-        <Header1 className="text-2xl font-semibold text-text-bright transition group-hover:text-white">
-          View our GitHub Discussions
-        </Header1>
-        <Paragraph variant="small">
-          Post your questions, feedback, and feature requests on GitHub.
-        </Paragraph>
-      </div>
-      <ChevronRightIcon className="size-5 text-charcoal-500 transition group-hover:translate-x-1 group-hover:text-text-bright" />
     </a>
   );
 }

apps/webapp/app/components/LoginPageLayout.tsx

@@ -2,7 +2,6 @@ import { useEffect, useState } from "react";
 import { ATAndTLogo } from "~/assets/logos/ATAndTLogo";
 import { AppsmithLogo } from "~/assets/logos/AppsmithLogo";
 import { CalComLogo } from "~/assets/logos/CalComLogo";
-import { OktaLogo } from "~/assets/logos/OktaLogo";
 import { ShopifyLogo } from "~/assets/logos/ShopifyLogo";
 import { VerizonLogo } from "~/assets/logos/VerizonLogo";
 import { LogoType } from "./LogoType";
@@ -75,7 +74,6 @@ export function LoginPageLayout({ children }: { children: React.ReactNode }) {
           <div className="flex w-full flex-wrap items-center justify-center gap-x-6 gap-y-3 text-charcoal-500 xl:justify-between xl:gap-0">
             <VerizonLogo />
             <ShopifyLogo />
-            <OktaLogo />
             <ATAndTLogo />
             <AppsmithLogo />
             <CalComLogo />

apps/webapp/app/components/SetupCommands.tsx

@@ -8,6 +8,7 @@ import {
 } from "./primitives/ClientTabs";
 import { ClipboardField } from "./primitives/ClipboardField";
 import { Paragraph } from "./primitives/Paragraph";
+import { useAppOrigin } from "~/hooks/useAppOrigin";
 
 export function InitCommand({ appOrigin, apiKey }: { appOrigin: string; apiKey: string }) {
   return (
@@ -133,9 +134,38 @@ export function TriggerDevStep({ extra }: { extra?: string }) {
 // Trigger.dev version 3 setup commands
 const v3PackageTag = "beta";
 
+function getApiUrlArg() {
+  const appOrigin = useAppOrigin();
+
+  let apiUrl: string | undefined = undefined;
+
+  switch (appOrigin) {
+    case "https://cloud.trigger.dev":
+      // don't display the arg, use the CLI default
+      break;
+    case "https://test-cloud.trigger.dev":
+      apiUrl = "https://test-api.trigger.dev";
+      break;
+    case "https://internal.trigger.dev":
+      apiUrl = "https://internal-api.trigger.dev";
+      break;
+    default:
+      apiUrl = appOrigin;
+      break;
+  }
+
+  return apiUrl ? `-a ${apiUrl}` : undefined;
+}
+
 export function InitCommandV3() {
   const project = useProject();
   const projectRef = project.ref;
+
+  const apiUrlArg = getApiUrlArg();
+
+  const initCommandParts = [`trigger.dev@${v3PackageTag}`, "init", `-p ${projectRef}`, apiUrlArg];
+  const initCommand = initCommandParts.filter(Boolean).join(" ");
+
   return (
     <ClientTabs defaultValue="npm">
       <ClientTabsList>
@@ -148,7 +178,7 @@ export function InitCommandV3() {
           variant="primary/medium"
           iconButton
           className="mb-4"
-          value={`npx trigger.dev@${v3PackageTag} init -p ${projectRef}`}
+          value={`npx ${initCommand}`}
         />
       </ClientTabsContent>
       <ClientTabsContent value={"pnpm"}>
@@ -156,7 +186,7 @@ export function InitCommandV3() {
           variant="primary/medium"
           iconButton
           className="mb-4"
-          value={`pnpm dlx trigger.dev@${v3PackageTag} init -p ${projectRef}`}
+          value={`pnpm dlx ${initCommand}`}
         />
       </ClientTabsContent>
       <ClientTabsContent value={"yarn"}>
@@ -164,7 +194,7 @@ export function InitCommandV3() {
           variant="primary/medium"
           iconButton
           className="mb-4"
-          value={`yarn dlx trigger.dev@${v3PackageTag} init -p ${projectRef}`}
+          value={`yarn dlx ${initCommand}`}
         />
       </ClientTabsContent>
     </ClientTabs>

apps/webapp/app/components/admin/debugTooltip.tsx

@@ -1,14 +1,16 @@
 import { ShieldCheckIcon } from "@heroicons/react/20/solid";
+import * as Property from "~/components/primitives/PropertyTable";
 import {
   Tooltip,
   TooltipContent,
   TooltipProvider,
   TooltipTrigger,
 } from "~/components/primitives/Tooltip";
-import { useIsImpersonating } from "~/hooks/useOrganizations";
-import { useHasAdminAccess } from "~/hooks/useUser";
+import { useIsImpersonating, useOptionalOrganization } from "~/hooks/useOrganizations";
+import { useOptionalProject } from "~/hooks/useProject";
+import { useHasAdminAccess, useUser } from "~/hooks/useUser";
 
-export function AdminDebugTooltip({ children }: { children: React.ReactNode }) {
+export function AdminDebugTooltip({ children }: { children?: React.ReactNode }) {
   const hasAdminAccess = useHasAdminAccess();
   const isImpersonating = useIsImpersonating();
 
@@ -22,10 +24,46 @@ export function AdminDebugTooltip({ children }: { children: React.ReactNode }) {
         <TooltipTrigger>
           <ShieldCheckIcon className="size-5" />
         </TooltipTrigger>
-        <TooltipContent className="flex max-h-[90vh] items-center gap-1 overflow-y-auto">
-          {children}
+        <TooltipContent className="max-h-[90vh] overflow-y-auto">
+          <Content>{children}</Content>
         </TooltipContent>
       </Tooltip>
     </TooltipProvider>
   );
 }
+
+function Content({ children }: { children: React.ReactNode }) {
+  const organization = useOptionalOrganization();
+  const project = useOptionalProject();
+  const user = useUser();
+
+  return (
+    <div className="flex flex-col gap-2 divide-y divide-slate-700">
+      <Property.Table>
+        <Property.Item>
+          <Property.Label>User ID</Property.Label>
+          <Property.Value>{user.id}</Property.Value>
+        </Property.Item>
+        {organization && (
+          <Property.Item>
+            <Property.Label>Org ID</Property.Label>
+            <Property.Value>{organization.id}</Property.Value>
+          </Property.Item>
+        )}
+        {project && (
+          <>
+            <Property.Item>
+              <Property.Label>Project ID</Property.Label>
+              <Property.Value>{project.id}</Property.Value>
+            </Property.Item>
+            <Property.Item>
+              <Property.Label>Project ref</Property.Label>
+              <Property.Value>{project.ref}</Property.Value>
+            </Property.Item>
+          </>
+        )}
+      </Property.Table>
+      <div className="pt-2">{children}</div>
+    </div>
+  );
+}

apps/webapp/app/components/billing/UpgradePrompt.tsx

@@ -1,61 +0,0 @@
-import { ExclamationCircleIcon } from "@heroicons/react/20/solid";
-import { ArrowUpCircleIcon } from "@heroicons/react/24/outline";
-import tileBgPath from "~/assets/images/error-banner-tile@2x.png";
-import { MatchedOrganization } from "~/hooks/useOrganizations";
-import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route";
-import { formatNumberCompact } from "~/utils/numberFormatter";
-import { LinkButton } from "../primitives/Buttons";
-import { Icon } from "../primitives/Icon";
-import { Paragraph } from "../primitives/Paragraph";
-
-type UpgradePromptProps = {
-  runsEnabled: boolean;
-  runCountCap: number;
-  planPath: string;
-};
-
-export function UpgradePrompt({ runsEnabled, runCountCap, planPath }: UpgradePromptProps) {
-  return (
-    <div
-      className="flex h-10 items-center justify-between border border-error bg-repeat py-0 pl-3 pr-2"
-      style={{ backgroundImage: `url(${tileBgPath})`, backgroundSize: "8px 8px" }}
-    >
-      <div className="flex items-center gap-2">
-        <Icon icon={ExclamationCircleIcon} className="h-5 w-5 text-error" />
-        <Paragraph variant="small" className="text-error">
-          {runsEnabled
-            ? `You have exceeded the monthly ${formatNumberCompact(runCountCap)} runs
-          limit`
-            : `No runs are executing because you have exceeded the free limit`}
-        </Paragraph>
-      </div>
-      <LinkButton
-        variant={"primary/small"}
-        LeadingIcon={ArrowUpCircleIcon}
-        leadingIconClassName="px-0"
-        to={planPath}
-      >
-        Upgrade
-      </LinkButton>
-    </div>
-  );
-}
-
-export function useShowUpgradePrompt(organization?: MatchedOrganization) {
-  const currentPlan = useCurrentPlan();
-  const shouldShow =
-    organization !== undefined &&
-    currentPlan !== undefined &&
-    currentPlan.usage.exceededRunCount &&
-    currentPlan.usage.runCountCap !== undefined;
-
-  if (!shouldShow) {
-    return { shouldShow };
-  }
-
-  return {
-    shouldShow,
-    runCountCap: currentPlan.usage.runCountCap!,
-    runsEnabled: organization.runsEnabled,
-  };
-}

apps/webapp/app/components/billing/v2/ConcurrentRunsChart.tsx

@@ -8,7 +8,7 @@ import {
   XAxis,
   YAxis,
 } from "recharts";
-import { Paragraph } from "../primitives/Paragraph";
+import { Paragraph } from "../../primitives/Paragraph";
 
 const tooltipStyle = {
   display: "flex",

apps/webapp/app/components/billing/v2/DailyRunsChat.tsx

@@ -1,5 +1,5 @@
 import { Label, Line, LineChart, ResponsiveContainer, Tooltip, XAxis, YAxis } from "recharts";
-import { Paragraph } from "../primitives/Paragraph";
+import { Paragraph } from "../../primitives/Paragraph";
 
 const tooltipStyle = {
   display: "flex",

apps/webapp/app/components/billing/v2/FreePlanUsage.tsx

@@ -1,6 +1,6 @@
 import { ArrowUpCircleIcon } from "@heroicons/react/24/outline";
 import { motion, useMotionValue, useTransform } from "framer-motion";
-import { Paragraph } from "../primitives/Paragraph";
+import { Paragraph } from "../../primitives/Paragraph";
 import { Link } from "@remix-run/react";
 import { cn } from "~/utils/cn";
 
@@ -28,7 +28,7 @@ export function FreePlanUsage({ to, percentage }: { to: string; percentage: numb
           <Paragraph className="text-2sm text-text-bright">Free Plan</Paragraph>
         </div>
         <Link to={to} className="text-2sm text-text-link">
-          Learn more
+          Upgrade
         </Link>
       </div>
       <div className="relative mt-3 h-1 rounded-full bg-background-dimmed">

apps/webapp/app/components/billing/v2/PricingCalculator.tsx

@@ -1,9 +1,9 @@
 import * as Slider from "@radix-ui/react-slider";
-import { Plans, estimate } from "@trigger.dev/billing";
+import { Plans, estimate } from "@trigger.dev/platform/v2";
 import { useCallback, useState } from "react";
-import { DefinitionTip } from "../DefinitionTooltip";
-import { Header2 } from "../primitives/Headers";
-import { Paragraph } from "../primitives/Paragraph";
+import { DefinitionTip } from "../../DefinitionTooltip";
+import { Header2 } from "../../primitives/Headers";
+import { Paragraph } from "../../primitives/Paragraph";
 import { formatCurrency, formatNumberCompact } from "~/utils/numberFormatter";
 import { cn } from "~/utils/cn";
 

apps/webapp/app/components/billing/v2/PricingTiers.tsx

@@ -2,17 +2,17 @@ import { useForm } from "@conform-to/react";
 import { parse } from "@conform-to/zod";
 import { CheckIcon, XMarkIcon } from "@heroicons/react/24/solid";
 import { Form, useActionData, useNavigation } from "@remix-run/react";
-import { ActiveSubscription, Plan, Plans, SetPlanBodySchema } from "@trigger.dev/billing";
+import { ActiveSubscription, Plan, Plans, SetPlanBodySchema } from "@trigger.dev/platform/v2";
 import { useState } from "react";
 import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route";
 import { cn } from "~/utils/cn";
 import { formatNumberCompact } from "~/utils/numberFormatter";
-import { DefinitionTip } from "../DefinitionTooltip";
-import { Feedback } from "../Feedback";
-import { Button, LinkButton } from "../primitives/Buttons";
-import SegmentedControl from "../primitives/SegmentedControl";
+import { DefinitionTip } from "../../DefinitionTooltip";
+import { Feedback } from "../../Feedback";
+import { Button, LinkButton } from "../../primitives/Buttons";
+import SegmentedControl from "../../primitives/SegmentedControl";
 import { RunsVolumeDiscountTable } from "./RunsVolumeDiscountTable";
-import { Spinner } from "../primitives/Spinner";
+import { Spinner } from "../../primitives/Spinner";
 
 const pricingDefinitions = {
   concurrentRuns: {

apps/webapp/app/components/billing/v2/RunsVolumeDiscountTable.tsx

@@ -1,6 +1,6 @@
-import { RunPriceBracket } from "@trigger.dev/billing";
-import { Header2 } from "../primitives/Headers";
-import { Paragraph } from "../primitives/Paragraph";
+import { RunPriceBracket } from "@trigger.dev/platform/v2";
+import { Header2 } from "../../primitives/Headers";
+import { Paragraph } from "../../primitives/Paragraph";
 import { formatNumberCompact } from "~/utils/numberFormatter";
 
 export function RunsVolumeDiscountTable({

apps/webapp/app/components/billing/v2/UsageBar.tsx

@@ -1,7 +1,7 @@
 import { cn } from "~/utils/cn";
 import { formatNumberCompact } from "~/utils/numberFormatter";
-import { Paragraph } from "../primitives/Paragraph";
-import { SimpleTooltip } from "../primitives/Tooltip";
+import { Paragraph } from "../../primitives/Paragraph";
+import { SimpleTooltip } from "../../primitives/Tooltip";
 import { motion } from "framer-motion";
 
 type UsageBarProps = {

apps/webapp/app/components/billing/v3/UpgradePrompt.tsx

@@ -0,0 +1,53 @@
+import { ExclamationCircleIcon } from "@heroicons/react/20/solid";
+import tileBgPath from "~/assets/images/error-banner-tile@2x.png";
+import { MatchedOrganization, useOrganization } from "~/hooks/useOrganizations";
+import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route";
+import { v3BillingPath } from "~/utils/pathBuilder";
+import { LinkButton } from "../../primitives/Buttons";
+import { Icon } from "../../primitives/Icon";
+import { Paragraph } from "../../primitives/Paragraph";
+import { DateTime } from "~/components/primitives/DateTime";
+
+export function UpgradePrompt() {
+  const organization = useOrganization();
+  const plan = useCurrentPlan();
+
+  if (!plan || !plan.v3Usage.hasExceededFreeTier) {
+    return null;
+  }
+
+  const nextMonth = new Date();
+  nextMonth.setUTCMonth(nextMonth.getMonth() + 1);
+  nextMonth.setUTCDate(1);
+  nextMonth.setUTCHours(0, 0, 0, 0);
+
+  return (
+    <div
+      className="flex h-10 items-center justify-between border border-error bg-repeat py-0 pl-3 pr-2"
+      style={{ backgroundImage: `url(${tileBgPath})`, backgroundSize: "8px 8px" }}
+    >
+      <div className="flex items-center gap-2">
+        <Icon icon={ExclamationCircleIcon} className="h-5 w-5 text-error" />
+        <Paragraph variant="small" className="text-error">
+          You have exceeded the monthly $
+          {(plan.v3Subscription?.plan?.limits.includedUsage ?? 500) / 100} free credits. No runs
+          will execute in Prod until{" "}
+          <DateTime date={nextMonth} includeTime={false} timeZone="utc" />, or you upgrade.
+        </Paragraph>
+      </div>
+      <LinkButton
+        variant={"primary/small"}
+        leadingIconClassName="px-0"
+        to={v3BillingPath(organization)}
+      >
+        Upgrade
+      </LinkButton>
+    </div>
+  );
+}
+
+export function useShowUpgradePrompt(organization?: MatchedOrganization) {
+  const currentPlan = useCurrentPlan();
+  const shouldShow = currentPlan?.v3Usage.hasExceededFreeTier === true;
+  return { shouldShow };
+}

apps/webapp/app/components/billing/v3/UsageBar.tsx

@@ -0,0 +1,170 @@
+import { cn } from "~/utils/cn";
+import { formatCurrency } from "~/utils/numberFormatter";
+import { Paragraph } from "../../primitives/Paragraph";
+import { SimpleTooltip } from "../../primitives/Tooltip";
+import { motion } from "framer-motion";
+
+type UsageBarProps = {
+  current: number;
+  billingLimit?: number;
+  tierLimit?: number;
+  projectedUsage?: number;
+  isPaying: boolean;
+};
+
+const startFactor = 4;
+
+export function UsageBar({
+  current,
+  billingLimit,
+  tierLimit,
+  projectedUsage,
+  isPaying,
+}: UsageBarProps) {
+  const getLargestNumber = Math.max(
+    current,
+    tierLimit ?? -Infinity,
+    projectedUsage ?? -Infinity,
+    billingLimit ?? -Infinity,
+    5
+  );
+  //creates a maximum range for the progress bar, add 10% to the largest number so the bar doesn't reach the end
+  const maxRange = Math.round(getLargestNumber * 1.1);
+  const tierRunLimitPercentage = tierLimit ? Math.round((tierLimit / maxRange) * 100) : 0;
+  const projectedRunsPercentage = projectedUsage
+    ? Math.round((projectedUsage / maxRange) * 100)
+    : 0;
+  const billingLimitPercentage =
+    billingLimit !== undefined ? Math.round((billingLimit / maxRange) * 100) : 0;
+  const usagePercentage = Math.round((current / maxRange) * 100);
+
+  //cap the usagePercentage to the freeRunLimitPercentage
+  const usageCappedToLimitPercentage = Math.min(usagePercentage, tierRunLimitPercentage);
+
+  return (
+    <div className="h-fit w-full py-12">
+      <div className="relative h-3 w-full rounded-sm bg-background-bright">
+        {billingLimit !== undefined && (
+          <motion.div
+            initial={{ width: billingLimitPercentage / startFactor + "%" }}
+            animate={{ width: billingLimitPercentage + "%" }}
+            transition={{ duration: 1.5, type: "spring" }}
+            style={{ width: `${billingLimitPercentage}%` }}
+            className="absolute h-3 rounded-l-sm"
+          >
+            <Legend
+              text="Billing limit:"
+              value={formatCurrency(billingLimit, false)}
+              position="bottomRow2"
+              percentage={billingLimitPercentage}
+            />
+          </motion.div>
+        )}
+        {tierLimit !== undefined && (
+          <motion.div
+            initial={{ width: tierRunLimitPercentage / startFactor + "%" }}
+            animate={{ width: tierRunLimitPercentage + "%" }}
+            transition={{ duration: 1.5, type: "spring" }}
+            style={{ width: `${tierRunLimitPercentage}%` }}
+            className="absolute h-3 rounded-l-sm bg-green-900/50"
+          >
+            <Legend
+              text={isPaying ? `Included usage:` : `Tier limit:`}
+              value={formatCurrency(tierLimit, false)}
+              position="bottomRow1"
+              percentage={tierRunLimitPercentage}
+            />
+          </motion.div>
+        )}
+        {projectedUsage !== undefined && projectedUsage !== 0 && (
+          <motion.div
+            initial={{ width: projectedRunsPercentage / startFactor + "%" }}
+            animate={{ width: projectedRunsPercentage + "%" }}
+            transition={{ duration: 1.5, type: "spring" }}
+            style={{ width: `${projectedRunsPercentage}%` }}
+            className="absolute h-3 rounded-l-sm"
+          >
+            <Legend
+              text="Projected:"
+              value={formatCurrency(projectedUsage, false)}
+              position="topRow2"
+              percentage={projectedRunsPercentage}
+            />
+          </motion.div>
+        )}
+        <motion.div
+          initial={{ width: usagePercentage / startFactor + "%" }}
+          animate={{ width: usagePercentage + "%" }}
+          transition={{ duration: 1.5, type: "spring" }}
+          style={{ width: `${usagePercentage}%` }}
+          className={cn(
+            "absolute h-3 rounded-l-sm",
+            tierLimit && current > tierLimit ? "bg-rose-600" : "bg-green-600"
+          )}
+        >
+          <Legend
+            text="Used:"
+            value={formatCurrency(current, false)}
+            position="topRow1"
+            percentage={usagePercentage}
+          />
+        </motion.div>
+        <motion.div
+          initial={{ width: usageCappedToLimitPercentage / startFactor + "%" }}
+          animate={{ width: usageCappedToLimitPercentage + "%" }}
+          transition={{ duration: 1.5, type: "spring" }}
+          style={{ width: `${usageCappedToLimitPercentage}%` }}
+          className="absolute h-3 rounded-l-sm bg-green-600"
+        />
+      </div>
+    </div>
+  );
+}
+
+const positions = {
+  topRow1: "bottom-0 h-9",
+  topRow2: "bottom-0 h-14",
+  bottomRow1: "top-0 h-9 items-end",
+  bottomRow2: "top-0 h-14 items-end",
+};
+
+type LegendProps = {
+  text: string;
+  value: number | string;
+  percentage: number;
+  position: keyof typeof positions;
+  tooltipContent?: string;
+};
+
+function Legend({ text, value, position, percentage, tooltipContent }: LegendProps) {
+  const flipLegendPositionValue = 80;
+  const flipLegendPosition = percentage > flipLegendPositionValue ? true : false;
+  return (
+    <div
+      className={cn(
+        "absolute left-full z-10 flex border-charcoal-500",
+        positions[position],
+        flipLegendPosition === true ? "-translate-x-full border-r" : "border-l"
+      )}
+    >
+      {tooltipContent ? (
+        <SimpleTooltip
+          button={
+            <Paragraph className="mr-px h-fit whitespace-nowrap bg-background-bright px-1.5 text-xs text-text-bright">
+              {text}
+              <span className="ml-1 text-text-dimmed">{value}</span>
+            </Paragraph>
+          }
+          side="top"
+          content={tooltipContent}
+          className="z-50 h-fit"
+        />
+      ) : (
+        <Paragraph className="mr-px h-fit whitespace-nowrap bg-background-bright px-1.5 text-xs text-text-bright">
+          {text}
+          <span className="ml-1 text-text-dimmed">{value}</span>
+        </Paragraph>
+      )}
+    </div>
+  );
+}