Update lockfile

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.changeset/afraid-sheep-joke.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fixes an issue with scoped packages in additionalPackages option

.changeset/big-tomatoes-deliver.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Hoist uncaughtException handler to the top of workers to better report error messages

.changeset/blue-pumas-whisper.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/shopify": patch
+---
+
+improved error messages when a shopify webhook fails to register

.changeset/brown-spies-burn.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+cli v3: increase otel force flush timeout to 30s from 500ms

.changeset/clever-apes-collect.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Fix error stack traces

.changeset/clever-carrots-travel.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Implement task.onSuccess/onFailure and config.onSuccess/onFailure

.changeset/clever-donkeys-hunt.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Remove "log" Log Level, unify log and info messages under the "info" log level

.changeset/cool-comics-burn.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fixing missing logs when importing client @opentelemetry/api

.changeset/dull-mangos-press.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Strip out server-only package from worker builds

.changeset/famous-boats-tease.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Better handle issues with resolving dependency versions during deploy

.changeset/five-toes-destroy.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+v3: Environment variable management API and SDK, along with resolveEnvVars CLI hook

.changeset/gorgeous-gorillas-compete.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Add runs.retrieve management API method to get info about a run by run ID

.changeset/hot-buckets-behave.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Support triggering tasks with non-URL friendly characters in the ID

.changeset/hot-fishes-retire.md

@@ -0,0 +1,5 @@
+---
+trigger.dev: patch
+---
+
+Fix TypeScript inclusion in tsconfig.json for `cli-v3 init`

.changeset/itchy-chairs-itch.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Fix for calling trigger and passing a custom queue

.changeset/lazy-files-lay.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Increased the timeout when canceling a checkpoint to 31s (to match the timeout on the server)

.changeset/little-crabs-cross.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Fix issues with consecutive waits

.changeset/long-fireants-search.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Adding some additional telemetry during deploy to help debug issues

.changeset/lovely-drinks-flash.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Add the "log" level back in as an alias to "info"

.changeset/mighty-flowers-train.md

@@ -0,0 +1,83 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Adds support for `emitDecoratorMetadata: true` and `experimentalDecorators: true` in your tsconfig using the [`@anatine/esbuild-decorators`](https://github.com/anatine/esbuildnx/tree/main/packages/esbuild-decorators) package. This allows you to use libraries like TypeORM: 
+
+```ts orm/index.ts
+import "reflect-metadata";
+import { DataSource } from "typeorm";
+import { Entity, Column, PrimaryColumn } from "typeorm";
+
+@Entity()
+export class Photo {
+  @PrimaryColumn()
+  id!: number;
+
+  @Column()
+  name!: string;
+
+  @Column()
+  description!: string;
+
+  @Column()
+  filename!: string;
+
+  @Column()
+  views!: number;
+
+  @Column()
+  isPublished!: boolean;
+}
+
+export const AppDataSource = new DataSource({
+  type: "postgres",
+  host: "localhost",
+  port: 5432,
+  username: "postgres",
+  password: "postgres",
+  database: "v3-catalog",
+  entities: [Photo],
+  synchronize: true,
+  logging: false,
+});
+```
+
+And then in your trigger.config.ts file you can initialize the datasource using the new `init` option:
+
+```ts trigger.config.ts
+import type { TriggerConfig } from "@trigger.dev/sdk/v3";
+import { AppDataSource } from "@/trigger/orm";
+
+export const config: TriggerConfig = {
+  // ... other options here
+  init: async (payload, { ctx }) => {
+    await AppDataSource.initialize();
+  },
+};
+```
+
+Now you are ready to use this in your tasks:
+
+```ts
+import { task } from "@trigger.dev/sdk/v3";
+import { AppDataSource, Photo } from "./orm";
+
+export const taskThatUsesDecorators = task({
+  id: "taskThatUsesDecorators",
+  run: async (payload: { message: string }) => {
+    console.log("Creating a photo...");
+
+    const photo = new Photo();
+    photo.id = 2;
+    photo.name = "Me and Bears";
+    photo.description = "I am near polar bears";
+    photo.filename = "photo-with-bears.jpg";
+    photo.views = 1;
+    photo.isPublished = true;
+
+    await AppDataSource.manager.save(photo);
+  },
+});
+```

.changeset/new-pants-beg.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Fix issue when using SDK in non-node environments by scoping the stream import with node:

.changeset/nice-bulldogs-turn.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Better handle uncaught exceptions

.changeset/pink-pumas-rhyme.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Add configure function to be able to configure the SDK manually

.changeset/plenty-ducks-beam.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Output stderr logs on dev worker failure

.changeset/polite-rockets-matter.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix dev CLI output when not printing update messages

.changeset/pre.json

@@ -44,63 +44,107 @@
     "@trigger.dev/yalt": "2.3.18"
   },
   "changesets": [
+    "afraid-sheep-joke",
     "angry-eagles-trade",
     "beige-pens-dance",
+    "big-tomatoes-deliver",
+    "blue-pumas-whisper",
     "breezy-gorillas-mate",
+    "brown-spies-burn",
     "chilled-hornets-move",
     "clean-pianos-listen",
+    "clever-apes-collect",
+    "clever-carrots-travel",
+    "clever-donkeys-hunt",
+    "cool-comics-burn",
     "cool-glasses-bake",
     "cuddly-feet-approve",
     "dry-walls-check",
+    "dull-mangos-press",
     "eight-pumas-float",
     "eleven-paws-join",
+    "famous-boats-tease",
     "few-students-share",
+    "five-toes-destroy",
     "funny-swans-destroy",
+    "gorgeous-gorillas-compete",
     "green-bags-wink",
+    "hot-buckets-behave",
+    "hot-fishes-retire",
+    "itchy-chairs-itch",
     "khaki-apricots-design",
     "khaki-poems-lay",
     "late-icons-lie",
     "late-steaks-behave",
+    "lazy-files-lay",
     "lemon-jobs-repair",
     "light-bulldogs-press",
     "light-dragons-complain",
+    "little-crabs-cross",
+    "long-fireants-search",
     "loud-actors-remember",
+    "lovely-drinks-flash",
     "many-ligers-pump",
     "mighty-camels-joke",
+    "mighty-flowers-train",
     "nasty-jars-pump",
+    "new-pants-beg",
     "new-rivers-tell",
+    "nice-bulldogs-turn",
     "ninety-pets-travel",
     "odd-poets-own",
+    "pink-pumas-rhyme",
+    "plenty-ducks-beam",
     "polite-ducks-switch",
+    "polite-rockets-matter",
     "poor-flowers-cross",
+    "purple-garlics-shop",
+    "rare-lamps-promise",
     "rare-roses-float",
     "real-planets-stare",
     "rich-kangaroos-unite",
     "rotten-beers-refuse",
     "rotten-dryers-exercise",
+    "rude-toys-compare",
+    "selfish-ducks-sort",
     "shaggy-spoons-taste",
     "sharp-emus-compare",
     "sharp-zebras-serve",
     "shiny-coats-cry",
     "silly-suits-switch",
+    "six-ligers-exist",
+    "sixty-insects-watch",
     "slow-buses-own",
+    "slow-sloths-retire",
     "smart-needles-move",
     "smart-olives-eat",
     "spicy-lamps-smoke",
+    "spicy-terms-bow",
     "strange-ghosts-matter",
+    "strange-sheep-pull",
     "strong-lemons-add",
+    "strong-owls-know",
+    "stupid-adults-sniff",
     "stupid-bulldogs-applaud",
     "sweet-lizards-press",
     "swift-dragons-peel",
     "tall-bees-wave",
     "tame-guests-know",
+    "tender-moose-tell",
     "tender-oranges-rhyme",
+    "thin-parents-heal",
+    "thirty-islands-kiss",
     "tidy-balloons-suffer",
     "tidy-dryers-sleep",
+    "tidy-tomatoes-explain",
     "tiny-doors-type",
     "tiny-elephants-scream",
     "tricky-bulldogs-heal",
+    "tricky-keys-attack",
     "tricky-ladybugs-unite",
-    "two-pumas-wait"
+    "two-pumas-wait",
+    "warm-olives-provide",
+    "warm-planes-taste",
+    "young-snails-sell"
   ]
 }

.changeset/purple-garlics-shop.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Add typescript as a dependency so the esbuild-decorator will work even when running in npx

.changeset/rare-lamps-promise.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Add a postInstall option to allow running scripts after dependencies have been installed in deployed images

.changeset/rude-toys-compare.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Capture and display stderr on index failures

.changeset/selfish-ducks-sort.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+---
+
+Lock SDK and CLI deps on exact core version

.changeset/six-ligers-exist.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Improve the display of non-object return types in the run trace viewer

.changeset/sixty-insects-watch.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Set the deploy timeout to 3mins from 1min

.changeset/slow-sloths-retire.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/core-apps": patch
+"@trigger.dev/core": patch
+---
+
+- Fix uncaught provider exception
+- Remove unused provider messages

.changeset/spicy-terms-bow.md

@@ -0,0 +1,9 @@
+---
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+---
+
+- Fix init command SDK pinning
+- Show --api-url / -a flag where needed
+- CLI now also respects `TRIGGER_TELEMETRY_DISABLED`
+- Dedicated docker checkpoint test function

.changeset/strange-sheep-pull.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+v2: Better handle recovering from platform communication errors by auto-yielding back to the platform in case of temporary API failures

.changeset/strong-owls-know.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+When a v2 run hits the rate limit, reschedule with the reset date

.changeset/stupid-adults-sniff.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Management SDK overhaul and adding the runs.list API

.changeset/tender-moose-tell.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Use locked package versions when resolving dependencies in deployed workers

.changeset/thin-parents-heal.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Ensure @trigger.dev/sdk and @trigger.dev/core are always in the list of deployed dependencies

.changeset/thirty-islands-kiss.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix for typo in v3 CLI login command

.changeset/tidy-tomatoes-explain.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Fix 3rd party otel propagation from breaking our Task Events data from being properly correlated to the correct trace

.changeset/tricky-keys-attack.md

@@ -0,0 +1,14 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+- Clear paused states before retry
+- Detect and handle unrecoverable worker errors
+- Remove checkpoints after successful push
+- Permanently switch to DO hosted busybox image
+- Fix IPC timeout issue, or at least handle it more gracefully
+- Handle checkpoint failures
+- Basic chaos monkey for checkpoint testing
+- Stack traces are back in the dashboard
+- Display final errors on root span

.changeset/warm-olives-provide.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Improve handling of IPC timeouts and fix checkpoint cancellation after failures

.changeset/warm-planes-taste.md

@@ -0,0 +1,7 @@
+---
+"trigger.dev": patch
+---
+
+v3: Prevent legacy-peer-deps=true from breaking deploys
+
+When a global `.npmrc` file includes `legacy-peer-deps=true`, deploys would fail on the `npm ci` step because the package-lock.json wouldn't match the `package.json` file. This is because inside the image build, the `.npmrc` file would not be picked up and so `legacy-peer-deps` would end up being false (which is the default). This change forces the `package-lock.json` file to be created using `legacy-peer-deps=false`

.changeset/young-snails-sell.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Increase cleanup IPC timeout

.env.example

@@ -11,15 +11,22 @@ REMIX_APP_PORT=3030
 APP_ENV=development
 APP_ORIGIN=http://localhost:3030
 NODE_ENV=development
+CLOUD_ENV=development
+V3_ENABLED=true
 
-# Redis is used for concurrency control
-# REDIS_HOST="localhost"
-# REDIS_PORT="6379"
-# REDIS_TLS_DISABLED="true"
+# Redis is used for the v3 queuing and v2 concurrency control
+REDIS_HOST="localhost"
+REDIS_PORT="6379"
+REDIS_TLS_DISABLED="true"
+
+DEV_OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:3030/otel"
+DEV_OTEL_BATCH_PROCESSING_ENABLED="0"
 
 # OPTIONAL VARIABLES
 # This is used for validating emails that are allowed to log in. Every email that do not match this regex will be rejected.
 # WHITELISTED_EMAILS="authorized@yahoo\.com|authorized@gmail\.com"
+# Accounts with these emails will get global admin rights. This grants access to the admin UI.
+# ADMIN_EMAILS="admin@example\.com|another-admin@example\.com"
 # This is used for logging in via GitHub. You can leave these commented out if you don't want to use GitHub for authentication.
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=

.github/workflows/publish-docker.yml

@@ -4,13 +4,15 @@ on:
 jobs:
   publish:
     runs-on: ubuntu-latest
+    env:
+      PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING: 1
     outputs:
       version: ${{ steps.get_version.outputs.version }}
       short_sha: ${{ steps.get_commit.outputs.sha_short }}
     steps:
       - name: Setup Depot CLI
         uses: depot/setup-action@v1
-        
+
       - name: ⬇️ Checkout repo
         uses: actions/checkout@v3
         with:
@@ -37,11 +39,25 @@ jobs:
             exit 1
           fi
           echo "::set-output name=version::${IMAGE_TAG}"
+
       - name: 🔢 Get the commit hash
         id: get_commit
         run: |
           echo ::set-output name=sha_short::$(echo ${{ github.sha }} | cut -c1-7)
 
+      - name: 📛 Set the tags
+        id: set_tags
+        run: |
+          ref_without_tag=ghcr.io/triggerdotdev/trigger.dev
+          image_tags=$ref_without_tag:${{ steps.get_version.outputs.version }}
+
+          # if it's a versioned tag, also tag it as latest
+          if [[ "${{ github.ref_name }}" == v.docker.* ]]; then
+            image_tags=$image_tags,$ref_without_tag:latest
+          fi
+          
+          echo "IMAGE_TAGS=${image_tags}" >> "$GITHUB_OUTPUT"
+
       - name: 🐙 Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:
@@ -54,6 +70,5 @@ jobs:
         with:
           file: ./docker/Dockerfile
           platforms: linux/amd64,linux/arm64
-          tags: |
-            ghcr.io/triggerdotdev/trigger.dev:${{ steps.get_version.outputs.version }}
+          tags: ${{ steps.set_tags.outputs.IMAGE_TAGS }}
           push: true

.github/workflows/publish-infra.yml

@@ -1,6 +1,7 @@
 name: "🚢 Publish Infra Images"
 
 on:
+  workflow_call:
   push:
     tags:
       - "infra-dev-*"
@@ -29,9 +30,6 @@ permissions:
   packages: write
   contents: read
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-
 env:
   AWS_REGION: us-east-1
 
@@ -39,7 +37,7 @@ jobs:
   build:
     strategy:
       matrix:
-        package: [coordinator, kubernetes-provider]
+        package: [coordinator, docker-provider, kubernetes-provider]
     runs-on: buildjet-16vcpu-ubuntu-2204
     env:
       DOCKER_BUILDKIT: "1"
@@ -48,20 +46,40 @@ jobs:
 
       - name: Generate image reference
         id: prep
-        # WARNING: This step expects the workflow to have been triggered by a specific tag format of: infra-${env}-*
         run: |
-          env=$(echo ${{ github.ref_name }} | cut -d- -f2)
-          sha=${GITHUB_SHA::7}
-          ts=$(date +%s)
+          # set image repo
           if [[ "${{ matrix.package }}" == *-provider ]]; then
-            provider_type=$(echo ${{ matrix.package }} | cut -d- -f1)
+            provider_type=$(echo "${{ matrix.package }}" | cut -d- -f1)
             repository=provider/${provider_type}
           else
-            repository=${{ matrix.package }}
+            repository="${{ matrix.package }}"
           fi
-          echo "IMAGE_TAG=${env}-${sha}-${ts}" >> "$GITHUB_OUTPUT"
           echo "REPOSITORY=${repository}" >> "$GITHUB_OUTPUT"
 
+          # set image tag
+          if [[ "${{ github.ref_type }}" == "tag" ]]; then
+            if [[ "${{ github.ref_name }}" == infra-*-* ]]; then
+              env=$(echo ${{ github.ref_name }} | cut -d- -f2)
+              sha=$(echo ${{ github.sha }} | head -c7)
+              ts=$(date +%s)
+              image_tag=${env}-${sha}-${ts}
+            elif [[ "${{ github.ref_name }}" == v.docker.* ]]; then
+              version="${GITHUB_REF_NAME#v.docker.}"
+              image_tag="v${version}"
+            elif [[ "${{ github.ref_name }}" == build-* ]]; then
+              image_tag="${GITHUB_REF_NAME#build-}"
+            else
+              echo "Invalid tag: ${{ github.ref_name }}"
+              exit 1
+            fi
+          elif [[ "${{ github.ref_name }}" == "main" ]]; then
+            image_tag="main"
+          else
+            echo "Invalid reference: ${{ github.ref }}"
+            exit 1
+          fi
+          echo "IMAGE_TAG=${image_tag}" >> "$GITHUB_OUTPUT"
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
@@ -92,3 +110,12 @@ jobs:
           REGISTRY: ghcr.io/triggerdotdev
           REPOSITORY: ${{ steps.prep.outputs.REPOSITORY }}
           IMAGE_TAG: ${{ steps.prep.outputs.IMAGE_TAG }}
+
+      - name: 🐙 Push 'latest' to GitHub Container Registry
+        if: startsWith(github.ref_name, 'v.docker.')
+        run: |
+          docker tag infra_image $REGISTRY/$REPOSITORY:latest
+          docker push $REGISTRY/$REPOSITORY:latest
+        env:
+          REGISTRY: ghcr.io/triggerdotdev
+          REPOSITORY: ${{ steps.prep.outputs.REPOSITORY }}

.github/workflows/publish.yml

@@ -57,3 +57,8 @@ jobs:
     needs: [typecheck, units]
     uses: ./.github/workflows/publish-docker.yml
     secrets: inherit
+
+  publish-infra:
+    needs: [typecheck, units]
+    uses: ./.github/workflows/publish-infra.yml
+    secrets: inherit

.vscode/launch.json

@@ -45,6 +45,14 @@
       "cwd": "${workspaceFolder}/references/v3-catalog",
       "sourceMaps": true
     },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 Management",
+      "command": "pnpm run management",
+      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "sourceMaps": true
+    },
     {
       "type": "node",
       "request": "attach",

CONTRIBUTING.md

@@ -10,13 +10,14 @@ Thank you for helping us make Trigger.dev even better! 🤩
 
 The development branch is `main`. This is the branch that all pull
 requests should be made against. The changes on the `main`
-branch are tagged into a release monthly.
+branch are tagged into a release periodically.
 
 ### Prerequisites
 
-- [Node.js](https://nodejs.org/en) version >=16.x
-- [pnpm package manager](https://pnpm.io/installation) version 7
+- [Node.js](https://nodejs.org/en) version 20.11.1
+- [pnpm package manager](https://pnpm.io/installation) version 8.15.5
 - [Docker](https://www.docker.com/get-started/)
+- [protobuf](https://github.com/protocolbuffers/protobuf)
 
 ### Setup
 
@@ -33,15 +34,19 @@ branch are tagged into a release monthly.
    ```
    cd trigger.dev
    ```
-3. Install the required packages using pnpm.
+3. Ensure you are on the correct version of Node.js (20.11.1). If you are using `nvm`, there is an `.nvmrc` file that will automatically select the correct version of Node.js when you navigate to the repository.
+
+4. Run `corepack enable` to use the correct version of pnpm (`8.15.5`) as specified in the root `package.json` file.
+
+5. Install the required packages using pnpm.
    ```
    pnpm i
    ```
-4. Create your `.env` file
+6. Create your `.env` file
    ```
    cp .env.example .env
    ```
-5. Open it and generate a new value for `ENCRYPTION_KEY`:
+7. Open it and generate a new value for `ENCRYPTION_KEY`:
 
    `ENCRYPTION_KEY` is used to two-way encrypt OAuth access tokens and so you'll probably want to actually generate a unique value, and it must be a random 16 byte hex string. You can generate one with the following command:
 
@@ -51,7 +56,7 @@ branch are tagged into a release monthly.
 
    Feel free to update `SESSION_SECRET` and `MAGIC_LINK_SECRET` as well using the same method.
 
-6. Start Docker. This starts the required services like Postgres. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)
+8. Start Docker. This starts the required services like Postgres & Redis. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)
 
    ```
    pnpm run docker
@@ -59,19 +64,15 @@ branch are tagged into a release monthly.
 
    This will also start and run a local instance of [pgAdmin](https://www.pgadmin.org/) on [localhost:5480](http://localhost:5480), preconfigured with email `admin@example.com` and pwd `admin`. Then use `postgres` as the password to the Trigger.dev server.
 
-7. Migrate the database
+9. Migrate the database
    ```
    pnpm run db:migrate
    ```
-8. Build the app
-   ```
-   pnpm run build --filter webapp
-   ```
-9. Run the seed script
-   ```
-   pnpm run db:seed
-   ```
-10. Run the app. See the section below.
+10. Build the server app
+    ```
+    pnpm run build --filter webapp
+    ```
+11. Run the app. See the section below.
 
 ## Running
 
@@ -83,99 +84,96 @@ branch are tagged into a release monthly.
 
    It should run on port `3030`: [http://localhost:3030](http://localhost:3030/)
 
-2. Once the app is running click the magic link button and enter your email.
-3. Check your terminal, the magic link email should have printed out as following:
+2. Once the app is running click the magic link button and enter your email. You will automatically be logged in, since you are running locally. Create an Org and your first project in the dashboard.
 
-   ```sh
-   webapp:dev: Log in to Trigger.dev
-   webapp:dev:
-   webapp:dev: Click here to log in with this magic link
-   webapp:dev: [http://localhost:3030/magic?token=U2FsdGVkX18OvB0JxgaswTLCSbaRz%2FY82TN0EZWhSzFyZYwgG%2BIzKVTkeiaOtWfotPw7F8RwFzCHh53aBpMEu%2B%2B%2FItb%2FcJYh89MSjc3Pz92bevoEjqxSQ%2Ff%2BZbks09JOpqlBbYC3FzGWC8vuSVFBlxqLXxteSDLthZSUaC%2BS2LaA%2BJgp%2BLO7hgjAaC2lXbCHrM7MTgTdXOFt7i0Dvvuwz6%2BWY25RnfomZOPqDsyH0xz8Q2rzPTz0Xu53WSXrZ1hd]
-   webapp:dev:
-   webapp:dev: If you didn't try to log in, you can safely ignore this email.
-   ```
+## Manual testing using v3-catalog
 
-   Paste the magic link shown in your terminal into your browser to login.
+We use the `<root>/references/v3-catalog` subdirectory as a staging ground for testing changes to the SDK (`@trigger.dev/sdk` at `<root>/packages/trigger-sdk`), the Core package (`@trigger.dev/core` at `<root>packages/core`), the CLI (`trigger.dev` at `<root>/packages/cli-v3`) and the platform (The remix app at `<root>/apps/webapp`). The instructions below will get you started on using the `v3-catalog` for local development of Trigger.dev (v3).
 
-## Adding and running migrations
+### First-time setup
 
-1. Modify packages/database/prisma/schema.prisma file
-2. Change directory to the packages/database folder
-   ```sh
-   cd packages/database
-   ```
-3. Generate the Prisma client
+First, make sure you are running the webapp according to the instructions above. Then:
 
-   ```sh
-   pnpm run generate
-   ```
+1. In Postgres go to the "Organizations" table and on your org set the `v3Enabled` column to `true`.
 
-   The above updates the prisma client generated into node_modules/.prisma/client folder. This helps with typing of relevant prisma models. It ensures typescript
-   recognizes fields added or removed from a model and type-checks appropriately.
+2. Visit http://localhost:3030 in your browser and create a new V3 project called "v3-catalog". If you don't see an option for V3, you haven't set the `v3Enabled` flag to true.
 
-4. Create and apply the migrations
+3. In Postgres go to the "Projects" table and for the project you create change the `externalRef` to `yubjwjsfkxnylobaqvqz`.
 
-   ```
-   pnpm run db:migrate:dev
-   ```
-
-   This creates a migration file and executes the migrations against your database and applies changes to the database schema(s)
-
-5. Commit generated migrations as well as changes to the schema.prisma file
-6. If you're using VSCode you may need to restart the Typescript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`.
-
-## Testing CLI changes
-
-To test CLI changes, follow the steps below:
-
-1. Build the CLI and watch for changes
+4. Build the CLI
 
 ```sh
-cd packages/cli
-pnpm run dev
-```
-
-2. Open a new Terminal window and run the webapp locally and then create a new project in the dashboard. Copy out the dev API key.
-
-3. Create a new temporary Next.js app in references directory
-
-```sh
-cd ./references
-pnpm create next-app@latest test-cli --ts --no-eslint --tailwind --app --src-dir --import-alias "@/*"
-```
-
-4. Then once that's finished, add the `@trigger.dev/cli` to the `devDependencies` of the newly created Next.js app's `package.json` file, like so:
-
-```json
-{
-  // other package.json properties
-  "devDependencies": { "@trigger.dev/cli": "workspace:*" }
-}
-```
-
-5. Back in the terminal, navigate into the reference, and initialize the CLI. When prompted, select `self-hosted` and enter `localhost:3030` if you are testing against the local instance of Trigger.dev, or you can just use the Trigger.dev cloud. When asked for an API key, use the key you copied earlier.
-
-```sh
-cd ./test-cli
+# Build the CLI
+pnpm run build --filter trigger.dev
+# Make it accessible to `pnpm exec`
 pnpm i
-pnpm exec trigger-cli init
 ```
 
-6. If you are just testing the `init` command, you can stop here. If you'd like to test the `dev` command, first start the Next.js app on port 3000:
+5. Change into the `<root>/references/v3-catalog` directory and authorize the CLI to the local server:
 
 ```sh
-pnpm run dev
+cd references/v3-catalog
+cp .env.example .env
+pnpm exec triggerdev login -a http://localhost:3030
 ```
 
-7. Open a new terminal window, and then run the `dev` command like so:
+This will open a new browser window and authorize the CLI against your local user account.
+
+You can optionally pass a `--profile` flag to the `login` command, which will allow you to use the CLI with separate accounts/servers. We suggest using a profile called `local` for your local development:
 
 ```sh
-pnpm exec trigger-cli dev
+cd references/v3-catalog
+pnpm exec triggerdev login -a http://localhost:3030 --profile local
+# later when you run the dev or deploy command:
+pnpm exec triggerdev dev --profile local
+pnpm exec triggerdev deploy --profile local
 ```
 
-8. Please remember to delete the temporary project you created after you've tested the changes, and before you raise a PR.
+### Running
 
-## Running end-to-end webapp tests
+The following steps should be followed any time you start working on a new feature you want to test in v3:
+
+1. Make sure the webapp is running on localhost:3030
+
+2. Open a terminal window and build the CLI and watch for changes
+
+```sh
+pnpm run dev --filter trigger.dev
+```
+
+2. Open a new terminal window, and anytime changes are made to the `@trigger.dev/core` package, you'll need to manually rebuild the CLI:
+
+```sh
+pnpm run build --filter trigger.dev
+```
+
+Note: You do not need to do the same for `@trigger.dev/sdk`, just core.
+
+3. Open another terminal window, and change into the `<root>/references/v3-catalog` directory.
+
+4. Run the `dev` command, which will register all the local tasks with the platform and allow you to start testing task execution:
+
+```sh
+# in <root>/references/v3-catalog
+pnpm exec triggerdev dev
+```
+
+If you want additional debug logging, you can use the `--log-level debug` flag:
+
+```sh
+# in <root>/references/v3-catalog
+pnpm exec triggerdev dev --log-level debug
+```
+
+5. If you make any changes in the CLI/Core/SDK, you'll need to `CTRL+C` to exit the `dev` command and restart it to pickup changes. Any changes to the files inside of the `v3-catalog/src/trigger` dir will automatically be rebuilt by the `dev` command.
+
+6. Navigate to the `v3-catalog` project in your local dashboard at localhost:3030 and you should see the list of tasks.
+
+7. Go to the "Test" page in the sidebar and select a task. Then enter a payload and click "Run test". You can tell what the payloads should be by looking at the relevant task file inside the `/references/v3-catalog/src/trigger` folder. Many of them accept an empty payload.
+
+8. Feel free to add additional files in `v3-catalog/src/trigger` to test out specific aspects of the system, or add in edge cases.
+
+## Running end-to-end webapp tests (deprecated)
 
 To run the end-to-end tests, follow the steps below:
 
@@ -223,6 +221,26 @@ The end-to-end tests use a `setup` and `teardown` script to seed the database wi
 pnpm run db:studio
 ```
 
+## Adding and running migrations
+
+1. Modify packages/database/prisma/schema.prisma file
+2. Change directory to the packages/database folder
+
+   ```sh
+   cd packages/database
+   ```
+
+3. Create and apply the migrations
+
+   ```
+   pnpm run db:migrate:dev
+   ```
+
+   This creates a migration file and executes the migrations against your database and applies changes to the database schema(s)
+
+4. Commit generated migrations as well as changes to the schema.prisma file
+5. If you're using VSCode you may need to restart the Typescript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`.
+
 ## Add sample jobs
 
 The [references/job-catalog](./references/job-catalog/) project defines simple jobs you can get started with.

README.md

@@ -5,7 +5,7 @@
   <img alt="Trigger.dev logo" src="https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/a45d1fa2-0ae8-4a39-4409-f4f934bfae00/public">
 </picture>
   
-### The open source background jobs framework
+### The open source background jobs platform
 
 [Discord](https://trigger.dev/discord) | [Website](https://trigger.dev) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Docs](https://trigger.dev/docs)
 
@@ -14,7 +14,9 @@
 
 </div>
 
-# About Trigger.dev
+> The Trigger.dev v3 developer preview is now open. For more information and to get early access, check out our [developer preview launch post](https://trigger.dev/blog/v3-developer-preview-launch/).
+
+## About Trigger.dev
 
 Create long-running jobs directly in your codebase with features like API integrations, webhooks, scheduling and delays.
 
@@ -48,30 +50,6 @@ View every Task in every Run so you can tell exactly what happened.
 
 Easily integrate with hundreds of third-party APIs – including your own. Use API keys (which never leave your server) or let us handle OAuth for you. Install our integration packages and easily subscribe to webhooks and perform common tasks, or you can easily use your existing favorite Node.JS SDKs and get resumability and idempotency through our `runTask` function.
 
-## Our progress
-
-We’re building the most comprehensive and easy-to-use background jobs framework for developers.
-
-Click the links to join the discussions about our upcoming features.
-
-| Feature                                                                              | What it does                                        | Status |
-| ------------------------------------------------------------------------------------ | --------------------------------------------------- | ------ |
-| Integration kit                                                                      | Official Trigger.dev integrations or build your own | ✅     |
-| Self-hosting                                                                         | Host the platform yourself                          | ✅     |
-| Cloud                                                                                | Just write code, no deployment required             | ✅     |
-| Dashboard                                                                            | View every Task in every Run                        | ✅     |
-| Serverless                                                                           | Long-running Jobs on your serverless backend        | ✅     |
-| React hooks                                                                          | Easily update your UI with Job progress             | ✅     |
-| React frameworks                                                                     | Support for Remix, Astro, RedwoodJS & more          | ✅     |
-| [Background tasks](https://github.com/triggerdotdev/trigger.dev/discussions/400)     | Offload long or intense Tasks to our infrastructure | 🛠️     |
-| [Long-running servers](https://github.com/triggerdotdev/trigger.dev/discussions/430) | Run Jobs on your long-running backend               | 🛠️     |
-| Polling Triggers                                                                     | Subscribe to changes without webhooks               | 🕝     |
-| Vercel integration                                                                   | Easy deploy and preview environment support         | 🕝     |
-| Streaming                                                                            | Receive data from your Jobs in realtime             | 🕝     |
-| 100+ integrations                                                                    | Comprehensive support for popular APIs              | 🕝     |
-| [Trigger.dev Connect](https://github.com/triggerdotdev/trigger.dev/discussions/441)  | Use integrations signed in as your users            | 🕝     |
-| File IO                                                                              | Create Tasks that have file outputs                 | 🕝     |
-
 # Getting started
 
 Visit our docs [here](https://trigger.dev/docs).

apps/coordinator/Containerfile

@@ -1,19 +1,19 @@
 # syntax=docker/dockerfile:labs
 
-FROM node:18-bullseye-slim@sha256:a4edd54dcfdcacc8a4100fee71498e8671d99556a1acf5614539214a70092426 AS node-18
+FROM node:20-bookworm-slim@sha256:72f2f046a5f8468db28730b990b37de63ce93fd1a72a40f531d6aa82afdf0d46 AS node-20
 
 WORKDIR /app
 
-FROM node-18 AS pruner
+FROM node-20 AS pruner
 
 COPY --chown=node:node . .
 RUN npx -q turbo@1.10.9 prune --scope=coordinator --docker
 RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
 
-FROM node-18 AS base
+FROM node-20 AS base
 
 RUN apt-get update \
-  && apt-get install -y buildah ca-certificates dumb-init \
+  && apt-get install -y buildah ca-certificates dumb-init docker.io \
   && rm -rf /var/lib/apt/lists/*
 
 COPY --chown=node:node .gitignore .gitignore

apps/coordinator/src/backoff.ts

@@ -0,0 +1,247 @@
+type ExponentialBackoffType = "NoJitter" | "FullJitter" | "EqualJitter";
+
+type ExponentialBackoffOptions = {
+  base: number;
+  factor: number;
+  min: number;
+  max: number;
+  maxRetries: number;
+  maxElapsed: number;
+};
+
+class StopRetrying extends Error {
+  constructor(message?: string) {
+    super(message);
+    this.name = "StopRetrying";
+  }
+}
+
+export class ExponentialBackoff {
+  #retries: number = 0;
+
+  #type: ExponentialBackoffType;
+  #base: number;
+  #factor: number;
+
+  #min: number;
+  #max: number;
+
+  #maxRetries: number;
+  #maxElapsed: number;
+
+  constructor(type?: ExponentialBackoffType, opts: Partial<ExponentialBackoffOptions> = {}) {
+    this.#type = type ?? "NoJitter";
+    this.#base = opts.base ?? 2;
+    this.#factor = opts.factor ?? 1;
+
+    this.#min = opts.min ?? -Infinity;
+    this.#max = opts.max ?? Infinity;
+
+    this.#maxRetries = opts.maxRetries ?? Infinity;
+    this.#maxElapsed = opts.maxElapsed ?? Infinity;
+  }
+
+  #clone() {
+    return new ExponentialBackoff(this.#type, {
+      base: this.#base,
+      factor: this.#factor,
+      min: this.#min,
+      max: this.#max,
+      maxRetries: this.#maxRetries,
+      maxElapsed: this.#maxElapsed,
+    });
+  }
+
+  type(type?: ExponentialBackoffType) {
+    if (typeof type !== "undefined") {
+      this.#type = type;
+    }
+    return this.#clone();
+  }
+
+  base(base?: number) {
+    if (typeof base !== "undefined") {
+      this.#base = base;
+    }
+    return this.#clone();
+  }
+
+  factor(factor?: number) {
+    if (typeof factor !== "undefined") {
+      this.#factor = factor;
+    }
+    return this.#clone();
+  }
+
+  min(min?: number) {
+    if (typeof min !== "undefined") {
+      this.#min = min;
+    }
+    return this.#clone();
+  }
+
+  max(max?: number) {
+    if (typeof max !== "undefined") {
+      this.#max = max;
+    }
+    return this.#clone();
+  }
+
+  maxRetries(maxRetries?: number) {
+    if (typeof maxRetries !== "undefined") {
+      this.#maxRetries = maxRetries;
+    }
+    return this.#clone();
+  }
+
+  maxElapsed(maxElapsed?: number) {
+    if (typeof maxElapsed !== "undefined") {
+      this.#maxElapsed = maxElapsed;
+    }
+    return this.#clone();
+  }
+
+  retries(retries?: number) {
+    if (typeof retries !== "undefined") {
+      if (retries > this.#maxRetries) {
+        console.error(
+          `Can't set retries ${retries} higher than maxRetries (${
+            this.#maxRetries
+          }), setting to maxRetries instead.`
+        );
+        this.#retries = this.#maxRetries;
+      } else {
+        this.#retries = retries;
+      }
+    }
+    return this.#clone();
+  }
+
+  async *retryAsync(maxRetries: number = this.#maxRetries ?? Infinity) {
+    let elapsed = 0;
+    let retry = 0;
+
+    while (retry <= maxRetries) {
+      const delay = this.delay(retry);
+      elapsed += delay;
+
+      if (elapsed > this.#maxElapsed) {
+        break;
+      }
+
+      yield {
+        delay: {
+          seconds: delay,
+          milliseconds: delay * 1000,
+        },
+        retry,
+      };
+
+      retry++;
+    }
+  }
+
+  async *[Symbol.asyncIterator]() {
+    yield* this.retryAsync();
+  }
+
+  delay(retries: number = this.#retries, jitter: boolean = true) {
+    if (retries > this.#maxRetries) {
+      console.error(
+        `Can't set retries ${retries} higher than maxRetries (${
+          this.#maxRetries
+        }), setting to maxRetries instead.`
+      );
+      retries = this.#maxRetries;
+    }
+
+    let delay = this.#factor * this.#base ** retries;
+
+    switch (this.#type) {
+      case "NoJitter": {
+        break;
+      }
+      case "FullJitter": {
+        if (!jitter) {
+          delay = 0;
+          break;
+        }
+
+        delay *= Math.random();
+        break;
+      }
+      case "EqualJitter": {
+        if (!jitter) {
+          delay *= 0.5;
+          break;
+        }
+
+        delay *= 0.5 * (1 + Math.random());
+        break;
+      }
+      default: {
+        throw new Error(`Unknown backoff type: ${this.#type}`);
+      }
+    }
+
+    delay = Math.min(delay, this.#max);
+    delay = Math.max(delay, this.#min);
+    delay = Math.round(delay);
+
+    return delay;
+  }
+
+  elapsed(retries: number = this.#retries, jitter: boolean = true) {
+    let elapsed = 0;
+
+    for (let i = 0; i <= retries; i++) {
+      elapsed += this.delay(i, jitter);
+    }
+
+    const total = elapsed;
+
+    let days = 0;
+    if (elapsed > 3600 * 24) {
+      days = Math.floor(elapsed / 3600 / 24);
+      elapsed -= days * 3600 * 24;
+    }
+
+    let hours = 0;
+    if (elapsed > 3600) {
+      hours = Math.floor(elapsed / 3600);
+      elapsed -= hours * 3600;
+    }
+
+    let minutes = 0;
+    if (elapsed > 60) {
+      minutes = Math.floor(elapsed / 60);
+      elapsed -= minutes * 60;
+    }
+
+    const seconds = elapsed;
+
+    return {
+      seconds,
+      minutes,
+      hours,
+      days,
+      total,
+    };
+  }
+
+  reset() {
+    this.#retries = 0;
+    return this;
+  }
+
+  next() {
+    this.#retries++;
+    return this.delay();
+  }
+
+  stop() {
+    throw new StopRetrying();
+  }
+
+  static StopRetrying = StopRetrying;
+}

apps/coordinator/src/index.ts

@@ -1,5 +1,6 @@
 import { createServer } from "node:http";
-import { $ } from "execa";
+import fs from "node:fs/promises";
+import { $, type ExecaChildProcess } from "execa";
 import { nanoid } from "nanoid";
 import { Server } from "socket.io";
 import {
@@ -11,7 +12,8 @@ import {
 } from "@trigger.dev/core/v3";
 import { ZodNamespace } from "@trigger.dev/core/v3/zodNamespace";
 import { ZodSocketConnection } from "@trigger.dev/core/v3/zodSocket";
-import { HttpReply, getTextBody, SimpleLogger } from "@trigger.dev/core-apps";
+import { HttpReply, getTextBody, SimpleLogger, testDockerCheckpoint } from "@trigger.dev/core-apps";
+import { ExponentialBackoff } from "./backoff";
 
 import { collectDefaultMetrics, register, Gauge } from "prom-client";
 collectDefaultMetrics();
@@ -19,8 +21,29 @@ collectDefaultMetrics();
 const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || 8020);
 const NODE_NAME = process.env.NODE_NAME || "coordinator";
 const DEFAULT_RETRY_DELAY_THRESHOLD_IN_MS = 30_000;
+const CHAOS_MONKEY_ENABLED = !!process.env.CHAOS_MONKEY_ENABLED;
+
+const FORCE_CHECKPOINT_SIMULATION = ["1", "true"].includes(
+  process.env.FORCE_CHECKPOINT_SIMULATION ?? "true"
+);
+const DISABLE_CHECKPOINT_SUPPORT = ["1", "true"].includes(
+  process.env.DISABLE_CHECKPOINT_SUPPORT ?? "false"
+);
+const SIMULATE_PUSH_FAILURE = ["1", "true"].includes(process.env.SIMULATE_PUSH_FAILURE ?? "false");
+const SIMULATE_PUSH_FAILURE_SECONDS = parseInt(
+  process.env.SIMULATE_PUSH_FAILURE_SECONDS ?? "300",
+  10
+);
+const SIMULATE_CHECKPOINT_FAILURE = ["1", "true"].includes(
+  process.env.SIMULATE_CHECKPOINT_FAILURE ?? "false"
+);
+const SIMULATE_CHECKPOINT_FAILURE_SECONDS = parseInt(
+  process.env.SIMULATE_CHECKPOINT_FAILURE_SECONDS ?? "300",
+  10
+);
 
 const REGISTRY_HOST = process.env.REGISTRY_HOST || "localhost:5000";
+const REGISTRY_NAMESPACE = process.env.REGISTRY_NAMESPACE || "trigger";
 const CHECKPOINT_PATH = process.env.CHECKPOINT_PATH || "/checkpoints";
 const REGISTRY_TLS_VERIFY = process.env.REGISTRY_TLS_VERIFY === "false" ? "false" : "true";
 
@@ -32,6 +55,10 @@ const SECURE_CONNECTION = ["1", "true"].includes(process.env.SECURE_CONNECTION ?
 
 const logger = new SimpleLogger(`[${NODE_NAME}]`);
 
+if (CHAOS_MONKEY_ENABLED) {
+  logger.log("🍌 Chaos monkey enabled");
+}
+
 type CheckpointerInitializeReturn = {
   canCheckpoint: boolean;
   willSimulate: boolean;
@@ -44,11 +71,52 @@ type CheckpointAndPushOptions = {
   deploymentVersion: string;
 };
 
+type CheckpointAndPushResult =
+  | { success: true; checkpoint: CheckpointData }
+  | {
+      success: false;
+      reason?: "CANCELED" | "DISABLED" | "ERROR" | "IN_PROGRESS" | "NO_SUPPORT" | "SKIP_RETRYING";
+    };
+
 type CheckpointData = {
   location: string;
   docker: boolean;
 };
 
+function isExecaChildProcess(maybeExeca: unknown): maybeExeca is Awaited<ExecaChildProcess> {
+  return typeof maybeExeca === "object" && maybeExeca !== null && "escapedCommand" in maybeExeca;
+}
+
+async function getFileSize(filePath: string): Promise<number> {
+  try {
+    const stats = await fs.stat(filePath);
+    return stats.size;
+  } catch (error) {
+    console.error("Error getting file size:", error);
+    return -1;
+  }
+}
+
+async function getParsedFileSize(filePath: string) {
+  const sizeInBytes = await getFileSize(filePath);
+
+  let message = `Size in bytes: ${sizeInBytes}`;
+
+  if (sizeInBytes > 1024 * 1024) {
+    const sizeInMB = (sizeInBytes / 1024 / 1024).toFixed(2);
+    message = `Size in MB (rounded): ${sizeInMB}`;
+  } else if (sizeInBytes > 1024) {
+    const sizeInKB = (sizeInBytes / 1024).toFixed(2);
+    message = `Size in KB (rounded): ${sizeInKB}`;
+  }
+
+  return {
+    path: filePath,
+    sizeInBytes,
+    message,
+  };
+}
+
 class Checkpointer {
   #initialized = false;
   #canCheckpoint = false;
@@ -56,73 +124,63 @@ class Checkpointer {
 
   #logger = new SimpleLogger("[checkptr]");
   #abortControllers = new Map<string, AbortController>();
+  #failedCheckpoints = new Map<string, unknown>();
+  #waitingForRetry = new Set<string>();
 
   constructor(private opts = { forceSimulate: false }) {}
 
-  async initialize(): Promise<CheckpointerInitializeReturn> {
+  async init(): Promise<CheckpointerInitializeReturn> {
     if (this.#initialized) {
-      return this.#getInitializeReturn();
+      return this.#getInitReturn(this.#canCheckpoint);
     }
 
     this.#logger.log(`${this.#dockerMode ? "Docker" : "Kubernetes"} mode`);
 
     if (this.#dockerMode) {
-      try {
-        await $`criu --version`;
-      } catch (error) {
-        this.#logger.error("No checkpoint support: Missing CRIU binary");
-        this.#logger.error("Will simulate instead");
-        this.#canCheckpoint = false;
-        this.#initialized = true;
+      const testCheckpoint = await testDockerCheckpoint();
 
-        return this.#getInitializeReturn();
+      if (testCheckpoint.ok) {
+        return this.#getInitReturn(true);
       }
 
-      try {
-        await $`docker checkpoint`;
-      } catch (error) {
-        this.#logger.error(
-          "No checkpoint support: Docker needs to have experimental features enabled"
-        );
-        this.#logger.error("Will simulate instead");
-        this.#canCheckpoint = false;
-        this.#initialized = true;
-
-        return this.#getInitializeReturn();
-      }
+      this.#logger.error(testCheckpoint.message, testCheckpoint.error ?? "");
+      return this.#getInitReturn(false);
     } else {
       try {
         await $`buildah login --get-login ${REGISTRY_HOST}`;
       } catch (error) {
         this.#logger.error(`No checkpoint support: Not logged in to registry ${REGISTRY_HOST}`);
-        this.#canCheckpoint = false;
-        this.#initialized = true;
-
-        return this.#getInitializeReturn();
+        return this.#getInitReturn(false);
       }
     }
 
-    this.#logger.log(
-      `Full checkpoint support${
-        this.#dockerMode && this.opts.forceSimulate ? " with forced simulation enabled." : "!"
-      }`
-    );
-
-    this.#initialized = true;
-    this.#canCheckpoint = true;
-
-    return this.#getInitializeReturn();
+    return this.#getInitReturn(true);
   }
 
-  #getInitializeReturn(): CheckpointerInitializeReturn {
+  #getInitReturn(canCheckpoint: boolean): CheckpointerInitializeReturn {
+    this.#initialized = true;
+    this.#canCheckpoint = canCheckpoint;
+
+    if (canCheckpoint) {
+      this.#logger.log("Full checkpoint support!");
+    }
+
+    const willSimulate = this.#dockerMode && (!this.#canCheckpoint || this.opts.forceSimulate);
+
+    if (willSimulate) {
+      this.#logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", {
+        forceSimulate: this.opts.forceSimulate,
+      });
+    }
+
     return {
-      canCheckpoint: this.#canCheckpoint,
-      willSimulate: this.#dockerMode && (!this.#canCheckpoint || this.opts.forceSimulate),
+      canCheckpoint,
+      willSimulate,
     };
   }
 
   #getImageRef(projectRef: string, deploymentVersion: string, shortCode: string) {
-    return `${REGISTRY_HOST}/trigger/${projectRef}:${deploymentVersion}.prod-${shortCode}`;
+    return `${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${projectRef}:${deploymentVersion}.prod-${shortCode}`;
   }
 
   #getExportLocation(projectRef: string, deploymentVersion: string, shortCode: string) {
@@ -139,7 +197,7 @@ class Checkpointer {
     const start = performance.now();
     logger.log(`checkpointAndPush() start`, { start, opts });
 
-    const result = await this.#checkpointAndPush(opts);
+    const result = await this.#checkpointAndPushWithBackoff(opts);
 
     const end = performance.now();
     logger.log(`checkpointAndPush() end`, {
@@ -147,26 +205,145 @@ class Checkpointer {
       end,
       diff: end - start,
       opts,
-      success: !!result,
+      success: result.success,
     });
 
-    return result;
+    if (!result.success) {
+      return;
+    }
+
+    return result.checkpoint;
   }
 
   isCheckpointing(runId: string) {
-    return this.#abortControllers.has(runId);
+    return this.#abortControllers.has(runId) || this.#waitingForRetry.has(runId);
   }
 
-  cancelCheckpoint(runId: string) {
+  cancelCheckpoint(runId: string): boolean {
+    // If the last checkpoint failed, pretend we canceled it
+    // This ensures tasks don't wait for external resume messages to continue
+    if (this.#hasFailedCheckpoint(runId)) {
+      this.#clearFailedCheckpoint(runId);
+      return true;
+    }
+
+    if (this.#waitingForRetry.has(runId)) {
+      this.#waitingForRetry.delete(runId);
+      return true;
+    }
+
     const controller = this.#abortControllers.get(runId);
 
     if (!controller) {
       logger.debug("Nothing to cancel", { runId });
-      return;
+      return false;
     }
 
     controller.abort("cancelCheckpointing()");
     this.#abortControllers.delete(runId);
+
+    return true;
+  }
+
+  async #checkpointAndPushWithBackoff({
+    runId,
+    leaveRunning = true, // This mirrors kubernetes behaviour more accurately
+    projectRef,
+    deploymentVersion,
+  }: CheckpointAndPushOptions): Promise<CheckpointAndPushResult> {
+    this.#logger.log("Checkpointing with backoff", {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+    });
+
+    const backoff = new ExponentialBackoff()
+      .type("EqualJitter")
+      .base(3)
+      .max(3 * 3600)
+      .maxElapsed(48 * 3600);
+
+    for await (const { delay, retry } of backoff) {
+      try {
+        if (retry > 0) {
+          this.#logger.error("Retrying checkpoint", {
+            runId,
+            retry,
+            delay,
+          });
+
+          this.#waitingForRetry.add(runId);
+          await new Promise((resolve) => setTimeout(resolve, delay.milliseconds));
+
+          if (!this.#waitingForRetry.has(runId)) {
+            this.#logger.log("Checkpoint canceled while waiting for retry", { runId });
+            return { success: false, reason: "CANCELED" };
+          } else {
+            this.#waitingForRetry.delete(runId);
+          }
+        }
+
+        const result = await this.#checkpointAndPush({
+          runId,
+          leaveRunning,
+          projectRef,
+          deploymentVersion,
+        });
+
+        if (result.success) {
+          return result;
+        }
+
+        if (result.reason === "CANCELED") {
+          this.#logger.log("Checkpoint canceled, won't retry", { runId });
+          // Don't fail the checkpoint, as it was canceled
+          return result;
+        }
+
+        if (result.reason === "IN_PROGRESS") {
+          this.#logger.log("Checkpoint already in progress, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        if (result.reason === "NO_SUPPORT") {
+          this.#logger.log("No checkpoint support, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        if (result.reason === "DISABLED") {
+          this.#logger.log("Checkpoint support disabled, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        if (result.reason === "SKIP_RETRYING") {
+          this.#logger.log("Skipping retrying", { runId });
+          return result;
+        }
+
+        continue;
+      } catch (error) {
+        this.#logger.error("Checkpoint error", {
+          retry,
+          runId,
+          delay,
+          error: error instanceof Error ? error.message : error,
+        });
+      }
+    }
+
+    this.#logger.error(`Checkpoint failed after exponential backoff`, {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+    });
+    this.#failCheckpoint(runId, "ERROR");
+
+    return { success: false, reason: "ERROR" };
   }
 
   async #checkpointAndPush({
@@ -174,24 +351,32 @@ class Checkpointer {
     leaveRunning = true, // This mirrors kubernetes behaviour more accurately
     projectRef,
     deploymentVersion,
-  }: CheckpointAndPushOptions): Promise<CheckpointData | undefined> {
-    await this.initialize();
+  }: CheckpointAndPushOptions): Promise<CheckpointAndPushResult> {
+    await this.init();
+
+    const options = {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+    };
 
     if (!this.#dockerMode && !this.#canCheckpoint) {
       this.#logger.error("No checkpoint support. Simulation requires docker.");
-      return;
+      return { success: false, reason: "NO_SUPPORT" };
     }
 
     if (this.#abortControllers.has(runId)) {
-      logger.error("Checkpoint procedure already in progress", {
-        options: {
-          runId,
-          leaveRunning,
-          projectRef,
-          deploymentVersion,
-        },
-      });
-      return;
+      logger.error("Checkpoint procedure already in progress", { options });
+      return { success: false, reason: "IN_PROGRESS" };
+    }
+
+    // This is a new checkpoint, clear any last failure for this run
+    this.#clearFailedCheckpoint(runId);
+
+    if (DISABLE_CHECKPOINT_SUPPORT) {
+      this.#logger.error("Checkpoint support disabled", { options });
+      return { success: false, reason: "DISABLED" };
     }
 
     const controller = new AbortController();
@@ -199,19 +384,44 @@ class Checkpointer {
 
     const $$ = $({ signal: controller.signal });
 
-    try {
-      const shortCode = nanoid(8);
-      const imageRef = this.#getImageRef(projectRef, deploymentVersion, shortCode);
-      const exportLocation = this.#getExportLocation(projectRef, deploymentVersion, shortCode);
+    const shortCode = nanoid(8);
+    const imageRef = this.#getImageRef(projectRef, deploymentVersion, shortCode);
+    const exportLocation = this.#getExportLocation(projectRef, deploymentVersion, shortCode);
 
-      this.#logger.log("Checkpointing:", {
-        options: {
-          runId,
-          leaveRunning,
-          projectRef,
-          deploymentVersion,
-        },
-      });
+    const cleanup = async () => {
+      if (this.#dockerMode) {
+        return;
+      }
+
+      try {
+        await $`rm ${exportLocation}`;
+        this.#logger.log("Deleted checkpoint archive", { exportLocation });
+
+        await $`buildah rmi ${imageRef}`;
+        this.#logger.log("Deleted checkpoint image", { imageRef });
+      } catch (error) {
+        this.#logger.error("Failure during checkpoint cleanup", { exportLocation, error });
+      }
+    };
+
+    try {
+      if (CHAOS_MONKEY_ENABLED) {
+        console.log("🍌 Chaos monkey wreaking havoc");
+
+        const random = Math.random();
+
+        if (random < 0.33) {
+          // Fake long checkpoint duration
+          await $$`sleep 300`;
+        } else if (random < 0.66) {
+          // Fake checkpoint error
+          await $$`false`;
+        } else {
+          // no-op
+        }
+      }
+
+      this.#logger.log("Checkpointing:", { options });
 
       const containterName = this.#getRunContainerName(runId);
 
@@ -222,6 +432,13 @@ class Checkpointer {
             this.#logger.log("Simulating checkpoint");
             this.#logger.debug(await $$`docker pause ${containterName}`);
           } else {
+            if (SIMULATE_CHECKPOINT_FAILURE) {
+              if (performance.now() < SIMULATE_CHECKPOINT_FAILURE_SECONDS * 1000) {
+                this.#logger.error("Simulating checkpoint failure", { options });
+                throw new Error("SIMULATE_CHECKPOINT_FAILURE");
+              }
+            }
+
             if (leaveRunning) {
               this.#logger.debug(
                 await $$`docker checkpoint create --leave-running ${containterName} ${exportLocation}`
@@ -232,9 +449,9 @@ class Checkpointer {
               );
             }
           }
-        } catch (error: any) {
-          this.#logger.error(error.stderr);
-          return;
+        } catch (error) {
+          this.#logger.error("Failed while creating docker checkpoint", { exportLocation });
+          throw error;
         }
 
         this.#logger.log("checkpoint created:", {
@@ -243,14 +460,18 @@ class Checkpointer {
         });
 
         return {
-          location: exportLocation,
-          docker: true,
+          success: true,
+          checkpoint: {
+            location: exportLocation,
+            docker: true,
+          },
         };
       }
 
       // Create checkpoint (CRI)
       if (!this.#canCheckpoint) {
-        throw new Error("No checkpoint support in kubernetes mode.");
+        this.#logger.error("No checkpoint support in kubernetes mode.");
+        return { success: false, reason: "SKIP_RETRYING" };
       }
 
       const containerId = this.#logger.debug(
@@ -261,57 +482,109 @@ class Checkpointer {
       );
 
       if (!containerId.stdout) {
-        throw new Error("could not find container id");
+        this.#logger.error("could not find container id", { options, containterName });
+        return { success: false, reason: "SKIP_RETRYING" };
       }
 
+      const start = performance.now();
+
+      if (SIMULATE_CHECKPOINT_FAILURE) {
+        if (performance.now() < SIMULATE_CHECKPOINT_FAILURE_SECONDS * 1000) {
+          this.#logger.error("Simulating checkpoint failure", { options });
+          throw new Error("SIMULATE_CHECKPOINT_FAILURE");
+        }
+      }
+
+      // Create checkpoint
       this.#logger.debug(await $$`crictl checkpoint --export=${exportLocation} ${containerId}`);
+      const postCheckpoint = performance.now();
+
+      // Print checkpoint size
+      const size = await getParsedFileSize(exportLocation);
+      this.#logger.log("checkpoint archive created", { size, options });
 
       // Create image from checkpoint
       const container = this.#logger.debug(await $$`buildah from scratch`);
+      const postFrom = performance.now();
+
       this.#logger.debug(await $$`buildah add ${container} ${exportLocation} /`);
+      const postAdd = performance.now();
+
       this.#logger.debug(
         await $$`buildah config --annotation=io.kubernetes.cri-o.annotations.checkpoint.name=counter ${container}`
       );
+      const postConfig = performance.now();
+
       this.#logger.debug(await $$`buildah commit ${container} ${imageRef}`);
+      const postCommit = performance.now();
+
       this.#logger.debug(await $$`buildah rm ${container}`);
+      const postRm = performance.now();
+
+      if (SIMULATE_PUSH_FAILURE) {
+        if (performance.now() < SIMULATE_PUSH_FAILURE_SECONDS * 1000) {
+          this.#logger.error("Simulating push failure", { options });
+          throw new Error("SIMULATE_PUSH_FAILURE");
+        }
+      }
 
       // Push checkpoint image
       this.#logger.debug(await $$`buildah push --tls-verify=${REGISTRY_TLS_VERIFY} ${imageRef}`);
+      const postPush = performance.now();
 
-      this.#logger.log("Checkpointed and pushed image to:", { location: imageRef });
+      const perf = {
+        "crictl checkpoint": postCheckpoint - start,
+        "buildah from": postFrom - postCheckpoint,
+        "buildah add": postAdd - postFrom,
+        "buildah config": postConfig - postAdd,
+        "buildah commit": postCommit - postConfig,
+        "buildah rm": postRm - postCommit,
+        "buildah push": postPush - postRm,
+      };
 
-      try {
-        await $$`rm ${exportLocation}`;
-        this.#logger.log("Deleted checkpoint archive", { exportLocation });
-
-        // Disabled for now as this will increase restore time by having to pull the image again
-        // await $`buildah rmi ${imageRef}`;
-        // this.#logger.log("Deleted checkpoint image", { imageRef });
-      } catch (error) {
-        this.#logger.error("Failed during checkpoint cleanup", { exportLocation });
-        this.#logger.debug(error);
-      }
+      this.#logger.log("Checkpointed and pushed image to:", { location: imageRef, perf });
 
       return {
-        location: imageRef,
-        docker: false,
+        success: true,
+        checkpoint: {
+          location: imageRef,
+          docker: false,
+        },
       };
     } catch (error) {
-      this.#logger.error("checkpoint failed", {
-        options: {
-          runId,
-          leaveRunning,
-          projectRef,
-          deploymentVersion,
-        },
-        error,
-      });
-      return;
+      if (isExecaChildProcess(error)) {
+        if (error.isCanceled) {
+          this.#logger.error("Checkpoint canceled", { options, error });
+
+          return { success: false, reason: "CANCELED" };
+        }
+
+        this.#logger.error("Checkpoint command error", { options, error });
+
+        return { success: false, reason: "ERROR" };
+      }
+
+      this.#logger.error("Unhandled checkpoint error", { options, error });
+
+      return { success: false, reason: "ERROR" };
     } finally {
       this.#abortControllers.delete(runId);
+      await cleanup();
     }
   }
 
+  #failCheckpoint(runId: string, error: unknown) {
+    this.#failedCheckpoints.set(runId, error);
+  }
+
+  #clearFailedCheckpoint(runId: string) {
+    this.#failedCheckpoints.delete(runId);
+  }
+
+  #hasFailedCheckpoint(runId: string) {
+    return this.#failedCheckpoints.has(runId);
+  }
+
   #getRunContainerName(suffix: string) {
     return `task-run-${suffix}`;
   }
@@ -319,7 +592,7 @@ class Checkpointer {
 
 class TaskCoordinator {
   #httpServer: ReturnType<typeof createServer>;
-  #checkpointer = new Checkpointer({ forceSimulate: true });
+  #checkpointer = new Checkpointer({ forceSimulate: FORCE_CHECKPOINT_SIMULATION });
 
   #prodWorkerNamespace: ZodNamespace<
     typeof ProdWorkerToCoordinatorMessages,
@@ -343,7 +616,7 @@ class TaskCoordinator {
     private host = "0.0.0.0"
   ) {
     this.#httpServer = this.#createHttpServer();
-    this.#checkpointer.initialize();
+    this.#checkpointer.init();
     this.#delayThresholdInMs = this.#getDelayThreshold();
 
     if (process.env.DELAY_THRESHOLD_IN_MS) {
@@ -440,6 +713,30 @@ class TaskCoordinator {
 
           taskSocket.emit("REQUEST_ATTEMPT_CANCELLATION", message);
         },
+        REQUEST_RUN_CANCELLATION: async (message) => {
+          const taskSocket = await this.#getRunSocket(message.runId);
+
+          if (!taskSocket) {
+            logger.log("Socket for run not found", {
+              runId: message.runId,
+            });
+            return;
+          }
+
+          this.#checkpointer.cancelCheckpoint(message.runId);
+
+          if (message.delayInMs) {
+            taskSocket.emit("REQUEST_EXIT", {
+              version: "v2",
+              delayInMs: message.delayInMs,
+            });
+          } else {
+            // If there's no delay, assume the worker doesn't support non-v1 messages
+            taskSocket.emit("REQUEST_EXIT", {
+              version: "v1",
+            });
+          }
+        },
         READY_FOR_RETRY: async (message) => {
           const taskSocket = await this.#getRunSocket(message.runId);
 
@@ -526,6 +823,20 @@ class TaskCoordinator {
       onConnection: async (socket, handler, sender) => {
         const logger = new SimpleLogger(`[prod-worker][${socket.id}]`);
 
+        const crashRun = async (error: { name: string; message: string; stack?: string }) => {
+          try {
+            this.#platformSocket?.send("RUN_CRASHED", {
+              version: "v1",
+              runId: socket.data.runId,
+              error,
+            });
+          } finally {
+            socket.emit("REQUEST_EXIT", {
+              version: "v1",
+            });
+          }
+        };
+
         const checkpointInProgress = () => {
           return this.#checkpointableTasks.has(socket.data.runId);
         };
@@ -594,8 +905,9 @@ class TaskCoordinator {
             if (!executionAck) {
               logger.error("no execution ack", { runId: socket.data.runId });
 
-              socket.emit("REQUEST_EXIT", {
-                version: "v1",
+              await crashRun({
+                name: "ReadyForExecutionError",
+                message: "No execution ack",
               });
 
               return;
@@ -604,8 +916,9 @@ class TaskCoordinator {
             if (!executionAck.success) {
               logger.error("failed to get execution payload", { runId: socket.data.runId });
 
-              socket.emit("REQUEST_EXIT", {
-                version: "v1",
+              await crashRun({
+                name: "ReadyForExecutionError",
+                message: "Failed to get execution payload",
               });
 
               return;
@@ -619,6 +932,62 @@ class TaskCoordinator {
             socket.data.attemptFriendlyId = executionAck.payload.execution.attempt.id;
           } catch (error) {
             logger.error("Error", { error });
+
+            await crashRun({
+              name: "ReadyForExecutionError",
+              message:
+                error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error",
+            });
+
+            return;
+          }
+        });
+
+        socket.on("READY_FOR_LAZY_ATTEMPT", async (message) => {
+          logger.log("[READY_FOR_LAZY_ATTEMPT]", message);
+
+          try {
+            const lazyAttempt = await this.#platformSocket?.sendWithAck("READY_FOR_LAZY_ATTEMPT", {
+              ...message,
+              envId: socket.data.envId,
+            });
+
+            if (!lazyAttempt) {
+              logger.error("no lazy attempt ack", { runId: socket.data.runId });
+
+              await crashRun({
+                name: "ReadyForLazyAttemptError",
+                message: "No lazy attempt ack",
+              });
+
+              return;
+            }
+
+            if (!lazyAttempt.success) {
+              logger.error("failed to get lazy attempt payload", { runId: socket.data.runId });
+
+              await crashRun({
+                name: "ReadyForLazyAttemptError",
+                message: "Failed to get lazy attempt payload",
+              });
+
+              return;
+            }
+
+            socket.emit("EXECUTE_TASK_RUN_LAZY_ATTEMPT", {
+              version: "v1",
+              lazyPayload: lazyAttempt.lazyPayload,
+            });
+          } catch (error) {
+            logger.error("Error", { error });
+
+            await crashRun({
+              name: "ReadyForLazyAttemptError",
+              message:
+                error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error",
+            });
+
+            return;
           }
         });
 
@@ -664,7 +1033,7 @@ class TaskCoordinator {
             return;
           }
 
-          const { canCheckpoint, willSimulate } = await this.#checkpointer.initialize();
+          const { canCheckpoint, willSimulate } = await this.#checkpointer.init();
 
           const willCheckpointAndRestore = canCheckpoint || willSimulate;
 
@@ -712,6 +1081,19 @@ class TaskCoordinator {
           }
         });
 
+        socket.on("TASK_RUN_FAILED_TO_RUN", async ({ completion }) => {
+          logger.log("completed task", { completionId: completion.id });
+
+          this.#platformSocket?.send("TASK_RUN_FAILED_TO_RUN", {
+            version: "v1",
+            completion,
+          });
+
+          socket.emit("REQUEST_EXIT", {
+            version: "v1",
+          });
+        });
+
         socket.on("READY_FOR_CHECKPOINT", async (message) => {
           logger.log("[READY_FOR_CHECKPOINT]", message);
 
@@ -725,10 +1107,18 @@ class TaskCoordinator {
           checkpointable.resolve();
         });
 
-        socket.on("CANCEL_CHECKPOINT", async (message) => {
+        socket.on("CANCEL_CHECKPOINT", async (message, callback) => {
           logger.log("[CANCEL_CHECKPOINT]", message);
 
-          this.#cancelCheckpoint(socket.data.runId);
+          if (message.version === "v1") {
+            this.#cancelCheckpoint(socket.data.runId);
+            // v1 has no callback
+            return;
+          }
+
+          const checkpointCanceled = this.#cancelCheckpoint(socket.data.runId);
+
+          callback({ version: "v2", checkpointCanceled });
         });
 
         socket.on("WAIT_FOR_DURATION", async (message, callback) => {
@@ -740,7 +1130,7 @@ class TaskCoordinator {
             return;
           }
 
-          const { canCheckpoint, willSimulate } = await this.#checkpointer.initialize();
+          const { canCheckpoint, willSimulate } = await this.#checkpointer.init();
 
           const willCheckpointAndRestore = canCheckpoint || willSimulate;
 
@@ -794,7 +1184,7 @@ class TaskCoordinator {
         socket.on("WAIT_FOR_TASK", async (message, callback) => {
           logger.log("[WAIT_FOR_TASK]", message);
 
-          const { canCheckpoint, willSimulate } = await this.#checkpointer.initialize();
+          const { canCheckpoint, willSimulate } = await this.#checkpointer.init();
 
           const willCheckpointAndRestore = canCheckpoint || willSimulate;
 
@@ -836,7 +1226,7 @@ class TaskCoordinator {
         socket.on("WAIT_FOR_BATCH", async (message, callback) => {
           logger.log("[WAIT_FOR_BATCH]", message);
 
-          const { canCheckpoint, willSimulate } = await this.#checkpointer.initialize();
+          const { canCheckpoint, willSimulate } = await this.#checkpointer.init();
 
           const willCheckpointAndRestore = canCheckpoint || willSimulate;
 
@@ -880,7 +1270,7 @@ class TaskCoordinator {
           logger.log("[INDEX_TASKS]", message);
 
           const workerAck = await this.#platformSocket?.sendWithAck("CREATE_WORKER", {
-            version: "v1",
+            version: "v2",
             projectRef: socket.data.projectRef,
             envId: socket.data.envId,
             deploymentId: message.deploymentId,
@@ -889,6 +1279,7 @@ class TaskCoordinator {
               packageVersion: message.packageVersion,
               tasks: message.tasks,
             },
+            supportsLazyAttempts: message.version !== "v1" && message.supportsLazyAttempts,
           });
 
           if (!workerAck) {
@@ -907,6 +1298,34 @@ class TaskCoordinator {
             error: message.error,
           });
         });
+
+        socket.on("CREATE_TASK_RUN_ATTEMPT", async (message, callback) => {
+          logger.log("[CREATE_TASK_RUN_ATTEMPT]", message);
+
+          const createAttempt = await this.#platformSocket?.sendWithAck("CREATE_TASK_RUN_ATTEMPT", {
+            runId: message.runId,
+            envId: socket.data.envId,
+          });
+
+          if (!createAttempt?.success) {
+            logger.debug("no ack while creating attempt", message);
+            callback({ success: false });
+            return;
+          }
+
+          socket.data.attemptFriendlyId = createAttempt.executionPayload.execution.attempt.id;
+
+          callback({
+            success: true,
+            executionPayload: createAttempt.executionPayload,
+          });
+        });
+
+        socket.on("UNRECOVERABLE_ERROR", async (message) => {
+          logger.log("[UNRECOVERABLE_ERROR]", message);
+
+          await crashRun(message.error);
+        });
       },
       onDisconnect: async (socket, handler, sender, logger) => {
         this.#platformSocket?.send("LOG", {
@@ -918,13 +1337,16 @@ class TaskCoordinator {
         TASK_HEARTBEAT: async (message) => {
           this.#platformSocket?.send("TASK_HEARTBEAT", message);
         },
+        TASK_RUN_HEARTBEAT: async (message) => {
+          this.#platformSocket?.send("TASK_RUN_HEARTBEAT", message);
+        },
       },
     });
 
     return provider;
   }
 
-  #cancelCheckpoint(runId: string) {
+  #cancelCheckpoint(runId: string): boolean {
     const checkpointWait = this.#checkpointableTasks.get(runId);
 
     if (checkpointWait) {
@@ -933,7 +1355,11 @@ class TaskCoordinator {
     }
 
     // Cancel checkpointing procedure
-    this.#checkpointer.cancelCheckpoint(runId);
+    const checkpointCanceled = this.#checkpointer.cancelCheckpoint(runId);
+
+    logger.log("cancelCheckpoint()", { runId, checkpointCanceled });
+
+    return checkpointCanceled;
   }
 
   #createHttpServer() {

apps/docker-provider/.env.example

@@ -4,6 +4,8 @@ PLATFORM_WS_PORT=3030
 PLATFORM_SECRET=provider-secret
 SECURE_CONNECTION=false
 
+OTEL_EXPORTER_OTLP_ENDPOINT=http://0.0.0.0:3030/otel
+
 # Use this if you are on macOS
 # COORDINATOR_HOST="host.docker.internal"
 # OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318"

apps/docker-provider/Containerfile

@@ -1,16 +1,47 @@
-# syntax=docker/dockerfile:labs
-
-FROM node:18-slim AS base
-
-RUN apt-get update \
-  && apt-get install -y dumb-init
-
-FROM base
+FROM node:20-alpine@sha256:7a91aa397f2e2dfbfcdad2e2d72599f374e0b0172be1d86eeb73f1d33f36a4b2 AS node-20-alpine
 
 WORKDIR /app
 
-COPY --chown=node dist/index.mjs /app/
+FROM node-20-alpine AS pruner
+
+COPY --chown=node:node . .
+RUN npx -q turbo@1.10.9 prune --scope=docker-provider --docker
+RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
+
+FROM node-20-alpine AS base
+
+RUN apk add --no-cache dumb-init docker
+
+COPY --chown=node:node .gitignore .gitignore
+COPY --from=pruner --chown=node:node /app/out/json/ .
+COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
+COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
+
+FROM base AS dev-deps
+RUN corepack enable
+ENV NODE_ENV development
+
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
+
+FROM base AS builder
+RUN corepack enable
+
+COPY --from=pruner --chown=node:node /app/out/full/ .
+COPY --from=dev-deps --chown=node:node /app/ .
+COPY --chown=node:node turbo.json turbo.json
+
+RUN pnpm run -r --filter docker-provider build:bundle
+
+FROM base AS runner
+
+RUN corepack enable
+ENV NODE_ENV production
+
+COPY --from=builder --chown=node:node /app/apps/docker-provider/dist/index.mjs ./index.mjs
 
 EXPOSE 8000
 
-ENTRYPOINT [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "/app/index.mjs" ]
+USER node
+
+CMD [ "/usr/bin/dumb-init", "--", "/usr/local/bin/node", "./index.mjs" ]

apps/docker-provider/src/index.ts

@@ -6,6 +6,8 @@ import {
   TaskOperationsRestoreOptions,
   TaskOperationsCreateOptions,
   TaskOperationsIndexOptions,
+  isExecaChildProcess,
+  testDockerCheckpoint,
 } from "@trigger.dev/core-apps";
 import { setTimeout } from "node:timers/promises";
 import { PostStartCauses, PreStopCauses } from "@trigger.dev/core/v3";
@@ -13,75 +15,68 @@ import { PostStartCauses, PreStopCauses } from "@trigger.dev/core/v3";
 const MACHINE_NAME = process.env.MACHINE_NAME || "local";
 const COORDINATOR_PORT = process.env.COORDINATOR_PORT || 8020;
 const COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
+
 const OTEL_EXPORTER_OTLP_ENDPOINT =
   process.env.OTEL_EXPORTER_OTLP_ENDPOINT || "http://0.0.0.0:4318";
 
+const FORCE_CHECKPOINT_SIMULATION = ["1", "true"].includes(
+  process.env.FORCE_CHECKPOINT_SIMULATION ?? "true"
+);
+
 const logger = new SimpleLogger(`[${MACHINE_NAME}]`);
 
-type InitializeReturn = {
+type TaskOperationsInitReturn = {
   canCheckpoint: boolean;
   willSimulate: boolean;
 };
 
-function isExecaChildProcess(maybeExeca: unknown): maybeExeca is Awaited<ExecaChildProcess> {
-  return typeof maybeExeca === "object" && maybeExeca !== null && "escapedCommand" in maybeExeca;
-}
-
 class DockerTaskOperations implements TaskOperations {
   #initialized = false;
   #canCheckpoint = false;
 
   constructor(private opts = { forceSimulate: false }) {}
 
-  async #initialize(): Promise<InitializeReturn> {
+  async init(): Promise<TaskOperationsInitReturn> {
     if (this.#initialized) {
-      return this.#getInitializeReturn();
+      return this.#getInitReturn(this.#canCheckpoint);
     }
 
     logger.log("Initializing task operations");
 
-    if (this.opts.forceSimulate) {
-      logger.log("Forced simulation enabled. Will simulate regardless of checkpoint support.");
+    const testCheckpoint = await testDockerCheckpoint();
+
+    if (testCheckpoint.ok) {
+      return this.#getInitReturn(true);
     }
 
-    try {
-      await $`criu --version`;
-    } catch (error) {
-      logger.error("No checkpoint support: Missing CRIU binary. Will simulate instead.");
-      this.#canCheckpoint = false;
-      this.#initialized = true;
-
-      return this.#getInitializeReturn();
-    }
-
-    try {
-      await $`docker checkpoint`;
-    } catch (error) {
-      logger.error("No checkpoint support: Docker needs to have experimental features enabled");
-      logger.error("Will simulate instead");
-      this.#canCheckpoint = false;
-      this.#initialized = true;
-
-      return this.#getInitializeReturn();
-    }
-
-    logger.log("Full checkpoint support!");
-
-    this.#initialized = true;
-    this.#canCheckpoint = true;
-
-    return this.#getInitializeReturn();
+    logger.error(testCheckpoint.message, testCheckpoint.error);
+    return this.#getInitReturn(false);
   }
 
-  #getInitializeReturn(): InitializeReturn {
+  #getInitReturn(canCheckpoint: boolean): TaskOperationsInitReturn {
+    this.#initialized = true;
+    this.#canCheckpoint = canCheckpoint;
+
+    if (canCheckpoint) {
+      logger.log("Full checkpoint support!");
+    }
+
+    const willSimulate = !canCheckpoint || this.opts.forceSimulate;
+
+    if (willSimulate) {
+      logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", {
+        forceSimulate: this.opts.forceSimulate,
+      });
+    }
+
     return {
-      canCheckpoint: this.#canCheckpoint,
-      willSimulate: !this.#canCheckpoint || this.opts.forceSimulate,
+      canCheckpoint,
+      willSimulate,
     };
   }
 
   async index(opts: TaskOperationsIndexOptions) {
-    await this.#initialize();
+    await this.init();
 
     const containerName = this.#getIndexContainerName(opts.shortCode);
 
@@ -90,41 +85,27 @@ class DockerTaskOperations implements TaskOperations {
       port: COORDINATOR_PORT,
     });
 
-    try {
-      logger.debug(
-        await execa("docker", [
-          "run",
-          "--network=host",
-          "--rm",
-          `--env=INDEX_TASKS=true`,
-          `--env=TRIGGER_SECRET_KEY=${opts.apiKey}`,
-          `--env=TRIGGER_API_URL=${opts.apiUrl}`,
-          `--env=TRIGGER_ENV_ID=${opts.envId}`,
-          `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
-          `--env=POD_NAME=${containerName}`,
-          `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
-          `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
-          `--name=${containerName}`,
-          `${opts.imageRef}`,
-        ])
-      );
-    } catch (error: any) {
-      if (!isExecaChildProcess(error)) {
-        throw error;
-      }
-
-      logger.error("Index failed:", {
-        opts,
-        exitCode: error.exitCode,
-        escapedCommand: error.escapedCommand,
-        stdout: error.stdout,
-        stderr: error.stderr,
-      });
-    }
+    logger.debug(
+      await execa("docker", [
+        "run",
+        "--network=host",
+        "--rm",
+        `--env=INDEX_TASKS=true`,
+        `--env=TRIGGER_SECRET_KEY=${opts.apiKey}`,
+        `--env=TRIGGER_API_URL=${opts.apiUrl}`,
+        `--env=TRIGGER_ENV_ID=${opts.envId}`,
+        `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
+        `--env=POD_NAME=${containerName}`,
+        `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
+        `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
+        `--name=${containerName}`,
+        `${opts.imageRef}`,
+      ])
+    );
   }
 
   async create(opts: TaskOperationsCreateOptions) {
-    await this.#initialize();
+    await this.init();
 
     const containerName = this.#getRunContainerName(opts.runId);
 
@@ -160,7 +141,7 @@ class DockerTaskOperations implements TaskOperations {
   }
 
   async restore(opts: TaskOperationsRestoreOptions) {
-    await this.#initialize();
+    await this.init();
 
     const containerName = this.#getRunContainerName(opts.runId);
 
@@ -189,7 +170,7 @@ class DockerTaskOperations implements TaskOperations {
   }
 
   async delete(opts: { runId: string }) {
-    await this.#initialize();
+    await this.init();
 
     const containerName = this.#getRunContainerName(opts.runId);
     await this.#sendPreStop(containerName);
@@ -198,7 +179,7 @@ class DockerTaskOperations implements TaskOperations {
   }
 
   async get(opts: { runId: string }) {
-    await this.#initialize();
+    await this.init();
 
     logger.log("noop: get");
   }
@@ -278,7 +259,7 @@ class DockerTaskOperations implements TaskOperations {
 }
 
 const provider = new ProviderShell({
-  tasks: new DockerTaskOperations({ forceSimulate: true }),
+  tasks: new DockerTaskOperations({ forceSimulate: FORCE_CHECKPOINT_SIMULATION }),
   type: "docker",
 });
 

apps/kubernetes-provider/Containerfile

@@ -1,14 +1,14 @@
-FROM node:18-alpine@sha256:ca9f6cb0466f9638e59e0c249d335a07c867cd50c429b5c7830dda1bed584649 AS node-18-alpine
+FROM node:20-alpine@sha256:7a91aa397f2e2dfbfcdad2e2d72599f374e0b0172be1d86eeb73f1d33f36a4b2 AS node-20-alpine
 
 WORKDIR /app
 
-FROM node-18-alpine AS pruner
+FROM node-20-alpine AS pruner
 
 COPY --chown=node:node . .
 RUN npx -q turbo@1.10.9 prune --scope=kubernetes-provider --docker
 RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
 
-FROM node-18-alpine AS base
+FROM node-20-alpine AS base
 
 RUN apk add --no-cache dumb-init
 

apps/kubernetes-provider/src/index.ts

@@ -10,6 +10,7 @@ import {
 import { Machine, PostStartCauses, PreStopCauses, EnvironmentType } from "@trigger.dev/core/v3";
 import { randomUUID } from "crypto";
 import { TaskMonitor } from "./taskMonitor";
+import { PodCleaner } from "./podCleaner";
 
 const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
 const NODE_NAME = process.env.NODE_NAME || "local";
@@ -46,6 +47,10 @@ class KubernetesTaskOperations implements TaskOperations {
     this.#k8sApi = this.#createK8sApi();
   }
 
+  async init() {
+    // noop
+  }
+
   async index(opts: TaskOperationsIndexOptions) {
     await this.#createJob(
       {
@@ -80,8 +85,8 @@ class KubernetesTaskOperations implements TaskOperations {
                   ],
                   resources: {
                     limits: {
-                      cpu: "250m",
-                      memory: "0.5G",
+                      cpu: "1",
+                      memory: "1G",
                       "ephemeral-storage": "2Gi",
                     },
                   },
@@ -211,7 +216,8 @@ class KubernetesTaskOperations implements TaskOperations {
             },
             {
               name: "populate-taskinfo",
-              image: "docker.io/library/busybox",
+              image: "registry.digitalocean.com/trigger/busybox",
+              imagePullPolicy: "IfNotPresent",
               command: ["/bin/sh", "-c"],
               args: ["printenv COORDINATOR_HOST | tee /etc/taskinfo/coordinator-host"],
               env: [
@@ -314,6 +320,9 @@ class KubernetesTaskOperations implements TaskOperations {
         {
           name: "registry-trigger",
         },
+        {
+          name: "registry-trigger-failover",
+        },
       ],
       nodeSelector: {
         nodetype: "worker",
@@ -542,3 +551,11 @@ const taskMonitor = new TaskMonitor({
 });
 
 taskMonitor.start();
+
+const podCleaner = new PodCleaner({
+  runtimeEnv: RUNTIME_ENV,
+  namespace: "default",
+  intervalInSeconds: 300,
+});
+
+podCleaner.start();

apps/kubernetes-provider/src/podCleaner.ts

@@ -0,0 +1,264 @@
+import * as k8s from "@kubernetes/client-node";
+import { SimpleLogger } from "@trigger.dev/core-apps";
+
+type PodCleanerOptions = {
+  runtimeEnv: "local" | "kubernetes";
+  namespace?: string;
+  intervalInSeconds?: number;
+};
+
+export class PodCleaner {
+  private enabled = false;
+  private namespace = "default";
+  private intervalInSeconds = 300;
+
+  private logger = new SimpleLogger("[PodCleaner]");
+  private k8sClient: {
+    core: k8s.CoreV1Api;
+    kubeConfig: k8s.KubeConfig;
+  };
+
+  constructor(private opts: PodCleanerOptions) {
+    if (opts.namespace) {
+      this.namespace = opts.namespace;
+    }
+
+    if (opts.intervalInSeconds) {
+      this.intervalInSeconds = opts.intervalInSeconds;
+    }
+
+    this.k8sClient = this.#createK8sClient();
+  }
+
+  #createK8sClient() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (this.opts.runtimeEnv === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (this.opts.runtimeEnv === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      kubeConfig: kubeConfig,
+    };
+  }
+
+  #isRecord(candidate: unknown): candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  #logK8sError(err: unknown, debugOnly = false) {
+    if (debugOnly) {
+      this.logger.debug("K8s API Error", err);
+    } else {
+      this.logger.error("K8s API Error", err);
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
+      this.#logK8sError(err);
+      return;
+    }
+
+    this.#logK8sError(err, true);
+
+    if (typeof err.body.message === "string") {
+      this.#logK8sError({ message: err.body.message });
+      return;
+    }
+
+    this.#logK8sError({ body: err.body });
+  }
+
+  async #deletePods(opts: {
+    namespace: string;
+    dryRun?: boolean;
+    fieldSelector?: string;
+    labelSelector?: string;
+  }) {
+    return await this.k8sClient.core
+      .deleteCollectionNamespacedPod(
+        opts.namespace,
+        undefined, // pretty
+        undefined, // continue
+        opts.dryRun ? "All" : undefined,
+        opts.fieldSelector,
+        undefined, // gracePeriodSeconds
+        opts.labelSelector
+      )
+      .catch(this.#handleK8sError.bind(this));
+  }
+
+  async #deleteCompletedRuns() {
+    this.logger.log("Deleting completed runs");
+
+    const start = Date.now();
+
+    const result = await this.#deletePods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=Succeeded",
+      labelSelector: "app=task-run",
+    });
+
+    const elapsedMs = Date.now() - start;
+
+    if (!result) {
+      this.logger.log("Deleting completed runs: No delete result", { elapsedMs });
+      return;
+    }
+
+    const total = (result.response as any)?.body?.items?.length ?? 0;
+
+    this.logger.log("Deleting completed runs: Done", { total, elapsedMs });
+  }
+
+  async #deleteFailedRuns() {
+    this.logger.log("Deleting failed runs");
+
+    const start = Date.now();
+
+    const result = await this.#deletePods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=Failed",
+      labelSelector: "app=task-run",
+    });
+
+    const elapsedMs = Date.now() - start;
+
+    if (!result) {
+      this.logger.log("Deleting failed runs: No delete result", { elapsedMs });
+      return;
+    }
+
+    const total = (result.response as any)?.body?.items?.length ?? 0;
+
+    this.logger.log("Deleting failed runs: Done", { total, elapsedMs });
+  }
+
+  async #deleteUnrecoverableRuns() {
+    await this.#deletePods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=?",
+      labelSelector: "app=task-run",
+    });
+  }
+
+  async start() {
+    this.enabled = true;
+    this.logger.log("Starting");
+
+    const completedInterval = setInterval(async () => {
+      if (!this.enabled) {
+        clearInterval(completedInterval);
+        return;
+      }
+
+      try {
+        await this.#deleteCompletedRuns();
+      } catch (error) {
+        this.logger.error("Error deleting completed runs", error);
+      }
+    }, this.intervalInSeconds * 1000);
+
+    const failedInterval = setInterval(
+      async () => {
+        if (!this.enabled) {
+          clearInterval(failedInterval);
+          return;
+        }
+
+        try {
+          await this.#deleteFailedRuns();
+        } catch (error) {
+          this.logger.error("Error deleting completed runs", error);
+        }
+      },
+      // Use a longer interval for failed runs. This is only a backup in case the task monitor fails.
+      2 * this.intervalInSeconds * 1000
+    );
+
+    // this.#launchTests();
+  }
+
+  async stop() {
+    if (!this.enabled) {
+      return;
+    }
+
+    this.enabled = false;
+    this.logger.log("Shutting down..");
+  }
+
+  async #launchTests() {
+    const createPod = async (
+      container: k8s.V1Container,
+      name: string,
+      labels?: Record<string, string>
+    ) => {
+      this.logger.log("Creating pod:", name);
+
+      const pod = {
+        metadata: {
+          name,
+          labels,
+        },
+        spec: {
+          restartPolicy: "Never",
+          automountServiceAccountToken: false,
+          terminationGracePeriodSeconds: 1,
+          containers: [container],
+        },
+      } satisfies k8s.V1Pod;
+
+      await this.k8sClient.core
+        .createNamespacedPod(this.namespace, pod)
+        .catch(this.#handleK8sError.bind(this));
+    };
+
+    const createIdlePod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: ["-c", "sleep infinity"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createCompletedPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: ["-c", "true"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createFailedPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: ["-c", "false"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    await createIdlePod("test-idle-1", { app: "task-run" });
+    await createFailedPod("test-failed-1", { app: "task-run" });
+    await createCompletedPod("test-completed-1", { app: "task-run" });
+  }
+}

apps/kubernetes-provider/src/taskMonitor.ts

@@ -30,10 +30,12 @@ type TaskMonitorOptions = {
 
 export class TaskMonitor {
   #enabled = false;
+
   #logger = new SimpleLogger("[TaskMonitor]");
   #taskInformer: ReturnType<typeof k8s.makeInformer<k8s.V1Pod>>;
   #processedPods = new Map<string, number>();
   #queue = new PQueue({ concurrency: 10 });
+
   #k8sClient: {
     core: k8s.CoreV1Api;
     kubeConfig: k8s.KubeConfig;
@@ -44,6 +46,10 @@ export class TaskMonitor {
   private labelSelector = "app in (task-index, task-run)";
 
   constructor(private opts: TaskMonitorOptions) {
+    if (opts.namespace) {
+      this.namespace = opts.namespace;
+    }
+
     this.#k8sClient = this.#createK8sClient();
 
     this.#taskInformer = this.#createTaskInformer();

apps/proxy/package.json

@@ -7,9 +7,9 @@
     "dev": "wrangler dev"
   },
   "devDependencies": {
-    "@cloudflare/workers-types": "^4.20230419.0",
+    "@cloudflare/workers-types": "^4.20240512.0",
     "typescript": "^5.0.4",
-    "wrangler": "^3.0.0"
+    "wrangler": "^3.57.1"
   },
   "dependencies": {
     "@aws-sdk/client-sqs": "^3.445.0",

apps/proxy/src/index.ts

@@ -1,5 +1,7 @@
 import { queueEvent } from "./events/queueEvent";
 import { queueEvents } from "./events/queueEvents";
+import { applyRateLimit } from "./rateLimit";
+import { Ratelimit } from "./rateLimiter";
 
 export interface Env {
   /** The hostname needs to be changed to allow requests to pass to the Trigger.dev platform */
@@ -9,6 +11,8 @@ export interface Env {
   AWS_SQS_SECRET_ACCESS_KEY: string;
   AWS_SQS_QUEUE_URL: string;
   AWS_SQS_REGION: string;
+  //rate limiter
+  API_RATE_LIMITER: Ratelimit;
 }
 
 export default {
@@ -25,13 +29,13 @@ export default {
     switch (url.pathname) {
       case "/api/v1/events": {
         if (request.method === "POST") {
-          return queueEvent(request, env);
+          return applyRateLimit(request, env, () => queueEvent(request, env));
         }
         break;
       }
       case "/api/v1/events/bulk": {
         if (request.method === "POST") {
-          return queueEvents(request, env);
+          return applyRateLimit(request, env, () => queueEvents(request, env));
         }
         break;
       }

apps/proxy/src/rateLimit.ts

@@ -0,0 +1,46 @@
+import { Env } from "src";
+import { getApiKeyFromRequest } from "./apikey";
+import { json } from "./json";
+
+export async function applyRateLimit(
+  request: Request,
+  env: Env,
+  fn: () => Promise<Response>
+): Promise<Response> {
+  const apiKey = getApiKeyFromRequest(request);
+  if (apiKey) {
+    const result = await env.API_RATE_LIMITER.limit({ key: `apikey-${apiKey.apiKey}` });
+    const { success } = result;
+    console.log(`Rate limiter`, {
+      success,
+      key: `${apiKey.apiKey.substring(0, 12)}...`,
+    });
+    if (!success) {
+      //60s in the future
+      const reset = Date.now() + 60 * 1000;
+      const secondsUntilReset = Math.max(0, (reset - new Date().getTime()) / 1000);
+
+      return json(
+        {
+          title: "Rate Limit Exceeded",
+          status: 429,
+          type: "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429",
+          detail: `Rate limit exceeded. Retry in ${secondsUntilReset} seconds.`,
+          error: `Rate limit exceeded. Retry in ${secondsUntilReset} seconds.`,
+          reset,
+        },
+        {
+          status: 429,
+          headers: {
+            "x-ratelimit-reset": reset.toString(),
+          },
+        }
+      );
+    }
+  } else {
+    console.log(`Rate limiter: no API key for request`);
+  }
+
+  //call the original function
+  return fn();
+}

apps/proxy/src/rateLimiter.ts

@@ -0,0 +1,23 @@
+export interface Ratelimit {
+  /*
+   * The ratelimit function
+   * @param {RatelimitOptions} options
+   * @returns {Promise<RatelimitResponse>}
+   */
+  limit: (options: RatelimitOptions) => Promise<RatelimitResponse>;
+}
+
+export interface RatelimitOptions {
+  /*
+   * The key to identify the user, can be an IP address, user ID, etc.
+   */
+  key: string;
+}
+
+export interface RatelimitResponse {
+  /*
+   * The ratelimit success status
+   * @returns {boolean}
+   */
+  success: boolean;
+}

apps/proxy/wrangler.toml

@@ -1,7 +1,33 @@
 name = "proxy"
 main = "src/index.ts"
-compatibility_date = "2023-10-30"
+compatibility_date = "2024-05-13"
 compatibility_flags = [ "nodejs_compat" ]
 
 [env.staging]
-[env.prod]
+  # The rate limiting API is in open beta.
+  [[env.staging.unsafe.bindings]]
+  name = "API_RATE_LIMITER"
+  type = "ratelimit"
+  # An identifier you define, that is unique to your Cloudflare account.
+  # Must be an integer.
+  namespace_id = "1"
+
+  # Limit: the number of tokens allowed within a given period in a single
+  # Cloudflare location
+  # Period: the duration of the period, in seconds. Must be either 10 or 60
+  simple = { limit = 100, period = 60 }
+
+
+[env.prod]
+  # The rate limiting API is in open beta.
+  [[env.prod.unsafe.bindings]]
+  name = "API_RATE_LIMITER"
+  type = "ratelimit"
+  # An identifier you define, that is unique to your Cloudflare account.
+  # Must be an integer.
+  namespace_id = "2"
+
+  # Limit: the number of tokens allowed within a given period in a single
+  # Cloudflare location
+  # Period: the duration of the period, in seconds. Must be either 10 or 60
+  simple = { limit = 300, period = 60 }

apps/webapp/.gitignore

@@ -17,4 +17,5 @@ build-storybook.log
 .storybook-out
 storybook-static
 
-/prisma/seed.js
+/prisma/seed.js
+/prisma/populate.js

apps/webapp/app/assets/icons/v3.svg

@@ -0,0 +1,4 @@
+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect x="0.861435" y="0.861435" width="30.2771" height="30.2771" rx="15.1386" stroke="#D7D9DD" stroke-width="1.72287"/>
+<path d="M14.9669 12.642L11.6417 21.7732H9.8155L6.49036 12.642H8.04094L10.7286 20.2571L13.4163 12.642H14.9669ZM20.534 14.5544C21.6884 14.5716 23.0666 14.0375 23.0666 12.4525C23.0666 11.3843 22.1018 10.7124 20.534 10.7124C19.2591 10.7124 18.3632 11.3326 18.2082 12.3146L16.6576 12.2113C16.8815 10.4884 18.4838 9.26516 20.534 9.26516C22.946 9.26516 24.6345 10.5745 24.6345 12.4525C24.6345 13.8135 23.8592 14.7267 22.3775 15.1574C24.0831 15.657 25.0479 16.8286 25.0479 18.4136C25.0479 20.55 23.1872 22.0489 20.534 22.0489C18.0876 22.0489 16.313 20.6361 16.2096 18.6203L17.743 18.517C17.8808 20.0159 19.3108 20.6016 20.534 20.6016C21.9812 20.6016 23.4974 19.947 23.4974 18.293C23.4974 16.6563 21.9812 15.9499 20.534 15.9844L19.5865 16.0016V14.5371L20.534 14.5544Z" fill="white"/>
+</svg>

apps/webapp/app/components/Feedback.tsx

@@ -1,10 +1,14 @@
 import { conform, useForm } from "@conform-to/react";
 import { parse } from "@conform-to/zod";
+import { BookOpenIcon } from "@heroicons/react/20/solid";
 import { ChevronRightIcon } from "@heroicons/react/24/solid";
 import { Form, useActionData, useLocation, useNavigation } from "@remix-run/react";
 import { DiscordIcon, GitHubLightIcon } from "@trigger.dev/companyicons";
+import { ActivityIcon } from "lucide-react";
 import { ReactNode, useState } from "react";
 import { FeedbackType, feedbackTypeLabel, schema } from "~/routes/resources.feedback";
+import { cn } from "~/utils/cn";
+import { docsPath } from "~/utils/pathBuilder";
 import { Button, LinkButton } from "./primitives/Buttons";
 import { Fieldset } from "./primitives/Fieldset";
 import { FormButtons } from "./primitives/FormButtons";
@@ -13,20 +17,9 @@ import { Header1, Header2 } from "./primitives/Headers";
 import { InputGroup } from "./primitives/InputGroup";
 import { Label } from "./primitives/Label";
 import { Paragraph } from "./primitives/Paragraph";
-import {
-  Select,
-  SelectContent,
-  SelectGroup,
-  SelectItem,
-  SelectTrigger,
-  SelectValue,
-} from "./primitives/Select";
+import { Select, SelectItem } from "./primitives/Select";
 import { Sheet, SheetBody, SheetContent, SheetTrigger } from "./primitives/Sheet";
 import { TextArea } from "./primitives/TextArea";
-import { cn } from "~/utils/cn";
-import { BookOpenIcon } from "@heroicons/react/20/solid";
-import { ActivityIcon, HeartPulseIcon } from "lucide-react";
-import { docsPath } from "~/utils/pathBuilder";
 
 type FeedbackProps = {
   button: ReactNode;
@@ -78,20 +71,20 @@ export function Feedback({ button, defaultValue = "bug" }: FeedbackProps) {
             <Fieldset className="max-w-full gap-y-3">
               <input value={location.pathname} {...conform.input(path, { type: "hidden" })} />
               <InputGroup className="max-w-full">
-                <SelectGroup>
-                  <Select {...conform.input(feedbackType)} defaultValue={defaultValue}>
-                    <SelectTrigger size="medium" width="full">
-                      <SelectValue placeholder="Type" />
-                    </SelectTrigger>
-                    <SelectContent>
-                      {Object.entries(feedbackTypeLabel).map(([key, value]) => (
-                        <SelectItem key={key} value={key}>
-                          {value}
-                        </SelectItem>
-                      ))}
-                    </SelectContent>
-                  </Select>
-                </SelectGroup>
+                <Select
+                  {...conform.select(feedbackType)}
+                  variant="tertiary/medium"
+                  defaultValue={defaultValue}
+                  placeholder="Select type"
+                  text={(value) => feedbackTypeLabel[value]}
+                  dropdownIcon
+                >
+                  {Object.entries(feedbackTypeLabel).map(([name, title]) => (
+                    <SelectItem key={name} value={name}>
+                      {title}
+                    </SelectItem>
+                  ))}
+                </Select>
                 <FormError id={feedbackType.errorId}>{feedbackType.error}</FormError>
               </InputGroup>
               <InputGroup className="max-w-full">
@@ -122,14 +115,7 @@ export function Feedback({ button, defaultValue = "bug" }: FeedbackProps) {
               Docs
             </LinkButton>
             <LinkButton
-              to={docsPath("v3/introduction")}
-              variant="tertiary/medium"
-              LeadingIcon={BookOpenIcon}
-            >
-              v3 Docs (Developer preview)
-            </LinkButton>
-            <LinkButton
-              to={"https://trigger.openstatus.dev/"}
+              to={"https://status.trigger.dev/"}
               variant="tertiary/medium"
               LeadingIcon={ActivityIcon}
             >

apps/webapp/app/components/SetupCommands.tsx

@@ -8,6 +8,7 @@ import {
 } from "./primitives/ClientTabs";
 import { ClipboardField } from "./primitives/ClipboardField";
 import { Paragraph } from "./primitives/Paragraph";
+import { useAppOrigin } from "~/hooks/useAppOrigin";
 
 export function InitCommand({ appOrigin, apiKey }: { appOrigin: string; apiKey: string }) {
   return (
@@ -133,9 +134,38 @@ export function TriggerDevStep({ extra }: { extra?: string }) {
 // Trigger.dev version 3 setup commands
 const v3PackageTag = "beta";
 
+function getApiUrlArg() {
+  const appOrigin = useAppOrigin();
+
+  let apiUrl: string | undefined = undefined;
+
+  switch (appOrigin) {
+    case "https://cloud.trigger.dev":
+      // don't display the arg, use the CLI default
+      break;
+    case "https://test-cloud.trigger.dev":
+      apiUrl = "https://test-api.trigger.dev";
+      break;
+    case "https://internal.trigger.dev":
+      apiUrl = "https://internal-api.trigger.dev";
+      break;
+    default:
+      apiUrl = appOrigin;
+      break;
+  }
+
+  return apiUrl ? `-a ${apiUrl}` : undefined;
+}
+
 export function InitCommandV3() {
   const project = useProject();
   const projectRef = project.ref;
+
+  const apiUrlArg = getApiUrlArg();
+
+  const initCommandParts = [`trigger.dev@${v3PackageTag}`, "init", `-p ${projectRef}`, apiUrlArg];
+  const initCommand = initCommandParts.filter(Boolean).join(" ");
+
   return (
     <ClientTabs defaultValue="npm">
       <ClientTabsList>
@@ -148,7 +178,7 @@ export function InitCommandV3() {
           variant="primary/medium"
           iconButton
           className="mb-4"
-          value={`npx trigger.dev@${v3PackageTag} init -p ${projectRef}`}
+          value={`npx ${initCommand}`}
         />
       </ClientTabsContent>
       <ClientTabsContent value={"pnpm"}>
@@ -156,7 +186,7 @@ export function InitCommandV3() {
           variant="primary/medium"
           iconButton
           className="mb-4"
-          value={`pnpm dlx trigger.dev@${v3PackageTag} init -p ${projectRef}`}
+          value={`pnpm dlx ${initCommand}`}
         />
       </ClientTabsContent>
       <ClientTabsContent value={"yarn"}>
@@ -164,7 +194,7 @@ export function InitCommandV3() {
           variant="primary/medium"
           iconButton
           className="mb-4"
-          value={`yarn dlx trigger.dev@${v3PackageTag} init -p ${projectRef}`}
+          value={`yarn dlx ${initCommand}`}
         />
       </ClientTabsContent>
     </ClientTabs>
@@ -206,3 +236,39 @@ export function TriggerDevStepV3() {
     </ClientTabs>
   );
 }
+
+export function TriggerLoginStepV3() {
+  return (
+    <ClientTabs defaultValue="npm">
+      <ClientTabsList>
+        <ClientTabsTrigger value={"npm"}>npm</ClientTabsTrigger>
+        <ClientTabsTrigger value={"pnpm"}>pnpm</ClientTabsTrigger>
+        <ClientTabsTrigger value={"yarn"}>yarn</ClientTabsTrigger>
+      </ClientTabsList>
+      <ClientTabsContent value={"npm"}>
+        <ClipboardField
+          variant="primary/medium"
+          iconButton
+          className="mb-4"
+          value={`npx trigger.dev@${v3PackageTag} login`}
+        />
+      </ClientTabsContent>
+      <ClientTabsContent value={"pnpm"}>
+        <ClipboardField
+          variant="primary/medium"
+          iconButton
+          className="mb-4"
+          value={`pnpm dlx trigger.dev@${v3PackageTag} login`}
+        />
+      </ClientTabsContent>
+      <ClientTabsContent value={"yarn"}>
+        <ClipboardField
+          variant="primary/medium"
+          iconButton
+          className="mb-4"
+          value={`yarn dlx trigger.dev@${v3PackageTag} login`}
+        />
+      </ClientTabsContent>
+    </ClientTabs>
+  );
+}

apps/webapp/app/components/admin/debugTooltip.tsx

@@ -0,0 +1,31 @@
+import { ShieldCheckIcon } from "@heroicons/react/20/solid";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger,
+} from "~/components/primitives/Tooltip";
+import { useIsImpersonating } from "~/hooks/useOrganizations";
+import { useHasAdminAccess } from "~/hooks/useUser";
+
+export function AdminDebugTooltip({ children }: { children: React.ReactNode }) {
+  const hasAdminAccess = useHasAdminAccess();
+  const isImpersonating = useIsImpersonating();
+
+  if (!hasAdminAccess && !isImpersonating) {
+    return null;
+  }
+
+  return (
+    <TooltipProvider>
+      <Tooltip>
+        <TooltipTrigger>
+          <ShieldCheckIcon className="size-5" />
+        </TooltipTrigger>
+        <TooltipContent className="flex max-h-[90vh] items-center gap-1 overflow-y-auto">
+          {children}
+        </TooltipContent>
+      </Tooltip>
+    </TooltipProvider>
+  );
+}

apps/webapp/app/components/billing/PricingTiers.tsx

@@ -290,6 +290,7 @@ export function TierPro({
           options={concurrencyTiers.map((c) => ({ label: `Up to ${c.upto}`, value: c.code }))}
           fullWidth
           value={concurrentBracketCode}
+          variant="primary"
           onChange={(v) => setConcurrentBracketCode(v)}
         />
         <div className="py-6">

apps/webapp/app/components/code/CodeBlock.tsx

@@ -1,14 +1,23 @@
 import { Clipboard, ClipboardCheck } from "lucide-react";
 import type { Language, PrismTheme } from "prism-react-renderer";
-import Highlight, { defaultProps } from "prism-react-renderer";
+import { Highlight, Prism } from "prism-react-renderer";
 import { forwardRef, useCallback, useState } from "react";
 import { cn } from "~/utils/cn";
-import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "../primitives/Tooltip";
 import { Paragraph } from "../primitives/Paragraph";
+import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "../primitives/Tooltip";
 
 //This is a fork of https://github.com/mantinedev/mantine/blob/master/src/mantine-prism/src/Prism/Prism.tsx
 //it didn't support highlighting lines by dimming the rest of the code, or animations on the highlighting
 
+async function setup() {
+  (typeof global !== "undefined" ? global : window).Prism = Prism;
+  //@ts-ignore
+  await import("prismjs/components/prism-json");
+  //@ts-ignore
+  await import("prismjs/components/prism-typescript");
+}
+setup();
+
 type CodeBlockProps = {
   /** Code which will be highlighted */
   code: string;
@@ -238,7 +247,7 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
         )}
 
         {shouldHighlight ? (
-          <Highlight {...defaultProps} theme={theme} code={code} language={language}>
+          <Highlight theme={theme} code={code} language={language}>
             {({
               className: inheritedClassName,
               style: inheritedStyle,
@@ -283,7 +292,7 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
 
                       return (
                         <div
-                          key={lineProps.key}
+                          key={lineNumber}
                           {...lineProps}
                           className={cn(
                             "flex w-full justify-start transition-opacity duration-500",
@@ -312,7 +321,7 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
                               const tokenProps = getTokenProps({ token, key });
                               return (
                                 <span
-                                  key={tokenProps.key}
+                                  key={key}
                                   {...tokenProps}
                                   style={{
                                     color: tokenProps?.style?.color as string,

apps/webapp/app/components/environments/EnvironmentLabel.tsx

@@ -1,5 +1,7 @@
-import { RuntimeEnvironment } from "~/models/runtimeEnvironment.server";
+import type { RuntimeEnvironment } from "~/models/runtimeEnvironment.server";
 import { cn } from "~/utils/cn";
+import { sortEnvironments } from "~/utils/environmentSort";
+import { SimpleTooltip } from "../primitives/Tooltip";
 
 type Environment = Pick<RuntimeEnvironment, "type">;
 const variants = {
@@ -7,6 +9,30 @@ const variants = {
   large: "h-6 text-xs px-1.5 rounded",
 };
 
+export function EnvironmentTypeLabel({
+  environment,
+  size = "small",
+  className,
+}: {
+  environment: Environment;
+  size?: keyof typeof variants;
+  className?: string;
+}) {
+  return (
+    <span
+      className={cn(
+        "text-midnight-900 inline-flex items-center justify-center whitespace-nowrap border font-medium uppercase tracking-wider",
+        environmentBorderClassName(environment),
+        environmentTextClassName(environment),
+        variants[size],
+        className
+      )}
+    >
+      {environmentTypeTitle(environment)}
+    </span>
+  );
+}
+
 export function EnvironmentLabel({
   environment,
   size = "small",
@@ -33,6 +59,74 @@ export function EnvironmentLabel({
   );
 }
 
+type EnvironmentWithUsername = Environment & { userName?: string };
+
+export function EnvironmentLabels({
+  environments,
+  size = "small",
+  className,
+}: {
+  environments: EnvironmentWithUsername[];
+  size?: keyof typeof variants;
+  className?: string;
+}) {
+  const devEnvironments = sortEnvironments(
+    environments.filter((env) => env.type === "DEVELOPMENT")
+  );
+  const firstDevEnvironment = devEnvironments[0];
+  const otherDevEnvironments = devEnvironments.slice(1);
+  const otherEnvironments = environments.filter((env) => env.type !== "DEVELOPMENT");
+
+  return (
+    <div className={cn("flex items-baseline gap-2", className)}>
+      {firstDevEnvironment && (
+        <EnvironmentLabel
+          environment={firstDevEnvironment}
+          userName={firstDevEnvironment.userName}
+          size={size}
+        />
+      )}
+      {otherDevEnvironments.length > 0 ? (
+        <SimpleTooltip
+          disableHoverableContent
+          button={
+            <span
+              className={cn(
+                "inline-flex items-center justify-center whitespace-nowrap border font-medium uppercase tracking-wider",
+                environmentBorderClassName({ type: "DEVELOPMENT" }),
+                environmentTextClassName({ type: "DEVELOPMENT" }),
+                variants[size]
+              )}
+            >
+              +{otherDevEnvironments.length}
+            </span>
+          }
+          content={
+            <div className="flex gap-1 py-1">
+              {otherDevEnvironments.map((environment, index) => (
+                <EnvironmentLabel
+                  key={index}
+                  environment={environment}
+                  userName={environment.userName}
+                  size={size}
+                />
+              ))}
+            </div>
+          }
+        />
+      ) : null}
+      {otherEnvironments.map((environment, index) => (
+        <EnvironmentLabel
+          key={index}
+          environment={environment}
+          userName={environment.userName}
+          size={size}
+        />
+      ))}
+    </div>
+  );
+}
+
 export function environmentTitle(environment: Environment, username?: string) {
   switch (environment.type) {
     case "PRODUCTION":
@@ -46,6 +140,19 @@ export function environmentTitle(environment: Environment, username?: string) {
   }
 }
 
+export function environmentTypeTitle(environment: Environment) {
+  switch (environment.type) {
+    case "PRODUCTION":
+      return "Prod";
+    case "STAGING":
+      return "Staging";
+    case "DEVELOPMENT":
+      return "Dev";
+    case "PREVIEW":
+      return "Preview";
+  }
+}
+
 export function environmentColorClassName(environment: Environment) {
   switch (environment.type) {
     case "PRODUCTION":

apps/webapp/app/components/events/EventsFilters.tsx

@@ -9,7 +9,7 @@ import {
   SelectItem,
   SelectTrigger,
   SelectValue,
-} from "../primitives/Select";
+} from "../primitives/SimpleSelect";
 import { EventListSearchSchema } from "./EventStatuses";
 import { environmentKeys, FilterableEnvironment } from "~/components/runs/RunStatuses";
 import { TimeFrameFilter } from "../runs/TimeFrameFilter";

apps/webapp/app/components/integrations/ConnectToOAuthForm.tsx

@@ -12,7 +12,7 @@ import { ApiAuthenticationMethodOAuth2, Integration, Scope } from "~/services/ex
 import { cn } from "~/utils/cn";
 import { CodeBlock } from "../code/CodeBlock";
 import { Button } from "../primitives/Buttons";
-import { Checkbox } from "../primitives/Checkbox";
+import { CheckboxWithLabel } from "../primitives/Checkbox";
 import { Fieldset } from "../primitives/Fieldset";
 import { FormError } from "../primitives/FormError";
 import { Header2, Header3 } from "../primitives/Headers";
@@ -123,7 +123,7 @@ export function ConnectToOAuthForm({
           <Paragraph variant="small" className="mb-2">
             To use your own OAuth app, check the option below and insert the details.
           </Paragraph>
-          <Checkbox
+          <CheckboxWithLabel
             id="hasCustomClient"
             label="Use my OAuth App"
             variant="simple/small"
@@ -200,7 +200,7 @@ export function ConnectToOAuthForm({
               )}
               {authMethod.scopes.map((s) => {
                 return (
-                  <Checkbox
+                  <CheckboxWithLabel
                     key={s.name}
                     id={s.name}
                     value={s.name}

apps/webapp/app/components/integrations/UpdateOAuthForm.tsx

@@ -10,7 +10,7 @@ import { ApiAuthenticationMethodOAuth2, Integration, Scope } from "~/services/ex
 import { cn } from "~/utils/cn";
 import { CodeBlock } from "../code/CodeBlock";
 import { Button } from "../primitives/Buttons";
-import { Checkbox } from "../primitives/Checkbox";
+import { CheckboxWithLabel } from "../primitives/Checkbox";
 import { Fieldset } from "../primitives/Fieldset";
 import { FormError } from "../primitives/FormError";
 import { Header2, Header3 } from "../primitives/Headers";
@@ -113,7 +113,7 @@ export function UpdateOAuthForm({
           <Paragraph variant="small" className="mb-2">
             To use your own OAuth app, check the option below and insert the details.
           </Paragraph>
-          <Checkbox
+          <CheckboxWithLabel
             id="hasCustomClient"
             label="Use my OAuth App"
             variant="simple/small"
@@ -189,7 +189,7 @@ export function UpdateOAuthForm({
               )}
               {authMethod.scopes.map((s) => {
                 return (
-                  <Checkbox
+                  <CheckboxWithLabel
                     key={s.name}
                     id={s.name}
                     value={s.name}

apps/webapp/app/components/layout/AppLayout.tsx

@@ -41,7 +41,7 @@ export function PageBody({
     <div
       className={cn(
         scrollable
-          ? "overflow-y-auto p-4 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600"
+          ? "overflow-y-auto p-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600"
           : "overflow-hidden",
         className
       )}

apps/webapp/app/components/navigation/SideMenu.tsx

@@ -3,6 +3,7 @@ import {
   ArrowRightIcon,
   ArrowRightOnRectangleIcon,
   BeakerIcon,
+  BellAlertIcon,
   ChartBarIcon,
   ClockIcon,
   CursorArrowRaysIcon,
@@ -45,6 +46,7 @@ import {
   v3ApiKeysPath,
   v3DeploymentsPath,
   v3EnvironmentVariablesPath,
+  v3ProjectAlertsPath,
   v3ProjectPath,
   v3ProjectSettingsPath,
   v3RunsPath,
@@ -231,31 +233,22 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
                   </div>
                   <hr className="border-charcoal-800" />
                   <div>
-                    <StepNumber stepNumber="1" title="Create a new Slack channel" />
+                    <StepNumber stepNumber="1" title="Email us" />
                     <StepContentContainer>
                       <Paragraph>
-                        In your Slack app, create a new channel from the main menu by going to File{" "}
-                        <ArrowRightIcon className="inline h-4 w-4 text-text-dimmed" /> New Channel
-                      </Paragraph>
-                    </StepContentContainer>
-                    <StepNumber stepNumber="2" title="Setup your channel" />
-                    <StepContentContainer>
-                      <Paragraph>
-                        Name your channel, set its visibility and click 'Create'.
-                      </Paragraph>
-                    </StepContentContainer>
-                    <StepNumber stepNumber="3" title="Invite Trigger.dev" />
-                    <StepContentContainer>
-                      <Paragraph>
-                        Invite this email address to your channel:{" "}
+                        Send us an email to this address from your Trigger.dev account email
+                        address:
                         <ClipboardField
                           variant="primary/medium"
-                          value="james@trigger.dev"
+                          value="priority-support@trigger.dev"
                           className="my-2"
                         />
                       </Paragraph>
+                    </StepContentContainer>
+                    <StepNumber stepNumber="2" title="Look out for an invite from Slack" />
+                    <StepContentContainer>
                       <Paragraph>
-                        As soon as we can, we'll accept your invitation and say hello!
+                        As soon as we can, we'll setup a Slack Connect channel and say hello!
                       </Paragraph>
                     </StepContentContainer>
                   </div>
@@ -549,6 +542,8 @@ function V3ProjectSideMenu({
   project: SideMenuProject;
   organization: MatchedOrganization;
 }) {
+  const { alertsEnabled } = useFeatures();
+
   return (
     <>
       <SideMenuHeader title={"Project (v3)"} />
@@ -601,6 +596,15 @@ function V3ProjectSideMenu({
         to={v3DeploymentsPath(organization, project)}
         data-action="deployments"
       />
+      {alertsEnabled && (
+        <SideMenuItem
+          name="Alerts"
+          icon={BellAlertIcon}
+          iconColor="text-red-500"
+          to={v3ProjectAlertsPath(organization, project)}
+          data-action="alerts"
+        />
+      )}
       <SideMenuItem
         name="Project settings"
         icon="settings"

apps/webapp/app/components/primitives/AppliedFilter.tsx

@@ -0,0 +1,56 @@
+import { XMarkIcon } from "@heroicons/react/20/solid";
+import { ReactNode } from "react";
+import { cn } from "~/utils/cn";
+
+const variants = {
+  "tertiary/small": {
+    box: "h-6 bg-tertiary rounded pl-1.5 gap-1.5 text-xs divide-x divide-black/15 group-hover:bg-charcoal-600",
+    clear: "size-6 text-text-dimmed hover:text-text-bright transition-colors",
+  },
+  "minimal/small": {
+    box: "h-6 hover:bg-tertiary rounded pl-1.5 gap-1.5 text-xs",
+    clear: "size-6 text-text-dimmed hover:text-text-bright transition-colors",
+  },
+};
+
+type Variant = keyof typeof variants;
+
+type AppliedFilterProps = {
+  label: ReactNode;
+  value: ReactNode;
+  removable?: boolean;
+  onRemove?: () => void;
+  variant?: Variant;
+  className?: string;
+};
+
+export function AppliedFilter({
+  label,
+  value,
+  removable = true,
+  onRemove,
+  variant = "tertiary/small",
+  className,
+}: AppliedFilterProps) {
+  const variantClassName = variants[variant];
+  return (
+    <div className={cn("flex items-center transition", variantClassName.box, className)}>
+      <div className="flex items-center gap-0.5">
+        <div className="text-text-dimmed">
+          <span>{label}</span>:
+        </div>
+        <div className="text-text-bright">
+          <div>{value}</div>
+        </div>
+      </div>
+      {removable && (
+        <button
+          className={cn("group flex size-6 items-center justify-center", variantClassName.clear)}
+          onClick={onRemove}
+        >
+          <XMarkIcon className="size-3.5" />
+        </button>
+      )}
+    </div>
+  );
+}

apps/webapp/app/components/primitives/Buttons.tsx

@@ -48,11 +48,11 @@ const theme = {
       "border-black/40 text-charcoal-900 group-hover:border-black/60 group-hover:text-charcoal-900",
   },
   secondary: {
-    textColor: "text-primary group-hover:text-apple-200 transition group-disabled:text-primary",
+    textColor: "text-secondary group-hover:text-secondary transition group-disabled:text-secondary",
     button:
-      "bg-transparent border border-primary group-hover:border-apple-200 group-hover:bg-apple-950 group-disabled:opacity-30 group-disabled:border-primary group-disabled:bg-transparent group-disabled:pointer-events-none",
+      "bg-transparent border border-secondary group-hover:border-secondary group-hover:bg-secondary/10 group-disabled:opacity-30 group-disabled:border-secondary group-disabled:bg-transparent group-disabled:pointer-events-none",
     shortcut:
-      "border-primary/30 text-apple-200 group-hover:text-text-bright/80 group-hover:border-dimmed/60",
+      "border-secondary/30 text-secondary group-hover:text-text-bright/80 group-hover:border-dimmed/60",
   },
   tertiary: {
     textColor: "text-text-bright transition group-disabled:text-text-dimmed/80",

apps/webapp/app/components/primitives/Checkbox.tsx

@@ -1,5 +1,5 @@
 import * as React from "react";
-import { useEffect, useState } from "react";
+import { forwardRef, useEffect, useState } from "react";
 import { cn } from "~/utils/cn";
 import { Badge } from "./Badge";
 import { Paragraph } from "./Paragraph";
@@ -53,18 +53,18 @@ export type CheckboxProps = Omit<
   React.InputHTMLAttributes<HTMLInputElement>,
   "checked" | "onChange"
 > & {
-  id: string;
+  id?: string;
   name?: string;
   value?: string;
   variant?: keyof typeof variants;
-  label?: React.ReactNode;
+  label: React.ReactNode;
   description?: string;
   badges?: string[];
   className?: string;
   onChange?: (isChecked: boolean) => void;
 };
 
-export const Checkbox = React.forwardRef<HTMLInputElement, CheckboxProps>(
+export const CheckboxWithLabel = React.forwardRef<HTMLInputElement, CheckboxProps>(
   (
     {
       id,
@@ -172,3 +172,21 @@ export const Checkbox = React.forwardRef<HTMLInputElement, CheckboxProps>(
     );
   }
 );
+
+type SimpleCheckboxProps = Omit<React.ComponentProps<"input">, "type">;
+
+export const Checkbox = forwardRef<HTMLInputElement, SimpleCheckboxProps>(
+  ({ className, ...props }: SimpleCheckboxProps, ref) => {
+    return (
+      <input
+        type="checkbox"
+        className={cn(
+          props.readOnly || props.disabled ? "cursor-default" : "cursor-pointer",
+          "read-only:border-charcoal-650 disabled:border-charcoal-650 rounded-sm border border-charcoal-600 bg-transparent transition checked:!bg-indigo-500 read-only:!bg-charcoal-700 group-hover:bg-charcoal-900 group-hover:checked:bg-indigo-500 group-focus:ring-1 focus:ring-indigo-500 focus:ring-offset-0 focus:ring-offset-transparent focus-visible:outline-none  focus-visible:ring-indigo-500 disabled:!bg-charcoal-700"
+        )}
+        {...props}
+        ref={ref}
+      />
+    );
+  }
+);

apps/webapp/app/components/primitives/DetailCell.tsx

@@ -49,12 +49,7 @@ export function DetailCell({
   const variation = variations[variant];
 
   return (
-    <div
-      className={cn(
-        "group flex h-11 w-full items-center gap-3 rounded-md p-1 pr-3 transition hover:bg-charcoal-900",
-        className
-      )}
-    >
+    <div className={cn("group flex h-11 w-full items-center gap-3 rounded-md p-1 pr-3", className)}>
       <IconInBox
         icon={leadingIcon}
         className={cn("flex-none transition group-hover:border-charcoal-750", leadingIconClassName)}
@@ -62,20 +57,14 @@ export function DetailCell({
       <div className="flex flex-1 flex-col">
         <Paragraph
           variant={variation.label.variant}
-          className={cn(
-            "flex-1 text-left transition group-hover:text-text-bright",
-            variation.label.className
-          )}
+          className={cn("flex-1 text-left", variation.label.className)}
         >
           {label}
         </Paragraph>
         {description && (
           <Paragraph
             variant={variation.description.variant}
-            className={cn(
-              "flex-1 text-left text-text-dimmed transition group-hover:text-text-bright",
-              variation.description.className
-            )}
+            className={cn("flex-1 text-left text-text-dimmed", variation.description.className)}
           >
             {description}
           </Paragraph>

apps/webapp/app/components/primitives/Dialog.tsx

@@ -43,7 +43,7 @@ const DialogContent = React.forwardRef<
     <DialogPrimitive.Content
       ref={ref}
       className={cn(
-        "fixed z-50 grid w-full gap-4 rounded-b-lg border bg-background-dimmed px-4 pb-4 pt-3.5 shadow-lg animate-in data-[state=open]:fade-in-90 data-[state=open]:slide-in-from-bottom-10 sm:max-w-lg sm:rounded-lg sm:zoom-in-90 data-[state=open]:sm:slide-in-from-bottom-0",
+        "fixed z-50 grid w-full gap-4 rounded-b-lg border bg-background-dimmed px-4 pb-4 pt-2.5 shadow-lg animate-in data-[state=open]:fade-in-90 data-[state=open]:slide-in-from-bottom-10 sm:max-w-lg sm:rounded-lg sm:zoom-in-90 data-[state=open]:sm:slide-in-from-bottom-0",
         className
       )}
       {...props}
@@ -74,7 +74,7 @@ DialogContent.displayName = DialogPrimitive.Content.displayName;
 
 const DialogHeader = ({ className, ...props }: React.HTMLAttributes<HTMLDivElement>) => (
   <div
-    className={cn("flex flex-col text-left font-medium text-text-dimmed", className)}
+    className={cn("flex flex-col text-left font-medium text-text-bright", className)}
     {...props}
   />
 );

apps/webapp/app/components/primitives/Hint.tsx

@@ -1,5 +1,9 @@
 import { Paragraph } from "./Paragraph";
 
-export function Hint({ children }: { children: React.ReactNode }) {
-  return <Paragraph variant="extra-small">{children}</Paragraph>;
+export function Hint({ children, className }: { children: React.ReactNode; className?: string }) {
+  return (
+    <Paragraph variant="extra-small" className={className}>
+      {children}
+    </Paragraph>
+  );
 }

apps/webapp/app/components/primitives/SegmentedControl.tsx

@@ -2,6 +2,19 @@ import { RadioGroup } from "@headlessui/react";
 import { motion } from "framer-motion";
 import { cn } from "~/utils/cn";
 
+const variants = {
+  primary: {
+    base: "bg-charcoal-700",
+    active: "text-text-bright hover:bg-charcoal-750/50",
+  },
+  secondary: {
+    base: "bg-charcoal-700/50",
+    active: "text-text-bright bg-charcoal-700 rounded-[2px] border border-charcoal-600/50",
+  },
+};
+
+type Variants = keyof typeof variants;
+
 type Options = {
   label: string;
   value: string;
@@ -12,6 +25,7 @@ type SegmentedControlProps = {
   value?: string;
   defaultValue?: string;
   options: Options[];
+  variant?: Variants;
   fullWidth?: boolean;
   onChange?: (value: string) => void;
 };
@@ -21,11 +35,18 @@ export default function SegmentedControl({
   value,
   defaultValue,
   options,
+  variant = "secondary",
   fullWidth,
   onChange,
 }: SegmentedControlProps) {
   return (
-    <div className={cn("flex h-10 rounded bg-charcoal-700", fullWidth ? "w-full" : "w-fit")}>
+    <div
+      className={cn(
+        "flex h-10 rounded text-text-bright",
+        variants[variant].base,
+        fullWidth ? "w-full" : "w-fit"
+      )}
+    >
       <RadioGroup
         value={value}
         defaultValue={defaultValue ?? options[0].value}
@@ -46,11 +67,11 @@ export default function SegmentedControl({
                 cn(
                   "relative flex h-full grow cursor-pointer text-center font-normal focus:outline-none",
                   active
-                    ? "ring-offset-2 focus-visible:ring focus-visible:ring-primary focus-visible:ring-opacity-60"
+                    ? "ring-offset-2 focus-visible:ring focus-visible:ring-secondary focus-visible:ring-opacity-60"
                     : "",
                   checked
-                    ? "text-text-bright"
-                    : "rounded-[2px] text-text-dimmed transition hover:bg-charcoal-750/50 hover:text-text-bright"
+                    ? variants[variant].active
+                    : "text-text-dimmed transition hover:text-text-bright"
                 )
               }
             >
@@ -60,12 +81,12 @@ export default function SegmentedControl({
                     <div className="z-10 flex h-full w-full items-center justify-center text-sm">
                       <RadioGroup.Label as="p">{option.label}</RadioGroup.Label>
                     </div>
-                    {checked && (
+                    {checked && variant === "primary" && (
                       <motion.div
                         layoutId={`segmented-control-${name}`}
                         transition={{ duration: 0.4, type: "spring" }}
                         className="absolute inset-0 rounded-[2px] shadow-md outline outline-3 outline-primary"
-                      ></motion.div>
+                      />
                     )}
                   </div>
                 </>

apps/webapp/app/components/primitives/Select.tsx

@@ -1,143 +1,624 @@
-"use client";
-
-import * as SelectPrimitive from "@radix-ui/react-select";
-import { Check, ChevronDown } from "lucide-react";
+import * as Ariakit from "@ariakit/react";
+import { SelectProps as AriaSelectProps } from "@ariakit/react";
+import { SelectValue } from "@ariakit/react-core/select/select-value";
+import { Link } from "@remix-run/react";
 import * as React from "react";
+import { Fragment, useMemo, useState } from "react";
+import { ShortcutDefinition, useShortcutKeys } from "~/hooks/useShortcutKeys";
 import { cn } from "~/utils/cn";
+import { ShortcutKey } from "./ShortcutKey";
+import { ChevronDown } from "lucide-react";
 
 const sizes = {
-  "secondary/small":
-    "text-xs h-6 bg-tertiary border border-tertiary group-hover:text-text-bright hover:border-charcoal-600 pr-2 pl-1.5",
-  medium: "text-sm h-8 bg-tertiary border border-tertiary hover:border-charcoal-600 px-2.5",
-  minimal: "text-xs h-6 bg-transparent hover:bg-tertiary pl-1.5 pr-2",
+  small: {
+    button: "h-6 rounded text-xs px-2 ",
+  },
+  medium: {
+    button: "h-8 rounded text-xs px-3 text-sm",
+  },
 };
 
-export type SelectProps = {
-  size?: keyof typeof sizes;
-  width?: "content" | "full";
+const style = {
+  tertiary: {
+    button:
+      "bg-tertiary focus-within:ring-charcoal-500 border border-tertiary hover:text-text-bright hover:border-charcoal-600",
+  },
+  minimal: {
+    button:
+      "bg-transparent focus-within:ring-charcoal-500 hover:bg-tertiary disabled:bg-transparent disabled:pointer-events-none",
+  },
 };
 
-const Select = SelectPrimitive.Root;
-const SelectGroup = SelectPrimitive.Group;
-const SelectValue = SelectPrimitive.Value;
-const SelectTrigger = React.forwardRef<
-  React.ElementRef<typeof SelectPrimitive.Trigger>,
-  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Trigger> & SelectProps
->(({ className, children, width = "content", size = "secondary/small", ...props }, ref) => {
-  const sizeClassName = sizes[size];
+const variants = {
+  "tertiary/small": {
+    button: cn(sizes.small.button, style.tertiary.button),
+  },
+  "tertiary/medium": {
+    button: cn(sizes.medium.button, style.tertiary.button),
+  },
+  "minimal/small": {
+    button: cn(sizes.small.button, style.minimal.button),
+  },
+  "minimal/medium": {
+    button: cn(sizes.medium.button, style.minimal.button),
+  },
+};
+
+type Variant = keyof typeof variants;
+
+type Section<TItem> = {
+  type: "section";
+  title?: string;
+  items: TItem[];
+};
+
+function isSection<TItem>(data: TItem[] | Section<TItem>[]): data is Section<TItem>[] {
+  const firstItem = data[0];
   return (
-    <SelectPrimitive.Trigger
-      ref={ref}
-      className={cn(
-        "ring-offset-background focus-visible:ring-ring group flex items-center justify-between gap-x-1 rounded text-text-dimmed transition placeholder:text-text-dimmed hover:text-text-bright focus-visible:bg-tertiary focus-visible:text-text-bright focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-offset-0 disabled:cursor-not-allowed disabled:opacity-50",
-        width === "full" ? "w-full" : "w-min",
-        sizeClassName,
-        className
-      )}
-      {...props}
-    >
-      {children}
-      <SelectPrimitive.Icon asChild>
-        <ChevronDown
-          className={cn(
-            "size-4 text-text-dimmed transition group-hover:text-text-bright group-focus:text-text-bright"
-          )}
-        />
-      </SelectPrimitive.Icon>
-    </SelectPrimitive.Trigger>
+    (firstItem as Section<TItem>).type === "section" &&
+    (firstItem as Section<TItem>).items !== undefined &&
+    Array.isArray((firstItem as Section<TItem>).items)
   );
-});
-SelectTrigger.displayName = SelectPrimitive.Trigger.displayName;
+}
 
-const SelectContent = React.forwardRef<
-  React.ElementRef<typeof SelectPrimitive.Content>,
-  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Content>
->(({ className, children, position = "popper", ...props }, ref) => (
-  <SelectPrimitive.Portal>
-    <SelectPrimitive.Content
-      ref={ref}
-      className={cn(
-        "relative z-50 min-w-max overflow-hidden rounded-md border border-charcoal-700 bg-background-dimmed text-text-bright shadow-md animate-in fade-in-40",
-        position === "popper" && "translate-y-1",
-        className
-      )}
-      position={position}
-      {...props}
+type ItemFromSection<TItemOrSection> = TItemOrSection extends Section<infer U> ? U : TItemOrSection;
+export interface SelectProps<TValue extends string | string[], TItem>
+  extends Omit<Ariakit.SelectProps, "children"> {
+  icon?: React.ReactNode;
+  text?: React.ReactNode | ((value: TValue) => React.ReactNode);
+  placeholder?: React.ReactNode;
+  value?: Ariakit.SelectProviderProps<TValue>["value"];
+  setValue?: Ariakit.SelectProviderProps<TValue>["setValue"];
+  defaultValue?: Ariakit.SelectProviderProps<TValue>["defaultValue"];
+  label?: string | Ariakit.SelectLabelProps["render"];
+  heading?: string;
+  showHeading?: boolean;
+  items?: TItem[] | Section<TItem>[];
+  empty?: React.ReactNode;
+  filter?: (item: ItemFromSection<TItem>, search: string, title?: string) => boolean;
+  children:
+    | React.ReactNode
+    | ((
+        items: ItemFromSection<TItem>[],
+        meta: {
+          shortcutsEnabled?: boolean;
+          section?: {
+            title?: string;
+            startIndex: number;
+            count: number;
+          };
+        }
+      ) => React.ReactNode);
+  variant?: Variant;
+  open?: boolean;
+  setOpen?: (open: boolean) => void;
+  shortcut?: ShortcutDefinition;
+  allowItemShortcuts?: boolean;
+  clearSearchOnSelection?: boolean;
+  dropdownIcon?: boolean | React.ReactNode;
+}
+
+export function Select<TValue extends string | string[], TItem>({
+  children,
+  icon,
+  text,
+  placeholder,
+  value,
+  setValue,
+  defaultValue,
+  label,
+  heading,
+  showHeading = false,
+  items,
+  filter,
+  empty = null,
+  variant = "tertiary/small",
+  open,
+  setOpen,
+  shortcut,
+  allowItemShortcuts = true,
+  disabled,
+  clearSearchOnSelection = true,
+  dropdownIcon,
+  ...props
+}: SelectProps<TValue, TItem>) {
+  const [searchValue, setSearchValue] = useState("");
+  const searchable = items !== undefined && filter !== undefined;
+
+  const matches = useMemo(() => {
+    if (!items) return [];
+    if (!searchValue || !filter) return items;
+
+    if (isSection(items)) {
+      return items
+        .map((section) => ({
+          ...section,
+          items: section.items.filter((item) =>
+            filter(item as ItemFromSection<TItem>, searchValue, section.title)
+          ),
+        }))
+        .filter((section) => section.items.length > 0);
+    }
+
+    return items.filter((item) => filter(item as ItemFromSection<TItem>, searchValue));
+  }, [searchValue, items]);
+
+  const enableItemShortcuts = allowItemShortcuts && matches.length === items?.length;
+
+  const select = (
+    <SelectProvider
+      open={open}
+      setOpen={setOpen}
+      virtualFocus={searchable}
+      value={value}
+      setValue={(v) => {
+        if (clearSearchOnSelection) {
+          setSearchValue("");
+        }
+
+        if (setValue) {
+          setValue(v as any);
+        }
+      }}
+      defaultValue={defaultValue}
     >
-      <SelectPrimitive.Viewport
-        className={cn(
-          "space-y-0.5 px-1 py-1",
-          position === "popper" &&
-            "h-[var(--radix-select-trigger-height)] w-full min-w-[var(--radix-select-trigger-width)]"
-        )}
+      {label && <SelectLabel render={typeof label === "string" ? <div>{label}</div> : label} />}
+      <SelectTrigger
+        icon={icon}
+        variant={variant}
+        text={text}
+        placeholder={placeholder}
+        shortcut={shortcut}
+        tooltipTitle={heading}
+        disabled={disabled}
+        dropdownIcon={dropdownIcon}
+        {...props}
+      />
+      <SelectPopover>
+        {!searchable && showHeading && heading && <SelectHeading render={<>{heading}</>} />}
+        {searchable && <ComboBox placeholder={heading} shortcut={shortcut} value={searchValue} />}
+
+        <SelectList>
+          {typeof children === "function" ? (
+            matches.length > 0 ? (
+              isSection(matches) ? (
+                <SelectGroupedRenderer
+                  items={matches}
+                  children={children}
+                  enableItemShortcuts={enableItemShortcuts}
+                />
+              ) : (
+                children(matches as ItemFromSection<TItem>[], {
+                  shortcutsEnabled: enableItemShortcuts,
+                })
+              )
+            ) : (
+              empty
+            )
+          ) : (
+            children
+          )}
+        </SelectList>
+      </SelectPopover>
+    </SelectProvider>
+  );
+
+  if (searchable) {
+    return (
+      <ComboboxProvider
+        resetValueOnHide
+        setValue={(value) => {
+          React.startTransition(() => {
+            setSearchValue(value);
+          });
+        }}
       >
-        {children}
-      </SelectPrimitive.Viewport>
-    </SelectPrimitive.Content>
-  </SelectPrimitive.Portal>
-));
-SelectContent.displayName = SelectPrimitive.Content.displayName;
+        {select}
+      </ComboboxProvider>
+    );
+  }
 
-const SelectLabel = React.forwardRef<
-  React.ElementRef<typeof SelectPrimitive.Label>,
-  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Label>
->(({ className, ...props }, ref) => (
-  <SelectPrimitive.Label
-    ref={ref}
-    className={cn(
-      "-ml-1 -mr-1 mb-1 bg-charcoal-900 py-1.5 pl-2 pr-2 font-sans text-xxs font-normal uppercase leading-normal tracking-wider text-text-dimmed first-of-type:-mt-0",
-      className
-    )}
-    {...props}
-  />
-));
-SelectLabel.displayName = SelectPrimitive.Label.displayName;
+  return select;
+}
 
-type SelectItemProps = React.ComponentPropsWithoutRef<typeof SelectPrimitive.Item> & {
-  contentClassName?: string;
-};
+export interface SelectTriggerProps<TValue = any> extends AriaSelectProps {
+  icon?: React.ReactNode;
+  text?: React.ReactNode | ((value: TValue) => React.ReactNode);
+  placeholder?: React.ReactNode;
+  variant?: Variant;
+  shortcut?: ShortcutDefinition;
+  tooltipTitle?: string;
+  dropdownIcon?: boolean | React.ReactNode;
+}
 
-const SelectItem = React.forwardRef<React.ElementRef<typeof SelectPrimitive.Item>, SelectItemProps>(
-  ({ className, children, contentClassName, ...props }, ref) => (
-    <SelectPrimitive.Item
-      ref={ref}
+export function SelectTrigger({
+  icon,
+  variant = "tertiary/small",
+  text,
+  shortcut,
+  tooltipTitle,
+  disabled,
+  placeholder,
+  dropdownIcon = false,
+  children,
+  className,
+  ...props
+}: SelectTriggerProps) {
+  const ref = React.useRef<HTMLButtonElement>(null);
+  useShortcutKeys({
+    shortcut: shortcut,
+    action: (e) => {
+      e.preventDefault();
+      e.stopPropagation();
+      if (ref.current) {
+        ref.current.click();
+      }
+    },
+    disabled,
+  });
+
+  const showTooltip = tooltipTitle || shortcut;
+  const variantClasses = variants[variant];
+
+  let content: React.ReactNode = "";
+  if (children) {
+    content = children;
+  } else if (text !== undefined) {
+    if (typeof text === "function") {
+      content = <SelectValue>{(value) => <>{text(value) ?? placeholder}</>}</SelectValue>;
+    } else {
+      content = text;
+    }
+  } else {
+    content = (
+      <SelectValue>
+        {(value) => (
+          <>
+            {typeof value === "string"
+              ? value ?? placeholder
+              : value.length === 0
+              ? placeholder
+              : value.join(", ")}
+          </>
+        )}
+      </SelectValue>
+    );
+  }
+
+  return (
+    <Ariakit.TooltipProvider timeout={200}>
+      <Ariakit.TooltipAnchor
+        className="button"
+        render={
+          <Ariakit.Select
+            className={cn(
+              "group flex items-center gap-1 outline-offset-0 focus-within:outline-none focus-within:ring-1 disabled:cursor-not-allowed disabled:opacity-50",
+              variantClasses.button,
+              className
+            )}
+            ref={ref}
+            {...props}
+          />
+        }
+      >
+        <div className="flex grow items-center gap-0.5">
+          {icon && <div className="-ml-1 flex-none">{icon}</div>}
+          <div className="truncate">{content}</div>
+        </div>
+        {dropdownIcon === true ? (
+          <ChevronDown
+            className={cn(
+              "size-4 flex-none text-text-dimmed transition group-hover:text-text-bright group-focus:text-text-bright"
+            )}
+          />
+        ) : !dropdownIcon ? null : (
+          dropdownIcon
+        )}
+      </Ariakit.TooltipAnchor>
+      {showTooltip && (
+        <Ariakit.Tooltip
+          disabled={shortcut === undefined}
+          className="z-40 cursor-default rounded border border-charcoal-700 bg-background-bright px-2 py-1.5 text-xs"
+        >
+          <div className="flex items-center gap-2">
+            <span>{tooltipTitle ?? "Open menu"}</span>
+            {shortcut && (
+              <ShortcutKey
+                className={cn("size-4 flex-none")}
+                shortcut={shortcut}
+                variant={"small"}
+              />
+            )}
+          </div>
+        </Ariakit.Tooltip>
+      )}
+    </Ariakit.TooltipProvider>
+  );
+}
+
+export interface SelectProviderProps<TValue extends string | string[]>
+  extends Ariakit.SelectProviderProps<TValue> {}
+export function SelectProvider<TValue extends string | string[]>(
+  props: SelectProviderProps<TValue>
+) {
+  return <Ariakit.SelectProvider {...props} />;
+}
+
+export interface ComboboxProviderProps extends Ariakit.ComboboxProviderProps {}
+export function ComboboxProvider(props: ComboboxProviderProps) {
+  return <Ariakit.ComboboxProvider {...props} />;
+}
+
+function SelectGroupedRenderer<TItem>({
+  items,
+  children,
+  enableItemShortcuts,
+}: {
+  items: Section<TItem>[];
+  children: (
+    items: ItemFromSection<TItem>[],
+    meta: {
+      shortcutsEnabled?: boolean;
+      section?: { title?: string; startIndex: number; count: number };
+    }
+  ) => React.ReactNode;
+  enableItemShortcuts: boolean;
+}) {
+  let count = 0;
+  return (
+    <>
+      {items.map((section, index) => {
+        const previousItem = items.at(index - 1);
+        count += previousItem ? previousItem.items.length : 0;
+        return (
+          <Fragment key={index}>
+            {children(section.items as ItemFromSection<TItem>[], {
+              shortcutsEnabled: enableItemShortcuts,
+              section: {
+                title: section.title,
+                startIndex: count - 1,
+                count: section.items.length,
+              },
+            })}
+          </Fragment>
+        );
+      })}
+    </>
+  );
+}
+
+export interface SelectListProps extends Omit<Ariakit.SelectListProps, "store"> {}
+export function SelectList(props: SelectListProps) {
+  const combobox = Ariakit.useComboboxContext();
+  const Component = combobox ? Ariakit.ComboboxList : Ariakit.SelectList;
+
+  return (
+    <Component
+      {...props}
       className={cn(
-        "relative flex w-full cursor-default select-none items-center rounded-sm py-1.5 pl-2 pr-12 text-sm outline-none transition data-[disabled]:pointer-events-none data-[disabled]:opacity-50 hover:bg-charcoal-750 focus:bg-charcoal-750/50",
+        "overflow-y-auto overscroll-contain outline-none scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600",
+        props.className
+      )}
+    />
+  );
+}
+
+export interface SelectItemProps extends Ariakit.SelectItemProps {
+  icon?: React.ReactNode;
+  checkIcon?: React.ReactNode;
+  shortcut?: ShortcutDefinition;
+}
+
+const selectItemClasses =
+  "group cursor-pointer px-1 pt-1 text-xs text-text-dimmed outline-none last:pb-1";
+
+export function SelectItem({
+  icon,
+  checkIcon = <Ariakit.SelectItemCheck className="size-8 flex-none text-white" />,
+  shortcut,
+  ...props
+}: SelectItemProps) {
+  const combobox = Ariakit.useComboboxContext();
+  const render = combobox ? <Ariakit.ComboboxItem render={props.render} /> : undefined;
+  const ref = React.useRef<HTMLDivElement>(null);
+
+  useShortcutKeys({
+    shortcut: shortcut,
+    action: (e) => {
+      e.preventDefault();
+      e.stopPropagation();
+      if (ref.current) {
+        ref.current.click();
+      }
+    },
+    disabled: props.disabled,
+    enabledOnInputElements: true,
+  });
+
+  return (
+    <Ariakit.SelectItem
+      {...props}
+      render={render}
+      blurOnHoverEnd={false}
+      className={cn(
+        selectItemClasses,
+        "[--padding-block:0.5rem] sm:[--padding-block:0.25rem]",
+        props.className
+      )}
+      ref={ref}
+    >
+      <div className="flex h-7 w-full items-center gap-1 rounded-sm px-2 group-data-[active-item=true]:bg-tertiary">
+        {icon}
+        <div className="grow truncate">{props.children || props.value}</div>
+        {checkIcon}
+        {shortcut && (
+          <ShortcutKey className={cn("size-4 flex-none")} shortcut={shortcut} variant={"small"} />
+        )}
+      </div>
+    </Ariakit.SelectItem>
+  );
+}
+
+export interface SelectLinkItemProps extends Ariakit.SelectItemProps {
+  icon?: React.ReactNode;
+  checkIcon?: React.ReactNode;
+  shortcut?: ShortcutDefinition;
+  to: string;
+}
+
+export function SelectLinkItem({
+  checkIcon = <Ariakit.SelectItemCheck className="size-8 flex-none text-white" />,
+  to,
+  ...props
+}: SelectLinkItemProps) {
+  const render = <Link to={to} className={cn("block", selectItemClasses, props.className)} />;
+
+  return (
+    <SelectItem
+      {...props}
+      render={render}
+      blurOnHoverEnd={false}
+      className={cn(selectItemClasses, props.className)}
+    />
+  );
+}
+
+export interface SelectButtonItemProps extends Omit<Ariakit.SelectItemProps, "onClick"> {
+  icon?: React.ReactNode;
+  checkIcon?: React.ReactNode;
+  shortcut?: ShortcutDefinition;
+  onClick: React.ComponentProps<"button">["onClick"];
+}
+
+export function SelectButtonItem({
+  checkIcon = <Ariakit.SelectItemCheck className="size-8 flex-none text-white" />,
+  onClick,
+  ...props
+}: SelectButtonItemProps) {
+  const render = (
+    <button
+      onClick={onClick}
+      className={cn("block w-full text-left", selectItemClasses, props.className)}
+    />
+  );
+
+  return (
+    <SelectItem
+      {...props}
+      render={render}
+      blurOnHoverEnd={false}
+      className={cn(selectItemClasses, props.className)}
+    />
+  );
+}
+
+export function shortcutFromIndex(
+  index: number,
+  meta: {
+    shortcutsEnabled?: boolean;
+    section?: { startIndex: number };
+  }
+): ShortcutDefinition | undefined {
+  if (!meta.shortcutsEnabled) return;
+
+  let adjustedIndex = index + (meta.section?.startIndex ?? 0);
+
+  if (adjustedIndex > 9) return;
+  if (adjustedIndex === 9) {
+    adjustedIndex = -1;
+  }
+
+  return { key: String(adjustedIndex + 1) };
+}
+
+export interface SelectSeparatorProps extends React.ComponentProps<"div"> {}
+
+export function SelectSeparator(props: SelectSeparatorProps) {
+  return <div {...props} className={cn("h-px bg-charcoal-700", props.className)} />;
+}
+
+export interface SelectGroupProps extends Ariakit.SelectGroupProps {}
+
+export function SelectGroup(props: SelectGroupProps) {
+  return <Ariakit.SelectGroup {...props} />;
+}
+
+export interface SelectGroupLabelProps extends Ariakit.SelectGroupLabelProps {}
+
+export function SelectGroupLabel(props: SelectGroupLabelProps) {
+  return (
+    <Ariakit.SelectGroupLabel
+      {...props}
+      className={cn(
+        "flex h-[1.375rem] items-center border-b border-charcoal-700 bg-charcoal-750 px-2.5 text-xxs uppercase text-text-bright",
+        props.className
+      )}
+    />
+  );
+}
+
+export interface SelectHeadingProps extends Ariakit.SelectHeadingProps {}
+export function SelectHeading({ render, ...props }: SelectHeadingProps) {
+  return (
+    <div className="flex h-[1.375rem] flex-none cursor-default items-center gap-2 border-b border-charcoal-700 bg-charcoal-750 px-2.5 text-xxs uppercase text-text-bright">
+      <Ariakit.SelectHeading render={<>{render}</>} />
+    </div>
+  );
+}
+
+export interface SelectPopoverProps extends Ariakit.SelectPopoverProps {}
+export function SelectPopover({
+  gutter = 5,
+  shift = 0,
+  unmountOnHide = true,
+  className,
+  ...props
+}: SelectPopoverProps) {
+  return (
+    <Ariakit.SelectPopover
+      gutter={gutter}
+      shift={shift}
+      unmountOnHide={unmountOnHide}
+      className={cn(
+        "z-50 flex flex-col overflow-clip rounded border border-charcoal-700 bg-background-bright shadow-md outline-none animate-in fade-in-40",
+        "min-w-[max(180px,calc(var(--popover-anchor-width)+0.5rem))]",
+        "max-w-[min(480px,var(--popover-available-width))]",
+        "max-h-[min(480px,var(--popover-available-height))]",
+        "origin-[var(--popover-transform-origin)]",
         className
       )}
       {...props}
-    >
-      <span className="absolute right-2 flex h-3.5 w-3.5 items-center justify-center">
-        <SelectPrimitive.ItemIndicator>
-          <Check className="h-4 w-4" />
-        </SelectPrimitive.ItemIndicator>
-      </span>
+    />
+  );
+}
 
-      <SelectPrimitive.ItemText>{children}</SelectPrimitive.ItemText>
-    </SelectPrimitive.Item>
-  )
-);
-SelectItem.displayName = SelectPrimitive.Item.displayName;
+export interface SelectLabelProps extends Ariakit.SelectLabelProps {}
+//currently unstyled
+export function SelectLabel(props: SelectLabelProps) {
+  return <Ariakit.SelectLabel {...props} />;
+}
 
-const SelectSeparator = React.forwardRef<
-  React.ElementRef<typeof SelectPrimitive.Separator>,
-  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Separator>
->(({ className, ...props }, ref) => (
-  <SelectPrimitive.Separator
-    ref={ref}
-    className={cn("bg-muted -mx-1 my-1 h-px", className)}
-    {...props}
-  />
-));
-SelectSeparator.displayName = SelectPrimitive.Separator.displayName;
+export interface ComboBoxProps extends Ariakit.ComboboxProps {
+  shortcut?: ShortcutDefinition;
+}
 
-export {
-  Select,
-  SelectContent,
-  SelectGroup,
-  SelectItem,
-  SelectLabel,
-  SelectSeparator,
-  SelectTrigger,
-  SelectValue,
-};
+export function ComboBox({
+  autoSelect = true,
+  placeholder = "Filter options",
+  shortcut,
+  ...props
+}: ComboBoxProps) {
+  return (
+    <div className="flex h-9 w-full flex-none items-center border-b border-grid-dimmed bg-transparent px-3 text-xs text-text-dimmed outline-none">
+      <Ariakit.Combobox
+        autoSelect={autoSelect}
+        render={<input placeholder={placeholder} />}
+        className="flex-1 bg-transparent text-xs text-text-dimmed outline-none"
+        {...props}
+      />
+      {shortcut && (
+        <ShortcutKey className={cn("size-4 flex-none")} shortcut={shortcut} variant={"small"} />
+      )}
+    </div>
+  );
+}

apps/webapp/app/components/primitives/SelectedItemsProvider.tsx

@@ -0,0 +1,146 @@
+"use client";
+
+import { createContext, useCallback, useContext, useReducer } from "react";
+
+type SelectedItemsContext = {
+  selectedItems: Set<string>;
+  select: (items: string | string[]) => void;
+  deselect: (items: string | string[]) => void;
+  toggle: (items: string | string[]) => void;
+  deselectAll: () => void;
+  has: (item: string) => boolean;
+  hasAll: (items: string[]) => boolean;
+};
+
+const SelectedItemsContext = createContext<SelectedItemsContext>({} as SelectedItemsContext);
+
+export function useSelectedItems(enabled = true) {
+  const context = useContext(SelectedItemsContext);
+  if (!context && enabled) {
+    throw new Error("useSelectedItems must be used within a SelectedItemsProvider");
+  }
+
+  return context;
+}
+
+export function SelectedItemsProvider({
+  initialSelectedItems,
+  maxSelectedItemCount,
+  children,
+}: {
+  initialSelectedItems: string[];
+  maxSelectedItemCount?: number;
+  children: React.ReactNode | ((context: SelectedItemsContext) => React.ReactNode);
+}) {
+  const [state, dispatch] = useReducer(selectedItemsReducer, {
+    items: new Set<string>(initialSelectedItems),
+    maxSelectedItemCount,
+  });
+
+  const select = useCallback((items: string | string[]) => {
+    dispatch({ type: "select", items: Array.isArray(items) ? items : [items] });
+  }, []);
+
+  const deselect = useCallback((items: string | string[]) => {
+    dispatch({ type: "deselect", items: Array.isArray(items) ? items : [items] });
+  }, []);
+
+  const toggle = useCallback((items: string | string[]) => {
+    dispatch({ type: "toggle", items: Array.isArray(items) ? items : [items] });
+  }, []);
+
+  const deselectAll = useCallback(() => {
+    dispatch({ type: "deselectAll" });
+  }, []);
+
+  const has = useCallback((item: string) => state.items.has(item), [state]);
+
+  const hasAll = useCallback(
+    (items: string[]) => items.every((item) => state.items.has(item)),
+    [state]
+  );
+
+  return (
+    <SelectedItemsContext.Provider
+      value={{ selectedItems: state.items, select, deselect, toggle, deselectAll, has, hasAll }}
+    >
+      {typeof children === "function"
+        ? children({
+            selectedItems: state.items,
+            select,
+            deselect,
+            toggle,
+            deselectAll,
+            has,
+            hasAll,
+          })
+        : children}
+    </SelectedItemsContext.Provider>
+  );
+}
+
+type SelectItemsAction = {
+  type: "select";
+  items: string[];
+};
+
+type DeSelectItemsAction = {
+  type: "deselect";
+  items: string[];
+};
+
+type DeselectAllItemsAction = {
+  type: "deselectAll";
+};
+
+type ToggleItemsAction = {
+  type: "toggle";
+  items: string[];
+};
+
+type Action = SelectItemsAction | DeSelectItemsAction | ToggleItemsAction | DeselectAllItemsAction;
+
+function selectedItemsReducer(
+  state: { items: Set<string>; maxSelectedItemCount?: number },
+  action: Action
+) {
+  switch (action.type) {
+    case "select":
+      const items = new Set([...state.items, ...action.items]);
+      return { ...state, items: cappedSet(items, state.maxSelectedItemCount) };
+    case "deselect":
+      const newItems = new Set(state.items);
+      action.items.forEach((item) => {
+        newItems.delete(item);
+      });
+      return { ...state, items: cappedSet(newItems, state.maxSelectedItemCount) };
+    case "toggle":
+      let newSet = new Set(state.items);
+      action.items.forEach((item) => {
+        if (newSet.has(item)) {
+          newSet.delete(item);
+        } else {
+          newSet.add(item);
+        }
+      });
+      return { ...state, items: cappedSet(newSet, state.maxSelectedItemCount) };
+    case "deselectAll":
+      return { ...state, items: new Set<string>() };
+    default:
+      return state;
+  }
+}
+
+function cappedSet(set: Set<string>, max?: number) {
+  if (!max) {
+    return set;
+  }
+
+  if (set.size <= max) {
+    return set;
+  }
+
+  console.warn(`Selected items exceeded the maximum count of ${max}.`);
+
+  return new Set([...set].slice(0, max));
+}

apps/webapp/app/components/primitives/SimpleSelect.tsx

@@ -0,0 +1,143 @@
+"use client";
+
+import * as SelectPrimitive from "@radix-ui/react-select";
+import { Check, ChevronDown } from "lucide-react";
+import * as React from "react";
+import { cn } from "~/utils/cn";
+
+const sizes = {
+  "secondary/small":
+    "text-xs h-6 bg-tertiary border border-tertiary group-hover:text-text-bright hover:border-charcoal-600 pr-2 pl-1.5",
+  medium: "text-sm h-8 bg-tertiary border border-tertiary hover:border-charcoal-600 px-2.5",
+  minimal: "text-xs h-6 bg-transparent hover:bg-tertiary pl-1.5 pr-2",
+};
+
+export type SelectProps = {
+  size?: keyof typeof sizes;
+  width?: "content" | "full";
+};
+
+const Select = SelectPrimitive.Root;
+const SelectGroup = SelectPrimitive.Group;
+const SelectValue = SelectPrimitive.Value;
+const SelectTrigger = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Trigger>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Trigger> & SelectProps
+>(({ className, children, width = "content", size = "secondary/small", ...props }, ref) => {
+  const sizeClassName = sizes[size];
+  return (
+    <SelectPrimitive.Trigger
+      ref={ref}
+      className={cn(
+        "ring-offset-background focus-visible:ring-ring group flex items-center justify-between gap-x-1 rounded text-text-dimmed transition placeholder:text-text-dimmed hover:text-text-bright focus-visible:bg-tertiary focus-visible:text-text-bright focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-offset-0 disabled:cursor-not-allowed disabled:opacity-50",
+        width === "full" ? "w-full" : "w-min",
+        sizeClassName,
+        className
+      )}
+      {...props}
+    >
+      {children}
+      <SelectPrimitive.Icon asChild>
+        <ChevronDown
+          className={cn(
+            "size-4 text-text-dimmed transition group-hover:text-text-bright group-focus:text-text-bright"
+          )}
+        />
+      </SelectPrimitive.Icon>
+    </SelectPrimitive.Trigger>
+  );
+});
+SelectTrigger.displayName = SelectPrimitive.Trigger.displayName;
+
+const SelectContent = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Content>
+>(({ className, children, position = "popper", ...props }, ref) => (
+  <SelectPrimitive.Portal>
+    <SelectPrimitive.Content
+      ref={ref}
+      className={cn(
+        "relative z-50 min-w-max overflow-hidden rounded-md border border-charcoal-700 bg-background-dimmed text-text-bright shadow-md animate-in fade-in-40",
+        position === "popper" && "translate-y-1",
+        className
+      )}
+      position={position}
+      {...props}
+    >
+      <SelectPrimitive.Viewport
+        className={cn(
+          "space-y-0.5 px-1 py-1",
+          position === "popper" &&
+            "h-[var(--radix-select-trigger-height)] w-full min-w-[var(--radix-select-trigger-width)]"
+        )}
+      >
+        {children}
+      </SelectPrimitive.Viewport>
+    </SelectPrimitive.Content>
+  </SelectPrimitive.Portal>
+));
+SelectContent.displayName = SelectPrimitive.Content.displayName;
+
+const SelectLabel = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Label>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Label>
+>(({ className, ...props }, ref) => (
+  <SelectPrimitive.Label
+    ref={ref}
+    className={cn(
+      "-ml-1 -mr-1 mb-1 bg-charcoal-900 py-1.5 pl-2 pr-2 font-sans text-xxs font-normal uppercase leading-normal tracking-wider text-text-dimmed first-of-type:-mt-0",
+      className
+    )}
+    {...props}
+  />
+));
+SelectLabel.displayName = SelectPrimitive.Label.displayName;
+
+type SelectItemProps = React.ComponentPropsWithoutRef<typeof SelectPrimitive.Item> & {
+  contentClassName?: string;
+};
+
+const SelectItem = React.forwardRef<React.ElementRef<typeof SelectPrimitive.Item>, SelectItemProps>(
+  ({ className, children, contentClassName, ...props }, ref) => (
+    <SelectPrimitive.Item
+      ref={ref}
+      className={cn(
+        "relative flex w-full cursor-default select-none items-center rounded-sm py-1.5 pl-2 pr-12 text-sm outline-none transition data-[disabled]:pointer-events-none data-[disabled]:opacity-50 hover:bg-charcoal-750 focus:bg-charcoal-750/50",
+        className
+      )}
+      {...props}
+    >
+      <span className="absolute right-2 flex h-3.5 w-3.5 items-center justify-center">
+        <SelectPrimitive.ItemIndicator>
+          <Check className="h-4 w-4" />
+        </SelectPrimitive.ItemIndicator>
+      </span>
+
+      <SelectPrimitive.ItemText>{children}</SelectPrimitive.ItemText>
+    </SelectPrimitive.Item>
+  )
+);
+SelectItem.displayName = SelectPrimitive.Item.displayName;
+
+const SelectSeparator = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Separator>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Separator>
+>(({ className, ...props }, ref) => (
+  <SelectPrimitive.Separator
+    ref={ref}
+    className={cn("bg-muted -mx-1 my-1 h-px", className)}
+    {...props}
+  />
+));
+SelectSeparator.displayName = SelectPrimitive.Separator.displayName;
+
+export {
+  Select,
+  SelectContent,
+  SelectGroup,
+  SelectItem,
+  SelectLabel,
+  SelectSeparator,
+  SelectTrigger,
+  SelectValue,
+};

apps/webapp/app/components/primitives/Table.tsx

@@ -106,7 +106,7 @@ export const TableHeaderCell = forwardRef<HTMLTableCellElement, TableHeaderCellP
         ref={ref}
         scope="col"
         className={cn(
-          "px-4 py-2 align-middle text-xxs font-normal uppercase tracking-wider text-text-dimmed",
+          "px-3 py-2 align-middle text-xxs font-normal uppercase tracking-wider text-text-dimmed",
           alignmentClassName,
           className
         )}
@@ -155,7 +155,7 @@ export const TableCell = forwardRef<HTMLTableCellElement, TableCellProps>(
     }
 
     const flexClasses = cn(
-      "flex w-full whitespace-nowrap px-4 py-3 text-xs text-text-dimmed",
+      "flex w-full whitespace-nowrap px-3 py-3 text-xs text-text-dimmed",
       alignment === "left"
         ? "justify-start text-left"
         : alignment === "center"
@@ -170,7 +170,7 @@ export const TableCell = forwardRef<HTMLTableCellElement, TableCellProps>(
           "text-xs text-charcoal-400",
           to || onClick || hasAction
             ? "cursor-pointer group-hover/table-row:bg-charcoal-900"
-            : "px-4 py-3 align-middle",
+            : "px-3 py-3 align-middle",
           !to && !onClick && alignmentClassName,
           isSticky && stickyStyles,
           className

apps/webapp/app/components/primitives/TooltipPortal.tsx

@@ -0,0 +1,103 @@
+import type { VirtualElement as IVirtualElement } from "@popperjs/core";
+import { ReactNode, useEffect, useState } from "react";
+import { createPortal } from "react-dom";
+import { usePopper } from "react-popper";
+import { useEvent } from "react-use";
+import useLazyRef from "~/hooks/useLazyRef";
+
+// Recharts 3.x will have portal support, but until then we're using this:
+//https://github.com/recharts/recharts/issues/2458#issuecomment-1063463873
+
+export interface PopperPortalProps {
+  active?: boolean;
+  children: ReactNode;
+}
+
+export default function TooltipPortal({ active = true, children }: PopperPortalProps) {
+  const [portalElement, setPortalElement] = useState<HTMLDivElement>();
+  const [popperElement, setPopperElement] = useState<HTMLDivElement | null>();
+  const virtualElementRef = useLazyRef(() => new VirtualElement());
+
+  const { styles, attributes, update } = usePopper(
+    virtualElementRef.current,
+    popperElement,
+    POPPER_OPTIONS
+  );
+
+  useEffect(() => {
+    const el = document.createElement("div");
+    document.body.appendChild(el);
+    setPortalElement(el);
+    return () => el.remove();
+  }, []);
+
+  useEvent("mousemove", ({ clientX: x, clientY: y }) => {
+    virtualElementRef.current?.update(x, y);
+    if (!active) return;
+    update?.();
+  });
+
+  useEffect(() => {
+    if (!active) return;
+    update?.();
+  }, [active, update]);
+
+  if (!portalElement) return null;
+
+  return createPortal(
+    <div
+      ref={setPopperElement}
+      {...attributes.popper}
+      style={{
+        ...styles.popper,
+        zIndex: 1000,
+        display: active ? "block" : "none",
+      }}
+    >
+      {children}
+    </div>,
+    portalElement
+  );
+}
+
+class VirtualElement implements IVirtualElement {
+  private rect = {
+    width: 0,
+    height: 0,
+    top: 0,
+    right: 0,
+    bottom: 0,
+    left: 0,
+    x: 0,
+    y: 0,
+    toJSON() {
+      return this;
+    },
+  };
+
+  update(x: number, y: number) {
+    this.rect.y = y;
+    this.rect.top = y;
+    this.rect.bottom = y;
+
+    this.rect.x = x;
+    this.rect.left = x;
+    this.rect.right = x;
+  }
+
+  getBoundingClientRect(): DOMRect {
+    return this.rect;
+  }
+}
+
+const POPPER_OPTIONS: Parameters<typeof usePopper>[2] = {
+  placement: "right-start",
+  modifiers: [
+    {
+      name: "offset",
+      options: {
+        offset: [8, 8],
+      },
+    },
+  ],
+};

apps/webapp/app/components/primitives/TreeView/TreeView.tsx

@@ -1,10 +1,9 @@
 import { VirtualItem, Virtualizer, useVirtualizer } from "@tanstack/react-virtual";
 import { motion } from "framer-motion";
 import { MutableRefObject, RefObject, useCallback, useEffect, useReducer, useRef } from "react";
-import { UnmountClosed } from "react-collapse";
 import { cn } from "~/utils/cn";
 import { NodeState, NodesState, reducer } from "./reducer";
-import { applyFilterToState, concreteStateFromInput, selectedIdFromState } from "./utils";
+import { concreteStateFromInput, selectedIdFromState } from "./utils";
 
 export type TreeViewProps<TData> = {
   tree: FlatTree<TData>;
@@ -104,23 +103,22 @@ export function TreeView<TData>({
             if (!node) return null;
             const state = nodes[node.id];
             if (!state) return null;
+            if (!state.visible) return null;
             return (
               <div
                 key={node.id}
                 data-index={virtualItem.index}
                 ref={virtualizer.measureElement}
-                className="overflow-clip [&_.ReactCollapse--collapse]:transition-all"
+                className="overflow-clip"
                 {...getNodeProps(node.id)}
               >
-                <UnmountClosed key={node.id} isOpened={state.visible}>
-                  {renderNode({
-                    node,
-                    state,
-                    index: virtualItem.index,
-                    virtualizer: virtualizer,
-                    virtualItem,
-                  })}
-                </UnmountClosed>
+                {renderNode({
+                  node,
+                  state,
+                  index: virtualItem.index,
+                  virtualizer: virtualizer,
+                  virtualItem,
+                })}
               </div>
             );
           })}
@@ -130,19 +128,23 @@ export function TreeView<TData>({
   );
 }
 
-type TreeStateHookProps<TData> = {
+export type Filter<TData, TFilterValue> = {
+  value?: TFilterValue;
+  fn: (value: TFilterValue, node: FlatTreeItem<TData>) => boolean;
+};
+
+type TreeStateHookProps<TData, TFilterValue> = {
   tree: FlatTree<TData>;
   selectedId?: string;
   collapsedIds?: string[];
   onSelectedIdChanged?: (selectedId: string | undefined) => void;
-  onCollapsedIdsChanged?: (collapsedIds: string[]) => void;
   estimatedRowHeight: (params: {
     node: FlatTreeItem<TData>;
     state: NodeState;
     index: number;
   }) => number;
   parentRef: RefObject<any>;
-  filter?: (node: FlatTreeItem<TData>) => boolean;
+  filter?: Filter<TData, TFilterValue>;
 };
 
 //this is so Framer Motion can be used to render the components
@@ -178,24 +180,24 @@ export type UseTreeStateOutput = {
   scrollToNode: (id: string) => void;
 };
 
-export function useTree<TData>({
+export function useTree<TData, TFilterValue>({
   tree,
   selectedId,
   collapsedIds,
   onSelectedIdChanged,
-  onCollapsedIdsChanged,
   parentRef,
   estimatedRowHeight,
   filter,
-}: TreeStateHookProps<TData>): UseTreeStateOutput {
+}: TreeStateHookProps<TData, TFilterValue>): UseTreeStateOutput {
   const previousNodeCount = useRef(tree.length);
   const previousSelectedId = useRef<string | undefined>(selectedId);
 
   const [state, dispatch] = useReducer(
     reducer,
-    concreteStateFromInput({ tree, selectedId, collapsedIds })
+    concreteStateFromInput({ tree, selectedId, collapsedIds, filter })
   );
 
+  //fire onSelectedIdChanged()
   useEffect(() => {
     const selectedId = selectedIdFromState(state.nodes);
     if (selectedId !== previousSelectedId.current) {
@@ -204,12 +206,7 @@ export function useTree<TData>({
     }
   }, [state.changes.selectedId]);
 
-  useEffect(() => {
-    if (state.changes.collapsedIds) {
-      onCollapsedIdsChanged?.(state.changes.collapsedIds);
-    }
-  }, [state.changes.collapsedIds]);
-
+  //update tree when the number of nodes changes
   useEffect(() => {
     if (tree.length !== previousNodeCount.current) {
       previousNodeCount.current = tree.length;
@@ -217,9 +214,25 @@ export function useTree<TData>({
     }
   }, [previousNodeCount.current, tree.length]);
 
+  //update the filter, if it's changed
+  const previousFilter = useRef(filter);
+  useEffect(() => {
+    //check if the value (not reference) of the filter is the same
+    const previousValue = previousFilter.current
+      ? JSON.stringify(previousFilter.current.value)
+      : undefined;
+    const newValue = filter ? JSON.stringify(filter.value) : undefined;
+
+    previousFilter.current = filter;
+
+    if (previousValue !== newValue) {
+      dispatch({ type: "UPDATE_FILTER", payload: { filter } });
+    }
+  }, [filter?.value]);
+
   const virtualizer = useVirtualizer({
-    count: tree.length,
-    getItemKey: (index) => tree[index].id,
+    count: state.visibleNodeIds.length,
+    getItemKey: (index) => state.visibleNodeIds[index],
     getScrollElement: () => parentRef.current,
     estimateSize: (index: number) => {
       return estimatedRowHeight({
@@ -233,7 +246,7 @@ export function useTree<TData>({
 
   const scrollToNodeFn = useCallback(
     (id: string) => {
-      const itemIndex = tree.findIndex((node) => node.id === id);
+      const itemIndex = state.visibleNodeIds.findIndex((n) => n === id);
 
       if (itemIndex !== -1) {
         virtualizer.scrollToIndex(itemIndex, { align: "auto" });
@@ -269,21 +282,21 @@ export function useTree<TData>({
 
   const expandNode = useCallback(
     (id: string, scrollToNode = true) => {
-      dispatch({ type: "EXPAND_NODE", payload: { id, tree, scrollToNode, scrollToNodeFn } });
+      dispatch({ type: "EXPAND_NODE", payload: { id, scrollToNode, scrollToNodeFn } });
     },
     [state]
   );
 
   const collapseNode = useCallback(
     (id: string) => {
-      dispatch({ type: "COLLAPSE_NODE", payload: { id, tree } });
+      dispatch({ type: "COLLAPSE_NODE", payload: { id } });
     },
     [state]
   );
 
   const toggleExpandNode = useCallback(
     (id: string, scrollToNode = true) => {
-      dispatch({ type: "TOGGLE_EXPAND_NODE", payload: { id, tree, scrollToNode, scrollToNodeFn } });
+      dispatch({ type: "TOGGLE_EXPAND_NODE", payload: { id, scrollToNode, scrollToNodeFn } });
     },
     [state]
   );
@@ -292,7 +305,7 @@ export function useTree<TData>({
     (scrollToNode = true) => {
       dispatch({
         type: "SELECT_FIRST_VISIBLE_NODE",
-        payload: { tree, scrollToNode, scrollToNodeFn },
+        payload: { scrollToNode, scrollToNodeFn },
       });
     },
     [tree, state]
@@ -302,7 +315,7 @@ export function useTree<TData>({
     (scrollToNode = true) => {
       dispatch({
         type: "SELECT_LAST_VISIBLE_NODE",
-        payload: { tree, scrollToNode, scrollToNodeFn },
+        payload: { scrollToNode, scrollToNodeFn },
       });
     },
     [tree, state]
@@ -312,7 +325,7 @@ export function useTree<TData>({
     (scrollToNode = true) => {
       dispatch({
         type: "SELECT_NEXT_VISIBLE_NODE",
-        payload: { tree, scrollToNode, scrollToNodeFn },
+        payload: { scrollToNode, scrollToNodeFn },
       });
     },
     [state]
@@ -322,7 +335,7 @@ export function useTree<TData>({
     (scrollToNode = true) => {
       dispatch({
         type: "SELECT_PREVIOUS_VISIBLE_NODE",
-        payload: { tree, scrollToNode, scrollToNodeFn },
+        payload: { scrollToNode, scrollToNodeFn },
       });
     },
     [state]
@@ -332,7 +345,7 @@ export function useTree<TData>({
     (scrollToNode = true) => {
       dispatch({
         type: "SELECT_PARENT_NODE",
-        payload: { tree, scrollToNode, scrollToNodeFn },
+        payload: { scrollToNode, scrollToNodeFn },
       });
     },
     [state]
@@ -340,35 +353,35 @@ export function useTree<TData>({
 
   const expandAllBelowDepth = useCallback(
     (depth: number) => {
-      dispatch({ type: "EXPAND_ALL_BELOW_DEPTH", payload: { tree, depth } });
+      dispatch({ type: "EXPAND_ALL_BELOW_DEPTH", payload: { depth } });
     },
     [state]
   );
 
   const collapseAllBelowDepth = useCallback(
     (depth: number) => {
-      dispatch({ type: "COLLAPSE_ALL_BELOW_DEPTH", payload: { tree, depth } });
+      dispatch({ type: "COLLAPSE_ALL_BELOW_DEPTH", payload: { depth } });
     },
     [state]
   );
 
   const expandLevel = useCallback(
     (level: number) => {
-      dispatch({ type: "EXPAND_LEVEL", payload: { tree, level } });
+      dispatch({ type: "EXPAND_LEVEL", payload: { level } });
     },
     [state]
   );
 
   const collapseLevel = useCallback(
     (level: number) => {
-      dispatch({ type: "COLLAPSE_LEVEL", payload: { tree, level } });
+      dispatch({ type: "COLLAPSE_LEVEL", payload: { level } });
     },
     [state]
   );
 
   const toggleExpandLevel = useCallback(
     (level: number) => {
-      dispatch({ type: "TOGGLE_EXPAND_LEVEL", payload: { tree, level } });
+      dispatch({ type: "TOGGLE_EXPAND_LEVEL", payload: { level } });
     },
     [state]
   );
@@ -480,7 +493,7 @@ export function useTree<TData>({
 
   return {
     selected: selectedIdFromState(state.nodes),
-    nodes: filter ? applyFilterToState(tree, state.nodes, filter) : state.nodes,
+    nodes: state.nodes,
     getTreeProps,
     getNodeProps,
     selectNode,

apps/webapp/app/components/primitives/TreeView/reducer.ts

@@ -1,5 +1,7 @@
-import { FlatTree } from "./TreeView";
+import assertNever from "assert-never";
+import { Filter, FlatTree } from "./TreeView";
 import {
+  applyFilterToState,
   applyVisibility,
   collapsedIdsFromState,
   concreteStateFromInput,
@@ -18,12 +20,15 @@ export type NodeState = {
 
 export type Changes = {
   selectedId: string | undefined;
-  collapsedIds: string[] | undefined;
 };
 
 export type TreeState = {
+  tree: FlatTree<any>;
   nodes: NodesState;
+  filteredNodes: NodesState;
   changes: Changes;
+  filter: Filter<any, any> | undefined;
+  visibleNodeIds: string[];
 };
 
 export type NodesState = Record<string, NodeState>;
@@ -71,7 +76,6 @@ type ExpandNodeAction = {
   type: "EXPAND_NODE";
   payload: {
     id: string;
-    tree: FlatTree<any>;
   } & WithScrollToNode;
 };
 
@@ -79,7 +83,6 @@ type CollapseNodeAction = {
   type: "COLLAPSE_NODE";
   payload: {
     id: string;
-    tree: FlatTree<any>;
   };
 };
 
@@ -87,7 +90,6 @@ type ToggleExpandNodeAction = {
   type: "TOGGLE_EXPAND_NODE";
   payload: {
     id: string;
-    tree: FlatTree<any>;
   } & WithScrollToNode;
 };
 
@@ -95,7 +97,6 @@ type ExpandAllBelowDepthAction = {
   type: "EXPAND_ALL_BELOW_DEPTH";
   payload: {
     depth: number;
-    tree: FlatTree<any>;
   };
 };
 
@@ -103,7 +104,6 @@ type CollapseAllBelowDepthAction = {
   type: "COLLAPSE_ALL_BELOW_DEPTH";
   payload: {
     depth: number;
-    tree: FlatTree<any>;
   };
 };
 
@@ -111,7 +111,6 @@ type ExpandLevelAction = {
   type: "EXPAND_LEVEL";
   payload: {
     level: number;
-    tree: FlatTree<any>;
   };
 };
 
@@ -119,7 +118,6 @@ type CollapseLevelAction = {
   type: "COLLAPSE_LEVEL";
   payload: {
     level: number;
-    tree: FlatTree<any>;
   };
 };
 
@@ -127,43 +125,39 @@ type ToggleExpandLevelAction = {
   type: "TOGGLE_EXPAND_LEVEL";
   payload: {
     level: number;
-    tree: FlatTree<any>;
   };
 };
 
 type SelectFirstVisibleNodeAction = {
   type: "SELECT_FIRST_VISIBLE_NODE";
-  payload: {
-    tree: FlatTree<any>;
-  } & WithScrollToNode;
+  payload: {} & WithScrollToNode;
 };
 
 type SelectLastVisibleNodeAction = {
   type: "SELECT_LAST_VISIBLE_NODE";
-  payload: {
-    tree: FlatTree<any>;
-  } & WithScrollToNode;
+  payload: {} & WithScrollToNode;
 };
 
 type SelectNextVisibleNodeAction = {
   type: "SELECT_NEXT_VISIBLE_NODE";
-  payload: {
-    tree: FlatTree<any>;
-  } & WithScrollToNode;
+  payload: {} & WithScrollToNode;
 };
 
 type SelectPreviousVisibleNodeAction = {
   type: "SELECT_PREVIOUS_VISIBLE_NODE";
-  payload: {
-    tree: FlatTree<any>;
-  } & WithScrollToNode;
+  payload: {} & WithScrollToNode;
 };
 
 type SelectParentNodeAction = {
   type: "SELECT_PARENT_NODE";
+  payload: {} & WithScrollToNode;
+};
+
+type UpdateFilterAction = {
+  type: "UPDATE_FILTER";
   payload: {
-    tree: FlatTree<any>;
-  } & WithScrollToNode;
+    filter: Filter<any, any> | undefined;
+  };
 };
 
 export type Action =
@@ -184,7 +178,8 @@ export type Action =
   | SelectLastVisibleNodeAction
   | SelectNextVisibleNodeAction
   | SelectPreviousVisibleNodeAction
-  | SelectParentNodeAction;
+  | SelectParentNodeAction
+  | UpdateFilterAction;
 
 export function reducer(state: TreeState, action: Action): TreeState {
   switch (action.type) {
@@ -204,7 +199,12 @@ export function reducer(state: TreeState, action: Action): TreeState {
         action.payload.scrollToNodeFn(action.payload.id);
       }
 
-      return { nodes: newNodes, changes: generateChanges(state.nodes, newNodes) };
+      return applyFilterToState({
+        ...state,
+        tree: state.tree,
+        nodes: newNodes,
+        changes: generateChanges(state.nodes, newNodes),
+      });
     }
     case "DESELECT_NODE": {
       const nodes = {
@@ -212,28 +212,36 @@ export function reducer(state: TreeState, action: Action): TreeState {
         [action.payload.id]: { ...state.nodes[action.payload.id], selected: false },
       };
 
-      return { nodes, changes: generateChanges(state.nodes, nodes) };
+      return applyFilterToState({
+        ...state,
+        nodes,
+        changes: generateChanges(state.nodes, nodes),
+      });
     }
     case "DESELECT_ALL_NODES": {
       const nodes = Object.fromEntries(
         Object.entries(state.nodes).map(([key, value]) => [key, { ...value, selected: false }])
       );
-      return { nodes, changes: generateChanges(state.nodes, nodes) };
+      return applyFilterToState({
+        ...state,
+        nodes,
+        changes: generateChanges(state.nodes, nodes),
+      });
     }
     case "TOGGLE_NODE_SELECTION": {
       const currentlySelected = state.nodes[action.payload.id]?.selected ?? false;
       if (currentlySelected) {
         return reducer(state, { type: "DESELECT_NODE", payload: { id: action.payload.id } });
-      } else {
-        return reducer(state, {
-          type: "SELECT_NODE",
-          payload: {
-            id: action.payload.id,
-            scrollToNode: action.payload.scrollToNode,
-            scrollToNodeFn: action.payload.scrollToNodeFn,
-          },
-        });
       }
+
+      return reducer(state, {
+        type: "SELECT_NODE",
+        payload: {
+          id: action.payload.id,
+          scrollToNode: action.payload.scrollToNode,
+          scrollToNodeFn: action.payload.scrollToNodeFn,
+        },
+      });
     }
     case "EXPAND_NODE": {
       const newNodes = {
@@ -245,37 +253,44 @@ export function reducer(state: TreeState, action: Action): TreeState {
         action.payload.scrollToNodeFn(action.payload.id);
       }
 
-      const visibleNodes = applyVisibility(action.payload.tree, newNodes);
-      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+      const visibleNodes = applyVisibility(state.tree, newNodes);
+      return applyFilterToState({
+        ...state,
+        nodes: visibleNodes,
+        changes: generateChanges(state.nodes, visibleNodes),
+      });
     }
     case "COLLAPSE_NODE": {
-      const visibleNodes = applyVisibility(action.payload.tree, {
+      const visibleNodes = applyVisibility(state.tree, {
         ...state.nodes,
         [action.payload.id]: { ...state.nodes[action.payload.id], expanded: false },
       });
-      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+      return applyFilterToState({
+        ...state,
+        nodes: visibleNodes,
+        changes: generateChanges(state.nodes, visibleNodes),
+      });
     }
     case "TOGGLE_EXPAND_NODE": {
       const currentlyExpanded = state.nodes[action.payload.id]?.expanded ?? true;
       if (currentlyExpanded) {
         return reducer(state, {
           type: "COLLAPSE_NODE",
-          payload: { id: action.payload.id, tree: action.payload.tree },
-        });
-      } else {
-        return reducer(state, {
-          type: "EXPAND_NODE",
-          payload: {
-            id: action.payload.id,
-            tree: action.payload.tree,
-            scrollToNode: action.payload.scrollToNode,
-            scrollToNodeFn: action.payload.scrollToNodeFn,
-          },
+          payload: { id: action.payload.id },
         });
       }
+
+      return reducer(state, {
+        type: "EXPAND_NODE",
+        payload: {
+          id: action.payload.id,
+          scrollToNode: action.payload.scrollToNode,
+          scrollToNodeFn: action.payload.scrollToNodeFn,
+        },
+      });
     }
     case "EXPAND_ALL_BELOW_DEPTH": {
-      const nodesToExpand = action.payload.tree.filter(
+      const nodesToExpand = state.tree.filter(
         (n) => n.level >= action.payload.depth && n.hasChildren
       );
 
@@ -289,11 +304,15 @@ export function reducer(state: TreeState, action: Action): TreeState {
         ])
       );
 
-      const visibleNodes = applyVisibility(action.payload.tree, newNodes);
-      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+      const visibleNodes = applyVisibility(state.tree, newNodes);
+      return applyFilterToState({
+        ...state,
+        nodes: visibleNodes,
+        changes: generateChanges(state.nodes, visibleNodes),
+      });
     }
     case "COLLAPSE_ALL_BELOW_DEPTH": {
-      const nodesToCollapse = action.payload.tree.filter(
+      const nodesToCollapse = state.tree.filter(
         (n) => n.level >= action.payload.depth && n.hasChildren
       );
 
@@ -307,11 +326,15 @@ export function reducer(state: TreeState, action: Action): TreeState {
         ])
       );
 
-      const visibleNodes = applyVisibility(action.payload.tree, newNodes);
-      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+      const visibleNodes = applyVisibility(state.tree, newNodes);
+      return applyFilterToState({
+        ...state,
+        nodes: visibleNodes,
+        changes: generateChanges(state.nodes, visibleNodes),
+      });
     }
     case "EXPAND_LEVEL": {
-      const nodesToExpand = action.payload.tree.filter(
+      const nodesToExpand = state.tree.filter(
         (n) => n.level <= action.payload.level && n.hasChildren
       );
 
@@ -325,11 +348,15 @@ export function reducer(state: TreeState, action: Action): TreeState {
         ])
       );
 
-      const visibleNodes = applyVisibility(action.payload.tree, newNodes);
-      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+      const visibleNodes = applyVisibility(state.tree, newNodes);
+      return applyFilterToState({
+        ...state,
+        nodes: visibleNodes,
+        changes: generateChanges(state.nodes, visibleNodes),
+      });
     }
     case "COLLAPSE_LEVEL": {
-      const nodesToCollapse = action.payload.tree.filter(
+      const nodesToCollapse = state.tree.filter(
         (n) => n.level === action.payload.level && n.hasChildren
       );
 
@@ -343,13 +370,17 @@ export function reducer(state: TreeState, action: Action): TreeState {
         ])
       );
 
-      const visibleNodes = applyVisibility(action.payload.tree, newNodes);
-      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+      const visibleNodes = applyVisibility(state.tree, newNodes);
+      return applyFilterToState({
+        ...state,
+        nodes: visibleNodes,
+        changes: generateChanges(state.nodes, visibleNodes),
+      });
     }
     case "TOGGLE_EXPAND_LEVEL": {
       //first get the first item at that level in the tree. If it is expanded, collapse all nodes at that level
       //if it is collapsed, expand all nodes at that level
-      const nodesAtLevel = action.payload.tree.filter(
+      const nodesAtLevel = state.tree.filter(
         (n) => n.level === action.payload.level && n.hasChildren
       );
       const firstNode = nodesAtLevel[0];
@@ -364,21 +395,19 @@ export function reducer(state: TreeState, action: Action): TreeState {
           type: "COLLAPSE_LEVEL",
           payload: {
             level: action.payload.level,
-            tree: action.payload.tree,
-          },
-        });
-      } else {
-        return reducer(state, {
-          type: "EXPAND_LEVEL",
-          payload: {
-            level: action.payload.level,
-            tree: action.payload.tree,
           },
         });
       }
+
+      return reducer(state, {
+        type: "EXPAND_LEVEL",
+        payload: {
+          level: action.payload.level,
+        },
+      });
     }
     case "SELECT_FIRST_VISIBLE_NODE": {
-      const node = firstVisibleNode(action.payload.tree, state.nodes);
+      const node = firstVisibleNode(state.tree, state.filteredNodes);
       if (node) {
         return reducer(state, {
           type: "SELECT_NODE",
@@ -389,9 +418,11 @@ export function reducer(state: TreeState, action: Action): TreeState {
           },
         });
       }
+
+      return state;
     }
     case "SELECT_LAST_VISIBLE_NODE": {
-      const node = lastVisibleNode(action.payload.tree, state.nodes);
+      const node = lastVisibleNode(state.tree, state.filteredNodes);
       if (node) {
         return reducer(state, {
           type: "SELECT_NODE",
@@ -402,6 +433,8 @@ export function reducer(state: TreeState, action: Action): TreeState {
           },
         });
       }
+
+      return state;
     }
     case "SELECT_NEXT_VISIBLE_NODE": {
       const selected = selectedIdFromState(state.nodes);
@@ -409,14 +442,13 @@ export function reducer(state: TreeState, action: Action): TreeState {
         return reducer(state, {
           type: "SELECT_FIRST_VISIBLE_NODE",
           payload: {
-            tree: action.payload.tree,
             scrollToNode: action.payload.scrollToNode,
             scrollToNodeFn: action.payload.scrollToNodeFn,
           },
         });
       }
 
-      const visible = visibleNodes(action.payload.tree, state.nodes);
+      const visible = visibleNodes(state.tree, state.filteredNodes);
       const selectedIndex = visible.findIndex((node) => node.id === selected);
       const nextNode = visible[selectedIndex + 1];
       if (nextNode) {
@@ -429,6 +461,8 @@ export function reducer(state: TreeState, action: Action): TreeState {
           },
         });
       }
+
+      return state;
     }
     case "SELECT_PREVIOUS_VISIBLE_NODE": {
       const selected = selectedIdFromState(state.nodes);
@@ -437,16 +471,15 @@ export function reducer(state: TreeState, action: Action): TreeState {
         return reducer(state, {
           type: "SELECT_FIRST_VISIBLE_NODE",
           payload: {
-            tree: action.payload.tree,
             scrollToNode: action.payload.scrollToNode,
             scrollToNodeFn: action.payload.scrollToNodeFn,
           },
         });
       }
 
-      const visible = visibleNodes(action.payload.tree, state.nodes);
+      const visible = visibleNodes(state.tree, state.filteredNodes);
       const selectedIndex = visible.findIndex((node) => node.id === selected);
-      const previousNode = visible[selectedIndex - 1];
+      const previousNode = visible[Math.max(0, selectedIndex - 1)];
       if (previousNode) {
         return reducer(state, {
           type: "SELECT_NODE",
@@ -467,19 +500,18 @@ export function reducer(state: TreeState, action: Action): TreeState {
         return reducer(state, {
           type: "SELECT_FIRST_VISIBLE_NODE",
           payload: {
-            tree: action.payload.tree,
             scrollToNode: action.payload.scrollToNode,
             scrollToNodeFn: action.payload.scrollToNodeFn,
           },
         });
       }
 
-      const selectedNode = action.payload.tree.find((node) => node.id === selected);
+      const selectedNode = state.tree.find((node) => node.id === selected);
       if (!selectedNode) {
         return state;
       }
 
-      const parentNode = action.payload.tree.find((node) => node.id === selectedNode.parentId);
+      const parentNode = state.tree.find((node) => node.id === selectedNode.parentId);
       if (parentNode) {
         return reducer(state, {
           type: "SELECT_NODE",
@@ -498,12 +530,23 @@ export function reducer(state: TreeState, action: Action): TreeState {
       const selectedId = selectedIdFromState(state.nodes);
       const collapsedIds = collapsedIdsFromState(state.nodes);
       const newState = concreteStateFromInput({
+        ...state,
         tree: action.payload.tree,
         selectedId,
         collapsedIds,
       });
       return newState;
     }
+    case "UPDATE_FILTER": {
+      const newState = applyFilterToState({
+        ...state,
+        filter: action.payload.filter,
+      });
+      return newState;
+    }
+    default: {
+      assertNever(action);
+    }
   }
 
   throw new Error(`Unhandled action type: ${(action as any).type}`);

apps/webapp/app/components/primitives/TreeView/utils.ts

@@ -1,4 +1,4 @@
-import { FlatTree, FlatTreeItem } from "./TreeView";
+import { Filter, FlatTree, FlatTreeItem } from "./TreeView";
 import { Changes, NodeState, NodesState, TreeState } from "./reducer";
 
 type PartialNodeState = Record<string, Partial<NodeState>>;
@@ -8,10 +8,12 @@ const defaultExpanded = true;
 
 export function concreteStateFromInput({
   tree,
+  filter,
   selectedId,
   collapsedIds,
 }: {
   tree: FlatTree<any>;
+  filter: Filter<any, any> | undefined;
   selectedId: string | undefined;
   collapsedIds: string[] | undefined;
 }): TreeState {
@@ -35,10 +37,15 @@ export function concreteStateFromInput({
       }
     }
   }
+  const nodes = concreteStateFromPartialState(tree, state);
 
   return {
-    nodes: concreteStateFromPartialState(tree, state),
-    changes: { selectedId, collapsedIds: [] },
+    tree,
+    nodes,
+    changes: { selectedId },
+    filter,
+    filteredNodes: nodes,
+    visibleNodeIds: visibleNodes(tree, nodes).map((node) => node.id),
   };
 }
 
@@ -82,28 +89,48 @@ export function selectedIdFromState(state: NodesState): string | undefined {
   return selected?.[0];
 }
 
-export function applyFilterToState<TData>(
-  tree: FlatTree<TData>,
-  inputNodes: NodesState,
-  filter: (node: FlatTreeItem<TData>) => boolean
-): NodesState {
+export function applyFilterToState<TData>({
+  tree,
+  nodes,
+  filter,
+  visibleNodeIds,
+  changes,
+}: TreeState): TreeState {
+  if (!filter || !filter.value) {
+    return {
+      tree,
+      nodes,
+      filteredNodes: nodes,
+      changes,
+      filter,
+      visibleNodeIds: visibleNodes(tree, nodes).map((node) => node.id),
+    };
+  }
+
   //we need to do two passes, first collect all the nodes that are results
   const newFilteredOut = new Set<string>();
   for (const node of tree) {
-    if (!filter(node)) {
+    if (!filter.fn(filter.value, node)) {
       newFilteredOut.add(node.id);
     }
   }
 
   //nothing is filtered out
   if (newFilteredOut.size === 0) {
-    return inputNodes;
+    return {
+      tree,
+      nodes,
+      filteredNodes: nodes,
+      changes,
+      filter,
+      visibleNodeIds: visibleNodes(tree, nodes).map((node) => node.id),
+    };
   }
 
   //copy of nodes
-  const nodes = { ...inputNodes };
+  const filteredNodes = { ...nodes };
 
-  const selected = selectedIdFromState(nodes);
+  const selected = selectedIdFromState(filteredNodes);
 
   const visible = new Set<string>();
   const expanded = new Set<string>();
@@ -148,28 +175,35 @@ export function applyFilterToState<TData>(
 
   //now set the visibility and expanded state
   for (const id of hidden) {
-    nodes[id] = { ...nodes[id], visible: false };
+    filteredNodes[id] = { ...filteredNodes[id], visible: false };
   }
   for (const id of visible) {
-    nodes[id] = { ...nodes[id], visible: true };
+    filteredNodes[id] = { ...filteredNodes[id], visible: true };
   }
 
   for (const id of collapsed) {
-    nodes[id] = { ...nodes[id], expanded: false };
+    filteredNodes[id] = { ...filteredNodes[id], expanded: false };
   }
   for (const id of expanded) {
-    nodes[id] = { ...nodes[id], expanded: true };
+    filteredNodes[id] = { ...filteredNodes[id], expanded: true };
   }
 
   if (selected) {
     if (visible.has(selected)) {
-      nodes[selected] = { ...nodes[selected], selected: true };
+      filteredNodes[selected] = { ...filteredNodes[selected], selected: true };
     } else {
-      nodes[selected] = { ...nodes[selected], selected: false };
+      filteredNodes[selected] = { ...filteredNodes[selected], selected: false };
     }
   }
 
-  return nodes;
+  return {
+    tree,
+    nodes,
+    filteredNodes,
+    changes,
+    filter,
+    visibleNodeIds: visibleNodes(tree, filteredNodes).map((node) => node.id),
+  };
 }
 
 export function visibleNodes(tree: FlatTree<any>, nodes: NodesState) {
@@ -215,6 +249,5 @@ export function generateChanges(a: NodesState, b: NodesState): Changes {
 
   return {
     selectedId: selectedIdA !== selectedIdB ? selectedIdB : undefined,
-    collapsedIds: collapsedChanges.length > 0 ? collapsedChanges : undefined,
   };
 }

apps/webapp/app/components/runs/RunFilters.tsx

@@ -20,7 +20,7 @@ import {
   SelectItem,
   SelectTrigger,
   SelectValue,
-} from "../primitives/Select";
+} from "../primitives/SimpleSelect";
 import { Spinner } from "../primitives/Spinner";
 import {
   FilterableEnvironment,

apps/webapp/app/components/runs/v3/BulkAction.tsx

@@ -0,0 +1,73 @@
+import { ArrowPathIcon, NoSymbolIcon } from "@heroicons/react/20/solid";
+import { BulkActionType } from "@trigger.dev/database";
+import assertNever from "assert-never";
+import { cn } from "~/utils/cn";
+
+export function BulkActionStatusCombo({
+  type,
+  className,
+  iconClassName,
+}: {
+  type: BulkActionType;
+  className?: string;
+  iconClassName?: string;
+}) {
+  return (
+    <span className={cn("flex items-center gap-1", className)}>
+      <BulkActionIcon type={type} className={cn("h-4 w-4", iconClassName)} />
+      <BulkActionLabel type={type} />
+    </span>
+  );
+}
+
+export function BulkActionLabel({ type }: { type: BulkActionType }) {
+  return <span className={bulkActionClassName(type)}>{bulkActionTitle(type)}</span>;
+}
+
+export function BulkActionIcon({ type, className }: { type: BulkActionType; className: string }) {
+  switch (type) {
+    case "REPLAY":
+      return <ArrowPathIcon className={cn(bulkActionClassName(type), className)} />;
+    case "CANCEL":
+      return <NoSymbolIcon className={cn(bulkActionClassName(type), className)} />;
+    default: {
+      assertNever(type);
+    }
+  }
+}
+
+export function bulkActionClassName(type: BulkActionType): string {
+  switch (type) {
+    case "REPLAY":
+      return "text-indigo-500";
+    case "CANCEL":
+      return "text-rose-500";
+    default: {
+      assertNever(type);
+    }
+  }
+}
+
+export function bulkActionTitle(type: BulkActionType): string {
+  switch (type) {
+    case "REPLAY":
+      return "Replay";
+    case "CANCEL":
+      return "Cancel";
+    default: {
+      assertNever(type);
+    }
+  }
+}
+
+export function bulkActionVerb(type: BulkActionType): string {
+  switch (type) {
+    case "REPLAY":
+      return "Replaying";
+    case "CANCEL":
+      return "Canceling";
+    default: {
+      assertNever(type);
+    }
+  }
+}

apps/webapp/app/components/runs/v3/DeploymentError.tsx

@@ -20,6 +20,17 @@ export function DeploymentError({ errorData }: DeploymentErrorProps) {
           maxLines={20}
         />
       )}
+      {errorData.stderr && (
+        <>
+          <DeploymentErrorHeader title="Error logs:" />
+          <CodeBlock
+            showCopyButton={false}
+            showLineNumbers={false}
+            code={errorData.stderr}
+            maxLines={20}
+          />
+        </>
+      )}
     </div>
   );
 }

apps/webapp/app/components/runs/v3/EnabledStatus.tsx

@@ -1,18 +1,31 @@
 import { BoltSlashIcon, CheckCircleIcon } from "@heroicons/react/20/solid";
 
-export function EnabledStatus({ enabled }: { enabled: boolean }) {
+type EnabledStatusProps = {
+  enabled: boolean;
+  enabledIcon?: React.ComponentType<any>;
+  disabledIcon?: React.ComponentType<any>;
+};
+
+export function EnabledStatus({
+  enabled,
+  enabledIcon = CheckCircleIcon,
+  disabledIcon = BoltSlashIcon,
+}: EnabledStatusProps) {
+  const EnabledIcon = enabledIcon;
+  const DisabledIcon = disabledIcon;
+
   switch (enabled) {
     case true:
       return (
         <div className="flex items-center gap-1 text-xs text-success">
-          <CheckCircleIcon className="h-4 w-4" />
+          <EnabledIcon className="size-4" />
           Enabled
         </div>
       );
     case false:
       return (
         <div className="text-dimmed flex items-center gap-1 text-xs">
-          <BoltSlashIcon className="h-4 w-4" />
+          <DisabledIcon className="size-4" />
           Disabled
         </div>
       );

apps/webapp/app/components/runs/v3/LiveTimer.tsx

@@ -1,18 +1,14 @@
 import { formatDuration } from "@trigger.dev/core/v3";
-import { useState, useEffect } from "react";
-import { Paragraph } from "~/components/primitives/Paragraph";
-import { cn } from "~/utils/cn";
+import { useEffect, useState } from "react";
 
 export function LiveTimer({
   startTime,
   endTime,
   updateInterval = 250,
-  className,
 }: {
   startTime: Date;
   endTime?: Date;
   updateInterval?: number;
-  className?: string;
 }) {
   const [now, setNow] = useState<Date>();
 
@@ -30,13 +26,13 @@ export function LiveTimer({
   }, [startTime]);
 
   return (
-    <Paragraph variant="extra-small" className={cn("whitespace-nowrap tabular-nums", className)}>
+    <>
       {formatDuration(startTime, now, {
         style: "short",
         maxDecimalPoints: 0,
         units: ["d", "h", "m", "s"],
       })}
-    </Paragraph>
+    </>
   );
 }