Release 4.0.0-v4-beta.17 (#2071)

* chore: Update version for release (v4-beta)

* Release 4.0.0-v4-beta.17

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: nicktrn <55853254+nicktrn@users.noreply.github.com>

.changeset/cuddly-boats-press.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Add external log exporters and fix missing external trace exporters in deployed tasks

.changeset/curvy-dogs-share.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+When you create a Waitpoint token using `wait.createToken()` you get a URL back that can be used to complete it by making an HTTP POST request.

.changeset/gentle-waves-suffer.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Fixed an issue with realtime streams that timeout and resume streaming dropping chunks

.changeset/gold-insects-invite.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Expose esbuild `keepNames` option (experimental)

.changeset/grumpy-wasps-fold.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Add `experimental_autoDetectExternal` trigger config option

.changeset/itchy-frogs-care.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Log images sizes for self-hosted deploys

.changeset/itchy-games-sort.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+---
+
+Display clickable links in Cursor terminal

.changeset/lazy-panthers-shop.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Improve structured logs

.changeset/orange-rocks-grow.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix init.ts detection when using the sentry esbuild plugin

.changeset/plenty-dolphins-act.md

@@ -0,0 +1,8 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+- Correctly resolve waitpoints that come in early
+- Ensure correct state before requesting suspension
+- Fix race conditions in snapshot processing

.changeset/pre.json

@@ -19,28 +19,48 @@
   "changesets": [
     "blue-eyes-tickle",
     "breezy-turtles-talk",
+    "cuddly-boats-press",
+    "curvy-dogs-share",
     "eighty-rings-divide",
     "four-needles-add",
+    "gentle-waves-suffer",
+    "gold-insects-invite",
     "green-lions-relate",
+    "grumpy-wasps-fold",
     "hip-cups-wave",
     "honest-files-decide",
+    "itchy-frogs-care",
+    "itchy-games-sort",
     "late-chairs-ring",
+    "lazy-panthers-shop",
     "moody-squids-count",
     "nice-colts-boil",
+    "orange-rocks-grow",
+    "plenty-dolphins-act",
     "polite-impalas-care",
     "polite-lies-fix",
+    "rare-beds-accept",
+    "real-rats-drop",
+    "red-chairs-begin",
     "red-wasps-cover",
     "shiny-kiwis-beam",
     "smart-coins-hammer",
     "sour-mirrors-accept",
     "spotty-ducks-punch",
     "spotty-pants-wink",
+    "sweet-dolphins-invent",
     "tender-jobs-collect",
+    "thick-bikes-laugh",
     "tidy-books-smell",
     "tiny-buckets-teach",
     "tricky-houses-invite",
+    "twelve-actors-hide",
     "two-tigers-dream",
     "weak-jobs-hide",
-    "wet-deers-think"
+    "weak-parents-sip",
+    "wet-deers-think",
+    "wet-steaks-reflect",
+    "wild-mirrors-return",
+    "witty-donkeys-unite"
   ]
 }

.changeset/rare-beds-accept.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Add verbose structured log level

.changeset/real-rats-drop.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Add onCancel lifecycle hook

.changeset/red-chairs-begin.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Added AI assistance link when you have build errors

.changeset/sweet-dolphins-invent.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Always print full deploy logs in CI

.changeset/thick-bikes-laugh.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/build": patch
+---
+
+Add ffmpeg v7 support to existing extension: `ffmpeg({ version: "7" })`

.changeset/twelve-actors-hide.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix init.ts auto-import for deployed workers

.changeset/weak-parents-sip.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Output esbuild metafile, can be inspected after `deploy --dry run`

.changeset/wet-steaks-reflect.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+If you pass a directory when calling deploy we validate it exists and give helpful hints

.changeset/wild-mirrors-return.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Expose esbuild `minify` option (experimental)

.changeset/witty-donkeys-unite.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/build": patch
+---
+
+Add playwright extension

.github/workflows/e2e.yml

@@ -1,5 +1,8 @@
 name: "E2E"
 
+permissions:
+  contents: read
+
 on:
   workflow_call:
     inputs:

.github/workflows/publish-webapp.yml

@@ -1,5 +1,10 @@
 name: "🐳 Publish Webapp"
 
+permissions:
+  contents: read
+  packages: write
+  id-token: write
+
 on:
   workflow_call:
     inputs:
@@ -43,9 +48,10 @@ jobs:
           ref_without_tag=ghcr.io/triggerdotdev/trigger.dev
           image_tags=$ref_without_tag:${{ steps.get_tag.outputs.tag }}
 
-          # if tag is a semver, also tag it as v3
+          # if tag is a semver, also tag it as v4
           if [[ "${{ steps.get_tag.outputs.is_semver }}" == true ]]; then
-            image_tags=$image_tags,$ref_without_tag:v3
+            # TODO: switch to v4 tag on GA
+            image_tags=$image_tags,$ref_without_tag:v4-beta
           fi
 
           echo "image_tags=${image_tags}" >> "$GITHUB_OUTPUT"

.github/workflows/publish-worker-v4.yml

@@ -1,4 +1,4 @@
-name: "⚒️ Publish Worker RE2"
+name: "⚒️ Publish Worker (v4)"
 
 on:
   workflow_call:
@@ -14,6 +14,7 @@ on:
       - "re2-prod-*"
 
 permissions:
+  id-token: write
   packages: write
   contents: read
 
@@ -35,6 +36,9 @@ jobs:
     env:
       DOCKER_BUILDKIT: "1"
     steps:
+      - name: 🏭 Setup Depot CLI
+        uses: depot/setup-action@v1
+
       - name: ⬇️ Checkout git repo
         uses: actions/checkout@v4
 
@@ -49,26 +53,26 @@ jobs:
           fi
           echo "repo=${repo}" >> "$GITHUB_OUTPUT"
 
-      - id: get_tag
+      - name: "#️⃣ Get image tag"
+        id: get_tag
         uses: ./.github/actions/get-image-tag
         with:
           tag: ${{ inputs.image_tag }}
 
-      - name: 🐋 Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      # ..to avoid rate limits when pulling images
-      - name: 🐳 Login to DockerHub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: 🚢 Build Container Image
+      - name: 📛 Set tags to push
+        id: set_tags
         run: |
-          docker build -t infra_image -f ./apps/${{ matrix.package }}/Containerfile .
+          ref_without_tag=ghcr.io/triggerdotdev/${{ steps.get_repository.outputs.repo }}
+          image_tags=$ref_without_tag:${{ steps.get_tag.outputs.tag }}
+
+          # if tag is a semver, also tag it as v4
+          if [[ "${{ steps.get_tag.outputs.is_semver }}" == true ]]; then
+            # TODO: switch to v4 tag on GA
+            image_tags=$image_tags,$ref_without_tag:v4-beta
+          fi
+
+          echo "image_tags=${image_tags}" >> "$GITHUB_OUTPUT"
 
-      # ..to push image
       - name: 🐙 Login to GitHub Container Registry
         uses: docker/login-action@v3
         with:
@@ -76,20 +80,10 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: 🐙 Push to GitHub Container Registry
-        run: |
-          docker tag infra_image "$REGISTRY/$REPOSITORY:$IMAGE_TAG"
-          docker push "$REGISTRY/$REPOSITORY:$IMAGE_TAG"
-        env:
-          REGISTRY: ghcr.io/triggerdotdev
-          REPOSITORY: ${{ steps.get_repository.outputs.repo }}
-          IMAGE_TAG: ${{ steps.get_tag.outputs.tag }}
-
-      - name: 🐙 Push 'v3' tag to GitHub Container Registry
-        if: steps.get_tag.outputs.is_semver == 'true'
-        run: |
-          docker tag infra_image "$REGISTRY/$REPOSITORY:v3"
-          docker push "$REGISTRY/$REPOSITORY:v3"
-        env:
-          REGISTRY: ghcr.io/triggerdotdev
-          REPOSITORY: ${{ steps.get_repository.outputs.repo }}
+      - name: 🐳 Build image and push to GitHub Container Registry
+        uses: depot/build-push-action@v1
+        with:
+          file: ./apps/${{ matrix.package }}/Containerfile
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.set_tags.outputs.image_tags }}
+          push: true

.github/workflows/publish-worker.yml

@@ -77,11 +77,11 @@ jobs:
           REPOSITORY: ${{ steps.get_repository.outputs.repo }}
           IMAGE_TAG: ${{ steps.get_tag.outputs.tag }}
 
-      - name: 🐙 Push 'v3' tag to GitHub Container Registry
-        if: steps.get_tag.outputs.is_semver == 'true'
-        run: |
-          docker tag infra_image "$REGISTRY/$REPOSITORY:v3"
-          docker push "$REGISTRY/$REPOSITORY:v3"
-        env:
-          REGISTRY: ghcr.io/triggerdotdev
-          REPOSITORY: ${{ steps.get_repository.outputs.repo }}
+      # - name: 🐙 Push 'v3' tag to GitHub Container Registry
+      #   if: steps.get_tag.outputs.is_semver == 'true'
+      #   run: |
+      #     docker tag infra_image "$REGISTRY/$REPOSITORY:v3"
+      #     docker push "$REGISTRY/$REPOSITORY:v3"
+      #   env:
+      #     REGISTRY: ghcr.io/triggerdotdev
+      #     REPOSITORY: ${{ steps.get_repository.outputs.repo }}

.github/workflows/publish.yml

@@ -68,3 +68,10 @@ jobs:
     secrets: inherit
     with:
       image_tag: ${{ inputs.image_tag }}
+
+  publish-worker-v4:
+    needs: [typecheck]
+    uses: ./.github/workflows/publish-worker-v4.yml
+    secrets: inherit
+    with:
+      image_tag: ${{ inputs.image_tag }}

.github/workflows/release.yml

@@ -1,5 +1,8 @@
 name: 🦋 Changesets Release
 
+permissions:
+  contents: write
+
 on:
   push:
     branches:
@@ -18,6 +21,10 @@ jobs:
   release:
     name: 🦋 Changesets Release
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      packages: write
+      pull-requests: write
     if: github.repository == 'triggerdotdev/trigger.dev'
     outputs:
       published: ${{ steps.changesets.outputs.published }}
@@ -89,13 +96,3 @@ jobs:
         run: |
           package_version=$(echo '${{ steps.changesets.outputs.publishedPackages }}' | jq -r '.[0].version')
           echo "package_version=${package_version}" >> "$GITHUB_OUTPUT"
-
-  publish:
-    needs: release
-    uses: ./.github/workflows/publish.yml
-    secrets: inherit
-    # if: needs.release.outputs.published == 'true'
-    # disable automatic publishing for now
-    if: false
-    with:
-      image_tag: v${{ needs.release.outputs.published_package_version }}

.github/workflows/typecheck.yml

@@ -3,6 +3,9 @@ name: "ʦ TypeScript"
 on:
   workflow_call:
 
+permissions:
+  contents: read
+
 jobs:
   typecheck:
     runs-on: ubuntu-latest

.github/workflows/unit-tests.yml

@@ -1,5 +1,8 @@
 name: "🧪 Unit Tests"
 
+permissions:
+  contents: read
+
 on:
   workflow_call:
 
@@ -7,7 +10,35 @@ jobs:
   unitTests:
     name: "🧪 Unit Tests"
     runs-on: ubuntu-latest
+    env:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
     steps:
+      - name: 🔧 Disable IPv6
+        run: |
+          sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
+          sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1
+          sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=1
+
+      - name: 🔧 Configure docker address pool
+        run: |
+          CONFIG='{
+            "default-address-pools" : [
+              {
+                "base" : "172.17.0.0/12",
+                "size" : 20
+              },
+              {
+                "base" : "192.168.0.0/16",
+                "size" : 24
+              }
+            ]
+          }'
+          mkdir -p /etc/docker
+          echo "$CONFIG" | sudo tee /etc/docker/daemon.json
+
+      - name: 🔧 Restart docker daemon
+        run: sudo systemctl restart docker
+
       - name: ⬇️ Checkout repo
         uses: actions/checkout@v4
         with:
@@ -26,10 +57,14 @@ jobs:
 
       # ..to avoid rate limits when pulling images
       - name: 🐳 Login to DockerHub
+        if: ${{ env.DOCKERHUB_USERNAME }}
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: 🐳 Skipping DockerHub login (no secrets available)
+        if: ${{ !env.DOCKERHUB_USERNAME }}
+        run: echo "DockerHub login skipped because secrets are not available."
 
       - name: 📥 Download deps
         run: pnpm install --frozen-lockfile

AGENTS.md

@@ -0,0 +1,68 @@
+# Guidance for Coding Agents
+
+This repository is a pnpm monorepo managed with Turbo. It contains multiple apps and packages that make up the Trigger.dev platform and SDK.
+
+## Repository layout
+- `apps/webapp` – Remix application that serves as the main API and dashboard.
+- `apps/supervisor` – Node application for executing built tasks.
+- `packages/*` – Published packages such as `@trigger.dev/sdk`, the CLI (`trigger.dev`), and shared libraries.
+- `internal-packages/*` – Internal-only packages used by the webapp and other apps.
+- `references/*` – Example projects for manual testing and development of new features.
+- `ai/references` – Contains additional documentation including an overview (`repo.md`) and testing guidelines (`tests.md`).
+
+See `ai/references/repo.md` for a more complete explanation of the workspaces.
+
+## Development setup
+1. Install dependencies with `pnpm i` (pnpm `8.15.5` and Node.js `20.11.1` are required).
+2. Copy `.env.example` to `.env` and generate a random 16 byte hex string for `ENCRYPTION_KEY` (`openssl rand -hex 16`). Update other secrets if needed.
+3. Start the local services with Docker:
+   ```bash
+   pnpm run docker
+   ```
+4. Run database migrations:
+   ```bash
+   pnpm run db:migrate
+   ```
+5. Build the webapp, CLI and SDK packages:
+   ```bash
+   pnpm run build --filter webapp && pnpm run build --filter trigger.dev && pnpm run build --filter @trigger.dev/sdk
+   ```
+6. Launch the development server:
+   ```bash
+   pnpm run dev --filter webapp
+   ```
+   The webapp runs on <http://localhost:3030>.
+
+For full setup instructions see `CONTRIBUTING.md`.
+
+## Running tests
+- Unit tests use **vitest**. Run all tests:
+  ```bash
+  pnpm run test
+  ```
+- Run tests for a specific workspace (example for `webapp`):
+  ```bash
+  pnpm run test --filter webapp
+  ```
+- Prefer running a single test file from within its directory:
+  ```bash
+  cd apps/webapp
+  pnpm run test ./src/components/Button.test.ts
+  ```
+  If packages in that workspace need to be built first, run `pnpm run build --filter webapp`.
+
+Refer to `ai/references/tests.md` for details on writing tests. Tests should avoid mocks or stubs and use the helpers from `@internal/testcontainers` when Redis or Postgres are needed.
+
+## Coding style
+- Formatting is enforced using Prettier. Run `pnpm run format` before committing.
+- Follow the existing project conventions. Test files live beside the files under test and use descriptive `describe` and `it` blocks.
+- Do not commit directly to the `main` branch. All changes should be made in a separate branch and go through a pull request.
+
+## Additional docs
+- The root `README.md` describes Trigger.dev and links to documentation.
+- The `docs` workspace contains our documentation site, which can be run locally with:
+  ```bash
+  pnpm run dev --filter docs
+  ```
+- `references/README.md` explains how to create new reference projects for manual testing.
+

CONTRIBUTING.md

@@ -68,9 +68,9 @@ branch are tagged into a release periodically.
    ```
    pnpm run db:migrate
    ```
-10. Build the server app
+10. Build everything
     ```
-    pnpm run build --filter webapp
+    pnpm run build --filter webapp && pnpm run build --filter trigger.dev && pnpm run build --filter @trigger.dev/sdk
     ```
 11. Run the app. See the section below.
 

apps/supervisor/.env.example

@@ -7,7 +7,7 @@ MANAGED_WORKER_SECRET=managed-secret
 # Point this at the webapp in prod
 TRIGGER_API_URL=http://localhost:3030
 
-# Point this at the OTel collector in prod
+# Point this at the webapp or an OTel collector in prod
 OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:3030/otel
 # Use this on macOS
 # OTEL_EXPORTER_OTLP_ENDPOINT=http://host.docker.internal:3030/otel

apps/supervisor/Containerfile

@@ -19,6 +19,7 @@ COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspa
 RUN corepack enable && corepack prepare --activate
 
 FROM base AS deps-fetcher
+RUN apk add --no-cache python3-dev py3-setuptools make g++ gcc linux-headers
 RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --frozen-lockfile
 
 FROM deps-fetcher AS dev-deps

apps/supervisor/README.md

@@ -19,6 +19,8 @@ curl -sS \
     -d "{\"name\": \"$wg_name\"}"
 ```
 
+If the worker group is newly created, the response will include a `token` field. If the group already exists, no token is returned.
+
 2. Create `.env` and set the worker token
 
 ```sh
@@ -34,7 +36,7 @@ TRIGGER_WORKER_TOKEN=tr_wgt_...
 pnpm dev
 ```
 
-4. Build CLI, then deploy a reference project
+4. Build CLI, then deploy a test project
 
 ```sh
 pnpm exec trigger deploy --self-hosted
@@ -43,16 +45,36 @@ pnpm exec trigger deploy --self-hosted
 pnpm exec trigger deploy --self-hosted --network host
 ```
 
-## Additional worker groups
+## Worker group management
 
-When adding more worker groups you might also want to make them the default for a specific project. This will allow you to test it without having to change the global default:
+### Shared variables
 
 ```sh
 api_url=http://localhost:3030
+admin_pat=tr_pat_... # edit this
+```
+
+- These are used by all commands
+
+### Create a worker group
+
+```sh
 wg_name=my-worker
 
-# edit these
-admin_pat=tr_pat_...
+curl -sS \
+    -X POST \
+    "$api_url/admin/api/v1/workers" \
+    -H "Authorization: Bearer $admin_pat" \
+    -H "Content-Type: application/json" \
+    -d "{\"name\": \"$wg_name\"}"
+```
+
+- If the worker group already exists, no token will be returned
+
+### Set a worker group as default for a project
+
+```sh
+wg_name=my-worker
 project_id=clsw6q8wz...
 
 curl -sS \
@@ -60,8 +82,24 @@ curl -sS \
     "$api_url/admin/api/v1/workers" \
     -H "Authorization: Bearer $admin_pat" \
     -H "Content-Type: application/json" \
-    -d "{
-        \"name\": \"$wg_name\",
-        \"makeDefaultForProjectId\": \"$project_id\"
-    }"
+    -d "{\"name\": \"$wg_name\", \"projectId\": \"$project_id\", \"makeDefaultForProject\": true}"
 ```
+
+- If the worker group doesn't exist, yet it will be created
+- If the worker group already exists, it will be attached to the project as default. No token will be returned.
+
+### Remove the default worker group from a project
+
+```sh
+project_id=clsw6q8wz...
+
+curl -sS \
+    -X POST \
+    "$api_url/admin/api/v1/workers" \
+    -H "Authorization: Bearer $admin_pat" \
+    -H "Content-Type: application/json" \
+    -d "{\"projectId\": \"$project_id\", \"removeDefaultFromProject\": true}"
+```
+
+- The project will then use the global default again
+- When `removeDefaultFromProject: true` no other actions will be performed

apps/supervisor/package.json

@@ -16,16 +16,13 @@
     "@kubernetes/client-node": "^1.0.0",
     "@trigger.dev/core": "workspace:*",
     "dockerode": "^4.0.3",
-    "nanoid": "^5.0.9",
     "prom-client": "^15.1.0",
     "socket.io": "4.7.4",
     "std-env": "^3.8.0",
-    "tinyexec": "^0.3.1",
     "zod": "3.23.8"
   },
   "devDependencies": {
     "@types/dockerode": "^3.3.33",
-    "docker-api-ts": "^0.2.2",
     "vitest": "^1.4.0"
   }
 }

apps/supervisor/src/clients/kubernetes.ts

@@ -3,9 +3,12 @@ import { Informer } from "@kubernetes/client-node";
 import { ListPromise } from "@kubernetes/client-node";
 import { KubernetesObject } from "@kubernetes/client-node";
 import { assertExhaustive } from "@trigger.dev/core/utils";
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
 
 export const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
 
+const logger = new SimpleStructuredLogger("kubernetes-client");
+
 export function createK8sApi() {
   const kubeConfig = getKubeConfig();
 
@@ -31,7 +34,7 @@ export function createK8sApi() {
 export type K8sApi = ReturnType<typeof createK8sApi>;
 
 function getKubeConfig() {
-  console.log("getKubeConfig()", { RUNTIME_ENV });
+  logger.debug("getKubeConfig()", { RUNTIME_ENV });
 
   const kubeConfig = new k8s.KubeConfig();
 

apps/supervisor/src/env.ts

@@ -6,6 +6,7 @@ import { AdditionalEnvVars, BoolEnv } from "./envUtil.js";
 const Env = z.object({
   // This will come from `spec.nodeName` in k8s
   TRIGGER_WORKER_INSTANCE_NAME: z.string().default(randomUUID()),
+  TRIGGER_WORKER_HEARTBEAT_INTERVAL_SECONDS: z.coerce.number().default(30),
 
   // Required settings
   TRIGGER_API_URL: z.string().url(),
@@ -28,10 +29,22 @@ const Env = z.object({
   RUNNER_SNAPSHOT_POLL_INTERVAL_SECONDS: z.coerce.number().optional(),
   RUNNER_ADDITIONAL_ENV_VARS: AdditionalEnvVars, // optional (csv)
   RUNNER_DOCKER_AUTOREMOVE: BoolEnv.default(true),
+  /**
+   * Network mode to use for all runners. Supported standard values are: `bridge`, `host`, `none`, and `container:<name|id>`.
+   * Any other value is taken as a custom network's name to which all runners should connect to.
+   *
+   * Accepts a list of comma-separated values to attach to multiple networks. Additional networks are interpreted as network names and will be attached after container creation.
+   *
+   * **WARNING**: Specifying multiple networks will slightly increase startup times.
+   *
+   * @default "host"
+   */
+  RUNNER_DOCKER_NETWORKS: z.string().default("host"),
 
   // Dequeue settings (provider mode)
   TRIGGER_DEQUEUE_ENABLED: BoolEnv.default("true"),
-  TRIGGER_DEQUEUE_INTERVAL_MS: z.coerce.number().int().default(1000),
+  TRIGGER_DEQUEUE_INTERVAL_MS: z.coerce.number().int().default(250),
+  TRIGGER_DEQUEUE_IDLE_INTERVAL_MS: z.coerce.number().int().default(1000),
   TRIGGER_DEQUEUE_MAX_RUN_COUNT: z.coerce.number().int().default(10),
   TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT: z.coerce.number().int().default(1),
 
@@ -41,14 +54,14 @@ const Env = z.object({
   TRIGGER_METADATA_URL: z.string().optional(),
 
   // Used by the workload manager, e.g docker/k8s
-  DOCKER_NETWORK: z.string().default("host"),
   OTEL_EXPORTER_OTLP_ENDPOINT: z.string().url(),
   ENFORCE_MACHINE_PRESETS: z.coerce.boolean().default(false),
   KUBERNETES_IMAGE_PULL_SECRETS: z.string().optional(), // csv
 
   // Used by the resource monitor
-  OVERRIDE_CPU_TOTAL: z.coerce.number().optional(),
-  OVERRIDE_MEMORY_TOTAL_GB: z.coerce.number().optional(),
+  RESOURCE_MONITOR_ENABLED: BoolEnv.default(false),
+  RESOURCE_MONITOR_OVERRIDE_CPU_TOTAL: z.coerce.number().optional(),
+  RESOURCE_MONITOR_OVERRIDE_MEMORY_TOTAL_GB: z.coerce.number().optional(),
 
   // Kubernetes specific settings
   KUBERNETES_FORCE_ENABLED: BoolEnv.default(false),

apps/supervisor/src/envUtil.ts

@@ -1,4 +1,7 @@
 import { z } from "zod";
+import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
+
+const logger = new SimpleStructuredLogger("env-util");
 
 export const BoolEnv = z.preprocess((val) => {
   if (typeof val !== "string") {
@@ -33,7 +36,7 @@ export const AdditionalEnvVars = z.preprocess((val) => {
     // Return undefined if no valid key-value pairs were found
     return Object.keys(result).length === 0 ? undefined : result;
   } catch (error) {
-    console.warn("Failed to parse additional env vars", { error, val });
+    logger.warn("Failed to parse additional env vars", { error, val });
     return undefined;
   }
 }, z.record(z.string(), z.string()).optional());

apps/supervisor/src/index.ts

@@ -9,6 +9,7 @@ import { type DequeuedMessage } from "@trigger.dev/core/v3";
 import {
   DockerResourceMonitor,
   KubernetesResourceMonitor,
+  NoopResourceMonitor,
   type ResourceMonitor,
 } from "./resourceMonitor.js";
 import { KubernetesWorkloadManager } from "./workloadManager/kubernetes.js";
@@ -33,7 +34,7 @@ class ManagedSupervisor {
   private readonly metricsServer?: HttpServer;
   private readonly workloadServer: WorkloadServer;
   private readonly workloadManager: WorkloadManager;
-  private readonly logger = new SimpleStructuredLogger("managed-worker");
+  private readonly logger = new SimpleStructuredLogger("managed-supervisor");
   private readonly resourceMonitor: ResourceMonitor;
   private readonly checkpointClient?: CheckpointClient;
 
@@ -47,11 +48,11 @@ class ManagedSupervisor {
     const { TRIGGER_WORKER_TOKEN, MANAGED_WORKER_SECRET, ...envWithoutSecrets } = env;
 
     if (env.DEBUG) {
-      console.debug("[ManagedSupervisor] Starting up", { envWithoutSecrets });
+      this.logger.debug("Starting up", { envWithoutSecrets });
     }
 
     if (this.warmStartUrl) {
-      this.logger.log("[ManagedWorker] 🔥 Warm starts enabled", {
+      this.logger.log("🔥 Warm starts enabled", {
         warmStartUrl: this.warmStartUrl,
       });
     }
@@ -69,9 +70,19 @@ class ManagedSupervisor {
       dockerAutoremove: env.RUNNER_DOCKER_AUTOREMOVE,
     } satisfies WorkloadManagerOptions;
 
+    this.resourceMonitor = env.RESOURCE_MONITOR_ENABLED
+      ? this.isKubernetes
+        ? new KubernetesResourceMonitor(createK8sApi(), env.TRIGGER_WORKER_INSTANCE_NAME)
+        : new DockerResourceMonitor(new Docker())
+      : new NoopResourceMonitor();
+
+    this.workloadManager = this.isKubernetes
+      ? new KubernetesWorkloadManager(workloadManagerOptions)
+      : new DockerWorkloadManager(workloadManagerOptions);
+
     if (this.isKubernetes) {
       if (env.POD_CLEANER_ENABLED) {
-        this.logger.log("[ManagedWorker] 🧹 Pod cleaner enabled", {
+        this.logger.log("🧹 Pod cleaner enabled", {
           namespace: env.KUBERNETES_NAMESPACE,
           batchSize: env.POD_CLEANER_BATCH_SIZE,
           intervalMs: env.POD_CLEANER_INTERVAL_MS,
@@ -83,11 +94,11 @@ class ManagedSupervisor {
           intervalMs: env.POD_CLEANER_INTERVAL_MS,
         });
       } else {
-        this.logger.warn("[ManagedWorker] Pod cleaner disabled");
+        this.logger.warn("Pod cleaner disabled");
       }
 
       if (env.FAILED_POD_HANDLER_ENABLED) {
-        this.logger.log("[ManagedWorker] 🔁 Failed pod handler enabled", {
+        this.logger.log("🔁 Failed pod handler enabled", {
           namespace: env.KUBERNETES_NAMESPACE,
           reconnectIntervalMs: env.FAILED_POD_HANDLER_RECONNECT_INTERVAL_MS,
         });
@@ -97,17 +108,14 @@ class ManagedSupervisor {
           reconnectIntervalMs: env.FAILED_POD_HANDLER_RECONNECT_INTERVAL_MS,
         });
       } else {
-        this.logger.warn("[ManagedWorker] Failed pod handler disabled");
+        this.logger.warn("Failed pod handler disabled");
       }
+    }
 
-      this.resourceMonitor = new KubernetesResourceMonitor(
-        createK8sApi(),
-        env.TRIGGER_WORKER_INSTANCE_NAME
+    if (env.TRIGGER_DEQUEUE_INTERVAL_MS > env.TRIGGER_DEQUEUE_IDLE_INTERVAL_MS) {
+      this.logger.warn(
+        `⚠️  TRIGGER_DEQUEUE_INTERVAL_MS (${env.TRIGGER_DEQUEUE_INTERVAL_MS}) is greater than TRIGGER_DEQUEUE_IDLE_INTERVAL_MS (${env.TRIGGER_DEQUEUE_IDLE_INTERVAL_MS}) - did you mix them up?`
       );
-      this.workloadManager = new KubernetesWorkloadManager(workloadManagerOptions);
-    } else {
-      this.resourceMonitor = new DockerResourceMonitor(new Docker());
-      this.workloadManager = new DockerWorkloadManager(workloadManagerOptions);
     }
 
     this.workerSession = new SupervisorSession({
@@ -116,17 +124,24 @@ class ManagedSupervisor {
       instanceName: env.TRIGGER_WORKER_INSTANCE_NAME,
       managedWorkerSecret: env.MANAGED_WORKER_SECRET,
       dequeueIntervalMs: env.TRIGGER_DEQUEUE_INTERVAL_MS,
+      dequeueIdleIntervalMs: env.TRIGGER_DEQUEUE_IDLE_INTERVAL_MS,
       queueConsumerEnabled: env.TRIGGER_DEQUEUE_ENABLED,
       maxRunCount: env.TRIGGER_DEQUEUE_MAX_RUN_COUNT,
       maxConsumerCount: env.TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT,
       runNotificationsEnabled: env.TRIGGER_WORKLOAD_API_ENABLED,
+      heartbeatIntervalSeconds: env.TRIGGER_WORKER_HEARTBEAT_INTERVAL_SECONDS,
       preDequeue: async () => {
+        if (!env.RESOURCE_MONITOR_ENABLED) {
+          return {};
+        }
+
         if (this.isKubernetes) {
           // Not used in k8s for now
           return {};
         }
 
         const resources = await this.resourceMonitor.getNodeResources();
+
         return {
           maxResources: {
             cpu: resources.cpuAvailable,
@@ -142,7 +157,7 @@ class ManagedSupervisor {
     });
 
     if (env.TRIGGER_CHECKPOINT_URL) {
-      this.logger.log("[ManagedWorker] 🥶 Checkpoints enabled", {
+      this.logger.log("🥶 Checkpoints enabled", {
         checkpointUrl: env.TRIGGER_CHECKPOINT_URL,
       });
 
@@ -153,43 +168,34 @@ class ManagedSupervisor {
       });
     }
 
-    // setInterval(async () => {
-    //   const resources = await this.resourceMonitor.getNodeResources(true);
-    //   this.logger.debug("[ManagedWorker] Current resources", { resources });
-    // }, 1000);
-
     this.workerSession.on("runNotification", async ({ time, run }) => {
-      this.logger.log("[ManagedWorker] runNotification", { time, run });
+      this.logger.log("runNotification", { time, run });
 
       this.workloadServer.notifyRun({ run });
     });
 
     this.workerSession.on("runQueueMessage", async ({ time, message }) => {
-      this.logger.log(
-        `[ManagedWorker] Received message with timestamp ${time.toLocaleString()}`,
-        message
-      );
+      this.logger.log(`Received message with timestamp ${time.toLocaleString()}`, message);
 
       if (message.completedWaitpoints.length > 0) {
-        this.logger.debug("[ManagedWorker] Run has completed waitpoints", {
+        this.logger.debug("Run has completed waitpoints", {
           runId: message.run.id,
           completedWaitpoints: message.completedWaitpoints.length,
         });
-        // TODO: Do something with them or if we don't need the data here, maybe we shouldn't even send it
       }
 
       if (!message.image) {
-        this.logger.error("[ManagedWorker] Run has no image", { runId: message.run.id });
+        this.logger.error("Run has no image", { runId: message.run.id });
         return;
       }
 
       const { checkpoint, ...rest } = message;
 
       if (checkpoint) {
-        this.logger.log("[ManagedWorker] Restoring run", { runId: message.run.id });
+        this.logger.log("Restoring run", { runId: message.run.id });
 
         if (!this.checkpointClient) {
-          this.logger.error("[ManagedWorker] No checkpoint client", { runId: message.run.id });
+          this.logger.error("No checkpoint client", { runId: message.run.id });
           return;
         }
 
@@ -204,23 +210,23 @@ class ManagedSupervisor {
           });
 
           if (didRestore) {
-            this.logger.log("[ManagedWorker] Restore successful", { runId: message.run.id });
+            this.logger.log("Restore successful", { runId: message.run.id });
           } else {
-            this.logger.error("[ManagedWorker] Restore failed", { runId: message.run.id });
+            this.logger.error("Restore failed", { runId: message.run.id });
           }
         } catch (error) {
-          this.logger.error("[ManagedWorker] Failed to restore run", { error });
+          this.logger.error("Failed to restore run", { error });
         }
 
         return;
       }
 
-      this.logger.log("[ManagedWorker] Scheduling run", { runId: message.run.id });
+      this.logger.log("Scheduling run", { runId: message.run.id });
 
       const didWarmStart = await this.tryWarmStart(message);
 
       if (didWarmStart) {
-        this.logger.log("[ManagedWorker] Warm start successful", { runId: message.run.id });
+        this.logger.log("Warm start successful", { runId: message.run.id });
         return;
       }
 
@@ -247,7 +253,7 @@ class ManagedSupervisor {
         //   memory: message.run.machine.memory,
         // });
       } catch (error) {
-        this.logger.error("[ManagedWorker] Failed to create workload", { error });
+        this.logger.error("Failed to create workload", { error });
       }
     });
 
@@ -275,12 +281,12 @@ class ManagedSupervisor {
   }
 
   async onRunConnected({ run }: { run: { friendlyId: string } }) {
-    this.logger.debug("[ManagedWorker] Run connected", { run });
+    this.logger.debug("Run connected", { run });
     this.workerSession.subscribeToRunNotifications([run.friendlyId]);
   }
 
   async onRunDisconnected({ run }: { run: { friendlyId: string } }) {
-    this.logger.debug("[ManagedWorker] Run disconnected", { run });
+    this.logger.debug("Run disconnected", { run });
     this.workerSession.unsubscribeFromRunNotifications([run.friendlyId]);
   }
 
@@ -301,7 +307,7 @@ class ManagedSupervisor {
       });
 
       if (!res.ok) {
-        this.logger.error("[ManagedWorker] Warm start failed", {
+        this.logger.error("Warm start failed", {
           runId: dequeuedMessage.run.id,
         });
         return false;
@@ -311,7 +317,7 @@ class ManagedSupervisor {
       const parsedData = z.object({ didWarmStart: z.boolean() }).safeParse(data);
 
       if (!parsedData.success) {
-        this.logger.error("[ManagedWorker] Warm start response invalid", {
+        this.logger.error("Warm start response invalid", {
           runId: dequeuedMessage.run.id,
           data,
         });
@@ -320,7 +326,7 @@ class ManagedSupervisor {
 
       return parsedData.data.didWarmStart;
     } catch (error) {
-      this.logger.error("[ManagedWorker] Warm start error", {
+      this.logger.error("Warm start error", {
         runId: dequeuedMessage.run.id,
         error,
       });
@@ -329,7 +335,7 @@ class ManagedSupervisor {
   }
 
   async start() {
-    this.logger.log("[ManagedWorker] Starting up");
+    this.logger.log("Starting up");
 
     // Optional services
     await this.podCleaner?.start();
@@ -337,21 +343,21 @@ class ManagedSupervisor {
     await this.metricsServer?.start();
 
     if (env.TRIGGER_WORKLOAD_API_ENABLED) {
-      this.logger.log("[ManagedWorker] Workload API enabled", {
+      this.logger.log("Workload API enabled", {
         protocol: env.TRIGGER_WORKLOAD_API_PROTOCOL,
         domain: env.TRIGGER_WORKLOAD_API_DOMAIN,
         port: env.TRIGGER_WORKLOAD_API_PORT_INTERNAL,
       });
       await this.workloadServer.start();
     } else {
-      this.logger.warn("[ManagedWorker] Workload API disabled");
+      this.logger.warn("Workload API disabled");
     }
 
     await this.workerSession.start();
   }
 
   async stop() {
-    this.logger.log("[ManagedWorker] Shutting down");
+    this.logger.log("Shutting down");
     await this.workerSession.stop();
 
     // Optional services

apps/supervisor/src/resourceMonitor.ts

@@ -1,5 +1,4 @@
 import type Docker from "dockerode";
-import type * as TDocker from "docker-api-ts";
 import type { MachineResources } from "@trigger.dev/core/v3";
 import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
 import { env } from "./env.js";
@@ -71,18 +70,21 @@ export abstract class ResourceMonitor {
   }
 
   protected applyOverrides(resources: NodeResources): NodeResources {
-    if (!env.OVERRIDE_CPU_TOTAL && !env.OVERRIDE_MEMORY_TOTAL_GB) {
+    if (
+      !env.RESOURCE_MONITOR_OVERRIDE_CPU_TOTAL &&
+      !env.RESOURCE_MONITOR_OVERRIDE_MEMORY_TOTAL_GB
+    ) {
       return resources;
     }
 
     logger.debug("[ResourceMonitor] 🛡️ Applying resource overrides", {
-      cpuTotal: env.OVERRIDE_CPU_TOTAL,
-      memoryTotalGb: env.OVERRIDE_MEMORY_TOTAL_GB,
+      cpuTotal: env.RESOURCE_MONITOR_OVERRIDE_CPU_TOTAL,
+      memoryTotalGb: env.RESOURCE_MONITOR_OVERRIDE_MEMORY_TOTAL_GB,
     });
 
-    const cpuTotal = env.OVERRIDE_CPU_TOTAL ?? resources.cpuTotal;
-    const memoryTotal = env.OVERRIDE_MEMORY_TOTAL_GB
-      ? this.gbToBytes(env.OVERRIDE_MEMORY_TOTAL_GB)
+    const cpuTotal = env.RESOURCE_MONITOR_OVERRIDE_CPU_TOTAL ?? resources.cpuTotal;
+    const memoryTotal = env.RESOURCE_MONITOR_OVERRIDE_MEMORY_TOTAL_GB
+      ? this.gbToBytes(env.RESOURCE_MONITOR_OVERRIDE_MEMORY_TOTAL_GB)
       : resources.memoryTotal;
 
     const cpuDiff = cpuTotal - resources.cpuTotal;
@@ -100,6 +102,11 @@ export abstract class ResourceMonitor {
   }
 }
 
+type SystemInfo = {
+  NCPU: number | undefined;
+  MemTotal: number | undefined;
+};
+
 export class DockerResourceMonitor extends ResourceMonitor {
   private docker: Docker;
 
@@ -114,7 +121,7 @@ export class DockerResourceMonitor extends ResourceMonitor {
       return this.cachedResources;
     }
 
-    const info: TDocker.SystemInfo = await this.docker.info();
+    const info: SystemInfo = await this.docker.info();
     const stats = await this.docker.listContainers({ all: true });
 
     // Get system-wide resources
@@ -208,6 +215,21 @@ export class KubernetesResourceMonitor extends ResourceMonitor {
   }
 }
 
+export class NoopResourceMonitor extends ResourceMonitor {
+  constructor() {
+    super(NoopResourceParser);
+  }
+
+  async getNodeResources(): Promise<NodeResources> {
+    return {
+      cpuTotal: 0,
+      cpuAvailable: Infinity,
+      memoryTotal: 0,
+      memoryAvailable: Infinity,
+    };
+  }
+}
+
 abstract class ResourceParser {
   abstract cpu(cpu: number | string): number;
   abstract memory(memory: number | string): number;
@@ -244,3 +266,13 @@ class KubernetesResourceParser extends ResourceParser {
     return parseInt(memory);
   }
 }
+
+class NoopResourceParser extends ResourceParser {
+  cpu(cpu: number): number {
+    return cpu;
+  }
+
+  memory(memory: number): number {
+    return memory;
+  }
+}

apps/supervisor/src/workloadManager/docker.ts

@@ -4,88 +4,172 @@ import {
   type WorkloadManagerCreateOptions,
   type WorkloadManagerOptions,
 } from "./types.js";
-import { x } from "tinyexec";
 import { env } from "../env.js";
 import { getDockerHostDomain, getRunnerId } from "../util.js";
+import Docker from "dockerode";
+import { tryCatch } from "@trigger.dev/core";
 
 export class DockerWorkloadManager implements WorkloadManager {
-  private readonly logger = new SimpleStructuredLogger("docker-workload-provider");
+  private readonly logger = new SimpleStructuredLogger("docker-workload-manager");
+  private readonly docker: Docker;
+
+  private readonly runnerNetworks: string[];
 
   constructor(private opts: WorkloadManagerOptions) {
+    this.docker = new Docker();
+
     if (opts.workloadApiDomain) {
-      this.logger.warn("[DockerWorkloadProvider] ⚠️ Custom workload API domain", {
+      this.logger.warn("⚠️ Custom workload API domain", {
         domain: opts.workloadApiDomain,
       });
     }
+
+    this.runnerNetworks = env.RUNNER_DOCKER_NETWORKS.split(",");
   }
 
   async create(opts: WorkloadManagerCreateOptions) {
-    this.logger.log("[DockerWorkloadProvider] Creating container", { opts });
+    this.logger.log("create()", { opts });
 
     const runnerId = getRunnerId(opts.runFriendlyId, opts.nextAttemptNumber);
 
-    const runArgs = [
-      "run",
-      "--detach",
-      `--network=${env.DOCKER_NETWORK}`,
-      `--env=TRIGGER_DEQUEUED_AT_MS=${opts.dequeuedAt.getTime()}`,
-      `--env=TRIGGER_POD_SCHEDULED_AT_MS=${Date.now()}`,
-      `--env=TRIGGER_ENV_ID=${opts.envId}`,
-      `--env=TRIGGER_RUN_ID=${opts.runFriendlyId}`,
-      `--env=TRIGGER_SNAPSHOT_ID=${opts.snapshotFriendlyId}`,
-      `--env=TRIGGER_SUPERVISOR_API_PROTOCOL=${this.opts.workloadApiProtocol}`,
-      `--env=TRIGGER_SUPERVISOR_API_PORT=${this.opts.workloadApiPort}`,
-      `--env=TRIGGER_SUPERVISOR_API_DOMAIN=${this.opts.workloadApiDomain ?? getDockerHostDomain()}`,
-      `--env=TRIGGER_WORKER_INSTANCE_NAME=${env.TRIGGER_WORKER_INSTANCE_NAME}`,
-      `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${env.OTEL_EXPORTER_OTLP_ENDPOINT}`,
-      `--env=TRIGGER_RUNNER_ID=${runnerId}`,
-      `--hostname=${runnerId}`,
-      `--name=${runnerId}`,
+    // Build environment variables
+    const envVars: string[] = [
+      `TRIGGER_DEQUEUED_AT_MS=${opts.dequeuedAt.getTime()}`,
+      `TRIGGER_POD_SCHEDULED_AT_MS=${Date.now()}`,
+      `TRIGGER_ENV_ID=${opts.envId}`,
+      `TRIGGER_RUN_ID=${opts.runFriendlyId}`,
+      `TRIGGER_SNAPSHOT_ID=${opts.snapshotFriendlyId}`,
+      `TRIGGER_SUPERVISOR_API_PROTOCOL=${this.opts.workloadApiProtocol}`,
+      `TRIGGER_SUPERVISOR_API_PORT=${this.opts.workloadApiPort}`,
+      `TRIGGER_SUPERVISOR_API_DOMAIN=${this.opts.workloadApiDomain ?? getDockerHostDomain()}`,
+      `TRIGGER_WORKER_INSTANCE_NAME=${env.TRIGGER_WORKER_INSTANCE_NAME}`,
+      `OTEL_EXPORTER_OTLP_ENDPOINT=${env.OTEL_EXPORTER_OTLP_ENDPOINT}`,
+      `TRIGGER_RUNNER_ID=${runnerId}`,
     ];
 
-    if (this.opts.dockerAutoremove) {
-      runArgs.push("--rm");
-    }
-
     if (this.opts.warmStartUrl) {
-      runArgs.push(`--env=TRIGGER_WARM_START_URL=${this.opts.warmStartUrl}`);
+      envVars.push(`TRIGGER_WARM_START_URL=${this.opts.warmStartUrl}`);
     }
 
     if (this.opts.metadataUrl) {
-      runArgs.push(`--env=TRIGGER_METADATA_URL=${this.opts.metadataUrl}`);
+      envVars.push(`TRIGGER_METADATA_URL=${this.opts.metadataUrl}`);
     }
 
     if (this.opts.heartbeatIntervalSeconds) {
-      runArgs.push(
-        `--env=TRIGGER_HEARTBEAT_INTERVAL_SECONDS=${this.opts.heartbeatIntervalSeconds}`
-      );
+      envVars.push(`TRIGGER_HEARTBEAT_INTERVAL_SECONDS=${this.opts.heartbeatIntervalSeconds}`);
     }
 
     if (this.opts.snapshotPollIntervalSeconds) {
-      runArgs.push(
-        `--env=TRIGGER_SNAPSHOT_POLL_INTERVAL_SECONDS=${this.opts.snapshotPollIntervalSeconds}`
+      envVars.push(
+        `TRIGGER_SNAPSHOT_POLL_INTERVAL_SECONDS=${this.opts.snapshotPollIntervalSeconds}`
       );
     }
 
     if (this.opts.additionalEnvVars) {
       Object.entries(this.opts.additionalEnvVars).forEach(([key, value]) => {
-        runArgs.push(`--env=${key}=${value}`);
+        envVars.push(`${key}=${value}`);
       });
     }
 
+    const hostConfig: Docker.HostConfig = {
+      AutoRemove: !!this.opts.dockerAutoremove,
+    };
+
+    const [firstNetwork, ...remainingNetworks] = this.runnerNetworks;
+
+    // Always attach the first network at container creation time. This has the following benefits:
+    // - If there is only a single network to attach, this will prevent having to make a separate request.
+    // - If there are multiple networks to attach, this will ensure the runner won't also be connected to the bridge network
+    hostConfig.NetworkMode = firstNetwork;
+
     if (env.ENFORCE_MACHINE_PRESETS) {
-      runArgs.push(`--cpus=${opts.machine.cpu}`, `--memory=${opts.machine.memory}G`);
-      runArgs.push(`--env=TRIGGER_MACHINE_CPU=${opts.machine.cpu}`);
-      runArgs.push(`--env=TRIGGER_MACHINE_MEMORY=${opts.machine.memory}`);
+      envVars.push(`TRIGGER_MACHINE_CPU=${opts.machine.cpu}`);
+      envVars.push(`TRIGGER_MACHINE_MEMORY=${opts.machine.memory}`);
+
+      hostConfig.NanoCpus = opts.machine.cpu * 1e9;
+      hostConfig.Memory = opts.machine.memory * 1024 * 1024 * 1024;
     }
 
-    runArgs.push(`${opts.image}`);
+    const containerCreateOpts: Docker.ContainerCreateOptions = {
+      Env: envVars,
+      name: runnerId,
+      Hostname: runnerId,
+      HostConfig: hostConfig,
+      Image: opts.image,
+      AttachStdout: false,
+      AttachStderr: false,
+      AttachStdin: false,
+    };
 
     try {
-      const { stdout, stderr } = await x("docker", runArgs);
-      this.logger.debug("[DockerWorkloadProvider] Create succeeded", { stdout, stderr });
+      // Create container
+      const container = await this.docker.createContainer(containerCreateOpts);
+
+      // If there are multiple networks to attach to we need to attach the remaining ones after creation
+      if (remainingNetworks.length > 0) {
+        await this.attachContainerToNetworks({
+          containerId: container.id,
+          networkNames: remainingNetworks,
+        });
+      }
+
+      // Start container
+      const startResult = await container.start();
+
+      this.logger.debug("create succeeded", {
+        opts,
+        startResult,
+        containerId: container.id,
+        containerCreateOpts,
+      });
     } catch (error) {
-      this.logger.error("[DockerWorkloadProvider] Create failed:", { opts, error });
+      this.logger.error("create failed:", { opts, error, containerCreateOpts });
     }
   }
+
+  private async attachContainerToNetworks({
+    containerId,
+    networkNames,
+  }: {
+    containerId: string;
+    networkNames: string[];
+  }) {
+    this.logger.debug("Attaching container to networks", { containerId, networkNames });
+
+    const [error, networkResults] = await tryCatch(
+      this.docker.listNetworks({
+        filters: {
+          // Full name matches only to prevent unexpected results
+          name: networkNames.map((name) => `^${name}$`),
+        },
+      })
+    );
+
+    if (error) {
+      this.logger.error("Failed to list networks", { networkNames });
+      return;
+    }
+
+    const results = await Promise.allSettled(
+      networkResults.map((networkInfo) => {
+        const network = this.docker.getNetwork(networkInfo.Id);
+        return network.connect({ Container: containerId });
+      })
+    );
+
+    if (results.some((r) => r.status === "rejected")) {
+      this.logger.error("Failed to attach container to some networks", {
+        containerId,
+        networkNames,
+        results,
+      });
+      return;
+    }
+
+    this.logger.debug("Attached container to networks", {
+      containerId,
+      networkNames,
+      results,
+    });
+  }
 }

apps/supervisor/src/workloadServer/index.ts

@@ -17,6 +17,7 @@ import {
   WorkloadRunAttemptStartRequestBody,
   type WorkloadRunAttemptStartResponseBody,
   type WorkloadRunLatestSnapshotResponseBody,
+  WorkloadRunSnapshotsSinceResponseBody,
   type WorkloadServerToClientEvents,
   type WorkloadSuspendRunResponseBody,
 } from "@trigger.dev/core/v3/workers";
@@ -62,6 +63,8 @@ type WorkloadServerOptions = {
 export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
   private checkpointClient?: CheckpointClient;
 
+  private readonly logger = new SimpleStructuredLogger("workload-server");
+
   private readonly httpServer: HttpServer;
   private readonly websocketServer: Namespace<
     WorkloadClientToServerEvents,
@@ -150,7 +153,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
             );
 
             if (!startResponse.success) {
-              console.error("Failed to start run", {
+              this.logger.error("Failed to start run", {
                 params,
                 error: startResponse.error,
               });
@@ -170,7 +173,6 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
           paramsSchema: WorkloadActionParams,
           bodySchema: WorkloadRunAttemptCompleteRequestBody,
           handler: async ({ req, reply, params, body }) => {
-            console.log("headers", req.headers);
             const completeResponse = await this.workerClient.completeRunAttempt(
               params.runFriendlyId,
               params.snapshotFriendlyId,
@@ -179,7 +181,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
             );
 
             if (!completeResponse.success) {
-              console.error("Failed to complete run", {
+              this.logger.error("Failed to complete run", {
                 params,
                 error: completeResponse.error,
               });
@@ -207,7 +209,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
             );
 
             if (!heartbeatResponse.success) {
-              console.error("Failed to heartbeat run", {
+              this.logger.error("Failed to heartbeat run", {
                 params,
                 error: heartbeatResponse.error,
               });
@@ -227,7 +229,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
         {
           paramsSchema: WorkloadActionParams,
           handler: async ({ reply, params, req }) => {
-            console.debug("Suspend request", { params, headers: req.headers });
+            this.logger.debug("Suspend request", { params, headers: req.headers });
 
             if (!this.checkpointClient) {
               reply.json(
@@ -246,7 +248,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
             const projectRef = this.projectRefFromRequest(req);
 
             if (!runnerId || !deploymentVersion || !projectRef) {
-              console.error("Invalid headers for suspend request", {
+              this.logger.error("Invalid headers for suspend request", {
                 ...params,
                 headers: req.headers,
               });
@@ -282,7 +284,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
             });
 
             if (!suspendResult) {
-              console.error("Failed to suspend run", { params });
+              this.logger.error("Failed to suspend run", { params });
               return;
             }
           },
@@ -294,7 +296,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
         {
           paramsSchema: WorkloadActionParams,
           handler: async ({ req, reply, params }) => {
-            console.debug("Run continuation request", { params });
+            this.logger.debug("Run continuation request", { params });
 
             const continuationResult = await this.workerClient.continueRunExecution(
               params.runFriendlyId,
@@ -303,7 +305,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
             );
 
             if (!continuationResult.success) {
-              console.error("Failed to continue run execution", { params });
+              this.logger.error("Failed to continue run execution", { params });
               reply.json(
                 {
                   ok: false,
@@ -328,7 +330,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
           );
 
           if (!latestSnapshotResponse.success) {
-            console.error("Failed to get latest snapshot", {
+            this.logger.error("Failed to get latest snapshot", {
               runId: params.runFriendlyId,
               error: latestSnapshotResponse.error,
             });
@@ -341,6 +343,31 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
           } satisfies WorkloadRunLatestSnapshotResponseBody);
         },
       })
+      .route(
+        "/api/v1/workload-actions/runs/:runFriendlyId/snapshots/since/:snapshotFriendlyId",
+        "GET",
+        {
+          paramsSchema: WorkloadActionParams,
+          handler: async ({ req, reply, params }) => {
+            const sinceSnapshotResponse = await this.workerClient.getSnapshotsSince(
+              params.runFriendlyId,
+              params.snapshotFriendlyId,
+              this.runnerIdFromRequest(req)
+            );
+
+            if (!sinceSnapshotResponse.success) {
+              this.logger.error("Failed to get snapshots since", {
+                runId: params.runFriendlyId,
+                error: sinceSnapshotResponse.error,
+              });
+              reply.empty(500);
+              return;
+            }
+
+            reply.json(sinceSnapshotResponse.data satisfies WorkloadRunSnapshotsSinceResponseBody);
+          },
+        }
+      )
       .route("/api/v1/workload-actions/runs/:runFriendlyId/logs/debug", "POST", {
         paramsSchema: WorkloadActionParams.pick({ runFriendlyId: true }),
         bodySchema: WorkloadDebugLogRequestBody,
@@ -367,7 +394,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
           );
 
           if (!dequeueResponse.success) {
-            console.error("Failed to get latest snapshot", {
+            this.logger.error("Failed to get latest snapshot", {
               deploymentId: params.deploymentId,
               error: dequeueResponse.error,
             });
@@ -391,14 +418,14 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
     > = io.of("/workload");
 
     websocketServer.on("disconnect", (socket) => {
-      console.log("[WorkloadSocket] disconnect", socket.id);
+      this.logger.log("[WS] disconnect", socket.id);
     });
     websocketServer.use(async (socket, next) => {
-      function setSocketDataFromHeader(
+      const setSocketDataFromHeader = (
         dataKey: keyof typeof socket.data,
         headerName: string,
         required: boolean = true
-      ) {
+      ) => {
         const value = socket.handshake.headers[headerName];
 
         if (value) {
@@ -414,27 +441,26 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
         }
 
         if (required) {
-          console.error("[WorkloadSocket] missing required header", { headerName });
+          this.logger.error("[WS] missing required header", { headerName });
           throw new Error("missing header");
         }
-      }
+      };
 
       try {
         setSocketDataFromHeader("deploymentId", WORKLOAD_HEADERS.DEPLOYMENT_ID);
         setSocketDataFromHeader("runnerId", WORKLOAD_HEADERS.RUNNER_ID);
       } catch (error) {
-        console.error("[WorkloadSocket] setSocketDataFromHeader error", { error });
+        this.logger.error("[WS] setSocketDataFromHeader error", { error });
         socket.disconnect(true);
         return;
       }
 
-      console.debug("[WorkloadSocket] auth success", socket.data);
+      this.logger.debug("[WS] auth success", socket.data);
 
       next();
     });
     websocketServer.on("connection", (socket) => {
-      const logger = new SimpleStructuredLogger("workload-namespace", undefined, {
-        namespace: "workload",
+      const socketLogger = this.logger.child({
         socketId: socket.id,
         socketData: socket.data,
       });
@@ -449,11 +475,11 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
       };
 
       const runConnected = (friendlyId: string) => {
-        logger.debug("runConnected", { ...getSocketMetadata() });
+        socketLogger.debug("runConnected", { ...getSocketMetadata() });
 
         // If there's already a run ID set, we should "disconnect" it from this socket
         if (socket.data.runFriendlyId && socket.data.runFriendlyId !== friendlyId) {
-          logger.debug("runConnected: disconnecting existing run", {
+          socketLogger.debug("runConnected: disconnecting existing run", {
             ...getSocketMetadata(),
             newRunId: friendlyId,
             oldRunId: socket.data.runFriendlyId,
@@ -467,14 +493,14 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
       };
 
       const runDisconnected = (friendlyId: string) => {
-        logger.debug("runDisconnected", { ...getSocketMetadata() });
+        socketLogger.debug("runDisconnected", { ...getSocketMetadata() });
 
         this.runSockets.delete(friendlyId);
         this.emit("runDisconnected", { run: { friendlyId } });
         socket.data.runFriendlyId = undefined;
       };
 
-      logger.log("wsServer socket connected", { ...getSocketMetadata() });
+      socketLogger.log("wsServer socket connected", { ...getSocketMetadata() });
 
       // FIXME: where does this get set?
       if (socket.data.runFriendlyId) {
@@ -482,7 +508,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
       }
 
       socket.on("disconnecting", (reason, description) => {
-        logger.log("Socket disconnecting", { ...getSocketMetadata(), reason, description });
+        socketLogger.log("Socket disconnecting", { ...getSocketMetadata(), reason, description });
 
         if (socket.data.runFriendlyId) {
           runDisconnected(socket.data.runFriendlyId);
@@ -490,11 +516,11 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
       });
 
       socket.on("disconnect", (reason, description) => {
-        logger.log("Socket disconnected", { ...getSocketMetadata(), reason, description });
+        socketLogger.log("Socket disconnected", { ...getSocketMetadata(), reason, description });
       });
 
       socket.on("error", (error) => {
-        logger.error("Socket error", {
+        socketLogger.error("Socket error", {
           ...getSocketMetadata(),
           error: {
             name: error.name,
@@ -505,7 +531,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
       });
 
       socket.on("run:start", async (message) => {
-        const log = logger.child({
+        const log = socketLogger.child({
           eventName: "run:start",
           ...getSocketMetadata(),
           ...message,
@@ -521,7 +547,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
       });
 
       socket.on("run:stop", async (message) => {
-        const log = logger.child({
+        const log = socketLogger.child({
           eventName: "run:stop",
           ...getSocketMetadata(),
           ...message,
@@ -545,7 +571,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
       const runSocket = this.runSockets.get(run.friendlyId);
 
       if (!runSocket) {
-        console.debug("[WorkloadServer] notifyRun: Run socket not found", { run });
+        this.logger.debug("notifyRun: Run socket not found", { run });
 
         this.workerClient.sendDebugLog(run.friendlyId, {
           time: new Date(),
@@ -556,14 +582,14 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
       }
 
       runSocket.emit("run:notify", { version: "1", run });
-      console.debug("[WorkloadServer] run:notify sent", { run });
+      this.logger.debug("run:notify sent", { run });
 
       this.workerClient.sendDebugLog(run.friendlyId, {
         time: new Date(),
         message: "run:notify supervisor -> runner",
       });
     } catch (error) {
-      console.error("[WorkloadServer] Error in notifyRun", { run, error });
+      this.logger.error("Error in notifyRun", { run, error });
 
       this.workerClient.sendDebugLog(run.friendlyId, {
         time: new Date(),

apps/webapp/.babelrc.json

@@ -1,16 +0,0 @@
-{
-  "sourceType": "unambiguous",
-  "presets": [
-    [
-      "@babel/preset-env",
-      {
-        "targets": {
-          "chrome": 100
-        }
-      }
-    ],
-    "@babel/preset-typescript",
-    "@babel/preset-react"
-  ],
-  "plugins": []
-}

apps/webapp/app/assets/icons/AISparkleIcon.tsx

@@ -1,53 +1,31 @@
 export function AISparkleIcon({ className }: { className?: string }) {
   return (
-    <svg className={className} viewBox="0 0 18 18" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <svg
+      className={className}
+      width="24"
+      height="24"
+      viewBox="0 0 24 24"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
       <path
-        d="M14.9806 0.803884C14.8871 0.33646 14.4767 0 14 0C13.5233 0 13.1129 0.33646 13.0194 0.803884L12.7809 1.99644C12.7017 2.3923 12.3923 2.70174 11.9964 2.78091L10.8039 3.01942C10.3365 3.1129 10 3.52332 10 4C10 4.47668 10.3365 4.8871 10.8039 4.98058L11.9964 5.21909C12.3923 5.29826 12.7017 5.6077 12.7809 6.00356L13.0194 7.19612C13.1129 7.66354 13.5233 8 14 8C14.4767 8 14.8871 7.66354 14.9806 7.19612L15.2191 6.00356C15.2983 5.6077 15.6077 5.29826 16.0036 5.21909L17.1961 4.98058C17.6635 4.8871 18 4.47668 18 4C18 3.52332 17.6635 3.1129 17.1961 3.01942L16.0036 2.78091C15.6077 2.70174 15.2983 2.3923 15.2191 1.99644L14.9806 0.803884Z"
-        fill="url(#paint0_linear_11402_36656)"
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M9.14286 4.85718C9.46177 4.85718 9.74205 5.06859 9.82966 5.37523L10.6041 8.08589C10.9431 9.27235 11.8705 10.1998 13.057 10.5388L15.7677 11.3132C16.0743 11.4008 16.2857 11.6811 16.2857 12C16.2857 12.319 16.0743 12.5992 15.7677 12.6868L13.057 13.4613C11.8705 13.8003 10.9431 14.7277 10.6041 15.9142L9.82966 18.6248C9.74205 18.9315 9.46177 19.1429 9.14286 19.1429C8.82394 19.1429 8.54367 18.9315 8.45605 18.6248L7.68158 15.9142C7.34259 14.7277 6.41517 13.8003 5.22871 13.4613L2.51806 12.6868C2.21141 12.5992 2 12.319 2 12C2 11.6811 2.21141 11.4008 2.51806 11.3132L5.22871 10.5388C6.41517 10.1998 7.34259 9.27235 7.68158 8.08589L8.45605 5.37523C8.54367 5.06859 8.82394 4.85718 9.14286 4.85718Z"
+        fill="#7655FD"
       />
       <path
-        d="M5.94868 4.68377C5.81257 4.27543 5.43043 4 5 4C4.56957 4 4.18743 4.27543 4.05132 4.68377L3.36754 6.73509C3.26801 7.03369 3.03369 7.26801 2.73509 7.36754L0.683772 8.05132C0.27543 8.18743 0 8.56957 0 9C0 9.43043 0.27543 9.81257 0.683772 9.94868L2.73509 10.6325C3.03369 10.732 3.26801 10.9663 3.36754 11.2649L4.05132 13.3162C4.18743 13.7246 4.56957 14 5 14C5.43043 14 5.81257 13.7246 5.94868 13.3162L6.63246 11.2649C6.73199 10.9663 6.96631 10.732 7.26491 10.6325L9.31623 9.94868C9.72457 9.81257 10 9.43043 10 9C10 8.56957 9.72457 8.18743 9.31623 8.05132L7.26491 7.36754C6.96631 7.26801 6.73199 7.03369 6.63246 6.73509L5.94868 4.68377Z"
-        fill="url(#paint1_linear_11402_36656)"
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M17.7143 2C18.0421 2 18.3278 2.22307 18.4072 2.54105L18.6538 3.5272C18.8777 4.42291 19.5771 5.12229 20.4728 5.34622L21.459 5.59276C21.7769 5.67225 22 5.95795 22 6.28571C22 6.61348 21.7769 6.89918 21.459 6.97867L20.4728 7.22521C19.5771 7.44914 18.8777 8.14851 18.6538 9.04423L18.4072 10.0304C18.3278 10.3484 18.0421 10.5714 17.7143 10.5714C17.3865 10.5714 17.1008 10.3484 17.0213 10.0304L16.7748 9.04423C16.5509 8.14852 15.8515 7.44914 14.9558 7.22521L13.9696 6.97867C13.6516 6.89918 13.4286 6.61348 13.4286 6.28571C13.4286 5.95795 13.6516 5.67225 13.9696 5.59276L14.9558 5.34622C15.8515 5.12229 16.5509 4.42291 16.7748 3.5272L17.0213 2.54105C17.1008 2.22307 17.3865 2 17.7143 2Z"
+        fill="#D946EF"
       />
       <path
-        d="M12.9487 12.6838C12.8126 12.2754 12.4304 12 12 12C11.5696 12 11.1874 12.2754 11.0513 12.6838L10.8675 13.2351C10.768 13.5337 10.5337 13.768 10.2351 13.8675L9.68377 14.0513C9.27543 14.1874 9 14.5696 9 15C9 15.4304 9.27543 15.8126 9.68377 15.9487L10.2351 16.1325C10.5337 16.232 10.768 16.4663 10.8675 16.7649L11.0513 17.3162C11.1874 17.7246 11.5696 18 12 18C12.4304 18 12.8126 17.7246 12.9487 17.3162L13.1325 16.7649C13.232 16.4663 13.4663 16.232 13.7649 16.1325L14.3162 15.9487C14.7246 15.8126 15 15.4304 15 15C15 14.5696 14.7246 14.1874 14.3162 14.0513L13.7649 13.8675C13.4663 13.768 13.232 13.5337 13.1325 13.2351L12.9487 12.6838Z"
-        fill="url(#paint2_linear_11402_36656)"
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M16.2857 14.8572C16.5932 14.8572 16.8661 15.0539 16.9633 15.3456L17.3388 16.472C17.481 16.8986 17.8157 17.2333 18.2423 17.3755L19.3687 17.751C19.6604 17.8482 19.8571 18.1212 19.8571 18.4286C19.8571 18.7361 19.6604 19.009 19.3687 19.1062L18.2423 19.4817C17.8157 19.6239 17.481 19.9586 17.3388 20.3852L16.9633 21.5116C16.8661 21.8033 16.5932 22 16.2857 22C15.9783 22 15.7053 21.8033 15.6081 21.5116L15.2326 20.3852C15.0904 19.9586 14.7557 19.6239 14.3291 19.4817L13.2027 19.1062C12.911 19.009 12.7143 18.7361 12.7143 18.4286C12.7143 18.1212 12.911 17.8482 13.2027 17.751L14.3291 17.3755C14.7557 17.2333 15.0904 16.8986 15.2326 16.472L15.6081 15.3456C15.7053 15.0539 15.9783 14.8572 16.2857 14.8572Z"
+        fill="#4F46E5"
       />
-      <defs>
-        <linearGradient
-          id="paint0_linear_11402_36656"
-          x1="9"
-          y1="0"
-          x2="9"
-          y2="18"
-          gradientUnits="userSpaceOnUse"
-        >
-          <stop stopColor="#E543FF" />
-          <stop offset="1" stopColor="#286399" />
-        </linearGradient>
-        <linearGradient
-          id="paint1_linear_11402_36656"
-          x1="9"
-          y1="0"
-          x2="9"
-          y2="18"
-          gradientUnits="userSpaceOnUse"
-        >
-          <stop stopColor="#E543FF" />
-          <stop offset="1" stopColor="#286399" />
-        </linearGradient>
-        <linearGradient
-          id="paint2_linear_11402_36656"
-          x1="9"
-          y1="0"
-          x2="9"
-          y2="18"
-          gradientUnits="userSpaceOnUse"
-        >
-          <stop stopColor="#E543FF" />
-          <stop offset="1" stopColor="#286399" />
-        </linearGradient>
-      </defs>
     </svg>
   );
 }

apps/webapp/app/assets/icons/KeyboardEnterIcon.tsx

@@ -0,0 +1,12 @@
+export function KeyboardEnterIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M12 5H18C19.6569 5 21 6.34315 21 8V13C21 14.6569 19.6569 16 18 16H8"
+        stroke="currentColor"
+        strokeWidth="2"
+      />
+      <path d="M2 16L8 12L8 20L2 16Z" fill="currentColor" />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/TextInlineIcon.tsx

@@ -0,0 +1,41 @@
+export function TextInlineIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M3 3V21"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M13 21L13 16"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M13 8L13 3"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M7 12L20 12"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M17.5 15.5L21 12L17.5 8.5"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/TextWrapIcon.tsx

@@ -0,0 +1,34 @@
+export function TextWrapIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M3 3V21"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M21 21V3"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M7 7H14C15.6569 7 17 8.34315 17 10V13C17 14.6569 15.6569 16 14 16H9"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M11 13L8 16L11 19"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+    </svg>
+  );
+}

apps/webapp/app/components/ImpersonationBanner.tsx

@@ -1,21 +1,29 @@
 import { UserMinusIcon } from "@heroicons/react/20/solid";
 import { Form } from "@remix-run/react";
 import { Button } from "./primitives/Buttons";
+import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "./primitives/Tooltip";
 
 export function ImpersonationBanner() {
   return (
-    <div className="w-full">
-      <Form action="/resources/impersonation" method="delete" reloadDocument className="w-full">
-        <Button
-          type="submit"
-          variant="small-menu-item"
-          LeadingIcon={UserMinusIcon}
-          fullWidth
-          textAlignLeft
-          className="text-amber-400"
-        >
-          Stop impersonating
-        </Button>
+    <div>
+      <Form action="/resources/impersonation" method="delete" reloadDocument>
+        <TooltipProvider disableHoverableContent={true}>
+          <Tooltip>
+            <TooltipTrigger>
+              <Button
+                type="submit"
+                variant="small-menu-item"
+                LeadingIcon={UserMinusIcon}
+                fullWidth
+                textAlignLeft
+                className="text-amber-400"
+              />
+            </TooltipTrigger>
+            <TooltipContent side="bottom" className={"text-xs"}>
+              Stop impersonating
+            </TooltipContent>
+          </Tooltip>
+        </TooltipProvider>
       </Form>
     </div>
   );

apps/webapp/app/components/Shortcuts.tsx

@@ -11,6 +11,8 @@ import {
 } from "./primitives/SheetV3";
 import { ShortcutKey } from "./primitives/ShortcutKey";
 import { Button } from "./primitives/Buttons";
+import { useState } from "react";
+import { useShortcutKeys } from "~/hooks/useShortcutKeys";
 
 export function Shortcuts() {
   return (
@@ -23,121 +25,147 @@ export function Shortcuts() {
           data-action="shortcuts"
           fullWidth
           textAlignLeft
-          shortcut={{ modifiers: ["shift"], key: "?" }}
+          shortcut={{ modifiers: ["shift"], key: "?", enabled: false }}
           className="gap-x-0 pl-0.5"
           iconSpacing="gap-x-0.5"
         >
           Shortcuts
         </Button>
       </SheetTrigger>
-      <SheetContent>
-        <SheetHeader>
-          <SheetTitle>
-            <div className="flex items-center gap-x-2">
-              <Keyboard className="size-5 text-indigo-500" />
-              <span className="font-sans text-base font-medium text-text-bright">
-                Keyboard shortcuts
-              </span>
-            </div>
-          </SheetTitle>
-          <div className="space-y-6 px-4 pb-4 pt-2">
-            <div className="space-y-3">
-              <Header3>General</Header3>
-              <Shortcut name="Close">
-                <ShortcutKey shortcut={{ key: "esc" }} variant="medium/bright" />
-              </Shortcut>
-              <Shortcut name="Confirm">
-                <ShortcutKey shortcut={{ modifiers: ["mod"] }} variant="medium/bright" />
-                <ShortcutKey shortcut={{ key: "enter" }} variant="medium/bright" />
-              </Shortcut>
-              <Shortcut name="Filter">
-                <ShortcutKey shortcut={{ key: "f" }} variant="medium/bright" />
-              </Shortcut>
-              <Shortcut name="Select filter">
-                <ShortcutKey shortcut={{ key: "1" }} variant="medium/bright" />
-                <Paragraph variant="small" className="ml-1.5">
-                  to
-                </Paragraph>
-                <ShortcutKey shortcut={{ key: "9" }} variant="medium/bright" />
-              </Shortcut>
-              <Shortcut name="Previous page">
-                <ShortcutKey shortcut={{ key: "j" }} variant="medium/bright" />
-              </Shortcut>
-              <Shortcut name="Next page">
-                <ShortcutKey shortcut={{ key: "k" }} variant="medium/bright" />
-              </Shortcut>
-              <Shortcut name="Help & Feedback">
-                <ShortcutKey shortcut={{ key: "h" }} variant="medium/bright" />
-              </Shortcut>
-            </div>
-            <div className="space-y-3">
-              <Header3>Runs page</Header3>
-              <Shortcut name="Bulk action: Cancel runs">
-                <ShortcutKey shortcut={{ key: "c" }} variant="medium/bright" />
-              </Shortcut>
-              <Shortcut name="Bulk action: Replay runs">
-                <ShortcutKey shortcut={{ key: "r" }} variant="medium/bright" />
-              </Shortcut>
-              <Shortcut name="Bulk action: Clear selection">
-                <ShortcutKey shortcut={{ key: "esc" }} variant="medium/bright" />
-              </Shortcut>
-            </div>
-            <div className="space-y-3">
-              <Header3>Run page</Header3>
-              <Shortcut name="Replay run">
-                <ShortcutKey shortcut={{ key: "r" }} variant="medium/bright" />
-              </Shortcut>
-              <Shortcut name="Overview">
-                <ShortcutKey shortcut={{ key: "o" }} variant="medium/bright" />
-              </Shortcut>
-              <Shortcut name="Details">
-                <ShortcutKey shortcut={{ key: "d" }} variant="medium/bright" />
-              </Shortcut>
-              <Shortcut name="Context">
-                <ShortcutKey shortcut={{ key: "c" }} variant="medium/bright" />
-              </Shortcut>
-              <Shortcut name="Metadata">
-                <ShortcutKey shortcut={{ key: "m" }} variant="medium/bright" />
-              </Shortcut>
-              <Shortcut name="Navigate">
-                <ShortcutKey shortcut={{ key: "arrowup" }} variant="medium/bright" />
-                <ShortcutKey shortcut={{ key: "arrowdown" }} variant="medium/bright" />
-                <ShortcutKey shortcut={{ key: "arrowleft" }} variant="medium/bright" />
-                <ShortcutKey shortcut={{ key: "arrowright" }} variant="medium/bright" />
-              </Shortcut>
-              <Shortcut name="Expand all">
-                <ShortcutKey shortcut={{ key: "e" }} variant="medium/bright" />
-              </Shortcut>
-              <Shortcut name="Collapse all">
-                <ShortcutKey shortcut={{ key: "w" }} variant="medium/bright" />
-              </Shortcut>
-              <Shortcut name="Toggle level">
-                <ShortcutKey shortcut={{ key: "0" }} variant="medium/bright" />
-                <Paragraph variant="small" className="ml-1.5">
-                  to
-                </Paragraph>
-                <ShortcutKey shortcut={{ key: "9" }} variant="medium/bright" />
-              </Shortcut>
-            </div>
-            <div className="space-y-3">
-              <Header3>Schedules page</Header3>
-              <Shortcut name="New schedule">
-                <ShortcutKey shortcut={{ key: "n" }} variant="medium/bright" />
-              </Shortcut>
-            </div>
-            <div className="space-y-3">
-              <Header3>Alerts page</Header3>
-              <Shortcut name="New alert">
-                <ShortcutKey shortcut={{ key: "n" }} variant="medium/bright" />
-              </Shortcut>
-            </div>
-          </div>
-        </SheetHeader>
-      </SheetContent>
+      <ShortcutContent />
     </Sheet>
   );
 }
 
+export function ShortcutsAutoOpen() {
+  const [isOpen, setIsOpen] = useState(false);
+
+  useShortcutKeys({
+    shortcut: { modifiers: ["shift"], key: "?" },
+    action: () => {
+      setIsOpen(true);
+    },
+  });
+
+  return (
+    <Sheet open={isOpen} onOpenChange={setIsOpen}>
+      <ShortcutContent />
+    </Sheet>
+  );
+}
+
+function ShortcutContent() {
+  return (
+    <SheetContent>
+      <SheetHeader>
+        <SheetTitle>
+          <div className="flex items-center gap-x-2">
+            <Keyboard className="size-5 text-indigo-500" />
+            <span className="font-sans text-base font-medium text-text-bright">
+              Keyboard shortcuts
+            </span>
+          </div>
+        </SheetTitle>
+        <div className="space-y-6 px-4 pb-4 pt-2">
+          <div className="space-y-3">
+            <Header3>General</Header3>
+            <Shortcut name="Close">
+              <ShortcutKey shortcut={{ key: "esc" }} variant="medium/bright" />
+            </Shortcut>
+            <Shortcut name="Confirm">
+              <ShortcutKey shortcut={{ modifiers: ["mod"] }} variant="medium/bright" />
+              <ShortcutKey shortcut={{ key: "enter" }} variant="medium/bright" />
+            </Shortcut>
+            <Shortcut name="Ask AI">
+              <ShortcutKey shortcut={{ modifiers: ["mod"], key: "/" }} variant="medium/bright" />
+            </Shortcut>
+            <Shortcut name="Filter">
+              <ShortcutKey shortcut={{ key: "f" }} variant="medium/bright" />
+            </Shortcut>
+            <Shortcut name="Select filter">
+              <ShortcutKey shortcut={{ key: "1" }} variant="medium/bright" />
+              <Paragraph variant="small" className="ml-1.5">
+                to
+              </Paragraph>
+              <ShortcutKey shortcut={{ key: "9" }} variant="medium/bright" />
+            </Shortcut>
+            <Shortcut name="Previous page">
+              <ShortcutKey shortcut={{ key: "j" }} variant="medium/bright" />
+            </Shortcut>
+            <Shortcut name="Next page">
+              <ShortcutKey shortcut={{ key: "k" }} variant="medium/bright" />
+            </Shortcut>
+            <Shortcut name="Help & Feedback">
+              <ShortcutKey shortcut={{ key: "h" }} variant="medium/bright" />
+            </Shortcut>
+          </div>
+          <div className="space-y-3">
+            <Header3>Runs page</Header3>
+            <Shortcut name="Bulk action: Cancel runs">
+              <ShortcutKey shortcut={{ key: "c" }} variant="medium/bright" />
+            </Shortcut>
+            <Shortcut name="Bulk action: Replay runs">
+              <ShortcutKey shortcut={{ key: "r" }} variant="medium/bright" />
+            </Shortcut>
+            <Shortcut name="Bulk action: Clear selection">
+              <ShortcutKey shortcut={{ key: "esc" }} variant="medium/bright" />
+            </Shortcut>
+          </div>
+          <div className="space-y-3">
+            <Header3>Run page</Header3>
+            <Shortcut name="Replay run">
+              <ShortcutKey shortcut={{ key: "r" }} variant="medium/bright" />
+            </Shortcut>
+            <Shortcut name="Overview">
+              <ShortcutKey shortcut={{ key: "o" }} variant="medium/bright" />
+            </Shortcut>
+            <Shortcut name="Details">
+              <ShortcutKey shortcut={{ key: "d" }} variant="medium/bright" />
+            </Shortcut>
+            <Shortcut name="Context">
+              <ShortcutKey shortcut={{ key: "c" }} variant="medium/bright" />
+            </Shortcut>
+            <Shortcut name="Metadata">
+              <ShortcutKey shortcut={{ key: "m" }} variant="medium/bright" />
+            </Shortcut>
+            <Shortcut name="Navigate">
+              <ShortcutKey shortcut={{ key: "arrowup" }} variant="medium/bright" />
+              <ShortcutKey shortcut={{ key: "arrowdown" }} variant="medium/bright" />
+              <ShortcutKey shortcut={{ key: "arrowleft" }} variant="medium/bright" />
+              <ShortcutKey shortcut={{ key: "arrowright" }} variant="medium/bright" />
+            </Shortcut>
+            <Shortcut name="Expand all">
+              <ShortcutKey shortcut={{ key: "e" }} variant="medium/bright" />
+            </Shortcut>
+            <Shortcut name="Collapse all">
+              <ShortcutKey shortcut={{ key: "w" }} variant="medium/bright" />
+            </Shortcut>
+            <Shortcut name="Toggle level">
+              <ShortcutKey shortcut={{ key: "0" }} variant="medium/bright" />
+              <Paragraph variant="small" className="ml-1.5">
+                to
+              </Paragraph>
+              <ShortcutKey shortcut={{ key: "9" }} variant="medium/bright" />
+            </Shortcut>
+          </div>
+          <div className="space-y-3">
+            <Header3>Schedules page</Header3>
+            <Shortcut name="New schedule">
+              <ShortcutKey shortcut={{ key: "n" }} variant="medium/bright" />
+            </Shortcut>
+          </div>
+          <div className="space-y-3">
+            <Header3>Alerts page</Header3>
+            <Shortcut name="New alert">
+              <ShortcutKey shortcut={{ key: "n" }} variant="medium/bright" />
+            </Shortcut>
+          </div>
+        </div>
+      </SheetHeader>
+    </SheetContent>
+  );
+}
+
 function Shortcut({ children, name }: { children: React.ReactNode; name: string }) {
   return (
     <div className="flex items-center justify-between gap-x-2">

apps/webapp/app/components/admin/debugRun.tsx

@@ -66,7 +66,15 @@ function DebugRunContent({ friendlyId }: { friendlyId: string }) {
   );
 }
 
-function DebugRunData({
+function DebugRunData(props: UseDataFunctionReturn<typeof loader>) {
+  if (props.engine === "V1") {
+    return <DebugRunDataEngineV1 {...props} />;
+  }
+
+  return <DebugRunDataEngineV2 {...props} />;
+}
+
+function DebugRunDataEngineV1({
   run,
   queueConcurrencyLimit,
   queueCurrentConcurrency,
@@ -338,3 +346,55 @@ function DebugRunData({
     </Property.Table>
   );
 }
+
+function DebugRunDataEngineV2({
+  run,
+  queueConcurrencyLimit,
+  queueCurrentConcurrency,
+  envConcurrencyLimit,
+  envCurrentConcurrency,
+  keys,
+}: UseDataFunctionReturn<typeof loader>) {
+  return (
+    <Property.Table>
+      <Property.Item>
+        <Property.Label>ID</Property.Label>
+        <Property.Value className="flex items-center gap-2">
+          <ClipboardField value={run.id} variant="tertiary/small" iconButton />
+        </Property.Value>
+      </Property.Item>
+      <Property.Item>
+        <Property.Label>Queue current concurrency</Property.Label>
+        <Property.Value className="flex items-center gap-2">
+          <span>{queueCurrentConcurrency ?? "0"}</span>
+        </Property.Value>
+      </Property.Item>
+      <Property.Item>
+        <Property.Label>Queue concurrency limit</Property.Label>
+        <Property.Value className="flex items-center gap-2">
+          <span>{queueConcurrencyLimit ?? "Not set"}</span>
+        </Property.Value>
+      </Property.Item>
+      <Property.Item>
+        <Property.Label>Env current concurrency</Property.Label>
+        <Property.Value className="flex items-center gap-2">
+          <span>{envCurrentConcurrency ?? "0"}</span>
+        </Property.Value>
+      </Property.Item>
+      <Property.Item>
+        <Property.Label>Env concurrency limit</Property.Label>
+        <Property.Value className="flex items-center gap-2">
+          <span>{envConcurrencyLimit ?? "Not set"}</span>
+        </Property.Value>
+      </Property.Item>
+      {keys.map((key) => (
+        <Property.Item>
+          <Property.Label>{key.label}</Property.Label>
+          <Property.Value className="flex items-center gap-2">
+            <ClipboardField value={key.key} variant="tertiary/small" iconButton />
+          </Property.Value>
+        </Property.Item>
+      ))}
+    </Property.Table>
+  );
+}

apps/webapp/app/components/code/CodeBlock.tsx

@@ -3,11 +3,13 @@ import { Clipboard, ClipboardCheck } from "lucide-react";
 import type { Language, PrismTheme } from "prism-react-renderer";
 import { Highlight, Prism } from "prism-react-renderer";
 import { forwardRef, ReactNode, useCallback, useEffect, useState } from "react";
+import { TextWrapIcon } from "~/assets/icons/TextWrapIcon";
 import { cn } from "~/utils/cn";
 import { Button } from "../primitives/Buttons";
 import { Dialog, DialogContent, DialogHeader, DialogTitle } from "../primitives/Dialog";
 import { Paragraph } from "../primitives/Paragraph";
 import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "../primitives/Tooltip";
+import { TextInlineIcon } from "~/assets/icons/TextInlineIcon";
 
 //This is a fork of https://github.com/mantinedev/mantine/blob/master/src/mantine-prism/src/Prism/Prism.tsx
 //it didn't support highlighting lines by dimming the rest of the code, or animations on the highlighting
@@ -31,6 +33,9 @@ type CodeBlockProps = {
   /** Show copy to clipboard button */
   showCopyButton?: boolean;
 
+  /** Show text wrapping button */
+  showTextWrapping?: boolean;
+
   /** Display line numbers */
   showLineNumbers?: boolean;
 
@@ -183,6 +188,7 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
   (
     {
       showCopyButton = true,
+      showTextWrapping = false,
       showLineNumbers = true,
       showOpenInModal = true,
       highlightedRanges,
@@ -202,6 +208,7 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
     const [copied, setCopied] = useState(false);
     const [modalCopied, setModalCopied] = useState(false);
     const [isModalOpen, setIsModalOpen] = useState(false);
+    const [isWrapped, setIsWrapped] = useState(false);
 
     const onCopied = useCallback(
       (event: React.MouseEvent<HTMLButtonElement>) => {
@@ -263,6 +270,25 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
               showChrome ? "right-1.5 top-1.5" : "top-2.5"
             )}
           >
+            {showTextWrapping && (
+              <TooltipProvider>
+                <Tooltip disableHoverableContent>
+                  <TooltipTrigger
+                    onClick={() => setIsWrapped(!isWrapped)}
+                    className="transition-colors focus-custom hover:cursor-pointer hover:text-text-bright"
+                  >
+                    {isWrapped ? (
+                      <TextInlineIcon className="size-4" />
+                    ) : (
+                      <TextWrapIcon className="size-4" />
+                    )}
+                  </TooltipTrigger>
+                  <TooltipContent side="left" className="text-xs">
+                    {isWrapped ? "Unwrap" : "Wrap"}
+                  </TooltipContent>
+                </Tooltip>
+              </TooltipProvider>
+            )}
             {showCopyButton && (
               <TooltipProvider>
                 <Tooltip open={copied || mouseOver} disableHoverableContent>
@@ -311,16 +337,27 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
               maxLineWidth={maxLineWidth}
               className="px-2 py-3"
               preClassName="text-xs"
+              isWrapped={isWrapped}
             />
           ) : (
             <div
               dir="ltr"
-              className="overflow-auto px-2 py-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600"
+              className={cn(
+                "px-2 py-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600",
+                !isWrapped && "overflow-x-auto",
+                isWrapped && "overflow-y-auto"
+              )}
               style={{
                 maxHeight,
               }}
             >
-              <pre className="relative mr-2 p-2 font-mono text-xs leading-relaxed" dir="ltr">
+              <pre
+                className={cn(
+                  "relative mr-2 p-2 font-mono text-xs leading-relaxed",
+                  isWrapped && "[&_span]:whitespace-pre-wrap [&_span]:break-words"
+                )}
+                dir="ltr"
+              >
                 {code}
               </pre>
             </div>
@@ -355,6 +392,7 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
                 maxLineWidth={maxLineWidth}
                 className="min-h-full"
                 preClassName="text-sm"
+                isWrapped={isWrapped}
               />
             ) : (
               <div
@@ -410,6 +448,7 @@ type HighlightCodeProps = {
   maxLineWidth?: number;
   className?: string;
   preClassName?: string;
+  isWrapped: boolean;
 };
 
 function HighlightCode({
@@ -421,11 +460,11 @@ function HighlightCode({
   maxLineWidth,
   className,
   preClassName,
+  isWrapped,
 }: HighlightCodeProps) {
   const [isLoaded, setIsLoaded] = useState(false);
 
   useEffect(() => {
-    // This ensures the language definitions are loaded
     Promise.all([
       //@ts-ignore
       import("prismjs/components/prism-json"),
@@ -434,16 +473,23 @@ function HighlightCode({
     ]).then(() => setIsLoaded(true));
   }, []);
 
+  const containerClasses = cn(
+    "px-3 py-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600",
+    !isWrapped && "overflow-x-auto",
+    isWrapped && "overflow-y-auto",
+    className
+  );
+
+  const preClasses = cn(
+    "relative mr-2 font-mono leading-relaxed",
+    preClassName,
+    isWrapped && "[&_span]:whitespace-pre-wrap [&_span]:break-words"
+  );
+
   if (!isLoaded) {
     return (
-      <div
-        dir="ltr"
-        className={cn(
-          "overflow-auto px-3 py-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600",
-          className
-        )}
-      >
-        <pre className={cn("relative mr-2 font-mono leading-relaxed", preClassName)}>{code}</pre>
+      <div dir="ltr" className={containerClasses}>
+        <pre className={preClasses}>{code}</pre>
       </div>
     );
   }
@@ -457,22 +503,8 @@ function HighlightCode({
         getLineProps,
         getTokenProps,
       }) => (
-        <div
-          dir="ltr"
-          className={cn(
-            "overflow-auto px-3 py-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600",
-            className
-          )}
-        >
-          <pre
-            className={cn(
-              "relative mr-2 font-mono leading-relaxed",
-              inheritedClassName,
-              preClassName
-            )}
-            style={inheritedStyle}
-            dir="ltr"
-          >
+        <div dir="ltr" className={containerClasses}>
+          <pre className={cn(preClasses, inheritedClassName)} style={inheritedStyle} dir="ltr">
             {tokens
               .map((line, index) => {
                 if (index === tokens.length - 1 && line.length === 1 && line[0].content === "\n") {
@@ -495,7 +527,8 @@ function HighlightCode({
                     {...lineProps}
                     className={cn(
                       "flex w-full justify-start transition-opacity duration-500",
-                      lineProps.className
+                      lineProps.className,
+                      isWrapped && "flex-wrap"
                     )}
                     style={{
                       opacity: shouldDim ? dimAmount : undefined,
@@ -504,9 +537,10 @@ function HighlightCode({
                   >
                     {showLineNumbers && (
                       <div
-                        className={
-                          "mr-2 flex-none select-none text-right text-charcoal-500 transition-opacity duration-500"
-                        }
+                        className={cn(
+                          "mr-2 flex-none select-none text-right text-charcoal-500 transition-opacity duration-500",
+                          isWrapped && "sticky left-0"
+                        )}
                         style={{
                           width: `calc(8 * ${(maxLineWidth as number) / 16}rem)`,
                         }}

apps/webapp/app/components/metrics/BigNumber.tsx

@@ -1,7 +1,10 @@
 import { type ReactNode } from "react";
-import { AnimatedNumber } from "../primitives/AnimatedNumber";
-import { Spinner } from "../primitives/Spinner";
 import { cn } from "~/utils/cn";
+import { formatNumber, formatNumberCompact } from "~/utils/numberFormatter";
+import { Header3 } from "../primitives/Headers";
+import { Spinner } from "../primitives/Spinner";
+import { SimpleTooltip } from "../primitives/Tooltip";
+import { AnimatedNumber } from "../primitives/AnimatedNumber";
 
 interface BigNumberProps {
   title: ReactNode;
@@ -13,6 +16,7 @@ interface BigNumberProps {
   accessory?: ReactNode;
   suffix?: string;
   suffixClassName?: string;
+  compactThreshold?: number;
 }
 
 export function BigNumber({
@@ -25,25 +29,39 @@ export function BigNumber({
   accessory,
   animate = false,
   loading = false,
+  compactThreshold,
 }: BigNumberProps) {
   const v = value ?? defaultValue;
+
+  const shouldCompact =
+    typeof compactThreshold === "number" && v !== undefined && v >= compactThreshold;
+
   return (
-    <div className="grid grid-rows-[1.5rem_auto] gap-4 rounded-sm border border-grid-dimmed bg-background-bright p-4">
-      <div className="flex items-center justify-between">
-        <div className="text-2sm text-text-dimmed">{title}</div>
+    <div className="flex flex-col justify-between gap-4 rounded-sm border border-grid-dimmed bg-background-bright p-4">
+      <div className="flex flex-wrap items-center justify-between gap-2">
+        <Header3 className="leading-6">{title}</Header3>
         {accessory && <div className="flex-shrink-0">{accessory}</div>}
       </div>
       <div
         className={cn(
-          "h-[3.75rem] text-[3.75rem] font-normal tabular-nums leading-none text-text-bright",
+          "text-[3.75rem] font-normal tabular-nums leading-none text-text-bright",
           valueClassName
         )}
       >
         {loading ? (
           <Spinner className="size-6" />
         ) : v !== undefined ? (
-          <div className="flex items-baseline gap-1">
-            {animate ? <AnimatedNumber value={v} /> : v}
+          <div className="flex flex-wrap items-baseline gap-2">
+            {shouldCompact ? (
+              <SimpleTooltip
+                button={animate ? <AnimatedNumber value={v} /> : formatNumberCompact(v)}
+                content={formatNumber(v)}
+              />
+            ) : animate ? (
+              <AnimatedNumber value={v} />
+            ) : (
+              formatNumber(v)
+            )}
             {suffix && <div className={cn("text-xs", suffixClassName)}>{suffix}</div>}
           </div>
         ) : (

apps/webapp/app/components/navigation/HelpAndFeedbackPopover.tsx

@@ -2,9 +2,9 @@ import {
   ArrowUpRightIcon,
   BookOpenIcon,
   CalendarDaysIcon,
-  ChatBubbleLeftEllipsisIcon,
   EnvelopeIcon,
   LightBulbIcon,
+  QuestionMarkCircleIcon,
   SignalIcon,
   StarIcon,
 } from "@heroicons/react/20/solid";
@@ -12,6 +12,7 @@ import { DiscordIcon, SlackIcon } from "@trigger.dev/companyicons";
 import { Fragment, useState } from "react";
 import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route";
 import { Feedback } from "../Feedback";
+import { Shortcuts } from "../Shortcuts";
 import { StepContentContainer } from "../StepContentContainer";
 import { Button } from "../primitives/Buttons";
 import { ClipboardField } from "../primitives/ClipboardField";
@@ -21,16 +22,21 @@ import { Paragraph } from "../primitives/Paragraph";
 import { Popover, PopoverContent, PopoverSideMenuTrigger } from "../primitives/Popover";
 import { StepNumber } from "../primitives/StepNumber";
 import { MenuCount, SideMenuItem } from "./SideMenuItem";
-import { Shortcuts } from "../Shortcuts";
-export function HelpAndFeedback() {
+
+export function HelpAndFeedback({ disableShortcut = false }: { disableShortcut?: boolean }) {
   const [isHelpMenuOpen, setHelpMenuOpen] = useState(false);
   const currentPlan = useCurrentPlan();
 
   return (
     <Popover onOpenChange={(open) => setHelpMenuOpen(open)}>
-      <PopoverSideMenuTrigger isOpen={isHelpMenuOpen} shortcut={{ key: "h" }}>
+      <PopoverSideMenuTrigger
+        isOpen={isHelpMenuOpen}
+        shortcut={{ key: "h", enabledOnInputElements: false }}
+        className="grow pr-2"
+        disabled={disableShortcut}
+      >
         <div className="flex items-center gap-1.5">
-          <ChatBubbleLeftEllipsisIcon className="size-4 text-success" />
+          <QuestionMarkCircleIcon className="size-4 text-success" />
           Help & Feedback
         </div>
       </PopoverSideMenuTrigger>

apps/webapp/app/components/navigation/SideMenu.tsx

@@ -16,10 +16,12 @@ import {
   RectangleStackIcon,
   ServerStackIcon,
   Squares2X2Icon,
+  UsersIcon,
 } from "@heroicons/react/20/solid";
 import { useNavigation } from "@remix-run/react";
 import { useEffect, useRef, useState, type ReactNode } from "react";
 import simplur from "simplur";
+import { AISparkleIcon } from "~/assets/icons/AISparkleIcon";
 import { RunsIconExtraSmall } from "~/assets/icons/RunsIcon";
 import { TaskIconSmall } from "~/assets/icons/TaskIcon";
 import { WaitpointTokenIcon } from "~/assets/icons/WaitpointTokenIcon";
@@ -27,12 +29,15 @@ import { Avatar } from "~/components/primitives/Avatar";
 import { type MatchedEnvironment } from "~/hooks/useEnvironment";
 import { type MatchedOrganization } from "~/hooks/useOrganizations";
 import { type MatchedProject } from "~/hooks/useProject";
+import { useHasAdminAccess } from "~/hooks/useUser";
 import { type User } from "~/models/user.server";
 import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route";
 import { type FeedbackType } from "~/routes/resources.feedback";
+import { IncidentStatusPanel } from "~/routes/resources.incidents";
 import { cn } from "~/utils/cn";
 import {
   accountPath,
+  adminPath,
   logoutPath,
   newOrganizationPath,
   newProjectPath,
@@ -55,6 +60,7 @@ import {
   v3UsagePath,
   v3WaitpointTokensPath,
 } from "~/utils/pathBuilder";
+import { useKapaWidget } from "../../hooks/useKapaWidget";
 import { FreePlanUsage } from "../billing/FreePlanUsage";
 import { ConnectionIcon, DevPresencePanel, useDevPresence } from "../DevPresence";
 import { ImpersonationBanner } from "../ImpersonationBanner";
@@ -68,8 +74,10 @@ import {
   PopoverMenuItem,
   PopoverTrigger,
 } from "../primitives/Popover";
+import { ShortcutKey } from "../primitives/ShortcutKey";
 import { TextLink } from "../primitives/TextLink";
 import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "../primitives/Tooltip";
+import { ShortcutsAutoOpen } from "../Shortcuts";
 import { UserProfilePhoto } from "../UserProfilePhoto";
 import { EnvironmentSelector } from "./EnvironmentSelector";
 import { HelpAndFeedback } from "./HelpAndFeedbackPopover";
@@ -106,6 +114,7 @@ export function SideMenu({
   const currentPlan = useCurrentPlan();
   const { isConnected } = useDevPresence();
   const isFreeUser = currentPlan?.v3Subscription?.isPaying === false;
+  const isAdmin = useHasAdminAccess();
 
   useEffect(() => {
     const handleScroll = () => {
@@ -139,6 +148,20 @@ export function SideMenu({
           project={project}
           user={user}
         />
+        {isAdmin && !user.isImpersonating ? (
+          <TooltipProvider disableHoverableContent={true}>
+            <Tooltip>
+              <TooltipTrigger>
+                <LinkButton variant="minimal/medium" to={adminPath()} TrailingIcon={UsersIcon} />
+              </TooltipTrigger>
+              <TooltipContent side="bottom" className={"text-xs"}>
+                Admin dashboard
+              </TooltipContent>
+            </Tooltip>
+          </TooltipProvider>
+        ) : isAdmin && user.isImpersonating ? (
+          <ImpersonationBanner />
+        ) : null}
       </div>
       <div
         className="overflow-hidden overflow-y-auto pt-2 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600"
@@ -275,14 +298,19 @@ export function SideMenu({
           </SideMenuSection>
         </div>
       </div>
-      <div className="flex flex-col gap-1 border-t border-grid-bright p-1">
-        <HelpAndFeedback />
-        {isFreeUser && (
-          <FreePlanUsage
-            to={v3BillingPath(organization)}
-            percentage={currentPlan.v3Usage.usagePercentage}
-          />
-        )}
+      <div>
+        <IncidentStatusPanel />
+        <div className="flex flex-col gap-1 border-t border-grid-bright p-1">
+          <div className="flex w-full items-center justify-between">
+            <HelpAndAI />
+          </div>
+          {isFreeUser && (
+            <FreePlanUsage
+              to={v3BillingPath(organization)}
+              percentage={currentPlan.v3Usage.usagePercentage}
+            />
+          )}
+        </div>
       </div>
     </div>
   );
@@ -319,10 +347,7 @@ function ProjectSelector({
       <PopoverArrowTrigger
         isOpen={isOrgMenuOpen}
         overflowHidden
-        className={cn(
-          "h-8 w-full justify-between py-1 pl-1.5",
-          user.isImpersonating && "border border-dashed border-amber-400"
-        )}
+        className="h-8 w-full justify-between py-1 pl-1.5"
       >
         <span className="flex items-center gap-1.5 overflow-hidden">
           <Avatar avatar={organization.avatar} size={1.25} orgName={organization.title} />
@@ -406,19 +431,24 @@ function ProjectSelector({
           <PopoverMenuItem to={newProjectPath(organization)} title="New project" icon={PlusIcon} />
         </div>
         <div className="border-t border-charcoal-700 p-1">
-          <SwitchOrganizations organizations={organizations} organization={organization} />
+          {organizations.length > 1 ? (
+            <SwitchOrganizations organizations={organizations} organization={organization} />
+          ) : (
+            <PopoverMenuItem
+              to={newOrganizationPath()}
+              title="New organization"
+              icon={PlusIcon}
+              leadingIconClassName="text-text-dimmed"
+            />
+          )}
         </div>
         <div className="border-t border-charcoal-700 p-1">
           <PopoverMenuItem
             to={accountPath()}
             title="Account"
             icon={UserProfilePhoto}
-            leadingIconClassName={cn(
-              "text-text-dimmed rounded-full border border-transparent",
-              user.isImpersonating && "rounded-full border-yellow-500"
-            )}
+            leadingIconClassName="text-text-dimmed rounded-full border border-transparent"
           />
-          {user.isImpersonating && <ImpersonationBanner />}
         </div>
         <div className="border-t border-charcoal-700 p-1">
           <PopoverMenuItem
@@ -513,7 +543,7 @@ function SwitchOrganizations({
           <div className="border-t border-charcoal-700 p-1">
             <PopoverMenuItem
               to={newOrganizationPath()}
-              title="New Organization"
+              title="New organization"
               icon={PlusIcon}
               leadingIconClassName="text-text-dimmed"
             />
@@ -538,3 +568,43 @@ function SelectorDivider() {
     </svg>
   );
 }
+
+function HelpAndAI() {
+  const { isKapaEnabled, isKapaOpen, openKapa } = useKapaWidget();
+
+  return (
+    <>
+      <ShortcutsAutoOpen />
+      <HelpAndFeedback disableShortcut={isKapaOpen} />
+      {isKapaEnabled && (
+        <TooltipProvider disableHoverableContent>
+          <Tooltip>
+            <TooltipTrigger asChild>
+              <div className="inline-flex">
+                <Button
+                  variant="small-menu-item"
+                  data-action="ask-ai"
+                  shortcut={{ modifiers: ["mod"], key: "/", enabledOnInputElements: true }}
+                  hideShortcutKey
+                  data-modal-override-open-class-ask-ai="true"
+                  onClick={() => {
+                    openKapa();
+                  }}
+                >
+                  <AISparkleIcon className="size-5" />
+                </Button>
+              </div>
+            </TooltipTrigger>
+            <TooltipContent
+              side="top"
+              className="flex items-center gap-1 py-1.5 pl-2.5 pr-2 text-xs"
+            >
+              Ask AI
+              <ShortcutKey shortcut={{ modifiers: ["mod"], key: "/" }} variant="medium/bright" />
+            </TooltipContent>
+          </Tooltip>
+        </TooltipProvider>
+      )}
+    </>
+  );
+}

apps/webapp/app/components/primitives/AnimatedNumber.tsx

@@ -1,4 +1,4 @@
-import { motion, useSpring, useTransform, useMotionValue, animate } from "framer-motion";
+import { animate, motion, useMotionValue, useTransform } from "framer-motion";
 import { useEffect } from "react";
 
 export function AnimatedNumber({ value }: { value: number }) {

apps/webapp/app/components/primitives/Buttons.tsx

@@ -177,6 +177,7 @@ export type ButtonContentPropsType = {
   shortcutPosition?: "before-trailing-icon" | "after-trailing-icon";
   tooltip?: ReactNode;
   iconSpacing?: string;
+  hideShortcutKey?: boolean;
 };
 
 export function ButtonContent(props: ButtonContentPropsType) {
@@ -192,6 +193,7 @@ export function ButtonContent(props: ButtonContentPropsType) {
     className,
     tooltip,
     iconSpacing,
+    hideShortcutKey,
   } = props;
   const variation = allVariants.variant[props.variant];
 
@@ -202,7 +204,8 @@ export function ButtonContent(props: ButtonContentPropsType) {
   const textColorClassName = variation.textColor;
 
   const renderShortcutKey = () =>
-    shortcut && (
+    shortcut &&
+    !hideShortcutKey && (
       <ShortcutKey
         className={cn(shortcutClassName)}
         shortcut={shortcut}

apps/webapp/app/components/primitives/ClipboardField.tsx

@@ -1,8 +1,6 @@
-import { CheckIcon } from "@heroicons/react/20/solid";
-import { useCallback, useEffect, useRef, useState } from "react";
+import { useEffect, useRef, useState } from "react";
 import { cn } from "~/utils/cn";
-import { Button } from "./Buttons";
-import { ClipboardCheckIcon, ClipboardIcon } from "lucide-react";
+import { CopyButton } from "./CopyButton";
 
 const variants = {
   "primary/small": {
@@ -10,61 +8,55 @@ const variants = {
       "flex items-center text-text-dimmed font-mono rounded border bg-charcoal-750 text-xs transition hover:bg-charcoal-700 focus-visible:outline-none focus-visible:ring-0 focus-visible:ring-offset-0 focus:border-transparent focus:outline-none focus:ring-0 focus:ring-transparent",
     input:
       "bg-transparent border-0 text-xs px-2 w-auto rounded-l h-6 leading-6 focus:ring-transparent",
-    buttonVariant: "primary/small" as const,
+    buttonVariant: "primary" as const,
+    size: "small" as const,
     button: "rounded-l-none",
-    iconSize: "h-3 w-3",
-    iconPadding: "pl-1",
   },
   "secondary/small": {
     container:
       "flex items-center text-text-dimmed font-mono rounded border bg-charcoal-750 text-xs transition hover:bg-charcoal-700 focus-visible:outline-none focus-visible:ring-0 focus-visible:ring-offset-0 focus:border-transparent focus:outline-none focus:ring-0 focus:ring-transparent",
     input:
       "bg-transparent border-0 text-xs px-2 w-auto rounded-l h-6 leading-6 focus:ring-transparent",
-    buttonVariant: "tertiary/small" as const,
+    buttonVariant: "tertiary" as const,
+    size: "small" as const,
     button: "rounded-l-none border-l border-charcoal-750",
-    iconSize: "h-3 w-3",
-    iconPadding: "pl-1",
   },
   "tertiary/small": {
     container:
       "group/clipboard flex items-center text-text-dimmed font-mono rounded bg-transparent border border-transparent text-xs transition duration-150 hover:border-charcoal-700 focus-visible:outline-none focus-visible:ring-0 focus-visible:ring-offset-0 focus:border-transparent focus:outline-none focus:ring-0 focus:ring-transparent",
     input:
       "bg-transparent border-0 text-xs px-2 w-auto rounded-l h-6 leading-6 focus:ring-transparent",
-    buttonVariant: "minimal/small" as const,
+    buttonVariant: "minimal" as const,
+    size: "small" as const,
     button:
       "rounded-l-none border-l border-transparent transition group-hover/clipboard:border-charcoal-700",
-    iconSize: "h-3 w-3",
-    iconPadding: "pl-1",
   },
   "primary/medium": {
     container:
       "flex items-center text-text-dimmed font-mono rounded border bg-charcoal-750 text-sm transition hover:bg-charcoal-700 focus-visible:outline-none focus-visible:ring-0 focus-visible:ring-offset-0 focus:border-transparent focus:outline-none focus:ring-0 focus:ring-transparent",
     input:
       "bg-transparent border-0 text-sm px-3 w-auto rounded-l h-8 leading-6 focus:ring-transparent",
-    buttonVariant: "primary/medium" as const,
+    buttonVariant: "primary" as const,
+    size: "medium" as const,
     button: "rounded-l-none",
-    iconSize: "h-4 w-4",
-    iconPadding: "pl-2",
   },
   "secondary/medium": {
     container:
       "flex items-center text-text-dimmed font-mono rounded bg-charcoal-750 text-sm transition hover:bg-charcoal-700 focus-visible:outline-none focus-visible:ring-0 focus-visible:ring-offset-0 focus:border-transparent focus:outline-none focus:ring-0 focus:ring-transparent",
     input:
       "bg-transparent border-0 text-sm px-3 w-auto rounded-l h-8 leading-6 focus:ring-transparent",
-    buttonVariant: "tertiary/medium" as const,
+    buttonVariant: "tertiary" as const,
+    size: "medium" as const,
     button: "rounded-l-none border-l border-charcoal-750",
-    iconSize: "h-4 w-4",
-    iconPadding: "pl-2",
   },
   "tertiary/medium": {
     container:
       "group flex items-center text-text-dimmed font-mono rounded bg-transparent border border-transparent text-sm transition hover:border-charcoal-700 focus-visible:outline-none focus-visible:ring-0 focus-visible:ring-offset-0 focus:border-transparent focus:outline-none focus:ring-0 focus:ring-transparent",
     input:
       "bg-transparent border-0 text-sm px-3 w-auto rounded-l h-8 leading-6 focus:ring-transparent",
-    buttonVariant: "minimal/medium" as const,
+    buttonVariant: "minimal" as const,
+    size: "medium" as const,
     button: "rounded-l-none border-l border-transparent transition group-hover:border-charcoal-700",
-    iconSize: "h-4 w-4",
-    iconPadding: "pl-2",
   },
 };
 
@@ -88,36 +80,19 @@ export function ClipboardField({
   fullWidth = true,
 }: ClipboardFieldProps) {
   const [isSecure, setIsSecure] = useState(secure !== undefined && secure);
-  const [copied, setCopied] = useState(false);
-
-  const copy = useCallback(
-    (event: React.MouseEvent<HTMLButtonElement>) => {
-      event.preventDefault();
-      event.stopPropagation();
-      navigator.clipboard.writeText(value);
-      setCopied(true);
-      setTimeout(() => {
-        setCopied(false);
-      }, 1500);
-    },
-    [value]
-  );
+  const inputIcon = useRef<HTMLInputElement>(null);
+  const { container, input, buttonVariant, button, size } = variants[variant];
 
   useEffect(() => {
     setIsSecure(secure !== undefined && secure);
   }, [secure]);
 
-  const { container, input, buttonVariant, button } = variants[variant];
-  const iconClassName = variants[variant].iconSize;
-  const iconPosition = variants[variant].iconPadding;
-  const inputIcon = useRef<HTMLInputElement>(null);
-
   return (
     <span className={cn(container, fullWidth ? "w-full" : "max-w-fit", className)}>
       {icon && (
         <span
           onClick={() => inputIcon.current && inputIcon.current.focus()}
-          className={cn(iconPosition, "flex items-center")}
+          className="flex items-center pl-1"
         >
           {icon}
         </span>
@@ -132,51 +107,26 @@ export function ClipboardField({
           fullWidth ? "w-full" : "max-w-fit",
           input
         )}
-        // size={value.length}
-        // maxLength={3}
         onFocus={(e) => {
           if (secure) {
-            setIsSecure((i) => false);
+            setIsSecure(false);
           }
           e.currentTarget.select();
         }}
         onBlur={() => {
           if (secure) {
-            setIsSecure((i) => true);
+            setIsSecure(true);
           }
         }}
       />
-      {iconButton ? (
-        <Button
-          variant={buttonVariant}
-          onClick={copy}
-          className={cn("shrink grow-0 px-1.5", button)}
-        >
-          {copied ? (
-            <ClipboardCheckIcon
-              className={cn(
-                "h-4 w-4",
-                buttonVariant === "primary/small" || buttonVariant === "primary/medium"
-                  ? "text-background-dimmed"
-                  : "text-green-500"
-              )}
-            />
-          ) : (
-            <ClipboardIcon
-              className={cn(
-                "h-4 w-4",
-                buttonVariant === "primary/small" || buttonVariant === "primary/medium"
-                  ? "text-background-dimmed"
-                  : "text-text-dimmed"
-              )}
-            />
-          )}
-        </Button>
-      ) : (
-        <Button variant={buttonVariant} onClick={copy} className={cn("shrink-0 grow-0", button)}>
-          {copied ? <CheckIcon className="mx-[0.4rem] h-4 w-4 text-green-500" /> : "Copy"}
-        </Button>
-      )}
+      <CopyButton
+        value={value}
+        variant={iconButton ? "icon" : "button"}
+        buttonVariant={buttonVariant}
+        size={size}
+        buttonClassName={button}
+        showTooltip={false}
+      />
     </span>
   );
 }

apps/webapp/app/components/primitives/CopyButton.tsx

@@ -0,0 +1,100 @@
+import { ClipboardCheckIcon, ClipboardIcon } from "lucide-react";
+import { useCopy } from "~/hooks/useCopy";
+import { cn } from "~/utils/cn";
+import { Button } from "./Buttons";
+import { SimpleTooltip } from "./Tooltip";
+
+const sizes = {
+  "extra-small": {
+    icon: "size-3",
+    button: "h-5 px-1",
+  },
+  small: {
+    icon: "size-3.5",
+    button: "h-6 px-1",
+  },
+  medium: {
+    icon: "size-4",
+    button: "h-8 px-1.5",
+  },
+};
+
+type CopyButtonProps = {
+  value: string;
+  variant?: "icon" | "button";
+  size?: keyof typeof sizes;
+  className?: string;
+  buttonClassName?: string;
+  showTooltip?: boolean;
+  buttonVariant?: "primary" | "secondary" | "tertiary" | "minimal";
+};
+
+export function CopyButton({
+  value,
+  variant = "button",
+  size = "medium",
+  className,
+  buttonClassName,
+  showTooltip = true,
+  buttonVariant = "tertiary",
+}: CopyButtonProps) {
+  const { copy, copied } = useCopy(value);
+
+  const { icon: iconSize, button: buttonSize } = sizes[size];
+
+  const button =
+    variant === "icon" ? (
+      <span
+        onClick={copy}
+        className={cn(
+          buttonSize,
+          "flex items-center justify-center rounded border border-charcoal-650 bg-charcoal-750",
+          copied
+            ? "text-green-500"
+            : "text-text-dimmed hover:border-charcoal-600 hover:bg-charcoal-700 hover:text-text-bright",
+          buttonClassName
+        )}
+      >
+        {copied ? (
+          <ClipboardCheckIcon className={iconSize} />
+        ) : (
+          <ClipboardIcon className={iconSize} />
+        )}
+      </span>
+    ) : (
+      <Button
+        variant={`${buttonVariant}/${size === "extra-small" ? "small" : size}`}
+        onClick={copy}
+        className={cn("shrink-0", buttonClassName)}
+      >
+        {copied ? (
+          <ClipboardCheckIcon
+            className={cn(
+              iconSize,
+              buttonVariant === "primary" ? "text-background-dimmed" : "text-green-500"
+            )}
+          />
+        ) : (
+          <ClipboardIcon
+            className={cn(
+              iconSize,
+              buttonVariant === "primary" ? "text-background-dimmed" : "text-text-dimmed"
+            )}
+          />
+        )}
+      </Button>
+    );
+
+  if (!showTooltip) return <span className={className}>{button}</span>;
+
+  return (
+    <span className={className}>
+      <SimpleTooltip
+        button={button}
+        content={copied ? "Copied!" : "Copy"}
+        className="font-sans"
+        disableHoverableContent
+      />
+    </span>
+  );
+}

apps/webapp/app/components/primitives/CopyableText.tsx

@@ -1,24 +1,12 @@
-import { useCallback, useState } from "react";
-import { SimpleTooltip } from "~/components/primitives/Tooltip";
 import { ClipboardCheckIcon, ClipboardIcon } from "lucide-react";
+import { useState } from "react";
+import { SimpleTooltip } from "~/components/primitives/Tooltip";
+import { useCopy } from "~/hooks/useCopy";
 import { cn } from "~/utils/cn";
 
 export function CopyableText({ value, className }: { value: string; className?: string }) {
   const [isHovered, setIsHovered] = useState(false);
-  const [copied, setCopied] = useState(false);
-
-  const copy = useCallback(
-    (e: React.MouseEvent) => {
-      e.preventDefault();
-      e.stopPropagation();
-      navigator.clipboard.writeText(value);
-      setCopied(true);
-      setTimeout(() => {
-        setCopied(false);
-      }, 1500);
-    },
-    [value]
-  );
+  const { copy, copied } = useCopy(value);
 
   return (
     <span

apps/webapp/app/components/primitives/DateTime.tsx

@@ -1,5 +1,10 @@
-import { Fragment, useEffect, useState } from "react";
+import { GlobeAltIcon, GlobeAmericasIcon } from "@heroicons/react/20/solid";
+import { Laptop } from "lucide-react";
+import { Fragment, type ReactNode, useEffect, useState } from "react";
+import { CopyButton } from "./CopyButton";
 import { useLocales } from "./LocaleProvider";
+import { Paragraph } from "./Paragraph";
+import { SimpleTooltip } from "./Tooltip";
 
 type DateTimeProps = {
   date: Date | string;
@@ -7,6 +12,7 @@ type DateTimeProps = {
   includeSeconds?: boolean;
   includeTime?: boolean;
   showTimezone?: boolean;
+  showTooltip?: boolean;
   previousDate?: Date | string | null; // Add optional previous date for comparison
 };
 
@@ -16,41 +22,43 @@ export const DateTime = ({
   includeSeconds = true,
   includeTime = true,
   showTimezone = false,
+  showTooltip = true,
 }: DateTimeProps) => {
   const locales = useLocales();
+  const [localTimeZone, setLocalTimeZone] = useState<string>("UTC");
 
   const realDate = typeof date === "string" ? new Date(date) : date;
 
-  const initialFormattedDateTime = formatDateTime(
-    realDate,
-    timeZone ?? "UTC",
-    locales,
-    includeSeconds,
-    includeTime
-  );
-
-  const [formattedDateTime, setFormattedDateTime] = useState<string>(initialFormattedDateTime);
-
   useEffect(() => {
     const resolvedOptions = Intl.DateTimeFormat().resolvedOptions();
+    setLocalTimeZone(resolvedOptions.timeZone);
+  }, []);
 
-    setFormattedDateTime(
-      formatDateTime(
+  const tooltipContent = (
+    <TooltipContent
+      realDate={realDate}
+      timeZone={timeZone}
+      localTimeZone={localTimeZone}
+      locales={locales}
+    />
+  );
+
+  const formattedDateTime = (
+    <Fragment>
+      {formatDateTime(
         realDate,
-        timeZone ?? resolvedOptions.timeZone,
+        timeZone ?? localTimeZone,
         locales,
         includeSeconds,
         includeTime
-      )
-    );
-  }, [locales, includeSeconds, realDate]);
-
-  return (
-    <Fragment>
-      {formattedDateTime.replace(/\s/g, String.fromCharCode(32))}
+      ).replace(/\s/g, String.fromCharCode(32))}
       {showTimezone ? ` (${timeZone ?? "UTC"})` : null}
     </Fragment>
   );
+
+  if (!showTooltip) return formattedDateTime;
+
+  return <SimpleTooltip button={formattedDateTime} content={tooltipContent} side="right" />;
 };
 
 export function formatDateTime(
@@ -71,6 +79,48 @@ export function formatDateTime(
   }).format(date);
 }
 
+export function formatDateTimeISO(date: Date, timeZone: string): string {
+  // Special handling for UTC
+  if (timeZone === "UTC") {
+    return date.toISOString();
+  }
+
+  // Get the date parts in the target timezone
+  const dateFormatter = new Intl.DateTimeFormat("en-US", {
+    timeZone,
+    year: "numeric",
+    month: "2-digit",
+    day: "2-digit",
+    hour: "2-digit",
+    minute: "2-digit",
+    second: "2-digit",
+    hour12: false,
+  });
+
+  // Get the timezone offset for this specific date
+  const timeZoneFormatter = new Intl.DateTimeFormat("en-US", {
+    timeZone,
+    timeZoneName: "longOffset",
+  });
+
+  const dateParts = Object.fromEntries(
+    dateFormatter.formatToParts(date).map(({ type, value }) => [type, value])
+  );
+
+  const timeZoneParts = timeZoneFormatter.formatToParts(date);
+  const offset =
+    timeZoneParts.find((part) => part.type === "timeZoneName")?.value.replace("GMT", "") ||
+    "+00:00";
+
+  // Format: YYYY-MM-DDThh:mm:ss.sss±hh:mm
+  return (
+    `${dateParts.year}-${dateParts.month}-${dateParts.day}T` +
+    `${dateParts.hour}:${dateParts.minute}:${dateParts.second}.${String(
+      date.getMilliseconds()
+    ).padStart(3, "0")}${offset}`
+  );
+}
+
 // New component that only shows date when it changes
 export const SmartDateTime = ({ date, previousDate = null, timeZone = "UTC" }: DateTimeProps) => {
   const locales = useLocales();
@@ -147,8 +197,10 @@ export const DateTimeAccurate = ({
   date,
   timeZone = "UTC",
   previousDate = null,
+  showTooltip = true,
 }: DateTimeProps) => {
   const locales = useLocales();
+  const [localTimeZone, setLocalTimeZone] = useState<string>("UTC");
   const realDate = typeof date === "string" ? new Date(date) : date;
   const realPrevDate = previousDate
     ? typeof previousDate === "string"
@@ -156,33 +208,37 @@ export const DateTimeAccurate = ({
       : previousDate
     : null;
 
-  // Use the new Smart formatting if previousDate is provided
-  const initialFormattedDateTime = realPrevDate
-    ? isSameDay(realDate, realPrevDate)
-      ? formatTimeOnly(realDate, timeZone, locales)
-      : formatDateTimeAccurate(realDate, timeZone, locales)
-    : formatDateTimeAccurate(realDate, timeZone, locales);
-
-  const [formattedDateTime, setFormattedDateTime] = useState<string>(initialFormattedDateTime);
-
   useEffect(() => {
     const resolvedOptions = Intl.DateTimeFormat().resolvedOptions();
-    const userTimeZone = resolvedOptions.timeZone;
+    setLocalTimeZone(resolvedOptions.timeZone);
+  }, []);
 
-    if (realPrevDate) {
-      // Smart formatting based on whether date changed
-      setFormattedDateTime(
-        isSameDay(realDate, realPrevDate)
-          ? formatTimeOnly(realDate, userTimeZone, locales)
-          : formatDateTimeAccurate(realDate, userTimeZone, locales)
-      );
-    } else {
-      // Default behavior when no previous date
-      setFormattedDateTime(formatDateTimeAccurate(realDate, userTimeZone, locales));
-    }
-  }, [locales, realDate, realPrevDate]);
+  // Smart formatting based on whether date changed
+  const formattedDateTime = realPrevDate
+    ? isSameDay(realDate, realPrevDate)
+      ? formatTimeOnly(realDate, localTimeZone, locales)
+      : formatDateTimeAccurate(realDate, localTimeZone, locales)
+    : formatDateTimeAccurate(realDate, localTimeZone, locales);
 
-  return <Fragment>{formattedDateTime.replace(/\s/g, String.fromCharCode(32))}</Fragment>;
+  if (!showTooltip)
+    return <Fragment>{formattedDateTime.replace(/\s/g, String.fromCharCode(32))}</Fragment>;
+
+  const tooltipContent = (
+    <TooltipContent
+      realDate={realDate}
+      timeZone={timeZone}
+      localTimeZone={localTimeZone}
+      locales={locales}
+    />
+  );
+
+  return (
+    <SimpleTooltip
+      button={<Fragment>{formattedDateTime.replace(/\s/g, String.fromCharCode(32))}</Fragment>}
+      content={tooltipContent}
+      side="right"
+    />
+  );
 };
 
 function formatDateTimeAccurate(date: Date, timeZone: string, locales: string[]): string {
@@ -226,3 +282,81 @@ function formatDateTimeShort(date: Date, timeZone: string, locales: string[]): s
 
   return formattedDateTime;
 }
+
+type DateTimeTooltipContentProps = {
+  title: string;
+  dateTime: string;
+  isoDateTime: string;
+  icon: ReactNode;
+};
+
+function DateTimeTooltipContent({
+  title,
+  dateTime,
+  isoDateTime,
+  icon,
+}: DateTimeTooltipContentProps) {
+  const getUtcOffset = () => {
+    if (title !== "Local") return "";
+    const offset = -new Date().getTimezoneOffset();
+    const sign = offset >= 0 ? "+" : "-";
+    const hours = Math.abs(Math.floor(offset / 60));
+    const minutes = Math.abs(offset % 60);
+    return `(UTC ${sign}${hours}${minutes ? `:${minutes.toString().padStart(2, "0")}` : ""})`;
+  };
+
+  return (
+    <div className="flex flex-col gap-1">
+      <div className="flex items-center gap-1 text-sm">
+        {icon}
+        <span className="font-medium">{title}</span>
+        <span className="font-normal text-text-dimmed">{getUtcOffset()}</span>
+      </div>
+      <div className="flex items-center justify-between gap-2">
+        <Paragraph variant="extra-small" className="text-text-dimmed">
+          {dateTime}
+        </Paragraph>
+        <CopyButton value={isoDateTime} variant="icon" size="extra-small" showTooltip={false} />
+      </div>
+    </div>
+  );
+}
+
+function TooltipContent({
+  realDate,
+  timeZone,
+  localTimeZone,
+  locales,
+}: {
+  realDate: Date;
+  timeZone?: string;
+  localTimeZone: string;
+  locales: string[];
+}) {
+  return (
+    <div className="flex flex-col gap-1">
+      <div className="flex flex-col gap-2.5 pb-1">
+        {timeZone && timeZone !== "UTC" && (
+          <DateTimeTooltipContent
+            title={timeZone}
+            dateTime={formatDateTime(realDate, timeZone, locales, true, true)}
+            isoDateTime={formatDateTimeISO(realDate, timeZone)}
+            icon={<GlobeAmericasIcon className="size-4 text-purple-500" />}
+          />
+        )}
+        <DateTimeTooltipContent
+          title="UTC"
+          dateTime={formatDateTime(realDate, "UTC", locales, true, true)}
+          isoDateTime={formatDateTimeISO(realDate, "UTC")}
+          icon={<GlobeAltIcon className="size-4 text-blue-500" />}
+        />
+        <DateTimeTooltipContent
+          title="Local"
+          dateTime={formatDateTime(realDate, localTimeZone, locales, true, true)}
+          isoDateTime={formatDateTimeISO(realDate, localTimeZone)}
+          icon={<Laptop className="size-4 text-green-500" />}
+        />
+      </div>
+    </div>
+  );
+}

apps/webapp/app/components/primitives/LoadingBarDivider.tsx

@@ -7,7 +7,7 @@ type LoadingBarDividerProps = {
 
 export function LoadingBarDivider({ isLoading }: LoadingBarDividerProps) {
   return (
-    <div className="relative h-px w-full bg-grid-bright">
+    <div className="relative h-px w-full overflow-hidden bg-grid-bright">
       <AnimationDivider isLoading={isLoading} />
     </div>
   );
@@ -18,14 +18,14 @@ export function AnimationDivider({ isLoading }: LoadingBarDividerProps) {
   const [isPresent, safeToRemove] = usePresence();
 
   useEffect(() => {
+    if (!scope.current) return;
+
     if (isPresent) {
       const enterAnimation = async () => {
         await animate(
-          [
-            [scope.current, { width: "30%", left: "35%" }, { duration: 1, ease: "easeIn" }],
-            [scope.current, { width: 0, left: "100%" }, { duration: 1, ease: "easeOut" }],
-          ],
-          { repeat: Infinity, repeatType: "reverse" }
+          scope.current,
+          { left: ["-100%", "100%"], width: "100%" },
+          { duration: 2, ease: "easeOut", repeat: Infinity }
         );
       };
       enterAnimation();

apps/webapp/app/components/primitives/ShortcutKey.tsx

@@ -3,9 +3,10 @@ import { KeyboardLeftIcon } from "~/assets/icons/KeyboardLeftIcon";
 import { KeyboardRightIcon } from "~/assets/icons/KeyboardRightIcon";
 import { KeyboardUpIcon } from "~/assets/icons/KeyboardUpIcon";
 import { KeyboardWindowsIcon } from "~/assets/icons/KeyboardWindowsIcon";
-import { Modifier, Shortcut } from "~/hooks/useShortcutKeys";
+import { type Modifier, type Shortcut } from "~/hooks/useShortcutKeys";
 import { cn } from "~/utils/cn";
 import { useOperatingSystem } from "./OperatingSystemProvider";
+import { KeyboardEnterIcon } from "~/assets/icons/KeyboardEnterIcon";
 
 const medium =
   "text-[0.75rem] font-medium min-w-[17px] rounded-[2px] tabular-nums px-1 ml-1 -mr-0.5 flex items-center gap-x-1.5 border border-dimmed/40 text-text-dimmed group-hover:text-text-bright/80 group-hover:border-dimmed/60 transition uppercase";
@@ -60,7 +61,11 @@ function keyString(key: string, isMac: boolean, variant: "small" | "medium" | "m
 
   switch (key) {
     case "enter":
-      return isMac ? "↵" : <span className="capitalize">Enter</span>;
+      return isMac ? (
+        <KeyboardEnterIcon className={className} />
+      ) : (
+        <span className="capitalize">Enter</span>
+      );
     case "esc":
       return <span className="capitalize">Esc</span>;
     case "del":

apps/webapp/app/components/primitives/ShortcutsProvider.tsx

@@ -0,0 +1,39 @@
+import { createContext, useContext, useState, useCallback, useMemo, type ReactNode } from "react";
+
+type ShortcutsContextType = {
+  areShortcutsEnabled: boolean;
+  disableShortcuts: () => void;
+  enableShortcuts: () => void;
+};
+
+const ShortcutsContext = createContext<ShortcutsContextType | null>(null);
+
+type ShortcutsProviderProps = {
+  children: ReactNode;
+};
+
+export function ShortcutsProvider({ children }: ShortcutsProviderProps) {
+  const [areShortcutsEnabled, setAreShortcutsEnabled] = useState(true);
+
+  const disableShortcuts = useCallback(() => setAreShortcutsEnabled(false), []);
+  const enableShortcuts = useCallback(() => setAreShortcutsEnabled(true), []);
+
+  const value = useMemo(
+    () => ({
+      areShortcutsEnabled,
+      disableShortcuts,
+      enableShortcuts,
+    }),
+    [areShortcutsEnabled, disableShortcuts, enableShortcuts]
+  );
+
+  return <ShortcutsContext.Provider value={value}>{children}</ShortcutsContext.Provider>;
+}
+
+const throwIfNoProvider = () => {
+  throw new Error("useShortcuts must be used within a ShortcutsProvider");
+};
+
+export const useShortcuts = () => {
+  return useContext(ShortcutsContext) ?? throwIfNoProvider();
+};

apps/webapp/app/components/primitives/Tooltip.tsx

@@ -1,7 +1,7 @@
-import * as React from "react";
-import * as TooltipPrimitive from "@radix-ui/react-tooltip";
-import { cn } from "~/utils/cn";
 import { InformationCircleIcon } from "@heroicons/react/20/solid";
+import * as TooltipPrimitive from "@radix-ui/react-tooltip";
+import * as React from "react";
+import { cn } from "~/utils/cn";
 
 const variantClasses = {
   basic:
@@ -115,4 +115,4 @@ export function InfoIconTooltip({
   );
 }
 
-export { Tooltip, TooltipTrigger, TooltipContent, TooltipProvider, TooltipArrow, SimpleTooltip };
+export { SimpleTooltip, Tooltip, TooltipArrow, TooltipContent, TooltipProvider, TooltipTrigger };

apps/webapp/app/components/runs/v3/PacketDisplay.tsx

@@ -44,6 +44,7 @@ export function PacketDisplay({
           code={data}
           maxLines={20}
           showLineNumbers={false}
+          showTextWrapping
         />
       );
     }

apps/webapp/app/components/runs/v3/RunIcon.tsx

@@ -19,6 +19,7 @@ import { FunctionIcon } from "~/assets/icons/FunctionIcon";
 import { TriggerIcon } from "~/assets/icons/TriggerIcon";
 import { PythonLogoIcon } from "~/assets/icons/PythonLogoIcon";
 import { TraceIcon } from "~/assets/icons/TraceIcon";
+import { WaitpointTokenIcon } from "~/assets/icons/WaitpointTokenIcon";
 
 type TaskIconProps = {
   name: string | undefined;
@@ -75,6 +76,10 @@ export function RunIcon({ name, className, spanName }: TaskIconProps) {
       return <TriggerIcon className={cn(className, "text-orange-500")} />;
     case "python":
       return <PythonLogoIcon className={className} />;
+    case "wait-token":
+      return <WaitpointTokenIcon className={cn(className, "text-sky-500")} />;
+    case "function":
+      return <FunctionIcon className={cn(className, "text-text-dimmed")} />;
     //log levels
     case "debug":
     case "log":
@@ -97,6 +102,7 @@ export function RunIcon({ name, className, spanName }: TaskIconProps) {
     case "task-hook-onResume":
     case "task-hook-onComplete":
     case "task-hook-cleanup":
+    case "task-hook-onCancel":
       return <FunctionIcon className={cn(className, "text-text-dimmed")} />;
     case "task-hook-onFailure":
     case "task-hook-catchError":

apps/webapp/app/components/runs/v3/ScheduleFilters.tsx

@@ -24,10 +24,6 @@ export const ScheduleListFilters = z.object({
     .string()
     .optional()
     .transform((value) => (value ? value.split(",") : undefined)),
-  environments: z
-    .string()
-    .optional()
-    .transform((value) => (value ? value.split(",") : undefined)),
   type: z.union([z.literal("declarative"), z.literal("imperative")]).optional(),
   search: z.string().optional(),
 });
@@ -44,7 +40,7 @@ export function ScheduleFilters({ possibleTasks }: ScheduleFiltersProps) {
   const navigate = useNavigate();
   const location = useOptimisticLocation();
   const searchParams = new URLSearchParams(location.search);
-  const { environments, tasks, page, search, type } = ScheduleListFilters.parse(
+  const { tasks, page, search, type } = ScheduleListFilters.parse(
     Object.fromEntries(searchParams.entries())
   );
 

apps/webapp/app/components/runs/v3/TaskTriggerSource.tsx

@@ -12,10 +12,12 @@ export function TaskTriggerSourceIcon({
 }) {
   switch (source) {
     case "STANDARD": {
-      return <TaskIconSmall className="size-[1.125rem] text-tasks" />;
+      return <TaskIconSmall className="size-[1.125rem] min-w-[1.125rem] text-tasks" />;
     }
     case "SCHEDULED": {
-      return <ClockIcon className={cn("size-[1.125rem] text-schedules", className)} />;
+      return (
+        <ClockIcon className={cn("size-[1.125rem] min-w-[1.125rem] text-schedules", className)} />
+      );
     }
   }
 }

apps/webapp/app/components/runs/v3/WaitpointDetails.tsx

@@ -11,6 +11,7 @@ import { v3WaitpointTokenPath, v3WaitpointTokensPath } from "~/utils/pathBuilder
 import { PacketDisplay } from "./PacketDisplay";
 import { WaitpointStatusCombo } from "./WaitpointStatus";
 import { RunTag } from "./RunTag";
+import { ClipboardField } from "~/components/primitives/ClipboardField";
 
 export function WaitpointDetailTable({
   waitpoint,
@@ -50,6 +51,14 @@ export function WaitpointDetailTable({
           )}
         </Property.Value>
       </Property.Item>
+      {waitpoint.type === "MANUAL" && (
+        <Property.Item>
+          <Property.Label>Callback URL</Property.Label>
+          <Property.Value className="my-1">
+            <ClipboardField value={waitpoint.url} variant={"secondary/small"} />
+          </Property.Value>
+        </Property.Item>
+      )}
       <Property.Item>
         <Property.Label>Idempotency key</Property.Label>
         <Property.Value>

apps/webapp/app/env.server.ts

@@ -1,7 +1,8 @@
-import { SecretStoreOptionsSchema } from "./services/secrets/secretStoreOptionsSchema.server";
 import { z } from "zod";
-import { isValidRegex } from "./utils/regex";
+import { SecretStoreOptionsSchema } from "./services/secrets/secretStoreOptionsSchema.server";
 import { isValidDatabaseUrl } from "./utils/db";
+import { isValidRegex } from "./utils/regex";
+import { BoolEnv } from "./utils/boolEnv";
 
 const EnvironmentSchema = z.object({
   NODE_ENV: z.union([z.literal("development"), z.literal("production"), z.literal("test")]),
@@ -50,7 +51,7 @@ const EnvironmentSchema = z.object({
   RESEND_API_KEY: z.string().optional(),
   SMTP_HOST: z.string().optional(),
   SMTP_PORT: z.coerce.number().optional(),
-  SMTP_SECURE: z.coerce.boolean().optional(),
+  SMTP_SECURE: BoolEnv.optional(),
   SMTP_USER: z.string().optional(),
   SMTP_PASSWORD: z.string().optional(),
 
@@ -338,7 +339,7 @@ const EnvironmentSchema = z.object({
   ALERT_RESEND_API_KEY: z.string().optional(),
   ALERT_SMTP_HOST: z.string().optional(),
   ALERT_SMTP_PORT: z.coerce.number().optional(),
-  ALERT_SMTP_SECURE: z.coerce.boolean().optional(),
+  ALERT_SMTP_SECURE: BoolEnv.optional(),
   ALERT_SMTP_USER: z.string().optional(),
   ALERT_SMTP_PASSWORD: z.string().optional(),
   ALERT_RATE_LIMITER_EMISSION_INTERVAL: z.coerce.number().int().default(2_500),
@@ -378,7 +379,7 @@ const EnvironmentSchema = z.object({
   MAX_SEQUENTIAL_INDEX_FAILURE_COUNT: z.coerce.number().default(96),
 
   LOOPS_API_KEY: z.string().optional(),
-  MARQS_DISABLE_REBALANCING: z.coerce.boolean().default(false),
+  MARQS_DISABLE_REBALANCING: BoolEnv.default(false),
   MARQS_VISIBILITY_TIMEOUT_MS: z.coerce
     .number()
     .int()
@@ -452,7 +453,11 @@ const EnvironmentSchema = z.object({
   RUN_ENGINE_TIMEOUT_PENDING_CANCEL: z.coerce.number().int().default(60_000),
   RUN_ENGINE_TIMEOUT_EXECUTING: z.coerce.number().int().default(60_000),
   RUN_ENGINE_TIMEOUT_EXECUTING_WITH_WAITPOINTS: z.coerce.number().int().default(60_000),
-  RUN_ENGINE_DEBUG_WORKER_NOTIFICATIONS: z.coerce.boolean().default(false),
+  RUN_ENGINE_TIMEOUT_SUSPENDED: z.coerce
+    .number()
+    .int()
+    .default(60_000 * 10),
+  RUN_ENGINE_DEBUG_WORKER_NOTIFICATIONS: BoolEnv.default(false),
   RUN_ENGINE_PARENT_QUEUE_LIMIT: z.coerce.number().int().default(1000),
   RUN_ENGINE_CONCURRENCY_LIMIT_BIAS: z.coerce.number().default(0.75),
   RUN_ENGINE_AVAILABLE_CAPACITY_BIAS: z.coerce.number().default(0.3),
@@ -460,7 +465,7 @@ const EnvironmentSchema = z.object({
   RUN_ENGINE_REUSE_SNAPSHOT_COUNT: z.coerce.number().int().default(0),
   RUN_ENGINE_MAXIMUM_ENV_COUNT: z.coerce.number().int().optional(),
   RUN_ENGINE_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000),
-  RUN_ENGINE_MAX_DEQUEUE_LOOP_ATTEMPTS: z.coerce.number().int().default(10),
+  RUN_ENGINE_RETRY_WARM_START_THRESHOLD_MS: z.coerce.number().int().default(30_000),
 
   RUN_ENGINE_WORKER_REDIS_HOST: z
     .string()
@@ -605,6 +610,11 @@ const EnvironmentSchema = z.object({
   RUN_ENGINE_RELEASE_CONCURRENCY_ENABLED: z.string().default("0"),
   RUN_ENGINE_RELEASE_CONCURRENCY_DISABLE_CONSUMERS: z.string().default("0"),
   RUN_ENGINE_RELEASE_CONCURRENCY_MAX_TOKENS_RATIO: z.coerce.number().default(1),
+  RUN_ENGINE_RELEASE_CONCURRENCY_RELEASINGS_MAX_AGE: z.coerce
+    .number()
+    .int()
+    .default(60_000 * 30),
+  RUN_ENGINE_RELEASE_CONCURRENCY_RELEASINGS_POLL_INTERVAL: z.coerce.number().int().default(60_000),
   RUN_ENGINE_RELEASE_CONCURRENCY_MAX_RETRIES: z.coerce.number().int().default(3),
   RUN_ENGINE_RELEASE_CONCURRENCY_CONSUMERS_COUNT: z.coerce.number().int().default(1),
   RUN_ENGINE_RELEASE_CONCURRENCY_POLL_INTERVAL: z.coerce.number().int().default(500),
@@ -715,6 +725,63 @@ const EnvironmentSchema = z.object({
 
   QUEUE_SSE_AUTORELOAD_INTERVAL_MS: z.coerce.number().int().default(5_000),
   QUEUE_SSE_AUTORELOAD_TIMEOUT_MS: z.coerce.number().int().default(60_000),
+
+  SLACK_BOT_TOKEN: z.string().optional(),
+  SLACK_SIGNUP_REASON_CHANNEL_ID: z.string().optional(),
+
+  // kapa.ai
+  KAPA_AI_WEBSITE_ID: z.string().optional(),
+
+  // BetterStack
+  BETTERSTACK_API_KEY: z.string().optional(),
+  BETTERSTACK_STATUS_PAGE_ID: z.string().optional(),
+
+  RUN_REPLICATION_REDIS_HOST: z
+    .string()
+    .optional()
+    .transform((v) => v ?? process.env.REDIS_HOST),
+  RUN_REPLICATION_REDIS_READER_HOST: z
+    .string()
+    .optional()
+    .transform((v) => v ?? process.env.REDIS_READER_HOST),
+  RUN_REPLICATION_REDIS_READER_PORT: z.coerce
+    .number()
+    .optional()
+    .transform(
+      (v) =>
+        v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined)
+    ),
+  RUN_REPLICATION_REDIS_PORT: z.coerce
+    .number()
+    .optional()
+    .transform((v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)),
+  RUN_REPLICATION_REDIS_USERNAME: z
+    .string()
+    .optional()
+    .transform((v) => v ?? process.env.REDIS_USERNAME),
+  RUN_REPLICATION_REDIS_PASSWORD: z
+    .string()
+    .optional()
+    .transform((v) => v ?? process.env.REDIS_PASSWORD),
+  RUN_REPLICATION_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"),
+
+  RUN_REPLICATION_CLICKHOUSE_URL: z.string().optional(),
+  RUN_REPLICATION_ENABLED: z.string().default("0"),
+  RUN_REPLICATION_SLOT_NAME: z.string().default("task_runs_to_clickhouse_v1"),
+  RUN_REPLICATION_PUBLICATION_NAME: z.string().default("task_runs_to_clickhouse_v1_publication"),
+  RUN_REPLICATION_MAX_FLUSH_CONCURRENCY: z.coerce.number().int().default(100),
+  RUN_REPLICATION_FLUSH_INTERVAL_MS: z.coerce.number().int().default(1000),
+  RUN_REPLICATION_FLUSH_BATCH_SIZE: z.coerce.number().int().default(100),
+  RUN_REPLICATION_LEADER_LOCK_TIMEOUT_MS: z.coerce.number().int().default(30_000),
+  RUN_REPLICATION_LEADER_LOCK_EXTEND_INTERVAL_MS: z.coerce.number().int().default(10_000),
+  RUN_REPLICATION_ACK_INTERVAL_SECONDS: z.coerce.number().int().default(10),
+  RUN_REPLICATION_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"),
+  RUN_REPLICATION_LEADER_LOCK_ADDITIONAL_TIME_MS: z.coerce.number().int().default(10_000),
+  RUN_REPLICATION_LEADER_LOCK_RETRY_INTERVAL_MS: z.coerce.number().int().default(500),
+  RUN_REPLICATION_WAIT_FOR_ASYNC_INSERT: z.string().default("0"),
+  RUN_REPLICATION_KEEP_ALIVE_ENABLED: z.string().default("1"),
+  RUN_REPLICATION_KEEP_ALIVE_IDLE_SOCKET_TTL_MS: z.coerce.number().int().optional(),
+  RUN_REPLICATION_MAX_OPEN_CONNECTIONS: z.coerce.number().int().default(10),
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/hooks/useCopy.ts

@@ -0,0 +1,22 @@
+import { useCallback, useState } from "react";
+
+export function useCopy(value: string, duration = 1500) {
+  const [copied, setCopied] = useState(false);
+
+  const copy = useCallback(
+    (e?: React.MouseEvent) => {
+      if (e) {
+        e.preventDefault();
+        e.stopPropagation();
+      }
+      navigator.clipboard.writeText(value);
+      setCopied(true);
+      setTimeout(() => {
+        setCopied(false);
+      }, duration);
+    },
+    [value, duration]
+  );
+
+  return { copy, copied };
+}

apps/webapp/app/hooks/useKapaWidget.tsx

@@ -0,0 +1,145 @@
+import { useMatches, useSearchParams } from "@remix-run/react";
+import { useCallback, useEffect, useState } from "react";
+import { useFeatures } from "~/hooks/useFeatures";
+import { type loader } from "~/root";
+import { useShortcuts } from "../components/primitives/ShortcutsProvider";
+import { useTypedMatchesData } from "./useTypedMatchData";
+
+type OpenOptions = { mode: string; query: string; submit: boolean };
+
+declare global {
+  interface Window {
+    Kapa: (
+      command: string,
+      options?: (() => void) | { onRender?: () => void } | OpenOptions,
+      remove?: string | { onRender?: () => void }
+    ) => void;
+  }
+}
+
+export function KapaScripts({ websiteId }: { websiteId?: string }) {
+  if (!websiteId) return null;
+
+  return (
+    <>
+      <script
+        async
+        src="https://widget.kapa.ai/kapa-widget.bundle.js"
+        data-website-id={websiteId}
+        data-project-name={"Trigger.dev"}
+        data-project-color={"#6366F1"}
+        data-project-logo={"https://content.trigger.dev/trigger-logo-circle.png"}
+        data-render-on-load={"false"}
+        data-button-hide={"true"}
+        data-modal-disclaimer-bg-color={"#D7D9DD"}
+        data-modal-header-bg-color={"#2C3034"}
+        data-modal-title-color={"#D7D9DD"}
+      ></script>
+      <script
+        dangerouslySetInnerHTML={{
+          __html: `
+                    (function () {
+                      let k = window.Kapa;
+                      if (!k) {
+                        let i = function () {
+                          i.c(arguments);
+                        };
+                        i.q = [];
+                        i.c = function (args) {
+                          i.q.push(args);
+                        };
+                        window.Kapa = i;
+                      }
+                    })();
+                  `,
+        }}
+      />
+    </>
+  );
+}
+
+export function useKapaConfig() {
+  const matches = useMatches();
+  const routeMatch = useTypedMatchesData<typeof loader>({
+    id: "root",
+    matches,
+  });
+  return routeMatch?.kapa;
+}
+
+export function useKapaWidget() {
+  const kapa = useKapaConfig();
+  const features = useFeatures();
+  const { disableShortcuts, enableShortcuts, areShortcutsEnabled } = useShortcuts();
+  const [isKapaOpen, setIsKapaOpen] = useState(false);
+  const [searchParams, setSearchParams] = useSearchParams();
+
+  const handleModalClose = useCallback(() => {
+    setIsKapaOpen(false);
+    enableShortcuts();
+  }, [enableShortcuts]);
+
+  const handleModalOpen = useCallback(() => {
+    setIsKapaOpen(true);
+    disableShortcuts();
+  }, [disableShortcuts]);
+
+  // Handle opening/closing
+  useEffect(() => {
+    if (!features.isManagedCloud || !kapa?.websiteId) return;
+
+    window.Kapa("render", {
+      onRender: () => {
+        //this auto-opens the Kapa widget on page load if the query contains ?aiHelp=
+        const aiHelp = searchParams.get("aiHelp");
+        if (aiHelp) {
+          setSearchParams((prev) => {
+            prev.delete("aiHelp");
+            return prev;
+          });
+
+          //we need to decode the aiHelp string because it's urlencoded
+          const decodedAiHelp = decodeURIComponent(aiHelp);
+          //sadly this delay is required, otherwise it fails the Kapa browser verification…
+          setTimeout(() => {
+            openKapa(decodedAiHelp);
+          }, 500);
+        }
+      },
+    });
+    window.Kapa("onModalOpen", handleModalOpen);
+    window.Kapa("onModalClose", handleModalClose);
+
+    return () => {
+      window.Kapa("onModalOpen", handleModalOpen, "remove");
+      window.Kapa("onModalClose", handleModalClose, "remove");
+    };
+  }, [features.isManagedCloud, kapa?.websiteId, searchParams, setSearchParams]);
+
+  // Handle opening the Kapa widget
+  const openKapa = useCallback(
+    (query?: string) => {
+      if (!features.isManagedCloud || !kapa?.websiteId) return;
+
+      window.Kapa(
+        "open",
+        query
+          ? {
+              mode: "ai",
+              query,
+              submit: true,
+            }
+          : undefined
+      );
+      setIsKapaOpen(true);
+      disableShortcuts();
+    },
+    [disableShortcuts, features.isManagedCloud, kapa?.websiteId]
+  );
+
+  return {
+    isKapaEnabled: features.isManagedCloud && kapa?.websiteId,
+    openKapa,
+    isKapaOpen,
+  };
+}

apps/webapp/app/hooks/useShortcutKeys.tsx

@@ -1,5 +1,6 @@
 import { useHotkeys } from "react-hotkeys-hook";
 import { useOperatingSystem } from "~/components/primitives/OperatingSystemProvider";
+import { useShortcuts } from "~/components/primitives/ShortcutsProvider";
 
 export type Modifier = "alt" | "ctrl" | "meta" | "shift" | "mod";
 
@@ -7,6 +8,7 @@ export type Shortcut = {
   key: string;
   modifiers?: Modifier[];
   enabledOnInputElements?: boolean;
+  enabled?: boolean;
 };
 
 export type ShortcutDefinition =
@@ -30,20 +32,26 @@ export function useShortcutKeys({
   enabledOnInputElements,
 }: useShortcutKeysProps) {
   const { platform } = useOperatingSystem();
+  const { areShortcutsEnabled } = useShortcuts();
   const isMac = platform === "mac";
   const relevantShortcut =
     shortcut && "mac" in shortcut ? (isMac ? shortcut.mac : shortcut.windows) : shortcut;
 
   const keys = createKeysFromShortcut(relevantShortcut);
+
+  const isEnabled = !disabled && areShortcutsEnabled && relevantShortcut?.enabled !== false;
+
   useHotkeys(
     keys,
     (event, hotkeysEvent) => {
       action(event);
     },
     {
-      enabled: !disabled,
-      enableOnFormTags: enabledOnInputElements ?? relevantShortcut?.enabledOnInputElements,
-      enableOnContentEditable: enabledOnInputElements ?? relevantShortcut?.enabledOnInputElements,
+      enabled: isEnabled,
+      enableOnFormTags:
+        isEnabled && (enabledOnInputElements ?? relevantShortcut?.enabledOnInputElements),
+      enableOnContentEditable:
+        isEnabled && (enabledOnInputElements ?? relevantShortcut?.enabledOnInputElements),
     }
   );
 }

apps/webapp/app/metrics.server.ts

@@ -4,7 +4,9 @@ import { env } from "./env.server";
 
 export const metricsRegister = singleton("metricsRegister", initializeMetricsRegister);
 
-function initializeMetricsRegister() {
+export type MetricsRegister = Registry<OpenMetricsContentType>;
+
+function initializeMetricsRegister(): MetricsRegister {
   const registry = new Registry<OpenMetricsContentType>();
 
   register.setDefaultLabels({

apps/webapp/app/models/user.server.ts

@@ -7,7 +7,7 @@ import {
   getDashboardPreferences,
 } from "~/services/dashboardPreferences.server";
 export type { User } from "@trigger.dev/database";
-
+import { assertEmailAllowed } from "~/utils/email";
 type FindOrCreateMagicLink = {
   authenticationMethod: "MAGIC_LINK";
   email: string;
@@ -38,31 +38,29 @@ export async function findOrCreateUser(input: FindOrCreateUser): Promise<LoggedI
   }
 }
 
-export async function findOrCreateMagicLinkUser(
-  input: FindOrCreateMagicLink
-): Promise<LoggedInUser> {
-  if (env.WHITELISTED_EMAILS && !new RegExp(env.WHITELISTED_EMAILS).test(input.email)) {
-    throw new Error("This email is unauthorized");
-  }
+export async function findOrCreateMagicLinkUser({
+  email,
+}: FindOrCreateMagicLink): Promise<LoggedInUser> {
+  assertEmailAllowed(email);
 
   const existingUser = await prisma.user.findFirst({
     where: {
-      email: input.email,
+      email,
     },
   });
 
   const adminEmailRegex = env.ADMIN_EMAILS ? new RegExp(env.ADMIN_EMAILS) : undefined;
-  const makeAdmin = adminEmailRegex ? adminEmailRegex.test(input.email) : false;
+  const makeAdmin = adminEmailRegex ? adminEmailRegex.test(email) : false;
 
   const user = await prisma.user.upsert({
     where: {
-      email: input.email,
+      email,
     },
     update: {
-      email: input.email,
+      email,
     },
     create: {
-      email: input.email,
+      email,
       authenticationMethod: "MAGIC_LINK",
       admin: makeAdmin, // only on create, to prevent automatically removing existing admins
     },
@@ -79,6 +77,8 @@ export async function findOrCreateGithubUser({
   authenticationProfile,
   authenticationExtraParams,
 }: FindOrCreateGithub): Promise<LoggedInUser> {
+  assertEmailAllowed(email);
+
   const name = authenticationProfile._json.name;
   let avatarUrl: string | undefined = undefined;
   if (authenticationProfile.photos[0]) {

apps/webapp/app/presenters/v3/ApiWaitpointListPresenter.server.ts

@@ -1,16 +1,12 @@
-import { RuntimeEnvironmentType, WaitpointTokenStatus } from "@trigger.dev/core/v3";
+import { type RuntimeEnvironmentType, WaitpointTokenStatus } from "@trigger.dev/core/v3";
+import { type RunEngineVersion } from "@trigger.dev/database";
 import { z } from "zod";
-import { BasePresenter } from "./basePresenter.server";
 import { CoercedDate } from "~/utils/zod";
-import { AuthenticatedEnvironment } from "@internal/run-engine";
-import {
-  WaitpointTokenListOptions,
-  WaitpointTokenListPresenter,
-} from "./WaitpointTokenListPresenter.server";
 import { ServiceValidationError } from "~/v3/services/baseService.server";
-import { RunEngineVersion } from "@trigger.dev/database";
+import { BasePresenter } from "./basePresenter.server";
+import { type WaitpointListOptions, WaitpointListPresenter } from "./WaitpointListPresenter.server";
 
-export const ApiWaitpointTokenListSearchParams = z.object({
+export const ApiWaitpointListSearchParams = z.object({
   "page[size]": z.coerce.number().int().positive().min(1).max(100).optional(),
   "page[after]": z.string().optional(),
   "page[before]": z.string().optional(),
@@ -61,9 +57,9 @@ export const ApiWaitpointTokenListSearchParams = z.object({
   "filter[createdAt][to]": CoercedDate,
 });
 
-type ApiWaitpointTokenListSearchParams = z.infer<typeof ApiWaitpointTokenListSearchParams>;
+type ApiWaitpointListSearchParams = z.infer<typeof ApiWaitpointListSearchParams>;
 
-export class ApiWaitpointTokenListPresenter extends BasePresenter {
+export class ApiWaitpointListPresenter extends BasePresenter {
   public async call(
     environment: {
       id: string;
@@ -72,11 +68,12 @@ export class ApiWaitpointTokenListPresenter extends BasePresenter {
         id: string;
         engine: RunEngineVersion;
       };
+      apiKey: string;
     },
-    searchParams: ApiWaitpointTokenListSearchParams
+    searchParams: ApiWaitpointListSearchParams
   ) {
     return this.trace("call", async (span) => {
-      const options: WaitpointTokenListOptions = {
+      const options: WaitpointListOptions = {
         environment,
       };
 
@@ -118,7 +115,7 @@ export class ApiWaitpointTokenListPresenter extends BasePresenter {
         options.to = searchParams["filter[createdAt][to]"].getTime();
       }
 
-      const presenter = new WaitpointTokenListPresenter();
+      const presenter = new WaitpointListPresenter();
       const result = await presenter.call(options);
 
       if (!result.success) {

apps/webapp/app/presenters/v3/ApiWaitpointPresenter.server.ts

@@ -2,8 +2,8 @@ import { logger, type RuntimeEnvironmentType } from "@trigger.dev/core/v3";
 import { type RunEngineVersion } from "@trigger.dev/database";
 import { ServiceValidationError } from "~/v3/services/baseService.server";
 import { BasePresenter } from "./basePresenter.server";
-import { WaitpointPresenter } from "./WaitpointPresenter.server";
-import { waitpointStatusToApiStatus } from "./WaitpointTokenListPresenter.server";
+import { waitpointStatusToApiStatus } from "./WaitpointListPresenter.server";
+import { generateHttpCallbackUrl } from "~/services/httpCallback.server";
 
 export class ApiWaitpointPresenter extends BasePresenter {
   public async call(
@@ -14,6 +14,7 @@ export class ApiWaitpointPresenter extends BasePresenter {
         id: string;
         engine: RunEngineVersion;
       };
+      apiKey: string;
     },
     waitpointId: string
   ) {
@@ -24,6 +25,7 @@ export class ApiWaitpointPresenter extends BasePresenter {
           environmentId: environment.id,
         },
         select: {
+          id: true,
           friendlyId: true,
           type: true,
           status: true,
@@ -62,6 +64,7 @@ export class ApiWaitpointPresenter extends BasePresenter {
       return {
         id: waitpoint.friendlyId,
         type: waitpoint.type,
+        url: generateHttpCallbackUrl(waitpoint.id, environment.apiKey),
         status: waitpointStatusToApiStatus(waitpoint.status, waitpoint.outputIsError),
         idempotencyKey: waitpoint.idempotencyKey,
         userProvidedIdempotencyKey: waitpoint.userProvidedIdempotencyKey,

apps/webapp/app/presenters/v3/EditSchedulePresenter.server.ts

@@ -1,13 +1,16 @@
-import { RuntimeEnvironmentType } from "@trigger.dev/database";
-import { PrismaClient, prisma } from "~/db.server";
-import { displayableEnvironment } from "~/models/runtimeEnvironment.server";
+import { type RuntimeEnvironmentType } from "@trigger.dev/database";
+import { type PrismaClient, prisma } from "~/db.server";
+import { displayableEnvironment, findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
 import { logger } from "~/services/logger.server";
 import { filterOrphanedEnvironments } from "~/utils/environmentSort";
 import { getTimezones } from "~/utils/timezones.server";
+import { findCurrentWorkerFromEnvironment } from "~/v3/models/workerDeployment.server";
+import { ServiceValidationError } from "~/v3/services/baseService.server";
 
 type EditScheduleOptions = {
   userId: string;
   projectSlug: string;
+  environmentSlug: string;
   friendlyId?: string;
 };
 
@@ -26,7 +29,7 @@ export class EditSchedulePresenter {
     this.#prismaClient = prismaClient;
   }
 
-  public async call({ userId, projectSlug, friendlyId }: EditScheduleOptions) {
+  public async call({ userId, projectSlug, environmentSlug, friendlyId }: EditScheduleOptions) {
     // Find the project scoped to the organization
     const project = await this.#prismaClient.project.findFirstOrThrow({
       select: {
@@ -62,13 +65,25 @@ export class EditSchedulePresenter {
       },
     });
 
-    const possibleTasks = await this.#prismaClient.backgroundWorkerTask.findMany({
-      distinct: ["slug"],
-      where: {
-        projectId: project.id,
-        triggerSource: "SCHEDULED",
-      },
-    });
+    const environment = await findEnvironmentBySlug(project.id, environmentSlug, userId);
+    if (!environment) {
+      throw new ServiceValidationError("No matching environment for project", 404);
+    }
+
+    //get the latest BackgroundWorker
+    const latestWorker = await findCurrentWorkerFromEnvironment(environment, this.#prismaClient);
+
+    //get all possible scheduled tasks
+    const possibleTasks = latestWorker
+      ? await this.#prismaClient.backgroundWorkerTask.findMany({
+          where: {
+            workerId: latestWorker.id,
+            projectId: project.id,
+            runtimeEnvironmentId: environment.id,
+            triggerSource: "SCHEDULED",
+          },
+        })
+      : [];
 
     const possibleEnvironments = filterOrphanedEnvironments(project.environments).map(
       (environment) => {
@@ -77,7 +92,7 @@ export class EditSchedulePresenter {
     );
 
     return {
-      possibleTasks: possibleTasks.map((task) => task.slug),
+      possibleTasks: possibleTasks.map((task) => task.slug).sort(),
       possibleEnvironments,
       possibleTimezones: getTimezones(),
       schedule: await this.#getExistingSchedule(friendlyId, possibleEnvironments),

apps/webapp/app/presenters/v3/QueueListPresenter.server.ts

@@ -16,17 +16,27 @@ export class QueueListPresenter extends BasePresenter {
 
   public async call({
     environment,
+    query,
     page,
   }: {
     environment: AuthenticatedEnvironment;
+    query?: string;
     page: number;
     perPage?: number;
   }) {
+    const hasFilters = query !== undefined && query.length > 0;
+
     // Get total count for pagination
     const totalQueues = await this._replica.taskQueue.count({
       where: {
         runtimeEnvironmentId: environment.id,
         version: "V2",
+        name: query
+          ? {
+              contains: query,
+              mode: "insensitive",
+            }
+          : undefined,
       },
     });
 
@@ -45,6 +55,7 @@ export class QueueListPresenter extends BasePresenter {
             success: false as const,
             code: "engine-version",
             totalQueues: 1,
+            hasFilters,
           };
         }
       }
@@ -53,26 +64,38 @@ export class QueueListPresenter extends BasePresenter {
         success: false as const,
         code: "engine-version",
         totalQueues,
+        hasFilters,
       };
     }
 
     return {
       success: true as const,
-      queues: await this.getQueuesWithPagination(environment, page),
+      queues: await this.getQueuesWithPagination(environment, query, page),
       pagination: {
         currentPage: page,
         totalPages: Math.ceil(totalQueues / this.perPage),
         count: totalQueues,
       },
       totalQueues,
+      hasFilters,
     };
   }
 
-  private async getQueuesWithPagination(environment: AuthenticatedEnvironment, page: number) {
+  private async getQueuesWithPagination(
+    environment: AuthenticatedEnvironment,
+    query: string | undefined,
+    page: number
+  ) {
     const queues = await this._replica.taskQueue.findMany({
       where: {
         runtimeEnvironmentId: environment.id,
         version: "V2",
+        name: query
+          ? {
+              contains: query,
+              mode: "insensitive",
+            }
+          : undefined,
       },
       select: {
         friendlyId: true,

apps/webapp/app/presenters/v3/RunPresenter.server.ts

@@ -32,6 +32,7 @@ export class RunPresenter {
     environmentSlug,
     runFriendlyId,
     showDeletedLogs,
+    showDebug,
   }: {
     userId: string;
     projectSlug: string;
@@ -39,6 +40,7 @@ export class RunPresenter {
     environmentSlug: string;
     runFriendlyId: string;
     showDeletedLogs: boolean;
+    showDebug: boolean;
   }) {
     const run = await this.#prismaClient.taskRun.findFirstOrThrow({
       select: {
@@ -131,7 +133,8 @@ export class RunPresenter {
       getTaskEventStoreTableForRun(run),
       run.traceId,
       run.rootTaskRun?.createdAt ?? run.createdAt,
-      run.completedAt ?? undefined
+      run.completedAt ?? undefined,
+      { includeDebugLogs: showDebug }
     );
     if (!traceSummary) {
       return {

apps/webapp/app/presenters/v3/ScheduleListPresenter.server.ts

@@ -6,9 +6,12 @@ import { getLimit } from "~/services/platform.v3.server";
 import { CheckScheduleService } from "~/v3/services/checkSchedule.server";
 import { calculateNextScheduledTimestamp } from "~/v3/utils/calculateNextSchedule.server";
 import { BasePresenter } from "./basePresenter.server";
+import { findCurrentWorkerFromEnvironment } from "~/v3/models/workerDeployment.server";
+import { ServiceValidationError } from "~/v3/services/baseService.server";
 
 type ScheduleListOptions = {
   projectId: string;
+  environmentId: string;
   userId?: string;
   pageSize?: number;
 } & ScheduleListFilters;
@@ -42,8 +45,8 @@ export class ScheduleListPresenter extends BasePresenter {
   public async call({
     userId,
     projectId,
+    environmentId,
     tasks,
-    environments,
     search,
     page,
     type,
@@ -84,16 +87,45 @@ export class ScheduleListPresenter extends BasePresenter {
       },
     });
 
+    const environment = project.environments.find((env) => env.id === environmentId);
+    if (!environment) {
+      throw new ServiceValidationError("No matching environment for project", 404);
+    }
+
     const schedulesCount = await CheckScheduleService.getUsedSchedulesCount({
       prisma: this._replica,
       environments: project.environments,
     });
 
+    const limit = await getLimit(project.organizationId, "schedules", 100_000_000);
+
+    //get the latest BackgroundWorker
+    const latestWorker = await findCurrentWorkerFromEnvironment(environment, this._replica);
+    if (!latestWorker) {
+      return {
+        currentPage: 1,
+        totalPages: 1,
+        totalCount: 0,
+        schedules: [],
+        possibleTasks: [],
+        hasFilters,
+        limits: {
+          used: schedulesCount,
+          limit,
+        },
+        filters: {
+          tasks,
+          search,
+        },
+      };
+    }
+
     //get all possible scheduled tasks
     const possibleTasks = await this._replica.backgroundWorkerTask.findMany({
-      distinct: ["slug"],
       where: {
+        workerId: latestWorker.id,
         projectId: project.id,
+        runtimeEnvironmentId: environmentId,
         triggerSource: "SCHEDULED",
       },
     });
@@ -107,7 +139,7 @@ export class ScheduleListPresenter extends BasePresenter {
         taskIdentifier: tasks ? { in: tasks } : undefined,
         instances: {
           some: {
-            environmentId: environments ? { in: environments } : undefined,
+            environmentId,
           },
         },
         type: filterType,
@@ -168,13 +200,11 @@ export class ScheduleListPresenter extends BasePresenter {
       where: {
         projectId: project.id,
         taskIdentifier: tasks ? { in: tasks } : undefined,
-        instances: environments
-          ? {
-              some: {
-                environmentId: environments ? { in: environments } : undefined,
-              },
-            }
-          : undefined,
+        instances: {
+          some: {
+            environmentId,
+          },
+        },
         type: filterType,
         AND: search
           ? {
@@ -242,17 +272,12 @@ export class ScheduleListPresenter extends BasePresenter {
       };
     });
 
-    const limit = await getLimit(project.organizationId, "schedules", 100_000_000);
-
     return {
       currentPage: page,
       totalPages: Math.ceil(totalCount / pageSize),
       totalCount: totalCount,
       schedules,
-      possibleTasks: possibleTasks.map((task) => task.slug),
-      possibleEnvironments: project.environments.map((environment) => {
-        return displayableEnvironment(environment, userId);
-      }),
+      possibleTasks: possibleTasks.map((task) => task.slug).sort((a, b) => a.localeCompare(b)),
       hasFilters,
       limits: {
         used: schedulesCount,
@@ -260,7 +285,6 @@ export class ScheduleListPresenter extends BasePresenter {
       },
       filters: {
         tasks,
-        environments,
         search,
       },
     };

apps/webapp/app/presenters/v3/SpanPresenter.server.ts

@@ -1,5 +1,4 @@
 import {
-  isWaitpointOutputTimeout,
   type MachinePresetName,
   prettyPrintPacket,
   SemanticInternalAttributes,
@@ -423,7 +422,8 @@ export class SpanPresenter extends BasePresenter {
       spanId,
       traceId,
       createdAt,
-      completedAt ?? undefined
+      completedAt ?? undefined,
+      { includeDebugLogs: true }
     );
     if (!span) {
       return;

apps/webapp/app/presenters/v3/WaitpointListPresenter.server.ts

@@ -11,10 +11,11 @@ import { BasePresenter } from "./basePresenter.server";
 import { type WaitpointSearchParams } from "~/components/runs/v3/WaitpointTokenFilters";
 import { determineEngineVersion } from "~/v3/engineVersion.server";
 import { type WaitpointTokenStatus, type WaitpointTokenItem } from "@trigger.dev/core/v3";
+import { generateHttpCallbackUrl } from "~/services/httpCallback.server";
 
 const DEFAULT_PAGE_SIZE = 25;
 
-export type WaitpointTokenListOptions = {
+export type WaitpointListOptions = {
   environment: {
     id: string;
     type: RuntimeEnvironmentType;
@@ -22,6 +23,7 @@ export type WaitpointTokenListOptions = {
       id: string;
       engine: RunEngineVersion;
     };
+    apiKey: string;
   };
   // filters
   id?: string;
@@ -63,7 +65,7 @@ type Result =
       filters: undefined;
     };
 
-export class WaitpointTokenListPresenter extends BasePresenter {
+export class WaitpointListPresenter extends BasePresenter {
   public async call({
     environment,
     id,
@@ -76,7 +78,7 @@ export class WaitpointTokenListPresenter extends BasePresenter {
     direction = "forward",
     cursor,
     pageSize = DEFAULT_PAGE_SIZE,
-  }: WaitpointTokenListOptions): Promise<Result> {
+  }: WaitpointListOptions): Promise<Result> {
     const engineVersion = await determineEngineVersion({ environment });
     if (engineVersion === "V1") {
       return {
@@ -165,8 +167,8 @@ export class WaitpointTokenListPresenter extends BasePresenter {
       ${sqlDatabaseSchema}."Waitpoint" w
     WHERE
       w."environmentId" = ${environment.id}
-      AND w.type = 'MANUAL'
-      -- cursor
+    AND w.type = 'MANUAL'      
+    -- cursor
       ${
         cursor
           ? direction === "forward"
@@ -263,6 +265,7 @@ export class WaitpointTokenListPresenter extends BasePresenter {
       success: true,
       tokens: tokensToReturn.map((token) => ({
         id: token.friendlyId,
+        url: generateHttpCallbackUrl(token.id, environment.apiKey),
         status: waitpointStatusToApiStatus(token.status, token.outputIsError),
         completedAt: token.completedAt ?? undefined,
         timeoutAt: token.completedAfter ?? undefined,

apps/webapp/app/presenters/v3/WaitpointPresenter.server.ts

@@ -1,8 +1,9 @@
 import { isWaitpointOutputTimeout, prettyPrintPacket } from "@trigger.dev/core/v3";
+import { generateHttpCallbackUrl } from "~/services/httpCallback.server";
 import { logger } from "~/services/logger.server";
 import { BasePresenter } from "./basePresenter.server";
 import { type RunListItem, RunListPresenter } from "./RunListPresenter.server";
-import { waitpointStatusToApiStatus } from "./WaitpointTokenListPresenter.server";
+import { waitpointStatusToApiStatus } from "./WaitpointListPresenter.server";
 
 export type WaitpointDetail = NonNullable<Awaited<ReturnType<WaitpointPresenter["call"]>>>;
 
@@ -22,6 +23,7 @@ export class WaitpointPresenter extends BasePresenter {
         environmentId,
       },
       select: {
+        id: true,
         friendlyId: true,
         type: true,
         status: true,
@@ -42,6 +44,11 @@ export class WaitpointPresenter extends BasePresenter {
           take: 5,
         },
         tags: true,
+        environment: {
+          select: {
+            apiKey: true,
+          },
+        },
       },
     });
 
@@ -83,6 +90,7 @@ export class WaitpointPresenter extends BasePresenter {
     return {
       id: waitpoint.friendlyId,
       type: waitpoint.type,
+      url: generateHttpCallbackUrl(waitpoint.id, waitpoint.environment.apiKey),
       status: waitpointStatusToApiStatus(waitpoint.status, waitpoint.outputIsError),
       idempotencyKey: waitpoint.idempotencyKey,
       userProvidedIdempotencyKey: waitpoint.userProvidedIdempotencyKey,

apps/webapp/app/root.tsx

@@ -1,19 +1,30 @@
 import type { LinksFunction, LoaderFunctionArgs, MetaFunction } from "@remix-run/node";
 import type { ShouldRevalidateFunction } from "@remix-run/react";
-import { Links, LiveReload, Meta, Outlet, Scripts, ScrollRestoration } from "@remix-run/react";
-import { UseDataFunctionReturn, typedjson, useTypedLoaderData } from "remix-typedjson";
+import {
+  Links,
+  LiveReload,
+  Meta,
+  Outlet,
+  Scripts,
+  ScrollRestoration,
+  useMatches,
+} from "@remix-run/react";
+import { type UseDataFunctionReturn, typedjson, useTypedLoaderData } from "remix-typedjson";
 import { ExternalScripts } from "remix-utils/external-scripts";
 import type { ToastMessage } from "~/models/message.server";
 import { commitSession, getSession } from "~/models/message.server";
 import tailwindStylesheetUrl from "~/tailwind.css";
 import { RouteErrorDisplay } from "./components/ErrorDisplay";
 import { AppContainer, MainCenteredContainer } from "./components/layout/AppLayout";
+import { ShortcutsProvider } from "./components/primitives/ShortcutsProvider";
 import { Toast } from "./components/primitives/Toast";
 import { env } from "./env.server";
 import { featuresForRequest } from "./features.server";
 import { usePostHog } from "./hooks/usePostHog";
+import { useTypedMatchesData } from "./hooks/useTypedMatchData";
 import { getUser } from "./services/session.server";
 import { appEnvTitleTag } from "./utils";
+import { KapaScripts } from "./hooks/useKapaWidget";
 
 export const links: LinksFunction = () => {
   return [{ rel: "stylesheet", href: tailwindStylesheetUrl }];
@@ -29,7 +40,10 @@ export const meta: MetaFunction = ({ data }) => {
     },
     {
       name: "robots",
-      content: typedData.features.isManagedCloud ? "index, follow" : "noindex, nofollow",
+      content:
+        typeof window === "undefined" || window.location.hostname !== "cloud.trigger.dev"
+          ? "noindex, nofollow"
+          : "index, follow",
     },
   ];
 };
@@ -40,6 +54,10 @@ export const loader = async ({ request }: LoaderFunctionArgs) => {
   const posthogProjectKey = env.POSTHOG_PROJECT_KEY;
   const features = featuresForRequest(request);
 
+  const kapa = {
+    websiteId: env.KAPA_AI_WEBSITE_ID,
+  };
+
   return typedjson(
     {
       user: await getUser(request),
@@ -48,6 +66,7 @@ export const loader = async ({ request }: LoaderFunctionArgs) => {
       features,
       appEnv: env.APP_ENV,
       appOrigin: env.APP_ORIGIN,
+      kapa,
     },
     { headers: { "Set-Cookie": await commitSession(session) } }
   );
@@ -73,7 +92,7 @@ export function ErrorBoundary() {
           <Meta />
           <Links />
         </head>
-        <body className="bg-darkBackground h-full overflow-hidden">
+        <body className="h-full overflow-hidden bg-background-dimmed">
           <AppContainer>
             <MainCenteredContainer>
               <RouteErrorDisplay />
@@ -86,8 +105,8 @@ export function ErrorBoundary() {
   );
 }
 
-function App() {
-  const { posthogProjectKey } = useTypedLoaderData<typeof loader>();
+export default function App() {
+  const { posthogProjectKey, kapa } = useTypedLoaderData<typeof loader>();
   usePostHog(posthogProjectKey);
 
   return (
@@ -96,10 +115,13 @@ function App() {
         <head>
           <Meta />
           <Links />
+          <KapaScripts websiteId={kapa.websiteId} />
         </head>
-        <body className="bg-darkBackground h-full overflow-hidden">
-          <Outlet />
-          <Toast />
+        <body className="h-full overflow-hidden bg-background-dimmed">
+          <ShortcutsProvider>
+            <Outlet />
+            <Toast />
+          </ShortcutsProvider>
           <ScrollRestoration />
           <ExternalScripts />
           <Scripts />
@@ -109,5 +131,3 @@ function App() {
     </>
   );
 }
-
-export default App;

apps/webapp/app/routes/_app.orgs.$organizationSlug.invite/route.tsx

@@ -134,6 +134,9 @@ export default function Page() {
     onValidate({ formData }) {
       return parse(formData, { schema });
     },
+    defaultValue: {
+      emails: [""],
+    },
   });
 
   const fieldValues = useRef<string[]>([""]);

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts.new/route.tsx

@@ -186,7 +186,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
   const project = await findProjectBySlug(organizationSlug, projectParam, userId);
 
   if (!project) {
-    submission.error.key = "Project not found";
+    submission.error.key = ["Project not found"];
     return json(submission);
   }
 
@@ -198,7 +198,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
   );
 
   if (!alertChannel) {
-    submission.error.key = "Failed to create alert channel";
+    submission.error.key = ["Failed to create alert channel"];
     return json(submission);
   }
 

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.alerts/route.tsx

@@ -122,7 +122,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
   const project = await findProjectBySlug(organizationSlug, projectParam, userId);
 
   if (!project) {
-    submission.error.key = "Project not found";
+    submission.error.key = ["Project not found"];
     return json(submission);
   }
 

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.apikeys/route.tsx

@@ -3,6 +3,7 @@ import { type MetaFunction } from "@remix-run/react";
 import { type LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
+import { InlineCode } from "~/components/code/InlineCode";
 import { EnvironmentCombo, environmentFullTitle } from "~/components/environments/EnvironmentLabel";
 import { RegenerateApiKeyModal } from "~/components/environments/RegenerateApiKeyModal";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
@@ -159,9 +160,9 @@ export default function Page() {
           <div className="flex flex-wrap justify-between">
             <InfoPanel icon={InformationCircleIcon} variant="minimal" panelClassName="max-w-fit">
               <Paragraph variant="small">
-                Secret keys should be used on your server. They give full API access and allow you
-                to <TextLink to={docsPath("v3/triggering")}>trigger tasks</TextLink> from your
-                backend.
+                Set your <TextLink to={docsPath("apikeys")}>Secret keys</TextLink> in your backend
+                by adding <InlineCode>TRIGGER_SECRET_KEY</InlineCode> env var in order to{" "}
+                <TextLink to={docsPath("v3/triggering")}>trigger tasks</TextLink>.
               </Paragraph>
             </InfoPanel>
           </div>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx

@@ -145,7 +145,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
     },
   });
   if (!project) {
-    submission.error.key = "Project not found";
+    submission.error.key = ["Project not found"];
     return json(submission);
   }
 
@@ -158,11 +158,11 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
         const index = submission.value.variables.findIndex((v) => v.key === key);
 
         if (index !== -1) {
-          submission.error[`variables[${index}].key`] = error;
+          submission.error[`variables[${index}].key`] = [error];
         }
       }
     } else {
-      submission.error.variables = result.error;
+      submission.error.variables = [result.error];
     }
 
     return json(submission);
@@ -197,6 +197,9 @@ export default function Page() {
       return parse(formData, { schema });
     },
     shouldRevalidate: "onSubmit",
+    defaultValue: {
+      variables: [{ key: "", value: "" }],
+    },
   });
 
   const [revealAll, setRevealAll] = useState(true);

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx

@@ -142,7 +142,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
     },
   });
   if (!project) {
-    submission.error.key = "Project not found";
+    submission.error.key = ["Project not found"];
     return json(submission);
   }
 
@@ -152,7 +152,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
       const result = await repository.editValue(project.id, submission.value);
 
       if (!result.success) {
-        submission.error.key = result.error;
+        submission.error.key = [result.error];
         return json(submission);
       }
 
@@ -175,7 +175,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
       const result = await repository.deleteValue(project.id, submission.value);
 
       if (!result.success) {
-        submission.error.key = result.error;
+        submission.error.key = [result.error];
         return json(submission);
       }
 

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.next.runs._index/route.tsx

@@ -0,0 +1,506 @@
+import { ArrowPathIcon, StopCircleIcon } from "@heroicons/react/20/solid";
+import { BeakerIcon, BookOpenIcon } from "@heroicons/react/24/solid";
+import { Form, type MetaFunction, useNavigation } from "@remix-run/react";
+import { type LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { IconCircleX } from "@tabler/icons-react";
+import { AnimatePresence, motion } from "framer-motion";
+import { ListChecks, ListX } from "lucide-react";
+import { Suspense, useState } from "react";
+import { TypedAwait, typeddefer, useTypedLoaderData } from "remix-typedjson";
+import { TaskIcon } from "~/assets/icons/TaskIcon";
+import { DevDisconnectedBanner, useDevPresence } from "~/components/DevPresence";
+import { StepContentContainer } from "~/components/StepContentContainer";
+import { MainCenteredContainer, PageBody } from "~/components/layout/AppLayout";
+import { Button, LinkButton } from "~/components/primitives/Buttons";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogFooter,
+  DialogHeader,
+  DialogTrigger,
+} from "~/components/primitives/Dialog";
+import { Header1, Header2 } from "~/components/primitives/Headers";
+import { InfoPanel } from "~/components/primitives/InfoPanel";
+import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
+import { Paragraph } from "~/components/primitives/Paragraph";
+import {
+  SelectedItemsProvider,
+  useSelectedItems,
+} from "~/components/primitives/SelectedItemsProvider";
+import { Spinner, SpinnerWhite } from "~/components/primitives/Spinner";
+import { StepNumber } from "~/components/primitives/StepNumber";
+import { TextLink } from "~/components/primitives/TextLink";
+import { RunsFilters, TaskRunListSearchFilters } from "~/components/runs/v3/RunFilters";
+import { TaskRunsTable } from "~/components/runs/v3/TaskRunsTable";
+import { BULK_ACTION_RUN_LIMIT } from "~/consts";
+import { useEnvironment } from "~/hooks/useEnvironment";
+import { useOrganization } from "~/hooks/useOrganizations";
+import { useProject } from "~/hooks/useProject";
+import { findProjectBySlug } from "~/models/project.server";
+import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
+import { RunListPresenter } from "~/presenters/v3/RunListPresenter.server";
+import {
+  getRootOnlyFilterPreference,
+  setRootOnlyFilterPreference,
+  uiPreferencesStorage,
+} from "~/services/preferences/uiPreferences.server";
+import { requireUserId } from "~/services/session.server";
+import { cn } from "~/utils/cn";
+import {
+  docsPath,
+  EnvironmentParamSchema,
+  v3ProjectPath,
+  v3RunsNextPath,
+  v3TestPath,
+} from "~/utils/pathBuilder";
+import { ListPagination } from "../../components/ListPagination";
+
+export const meta: MetaFunction = () => {
+  return [
+    {
+      title: `Runs | Trigger.dev`,
+    },
+  ];
+};
+
+export const loader = async ({ request, params }: LoaderFunctionArgs) => {
+  const userId = await requireUserId(request);
+  const { projectParam, organizationSlug, envParam } = EnvironmentParamSchema.parse(params);
+
+  const url = new URL(request.url);
+
+  let rootOnlyValue = false;
+  if (url.searchParams.has("rootOnly")) {
+    rootOnlyValue = url.searchParams.get("rootOnly") === "true";
+  } else {
+    rootOnlyValue = await getRootOnlyFilterPreference(request);
+  }
+
+  const project = await findProjectBySlug(organizationSlug, projectParam, userId);
+  if (!project) {
+    throw new Error("Project not found");
+  }
+
+  const environment = await findEnvironmentBySlug(project.id, envParam, userId);
+  if (!environment) {
+    throw new Error("Environment not found");
+  }
+
+  const s = {
+    cursor: url.searchParams.get("cursor") ?? undefined,
+    direction: url.searchParams.get("direction") ?? undefined,
+    statuses: url.searchParams.getAll("statuses"),
+    environments: [environment.id],
+    tasks: url.searchParams.getAll("tasks"),
+    period: url.searchParams.get("period") ?? undefined,
+    bulkId: url.searchParams.get("bulkId") ?? undefined,
+    tags: url.searchParams.getAll("tags").map((t) => decodeURIComponent(t)),
+    from: url.searchParams.get("from") ?? undefined,
+    to: url.searchParams.get("to") ?? undefined,
+    rootOnly: rootOnlyValue,
+    runId: url.searchParams.get("runId") ?? undefined,
+    batchId: url.searchParams.get("batchId") ?? undefined,
+    scheduleId: url.searchParams.get("scheduleId") ?? undefined,
+  };
+  const {
+    tasks,
+    versions,
+    statuses,
+    environments,
+    tags,
+    period,
+    bulkId,
+    from,
+    to,
+    cursor,
+    direction,
+    rootOnly,
+    runId,
+    batchId,
+    scheduleId,
+  } = TaskRunListSearchFilters.parse(s);
+
+  const presenter = new RunListPresenter();
+  const list = presenter.call({
+    userId,
+    projectId: project.id,
+    tasks,
+    versions,
+    statuses,
+    environments,
+    tags,
+    period,
+    bulkId,
+    from,
+    to,
+    batchId,
+    runIds: runId ? [runId] : undefined,
+    scheduleId,
+    rootOnly,
+    direction: direction,
+    cursor: cursor,
+  });
+
+  const session = await setRootOnlyFilterPreference(rootOnlyValue, request);
+  const cookieValue = await uiPreferencesStorage.commitSession(session);
+
+  return typeddefer(
+    {
+      data: list,
+      rootOnlyDefault: rootOnlyValue,
+    },
+    {
+      headers: {
+        "Set-Cookie": cookieValue,
+      },
+    }
+  );
+};
+
+export default function Page() {
+  const { data, rootOnlyDefault } = useTypedLoaderData<typeof loader>();
+  const navigation = useNavigation();
+  const isLoading = navigation.state !== "idle";
+  const { isConnected } = useDevPresence();
+  const project = useProject();
+  const environment = useEnvironment();
+
+  return (
+    <>
+      <NavBar>
+        <PageTitle title="Runs" />
+        {environment.type === "DEVELOPMENT" && project.engine === "V2" && (
+          <DevDisconnectedBanner isConnected={isConnected} />
+        )}
+        <PageAccessories>
+          <LinkButton
+            variant={"docs/small"}
+            LeadingIcon={BookOpenIcon}
+            to={docsPath("/runs-and-attempts")}
+          >
+            Runs docs
+          </LinkButton>
+        </PageAccessories>
+      </NavBar>
+      <PageBody scrollable={false}>
+        <SelectedItemsProvider
+          initialSelectedItems={[]}
+          maxSelectedItemCount={BULK_ACTION_RUN_LIMIT}
+        >
+          {({ selectedItems }) => (
+            <div
+              className={cn(
+                "grid h-full max-h-full overflow-hidden",
+                selectedItems.size === 0 ? "grid-rows-1" : "grid-rows-[1fr_auto]"
+              )}
+            >
+              <Suspense
+                fallback={
+                  <div className="flex items-center justify-center py-2">
+                    <div className="mx-auto flex items-center gap-2">
+                      <Spinner />
+                      <Paragraph variant="small">Loading runs</Paragraph>
+                    </div>
+                  </div>
+                }
+              >
+                <TypedAwait resolve={data}>
+                  {(list) => (
+                    <>
+                      {list.runs.length === 0 && !list.hasAnyRuns ? (
+                        list.possibleTasks.length === 0 ? (
+                          <CreateFirstTaskInstructions />
+                        ) : (
+                          <RunTaskInstructions />
+                        )
+                      ) : (
+                        <div
+                          className={cn(
+                            "grid h-full max-h-full grid-rows-[auto_1fr] overflow-hidden"
+                          )}
+                        >
+                          <div className="flex items-start justify-between gap-x-2 p-2">
+                            <RunsFilters
+                              possibleTasks={list.possibleTasks}
+                              bulkActions={list.bulkActions}
+                              hasFilters={list.hasFilters}
+                              rootOnlyDefault={rootOnlyDefault}
+                            />
+                            <div className="flex items-center justify-end gap-x-2">
+                              <ListPagination list={list} />
+                            </div>
+                          </div>
+
+                          <TaskRunsTable
+                            total={list.runs.length}
+                            hasFilters={list.hasFilters}
+                            filters={list.filters}
+                            runs={list.runs}
+                            isLoading={isLoading}
+                            allowSelection
+                          />
+                        </div>
+                      )}
+                    </>
+                  )}
+                </TypedAwait>
+              </Suspense>
+              <BulkActionBar />
+            </div>
+          )}
+        </SelectedItemsProvider>
+      </PageBody>
+    </>
+  );
+}
+
+function BulkActionBar() {
+  const { selectedItems, deselectAll } = useSelectedItems();
+  const [barState, setBarState] = useState<"none" | "replay" | "cancel">("none");
+
+  const hasSelectedMaximum = selectedItems.size >= BULK_ACTION_RUN_LIMIT;
+
+  return (
+    <AnimatePresence>
+      {selectedItems.size > 0 && (
+        <motion.div
+          initial={{ translateY: "100%" }}
+          animate={{ translateY: 0 }}
+          exit={{ translateY: "100%" }}
+          className="flex items-center justify-between gap-3 border-t border-grid-bright bg-background-bright py-3 pl-4 pr-3"
+        >
+          <div className="flex items-center gap-1.5 text-sm text-text-bright">
+            <ListChecks className="mr-1 size-7 text-indigo-400" />
+            <Header2>Bulk actions:</Header2>
+            {hasSelectedMaximum ? (
+              <Paragraph className="text-warning">
+                Maximum of {selectedItems.size} runs selected
+              </Paragraph>
+            ) : (
+              <Paragraph className="">{selectedItems.size} runs selected</Paragraph>
+            )}
+          </div>
+          <div className="flex items-center gap-3">
+            <CancelRuns
+              onOpen={(o) => {
+                if (o) {
+                  setBarState("cancel");
+                } else {
+                  setBarState("none");
+                }
+              }}
+            />
+            <ReplayRuns
+              onOpen={(o) => {
+                if (o) {
+                  setBarState("replay");
+                } else {
+                  setBarState("none");
+                }
+              }}
+            />
+            <Button
+              variant="tertiary/medium"
+              shortcut={{ key: "esc", enabledOnInputElements: true }}
+              onClick={() => {
+                if (barState !== "none") return;
+                deselectAll();
+              }}
+              LeadingIcon={ListX}
+              leadingIconClassName="text-indigo-400 w-6 h-6"
+            >
+              Clear selection
+            </Button>
+          </div>
+        </motion.div>
+      )}
+    </AnimatePresence>
+  );
+}
+
+function CancelRuns({ onOpen }: { onOpen: (open: boolean) => void }) {
+  const { selectedItems } = useSelectedItems();
+
+  const organization = useOrganization();
+  const project = useProject();
+  const environment = useEnvironment();
+  const failedRedirect = v3RunsNextPath(organization, project, environment);
+
+  const formAction = `/resources/taskruns/bulk/cancel`;
+
+  const navigation = useNavigation();
+  const isLoading = navigation.formAction === formAction;
+
+  return (
+    <Dialog onOpenChange={(o) => onOpen(o)}>
+      <DialogTrigger asChild>
+        <Button
+          type="button"
+          variant="tertiary/medium"
+          shortcut={{ key: "c", enabledOnInputElements: true }}
+          LeadingIcon={IconCircleX}
+          leadingIconClassName="text-error w-[1.3rem] h-[1.3rem]"
+        >
+          Cancel runs
+        </Button>
+      </DialogTrigger>
+      <DialogContent key="replay">
+        <DialogHeader>Cancel {selectedItems.size} runs?</DialogHeader>
+        <DialogDescription className="pt-2">
+          Canceling these runs will stop them from running. Only runs that are not already finished
+          will be canceled, the others will remain in their existing state.
+        </DialogDescription>
+        <DialogFooter>
+          <Form action={formAction} method="post" reloadDocument>
+            <input type="hidden" name="failedRedirect" value={failedRedirect} />
+            <input type="hidden" name="organizationSlug" value={organization.slug} />
+            <input type="hidden" name="projectSlug" value={project.slug} />
+            <input type="hidden" name="environmentSlug" value={environment.slug} />
+            {[...selectedItems].map((runId) => (
+              <input key={runId} type="hidden" name="runIds" value={runId} />
+            ))}
+            <Button
+              type="submit"
+              variant="danger/medium"
+              LeadingIcon={isLoading ? SpinnerWhite : StopCircleIcon}
+              disabled={isLoading}
+              shortcut={{ modifiers: ["mod"], key: "enter" }}
+            >
+              {isLoading ? "Canceling..." : `Cancel ${selectedItems.size} runs`}
+            </Button>
+          </Form>
+        </DialogFooter>
+      </DialogContent>
+    </Dialog>
+  );
+}
+
+function ReplayRuns({ onOpen }: { onOpen: (open: boolean) => void }) {
+  const { selectedItems } = useSelectedItems();
+
+  const organization = useOrganization();
+  const project = useProject();
+  const environment = useEnvironment();
+  const failedRedirect = v3RunsNextPath(organization, project, environment);
+
+  const formAction = `/resources/taskruns/bulk/replay`;
+
+  const navigation = useNavigation();
+  const isLoading = navigation.formAction === formAction;
+
+  return (
+    <Dialog onOpenChange={(o) => onOpen(o)}>
+      <DialogTrigger asChild>
+        <Button
+          type="button"
+          variant="tertiary/medium"
+          shortcut={{ key: "r", enabledOnInputElements: true }}
+          LeadingIcon={ArrowPathIcon}
+          leadingIconClassName="text-blue-400 w-[1.3rem] h-[1.3rem]"
+        >
+          <span className="text-text-bright">Replay {selectedItems.size} runs</span>
+        </Button>
+      </DialogTrigger>
+      <DialogContent key="replay">
+        <DialogHeader>Replay runs?</DialogHeader>
+        <DialogDescription className="pt-2">
+          Replaying these runs will create a new run for each with the same payload and environment
+          as the original. It will use the latest version of the code for each task.
+        </DialogDescription>
+        <DialogFooter>
+          <Form action={formAction} method="post" reloadDocument>
+            <input type="hidden" name="failedRedirect" value={failedRedirect} />
+            <input type="hidden" name="organizationSlug" value={organization.slug} />
+            <input type="hidden" name="projectSlug" value={project.slug} />
+            <input type="hidden" name="environmentSlug" value={environment.slug} />
+            {[...selectedItems].map((runId) => (
+              <input key={runId} type="hidden" name="runIds" value={runId} />
+            ))}
+            <Button
+              type="submit"
+              variant="primary/medium"
+              LeadingIcon={isLoading ? SpinnerWhite : ArrowPathIcon}
+              disabled={isLoading}
+              shortcut={{ modifiers: ["mod"], key: "enter" }}
+            >
+              {isLoading ? "Replaying..." : `Replay ${selectedItems.size} runs`}
+            </Button>
+          </Form>
+        </DialogFooter>
+      </DialogContent>
+    </Dialog>
+  );
+}
+
+function CreateFirstTaskInstructions() {
+  const organization = useOrganization();
+  const project = useProject();
+  return (
+    <MainCenteredContainer className="max-w-md">
+      <InfoPanel
+        icon={TaskIcon}
+        iconClassName="text-blue-500"
+        panelClassName="max-full"
+        title="Create your first task"
+        accessory={
+          <LinkButton to={v3ProjectPath(organization, project)} variant="primary/small">
+            Create a task
+          </LinkButton>
+        }
+      >
+        <Paragraph variant="small">
+          Before running a task, you must first create one. Follow the instructions on the{" "}
+          <TextLink to={v3ProjectPath(organization, project)}>Tasks</TextLink> page to create a
+          task, then return here to run it.
+        </Paragraph>
+      </InfoPanel>
+    </MainCenteredContainer>
+  );
+}
+
+function RunTaskInstructions() {
+  const organization = useOrganization();
+  const project = useProject();
+  const environment = useEnvironment();
+  return (
+    <MainCenteredContainer className="max-w-prose">
+      <Header1 className="mb-6 border-b py-2">How to run your tasks</Header1>
+      <StepNumber stepNumber="A" title="Trigger a test run" />
+      <StepContentContainer>
+        <Paragraph spacing>
+          Perform a test run with a payload directly from the dashboard.
+        </Paragraph>
+        <LinkButton
+          to={v3TestPath(organization, project, environment)}
+          variant="secondary/medium"
+          LeadingIcon={BeakerIcon}
+          leadingIconClassName="text-lime-500"
+          className="inline-flex"
+        >
+          Test
+        </LinkButton>
+        <div className="mt-6 flex items-center gap-2">
+          <hr className="w-full" />
+          <Paragraph variant="extra-extra-small/dimmed/caps">OR</Paragraph>
+          <hr className="w-full" />
+        </div>
+      </StepContentContainer>
+
+      <StepNumber stepNumber="B" title="Trigger your task for real" />
+      <StepContentContainer>
+        <Paragraph spacing>
+          Performing a real run depends on the type of trigger your task is using.
+        </Paragraph>
+        <LinkButton
+          to={docsPath("/triggering")}
+          variant="docs/medium"
+          LeadingIcon={BookOpenIcon}
+          className="inline-flex"
+        >
+          How to trigger a task
+        </LinkButton>
+      </StepContentContainer>
+    </MainCenteredContainer>
+  );
+}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.next.runs/route.tsx

@@ -0,0 +1,10 @@
+import { Outlet } from "@remix-run/react";
+import { PageContainer } from "~/components/layout/AppLayout";
+
+export default function Page() {
+  return (
+    <PageContainer>
+      <Outlet />
+    </PageContainer>
+  );
+}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx

@@ -2,12 +2,20 @@ import {
   ArrowUpCircleIcon,
   BookOpenIcon,
   ChatBubbleLeftEllipsisIcon,
+  MagnifyingGlassIcon,
   PauseIcon,
   PlayIcon,
   RectangleStackIcon,
 } from "@heroicons/react/20/solid";
 import { DialogClose } from "@radix-ui/react-dialog";
-import { Form, useNavigation, useRevalidator, type MetaFunction } from "@remix-run/react";
+import {
+  Form,
+  useNavigate,
+  useNavigation,
+  useRevalidator,
+  useSearchParams,
+  type MetaFunction,
+} from "@remix-run/react";
 import { type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { type RuntimeEnvironmentType } from "@trigger.dev/database";
 import { useEffect, useState } from "react";
@@ -61,8 +69,12 @@ import { docsPath, EnvironmentParamSchema, v3BillingPath } from "~/utils/pathBui
 import { PauseEnvironmentService } from "~/v3/services/pauseEnvironment.server";
 import { PauseQueueService } from "~/v3/services/pauseQueue.server";
 import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route";
+import { Header3 } from "~/components/primitives/Headers";
+import { Input } from "~/components/primitives/Input";
+import { useThrottle } from "~/hooks/useThrottle";
 
 const SearchParamsSchema = z.object({
+  query: z.string().optional(),
   page: z.coerce.number().min(1).default(1),
 });
 
@@ -79,7 +91,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const { organizationSlug, projectParam, envParam } = EnvironmentParamSchema.parse(params);
 
   const url = new URL(request.url);
-  const { page } = SearchParamsSchema.parse(Object.fromEntries(url.searchParams));
+  const { page, query } = SearchParamsSchema.parse(Object.fromEntries(url.searchParams));
 
   const project = await findProjectBySlug(organizationSlug, projectParam, userId);
   if (!project) {
@@ -101,6 +113,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
     const queueListPresenter = new QueueListPresenter();
     const queues = await queueListPresenter.call({
       environment,
+      query,
       page,
     });
 
@@ -198,7 +211,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
 };
 
 export default function Page() {
-  const { environment, queues, success, pagination, code, totalQueues } =
+  const { environment, queues, success, pagination, code, totalQueues, hasFilters } =
     useTypedLoaderData<typeof loader>();
 
   const organization = useOrganization();
@@ -245,13 +258,30 @@ export default function Page() {
               suffix={env.paused && environment.queued > 0 ? "paused" : undefined}
               animate
               accessory={<EnvironmentPauseResumeButton env={env} />}
-              valueClassName={env.paused ? "text-amber-500" : undefined}
+              valueClassName={env.paused ? "text-warning" : undefined}
+              compactThreshold={1000000}
+            />
+            <BigNumber
+              title="Running"
+              value={environment.running}
+              animate
+              valueClassName={
+                environment.running === environment.concurrencyLimit ? "text-warning" : undefined
+              }
+              suffix={
+                environment.running === environment.concurrencyLimit
+                  ? "At concurrency limit"
+                  : undefined
+              }
+              compactThreshold={1000000}
             />
-            <BigNumber title="Running" value={environment.running} animate />
             <BigNumber
               title="Concurrency limit"
               value={environment.concurrencyLimit}
               animate
+              valueClassName={
+                environment.running === environment.concurrencyLimit ? "text-warning" : undefined
+              }
               accessory={
                 plan ? (
                   plan?.v3Subscription?.plan?.limits.concurrentRuns.canExceed ? (
@@ -285,16 +315,47 @@ export default function Page() {
           {success ? (
             <div
               className={cn(
-                "grid max-h-full min-h-full grid-rows-[1fr] overflow-x-auto",
-                pagination.totalPages > 1 && "grid-rows-[1fr_auto]"
+                "grid max-h-full min-h-full grid-rows-[auto_1fr] overflow-x-auto",
+                pagination.totalPages > 1 && "grid-rows-[auto_1fr_auto]"
               )}
             >
+              <QueueFilters />
               <Table containerClassName="border-t">
                 <TableHeader>
                   <TableRow>
                     <TableHeaderCell>Name</TableHeaderCell>
                     <TableHeaderCell alignment="right">Queued</TableHeaderCell>
-                    <TableHeaderCell alignment="right">Running</TableHeaderCell>
+                    <TableHeaderCell alignment="right">Running/limit</TableHeaderCell>
+                    <TableHeaderCell
+                      alignment="right"
+                      tooltip={
+                        <div className="max-w-xs space-y-2 p-1 text-left">
+                          <div className="space-y-0.5">
+                            <Header3>Environment</Header3>
+                            <Paragraph
+                              variant="small"
+                              className="!text-wrap text-text-dimmed"
+                              spacing
+                            >
+                              This queue is limited by your environment's concurrency limit of{" "}
+                              {environment.concurrencyLimit}.
+                            </Paragraph>
+                          </div>
+                          <div className="space-y-0.5">
+                            <Header3>User</Header3>
+                            <Paragraph
+                              variant="small"
+                              className="!text-wrap text-text-dimmed"
+                              spacing
+                            >
+                              This queue is limited by a concurrency limit set in your code.
+                            </Paragraph>
+                          </div>
+                        </div>
+                      }
+                    >
+                      Limited by
+                    </TableHeaderCell>
                     <TableHeaderCell
                       alignment="right"
                       tooltip={
@@ -321,7 +382,6 @@ export default function Page() {
                     >
                       Release on waitpoint
                     </TableHeaderCell>
-                    <TableHeaderCell alignment="right">Concurrency limit</TableHeaderCell>
                     <TableHeaderCell className="w-[1%] pl-24">
                       <span className="sr-only">Pause/resume</span>
                     </TableHeaderCell>
@@ -329,85 +389,112 @@ export default function Page() {
                 </TableHeader>
                 <TableBody>
                   {queues.length > 0 ? (
-                    queues.map((queue) => (
-                      <TableRow key={queue.name}>
-                        <TableCell>
-                          <span className="flex items-center gap-2">
-                            {queue.type === "task" ? (
-                              <SimpleTooltip
-                                button={
-                                  <TaskIconSmall
-                                    className={cn(
-                                      "size-[1.125rem] text-blue-500",
-                                      queue.paused && "opacity-50"
-                                    )}
-                                  />
-                                }
-                                content={`This queue was automatically created from your "${queue.name}" task`}
-                              />
-                            ) : (
-                              <SimpleTooltip
-                                button={
-                                  <RectangleStackIcon
-                                    className={cn(
-                                      "size-[1.125rem] text-purple-500",
-                                      queue.paused && "opacity-50"
-                                    )}
-                                  />
-                                }
-                                content={`This is a custom queue you added in your code.`}
-                              />
+                    queues.map((queue) => {
+                      const limit = queue.concurrencyLimit ?? environment.concurrencyLimit;
+                      const isAtLimit = queue.running === limit;
+                      return (
+                        <TableRow key={queue.name}>
+                          <TableCell>
+                            <span className="flex items-center gap-2">
+                              {queue.type === "task" ? (
+                                <SimpleTooltip
+                                  button={
+                                    <TaskIconSmall
+                                      className={cn(
+                                        "size-[1.125rem] text-blue-500",
+                                        queue.paused && "opacity-50"
+                                      )}
+                                    />
+                                  }
+                                  content={`This queue was automatically created from your "${queue.name}" task`}
+                                />
+                              ) : (
+                                <SimpleTooltip
+                                  button={
+                                    <RectangleStackIcon
+                                      className={cn(
+                                        "size-[1.125rem] text-purple-500",
+                                        queue.paused && "opacity-50"
+                                      )}
+                                    />
+                                  }
+                                  content={`This is a custom queue you added in your code.`}
+                                />
+                              )}
+                              <span className={queue.paused ? "opacity-50" : undefined}>
+                                {queue.name}
+                              </span>
+                              {queue.paused ? (
+                                <Badge variant="extra-small" className="text-warning">
+                                  Paused
+                                </Badge>
+                              ) : null}
+                              {isAtLimit ? (
+                                <Badge variant="extra-small" className="text-warning">
+                                  At concurrency limit
+                                </Badge>
+                              ) : null}
+                            </span>
+                          </TableCell>
+                          <TableCell
+                            alignment="right"
+                            className={queue.paused ? "opacity-50" : undefined}
+                          >
+                            {queue.queued}
+                          </TableCell>
+                          <TableCell
+                            alignment="right"
+                            className={cn(
+                              queue.paused ? "tabular-nums opacity-50" : undefined,
+                              isAtLimit && "text-warning"
                             )}
-                            <span className={queue.paused ? "opacity-50" : undefined}>
-                              {queue.name}
+                          >
+                            {queue.running}/
+                            <span
+                              className={cn(
+                                "tabular-nums text-text-dimmed",
+                                isAtLimit && "text-warning"
+                              )}
+                            >
+                              {limit}
                             </span>
-                            {queue.paused ? (
-                              <Badge variant="extra-small" className="text-warning">
-                                Paused
-                              </Badge>
-                            ) : null}
-                          </span>
-                        </TableCell>
-                        <TableCell
-                          alignment="right"
-                          className={queue.paused ? "opacity-50" : undefined}
-                        >
-                          {queue.queued}
-                        </TableCell>
-                        <TableCell
-                          alignment="right"
-                          className={queue.paused ? "opacity-50" : undefined}
-                        >
-                          {queue.running}
-                        </TableCell>
-                        <TableCell
-                          alignment="right"
-                          className={queue.paused ? "opacity-50" : undefined}
-                        >
-                          {queue.releaseConcurrencyOnWaitpoint ? "Yes" : "No"}
-                        </TableCell>
-                        <TableCell
-                          alignment="right"
-                          className={queue.paused ? "opacity-50" : undefined}
-                        >
-                          {queue.concurrencyLimit ?? (
-                            <span className="text-text-dimmed">
-                              Max ({environment.concurrencyLimit})
-                            </span>
-                          )}
-                        </TableCell>
-                        <TableCellMenu
-                          isSticky
-                          visibleButtons={queue.paused && <QueuePauseResumeButton queue={queue} />}
-                          hiddenButtons={!queue.paused && <QueuePauseResumeButton queue={queue} />}
-                        />
-                      </TableRow>
-                    ))
+                          </TableCell>
+                          <TableCell
+                            alignment="right"
+                            className={cn(
+                              queue.paused ? "opacity-50" : undefined,
+                              isAtLimit && "text-warning"
+                            )}
+                          >
+                            {queue.concurrencyLimit ? "User" : "Environment"}
+                          </TableCell>
+                          <TableCell
+                            alignment="right"
+                            className={queue.paused ? "opacity-50" : undefined}
+                          >
+                            {queue.releaseConcurrencyOnWaitpoint ? "Yes" : "No"}
+                          </TableCell>
+                          <TableCellMenu
+                            isSticky
+                            visibleButtons={
+                              queue.paused && <QueuePauseResumeButton queue={queue} />
+                            }
+                            hiddenButtons={
+                              !queue.paused && <QueuePauseResumeButton queue={queue} />
+                            }
+                          />
+                        </TableRow>
+                      );
+                    })
                   ) : (
                     <TableRow>
                       <TableCell colSpan={6}>
                         <div className="grid place-items-center py-6 text-text-dimmed">
-                          <Paragraph>No queues found</Paragraph>
+                          <Paragraph>
+                            {hasFilters
+                              ? "No queues found matching your filters"
+                              : "No queues found"}
+                          </Paragraph>
                         </div>
                       </TableCell>
                     </TableRow>
@@ -486,7 +573,7 @@ function EnvironmentPauseResumeButton({
                     type="button"
                     variant="secondary/small"
                     LeadingIcon={env.paused ? PlayIcon : PauseIcon}
-                    leadingIconClassName={env.paused ? "text-success" : "text-amber-500"}
+                    leadingIconClassName={env.paused ? "text-success" : "text-warning"}
                   >
                     {env.paused ? "Resume..." : "Pause environment..."}
                   </Button>
@@ -495,8 +582,8 @@ function EnvironmentPauseResumeButton({
             </TooltipTrigger>
             <TooltipContent className={"text-xs"}>
               {env.paused
-                ? `Resume processing runs in ${environmentFullTitle(env)}.`
-                : `Pause processing runs in ${environmentFullTitle(env)}.`}
+                ? `Resume processing runs in ${environmentFullTitle(env)}`
+                : `Pause processing runs in ${environmentFullTitle(env)}`}
             </TooltipContent>
           </Tooltip>
         </TooltipProvider>
@@ -523,7 +610,9 @@ function EnvironmentPauseResumeButton({
                   type="submit"
                   disabled={isLoading}
                   variant={env.paused ? "primary/medium" : "danger/medium"}
-                  LeadingIcon={isLoading ? <Spinner /> : env.paused ? PlayIcon : PauseIcon}
+                  LeadingIcon={
+                    isLoading ? <Spinner color="white" /> : env.paused ? PlayIcon : PauseIcon
+                  }
                   shortcut={{ modifiers: ["mod"], key: "enter" }}
                 >
                   {env.paused ? "Resume environment" : "Pause environment"}
@@ -565,7 +654,7 @@ function QueuePauseResumeButton({
                     type="button"
                     variant="tertiary/small"
                     LeadingIcon={queue.paused ? PlayIcon : PauseIcon}
-                    leadingIconClassName={queue.paused ? "text-success" : "text-amber-500"}
+                    leadingIconClassName={queue.paused ? "text-success" : "text-warning"}
                   >
                     {queue.paused ? "Resume..." : "Pause..."}
                   </Button>
@@ -663,3 +752,39 @@ export function isEnvironmentPauseResumeFormSubmission(
       formData.get("action") === "environment-resume")
   );
 }
+
+export function QueueFilters() {
+  const [searchParams, setSearchParams] = useSearchParams();
+
+  const handleSearchChange = useThrottle((value: string) => {
+    if (value) {
+      setSearchParams((prev) => {
+        prev.set("query", value);
+        prev.delete("page");
+        return prev;
+      });
+    } else {
+      setSearchParams((prev) => {
+        prev.delete("query");
+        prev.delete("page");
+        return prev;
+      });
+    }
+  }, 300);
+
+  const search = searchParams.get("query") ?? "";
+
+  return (
+    <div className="flex w-full border-t border-grid-dimmed px-1.5 py-1.5">
+      <Input
+        name="search"
+        placeholder="Search queue name"
+        icon={MagnifyingGlassIcon}
+        variant="tertiary"
+        className="grow"
+        defaultValue={search}
+        onChange={(e) => handleSearchChange(e.target.value)}
+      />
+    </div>
+  );
+}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx

@@ -95,6 +95,7 @@ import {
 } from "~/utils/pathBuilder";
 import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route";
 import { SpanView } from "../resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route";
+import { useSearchParams } from "~/hooks/useSearchParam";
 
 const resizableSettings = {
   parent: {
@@ -133,6 +134,9 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const impersonationId = await getImpersonationId(request);
   const { projectParam, organizationSlug, envParam, runParam } = v3RunParamsSchema.parse(params);
 
+  const url = new URL(request.url);
+  const showDebug = url.searchParams.get("showDebug") === "true";
+
   const presenter = new RunPresenter();
   const [error, result] = await tryCatch(
     presenter.call({
@@ -142,6 +146,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
       projectSlug: projectParam,
       runFriendlyId: runParam,
       environmentSlug: envParam,
+      showDebug,
     })
   );
 
@@ -505,15 +510,18 @@ function TasksTreeView({
   const isAdmin = useHasAdminAccess();
   const [filterText, setFilterText] = useState("");
   const [errorsOnly, setErrorsOnly] = useState(false);
-  const [showDebug, setShowDebug] = useState(false);
   const [showDurations, setShowDurations] = useState(true);
   const [showQueueTime, setShowQueueTime] = useState(false);
   const [scale, setScale] = useState(0);
   const parentRef = useRef<HTMLDivElement>(null);
   const treeScrollRef = useRef<HTMLDivElement>(null);
   const timelineScrollRef = useRef<HTMLDivElement>(null);
+  const { value, replace } = useSearchParams();
 
-  const displayEvents = showDebug ? events : events.filter((event) => !event.data.isDebug);
+  const searchValue = value("showDebug");
+  const showDebug = searchValue !== undefined ? searchValue === "true" : false;
+
+  const displayEvents = events;
   const queuedTime = showQueueTime ? undefined : queuedDuration;
 
   const {
@@ -560,7 +568,11 @@ function TasksTreeView({
             label="Debug"
             shortcut={{ modifiers: ["shift"], key: "D" }}
             checked={showDebug}
-            onCheckedChange={(e) => setShowDebug(e.valueOf())}
+            onCheckedChange={(checked) => {
+              replace({
+                showDebug: checked ? "true" : "false",
+              });
+            }}
           />
         )}
         <Switch

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules.edit.$scheduleParam/route.tsx

@@ -15,6 +15,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const result = await presenter.call({
     userId,
     projectSlug: projectParam,
+    environmentSlug: envParam,
     friendlyId: scheduleParam,
   });
 

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules.new/route.tsx

@@ -2,18 +2,19 @@ import { type LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { EditSchedulePresenter } from "~/presenters/v3/EditSchedulePresenter.server";
 import { requireUserId } from "~/services/session.server";
-import { ProjectParamSchema } from "~/utils/pathBuilder";
+import { EnvironmentParamSchema } from "~/utils/pathBuilder";
 import { humanToCronSupported } from "~/v3/humanToCron.server";
 import { UpsertScheduleForm } from "../resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules.new/route";
 
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const userId = await requireUserId(request);
-  const { projectParam, organizationSlug } = ProjectParamSchema.parse(params);
+  const { projectParam, envParam, organizationSlug } = EnvironmentParamSchema.parse(params);
 
   const presenter = new EditSchedulePresenter();
   const result = await presenter.call({
     userId,
     projectSlug: projectParam,
+    environmentSlug: envParam,
   });
 
   return typedjson({ ...result, showGenerateField: humanToCronSupported });

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules/route.tsx

@@ -38,7 +38,7 @@ import {
   TableHeaderCell,
   TableRow,
 } from "~/components/primitives/Table";
-import { SimpleTooltip } from "~/components/primitives/Tooltip";
+import { InfoIconTooltip, SimpleTooltip } from "~/components/primitives/Tooltip";
 import { EnabledStatus } from "~/components/runs/v3/EnabledStatus";
 import { ScheduleFilters, ScheduleListFilters } from "~/components/runs/v3/ScheduleFilters";
 import {
@@ -93,12 +93,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const url = new URL(request.url);
   const s = Object.fromEntries(url.searchParams.entries());
   const filters = ScheduleListFilters.parse(s);
-  filters.environments = [environment.id];
 
   const presenter = new ScheduleListPresenter();
   const list = await presenter.call({
     userId,
     projectId: project.id,
+    environmentId: environment.id,
     ...filters,
   });
 
@@ -106,15 +106,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
 };
 
 export default function Page() {
-  const {
-    schedules,
-    possibleTasks,
-    possibleEnvironments,
-    hasFilters,
-    limits,
-    currentPage,
-    totalPages,
-  } = useTypedLoaderData<typeof loader>();
+  const { schedules, possibleTasks, hasFilters, limits, currentPage, totalPages } =
+    useTypedLoaderData<typeof loader>();
   const location = useLocation();
   const organization = useOrganization();
   const project = useProject();
@@ -281,9 +274,12 @@ export default function Page() {
                             plan to enable more.
                           </Header3>
                         ) : (
-                          <Header3>
-                            You've used {limits.used}/{limits.limit} of your schedules.
-                          </Header3>
+                          <div className="flex items-center gap-1">
+                            <Header3>
+                              You've used {limits.used}/{limits.limit} of your schedules
+                            </Header3>
+                            <InfoIconTooltip content="Schedules created in Dev don't count towards your limit." />
+                          </div>
                         )}
 
                         {canUpgrade ? (
@@ -291,6 +287,7 @@ export default function Page() {
                             to={v3BillingPath(organization)}
                             variant="secondary/small"
                             LeadingIcon={ArrowUpCircleIcon}
+                            leadingIconClassName="text-indigo-500"
                           >
                             Upgrade
                           </LinkButton>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.tokens/route.tsx

@@ -7,6 +7,7 @@ import { NoWaitpointTokens } from "~/components/BlankStatePanels";
 import { MainCenteredContainer, PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { ListPagination } from "~/components/ListPagination";
 import { LinkButton } from "~/components/primitives/Buttons";
+import { ClipboardField } from "~/components/primitives/ClipboardField";
 import { CopyableText } from "~/components/primitives/CopyableText";
 import { DateTime } from "~/components/primitives/DateTime";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
@@ -36,7 +37,7 @@ import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
 import { findProjectBySlug } from "~/models/project.server";
 import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
-import { WaitpointTokenListPresenter } from "~/presenters/v3/WaitpointTokenListPresenter.server";
+import { WaitpointListPresenter } from "~/presenters/v3/WaitpointListPresenter.server";
 import { requireUserId } from "~/services/session.server";
 import { docsPath, EnvironmentParamSchema, v3WaitpointTokenPath } from "~/utils/pathBuilder";
 
@@ -84,7 +85,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   }
 
   try {
-    const presenter = new WaitpointTokenListPresenter();
+    const presenter = new WaitpointListPresenter();
     const result = await presenter.call({
       environment,
       ...searchParams,
@@ -143,6 +144,7 @@ export default function Page() {
                       <TableRow>
                         <TableHeaderCell className="w-[1%]">Created</TableHeaderCell>
                         <TableHeaderCell className="w-[20%]">ID</TableHeaderCell>
+                        <TableHeaderCell className="w-[20%]">Callback URL</TableHeaderCell>
                         <TableHeaderCell className="w-[20%]">Status</TableHeaderCell>
                         <TableHeaderCell className="w-[20%]">Completed</TableHeaderCell>
                         <TableHeaderCell className="w-[20%]">Idempotency Key</TableHeaderCell>
@@ -178,6 +180,9 @@ export default function Page() {
                               <TableCell to={path}>
                                 <CopyableText value={token.id} className="font-mono" />
                               </TableCell>
+                              <TableCell to={path}>
+                                <ClipboardField value={token.url} variant={"secondary/small"} />
+                              </TableCell>
                               <TableCell to={path}>
                                 <WaitpointStatusCombo status={token.status} className="text-xs" />
                               </TableCell>

apps/webapp/app/routes/_app.orgs.new/route.tsx

@@ -5,7 +5,6 @@ import { RadioGroup } from "@radix-ui/react-radio-group";
 import type { ActionFunction, LoaderFunctionArgs } from "@remix-run/node";
 import { json, redirect } from "@remix-run/node";
 import { Form, useActionData, useNavigation } from "@remix-run/react";
-import { uiComponent } from "@team-plain/typescript-sdk";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { z } from "zod";
 import { MainCenteredContainer } from "~/components/layout/AppLayout";
@@ -23,10 +22,9 @@ import { TextArea } from "~/components/primitives/TextArea";
 import { useFeatures } from "~/hooks/useFeatures";
 import { createOrganization } from "~/models/organization.server";
 import { NewOrganizationPresenter } from "~/presenters/NewOrganizationPresenter.server";
-import { logger } from "~/services/logger.server";
 import { requireUser, requireUserId } from "~/services/session.server";
+import { sendNewOrgMessage } from "~/services/slack.server";
 import { organizationPath, rootPath } from "~/utils/pathBuilder";
-import { sendToPlain } from "~/utils/plain.server";
 
 const schema = z.object({
   orgName: z.string().min(3).max(50),
@@ -63,32 +61,11 @@ export const action: ActionFunction = async ({ request }) => {
     const whyUseUs = formData.get("whyUseUs");
 
     if (whyUseUs) {
-      try {
-        await sendToPlain({
-          userId: user.id,
-          email: user.email,
-          name: user.name ?? user.displayName ?? user.email,
-          title: "New org feedback",
-          components: [
-            uiComponent.text({
-              text: `${submission.value.orgName} just created a new organization.`,
-            }),
-            uiComponent.divider({ spacingSize: "M" }),
-            uiComponent.text({
-              size: "L",
-              color: "NORMAL",
-              text: "What problem are you trying to solve?",
-            }),
-            uiComponent.text({
-              size: "L",
-              color: "NORMAL",
-              text: whyUseUs.toString(),
-            }),
-          ],
-        });
-      } catch (error) {
-        logger.error("Error sending data to Plain when creating an org:", { error });
-      }
+      await sendNewOrgMessage({
+        orgName: submission.value.orgName,
+        whyUseUs: whyUseUs.toString(),
+        userEmail: user.email,
+      });
     }
 
     return redirect(organizationPath(organization));