Release 3.0.0-beta.36

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.changeset/afraid-sheep-joke.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fixes an issue with scoped packages in additionalPackages option

.changeset/cool-comics-burn.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fixing missing logs when importing client @opentelemetry/api

.changeset/famous-boats-tease.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Better handle issues with resolving dependency versions during deploy

.changeset/five-toes-destroy.md

@@ -0,0 +1,7 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+v3: Environment variable management API and SDK, along with resolveEnvVars CLI hook

.changeset/hot-buckets-behave.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Support triggering tasks with non-URL friendly characters in the ID

.changeset/hot-fishes-retire.md

@@ -0,0 +1,5 @@
+---
+trigger.dev: patch
+---
+
+Fix TypeScript inclusion in tsconfig.json for `cli-v3 init`

.changeset/itchy-chairs-itch.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Fix for calling trigger and passing a custom queue

.changeset/new-pants-beg.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Fix issue when using SDK in non-node environments by scoping the stream import with node:

.changeset/pre.json

@@ -44,6 +44,7 @@
     "@trigger.dev/yalt": "2.3.18"
   },
   "changesets": [
+    "afraid-sheep-joke",
     "angry-eagles-trade",
     "beige-pens-dance",
     "big-tomatoes-deliver",
@@ -54,16 +55,22 @@
     "clever-apes-collect",
     "clever-carrots-travel",
     "clever-donkeys-hunt",
+    "cool-comics-burn",
     "cool-glasses-bake",
     "cuddly-feet-approve",
     "dry-walls-check",
     "dull-mangos-press",
     "eight-pumas-float",
     "eleven-paws-join",
+    "famous-boats-tease",
     "few-students-share",
+    "five-toes-destroy",
     "funny-swans-destroy",
     "gorgeous-gorillas-compete",
     "green-bags-wink",
+    "hot-buckets-behave",
+    "hot-fishes-retire",
+    "itchy-chairs-itch",
     "khaki-apricots-design",
     "khaki-poems-lay",
     "late-icons-lie",
@@ -80,6 +87,7 @@
     "mighty-camels-joke",
     "mighty-flowers-train",
     "nasty-jars-pump",
+    "new-pants-beg",
     "new-rivers-tell",
     "nice-bulldogs-turn",
     "ninety-pets-travel",
@@ -102,12 +110,16 @@
     "shiny-coats-cry",
     "silly-suits-switch",
     "six-ligers-exist",
+    "sixty-insects-watch",
     "slow-buses-own",
     "smart-needles-move",
     "smart-olives-eat",
     "spicy-lamps-smoke",
     "strange-ghosts-matter",
+    "strange-sheep-pull",
     "strong-lemons-add",
+    "strong-owls-know",
+    "stupid-adults-sniff",
     "stupid-bulldogs-applaud",
     "sweet-lizards-press",
     "swift-dragons-peel",
@@ -116,12 +128,18 @@
     "tender-moose-tell",
     "tender-oranges-rhyme",
     "thin-parents-heal",
+    "thirty-islands-kiss",
     "tidy-balloons-suffer",
     "tidy-dryers-sleep",
+    "tidy-tomatoes-explain",
     "tiny-doors-type",
     "tiny-elephants-scream",
     "tricky-bulldogs-heal",
+    "tricky-keys-attack",
     "tricky-ladybugs-unite",
-    "two-pumas-wait"
+    "two-pumas-wait",
+    "warm-olives-provide",
+    "warm-planes-taste",
+    "young-snails-sell"
   ]
 }

.changeset/sixty-insects-watch.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Set the deploy timeout to 3mins from 1min

.changeset/strange-sheep-pull.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+v2: Better handle recovering from platform communication errors by auto-yielding back to the platform in case of temporary API failures

.changeset/strong-owls-know.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+When a v2 run hits the rate limit, reschedule with the reset date

.changeset/stupid-adults-sniff.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+Management SDK overhaul and adding the runs.list API

.changeset/thirty-islands-kiss.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix for typo in v3 CLI login command

.changeset/tidy-tomatoes-explain.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Fix 3rd party otel propagation from breaking our Task Events data from being properly correlated to the correct trace

.changeset/tricky-keys-attack.md

@@ -0,0 +1,14 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+- Clear paused states before retry
+- Detect and handle unrecoverable worker errors
+- Remove checkpoints after successful push
+- Permanently switch to DO hosted busybox image
+- Fix IPC timeout issue, or at least handle it more gracefully
+- Handle checkpoint failures
+- Basic chaos monkey for checkpoint testing
+- Stack traces are back in the dashboard
+- Display final errors on root span

.changeset/warm-olives-provide.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Improve handling of IPC timeouts and fix checkpoint cancellation after failures

.changeset/warm-planes-taste.md

@@ -0,0 +1,7 @@
+---
+"trigger.dev": patch
+---
+
+v3: Prevent legacy-peer-deps=true from breaking deploys
+
+When a global `.npmrc` file includes `legacy-peer-deps=true`, deploys would fail on the `npm ci` step because the package-lock.json wouldn't match the `package.json` file. This is because inside the image build, the `.npmrc` file would not be picked up and so `legacy-peer-deps` would end up being false (which is the default). This change forces the `package-lock.json` file to be created using `legacy-peer-deps=false`

.changeset/young-snails-sell.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Increase cleanup IPC timeout

.env.example

@@ -11,12 +11,17 @@ REMIX_APP_PORT=3030
 APP_ENV=development
 APP_ORIGIN=http://localhost:3030
 NODE_ENV=development
+CLOUD_ENV=development
+V3_ENABLED=true
 
 # Redis is used for the v3 queuing and v2 concurrency control
 REDIS_HOST="localhost"
 REDIS_PORT="6379"
 REDIS_TLS_DISABLED="true"
 
+DEV_OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:3030/otel"
+DEV_OTEL_BATCH_PROCESSING_ENABLED="0"
+
 # OPTIONAL VARIABLES
 # This is used for validating emails that are allowed to log in. Every email that do not match this regex will be rejected.
 # WHITELISTED_EMAILS="authorized@yahoo\.com|authorized@gmail\.com"

.vscode/launch.json

@@ -45,6 +45,14 @@
       "cwd": "${workspaceFolder}/references/v3-catalog",
       "sourceMaps": true
     },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug V3 Management",
+      "command": "pnpm run management",
+      "cwd": "${workspaceFolder}/references/v3-catalog",
+      "sourceMaps": true
+    },
     {
       "type": "node",
       "request": "attach",

CONTRIBUTING.md

@@ -10,13 +10,14 @@ Thank you for helping us make Trigger.dev even better! 🤩
 
 The development branch is `main`. This is the branch that all pull
 requests should be made against. The changes on the `main`
-branch are tagged into a release monthly.
+branch are tagged into a release periodically.
 
 ### Prerequisites
 
-- [Node.js](https://nodejs.org/en) version >=16.x
-- [pnpm package manager](https://pnpm.io/installation) version 7
+- [Node.js](https://nodejs.org/en) version 20.11.1
+- [pnpm package manager](https://pnpm.io/installation) version 8.15.5
 - [Docker](https://www.docker.com/get-started/)
+- [protobuf](https://github.com/protocolbuffers/protobuf)
 
 ### Setup
 
@@ -33,15 +34,19 @@ branch are tagged into a release monthly.
    ```
    cd trigger.dev
    ```
-3. Install the required packages using pnpm.
+3. Ensure you are on the correct version of Node.js (20.11.1). If you are using `nvm`, there is an `.nvmrc` file that will automatically select the correct version of Node.js when you navigate to the repository.
+
+4. Run `corepack enable` to use the correct version of pnpm (`8.15.5`) as specified in the root `package.json` file.
+
+5. Install the required packages using pnpm.
    ```
    pnpm i
    ```
-4. Create your `.env` file
+6. Create your `.env` file
    ```
    cp .env.example .env
    ```
-5. Open it and generate a new value for `ENCRYPTION_KEY`:
+7. Open it and generate a new value for `ENCRYPTION_KEY`:
 
    `ENCRYPTION_KEY` is used to two-way encrypt OAuth access tokens and so you'll probably want to actually generate a unique value, and it must be a random 16 byte hex string. You can generate one with the following command:
 
@@ -51,7 +56,7 @@ branch are tagged into a release monthly.
 
    Feel free to update `SESSION_SECRET` and `MAGIC_LINK_SECRET` as well using the same method.
 
-6. Start Docker. This starts the required services like Postgres. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)
+8. Start Docker. This starts the required services like Postgres & Redis. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)
 
    ```
    pnpm run docker
@@ -59,19 +64,15 @@ branch are tagged into a release monthly.
 
    This will also start and run a local instance of [pgAdmin](https://www.pgadmin.org/) on [localhost:5480](http://localhost:5480), preconfigured with email `admin@example.com` and pwd `admin`. Then use `postgres` as the password to the Trigger.dev server.
 
-7. Migrate the database
+9. Migrate the database
    ```
    pnpm run db:migrate
    ```
-8. Build the app
-   ```
-   pnpm run build --filter webapp
-   ```
-9. Run the seed script
-   ```
-   pnpm run db:seed
-   ```
-10. Run the app. See the section below.
+10. Build the server app
+    ```
+    pnpm run build --filter webapp
+    ```
+11. Run the app. See the section below.
 
 ## Running
 
@@ -83,99 +84,96 @@ branch are tagged into a release monthly.
 
    It should run on port `3030`: [http://localhost:3030](http://localhost:3030/)
 
-2. Once the app is running click the magic link button and enter your email.
-3. Check your terminal, the magic link email should have printed out as following:
+2. Once the app is running click the magic link button and enter your email. You will automatically be logged in, since you are running locally. Create an Org and your first project in the dashboard.
 
-   ```sh
-   webapp:dev: Log in to Trigger.dev
-   webapp:dev:
-   webapp:dev: Click here to log in with this magic link
-   webapp:dev: [http://localhost:3030/magic?token=U2FsdGVkX18OvB0JxgaswTLCSbaRz%2FY82TN0EZWhSzFyZYwgG%2BIzKVTkeiaOtWfotPw7F8RwFzCHh53aBpMEu%2B%2B%2FItb%2FcJYh89MSjc3Pz92bevoEjqxSQ%2Ff%2BZbks09JOpqlBbYC3FzGWC8vuSVFBlxqLXxteSDLthZSUaC%2BS2LaA%2BJgp%2BLO7hgjAaC2lXbCHrM7MTgTdXOFt7i0Dvvuwz6%2BWY25RnfomZOPqDsyH0xz8Q2rzPTz0Xu53WSXrZ1hd]
-   webapp:dev:
-   webapp:dev: If you didn't try to log in, you can safely ignore this email.
-   ```
+## Manual testing using v3-catalog
 
-   Paste the magic link shown in your terminal into your browser to login.
+We use the `<root>/references/v3-catalog` subdirectory as a staging ground for testing changes to the SDK (`@trigger.dev/sdk` at `<root>/packages/trigger-sdk`), the Core package (`@trigger.dev/core` at `<root>packages/core`), the CLI (`trigger.dev` at `<root>/packages/cli-v3`) and the platform (The remix app at `<root>/apps/webapp`). The instructions below will get you started on using the `v3-catalog` for local development of Trigger.dev (v3).
 
-## Adding and running migrations
+### First-time setup
 
-1. Modify packages/database/prisma/schema.prisma file
-2. Change directory to the packages/database folder
-   ```sh
-   cd packages/database
-   ```
-3. Generate the Prisma client
+First, make sure you are running the webapp according to the instructions above. Then:
 
-   ```sh
-   pnpm run generate
-   ```
+1. In Postgres go to the "Organizations" table and on your org set the `v3Enabled` column to `true`.
 
-   The above updates the prisma client generated into node_modules/.prisma/client folder. This helps with typing of relevant prisma models. It ensures typescript
-   recognizes fields added or removed from a model and type-checks appropriately.
+2. Visit http://localhost:3030 in your browser and create a new V3 project called "v3-catalog". If you don't see an option for V3, you haven't set the `v3Enabled` flag to true.
 
-4. Create and apply the migrations
+3. In Postgres go to the "Projects" table and for the project you create change the `externalRef` to `yubjwjsfkxnylobaqvqz`.
 
-   ```
-   pnpm run db:migrate:dev
-   ```
-
-   This creates a migration file and executes the migrations against your database and applies changes to the database schema(s)
-
-5. Commit generated migrations as well as changes to the schema.prisma file
-6. If you're using VSCode you may need to restart the Typescript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`.
-
-## Testing CLI changes
-
-To test CLI changes, follow the steps below:
-
-1. Build the CLI and watch for changes
+4. Build the CLI
 
 ```sh
-cd packages/cli
-pnpm run dev
-```
-
-2. Open a new Terminal window and run the webapp locally and then create a new project in the dashboard. Copy out the dev API key.
-
-3. Create a new temporary Next.js app in references directory
-
-```sh
-cd ./references
-pnpm create next-app@latest test-cli --ts --no-eslint --tailwind --app --src-dir --import-alias "@/*"
-```
-
-4. Then once that's finished, add the `@trigger.dev/cli` to the `devDependencies` of the newly created Next.js app's `package.json` file, like so:
-
-```json
-{
-  // other package.json properties
-  "devDependencies": { "@trigger.dev/cli": "workspace:*" }
-}
-```
-
-5. Back in the terminal, navigate into the reference, and initialize the CLI. When prompted, select `self-hosted` and enter `localhost:3030` if you are testing against the local instance of Trigger.dev, or you can just use the Trigger.dev cloud. When asked for an API key, use the key you copied earlier.
-
-```sh
-cd ./test-cli
+# Build the CLI
+pnpm run build --filter trigger.dev
+# Make it accessible to `pnpm exec`
 pnpm i
-pnpm exec trigger-cli init
 ```
 
-6. If you are just testing the `init` command, you can stop here. If you'd like to test the `dev` command, first start the Next.js app on port 3000:
+5. Change into the `<root>/references/v3-catalog` directory and authorize the CLI to the local server:
 
 ```sh
-pnpm run dev
+cd references/v3-catalog
+cp .env.example .env
+pnpm exec triggerdev login -a http://localhost:3030
 ```
 
-7. Open a new terminal window, and then run the `dev` command like so:
+This will open a new browser window and authorize the CLI against your local user account.
+
+You can optionally pass a `--profile` flag to the `login` command, which will allow you to use the CLI with separate accounts/servers. We suggest using a profile called `local` for your local development:
 
 ```sh
-pnpm exec trigger-cli dev
+cd references/v3-catalog
+pnpm exec triggerdev login -a http://localhost:3030 --profile local
+# later when you run the dev or deploy command:
+pnpm exec triggerdev dev --profile local
+pnpm exec triggerdev deploy --profile local
 ```
 
-8. Please remember to delete the temporary project you created after you've tested the changes, and before you raise a PR.
+### Running
 
-## Running end-to-end webapp tests
+The following steps should be followed any time you start working on a new feature you want to test in v3:
+
+1. Make sure the webapp is running on localhost:3030
+
+2. Open a terminal window and build the CLI and watch for changes
+
+```sh
+pnpm run dev --filter trigger.dev
+```
+
+2. Open a new terminal window, and anytime changes are made to the `@trigger.dev/core` package, you'll need to manually rebuild the CLI:
+
+```sh
+pnpm run build --filter trigger.dev
+```
+
+Note: You do not need to do the same for `@trigger.dev/sdk`, just core.
+
+3. Open another terminal window, and change into the `<root>/references/v3-catalog` directory.
+
+4. Run the `dev` command, which will register all the local tasks with the platform and allow you to start testing task execution:
+
+```sh
+# in <root>/references/v3-catalog
+pnpm exec triggerdev dev
+```
+
+If you want additional debug logging, you can use the `--log-level debug` flag:
+
+```sh
+# in <root>/references/v3-catalog
+pnpm exec triggerdev dev --log-level debug
+```
+
+5. If you make any changes in the CLI/Core/SDK, you'll need to `CTRL+C` to exit the `dev` command and restart it to pickup changes. Any changes to the files inside of the `v3-catalog/src/trigger` dir will automatically be rebuilt by the `dev` command.
+
+6. Navigate to the `v3-catalog` project in your local dashboard at localhost:3030 and you should see the list of tasks.
+
+7. Go to the "Test" page in the sidebar and select a task. Then enter a payload and click "Run test". You can tell what the payloads should be by looking at the relevant task file inside the `/references/v3-catalog/src/trigger` folder. Many of them accept an empty payload.
+
+8. Feel free to add additional files in `v3-catalog/src/trigger` to test out specific aspects of the system, or add in edge cases.
+
+## Running end-to-end webapp tests (deprecated)
 
 To run the end-to-end tests, follow the steps below:
 
@@ -223,6 +221,26 @@ The end-to-end tests use a `setup` and `teardown` script to seed the database wi
 pnpm run db:studio
 ```
 
+## Adding and running migrations
+
+1. Modify packages/database/prisma/schema.prisma file
+2. Change directory to the packages/database folder
+
+   ```sh
+   cd packages/database
+   ```
+
+3. Create and apply the migrations
+
+   ```
+   pnpm run db:migrate:dev
+   ```
+
+   This creates a migration file and executes the migrations against your database and applies changes to the database schema(s)
+
+4. Commit generated migrations as well as changes to the schema.prisma file
+5. If you're using VSCode you may need to restart the Typescript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`.
+
 ## Add sample jobs
 
 The [references/job-catalog](./references/job-catalog/) project defines simple jobs you can get started with.

apps/coordinator/src/backoff.ts

@@ -0,0 +1,247 @@
+type ExponentialBackoffType = "NoJitter" | "FullJitter" | "EqualJitter";
+
+type ExponentialBackoffOptions = {
+  base: number;
+  factor: number;
+  min: number;
+  max: number;
+  maxRetries: number;
+  maxElapsed: number;
+};
+
+class StopRetrying extends Error {
+  constructor(message?: string) {
+    super(message);
+    this.name = "StopRetrying";
+  }
+}
+
+export class ExponentialBackoff {
+  #retries: number = 0;
+
+  #type: ExponentialBackoffType;
+  #base: number;
+  #factor: number;
+
+  #min: number;
+  #max: number;
+
+  #maxRetries: number;
+  #maxElapsed: number;
+
+  constructor(type?: ExponentialBackoffType, opts: Partial<ExponentialBackoffOptions> = {}) {
+    this.#type = type ?? "NoJitter";
+    this.#base = opts.base ?? 2;
+    this.#factor = opts.factor ?? 1;
+
+    this.#min = opts.min ?? -Infinity;
+    this.#max = opts.max ?? Infinity;
+
+    this.#maxRetries = opts.maxRetries ?? Infinity;
+    this.#maxElapsed = opts.maxElapsed ?? Infinity;
+  }
+
+  #clone() {
+    return new ExponentialBackoff(this.#type, {
+      base: this.#base,
+      factor: this.#factor,
+      min: this.#min,
+      max: this.#max,
+      maxRetries: this.#maxRetries,
+      maxElapsed: this.#maxElapsed,
+    });
+  }
+
+  type(type?: ExponentialBackoffType) {
+    if (typeof type !== "undefined") {
+      this.#type = type;
+    }
+    return this.#clone();
+  }
+
+  base(base?: number) {
+    if (typeof base !== "undefined") {
+      this.#base = base;
+    }
+    return this.#clone();
+  }
+
+  factor(factor?: number) {
+    if (typeof factor !== "undefined") {
+      this.#factor = factor;
+    }
+    return this.#clone();
+  }
+
+  min(min?: number) {
+    if (typeof min !== "undefined") {
+      this.#min = min;
+    }
+    return this.#clone();
+  }
+
+  max(max?: number) {
+    if (typeof max !== "undefined") {
+      this.#max = max;
+    }
+    return this.#clone();
+  }
+
+  maxRetries(maxRetries?: number) {
+    if (typeof maxRetries !== "undefined") {
+      this.#maxRetries = maxRetries;
+    }
+    return this.#clone();
+  }
+
+  maxElapsed(maxElapsed?: number) {
+    if (typeof maxElapsed !== "undefined") {
+      this.#maxElapsed = maxElapsed;
+    }
+    return this.#clone();
+  }
+
+  retries(retries?: number) {
+    if (typeof retries !== "undefined") {
+      if (retries > this.#maxRetries) {
+        console.error(
+          `Can't set retries ${retries} higher than maxRetries (${
+            this.#maxRetries
+          }), setting to maxRetries instead.`
+        );
+        this.#retries = this.#maxRetries;
+      } else {
+        this.#retries = retries;
+      }
+    }
+    return this.#clone();
+  }
+
+  async *retryAsync(maxRetries: number = this.#maxRetries ?? Infinity) {
+    let elapsed = 0;
+    let retry = 0;
+
+    while (retry <= maxRetries) {
+      const delay = this.delay(retry);
+      elapsed += delay;
+
+      if (elapsed > this.#maxElapsed) {
+        break;
+      }
+
+      yield {
+        delay: {
+          seconds: delay,
+          milliseconds: delay * 1000,
+        },
+        retry,
+      };
+
+      retry++;
+    }
+  }
+
+  async *[Symbol.asyncIterator]() {
+    yield* this.retryAsync();
+  }
+
+  delay(retries: number = this.#retries, jitter: boolean = true) {
+    if (retries > this.#maxRetries) {
+      console.error(
+        `Can't set retries ${retries} higher than maxRetries (${
+          this.#maxRetries
+        }), setting to maxRetries instead.`
+      );
+      retries = this.#maxRetries;
+    }
+
+    let delay = this.#factor * this.#base ** retries;
+
+    switch (this.#type) {
+      case "NoJitter": {
+        break;
+      }
+      case "FullJitter": {
+        if (!jitter) {
+          delay = 0;
+          break;
+        }
+
+        delay *= Math.random();
+        break;
+      }
+      case "EqualJitter": {
+        if (!jitter) {
+          delay *= 0.5;
+          break;
+        }
+
+        delay *= 0.5 * (1 + Math.random());
+        break;
+      }
+      default: {
+        throw new Error(`Unknown backoff type: ${this.#type}`);
+      }
+    }
+
+    delay = Math.min(delay, this.#max);
+    delay = Math.max(delay, this.#min);
+    delay = Math.round(delay);
+
+    return delay;
+  }
+
+  elapsed(retries: number = this.#retries, jitter: boolean = true) {
+    let elapsed = 0;
+
+    for (let i = 0; i <= retries; i++) {
+      elapsed += this.delay(i, jitter);
+    }
+
+    const total = elapsed;
+
+    let days = 0;
+    if (elapsed > 3600 * 24) {
+      days = Math.floor(elapsed / 3600 / 24);
+      elapsed -= days * 3600 * 24;
+    }
+
+    let hours = 0;
+    if (elapsed > 3600) {
+      hours = Math.floor(elapsed / 3600);
+      elapsed -= hours * 3600;
+    }
+
+    let minutes = 0;
+    if (elapsed > 60) {
+      minutes = Math.floor(elapsed / 60);
+      elapsed -= minutes * 60;
+    }
+
+    const seconds = elapsed;
+
+    return {
+      seconds,
+      minutes,
+      hours,
+      days,
+      total,
+    };
+  }
+
+  reset() {
+    this.#retries = 0;
+    return this;
+  }
+
+  next() {
+    this.#retries++;
+    return this.delay();
+  }
+
+  stop() {
+    throw new StopRetrying();
+  }
+
+  static StopRetrying = StopRetrying;
+}

apps/coordinator/src/index.ts

@@ -1,5 +1,6 @@
 import { createServer } from "node:http";
-import { $ } from "execa";
+import fs from "node:fs/promises";
+import { $, type ExecaChildProcess } from "execa";
 import { nanoid } from "nanoid";
 import { Server } from "socket.io";
 import {
@@ -12,6 +13,7 @@ import {
 import { ZodNamespace } from "@trigger.dev/core/v3/zodNamespace";
 import { ZodSocketConnection } from "@trigger.dev/core/v3/zodSocket";
 import { HttpReply, getTextBody, SimpleLogger } from "@trigger.dev/core-apps";
+import { ExponentialBackoff } from "./backoff";
 
 import { collectDefaultMetrics, register, Gauge } from "prom-client";
 collectDefaultMetrics();
@@ -19,6 +21,26 @@ collectDefaultMetrics();
 const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || 8020);
 const NODE_NAME = process.env.NODE_NAME || "coordinator";
 const DEFAULT_RETRY_DELAY_THRESHOLD_IN_MS = 30_000;
+const CHAOS_MONKEY_ENABLED = !!process.env.CHAOS_MONKEY_ENABLED;
+
+const FORCE_CHECKPOINT_SIMULATION = ["1", "true"].includes(
+  process.env.FORCE_CHECKPOINT_SIMULATION ?? "true"
+);
+const DISABLE_CHECKPOINT_SUPPORT = ["1", "true"].includes(
+  process.env.DISABLE_CHECKPOINT_SUPPORT ?? "false"
+);
+const SIMULATE_PUSH_FAILURE = ["1", "true"].includes(process.env.SIMULATE_PUSH_FAILURE ?? "false");
+const SIMULATE_PUSH_FAILURE_SECONDS = parseInt(
+  process.env.SIMULATE_PUSH_FAILURE_SECONDS ?? "300",
+  10
+);
+const SIMULATE_CHECKPOINT_FAILURE = ["1", "true"].includes(
+  process.env.SIMULATE_CHECKPOINT_FAILURE ?? "false"
+);
+const SIMULATE_CHECKPOINT_FAILURE_SECONDS = parseInt(
+  process.env.SIMULATE_CHECKPOINT_FAILURE_SECONDS ?? "300",
+  10
+);
 
 const REGISTRY_HOST = process.env.REGISTRY_HOST || "localhost:5000";
 const CHECKPOINT_PATH = process.env.CHECKPOINT_PATH || "/checkpoints";
@@ -32,6 +54,10 @@ const SECURE_CONNECTION = ["1", "true"].includes(process.env.SECURE_CONNECTION ?
 
 const logger = new SimpleLogger(`[${NODE_NAME}]`);
 
+if (CHAOS_MONKEY_ENABLED) {
+  logger.log("🍌 Chaos monkey enabled");
+}
+
 type CheckpointerInitializeReturn = {
   canCheckpoint: boolean;
   willSimulate: boolean;
@@ -44,11 +70,49 @@ type CheckpointAndPushOptions = {
   deploymentVersion: string;
 };
 
+type CheckpointAndPushResult =
+  | { success: true; checkpoint: CheckpointData }
+  | { success: false; reason?: "CANCELED" | "DISABLED" | "ERROR" | "IN_PROGRESS" | "NO_SUPPORT" };
+
 type CheckpointData = {
   location: string;
   docker: boolean;
 };
 
+function isExecaChildProcess(maybeExeca: unknown): maybeExeca is Awaited<ExecaChildProcess> {
+  return typeof maybeExeca === "object" && maybeExeca !== null && "escapedCommand" in maybeExeca;
+}
+
+async function getFileSize(filePath: string): Promise<number> {
+  try {
+    const stats = await fs.stat(filePath);
+    return stats.size;
+  } catch (error) {
+    console.error("Error getting file size:", error);
+    return -1;
+  }
+}
+
+async function getParsedFileSize(filePath: string) {
+  const sizeInBytes = await getFileSize(filePath);
+
+  let message = `Size in bytes: ${sizeInBytes}`;
+
+  if (sizeInBytes > 1024 * 1024) {
+    const sizeInMB = (sizeInBytes / 1024 / 1024).toFixed(2);
+    message = `Size in MB (rounded): ${sizeInMB}`;
+  } else if (sizeInBytes > 1024) {
+    const sizeInKB = (sizeInBytes / 1024).toFixed(2);
+    message = `Size in KB (rounded): ${sizeInKB}`;
+  }
+
+  return {
+    path: filePath,
+    sizeInBytes,
+    message,
+  };
+}
+
 class Checkpointer {
   #initialized = false;
   #canCheckpoint = false;
@@ -56,6 +120,8 @@ class Checkpointer {
 
   #logger = new SimpleLogger("[checkptr]");
   #abortControllers = new Map<string, AbortController>();
+  #failedCheckpoints = new Map<string, unknown>();
+  #waitingForRetry = new Set<string>();
 
   constructor(private opts = { forceSimulate: false }) {}
 
@@ -139,7 +205,7 @@ class Checkpointer {
     const start = performance.now();
     logger.log(`checkpointAndPush() start`, { start, opts });
 
-    const result = await this.#checkpointAndPush(opts);
+    const result = await this.#checkpointAndPushWithBackoff(opts);
 
     const end = performance.now();
     logger.log(`checkpointAndPush() end`, {
@@ -147,17 +213,33 @@ class Checkpointer {
       end,
       diff: end - start,
       opts,
-      success: !!result,
+      success: result.success,
     });
 
-    return result;
+    if (!result.success) {
+      return;
+    }
+
+    return result.checkpoint;
   }
 
   isCheckpointing(runId: string) {
-    return this.#abortControllers.has(runId);
+    return this.#abortControllers.has(runId) || this.#waitingForRetry.has(runId);
   }
 
   cancelCheckpoint(runId: string): boolean {
+    // If the last checkpoint failed, pretend we canceled it
+    // This ensures tasks don't wait for external resume messages to continue
+    if (this.#hasFailedCheckpoint(runId)) {
+      this.#clearFailedCheckpoint(runId);
+      return true;
+    }
+
+    if (this.#waitingForRetry.has(runId)) {
+      this.#waitingForRetry.delete(runId);
+      return true;
+    }
+
     const controller = this.#abortControllers.get(runId);
 
     if (!controller) {
@@ -171,29 +253,133 @@ class Checkpointer {
     return true;
   }
 
+  async #checkpointAndPushWithBackoff({
+    runId,
+    leaveRunning = true, // This mirrors kubernetes behaviour more accurately
+    projectRef,
+    deploymentVersion,
+  }: CheckpointAndPushOptions): Promise<CheckpointAndPushResult> {
+    this.#logger.log("Checkpointing with backoff", {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+    });
+
+    const backoff = new ExponentialBackoff()
+      .type("EqualJitter")
+      .base(3)
+      .max(3 * 3600)
+      .maxElapsed(48 * 3600);
+
+    for await (const { delay, retry } of backoff) {
+      try {
+        if (retry > 0) {
+          this.#logger.error("Retrying checkpoint", {
+            runId,
+            retry,
+            delay,
+          });
+
+          this.#waitingForRetry.add(runId);
+          await new Promise((resolve) => setTimeout(resolve, delay.milliseconds));
+
+          if (!this.#waitingForRetry.has(runId)) {
+            this.#logger.log("Checkpoint canceled while waiting for retry", { runId });
+            return { success: false, reason: "CANCELED" };
+          } else {
+            this.#waitingForRetry.delete(runId);
+          }
+        }
+
+        const result = await this.#checkpointAndPush({
+          runId,
+          leaveRunning,
+          projectRef,
+          deploymentVersion,
+        });
+
+        if (result.success) {
+          return result;
+        }
+
+        if (result.reason === "CANCELED") {
+          this.#logger.log("Checkpoint canceled, won't retry", { runId });
+          // Don't fail the checkpoint, as it was canceled
+          return result;
+        }
+
+        if (result.reason === "IN_PROGRESS") {
+          this.#logger.log("Checkpoint already in progress, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        if (result.reason === "NO_SUPPORT") {
+          this.#logger.log("No checkpoint support, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        if (result.reason === "DISABLED") {
+          this.#logger.log("Checkpoint support disabled, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        continue;
+      } catch (error) {
+        this.#logger.error("Checkpoint error", {
+          retry,
+          runId,
+          delay,
+          error: error instanceof Error ? error.message : error,
+        });
+      }
+    }
+
+    this.#logger.error(`Checkpoint failed after exponential backoff`, {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+    });
+    this.#failCheckpoint(runId, "ERROR");
+
+    return { success: false, reason: "ERROR" };
+  }
+
   async #checkpointAndPush({
     runId,
     leaveRunning = true, // This mirrors kubernetes behaviour more accurately
     projectRef,
     deploymentVersion,
-  }: CheckpointAndPushOptions): Promise<CheckpointData | undefined> {
+  }: CheckpointAndPushOptions): Promise<CheckpointAndPushResult> {
     await this.initialize();
 
+    const options = {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+    };
+
     if (!this.#dockerMode && !this.#canCheckpoint) {
       this.#logger.error("No checkpoint support. Simulation requires docker.");
-      return;
+      return { success: false, reason: "NO_SUPPORT" };
     }
 
     if (this.#abortControllers.has(runId)) {
-      logger.error("Checkpoint procedure already in progress", {
-        options: {
-          runId,
-          leaveRunning,
-          projectRef,
-          deploymentVersion,
-        },
-      });
-      return;
+      logger.error("Checkpoint procedure already in progress", { options });
+      return { success: false, reason: "IN_PROGRESS" };
+    }
+
+    // This is a new checkpoint, clear any last failure for this run
+    this.#clearFailedCheckpoint(runId);
+
+    if (DISABLE_CHECKPOINT_SUPPORT) {
+      this.#logger.error("Checkpoint support disabled", { options });
+      return { success: false, reason: "DISABLED" };
     }
 
     const controller = new AbortController();
@@ -201,19 +387,44 @@ class Checkpointer {
 
     const $$ = $({ signal: controller.signal });
 
-    try {
-      const shortCode = nanoid(8);
-      const imageRef = this.#getImageRef(projectRef, deploymentVersion, shortCode);
-      const exportLocation = this.#getExportLocation(projectRef, deploymentVersion, shortCode);
+    const shortCode = nanoid(8);
+    const imageRef = this.#getImageRef(projectRef, deploymentVersion, shortCode);
+    const exportLocation = this.#getExportLocation(projectRef, deploymentVersion, shortCode);
 
-      this.#logger.log("Checkpointing:", {
-        options: {
-          runId,
-          leaveRunning,
-          projectRef,
-          deploymentVersion,
-        },
-      });
+    const cleanup = async () => {
+      if (this.#dockerMode) {
+        return;
+      }
+
+      try {
+        await $`rm ${exportLocation}`;
+        this.#logger.log("Deleted checkpoint archive", { exportLocation });
+
+        await $`buildah rmi ${imageRef}`;
+        this.#logger.log("Deleted checkpoint image", { imageRef });
+      } catch (error) {
+        this.#logger.error("Failure during checkpoint cleanup", { exportLocation, error });
+      }
+    };
+
+    try {
+      if (CHAOS_MONKEY_ENABLED) {
+        console.log("🍌 Chaos monkey wreaking havoc");
+
+        const random = Math.random();
+
+        if (random < 0.33) {
+          // Fake long checkpoint duration
+          await $$`sleep 300`;
+        } else if (random < 0.66) {
+          // Fake checkpoint error
+          await $$`false`;
+        } else {
+          // no-op
+        }
+      }
+
+      this.#logger.log("Checkpointing:", { options });
 
       const containterName = this.#getRunContainerName(runId);
 
@@ -224,6 +435,13 @@ class Checkpointer {
             this.#logger.log("Simulating checkpoint");
             this.#logger.debug(await $$`docker pause ${containterName}`);
           } else {
+            if (SIMULATE_CHECKPOINT_FAILURE) {
+              if (performance.now() < SIMULATE_CHECKPOINT_FAILURE_SECONDS * 1000) {
+                this.#logger.error("Simulating checkpoint failure", { options });
+                throw new Error("SIMULATE_CHECKPOINT_FAILURE");
+              }
+            }
+
             if (leaveRunning) {
               this.#logger.debug(
                 await $$`docker checkpoint create --leave-running ${containterName} ${exportLocation}`
@@ -234,9 +452,9 @@ class Checkpointer {
               );
             }
           }
-        } catch (error: any) {
-          this.#logger.error(error.stderr);
-          return;
+        } catch (error) {
+          this.#logger.error("Failed while creating docker checkpoint", { exportLocation });
+          throw error;
         }
 
         this.#logger.log("checkpoint created:", {
@@ -245,8 +463,11 @@ class Checkpointer {
         });
 
         return {
-          location: exportLocation,
-          docker: true,
+          success: true,
+          checkpoint: {
+            location: exportLocation,
+            docker: true,
+          },
         };
       }
 
@@ -266,54 +487,105 @@ class Checkpointer {
         throw new Error("could not find container id");
       }
 
+      const start = performance.now();
+
+      if (SIMULATE_CHECKPOINT_FAILURE) {
+        if (performance.now() < SIMULATE_CHECKPOINT_FAILURE_SECONDS * 1000) {
+          this.#logger.error("Simulating checkpoint failure", { options });
+          throw new Error("SIMULATE_CHECKPOINT_FAILURE");
+        }
+      }
+
+      // Create checkpoint
       this.#logger.debug(await $$`crictl checkpoint --export=${exportLocation} ${containerId}`);
+      const postCheckpoint = performance.now();
+
+      // Print checkpoint size
+      const size = await getParsedFileSize(exportLocation);
+      this.#logger.log("checkpoint archive created", { size, options });
 
       // Create image from checkpoint
       const container = this.#logger.debug(await $$`buildah from scratch`);
+      const postFrom = performance.now();
+
       this.#logger.debug(await $$`buildah add ${container} ${exportLocation} /`);
+      const postAdd = performance.now();
+
       this.#logger.debug(
         await $$`buildah config --annotation=io.kubernetes.cri-o.annotations.checkpoint.name=counter ${container}`
       );
+      const postConfig = performance.now();
+
       this.#logger.debug(await $$`buildah commit ${container} ${imageRef}`);
+      const postCommit = performance.now();
+
       this.#logger.debug(await $$`buildah rm ${container}`);
+      const postRm = performance.now();
+
+      if (SIMULATE_PUSH_FAILURE) {
+        if (performance.now() < SIMULATE_PUSH_FAILURE_SECONDS * 1000) {
+          this.#logger.error("Simulating push failure", { options });
+          throw new Error("SIMULATE_PUSH_FAILURE");
+        }
+      }
 
       // Push checkpoint image
       this.#logger.debug(await $$`buildah push --tls-verify=${REGISTRY_TLS_VERIFY} ${imageRef}`);
+      const postPush = performance.now();
 
-      this.#logger.log("Checkpointed and pushed image to:", { location: imageRef });
+      const perf = {
+        "crictl checkpoint": postCheckpoint - start,
+        "buildah from": postFrom - postCheckpoint,
+        "buildah add": postAdd - postFrom,
+        "buildah config": postConfig - postAdd,
+        "buildah commit": postCommit - postConfig,
+        "buildah rm": postRm - postCommit,
+        "buildah push": postPush - postRm,
+      };
 
-      try {
-        await $$`rm ${exportLocation}`;
-        this.#logger.log("Deleted checkpoint archive", { exportLocation });
-
-        // Disabled for now as this will increase restore time by having to pull the image again
-        // await $`buildah rmi ${imageRef}`;
-        // this.#logger.log("Deleted checkpoint image", { imageRef });
-      } catch (error) {
-        this.#logger.error("Failed during checkpoint cleanup", { exportLocation });
-        this.#logger.debug(error);
-      }
+      this.#logger.log("Checkpointed and pushed image to:", { location: imageRef, perf });
 
       return {
-        location: imageRef,
-        docker: false,
+        success: true,
+        checkpoint: {
+          location: imageRef,
+          docker: false,
+        },
       };
     } catch (error) {
-      this.#logger.error("checkpoint failed", {
-        options: {
-          runId,
-          leaveRunning,
-          projectRef,
-          deploymentVersion,
-        },
-        error,
-      });
-      return;
+      if (isExecaChildProcess(error)) {
+        if (error.isCanceled) {
+          this.#logger.error("Checkpoint canceled", { options, error });
+
+          return { success: false, reason: "CANCELED" };
+        }
+
+        this.#logger.error("Checkpoint command error", { options, error });
+
+        return { success: false, reason: "ERROR" };
+      }
+
+      this.#logger.error("Unhandled checkpoint error", { options, error });
+
+      return { success: false, reason: "ERROR" };
     } finally {
       this.#abortControllers.delete(runId);
+      await cleanup();
     }
   }
 
+  #failCheckpoint(runId: string, error: unknown) {
+    this.#failedCheckpoints.set(runId, error);
+  }
+
+  #clearFailedCheckpoint(runId: string) {
+    this.#failedCheckpoints.delete(runId);
+  }
+
+  #hasFailedCheckpoint(runId: string) {
+    return this.#failedCheckpoints.has(runId);
+  }
+
   #getRunContainerName(suffix: string) {
     return `task-run-${suffix}`;
   }
@@ -321,7 +593,7 @@ class Checkpointer {
 
 class TaskCoordinator {
   #httpServer: ReturnType<typeof createServer>;
-  #checkpointer = new Checkpointer({ forceSimulate: true });
+  #checkpointer = new Checkpointer({ forceSimulate: FORCE_CHECKPOINT_SIMULATION });
 
   #prodWorkerNamespace: ZodNamespace<
     typeof ProdWorkerToCoordinatorMessages,
@@ -442,6 +714,30 @@ class TaskCoordinator {
 
           taskSocket.emit("REQUEST_ATTEMPT_CANCELLATION", message);
         },
+        REQUEST_RUN_CANCELLATION: async (message) => {
+          const taskSocket = await this.#getRunSocket(message.runId);
+
+          if (!taskSocket) {
+            logger.log("Socket for run not found", {
+              runId: message.runId,
+            });
+            return;
+          }
+
+          this.#checkpointer.cancelCheckpoint(message.runId);
+
+          if (message.delayInMs) {
+            taskSocket.emit("REQUEST_EXIT", {
+              version: "v2",
+              delayInMs: message.delayInMs,
+            });
+          } else {
+            // If there's no delay, assume the worker doesn't support non-v1 messages
+            taskSocket.emit("REQUEST_EXIT", {
+              version: "v1",
+            });
+          }
+        },
         READY_FOR_RETRY: async (message) => {
           const taskSocket = await this.#getRunSocket(message.runId);
 
@@ -528,6 +824,20 @@ class TaskCoordinator {
       onConnection: async (socket, handler, sender) => {
         const logger = new SimpleLogger(`[prod-worker][${socket.id}]`);
 
+        const crashRun = async (error: { name: string; message: string; stack?: string }) => {
+          try {
+            this.#platformSocket?.send("RUN_CRASHED", {
+              version: "v1",
+              runId: socket.data.runId,
+              error,
+            });
+          } finally {
+            socket.emit("REQUEST_EXIT", {
+              version: "v1",
+            });
+          }
+        };
+
         const checkpointInProgress = () => {
           return this.#checkpointableTasks.has(socket.data.runId);
         };
@@ -596,8 +906,9 @@ class TaskCoordinator {
             if (!executionAck) {
               logger.error("no execution ack", { runId: socket.data.runId });
 
-              socket.emit("REQUEST_EXIT", {
-                version: "v1",
+              await crashRun({
+                name: "ReadyForExecutionError",
+                message: "No execution ack",
               });
 
               return;
@@ -606,8 +917,9 @@ class TaskCoordinator {
             if (!executionAck.success) {
               logger.error("failed to get execution payload", { runId: socket.data.runId });
 
-              socket.emit("REQUEST_EXIT", {
-                version: "v1",
+              await crashRun({
+                name: "ReadyForExecutionError",
+                message: "Failed to get execution payload",
               });
 
               return;
@@ -621,6 +933,62 @@ class TaskCoordinator {
             socket.data.attemptFriendlyId = executionAck.payload.execution.attempt.id;
           } catch (error) {
             logger.error("Error", { error });
+
+            await crashRun({
+              name: "ReadyForExecutionError",
+              message:
+                error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error",
+            });
+
+            return;
+          }
+        });
+
+        socket.on("READY_FOR_LAZY_ATTEMPT", async (message) => {
+          logger.log("[READY_FOR_LAZY_ATTEMPT]", message);
+
+          try {
+            const lazyAttempt = await this.#platformSocket?.sendWithAck("READY_FOR_LAZY_ATTEMPT", {
+              ...message,
+              envId: socket.data.envId,
+            });
+
+            if (!lazyAttempt) {
+              logger.error("no lazy attempt ack", { runId: socket.data.runId });
+
+              await crashRun({
+                name: "ReadyForLazyAttemptError",
+                message: "No lazy attempt ack",
+              });
+
+              return;
+            }
+
+            if (!lazyAttempt.success) {
+              logger.error("failed to get lazy attempt payload", { runId: socket.data.runId });
+
+              await crashRun({
+                name: "ReadyForLazyAttemptError",
+                message: "Failed to get lazy attempt payload",
+              });
+
+              return;
+            }
+
+            socket.emit("EXECUTE_TASK_RUN_LAZY_ATTEMPT", {
+              version: "v1",
+              lazyPayload: lazyAttempt.lazyPayload,
+            });
+          } catch (error) {
+            logger.error("Error", { error });
+
+            await crashRun({
+              name: "ReadyForLazyAttemptError",
+              message:
+                error instanceof Error ? `Unexpected error: ${error.message}` : "Unexpected error",
+            });
+
+            return;
           }
         });
 
@@ -714,6 +1082,19 @@ class TaskCoordinator {
           }
         });
 
+        socket.on("TASK_RUN_FAILED_TO_RUN", async ({ completion }) => {
+          logger.log("completed task", { completionId: completion.id });
+
+          this.#platformSocket?.send("TASK_RUN_FAILED_TO_RUN", {
+            version: "v1",
+            completion,
+          });
+
+          socket.emit("REQUEST_EXIT", {
+            version: "v1",
+          });
+        });
+
         socket.on("READY_FOR_CHECKPOINT", async (message) => {
           logger.log("[READY_FOR_CHECKPOINT]", message);
 
@@ -890,7 +1271,7 @@ class TaskCoordinator {
           logger.log("[INDEX_TASKS]", message);
 
           const workerAck = await this.#platformSocket?.sendWithAck("CREATE_WORKER", {
-            version: "v1",
+            version: "v2",
             projectRef: socket.data.projectRef,
             envId: socket.data.envId,
             deploymentId: message.deploymentId,
@@ -899,6 +1280,7 @@ class TaskCoordinator {
               packageVersion: message.packageVersion,
               tasks: message.tasks,
             },
+            supportsLazyAttempts: message.version !== "v1" && message.supportsLazyAttempts,
           });
 
           if (!workerAck) {
@@ -917,6 +1299,34 @@ class TaskCoordinator {
             error: message.error,
           });
         });
+
+        socket.on("CREATE_TASK_RUN_ATTEMPT", async (message, callback) => {
+          logger.log("[CREATE_TASK_RUN_ATTEMPT]", message);
+
+          const createAttempt = await this.#platformSocket?.sendWithAck("CREATE_TASK_RUN_ATTEMPT", {
+            runId: message.runId,
+            envId: socket.data.envId,
+          });
+
+          if (!createAttempt?.success) {
+            logger.debug("no ack while creating attempt", message);
+            callback({ success: false });
+            return;
+          }
+
+          socket.data.attemptFriendlyId = createAttempt.executionPayload.execution.attempt.id;
+
+          callback({
+            success: true,
+            executionPayload: createAttempt.executionPayload,
+          });
+        });
+
+        socket.on("UNRECOVERABLE_ERROR", async (message) => {
+          logger.log("[UNRECOVERABLE_ERROR]", message);
+
+          await crashRun(message.error);
+        });
       },
       onDisconnect: async (socket, handler, sender, logger) => {
         this.#platformSocket?.send("LOG", {
@@ -928,13 +1338,16 @@ class TaskCoordinator {
         TASK_HEARTBEAT: async (message) => {
           this.#platformSocket?.send("TASK_HEARTBEAT", message);
         },
+        TASK_RUN_HEARTBEAT: async (message) => {
+          this.#platformSocket?.send("TASK_RUN_HEARTBEAT", message);
+        },
       },
     });
 
     return provider;
   }
 
-  #cancelCheckpoint(runId: string) {
+  #cancelCheckpoint(runId: string): boolean {
     const checkpointWait = this.#checkpointableTasks.get(runId);
 
     if (checkpointWait) {
@@ -945,6 +1358,8 @@ class TaskCoordinator {
     // Cancel checkpointing procedure
     const checkpointCanceled = this.#checkpointer.cancelCheckpoint(runId);
 
+    logger.log("cancelCheckpoint()", { runId, checkpointCanceled });
+
     return checkpointCanceled;
   }
 

apps/docker-provider/.env.example

@@ -4,6 +4,8 @@ PLATFORM_WS_PORT=3030
 PLATFORM_SECRET=provider-secret
 SECURE_CONNECTION=false
 
+OTEL_EXPORTER_OTLP_ENDPOINT=http://0.0.0.0:3030/otel
+
 # Use this if you are on macOS
 # COORDINATOR_HOST="host.docker.internal"
 # OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318"

apps/docker-provider/src/index.ts

@@ -13,9 +13,14 @@ import { PostStartCauses, PreStopCauses } from "@trigger.dev/core/v3";
 const MACHINE_NAME = process.env.MACHINE_NAME || "local";
 const COORDINATOR_PORT = process.env.COORDINATOR_PORT || 8020;
 const COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
+
 const OTEL_EXPORTER_OTLP_ENDPOINT =
   process.env.OTEL_EXPORTER_OTLP_ENDPOINT || "http://0.0.0.0:4318";
 
+const FORCE_CHECKPOINT_SIMULATION = ["1", "true"].includes(
+  process.env.FORCE_CHECKPOINT_SIMULATION ?? "true"
+);
+
 const logger = new SimpleLogger(`[${MACHINE_NAME}]`);
 
 type InitializeReturn = {
@@ -278,7 +283,7 @@ class DockerTaskOperations implements TaskOperations {
 }
 
 const provider = new ProviderShell({
-  tasks: new DockerTaskOperations({ forceSimulate: true }),
+  tasks: new DockerTaskOperations({ forceSimulate: FORCE_CHECKPOINT_SIMULATION }),
   type: "docker",
 });
 

apps/kubernetes-provider/src/index.ts

@@ -212,7 +212,7 @@ class KubernetesTaskOperations implements TaskOperations {
             },
             {
               name: "populate-taskinfo",
-              image: "docker.io/library/busybox",
+              image: "registry.digitalocean.com/trigger/busybox",
               imagePullPolicy: "IfNotPresent",
               command: ["/bin/sh", "-c"],
               args: ["printenv COORDINATOR_HOST | tee /etc/taskinfo/coordinator-host"],
@@ -316,6 +316,9 @@ class KubernetesTaskOperations implements TaskOperations {
         {
           name: "registry-trigger",
         },
+        {
+          name: "registry-trigger-failover",
+        },
       ],
       nodeSelector: {
         nodetype: "worker",

apps/proxy/package.json

@@ -7,9 +7,9 @@
     "dev": "wrangler dev"
   },
   "devDependencies": {
-    "@cloudflare/workers-types": "^4.20230419.0",
+    "@cloudflare/workers-types": "^4.20240512.0",
     "typescript": "^5.0.4",
-    "wrangler": "^3.0.0"
+    "wrangler": "^3.57.1"
   },
   "dependencies": {
     "@aws-sdk/client-sqs": "^3.445.0",

apps/proxy/src/index.ts

@@ -1,5 +1,7 @@
 import { queueEvent } from "./events/queueEvent";
 import { queueEvents } from "./events/queueEvents";
+import { applyRateLimit } from "./rateLimit";
+import { Ratelimit } from "./rateLimiter";
 
 export interface Env {
   /** The hostname needs to be changed to allow requests to pass to the Trigger.dev platform */
@@ -9,6 +11,8 @@ export interface Env {
   AWS_SQS_SECRET_ACCESS_KEY: string;
   AWS_SQS_QUEUE_URL: string;
   AWS_SQS_REGION: string;
+  //rate limiter
+  API_RATE_LIMITER: Ratelimit;
 }
 
 export default {
@@ -25,13 +29,13 @@ export default {
     switch (url.pathname) {
       case "/api/v1/events": {
         if (request.method === "POST") {
-          return queueEvent(request, env);
+          return applyRateLimit(request, env, () => queueEvent(request, env));
         }
         break;
       }
       case "/api/v1/events/bulk": {
         if (request.method === "POST") {
-          return queueEvents(request, env);
+          return applyRateLimit(request, env, () => queueEvents(request, env));
         }
         break;
       }

apps/proxy/src/rateLimit.ts

@@ -0,0 +1,46 @@
+import { Env } from "src";
+import { getApiKeyFromRequest } from "./apikey";
+import { json } from "./json";
+
+export async function applyRateLimit(
+  request: Request,
+  env: Env,
+  fn: () => Promise<Response>
+): Promise<Response> {
+  const apiKey = getApiKeyFromRequest(request);
+  if (apiKey) {
+    const result = await env.API_RATE_LIMITER.limit({ key: `apikey-${apiKey.apiKey}` });
+    const { success } = result;
+    console.log(`Rate limiter`, {
+      success,
+      key: `${apiKey.apiKey.substring(0, 12)}...`,
+    });
+    if (!success) {
+      //60s in the future
+      const reset = Date.now() + 60 * 1000;
+      const secondsUntilReset = Math.max(0, (reset - new Date().getTime()) / 1000);
+
+      return json(
+        {
+          title: "Rate Limit Exceeded",
+          status: 429,
+          type: "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429",
+          detail: `Rate limit exceeded. Retry in ${secondsUntilReset} seconds.`,
+          error: `Rate limit exceeded. Retry in ${secondsUntilReset} seconds.`,
+          reset,
+        },
+        {
+          status: 429,
+          headers: {
+            "x-ratelimit-reset": reset.toString(),
+          },
+        }
+      );
+    }
+  } else {
+    console.log(`Rate limiter: no API key for request`);
+  }
+
+  //call the original function
+  return fn();
+}

apps/proxy/src/rateLimiter.ts

@@ -0,0 +1,23 @@
+export interface Ratelimit {
+  /*
+   * The ratelimit function
+   * @param {RatelimitOptions} options
+   * @returns {Promise<RatelimitResponse>}
+   */
+  limit: (options: RatelimitOptions) => Promise<RatelimitResponse>;
+}
+
+export interface RatelimitOptions {
+  /*
+   * The key to identify the user, can be an IP address, user ID, etc.
+   */
+  key: string;
+}
+
+export interface RatelimitResponse {
+  /*
+   * The ratelimit success status
+   * @returns {boolean}
+   */
+  success: boolean;
+}

apps/proxy/wrangler.toml

@@ -1,7 +1,33 @@
 name = "proxy"
 main = "src/index.ts"
-compatibility_date = "2023-10-30"
+compatibility_date = "2024-05-13"
 compatibility_flags = [ "nodejs_compat" ]
 
 [env.staging]
-[env.prod]
+  # The rate limiting API is in open beta.
+  [[env.staging.unsafe.bindings]]
+  name = "API_RATE_LIMITER"
+  type = "ratelimit"
+  # An identifier you define, that is unique to your Cloudflare account.
+  # Must be an integer.
+  namespace_id = "1"
+
+  # Limit: the number of tokens allowed within a given period in a single
+  # Cloudflare location
+  # Period: the duration of the period, in seconds. Must be either 10 or 60
+  simple = { limit = 100, period = 60 }
+
+
+[env.prod]
+  # The rate limiting API is in open beta.
+  [[env.prod.unsafe.bindings]]
+  name = "API_RATE_LIMITER"
+  type = "ratelimit"
+  # An identifier you define, that is unique to your Cloudflare account.
+  # Must be an integer.
+  namespace_id = "2"
+
+  # Limit: the number of tokens allowed within a given period in a single
+  # Cloudflare location
+  # Period: the duration of the period, in seconds. Must be either 10 or 60
+  simple = { limit = 300, period = 60 }

apps/webapp/app/assets/icons/v3.svg

@@ -0,0 +1,4 @@
+<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect x="0.861435" y="0.861435" width="30.2771" height="30.2771" rx="15.1386" stroke="#D7D9DD" stroke-width="1.72287"/>
+<path d="M14.9669 12.642L11.6417 21.7732H9.8155L6.49036 12.642H8.04094L10.7286 20.2571L13.4163 12.642H14.9669ZM20.534 14.5544C21.6884 14.5716 23.0666 14.0375 23.0666 12.4525C23.0666 11.3843 22.1018 10.7124 20.534 10.7124C19.2591 10.7124 18.3632 11.3326 18.2082 12.3146L16.6576 12.2113C16.8815 10.4884 18.4838 9.26516 20.534 9.26516C22.946 9.26516 24.6345 10.5745 24.6345 12.4525C24.6345 13.8135 23.8592 14.7267 22.3775 15.1574C24.0831 15.657 25.0479 16.8286 25.0479 18.4136C25.0479 20.55 23.1872 22.0489 20.534 22.0489C18.0876 22.0489 16.313 20.6361 16.2096 18.6203L17.743 18.517C17.8808 20.0159 19.3108 20.6016 20.534 20.6016C21.9812 20.6016 23.4974 19.947 23.4974 18.293C23.4974 16.6563 21.9812 15.9499 20.534 15.9844L19.5865 16.0016V14.5371L20.534 14.5544Z" fill="white"/>
+</svg>

apps/webapp/app/components/Feedback.tsx

@@ -115,14 +115,7 @@ export function Feedback({ button, defaultValue = "bug" }: FeedbackProps) {
               Docs
             </LinkButton>
             <LinkButton
-              to={docsPath("v3/introduction")}
-              variant="tertiary/medium"
-              LeadingIcon={BookOpenIcon}
-            >
-              v3 Docs (Developer preview)
-            </LinkButton>
-            <LinkButton
-              to={"https://trigger.openstatus.dev/"}
+              to={"https://status.trigger.dev/"}
               variant="tertiary/medium"
               LeadingIcon={ActivityIcon}
             >

apps/webapp/app/components/admin/debugTooltip.tsx

@@ -20,9 +20,11 @@ export function AdminDebugTooltip({ children }: { children: React.ReactNode }) {
     <TooltipProvider>
       <Tooltip>
         <TooltipTrigger>
-          <ShieldCheckIcon className="h-5 w-5" />
+          <ShieldCheckIcon className="size-5" />
         </TooltipTrigger>
-        <TooltipContent className="flex items-center gap-1">{children}</TooltipContent>
+        <TooltipContent className="flex max-h-[90vh] items-center gap-1 overflow-y-auto">
+          {children}
+        </TooltipContent>
       </Tooltip>
     </TooltipProvider>
   );

apps/webapp/app/components/code/CodeBlock.tsx

@@ -1,14 +1,23 @@
 import { Clipboard, ClipboardCheck } from "lucide-react";
 import type { Language, PrismTheme } from "prism-react-renderer";
-import Highlight, { defaultProps } from "prism-react-renderer";
+import { Highlight, Prism } from "prism-react-renderer";
 import { forwardRef, useCallback, useState } from "react";
 import { cn } from "~/utils/cn";
-import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "../primitives/Tooltip";
 import { Paragraph } from "../primitives/Paragraph";
+import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "../primitives/Tooltip";
 
 //This is a fork of https://github.com/mantinedev/mantine/blob/master/src/mantine-prism/src/Prism/Prism.tsx
 //it didn't support highlighting lines by dimming the rest of the code, or animations on the highlighting
 
+async function setup() {
+  (typeof global !== "undefined" ? global : window).Prism = Prism;
+  //@ts-ignore
+  await import("prismjs/components/prism-json");
+  //@ts-ignore
+  await import("prismjs/components/prism-typescript");
+}
+setup();
+
 type CodeBlockProps = {
   /** Code which will be highlighted */
   code: string;
@@ -238,7 +247,7 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
         )}
 
         {shouldHighlight ? (
-          <Highlight {...defaultProps} theme={theme} code={code} language={language}>
+          <Highlight theme={theme} code={code} language={language}>
             {({
               className: inheritedClassName,
               style: inheritedStyle,
@@ -283,7 +292,7 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
 
                       return (
                         <div
-                          key={lineProps.key}
+                          key={lineNumber}
                           {...lineProps}
                           className={cn(
                             "flex w-full justify-start transition-opacity duration-500",
@@ -312,7 +321,7 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
                               const tokenProps = getTokenProps({ token, key });
                               return (
                                 <span
-                                  key={tokenProps.key}
+                                  key={key}
                                   {...tokenProps}
                                   style={{
                                     color: tokenProps?.style?.color as string,

apps/webapp/app/components/environments/EnvironmentLabel.tsx

@@ -9,6 +9,30 @@ const variants = {
   large: "h-6 text-xs px-1.5 rounded",
 };
 
+export function EnvironmentTypeLabel({
+  environment,
+  size = "small",
+  className,
+}: {
+  environment: Environment;
+  size?: keyof typeof variants;
+  className?: string;
+}) {
+  return (
+    <span
+      className={cn(
+        "text-midnight-900 inline-flex items-center justify-center whitespace-nowrap border font-medium uppercase tracking-wider",
+        environmentBorderClassName(environment),
+        environmentTextClassName(environment),
+        variants[size],
+        className
+      )}
+    >
+      {environmentTypeTitle(environment)}
+    </span>
+  );
+}
+
 export function EnvironmentLabel({
   environment,
   size = "small",
@@ -116,6 +140,19 @@ export function environmentTitle(environment: Environment, username?: string) {
   }
 }
 
+export function environmentTypeTitle(environment: Environment) {
+  switch (environment.type) {
+    case "PRODUCTION":
+      return "Prod";
+    case "STAGING":
+      return "Staging";
+    case "DEVELOPMENT":
+      return "Dev";
+    case "PREVIEW":
+      return "Preview";
+  }
+}
+
 export function environmentColorClassName(environment: Environment) {
   switch (environment.type) {
     case "PRODUCTION":

apps/webapp/app/components/integrations/ConnectToOAuthForm.tsx

@@ -12,7 +12,7 @@ import { ApiAuthenticationMethodOAuth2, Integration, Scope } from "~/services/ex
 import { cn } from "~/utils/cn";
 import { CodeBlock } from "../code/CodeBlock";
 import { Button } from "../primitives/Buttons";
-import { Checkbox } from "../primitives/Checkbox";
+import { CheckboxWithLabel } from "../primitives/Checkbox";
 import { Fieldset } from "../primitives/Fieldset";
 import { FormError } from "../primitives/FormError";
 import { Header2, Header3 } from "../primitives/Headers";
@@ -123,7 +123,7 @@ export function ConnectToOAuthForm({
           <Paragraph variant="small" className="mb-2">
             To use your own OAuth app, check the option below and insert the details.
           </Paragraph>
-          <Checkbox
+          <CheckboxWithLabel
             id="hasCustomClient"
             label="Use my OAuth App"
             variant="simple/small"
@@ -200,7 +200,7 @@ export function ConnectToOAuthForm({
               )}
               {authMethod.scopes.map((s) => {
                 return (
-                  <Checkbox
+                  <CheckboxWithLabel
                     key={s.name}
                     id={s.name}
                     value={s.name}

apps/webapp/app/components/integrations/UpdateOAuthForm.tsx

@@ -10,7 +10,7 @@ import { ApiAuthenticationMethodOAuth2, Integration, Scope } from "~/services/ex
 import { cn } from "~/utils/cn";
 import { CodeBlock } from "../code/CodeBlock";
 import { Button } from "../primitives/Buttons";
-import { Checkbox } from "../primitives/Checkbox";
+import { CheckboxWithLabel } from "../primitives/Checkbox";
 import { Fieldset } from "../primitives/Fieldset";
 import { FormError } from "../primitives/FormError";
 import { Header2, Header3 } from "../primitives/Headers";
@@ -113,7 +113,7 @@ export function UpdateOAuthForm({
           <Paragraph variant="small" className="mb-2">
             To use your own OAuth app, check the option below and insert the details.
           </Paragraph>
-          <Checkbox
+          <CheckboxWithLabel
             id="hasCustomClient"
             label="Use my OAuth App"
             variant="simple/small"
@@ -189,7 +189,7 @@ export function UpdateOAuthForm({
               )}
               {authMethod.scopes.map((s) => {
                 return (
-                  <Checkbox
+                  <CheckboxWithLabel
                     key={s.name}
                     id={s.name}
                     value={s.name}

apps/webapp/app/components/navigation/SideMenu.tsx

@@ -233,31 +233,22 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
                   </div>
                   <hr className="border-charcoal-800" />
                   <div>
-                    <StepNumber stepNumber="1" title="Create a new Slack channel" />
+                    <StepNumber stepNumber="1" title="Email us" />
                     <StepContentContainer>
                       <Paragraph>
-                        In your Slack app, create a new channel from the main menu by going to File{" "}
-                        <ArrowRightIcon className="inline h-4 w-4 text-text-dimmed" /> New Channel
-                      </Paragraph>
-                    </StepContentContainer>
-                    <StepNumber stepNumber="2" title="Setup your channel" />
-                    <StepContentContainer>
-                      <Paragraph>
-                        Name your channel, set its visibility and click 'Create'.
-                      </Paragraph>
-                    </StepContentContainer>
-                    <StepNumber stepNumber="3" title="Invite Trigger.dev" />
-                    <StepContentContainer>
-                      <Paragraph>
-                        Invite this email address to your channel:{" "}
+                        Send us an email to this address from your Trigger.dev account email
+                        address:
                         <ClipboardField
                           variant="primary/medium"
-                          value="james@trigger.dev"
+                          value="priority-support@trigger.dev"
                           className="my-2"
                         />
                       </Paragraph>
+                    </StepContentContainer>
+                    <StepNumber stepNumber="2" title="Look out for an invite from Slack" />
+                    <StepContentContainer>
                       <Paragraph>
-                        As soon as we can, we'll accept your invitation and say hello!
+                        As soon as we can, we'll setup a Slack Connect channel and say hello!
                       </Paragraph>
                     </StepContentContainer>
                   </div>
@@ -551,6 +542,8 @@ function V3ProjectSideMenu({
   project: SideMenuProject;
   organization: MatchedOrganization;
 }) {
+  const { alertsEnabled } = useFeatures();
+
   return (
     <>
       <SideMenuHeader title={"Project (v3)"} />
@@ -603,13 +596,15 @@ function V3ProjectSideMenu({
         to={v3DeploymentsPath(organization, project)}
         data-action="deployments"
       />
-      <SideMenuItem
-        name="Alerts"
-        icon={BellAlertIcon}
-        iconColor="text-red-500"
-        to={v3ProjectAlertsPath(organization, project)}
-        data-action="alerts"
-      />
+      {alertsEnabled && (
+        <SideMenuItem
+          name="Alerts"
+          icon={BellAlertIcon}
+          iconColor="text-red-500"
+          to={v3ProjectAlertsPath(organization, project)}
+          data-action="alerts"
+        />
+      )}
       <SideMenuItem
         name="Project settings"
         icon="settings"

apps/webapp/app/components/primitives/Checkbox.tsx

@@ -1,5 +1,5 @@
 import * as React from "react";
-import { useEffect, useState } from "react";
+import { forwardRef, useEffect, useState } from "react";
 import { cn } from "~/utils/cn";
 import { Badge } from "./Badge";
 import { Paragraph } from "./Paragraph";
@@ -53,18 +53,18 @@ export type CheckboxProps = Omit<
   React.InputHTMLAttributes<HTMLInputElement>,
   "checked" | "onChange"
 > & {
-  id: string;
+  id?: string;
   name?: string;
   value?: string;
   variant?: keyof typeof variants;
-  label?: React.ReactNode;
+  label: React.ReactNode;
   description?: string;
   badges?: string[];
   className?: string;
   onChange?: (isChecked: boolean) => void;
 };
 
-export const Checkbox = React.forwardRef<HTMLInputElement, CheckboxProps>(
+export const CheckboxWithLabel = React.forwardRef<HTMLInputElement, CheckboxProps>(
   (
     {
       id,
@@ -172,3 +172,21 @@ export const Checkbox = React.forwardRef<HTMLInputElement, CheckboxProps>(
     );
   }
 );
+
+type SimpleCheckboxProps = Omit<React.ComponentProps<"input">, "type">;
+
+export const Checkbox = forwardRef<HTMLInputElement, SimpleCheckboxProps>(
+  ({ className, ...props }: SimpleCheckboxProps, ref) => {
+    return (
+      <input
+        type="checkbox"
+        className={cn(
+          props.readOnly || props.disabled ? "cursor-default" : "cursor-pointer",
+          "read-only:border-charcoal-650 disabled:border-charcoal-650 rounded-sm border border-charcoal-600 bg-transparent transition checked:!bg-indigo-500 read-only:!bg-charcoal-700 group-hover:bg-charcoal-900 group-hover:checked:bg-indigo-500 group-focus:ring-1 focus:ring-indigo-500 focus:ring-offset-0 focus:ring-offset-transparent focus-visible:outline-none  focus-visible:ring-indigo-500 disabled:!bg-charcoal-700"
+        )}
+        {...props}
+        ref={ref}
+      />
+    );
+  }
+);

apps/webapp/app/components/primitives/Dialog.tsx

@@ -43,7 +43,7 @@ const DialogContent = React.forwardRef<
     <DialogPrimitive.Content
       ref={ref}
       className={cn(
-        "fixed z-50 grid w-full gap-4 rounded-b-lg border bg-background-dimmed px-4 pb-4 pt-3.5 shadow-lg animate-in data-[state=open]:fade-in-90 data-[state=open]:slide-in-from-bottom-10 sm:max-w-lg sm:rounded-lg sm:zoom-in-90 data-[state=open]:sm:slide-in-from-bottom-0",
+        "fixed z-50 grid w-full gap-4 rounded-b-lg border bg-background-dimmed px-4 pb-4 pt-2.5 shadow-lg animate-in data-[state=open]:fade-in-90 data-[state=open]:slide-in-from-bottom-10 sm:max-w-lg sm:rounded-lg sm:zoom-in-90 data-[state=open]:sm:slide-in-from-bottom-0",
         className
       )}
       {...props}
@@ -74,7 +74,7 @@ DialogContent.displayName = DialogPrimitive.Content.displayName;
 
 const DialogHeader = ({ className, ...props }: React.HTMLAttributes<HTMLDivElement>) => (
   <div
-    className={cn("flex flex-col text-left font-medium text-text-dimmed", className)}
+    className={cn("flex flex-col text-left font-medium text-text-bright", className)}
     {...props}
   />
 );

apps/webapp/app/components/primitives/SelectedItemsProvider.tsx

@@ -0,0 +1,146 @@
+"use client";
+
+import { createContext, useCallback, useContext, useReducer } from "react";
+
+type SelectedItemsContext = {
+  selectedItems: Set<string>;
+  select: (items: string | string[]) => void;
+  deselect: (items: string | string[]) => void;
+  toggle: (items: string | string[]) => void;
+  deselectAll: () => void;
+  has: (item: string) => boolean;
+  hasAll: (items: string[]) => boolean;
+};
+
+const SelectedItemsContext = createContext<SelectedItemsContext>({} as SelectedItemsContext);
+
+export function useSelectedItems(enabled = true) {
+  const context = useContext(SelectedItemsContext);
+  if (!context && enabled) {
+    throw new Error("useSelectedItems must be used within a SelectedItemsProvider");
+  }
+
+  return context;
+}
+
+export function SelectedItemsProvider({
+  initialSelectedItems,
+  maxSelectedItemCount,
+  children,
+}: {
+  initialSelectedItems: string[];
+  maxSelectedItemCount?: number;
+  children: React.ReactNode | ((context: SelectedItemsContext) => React.ReactNode);
+}) {
+  const [state, dispatch] = useReducer(selectedItemsReducer, {
+    items: new Set<string>(initialSelectedItems),
+    maxSelectedItemCount,
+  });
+
+  const select = useCallback((items: string | string[]) => {
+    dispatch({ type: "select", items: Array.isArray(items) ? items : [items] });
+  }, []);
+
+  const deselect = useCallback((items: string | string[]) => {
+    dispatch({ type: "deselect", items: Array.isArray(items) ? items : [items] });
+  }, []);
+
+  const toggle = useCallback((items: string | string[]) => {
+    dispatch({ type: "toggle", items: Array.isArray(items) ? items : [items] });
+  }, []);
+
+  const deselectAll = useCallback(() => {
+    dispatch({ type: "deselectAll" });
+  }, []);
+
+  const has = useCallback((item: string) => state.items.has(item), [state]);
+
+  const hasAll = useCallback(
+    (items: string[]) => items.every((item) => state.items.has(item)),
+    [state]
+  );
+
+  return (
+    <SelectedItemsContext.Provider
+      value={{ selectedItems: state.items, select, deselect, toggle, deselectAll, has, hasAll }}
+    >
+      {typeof children === "function"
+        ? children({
+            selectedItems: state.items,
+            select,
+            deselect,
+            toggle,
+            deselectAll,
+            has,
+            hasAll,
+          })
+        : children}
+    </SelectedItemsContext.Provider>
+  );
+}
+
+type SelectItemsAction = {
+  type: "select";
+  items: string[];
+};
+
+type DeSelectItemsAction = {
+  type: "deselect";
+  items: string[];
+};
+
+type DeselectAllItemsAction = {
+  type: "deselectAll";
+};
+
+type ToggleItemsAction = {
+  type: "toggle";
+  items: string[];
+};
+
+type Action = SelectItemsAction | DeSelectItemsAction | ToggleItemsAction | DeselectAllItemsAction;
+
+function selectedItemsReducer(
+  state: { items: Set<string>; maxSelectedItemCount?: number },
+  action: Action
+) {
+  switch (action.type) {
+    case "select":
+      const items = new Set([...state.items, ...action.items]);
+      return { ...state, items: cappedSet(items, state.maxSelectedItemCount) };
+    case "deselect":
+      const newItems = new Set(state.items);
+      action.items.forEach((item) => {
+        newItems.delete(item);
+      });
+      return { ...state, items: cappedSet(newItems, state.maxSelectedItemCount) };
+    case "toggle":
+      let newSet = new Set(state.items);
+      action.items.forEach((item) => {
+        if (newSet.has(item)) {
+          newSet.delete(item);
+        } else {
+          newSet.add(item);
+        }
+      });
+      return { ...state, items: cappedSet(newSet, state.maxSelectedItemCount) };
+    case "deselectAll":
+      return { ...state, items: new Set<string>() };
+    default:
+      return state;
+  }
+}
+
+function cappedSet(set: Set<string>, max?: number) {
+  if (!max) {
+    return set;
+  }
+
+  if (set.size <= max) {
+    return set;
+  }
+
+  console.warn(`Selected items exceeded the maximum count of ${max}.`);
+
+  return new Set([...set].slice(0, max));
+}

apps/webapp/app/components/primitives/Table.tsx

@@ -106,7 +106,7 @@ export const TableHeaderCell = forwardRef<HTMLTableCellElement, TableHeaderCellP
         ref={ref}
         scope="col"
         className={cn(
-          "px-4 py-2 align-middle text-xxs font-normal uppercase tracking-wider text-text-dimmed",
+          "px-3 py-2 align-middle text-xxs font-normal uppercase tracking-wider text-text-dimmed",
           alignmentClassName,
           className
         )}
@@ -155,7 +155,7 @@ export const TableCell = forwardRef<HTMLTableCellElement, TableCellProps>(
     }
 
     const flexClasses = cn(
-      "flex w-full whitespace-nowrap px-4 py-3 text-xs text-text-dimmed",
+      "flex w-full whitespace-nowrap px-3 py-3 text-xs text-text-dimmed",
       alignment === "left"
         ? "justify-start text-left"
         : alignment === "center"
@@ -170,7 +170,7 @@ export const TableCell = forwardRef<HTMLTableCellElement, TableCellProps>(
           "text-xs text-charcoal-400",
           to || onClick || hasAction
             ? "cursor-pointer group-hover/table-row:bg-charcoal-900"
-            : "px-4 py-3 align-middle",
+            : "px-3 py-3 align-middle",
           !to && !onClick && alignmentClassName,
           isSticky && stickyStyles,
           className

apps/webapp/app/components/runs/v3/BulkAction.tsx

@@ -0,0 +1,73 @@
+import { ArrowPathIcon, NoSymbolIcon } from "@heroicons/react/20/solid";
+import { BulkActionType } from "@trigger.dev/database";
+import assertNever from "assert-never";
+import { cn } from "~/utils/cn";
+
+export function BulkActionStatusCombo({
+  type,
+  className,
+  iconClassName,
+}: {
+  type: BulkActionType;
+  className?: string;
+  iconClassName?: string;
+}) {
+  return (
+    <span className={cn("flex items-center gap-1", className)}>
+      <BulkActionIcon type={type} className={cn("h-4 w-4", iconClassName)} />
+      <BulkActionLabel type={type} />
+    </span>
+  );
+}
+
+export function BulkActionLabel({ type }: { type: BulkActionType }) {
+  return <span className={bulkActionClassName(type)}>{bulkActionTitle(type)}</span>;
+}
+
+export function BulkActionIcon({ type, className }: { type: BulkActionType; className: string }) {
+  switch (type) {
+    case "REPLAY":
+      return <ArrowPathIcon className={cn(bulkActionClassName(type), className)} />;
+    case "CANCEL":
+      return <NoSymbolIcon className={cn(bulkActionClassName(type), className)} />;
+    default: {
+      assertNever(type);
+    }
+  }
+}
+
+export function bulkActionClassName(type: BulkActionType): string {
+  switch (type) {
+    case "REPLAY":
+      return "text-indigo-500";
+    case "CANCEL":
+      return "text-rose-500";
+    default: {
+      assertNever(type);
+    }
+  }
+}
+
+export function bulkActionTitle(type: BulkActionType): string {
+  switch (type) {
+    case "REPLAY":
+      return "Replay";
+    case "CANCEL":
+      return "Cancel";
+    default: {
+      assertNever(type);
+    }
+  }
+}
+
+export function bulkActionVerb(type: BulkActionType): string {
+  switch (type) {
+    case "REPLAY":
+      return "Replaying";
+    case "CANCEL":
+      return "Canceling";
+    default: {
+      assertNever(type);
+    }
+  }
+}

apps/webapp/app/components/runs/v3/EnabledStatus.tsx

@@ -1,18 +1,31 @@
 import { BoltSlashIcon, CheckCircleIcon } from "@heroicons/react/20/solid";
 
-export function EnabledStatus({ enabled }: { enabled: boolean }) {
+type EnabledStatusProps = {
+  enabled: boolean;
+  enabledIcon?: React.ComponentType<any>;
+  disabledIcon?: React.ComponentType<any>;
+};
+
+export function EnabledStatus({
+  enabled,
+  enabledIcon = CheckCircleIcon,
+  disabledIcon = BoltSlashIcon,
+}: EnabledStatusProps) {
+  const EnabledIcon = enabledIcon;
+  const DisabledIcon = disabledIcon;
+
   switch (enabled) {
     case true:
       return (
         <div className="flex items-center gap-1 text-xs text-success">
-          <CheckCircleIcon className="h-4 w-4" />
+          <EnabledIcon className="size-4" />
           Enabled
         </div>
       );
     case false:
       return (
         <div className="text-dimmed flex items-center gap-1 text-xs">
-          <BoltSlashIcon className="h-4 w-4" />
+          <DisabledIcon className="size-4" />
           Disabled
         </div>
       );

apps/webapp/app/components/runs/v3/RunFilters.tsx

@@ -1,9 +1,21 @@
 import * as Ariakit from "@ariakit/react";
-import { CalendarIcon, CpuChipIcon, XMarkIcon } from "@heroicons/react/20/solid";
+import {
+  ArrowPathIcon,
+  CalendarIcon,
+  CpuChipIcon,
+  InboxStackIcon,
+  XMarkIcon,
+} from "@heroicons/react/20/solid";
 import { Form } from "@remix-run/react";
-import { RuntimeEnvironment, TaskRunStatus, TaskTriggerSource } from "@trigger.dev/database";
+import type {
+  RuntimeEnvironment,
+  TaskTriggerSource,
+  TaskRunStatus,
+  BulkActionType,
+} from "@trigger.dev/database";
 import { ListFilterIcon } from "lucide-react";
-import { ReactNode, startTransition, useCallback, useMemo, useState } from "react";
+import type { ReactNode } from "react";
+import { startTransition, useCallback, useMemo, useState } from "react";
 import { z } from "zod";
 import { TaskIcon } from "~/assets/icons/TaskIcon";
 import { EnvironmentLabel, environmentTitle } from "~/components/environments/EnvironmentLabel";
@@ -32,12 +44,15 @@ import { Button } from "../../primitives/Buttons";
 import {
   TaskRunStatusCombo,
   allTaskRunStatuses,
+  filterableTaskRunStatuses,
   descriptionForTaskRunStatus,
   runStatusTitle,
 } from "./TaskRunStatus";
 import { TaskTriggerSourceIcon } from "./TaskTriggerSource";
+import { DateTime } from "~/components/primitives/DateTime";
+import { BulkActionStatusCombo } from "./BulkAction";
 
-export const TaskAttemptStatus = z.nativeEnum(TaskRunStatus);
+export const TaskAttemptStatus = z.enum(allTaskRunStatuses);
 
 export const TaskRunListSearchFilters = z.object({
   cursor: z.string().optional(),
@@ -59,6 +74,7 @@ export const TaskRunListSearchFilters = z.object({
     TaskAttemptStatus.array().optional()
   ),
   period: z.preprocess((value) => (value === "all" ? undefined : value), z.string().optional()),
+  bulkId: z.string().optional(),
   from: z.coerce.number().optional(),
   to: z.coerce.number().optional(),
 });
@@ -72,6 +88,11 @@ type DisplayableEnvironment = Pick<RuntimeEnvironment, "type" | "id"> & {
 type RunFiltersProps = {
   possibleEnvironments: DisplayableEnvironment[];
   possibleTasks: { slug: string; triggerSource: TaskTriggerSource }[];
+  bulkActions: {
+    id: string;
+    type: BulkActionType;
+    createdAt: Date;
+  }[];
   hasFilters: boolean;
 };
 
@@ -82,7 +103,8 @@ export function RunsFilters(props: RunFiltersProps) {
     searchParams.has("statuses") ||
     searchParams.has("environments") ||
     searchParams.has("tasks") ||
-    searchParams.has("period");
+    searchParams.has("period") ||
+    searchParams.has("bulkId");
 
   return (
     <div className="flex flex-row flex-wrap items-center gap-1">
@@ -112,7 +134,8 @@ const filterTypes = [
   { name: "environments", title: "Environment", icon: <CpuChipIcon className="size-4" /> },
   { name: "tasks", title: "Tasks", icon: <TaskIcon className="size-4" /> },
   { name: "created", title: "Created", icon: <CalendarIcon className="size-4" /> },
-];
+  { name: "bulk", title: "Bulk action", icon: <InboxStackIcon className="size-4" /> },
+] as const;
 
 type FilterType = (typeof filterTypes)[number]["name"];
 
@@ -123,6 +146,7 @@ function FilterMenu(props: RunFiltersProps) {
 
   const filterTrigger = (
     <SelectTrigger
+      autoFocus
       icon={
         <div className="flex size-4 items-center justify-center">
           <ListFilterIcon className="size-3.5" />
@@ -180,13 +204,14 @@ function FilterMenuProvider({
   );
 }
 
-function AppliedFilters({ possibleEnvironments, possibleTasks }: RunFiltersProps) {
+function AppliedFilters({ possibleEnvironments, possibleTasks, bulkActions }: RunFiltersProps) {
   return (
     <>
       <AppliedStatusFilter />
       <AppliedEnvironmentFilter possibleEnvironments={possibleEnvironments} />
       <AppliedTaskFilter possibleTasks={possibleTasks} />
       <AppliedPeriodFilter />
+      <AppliedBulkActionsFilter bulkActions={bulkActions} />
     </>
   );
 }
@@ -211,6 +236,8 @@ function Menu(props: MenuProps) {
       return <TasksDropdown onClose={() => props.setFilterType(undefined)} {...props} />;
     case "created":
       return <CreatedDropdown onClose={() => props.setFilterType(undefined)} {...props} />;
+    case "bulk":
+      return <BulkActionsDropdown onClose={() => props.setFilterType(undefined)} {...props} />;
   }
 }
 
@@ -246,7 +273,7 @@ function MainMenu({ searchValue, trigger, clearSearchValue, setFilterType }: Men
   );
 }
 
-const statuses = allTaskRunStatuses.map((status) => ({
+const statuses = filterableTaskRunStatuses.map((status) => ({
   title: runStatusTitle(status),
   value: status,
 }));
@@ -529,6 +556,99 @@ function AppliedTaskFilter({ possibleTasks }: Pick<RunFiltersProps, "possibleTas
   );
 }
 
+function BulkActionsDropdown({
+  trigger,
+  clearSearchValue,
+  searchValue,
+  onClose,
+  bulkActions,
+}: {
+  trigger: ReactNode;
+  clearSearchValue: () => void;
+  searchValue: string;
+  onClose?: () => void;
+  bulkActions: RunFiltersProps["bulkActions"];
+}) {
+  const { value, replace } = useSearchParams();
+
+  const handleChange = (value: string) => {
+    clearSearchValue();
+    replace({ bulkId: value, cursor: undefined, direction: undefined });
+  };
+
+  const filtered = useMemo(() => {
+    return bulkActions.filter((item) => {
+      return (
+        item.type.toLowerCase().includes(searchValue.toLowerCase()) ||
+        item.createdAt.toISOString().includes(searchValue)
+      );
+    });
+  }, [searchValue, bulkActions]);
+
+  return (
+    <SelectProvider value={value("bulkId")} setValue={handleChange} virtualFocus={true}>
+      {trigger}
+      <SelectPopover
+        className="min-w-0 max-w-[min(240px,var(--popover-available-width))]"
+        hideOnEscape={() => {
+          if (onClose) {
+            onClose();
+            return false;
+          }
+
+          return true;
+        }}
+      >
+        <ComboBox placeholder={"Filter by bulk action..."} value={searchValue} />
+        <SelectList>
+          <SelectItem value={""}>None</SelectItem>
+          {filtered.map((item, index) => (
+            <SelectItem key={item.id} value={item.id}>
+              <div className="flex gap-3">
+                <BulkActionStatusCombo type={item.type} iconClassName="size-4" />
+                <DateTime date={item.createdAt} />
+              </div>
+            </SelectItem>
+          ))}
+        </SelectList>
+      </SelectPopover>
+    </SelectProvider>
+  );
+}
+
+function AppliedBulkActionsFilter({ bulkActions }: Pick<RunFiltersProps, "bulkActions">) {
+  const { value, del } = useSearchParams();
+
+  const bulkId = value("bulkId");
+
+  if (!bulkId) {
+    return null;
+  }
+
+  const action = bulkActions.find((action) => action.id === bulkId);
+
+  return (
+    <FilterMenuProvider>
+      {(search, setSearch) => (
+        <BulkActionsDropdown
+          trigger={
+            <Ariakit.Select render={<div className="group cursor-pointer" />}>
+              <AppliedFilter
+                label="Bulk action"
+                value={bulkId}
+                onRemove={() => del(["bulkId", "cursor", "direction"])}
+              />
+            </Ariakit.Select>
+          }
+          searchValue={search}
+          clearSearchValue={() => setSearch("")}
+          bulkActions={bulkActions}
+        />
+      )}
+    </FilterMenuProvider>
+  );
+}
+
 const timePeriods = [
   {
     label: "All periods",

apps/webapp/app/components/runs/v3/TaskRunAttemptStatus.tsx

@@ -7,7 +7,7 @@ import {
   XCircleIcon,
 } from "@heroicons/react/20/solid";
 import type { TaskRunAttemptStatus as TaskRunAttemptStatusType } from "@trigger.dev/database";
-import { TaskRunAttemptStatus } from "@trigger.dev/database";
+import { TaskRunAttemptStatus } from "~/database-types";
 import assertNever from "assert-never";
 import { SnowflakeIcon } from "lucide-react";
 import { Spinner } from "~/components/primitives/Spinner";
@@ -17,7 +17,7 @@ export const allTaskRunAttemptStatuses = Object.values(
   TaskRunAttemptStatus
 ) as TaskRunAttemptStatusType[];
 
-export type ExtendedTaskAttemptStatus = (typeof allTaskRunAttemptStatuses)[number] | "ENQUEUED";
+export type ExtendedTaskAttemptStatus = TaskRunAttemptStatusType | "ENQUEUED";
 
 export function TaskRunAttemptStatusCombo({
   status,

apps/webapp/app/components/runs/v3/TaskRunStatus.tsx

@@ -25,9 +25,24 @@ export const allTaskRunStatuses = [
   "CANCELED",
   "COMPLETED_WITH_ERRORS",
   "CRASHED",
+  "PAUSED",
   "INTERRUPTED",
   "SYSTEM_FAILURE",
-] as TaskRunStatus[];
+] as const satisfies Readonly<Array<TaskRunStatus>>;
+
+export const filterableTaskRunStatuses = [
+  "WAITING_FOR_DEPLOY",
+  "PENDING",
+  "EXECUTING",
+  "RETRYING_AFTER_FAILURE",
+  "WAITING_TO_RESUME",
+  "COMPLETED_SUCCESSFULLY",
+  "CANCELED",
+  "COMPLETED_WITH_ERRORS",
+  "CRASHED",
+  "INTERRUPTED",
+  "SYSTEM_FAILURE",
+] as const satisfies Readonly<Array<TaskRunStatus>>;
 
 const taskRunStatusDescriptions: Record<TaskRunStatus, string> = {
   PENDING: "Task is waiting to be executed",

apps/webapp/app/components/runs/v3/TaskRunsTable.tsx

@@ -28,6 +28,11 @@ import { CancelRunDialog } from "./CancelRunDialog";
 import { ReplayRunDialog } from "./ReplayRunDialog";
 import { TaskRunStatusCombo } from "./TaskRunStatus";
 import { LiveTimer } from "./LiveTimer";
+import { useSelectedItems } from "~/components/primitives/SelectedItemsProvider";
+import { Checkbox } from "~/components/primitives/Checkbox";
+import { useCallback, useRef } from "react";
+import { run } from "@remix-run/dev/dist/cli/run";
+import { formatNumber } from "~/utils/numberFormatter";
 
 type RunsTableProps = {
   total: number;
@@ -36,6 +41,7 @@ type RunsTableProps = {
   showJob?: boolean;
   runs: RunListItem[];
   isLoading?: boolean;
+  allowSelection?: boolean;
 };
 
 export function TaskRunsTable({
@@ -44,15 +50,66 @@ export function TaskRunsTable({
   filters,
   runs,
   isLoading = false,
+  allowSelection = false,
 }: RunsTableProps) {
   const organization = useOrganization();
   const project = useProject();
+  const checkboxes = useRef<(HTMLInputElement | null)[]>([]);
+  const { selectedItems, has, hasAll, select, deselect, toggle } = useSelectedItems(allowSelection);
+
+  const navigateCheckboxes = useCallback(
+    (event: React.KeyboardEvent<HTMLInputElement>, index: number) => {
+      //indexes are out by one because of the header row
+      if (event.key === "ArrowUp" && index > 0) {
+        checkboxes.current[index - 1]?.focus();
+
+        if (event.shiftKey) {
+          const oldItem = runs.at(index - 1);
+          const newItem = runs.at(index - 2);
+          const itemsIds = [oldItem?.id, newItem?.id].filter(Boolean);
+          select(itemsIds);
+        }
+      } else if (event.key === "ArrowDown" && index < checkboxes.current.length - 1) {
+        checkboxes.current[index + 1]?.focus();
+
+        if (event.shiftKey) {
+          const oldItem = runs.at(index - 1);
+          const newItem = runs.at(index);
+          const itemsIds = [oldItem?.id, newItem?.id].filter(Boolean);
+          select(itemsIds);
+        }
+      }
+    },
+    [checkboxes, runs]
+  );
 
   return (
     <Table>
       <TableHeader>
         <TableRow>
-          <TableHeaderCell>Run</TableHeaderCell>
+          {allowSelection && (
+            <TableHeaderCell className="pl-2 pr-0">
+              {runs.length > 0 && (
+                <Checkbox
+                  checked={hasAll(runs.map((r) => r.id))}
+                  onChange={(element) => {
+                    const ids = runs.map((r) => r.id);
+                    const checked = element.currentTarget.checked;
+                    if (checked) {
+                      select(ids);
+                    } else {
+                      deselect(ids);
+                    }
+                  }}
+                  ref={(r) => {
+                    checkboxes.current[0] = r;
+                  }}
+                  onKeyDown={(event) => navigateCheckboxes(event, 0)}
+                />
+              )}
+            </TableHeaderCell>
+          )}
+          <TableHeaderCell alignment="right">Run #</TableHeaderCell>
           <TableHeaderCell>Task ID</TableHeaderCell>
           <TableHeaderCell>Version</TableHeaderCell>
           <TableHeaderCell>Env</TableHeaderCell>
@@ -74,11 +131,27 @@ export function TaskRunsTable({
         ) : runs.length === 0 ? (
           <BlankState isLoading={isLoading} filters={filters} />
         ) : (
-          runs.map((run) => {
+          runs.map((run, index) => {
             const path = v3RunSpanPath(organization, project, run, { spanId: run.spanId });
             return (
               <TableRow key={run.id}>
-                <TableCell to={path}>#{run.number}</TableCell>
+                {allowSelection && (
+                  <TableCell className="pl-2 pr-0">
+                    <Checkbox
+                      checked={has(run.id)}
+                      onChange={(element) => {
+                        toggle(run.id);
+                      }}
+                      ref={(r) => {
+                        checkboxes.current[index + 1] = r;
+                      }}
+                      onKeyDown={(event) => navigateCheckboxes(event, index + 1)}
+                    />
+                  </TableCell>
+                )}
+                <TableCell to={path} alignment="right">
+                  {formatNumber(run.number)}
+                </TableCell>
                 <TableCell to={path}>{run.taskIdentifier}</TableCell>
                 <TableCell to={path}>{run.version ?? "–"}</TableCell>
                 <TableCell to={path}>
@@ -194,7 +267,7 @@ function BlankState({ isLoading, filters }: Pick<RunsTableProps, "isLoading" | "
   ) {
     const environment = envs?.find((env) => env.id === filters.environments[0]);
     return (
-      <TableBlankRow colSpan={9}>
+      <TableBlankRow colSpan={10}>
         <div className="py-14">
           <Paragraph className="w-auto" variant="base/bright" spacing>
             There are no runs for {filters.tasks[0]}
@@ -235,8 +308,21 @@ function BlankState({ isLoading, filters }: Pick<RunsTableProps, "isLoading" | "
   }
 
   return (
-    <TableBlankRow colSpan={9}>
-      <NoRuns title="No runs match your filters" />
+    <TableBlankRow colSpan={10}>
+      <div className="flex flex-col items-center justify-center gap-2">
+        <Paragraph className="w-auto" variant="small">
+          No runs currently match your filters. Try refreshing or modifying your filters.
+        </Paragraph>
+        <Button
+          LeadingIcon={ArrowPathIcon}
+          variant="tertiary/small"
+          onClick={() => {
+            window.location.reload();
+          }}
+        >
+          Refresh
+        </Button>
+      </div>
     </TableBlankRow>
   );
 }

apps/webapp/app/consts.ts

@@ -12,3 +12,5 @@ export const MAX_RUN_CHUNK_EXECUTION_LIMIT = 120000; // 2 minutes
 export const VERCEL_RESPONSE_TIMEOUT_STATUS_CODES = [408, 504];
 export const MAX_BATCH_TRIGGER_ITEMS = 100;
 export const MAX_TASK_RUN_ATTEMPTS = 250;
+export const BULK_ACTION_RUN_LIMIT = 250;
+export const MAX_JOB_RUN_EXECUTION_COUNT = 250;

apps/webapp/app/database-types.ts

@@ -0,0 +1,68 @@
+// There's a weird issue with importing values from the prisma client
+// when using Remix Vite + pnpm + prisma
+// As long as they're only used as types it's ok
+// Import types here and validate hardcoded enums
+
+import type {
+  BatchTaskRunItemStatus as BatchTaskRunItemStatusType,
+  TaskRunAttemptStatus as TaskRunAttemptStatusType,
+  TaskRunStatus as TaskRunStatusType,
+  JobRunStatus as JobRunStatusType,
+  RuntimeEnvironmentType as RuntimeEnvironmentTypeType,
+} from "@trigger.dev/database";
+
+export const BatchTaskRunItemStatus = {
+  PENDING: "PENDING",
+  FAILED: "FAILED",
+  CANCELED: "CANCELED",
+  COMPLETED: "COMPLETED",
+} as const satisfies Record<BatchTaskRunItemStatusType, BatchTaskRunItemStatusType>;
+
+export const TaskRunAttemptStatus = {
+  PENDING: "PENDING",
+  EXECUTING: "EXECUTING",
+  PAUSED: "PAUSED",
+  FAILED: "FAILED",
+  CANCELED: "CANCELED",
+  COMPLETED: "COMPLETED",
+} as const satisfies Record<TaskRunAttemptStatusType, TaskRunAttemptStatusType>;
+
+export const TaskRunStatus = {
+  PENDING: "PENDING",
+  WAITING_FOR_DEPLOY: "WAITING_FOR_DEPLOY",
+  EXECUTING: "EXECUTING",
+  WAITING_TO_RESUME: "WAITING_TO_RESUME",
+  RETRYING_AFTER_FAILURE: "RETRYING_AFTER_FAILURE",
+  PAUSED: "PAUSED",
+  CANCELED: "CANCELED",
+  INTERRUPTED: "INTERRUPTED",
+  COMPLETED_SUCCESSFULLY: "COMPLETED_SUCCESSFULLY",
+  COMPLETED_WITH_ERRORS: "COMPLETED_WITH_ERRORS",
+  SYSTEM_FAILURE: "SYSTEM_FAILURE",
+  CRASHED: "CRASHED",
+} as const satisfies Record<TaskRunStatusType, TaskRunStatusType>;
+
+export const JobRunStatus = {
+  PENDING: "PENDING",
+  QUEUED: "QUEUED",
+  WAITING_ON_CONNECTIONS: "WAITING_ON_CONNECTIONS",
+  PREPROCESSING: "PREPROCESSING",
+  STARTED: "STARTED",
+  EXECUTING: "EXECUTING",
+  WAITING_TO_CONTINUE: "WAITING_TO_CONTINUE",
+  WAITING_TO_EXECUTE: "WAITING_TO_EXECUTE",
+  SUCCESS: "SUCCESS",
+  FAILURE: "FAILURE",
+  TIMED_OUT: "TIMED_OUT",
+  ABORTED: "ABORTED",
+  CANCELED: "CANCELED",
+  UNRESOLVED_AUTH: "UNRESOLVED_AUTH",
+  INVALID_PAYLOAD: "INVALID_PAYLOAD",
+} as const satisfies Record<JobRunStatusType, JobRunStatusType>;
+
+export const RuntimeEnvironmentType = {
+  PRODUCTION: "PRODUCTION",
+  STAGING: "STAGING",
+  DEVELOPMENT: "DEVELOPMENT",
+  PREVIEW: "PREVIEW",
+} as const satisfies Record<RuntimeEnvironmentTypeType, RuntimeEnvironmentTypeType>;

apps/webapp/app/db.server.ts

@@ -70,7 +70,9 @@ export { Prisma };
 
 export const prisma = singleton("prisma", getClient);
 
-export const $replica: Omit<PrismaClient, "$transaction"> = singleton(
+export type PrismaReplicaClient = Omit<PrismaClient, "$transaction">;
+
+export const $replica: PrismaReplicaClient = singleton(
   "replica",
   () => getReplicaClient() ?? prisma
 );

apps/webapp/app/env.server.ts

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { SecretStoreOptionsSchema } from "./services/secrets/secretStore.server";
+import { SecretStoreOptionsSchema } from "./services/secrets/secretStoreOptionsSchema.server";
 import { isValidRegex } from "./utils/regex";
 import { isValidDatabaseUrl } from "./utils/db";
 
@@ -47,6 +47,8 @@ const EnvironmentSchema = z.object({
   WORKER_SCHEMA: z.string().default("graphile_worker"),
   WORKER_CONCURRENCY: z.coerce.number().int().default(10),
   WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000),
+  /** The number of days a failed Graphile task should stay before getting cleaned up */
+  WORKER_CLEANUP_TTL_DAYS: z.coerce.number().int().default(3),
   EXECUTION_WORKER_CONCURRENCY: z.coerce.number().int().default(10),
   EXECUTION_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000),
   WORKER_ENABLED: z.string().default("true"),
@@ -98,6 +100,10 @@ const EnvironmentSchema = z.object({
   API_RATE_LIMIT_REQUEST_LOGS_ENABLED: z.string().default("0"),
   API_RATE_LIMIT_REJECTION_LOGS_ENABLED: z.string().default("1"),
 
+  //Ingesting event rate limit
+  INGEST_EVENT_RATE_LIMIT_WINDOW: z.string().default("60s"),
+  INGEST_EVENT_RATE_LIMIT_MAX: z.coerce.number().int().optional(),
+
   //v3
   V3_ENABLED: z.string().default("false"),
   PROVIDER_SECRET: z.string().default("provider-secret"),
@@ -109,6 +115,7 @@ const EnvironmentSchema = z.object({
   CONTAINER_REGISTRY_USERNAME: z.string().optional(),
   CONTAINER_REGISTRY_PASSWORD: z.string().optional(),
   DEPLOY_REGISTRY_HOST: z.string().optional(),
+  DEPLOY_REGISTRY_NAMESPACE: z.string().default("trigger"),
   OBJECT_STORE_BASE_URL: z.string().optional(),
   OBJECT_STORE_ACCESS_KEY_ID: z.string().optional(),
   OBJECT_STORE_SECRET_ACCESS_KEY: z.string().optional(),
@@ -156,6 +163,28 @@ const EnvironmentSchema = z.object({
 
   ORG_SLACK_INTEGRATION_CLIENT_ID: z.string().optional(),
   ORG_SLACK_INTEGRATION_CLIENT_SECRET: z.string().optional(),
+
+  /** These enable the alerts feature in v3 */
+  ALERT_FROM_EMAIL: z.string().optional(),
+  ALERT_RESEND_API_KEY: z.string().optional(),
+
+  MAX_SEQUENTIAL_INDEX_FAILURE_COUNT: z.coerce.number().default(96),
+
+  LOOPS_API_KEY: z.string().optional(),
+  MARQS_DISABLE_REBALANCING: z.coerce.boolean().default(false),
+
+  VERBOSE_GRAPHILE_LOGGING: z.string().default("false"),
+  V2_MARQS_ENABLED: z.string().default("0"),
+  V2_MARQS_CONSUMER_POOL_ENABLED: z.string().default("0"),
+  V2_MARQS_CONSUMER_POOL_SIZE: z.coerce.number().int().default(10),
+  V2_MARQS_CONSUMER_POLL_INTERVAL_MS: z.coerce.number().int().default(1000),
+  V2_MARQS_QUEUE_SELECTION_COUNT: z.coerce.number().int().default(36),
+  V2_MARQS_VISIBILITY_TIMEOUT_MS: z.coerce
+    .number()
+    .int()
+    .default(60 * 1000 * 15),
+  V2_MARQS_DEFAULT_ENV_CONCURRENCY: z.coerce.number().int().default(100),
+  V2_MARQS_VERBOSE: z.string().default("0"),
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/features.server.ts

@@ -4,6 +4,7 @@ import { requestUrl } from "./utils/requestUrl.server";
 export type TriggerFeatures = {
   isManagedCloud: boolean;
   v3Enabled: boolean;
+  alertsEnabled: boolean;
 };
 
 // If the request host is cloud.trigger.dev then we are on the managed cloud
@@ -20,5 +21,6 @@ export function featuresForRequest(request: Request): TriggerFeatures {
   return {
     isManagedCloud,
     v3Enabled: env.V3_ENABLED === "true",
+    alertsEnabled: env.ALERT_FROM_EMAIL !== undefined && env.ALERT_RESEND_API_KEY !== undefined,
   };
 }

apps/webapp/app/hooks/useFeatures.ts

@@ -5,5 +5,5 @@ import type { TriggerFeatures } from "~/features.server";
 export function useFeatures(): TriggerFeatures {
   const routeMatch = useTypedRouteLoaderData<typeof loader>("root");
 
-  return routeMatch?.features ?? { isManagedCloud: false, v3Enabled: false };
+  return routeMatch?.features ?? { isManagedCloud: false, v3Enabled: false, alertsEnabled: false };
 }

apps/webapp/app/models/api-key.server.ts

@@ -1,6 +1,7 @@
-import { RuntimeEnvironmentType, type RuntimeEnvironment } from "@trigger.dev/database";
+import type { RuntimeEnvironment } from "@trigger.dev/database";
 import { prisma } from "~/db.server";
 import { customAlphabet } from "nanoid";
+import { RuntimeEnvironmentType } from "~/database-types";
 
 const apiKeyId = customAlphabet(
   "1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ",

apps/webapp/app/models/orgIntegration.server.ts

@@ -27,7 +27,7 @@ type SlackSecret = z.infer<typeof SlackSecretSchema>;
 
 const REDIRECT_AFTER_AUTH_KEY = "redirect-back-after-auth";
 
-type OrganizationIntegrationForService<TService extends IntegrationService> = Omit<
+export type OrganizationIntegrationForService<TService extends IntegrationService> = Omit<
   AuthenticatableIntegration,
   "service"
 > & {
@@ -83,7 +83,14 @@ export class OrgIntegrationRepository {
 
   static slackAuthorizationUrl(
     state: string,
-    scopes: string[] = ["channels:read", "groups:read", "im:read", "mpim:read", "chat:write"],
+    scopes: string[] = [
+      "channels:read",
+      "groups:read",
+      "im:read",
+      "mpim:read",
+      "chat:write",
+      "chat:write.public",
+    ],
     userScopes: string[] = ["channels:read", "groups:read", "im:read", "mpim:read", "chat:write"]
   ) {
     return `https://slack.com/oauth/v2/authorize?client_id=${

apps/webapp/app/models/organization.server.ts

@@ -21,16 +21,12 @@ export async function createOrganization(
   {
     title,
     userId,
-    projectName,
     companySize,
-    projectVersion,
   }: Pick<Organization, "title" | "companySize"> & {
     userId: User["id"];
-    projectName: string;
-    projectVersion: "v2" | "v3";
   },
   attemptCount = 0
-): Promise<Organization & { projects: Project[] }> {
+): Promise<Organization> {
   if (typeof process.env.BLOCKED_USERS === "string" && process.env.BLOCKED_USERS.includes(userId)) {
     throw new Error("Organization could not be created.");
   }
@@ -50,9 +46,7 @@ export async function createOrganization(
       {
         title,
         userId,
-        projectName,
         companySize,
-        projectVersion,
       },
       attemptCount + 1
     );
@@ -76,14 +70,7 @@ export async function createOrganization(
     },
   });
 
-  const project = await createProject({
-    organizationSlug: organization.slug,
-    name: projectName,
-    userId,
-    version: projectVersion,
-  });
-
-  return { ...organization, projects: [project] };
+  return { ...organization };
 }
 
 export async function createEnvironment(

apps/webapp/app/models/runtimeEnvironment.server.ts

@@ -124,6 +124,7 @@ type DisplayableInputEnvironment = Prisma.RuntimeEnvironmentGetPayload<{
   select: {
     id: true;
     type: true;
+    slug: true;
     orgMember: {
       select: {
         user: {
@@ -138,17 +139,24 @@ type DisplayableInputEnvironment = Prisma.RuntimeEnvironmentGetPayload<{
   };
 }>;
 
-export function displayableEnvironments(
+export function displayableEnvironment(
   environment: DisplayableInputEnvironment,
   userId: string | undefined
 ) {
+  let userName: string | undefined = undefined;
+
+  if (environment.type === "DEVELOPMENT") {
+    if (!environment.orgMember) {
+      userName = "Deleted";
+    } else if (environment.orgMember.user.id !== userId) {
+      userName = getUsername(environment.orgMember.user);
+    }
+  }
+
   return {
     id: environment.id,
     type: environment.type,
-    userName: environment.orgMember
-      ? environment.orgMember.user.id === userId
-        ? undefined
-        : getUsername(environment.orgMember.user)
-      : undefined,
+    slug: environment.slug,
+    userName,
   };
 }

apps/webapp/app/models/taskRun.server.ts

@@ -1,17 +1,20 @@
-import {
-  TaskRunError,
+import type {
   TaskRunExecutionResult,
   TaskRunFailedExecutionResult,
   TaskRunSuccessfulExecutionResult,
 } from "@trigger.dev/core/v3";
-import {
-  BatchTaskRunItemStatus,
+import { TaskRunError } from "@trigger.dev/core/v3";
+
+import type {
   TaskRun,
   TaskRunAttempt,
-  TaskRunAttemptStatus,
-  TaskRunStatus,
+  TaskRunAttemptStatus as TaskRunAttemptStatusType,
+  TaskRunStatus as TaskRunStatusType,
+  BatchTaskRunItemStatus as BatchTaskRunItemStatusType,
 } from "@trigger.dev/database";
+
 import { assertNever } from "assert-never";
+import { BatchTaskRunItemStatus, TaskRunAttemptStatus, TaskRunStatus } from "~/database-types";
 import { logger } from "~/services/logger.server";
 
 const SUCCESSFUL_STATUSES = [TaskRunStatus.COMPLETED_SUCCESSFULLY];
@@ -104,7 +107,9 @@ export function executionResultForTaskRun(
   }
 }
 
-export function batchTaskRunItemStatusForRunStatus(status: TaskRunStatus): BatchTaskRunItemStatus {
+export function batchTaskRunItemStatusForRunStatus(
+  status: TaskRunStatusType
+): BatchTaskRunItemStatusType {
   switch (status) {
     case TaskRunStatus.COMPLETED_SUCCESSFULLY:
       return BatchTaskRunItemStatus.COMPLETED;
@@ -113,7 +118,6 @@ export function batchTaskRunItemStatusForRunStatus(status: TaskRunStatus): Batch
     case TaskRunStatus.COMPLETED_WITH_ERRORS:
     case TaskRunStatus.SYSTEM_FAILURE:
     case TaskRunStatus.CRASHED:
-    case TaskRunStatus.COMPLETED_WITH_ERRORS:
       return BatchTaskRunItemStatus.FAILED;
     case TaskRunStatus.PENDING:
     case TaskRunStatus.WAITING_FOR_DEPLOY:

apps/webapp/app/platform/zodWorker.server.ts

@@ -10,7 +10,12 @@ import type {
   TaskSpec,
   WorkerUtils,
 } from "graphile-worker";
-import { run as graphileRun, makeWorkerUtils, parseCronItems } from "graphile-worker";
+import {
+  run as graphileRun,
+  makeWorkerUtils,
+  parseCronItems,
+  Logger as GraphileLogger,
+} from "graphile-worker";
 import { SpanKind, trace } from "@opentelemetry/api";
 
 import omit from "lodash.omit";
@@ -19,6 +24,7 @@ import { $replica, PrismaClient, PrismaClientOrTransaction } from "~/db.server";
 import { PgListenService } from "~/services/db/pgListen.server";
 import { workerLogger as logger } from "~/services/logger.server";
 import { flattenAttributes } from "@trigger.dev/core/v3";
+import { env } from "~/env.server";
 
 const tracer = trace.getTracer("zodWorker", "3.0.0.dp.1");
 
@@ -56,7 +62,6 @@ const AddJobResultsSchema = z.array(GraphileJobSchema);
 
 export type ZodTasks<TConsumerSchema extends MessageCatalogSchema> = {
   [K in keyof TConsumerSchema]: {
-    queueName?: string | ((payload: z.infer<TConsumerSchema[K]>) => string);
     jobKey?: string | ((payload: z.infer<TConsumerSchema[K]>) => string | undefined);
     priority?: number;
     maxAttempts?: number;
@@ -79,7 +84,9 @@ export type ZodRecurringTasks = {
   };
 };
 
-export type ZodWorkerEnqueueOptions = TaskSpec & {
+type ZodTaskSpec = Omit<TaskSpec, "queueName">;
+
+export type ZodWorkerEnqueueOptions = ZodTaskSpec & {
   tx?: PrismaClientOrTransaction;
 };
 
@@ -162,12 +169,25 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
 
     this.#workerUtils = await makeWorkerUtils(this.#runnerOptions);
 
+    const graphileLogger = new GraphileLogger((scope) => {
+      return (level, message, meta) => {
+        if (env.VERBOSE_GRAPHILE_LOGGING !== "true") return;
+
+        logger.debug(`[graphile-worker][${this.#name}][${level}] ${message}`, {
+          scope,
+          meta,
+          workerName: this.#name,
+        });
+      };
+    });
+
     this.#runner = await graphileRun({
       ...this.#runnerOptions,
       noHandleSignals: true,
       taskList: this.#createTaskListFromTasks(),
       parsedCronItems,
       forbiddenFlags: this.#rateLimiter?.forbiddenFlags.bind(this.#rateLimiter),
+      logger: graphileLogger,
     });
 
     if (!this.#runner) {
@@ -237,6 +257,20 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
       this.#logDebug("stop");
     });
 
+    this.#runner?.events.on("worker:getJob:error", ({ worker, error }) => {
+      this.#logDebug("worker:getJob:error", { workerId: worker.workerId, error });
+    });
+
+    this.#runner?.events.on("worker:getJob:start", ({ worker }) => {
+      if (env.VERBOSE_GRAPHILE_LOGGING !== "true") return;
+      this.#logDebug("worker:getJob:start", { workerId: worker.workerId });
+    });
+
+    this.#runner?.events.on("job:start", ({ worker, job }) => {
+      if (env.VERBOSE_GRAPHILE_LOGGING !== "true") return;
+      this.#logDebug("job:start", { workerId: worker.workerId, job });
+    });
+
     process.on("SIGTERM", this._handleSignal.bind(this));
     process.on("SIGINT", this._handleSignal.bind(this));
 
@@ -250,16 +284,18 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
 
     this.#shuttingDown = true;
 
+    this.#logDebug(
+      `Received ${signal}, shutting down zodWorker with timeout ${this.#shutdownTimeoutInMs}ms`
+    );
+
     if (this.#shutdownTimeoutInMs) {
       setTimeout(() => {
-        this.#logDebug("Shutdown timeout reached, exiting process");
+        this.#logDebug(`Shutdown timeout of ${this.#shutdownTimeoutInMs} reached, exiting process`);
 
         process.exit(0);
       }, this.#shutdownTimeoutInMs);
     }
 
-    this.#logDebug(`Received ${signal}, shutting down zodWorker...`);
-
     this.stop().finally(() => {
       this.#logDebug("zodWorker stopped");
     });
@@ -286,10 +322,6 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
       ...optionsWithoutTx,
     };
 
-    if (typeof task.queueName === "function") {
-      spec.queueName = task.queueName(payload);
-    }
-
     if (typeof task.jobKey === "function") {
       const jobKey = task.jobKey(payload);
 
@@ -345,17 +377,15 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
       `SELECT * FROM ${this.graphileWorkerSchema}.add_job(
           identifier => $1::text,
           payload => $2::json,
-          queue_name => $3::text,
-          run_at => $4::timestamptz,
-          max_attempts => $5::int,
-          job_key => $6::text,
-          priority => $7::int,
-          flags => $8::text[],
-          job_key_mode => $9::text
+          run_at => $3::timestamptz,
+          max_attempts => $4::int,
+          job_key => $5::text,
+          priority => $6::int,
+          flags => $7::text[],
+          job_key_mode => $8::text
         )`,
       identifier,
       JSON.stringify(payload),
-      spec.queueName || null,
       spec.runAt || null,
       spec.maxAttempts || null,
       spec.jobKey || null,
@@ -447,33 +477,15 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
     return taskList;
   }
 
-  async #getQueueName(queueId: number | null) {
-    if (queueId === null) {
-      return;
-    }
-
-    const schema = z.array(z.object({ queue_name: z.string() }));
-
-    const rawQueueNameResults = await $replica.$queryRawUnsafe(
-      `SELECT queue_name FROM ${this.graphileWorkerSchema}._private_job_queues WHERE id = $1`,
-      queueId
-    );
-
-    const queueNameResults = schema.parse(rawQueueNameResults);
-
-    return queueNameResults[0]?.queue_name;
-  }
-
   async #rescheduleTask(payload: unknown, helpers: JobHelpers) {
     this.#logDebug("Rescheduling task", { payload, job: helpers.job });
 
     await this.enqueue(helpers.job.task_identifier, payload, {
-      runAt: new Date(Date.now() + 1000 * 10),
-      queueName: await this.#getQueueName(helpers.job.job_queue_id),
+      runAt: helpers.job.run_at,
       priority: helpers.job.priority,
       jobKey: helpers.job.key ?? undefined,
       flags: Object.keys(helpers.job.flags ?? []),
-      maxAttempts: helpers.job.max_attempts,
+      maxAttempts: helpers.job.max_attempts - (helpers.job.attempts - 1),
     });
   }
 
@@ -577,6 +589,17 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
             span.recordException(new Error(String(error)));
           }
 
+          if (job.attempts >= job.max_attempts) {
+            logger.debug("Job failed after max attempts", {
+              job,
+              attempts: job.attempts,
+              max_attempts: job.max_attempts,
+              error: error instanceof Error ? error.message : error,
+            });
+
+            return;
+          }
+
           throw error;
         } finally {
           span.end();

apps/webapp/app/presenters/OrganizationsPresenter.server.ts

@@ -1,8 +1,8 @@
 import { PrismaClient } from "@trigger.dev/database";
 import { redirect } from "remix-typedjson";
 import { prisma } from "~/db.server";
+import { redirectWithErrorMessage } from "~/models/message.server";
 import {
-  clearCurrentProjectId,
   commitCurrentProjectSession,
   getCurrentProjectId,
   setCurrentProjectId,
@@ -10,8 +10,6 @@ import {
 import { logger } from "~/services/logger.server";
 import { newProjectPath } from "~/utils/pathBuilder";
 import { ProjectPresenter } from "./ProjectPresenter.server";
-import { redirectWithErrorMessage } from "~/models/message.server";
-import { match } from "assert";
 
 export class OrganizationsPresenter {
   #prismaClient: PrismaClient;
@@ -64,6 +62,10 @@ export class OrganizationsPresenter {
       );
     }
 
+    if (project.organizationId !== organization.id) {
+      throw redirect(newProjectPath({ slug: organizationSlug }), request);
+    }
+
     return { organizations, organization, project };
   }
 

apps/webapp/app/presenters/ProjectPresenter.server.ts

@@ -1,6 +1,6 @@
 import { PrismaClient, prisma } from "~/db.server";
 import { Project } from "~/models/project.server";
-import { displayableEnvironments } from "~/models/runtimeEnvironment.server";
+import { displayableEnvironment } from "~/models/runtimeEnvironment.server";
 import { User } from "~/models/user.server";
 import { sortEnvironments } from "~/utils/environmentSort";
 
@@ -86,7 +86,7 @@ export class ProjectPresenter {
       httpEndpointCount: project._count.httpEndpoints,
       environments: sortEnvironments(
         project.environments.map((environment) => ({
-          ...displayableEnvironments(environment, userId),
+          ...displayableEnvironment(environment, userId),
           userId: environment.orgMember?.user.id,
         }))
       ),

apps/webapp/app/presenters/v3/ApiAlertChannelPresenter.server.ts

@@ -16,6 +16,10 @@ export const ApiAlertType = z.enum(["attempt_failure", "deployment_failure", "de
 
 export type ApiAlertType = z.infer<typeof ApiAlertType>;
 
+export const ApiAlertEnvironmentType = z.enum(["STAGING", "PRODUCTION"]);
+
+export type ApiAlertEnvironmentType = z.infer<typeof ApiAlertEnvironmentType>;
+
 export const ApiAlertChannel = z.enum(["email", "webhook"]);
 
 export type ApiAlertChannel = z.infer<typeof ApiAlertChannel>;
@@ -34,6 +38,7 @@ export const ApiCreateAlertChannel = z.object({
   channel: ApiAlertChannel,
   channelData: ApiAlertChannelData,
   deduplicationKey: z.string().optional(),
+  environmentTypes: ApiAlertEnvironmentType.array().default(["STAGING", "PRODUCTION"]),
 });
 
 export type ApiCreateAlertChannel = z.infer<typeof ApiCreateAlertChannel>;

apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts

@@ -1,5 +1,15 @@
-import { AttemptStatus, RetrieveRunResponse, RunStatus, logger } from "@trigger.dev/core/v3";
-import { TaskRunAttemptStatus, TaskRunStatus } from "@trigger.dev/database";
+import {
+  AttemptStatus,
+  RetrieveRunResponse,
+  RunStatus,
+  SerializedError,
+  TaskRunError,
+  conditionallyImportPacket,
+  createJsonErrorObject,
+  logger,
+  parsePacket,
+} from "@trigger.dev/core/v3";
+import { Prisma, TaskRunAttemptStatus, TaskRunStatus } from "@trigger.dev/database";
 import assertNever from "assert-never";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { BasePresenter } from "./basePresenter.server";
@@ -23,6 +33,7 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
             },
           },
           lockedToVersion: true,
+          schedule: true,
         },
       });
 
@@ -32,14 +43,65 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
         return undefined;
       }
 
+      let $payload: any;
+      let $output: any;
+
+      if (showSecretDetails) {
+        const payloadPacket = await conditionallyImportPacket({
+          data: taskRun.payload,
+          dataType: taskRun.payloadType,
+        });
+
+        $payload = await parsePacket(payloadPacket);
+
+        if (taskRun.status === "COMPLETED_SUCCESSFULLY") {
+          const completedAttempt = taskRun.attempts.find(
+            (a) => a.status === "COMPLETED" && typeof a.output !== null
+          );
+
+          if (completedAttempt && completedAttempt.output) {
+            const outputPacket = await conditionallyImportPacket({
+              data: completedAttempt.output,
+              dataType: completedAttempt.outputType,
+            });
+
+            $output = await parsePacket(outputPacket);
+          }
+        }
+      }
+
+      const apiStatus = ApiRetrieveRunPresenter.apiStatusFromRunStatus(taskRun.status);
+
       return {
         id: taskRun.friendlyId,
-        status: ApiRetrieveRunPresenter.apiStatusFromRunStatus(taskRun.status),
+        status: apiStatus,
         taskIdentifier: taskRun.taskIdentifier,
         idempotencyKey: taskRun.idempotencyKey ?? undefined,
         version: taskRun.lockedToVersion ? taskRun.lockedToVersion.version : undefined,
         createdAt: taskRun.createdAt ?? undefined,
         updatedAt: taskRun.updatedAt ?? undefined,
+        startedAt: taskRun.lockedAt ?? undefined,
+        finishedAt: ApiRetrieveRunPresenter.isStatusFinished(apiStatus)
+          ? taskRun.updatedAt
+          : undefined,
+        payload: $payload,
+        output: $output,
+        isTest: taskRun.isTest,
+        schedule: taskRun.schedule
+          ? {
+              id: taskRun.schedule.friendlyId,
+              externalId: taskRun.schedule.externalId ?? undefined,
+              deduplicationKey: taskRun.schedule.userProvidedDeduplicationKey
+                ? taskRun.schedule.deduplicationKey
+                : undefined,
+              generator: {
+                type: "CRON",
+                expression: taskRun.schedule.generatorExpression,
+                description: taskRun.schedule.generatorDescription,
+              },
+            }
+          : undefined,
+        ...ApiRetrieveRunPresenter.apiBooleanHelpersFromRunStatus(apiStatus),
         attempts: !showSecretDetails
           ? []
           : taskRun.attempts.map((a) => ({
@@ -49,34 +111,68 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
               updatedAt: a.updatedAt ?? undefined,
               startedAt: a.startedAt ?? undefined,
               completedAt: a.completedAt ?? undefined,
+              error: ApiRetrieveRunPresenter.apiErrorFromError(a.error),
             })),
       };
     });
   }
 
+  static apiErrorFromError(error: Prisma.JsonValue): SerializedError | undefined {
+    if (!error) {
+      return;
+    }
+
+    const errorData = TaskRunError.safeParse(error);
+
+    if (errorData.success) {
+      return createJsonErrorObject(errorData.data);
+    }
+  }
+
+  static isStatusFinished(status: RunStatus) {
+    return (
+      status === "COMPLETED" ||
+      status === "FAILED" ||
+      status === "CANCELED" ||
+      status === "INTERRUPTED" ||
+      status === "CRASHED" ||
+      status === "SYSTEM_FAILURE"
+    );
+  }
+
   static apiStatusFromRunStatus(status: TaskRunStatus): RunStatus {
     switch (status) {
-      case "WAITING_FOR_DEPLOY":
-      case "PENDING": {
-        return "PENDING";
+      case "WAITING_FOR_DEPLOY": {
+        return "WAITING_FOR_DEPLOY";
+      }
+      case "PENDING": {
+        return "QUEUED";
+      }
+      case "PAUSED":
+      case "WAITING_TO_RESUME": {
+        return "FROZEN";
+      }
+      case "RETRYING_AFTER_FAILURE": {
+        return "REATTEMPTING";
       }
-      case "RETRYING_AFTER_FAILURE":
       case "EXECUTING": {
         return "EXECUTING";
       }
-      case "WAITING_TO_RESUME":
-      case "PAUSED": {
-        return "PAUSED";
-      }
       case "CANCELED": {
         return "CANCELED";
       }
       case "COMPLETED_SUCCESSFULLY": {
         return "COMPLETED";
       }
-      case "SYSTEM_FAILURE":
-      case "INTERRUPTED":
-      case "CRASHED":
+      case "SYSTEM_FAILURE": {
+        return "SYSTEM_FAILURE";
+      }
+      case "INTERRUPTED": {
+        return "INTERRUPTED";
+      }
+      case "CRASHED": {
+        return "CRASHED";
+      }
       case "COMPLETED_WITH_ERRORS": {
         return "FAILED";
       }
@@ -86,6 +182,30 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
     }
   }
 
+  static apiBooleanHelpersFromRunStatus(status: RunStatus) {
+    const isQueued = status === "QUEUED" || status === "WAITING_FOR_DEPLOY";
+    const isExecuting = status === "EXECUTING" || status === "REATTEMPTING" || status === "FROZEN";
+    const isCompleted =
+      status === "COMPLETED" ||
+      status === "CANCELED" ||
+      status === "FAILED" ||
+      status === "CRASHED" ||
+      status === "INTERRUPTED" ||
+      status === "SYSTEM_FAILURE";
+    const isFailed = isCompleted && status !== "COMPLETED";
+    const isSuccess = isCompleted && status === "COMPLETED";
+    const isCancelled = status === "CANCELED";
+
+    return {
+      isQueued,
+      isExecuting,
+      isCompleted,
+      isFailed,
+      isSuccess,
+      isCancelled,
+    };
+  }
+
   static apiStatusFromAttemptStatus(status: TaskRunAttemptStatus): AttemptStatus {
     switch (status) {
       case "PENDING": {

apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts

@@ -0,0 +1,274 @@
+import { ListRunResponse, ListRunResponseItem, RunStatus } from "@trigger.dev/core/v3";
+import { Project, RuntimeEnvironment, TaskRunStatus } from "@trigger.dev/database";
+import assertNever from "assert-never";
+import { z } from "zod";
+import { fromZodError } from "zod-validation-error";
+import { logger } from "~/services/logger.server";
+import { ApiRetrieveRunPresenter } from "./ApiRetrieveRunPresenter.server";
+import { RunListOptions, RunListPresenter } from "./RunListPresenter.server";
+import { BasePresenter } from "./basePresenter.server";
+
+const SearchParamsSchema = z.object({
+  "page[size]": z.coerce.number().int().positive().min(1).max(100).optional(),
+  "page[after]": z.string().optional(),
+  "page[before]": z.string().optional(),
+  "filter[status]": z
+    .string()
+    .optional()
+    .transform((value, ctx) => {
+      if (!value) {
+        return undefined;
+      }
+
+      const statuses = value.split(",");
+      const parsedStatuses = statuses.map((status) => RunStatus.safeParse(status));
+
+      if (parsedStatuses.some((result) => !result.success)) {
+        const invalidStatuses: string[] = [];
+
+        for (const [index, result] of parsedStatuses.entries()) {
+          if (!result.success) {
+            invalidStatuses.push(statuses[index]);
+          }
+        }
+
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid status values: ${invalidStatuses.join(", ")}`,
+        });
+
+        return z.NEVER;
+      }
+
+      const $statuses = parsedStatuses
+        .map((result) => (result.success ? result.data : undefined))
+        .filter(Boolean);
+
+      return Array.from(new Set($statuses));
+    }),
+  "filter[env]": z
+    .string()
+    .optional()
+    .transform((value) => {
+      return value ? value.split(",") : undefined;
+    }),
+  "filter[taskIdentifier]": z
+    .string()
+    .optional()
+    .transform((value) => {
+      return value ? value.split(",") : undefined;
+    }),
+  "filter[version]": z
+    .string()
+    .optional()
+    .transform((value) => {
+      return value ? value.split(",") : undefined;
+    }),
+  "filter[bulkAction]": z.string().optional(),
+  "filter[schedule]": z.string().optional(),
+  "filter[isTest]": z
+    .string()
+    .optional()
+    .transform((value, ctx) => {
+      if (!value) {
+        return undefined;
+      }
+
+      if (value === "true") {
+        return true;
+      }
+
+      if (value === "false") {
+        return false;
+      }
+
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Invalid value for isTest: ${value}`,
+      });
+
+      return z.NEVER;
+    }),
+  "filter[createdAt][from]": z.coerce.date().optional(),
+  "filter[createdAt][to]": z.coerce.date().optional(),
+  "filter[createdAt][period]": z.string().optional(),
+});
+
+type SearchParamsSchema = z.infer<typeof SearchParamsSchema>;
+
+export class ApiRunListPresenter extends BasePresenter {
+  public async call(
+    project: Project,
+    searchParams: URLSearchParams,
+    environment?: RuntimeEnvironment
+  ): Promise<ListRunResponse> {
+    return this.trace("call", async (span) => {
+      const rawSearchParams = Object.fromEntries(searchParams.entries());
+      const $searchParams = SearchParamsSchema.safeParse(rawSearchParams);
+
+      if (!$searchParams.success) {
+        logger.error("Invalid search params", {
+          searchParams: rawSearchParams,
+          errors: $searchParams.error.errors,
+        });
+
+        throw fromZodError($searchParams.error);
+      }
+
+      logger.debug("Valid search params", { searchParams: $searchParams.data });
+
+      const options: RunListOptions = {
+        projectId: project.id,
+      };
+
+      // pagination
+      if ($searchParams.data["page[size]"]) {
+        options.pageSize = $searchParams.data["page[size]"];
+      }
+
+      if ($searchParams.data["page[after]"]) {
+        options.cursor = $searchParams.data["page[after]"];
+        options.direction = "forward";
+      }
+
+      if ($searchParams.data["page[before]"]) {
+        options.cursor = $searchParams.data["page[before]"];
+        options.direction = "backward";
+      }
+
+      // filters
+      if (environment) {
+        options.environments = [environment.id];
+      } else {
+        if ($searchParams.data["filter[env]"]) {
+          const environments = await this._prisma.runtimeEnvironment.findMany({
+            where: {
+              projectId: project.id,
+              slug: {
+                in: $searchParams.data["filter[env]"],
+              },
+            },
+          });
+
+          options.environments = environments.map((env) => env.id);
+        }
+      }
+
+      if ($searchParams.data["filter[status]"]) {
+        options.statuses = $searchParams.data["filter[status]"].flatMap((status) =>
+          ApiRunListPresenter.apiStatusToRunStatuses(status)
+        );
+      }
+
+      if ($searchParams.data["filter[taskIdentifier]"]) {
+        options.tasks = $searchParams.data["filter[taskIdentifier]"];
+      }
+
+      if ($searchParams.data["filter[version]"]) {
+        options.versions = $searchParams.data["filter[version]"];
+      }
+
+      if ($searchParams.data["filter[bulkAction]"]) {
+        options.bulkId = $searchParams.data["filter[bulkAction]"];
+      }
+
+      if ($searchParams.data["filter[schedule]"]) {
+        options.scheduleId = $searchParams.data["filter[schedule]"];
+      }
+
+      if ($searchParams.data["filter[createdAt][from]"]) {
+        options.from = $searchParams.data["filter[createdAt][from]"].getTime();
+      }
+
+      if ($searchParams.data["filter[createdAt][to]"]) {
+        options.to = $searchParams.data["filter[createdAt][to]"].getTime();
+      }
+
+      if ($searchParams.data["filter[createdAt][period]"]) {
+        options.period = $searchParams.data["filter[createdAt][period]"];
+      }
+
+      if (typeof $searchParams.data["filter[isTest]"] === "boolean") {
+        options.isTest = $searchParams.data["filter[isTest]"];
+      }
+
+      const presenter = new RunListPresenter();
+
+      logger.debug("Calling RunListPresenter", { options });
+
+      const results = await presenter.call(options);
+
+      const data: ListRunResponseItem[] = results.runs.map((run) => {
+        return {
+          id: run.friendlyId,
+          status: ApiRetrieveRunPresenter.apiStatusFromRunStatus(run.status),
+          taskIdentifier: run.taskIdentifier,
+          idempotencyKey: run.idempotencyKey,
+          version: run.version ?? undefined,
+          createdAt: new Date(run.createdAt),
+          updatedAt: new Date(run.updatedAt),
+          startedAt: run.startedAt ? new Date(run.startedAt) : undefined,
+          finishedAt: run.finishedAt ? new Date(run.finishedAt) : undefined,
+          isTest: run.isTest,
+          env: {
+            id: run.environment.id,
+            name: run.environment.slug,
+            user: run.environment.userName,
+          },
+          ...ApiRetrieveRunPresenter.apiBooleanHelpersFromRunStatus(
+            ApiRetrieveRunPresenter.apiStatusFromRunStatus(run.status)
+          ),
+        };
+      });
+
+      return {
+        data,
+        pagination: {
+          next: results.pagination.next,
+          previous: results.pagination.previous,
+        },
+      };
+    });
+  }
+
+  static apiStatusToRunStatuses(status: RunStatus): TaskRunStatus[] | TaskRunStatus {
+    switch (status) {
+      case "WAITING_FOR_DEPLOY": {
+        return "WAITING_FOR_DEPLOY";
+      }
+      case "QUEUED": {
+        return "PENDING";
+      }
+      case "EXECUTING": {
+        return "EXECUTING";
+      }
+      case "REATTEMPTING": {
+        return "RETRYING_AFTER_FAILURE";
+      }
+      case "FROZEN": {
+        return ["PAUSED", "WAITING_TO_RESUME"];
+      }
+      case "CANCELED": {
+        return "CANCELED";
+      }
+      case "COMPLETED": {
+        return "COMPLETED_SUCCESSFULLY";
+      }
+      case "SYSTEM_FAILURE": {
+        return "SYSTEM_FAILURE";
+      }
+      case "INTERRUPTED": {
+        return "INTERRUPTED";
+      }
+      case "CRASHED": {
+        return "CRASHED";
+      }
+      case "FAILED": {
+        return "COMPLETED_WITH_ERRORS";
+      }
+      default: {
+        assertNever(status);
+      }
+    }
+  }
+}

apps/webapp/app/presenters/v3/EditSchedulePresenter.server.ts

@@ -1,5 +1,6 @@
 import { RuntimeEnvironmentType } from "@trigger.dev/database";
 import { PrismaClient, prisma } from "~/db.server";
+import { displayableEnvironment } from "~/models/runtimeEnvironment.server";
 
 type EditScheduleOptions = {
   userId: string;
@@ -67,19 +68,7 @@ export class EditSchedulePresenter {
     });
 
     const possibleEnvironments = project.environments.map((environment) => {
-      let userName: undefined | string;
-      if (environment.orgMember) {
-        if (environment.orgMember.user.id !== userId) {
-          userName =
-            environment.orgMember.user.displayName ?? environment.orgMember.user.name ?? undefined;
-        }
-      }
-
-      return {
-        id: environment.id,
-        type: environment.type,
-        userName,
-      };
+      return displayableEnvironment(environment, userId);
     });
 
     return {

apps/webapp/app/presenters/v3/EnvironmentVariablesPresenter.server.ts

@@ -79,6 +79,19 @@ export class EnvironmentVariablesPresenter {
         project: {
           slug: projectSlug,
         },
+        OR: [
+          {
+            type: {
+              in: ["PREVIEW", "STAGING", "PRODUCTION"],
+            },
+          },
+          {
+            type: "DEVELOPMENT",
+            orgMember: {
+              userId,
+            },
+          },
+        ],
       },
     });
 
@@ -87,7 +100,7 @@ export class EnvironmentVariablesPresenter {
     );
 
     const repository = new EnvironmentVariablesRepository(this.#prismaClient);
-    const variables = await repository.getProject(project.id, userId);
+    const variables = await repository.getProject(project.id);
 
     return {
       environmentVariables: environmentVariables.map((environmentVariable) => {

apps/webapp/app/presenters/v3/RunListPresenter.server.ts

@@ -3,13 +3,13 @@ import parse from "parse-duration";
 import { Direction } from "~/components/runs/RunStatuses";
 import { FINISHED_STATUSES } from "~/components/runs/v3/TaskRunStatus";
 import { sqlDatabaseSchema } from "~/db.server";
-import { displayableEnvironments } from "~/models/runtimeEnvironment.server";
+import { displayableEnvironment } from "~/models/runtimeEnvironment.server";
 import { CANCELLABLE_STATUSES } from "~/v3/services/cancelTaskRun.server";
 import { BasePresenter } from "./basePresenter.server";
 
-type RunListOptions = {
+export type RunListOptions = {
   userId?: string;
-  projectSlug: string;
+  projectId: string;
   //filters
   tasks?: string[];
   versions?: string[];
@@ -17,15 +17,17 @@ type RunListOptions = {
   environments?: string[];
   scheduleId?: string;
   period?: string;
+  bulkId?: string;
   from?: number;
   to?: number;
+  isTest?: boolean;
   //pagination
   direction?: Direction;
   cursor?: string;
   pageSize?: number;
 };
 
-const DEFAULT_PAGE_SIZE = 20;
+const DEFAULT_PAGE_SIZE = 25;
 
 export type RunList = Awaited<ReturnType<RunListPresenter["call"]>>;
 export type RunListItem = RunList["runs"][0];
@@ -34,13 +36,15 @@ export type RunListAppliedFilters = RunList["filters"];
 export class RunListPresenter extends BasePresenter {
   public async call({
     userId,
-    projectSlug,
+    projectId,
     tasks,
     versions,
     statuses,
     environments,
     scheduleId,
     period,
+    bulkId,
+    isTest,
     from,
     to,
     direction = "forward",
@@ -55,8 +59,11 @@ export class RunListPresenter extends BasePresenter {
       hasStatusFilters ||
       (environments !== undefined && environments.length > 0) ||
       (period !== undefined && period !== "all") ||
+      (bulkId !== undefined && bulkId !== "") ||
       from !== undefined ||
-      to !== undefined;
+      to !== undefined ||
+      (scheduleId !== undefined && scheduleId !== "") ||
+      typeof isTest === "boolean";
 
     // Find the project scoped to the organization
     const project = await this._replica.project.findFirstOrThrow({
@@ -82,18 +89,60 @@ export class RunListPresenter extends BasePresenter {
         },
       },
       where: {
-        slug: projectSlug,
+        id: projectId,
       },
     });
 
     //get all possible tasks
-    const possibleTasks = await this._replica.backgroundWorkerTask.findMany({
+    const possibleTasksAsync = this._replica.backgroundWorkerTask.findMany({
       distinct: ["slug"],
       where: {
         projectId: project.id,
       },
     });
 
+    //get possible bulk actions
+    const bulkActionsAsync = this._replica.bulkActionGroup.findMany({
+      select: {
+        friendlyId: true,
+        type: true,
+        createdAt: true,
+      },
+      where: {
+        projectId: project.id,
+      },
+      orderBy: {
+        createdAt: "desc",
+      },
+      take: 20,
+    });
+
+    const [possibleTasks, bulkActions] = await Promise.all([possibleTasksAsync, bulkActionsAsync]);
+
+    //we can restrict to specific runs using bulkId, or batchId
+    let restrictToRunIds: undefined | string[] = undefined;
+
+    //bulk id
+    if (bulkId) {
+      const bulkAction = await this._replica.bulkActionGroup.findUnique({
+        select: {
+          items: {
+            select: {
+              destinationRunId: true,
+            },
+          },
+        },
+        where: {
+          friendlyId: bulkId,
+        },
+      });
+
+      if (bulkAction) {
+        const runIds = bulkAction.items.map((item) => item.destinationRunId).filter(Boolean);
+        restrictToRunIds = runIds;
+      }
+    }
+
     const periodMs = period ? parse(period) : undefined;
 
     //get the runs
@@ -111,6 +160,7 @@ export class RunListPresenter extends BasePresenter {
         updatedAt: Date;
         isTest: boolean;
         spanId: string;
+        idempotencyKey: string | null;
       }[]
     >`
     SELECT
@@ -125,7 +175,8 @@ export class RunListPresenter extends BasePresenter {
     tr."lockedAt" AS "lockedAt",
     tr."updatedAt" AS "updatedAt",
     tr."isTest" AS "isTest",
-    tr."spanId" AS "spanId"
+    tr."spanId" AS "spanId",
+    tr."idempotencyKey" AS "idempotencyKey"
   FROM
     ${sqlDatabaseSchema}."TaskRun" tr
   LEFT JOIN
@@ -142,6 +193,13 @@ export class RunListPresenter extends BasePresenter {
           : Prisma.empty
       }
       -- filters
+      ${
+        restrictToRunIds
+          ? restrictToRunIds.length === 0
+            ? Prisma.sql`AND tr.id = ''`
+            : Prisma.sql`AND tr.id IN (${Prisma.join(restrictToRunIds)})`
+          : Prisma.empty
+      }
       ${
         tasks && tasks.length > 0
           ? Prisma.sql`AND tr."taskIdentifier" IN (${Prisma.join(tasks)})`
@@ -158,6 +216,7 @@ export class RunListPresenter extends BasePresenter {
           : Prisma.empty
       }
       ${scheduleId ? Prisma.sql`AND tr."scheduleId" = ${scheduleId}` : Prisma.empty}
+      ${typeof isTest === "boolean" ? Prisma.sql`AND tr."isTest" = ${isTest}` : Prisma.empty}
       ${
         periodMs
           ? Prisma.sql`AND tr."createdAt" >= NOW() - INTERVAL '1 millisecond' * ${periodMs}`
@@ -218,6 +277,7 @@ export class RunListPresenter extends BasePresenter {
           friendlyId: run.runFriendlyId,
           number: Number(run.number),
           createdAt: run.createdAt.toISOString(),
+          updatedAt: run.updatedAt.toISOString(),
           startedAt: run.lockedAt ? run.lockedAt.toISOString() : undefined,
           hasFinished,
           finishedAt: hasFinished ? run.updatedAt.toISOString() : undefined,
@@ -228,7 +288,8 @@ export class RunListPresenter extends BasePresenter {
           spanId: run.spanId,
           isReplayable: true,
           isCancellable: CANCELLABLE_STATUSES.includes(run.status),
-          environment: displayableEnvironments(environment, userId),
+          environment: displayableEnvironment(environment, userId),
+          idempotencyKey: run.idempotencyKey ? run.idempotencyKey : undefined,
         };
       }),
       pagination: {
@@ -240,6 +301,11 @@ export class RunListPresenter extends BasePresenter {
         .sort((a, b) => {
           return a.slug.localeCompare(b.slug);
         }),
+      bulkActions: bulkActions.map((bulkAction) => ({
+        id: bulkAction.friendlyId,
+        type: bulkAction.type,
+        createdAt: bulkAction.createdAt,
+      })),
       filters: {
         tasks: tasks || [],
         versions: versions || [],

apps/webapp/app/presenters/v3/ScheduleListPresenter.server.ts

@@ -1,6 +1,7 @@
 import { Prisma, RuntimeEnvironmentType } from "@trigger.dev/database";
 import { ScheduleListFilters } from "~/components/runs/v3/ScheduleFilters";
 import { PrismaClient, prisma, sqlDatabaseSchema } from "~/db.server";
+import { displayableEnvironment } from "~/models/runtimeEnvironment.server";
 import { getUsername } from "~/utils/username";
 import { calculateNextScheduledTimestamp } from "~/v3/utils/calculateNextSchedule.server";
 
@@ -233,14 +234,7 @@ export class ScheduleListPresenter {
             );
           }
 
-          return {
-            id: instance.environmentId,
-            type: environment.type,
-            userName:
-              environment.orgMember?.user.id === userId
-                ? undefined
-                : getUsername(environment.orgMember?.user),
-          };
+          return displayableEnvironment(environment, userId);
         }),
       };
     });
@@ -252,14 +246,7 @@ export class ScheduleListPresenter {
       schedules,
       possibleTasks: possibleTasks.map((task) => task.slug),
       possibleEnvironments: project.environments.map((environment) => {
-        return {
-          id: environment.id,
-          type: environment.type,
-          userName:
-            environment.orgMember?.user.id === userId
-              ? undefined
-              : getUsername(environment.orgMember?.user),
-        };
+        return displayableEnvironment(environment, userId);
       }),
       hasFilters,
       filters: {

apps/webapp/app/presenters/v3/SpanPresenter.server.ts

@@ -1,29 +1,26 @@
 import { prettyPrintPacket } from "@trigger.dev/core/v3";
 import { PrismaClient, prisma } from "~/db.server";
 import { eventRepository } from "~/v3/eventRepository.server";
+import { BasePresenter } from "./basePresenter.server";
 
 type Result = Awaited<ReturnType<SpanPresenter["call"]>>;
 export type Span = NonNullable<Result>["event"];
 
-export class SpanPresenter {
-  #prismaClient: PrismaClient;
-
-  constructor(prismaClient: PrismaClient = prisma) {
-    this.#prismaClient = prismaClient;
-  }
-
+export class SpanPresenter extends BasePresenter {
   public async call({
     userId,
     projectSlug,
     organizationSlug,
     spanId,
+    runFriendlyId,
   }: {
     userId: string;
     projectSlug: string;
     organizationSlug: string;
     spanId: string;
+    runFriendlyId: string;
   }) {
-    const project = await this.#prismaClient.project.findUnique({
+    const project = await this._replica.project.findUnique({
       where: {
         slug: projectSlug,
       },
@@ -33,7 +30,20 @@ export class SpanPresenter {
       throw new Error("Project not found");
     }
 
-    const span = await eventRepository.getSpan(spanId);
+    const run = await this._prisma.taskRun.findFirst({
+      select: {
+        traceId: true,
+      },
+      where: {
+        friendlyId: runFriendlyId,
+      },
+    });
+
+    if (!run) {
+      return;
+    }
+
+    const span = await eventRepository.getSpan(spanId, run.traceId);
 
     if (!span) {
       return;

apps/webapp/app/presenters/v3/TaskListPresenter.server.ts

@@ -1,18 +1,19 @@
-import {
-  Prisma,
+import type {
   RuntimeEnvironmentType,
-  TaskRunStatus,
   TaskTriggerSource,
+  TaskRunStatus as TaskRunStatusType,
 } from "@trigger.dev/database";
+import { Prisma } from "@trigger.dev/database";
 import { QUEUED_STATUSES, RUNNING_STATUSES } from "~/components/runs/v3/TaskRunStatus";
 import { sqlDatabaseSchema } from "~/db.server";
-import { Organization } from "~/models/organization.server";
-import { Project } from "~/models/project.server";
-import { displayableEnvironments } from "~/models/runtimeEnvironment.server";
-import { User } from "~/models/user.server";
-import { sortEnvironments } from "~/utils/environmentSort";
+import type { Organization } from "~/models/organization.server";
+import type { Project } from "~/models/project.server";
+import { displayableEnvironment } from "~/models/runtimeEnvironment.server";
+import type { User } from "~/models/user.server";
+import { filterOrphanedEnvironments, sortEnvironments } from "~/utils/environmentSort";
 import { logger } from "~/services/logger.server";
 import { BasePresenter } from "./basePresenter.server";
+import { TaskRunStatus } from "~/database-types";
 
 export type Task = {
   slug: string;
@@ -85,7 +86,9 @@ export class TaskListPresenter extends BasePresenter {
     WITH workers AS (
       SELECT DISTINCT ON ("runtimeEnvironmentId") id, "runtimeEnvironmentId", version
       FROM ${sqlDatabaseSchema}."BackgroundWorker"
-      WHERE "runtimeEnvironmentId" IN (${Prisma.join(project.environments.map((e) => e.id))})
+      WHERE "runtimeEnvironmentId" IN (${Prisma.join(
+        filterOrphanedEnvironments(project.environments).map((e) => e.id)
+      )})
       ORDER BY "runtimeEnvironmentId", "createdAt" DESC
     )
     SELECT tasks.id, slug, "filePath", "exportName", "triggerSource", tasks."runtimeEnvironmentId", tasks."createdAt"
@@ -110,7 +113,15 @@ export class TaskListPresenter extends BasePresenter {
         acc.push(existingTask);
       }
 
-      existingTask.environments.push(displayableEnvironments(environment, userId));
+      //favour newer tasks
+      if (task.createdAt > existingTask.createdAt) {
+        existingTask.createdAt = task.createdAt;
+        existingTask.exportName = task.exportName;
+        existingTask.filePath = task.filePath;
+        existingTask.triggerSource = task.triggerSource;
+      }
+
+      existingTask.environments.push(displayableEnvironment(environment, userId));
 
       //order the environments
       existingTask.environments = sortEnvironments(existingTask.environments);
@@ -150,7 +161,7 @@ export class TaskListPresenter extends BasePresenter {
     const activity = await this._replica.$queryRaw<
       {
         taskIdentifier: string;
-        status: TaskRunStatus;
+        status: TaskRunStatusType;
         day: Date;
         count: BigInt;
       }[]
@@ -193,7 +204,7 @@ export class TaskListPresenter extends BasePresenter {
           existingTask.push({
             day: day.toISOString(),
             [TaskRunStatus.COMPLETED_SUCCESSFULLY]: 0,
-          } as { day: string } & Record<TaskRunStatus, number>);
+          } as { day: string } & Record<TaskRunStatusType, number>);
         }
 
         acc[a.taskIdentifier] = existingTask;
@@ -214,7 +225,7 @@ export class TaskListPresenter extends BasePresenter {
       day[a.status] = Number(a.count);
 
       return acc;
-    }, {} as Record<string, ({ day: string } & Record<TaskRunStatus, number>)[]>);
+    }, {} as Record<string, ({ day: string } & Record<TaskRunStatusType, number>)[]>);
   }
 
   async #getRunningStats(tasks: string[], projectId: string) {
@@ -225,7 +236,7 @@ export class TaskListPresenter extends BasePresenter {
     const statuses = await this._replica.$queryRaw<
       {
         taskIdentifier: string;
-        status: TaskRunStatus;
+        status: TaskRunStatusType;
         count: BigInt;
       }[]
     >`

apps/webapp/app/presenters/v3/TestPresenter.server.ts

@@ -36,9 +36,12 @@ export class TestPresenter {
           where: {
             OR: [
               {
-                orgMember: null,
+                type: {
+                  in: ["PREVIEW", "STAGING", "PRODUCTION"],
+                },
               },
               {
+                type: "DEVELOPMENT",
                 orgMember: {
                   userId,
                 },

apps/webapp/app/presenters/v3/ViewSchedulePresenter.server.ts

@@ -2,6 +2,7 @@ import { PrismaClient, prisma } from "~/db.server";
 import { nextScheduledTimestamps } from "~/v3/utils/calculateNextSchedule.server";
 import { RunListPresenter } from "./RunListPresenter.server";
 import { ScheduleObject } from "@trigger.dev/core/v3";
+import { displayableEnvironment } from "~/models/runtimeEnvironment.server";
 
 type ViewScheduleOptions = {
   userId?: string;
@@ -29,7 +30,7 @@ export class ViewSchedulePresenter {
         taskIdentifier: true,
         project: {
           select: {
-            slug: true,
+            id: true,
           },
         },
         instances: {
@@ -38,6 +39,7 @@ export class ViewSchedulePresenter {
               select: {
                 id: true,
                 type: true,
+                slug: true,
                 orgMember: {
                   select: {
                     user: {
@@ -70,8 +72,9 @@ export class ViewSchedulePresenter {
       : [];
 
     const runPresenter = new RunListPresenter(this.#prismaClient);
+
     const { runs } = await runPresenter.call({
-      projectSlug: schedule.project.slug,
+      projectId: schedule.project.id,
       scheduleId: schedule.id,
       pageSize: 5,
     });
@@ -85,21 +88,7 @@ export class ViewSchedulePresenter {
         runs,
         environments: schedule.instances.map((instance) => {
           const environment = instance.environment;
-          let userName: undefined | string;
-          if (environment.orgMember) {
-            if (environment.orgMember.user.id !== userId) {
-              userName =
-                environment.orgMember.user.displayName ??
-                environment.orgMember.user.name ??
-                undefined;
-            }
-          }
-
-          return {
-            id: environment.id,
-            type: environment.type,
-            userName,
-          };
+          return displayableEnvironment(environment, userId);
         }),
       },
     };

apps/webapp/app/presenters/v3/basePresenter.server.ts

@@ -34,4 +34,26 @@ export abstract class BasePresenter {
       }
     );
   }
+
+  protected async trace<T>(trace: string, fn: (span: Span) => Promise<T>): Promise<T> {
+    return tracer.startActiveSpan(
+      `${this.constructor.name}.${trace}`,
+      { kind: SpanKind.SERVER },
+      async (span) => {
+        try {
+          return await fn(span);
+        } catch (e) {
+          if (e instanceof Error) {
+            span.recordException(e);
+          } else {
+            span.recordException(new Error(String(e)));
+          }
+
+          throw e;
+        } finally {
+          span.end();
+        }
+      }
+    );
+  }
 }

apps/webapp/app/routes/_app._index/route.tsx

@@ -1,8 +1,9 @@
 import { LoaderFunctionArgs, redirect } from "@remix-run/server-runtime";
+import { prisma } from "~/db.server";
 import { getUsersInvites } from "~/models/member.server";
 import { SelectBestProjectPresenter } from "~/presenters/SelectBestProjectPresenter.server";
 import { requireUser } from "~/services/session.server";
-import { invitesPath, newOrganizationPath, projectPath } from "~/utils/pathBuilder";
+import { invitesPath, newOrganizationPath, newProjectPath, projectPath } from "~/utils/pathBuilder";
 
 //this loader chooses the best project to redirect you to, ideally based on the cookie
 export const loader = async ({ request }: LoaderFunctionArgs) => {
@@ -20,6 +21,24 @@ export const loader = async ({ request }: LoaderFunctionArgs) => {
     //redirect them to the most appropriate project
     return redirect(projectPath(organization, project));
   } catch (e) {
+    const organization = await prisma.organization.findFirst({
+      where: {
+        members: {
+          some: {
+            userId: user.id,
+          },
+        },
+        deletedAt: null,
+      },
+      orderBy: {
+        createdAt: "desc",
+      },
+    });
+
+    if (organization) {
+      return redirect(newProjectPath(organization));
+    }
+
     //this should only happen if the user has no projects, and no invites
     return redirect(newOrganizationPath());
   }

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.alerts.new/route.tsx

@@ -10,8 +10,8 @@ import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { z } from "zod";
 import { InlineCode } from "~/components/code/InlineCode";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
-import { Callout } from "~/components/primitives/Callout";
-import { Checkbox } from "~/components/primitives/Checkbox";
+import { Callout, variantClasses } from "~/components/primitives/Callout";
+import { CheckboxWithLabel } from "~/components/primitives/Checkbox";
 import { Dialog, DialogContent, DialogHeader } from "~/components/primitives/Dialog";
 import { Fieldset } from "~/components/primitives/Fieldset";
 import { FormButtons } from "~/components/primitives/FormButtons";
@@ -20,14 +20,17 @@ import { Hint } from "~/components/primitives/Hint";
 import { Input } from "~/components/primitives/Input";
 import { InputGroup } from "~/components/primitives/InputGroup";
 import { Label } from "~/components/primitives/Label";
+import { Paragraph } from "~/components/primitives/Paragraph";
 import SegmentedControl from "~/components/primitives/SegmentedControl";
 import { Select, SelectItem } from "~/components/primitives/Select";
+import { InfoIconTooltip } from "~/components/primitives/Tooltip";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
 import { redirectWithSuccessMessage } from "~/models/message.server";
 import { findProjectBySlug } from "~/models/project.server";
 import { NewAlertChannelPresenter } from "~/presenters/v3/NewAlertChannelPresenter.server";
 import { requireUserId } from "~/services/session.server";
+import { cn } from "~/utils/cn";
 import { ProjectParamSchema, v3ProjectAlertsPath } from "~/utils/pathBuilder";
 import {
   CreateAlertChannelOptions,
@@ -40,6 +43,10 @@ const FormSchema = z
       .array(z.enum(["TASK_RUN_ATTEMPT", "DEPLOYMENT_FAILURE", "DEPLOYMENT_SUCCESS"]))
       .min(1)
       .or(z.enum(["TASK_RUN_ATTEMPT", "DEPLOYMENT_FAILURE", "DEPLOYMENT_SUCCESS"])),
+    environmentTypes: z
+      .array(z.enum(["STAGING", "PRODUCTION"]))
+      .min(1)
+      .or(z.enum(["STAGING", "PRODUCTION"])),
     type: z.enum(["WEBHOOK", "SLACK", "EMAIL"]).default("EMAIL"),
     channelValue: z.string().nonempty(),
     integrationId: z.string().optional(),
@@ -81,6 +88,9 @@ function formDataToCreateAlertChannelOptions(
         alertTypes: Array.isArray(formData.alertTypes)
           ? formData.alertTypes
           : [formData.alertTypes],
+        environmentTypes: Array.isArray(formData.environmentTypes)
+          ? formData.environmentTypes
+          : [formData.environmentTypes],
         channel: {
           type: "WEBHOOK",
           url: formData.channelValue,
@@ -93,6 +103,9 @@ function formDataToCreateAlertChannelOptions(
         alertTypes: Array.isArray(formData.alertTypes)
           ? formData.alertTypes
           : [formData.alertTypes],
+        environmentTypes: Array.isArray(formData.environmentTypes)
+          ? formData.environmentTypes
+          : [formData.environmentTypes],
         channel: {
           type: "EMAIL",
           email: formData.channelValue,
@@ -107,6 +120,9 @@ function formDataToCreateAlertChannelOptions(
         alertTypes: Array.isArray(formData.alertTypes)
           ? formData.alertTypes
           : [formData.alertTypes],
+        environmentTypes: Array.isArray(formData.environmentTypes)
+          ? formData.environmentTypes
+          : [formData.environmentTypes],
         channel: {
           type: "SLACK",
           channelId,
@@ -193,20 +209,27 @@ export default function Page() {
   const project = useProject();
   const [currentAlertChannel, setCurrentAlertChannel] = useState<string | null>(option ?? "EMAIL");
 
+  const [selectedSlackChannelValue, setSelectedSlackChannelValue] = useState<string | undefined>();
+
+  const selectedSlackChannel = slack.channels?.find(
+    (s) => selectedSlackChannelValue === `${s.id}/${s.name}`
+  );
+
   const isLoading =
     navigation.state !== "idle" &&
     navigation.formMethod === "post" &&
     navigation.formData?.get("action") === "create";
 
-  const [form, { channelValue: channelValue, alertTypes, type, integrationId }] = useForm({
-    id: "create-alert",
-    // TODO: type this
-    lastSubmission: lastSubmission as any,
-    onValidate({ formData }) {
-      return parse(formData, { schema: FormSchema });
-    },
-    shouldRevalidate: "onSubmit",
-  });
+  const [form, { channelValue: channelValue, alertTypes, environmentTypes, type, integrationId }] =
+    useForm({
+      id: "create-alert",
+      // TODO: type this
+      lastSubmission: lastSubmission as any,
+      onValidate({ formData }) {
+        return parse(formData, { schema: FormSchema });
+      },
+      shouldRevalidate: "onSubmit",
+    });
 
   useEffect(() => {
     setIsOpen(true);
@@ -271,6 +294,9 @@ export default function Page() {
                       dropdownIcon
                       variant="tertiary/medium"
                       items={slack.channels}
+                      setValue={(value) => {
+                        typeof value === "string" && setSelectedSlackChannelValue(value);
+                      }}
                       filter={(channel, search) =>
                         channel.name?.toLowerCase().includes(search.toLowerCase()) ?? false
                       }
@@ -290,10 +316,19 @@ export default function Page() {
                         </>
                       )}
                     </Select>
-                    <Hint className="leading-relaxed">
-                      If selecting a private channel, you will need to invite the bot to the channel
-                      using <InlineCode variant="extra-small">/invite @Trigger.dev</InlineCode>
-                    </Hint>
+                    {selectedSlackChannel && selectedSlackChannel.is_private && (
+                      <Callout
+                        variant="warning"
+                        className={cn("text-sm", variantClasses.warning.textColor)}
+                      >
+                        To receive alerts in the{" "}
+                        <InlineCode variant="extra-small">{selectedSlackChannel.name}</InlineCode>{" "}
+                        channel, you need to invite the @Trigger.dev Slack Bot. Go to the channel in
+                        Slack and type:{" "}
+                        <InlineCode variant="extra-small">/invite @Trigger.dev</InlineCode>.
+                      </Callout>
+                    )}
+
                     <FormError id={channelValue.errorId}>{channelValue.error}</FormError>
                     <input type="hidden" name="integrationId" value={slack.integrationId} />
                   </>
@@ -324,39 +359,63 @@ export default function Page() {
               </InputGroup>
             )}
 
-            <InputGroup fullWidth>
-              <Label>Events</Label>
+            <InputGroup>
+              <Label>Alert me when</Label>
 
-              <Checkbox
-                name={alertTypes.name}
-                id="TASK_RUN_ATTEMPT"
-                value="TASK_RUN_ATTEMPT"
-                variant="simple/small"
-                label="Task run failure"
-                defaultChecked
-              />
+              <div className="flex items-center gap-1">
+                <CheckboxWithLabel
+                  name={alertTypes.name}
+                  id="TASK_RUN_ATTEMPT"
+                  value="TASK_RUN_ATTEMPT"
+                  variant="simple/small"
+                  label="Task run attempts fail"
+                  defaultChecked
+                  className="pr-0"
+                />
+                <InfoIconTooltip content="You'll receive an alert every time an attempt fails on a run." />
+              </div>
 
-              <Checkbox
+              <CheckboxWithLabel
                 name={alertTypes.name}
                 id="DEPLOYMENT_FAILURE"
                 value="DEPLOYMENT_FAILURE"
                 variant="simple/small"
-                label="Deployment failure"
+                label="Deployments fail"
                 defaultChecked
               />
 
-              <Checkbox
+              <CheckboxWithLabel
                 name={alertTypes.name}
                 id="DEPLOYMENT_SUCCESS"
                 value="DEPLOYMENT_SUCCESS"
                 variant="simple/small"
-                label="Deployment success"
+                label="Deployments succeed"
                 defaultChecked
               />
 
               <FormError id={alertTypes.errorId}>{alertTypes.error}</FormError>
             </InputGroup>
+            <InputGroup>
+              <Label>Environments</Label>
+              <CheckboxWithLabel
+                name={environmentTypes.name}
+                id="PRODUCTION"
+                value="PRODUCTION"
+                variant="simple/small"
+                label="PROD"
+                defaultChecked
+              />
+              <CheckboxWithLabel
+                name={environmentTypes.name}
+                id="STAGING"
+                value="STAGING"
+                variant="simple/small"
+                label="STAGING"
+                defaultChecked
+              />
 
+              <FormError id={environmentTypes.errorId}>{environmentTypes.error}</FormError>
+            </InputGroup>
             <FormError>{form.error}</FormError>
             <div className="border-t border-grid-bright pt-3">
               <FormButtons

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.alerts/route.tsx

@@ -1,8 +1,9 @@
 import { useForm } from "@conform-to/react";
 import { parse } from "@conform-to/zod";
 import {
-  BoltIcon,
-  BoltSlashIcon,
+  ArrowUpRightIcon,
+  BellAlertIcon,
+  BellSlashIcon,
   BookOpenIcon,
   EnvelopeIcon,
   GlobeAltIcon,
@@ -17,10 +18,12 @@ import { ProjectAlertChannelType, ProjectAlertType } from "@trigger.dev/database
 import assertNever from "assert-never";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { z } from "zod";
+import { EnvironmentTypeLabel } from "~/components/environments/EnvironmentLabel";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
 import { ClipboardField } from "~/components/primitives/ClipboardField";
 import { DetailCell } from "~/components/primitives/DetailCell";
+import { Header2 } from "~/components/primitives/Headers";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
 import {
@@ -155,7 +158,7 @@ export default function Page() {
         <PageAccessories>
           <LinkButton
             LeadingIcon={BookOpenIcon}
-            to={docsPath("v3/project-alerts")}
+            to={docsPath("v3/troubleshooting-alerts")}
             variant="minimal/small"
           >
             Alerts docs
@@ -164,21 +167,25 @@ export default function Page() {
       </NavBar>
       <PageBody>
         <div className={cn("flex h-full flex-col gap-3")}>
-          <div className="flex items-center justify-end gap-2">
-            <LinkButton
-              to={v3NewProjectAlertPath(organization, project)}
-              variant="primary/small"
-              LeadingIcon={PlusIcon}
-              shortcut={{ key: "n" }}
-            >
-              New alert
-            </LinkButton>
-          </div>
+          {alertChannels.length > 0 && alertChannels.length < 10 && (
+            <div className="flex items-end justify-between">
+              <Header2 className="">Project alerts</Header2>
+              <LinkButton
+                to={v3NewProjectAlertPath(organization, project)}
+                variant="primary/small"
+                LeadingIcon={PlusIcon}
+                shortcut={{ key: "n" }}
+              >
+                New alert
+              </LinkButton>
+            </div>
+          )}
           <Table>
             <TableHeader>
               <TableRow>
                 <TableHeaderCell>Name</TableHeaderCell>
-                <TableHeaderCell>Alert Types</TableHeaderCell>
+                <TableHeaderCell>Alert types</TableHeaderCell>
+                <TableHeaderCell>Environments</TableHeaderCell>
                 <TableHeaderCell>Channel</TableHeaderCell>
                 <TableHeaderCell>Enabled</TableHeaderCell>
                 <TableHeaderCell hiddenLabel>Actions</TableHeaderCell>
@@ -194,11 +201,25 @@ export default function Page() {
                     <TableCell className={alertChannel.enabled ? "" : "opacity-50"}>
                       {alertChannel.alertTypes.map((type) => alertTypeTitle(type)).join(", ")}
                     </TableCell>
+                    <TableCell
+                      className={cn("space-x-2", alertChannel.enabled ? "" : "opacity-50")}
+                    >
+                      {alertChannel.environmentTypes.map((environmentType) => (
+                        <EnvironmentTypeLabel
+                          key={environmentType}
+                          environment={{ type: environmentType }}
+                        />
+                      ))}
+                    </TableCell>
                     <TableCell className={alertChannel.enabled ? "" : "opacity-50"}>
                       <AlertChannelDetails alertChannel={alertChannel} />
                     </TableCell>
                     <TableCell className={alertChannel.enabled ? "" : "opacity-50"}>
-                      <EnabledStatus enabled={alertChannel.enabled} />
+                      <EnabledStatus
+                        enabled={alertChannel.enabled}
+                        enabledIcon={BellAlertIcon}
+                        disabledIcon={BellSlashIcon}
+                      />
                     </TableCell>
                     <TableCellMenu isSticky>
                       {alertChannel.enabled ? (
@@ -214,14 +235,44 @@ export default function Page() {
               ) : (
                 <TableRow>
                   <TableCell colSpan={5}>
-                    <div className="flex items-center justify-center">
-                      <Paragraph>No alerts have been created</Paragraph>
+                    <div className="flex flex-col items-center justify-center py-6">
+                      <Header2 spacing className="text-text-bright">
+                        You haven't created any project alerts yet
+                      </Header2>
+                      <Paragraph variant="small" className="mb-4">
+                        Get alerted when runs or deployments fail, or when deployments succeed in
+                        both Prod and Staging environments.
+                      </Paragraph>
+                      <LinkButton
+                        to={v3NewProjectAlertPath(organization, project)}
+                        variant="primary/medium"
+                        LeadingIcon={PlusIcon}
+                        shortcut={{ key: "n" }}
+                      >
+                        New alert
+                      </LinkButton>
                     </div>
                   </TableCell>
                 </TableRow>
               )}
             </TableBody>
           </Table>
+          <div className="mt-4">
+            <Header2 className="mb-1">Platform alerts</Header2>
+            <Paragraph variant="small" className="mb-4">
+              Subscribe to get email notifications when Trigger.dev creates, updates or resolves a
+              platform incident.
+            </Paragraph>
+            <LinkButton
+              variant="tertiary/medium"
+              TrailingIcon={ArrowUpRightIcon}
+              to="https://status.trigger.dev/"
+              target="_blank"
+              className="inline-flex"
+            >
+              Subscribe
+            </LinkButton>
+          </div>
         </div>
         <Outlet />
       </PageBody>
@@ -294,7 +345,7 @@ function DisableAlertChannelButton(props: { id: string }) {
         value="disable"
         type="submit"
         variant="small-menu-item"
-        LeadingIcon={BoltSlashIcon}
+        LeadingIcon={BellSlashIcon}
         leadingIconClassName="text-dimmed"
         className="text-xs"
       >
@@ -332,7 +383,7 @@ function EnableAlertChannelButton(props: { id: string }) {
         value="enable"
         type="submit"
         variant="small-menu-item"
-        LeadingIcon={BoltIcon}
+        LeadingIcon={BellAlertIcon}
         leadingIconClassName="text-success"
         className="text-xs"
       >

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.environment-variables.new/route.tsx

@@ -18,7 +18,7 @@ import {
   environmentTitle,
 } from "~/components/environments/EnvironmentLabel";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
-import { Checkbox } from "~/components/primitives/Checkbox";
+import { CheckboxWithLabel } from "~/components/primitives/Checkbox";
 import { Dialog, DialogContent, DialogHeader } from "~/components/primitives/Dialog";
 import { Fieldset } from "~/components/primitives/Fieldset";
 import { FormButtons } from "~/components/primitives/FormButtons";
@@ -70,7 +70,7 @@ const Variable = z.object({
 type Variable = z.infer<typeof Variable>;
 
 const schema = z.object({
-  overwrite: z.preprocess((i) => {
+  override: z.preprocess((i) => {
     if (i === "true") return true;
     if (i === "false") return false;
     return;
@@ -115,6 +115,13 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
   const project = await prisma.project.findUnique({
     where: {
       slug: params.projectParam,
+      organization: {
+        members: {
+          some: {
+            userId,
+          },
+        },
+      },
     },
     select: {
       id: true,
@@ -126,7 +133,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
   }
 
   const repository = new EnvironmentVariablesRepository(prisma);
-  const result = await repository.create(project.id, userId, submission.value);
+  const result = await repository.create(project.id, submission.value);
 
   if (!result.success) {
     if (result.variableErrors) {
@@ -195,7 +202,7 @@ export default function Page() {
               <Label>Environments</Label>
               <div className="flex flex-wrap items-center gap-2">
                 {environments.map((environment) => (
-                  <Checkbox
+                  <CheckboxWithLabel
                     key={environment.id}
                     id={environment.id}
                     value={environment.id}
@@ -249,7 +256,7 @@ export default function Page() {
                     type="submit"
                     variant="primary/small"
                     disabled={isLoading}
-                    name="overwrite"
+                    name="override"
                     value="false"
                   >
                     {isLoading ? "Saving" : "Save"}
@@ -257,10 +264,10 @@ export default function Page() {
                   <Button
                     variant="secondary/small"
                     disabled={isLoading}
-                    name="overwrite"
+                    name="override"
                     value="true"
                   >
-                    {isLoading ? "Overwriting" : "Overwrite"}
+                    {isLoading ? "Overriding" : "Override"}
                   </Button>
                 </div>
               }

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.environment-variables/route.tsx

@@ -106,6 +106,13 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
   const project = await prisma.project.findUnique({
     where: {
       slug: params.projectParam,
+      organization: {
+        members: {
+          some: {
+            userId,
+          },
+        },
+      },
     },
     select: {
       id: true,
@@ -119,7 +126,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
   switch (submission.value.action) {
     case "edit": {
       const repository = new EnvironmentVariablesRepository(prisma);
-      const result = await repository.edit(project.id, userId, submission.value);
+      const result = await repository.edit(project.id, submission.value);
 
       if (!result.success) {
         submission.error.key = result.error;
@@ -138,7 +145,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
     }
     case "delete": {
       const repository = new EnvironmentVariablesRepository(prisma);
-      const result = await repository.delete(project.id, userId, submission.value);
+      const result = await repository.delete(project.id, submission.value);
 
       if (!result.success) {
         submission.error.key = result.error;
@@ -334,6 +341,7 @@ function EditEnvironmentVariablePanel({
                         name={`values[${index}].value`}
                         placeholder="Not set"
                         defaultValue={value}
+                        type="password"
                       />
                     </Fragment>
                   );

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs._index/route.tsx

@@ -1,16 +1,33 @@
+import { ArrowPathIcon, StopCircleIcon } from "@heroicons/react/20/solid";
 import { BeakerIcon, BookOpenIcon } from "@heroicons/react/24/solid";
-import { useNavigation } from "@remix-run/react";
+import { Form, useNavigation } from "@remix-run/react";
 import { LoaderFunctionArgs } from "@remix-run/server-runtime";
-import { TypedAwait, typeddefer, typedjson, useTypedLoaderData } from "remix-typedjson";
+import { AnimatePresence, motion } from "framer-motion";
+import { Suspense, useState } from "react";
+import { TypedAwait, typeddefer, useTypedLoaderData } from "remix-typedjson";
 import { TaskIcon } from "~/assets/icons/TaskIcon";
 import { BlankstateInstructions } from "~/components/BlankstateInstructions";
 import { StepContentContainer } from "~/components/StepContentContainer";
 import { MainCenteredContainer, PageBody } from "~/components/layout/AppLayout";
-import { LinkButton } from "~/components/primitives/Buttons";
+import { Button, LinkButton } from "~/components/primitives/Buttons";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogFooter,
+  DialogHeader,
+  DialogTrigger,
+} from "~/components/primitives/Dialog";
 import { Header1 } from "~/components/primitives/Headers";
 import { NavBar, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
+import {
+  SelectedItemsProvider,
+  useSelectedItems,
+} from "~/components/primitives/SelectedItemsProvider";
+import { Spinner } from "~/components/primitives/Spinner";
 import { StepNumber } from "~/components/primitives/StepNumber";
+import { TextLink } from "~/components/primitives/TextLink";
 import { RunsFilters, TaskRunListSearchFilters } from "~/components/runs/v3/RunFilters";
 import { TaskRunsTable } from "~/components/runs/v3/TaskRunsTable";
 import { useOrganization } from "~/hooks/useOrganizations";
@@ -19,11 +36,10 @@ import { useUser } from "~/hooks/useUser";
 import { RunListPresenter } from "~/presenters/v3/RunListPresenter.server";
 import { requireUserId } from "~/services/session.server";
 import { cn } from "~/utils/cn";
-import { ProjectParamSchema, v3ProjectPath, v3TestPath } from "~/utils/pathBuilder";
+import { ProjectParamSchema, v3ProjectPath, v3RunsPath, v3TestPath } from "~/utils/pathBuilder";
 import { ListPagination } from "../../components/ListPagination";
-import { TextLink } from "~/components/primitives/TextLink";
-import { Spinner } from "~/components/primitives/Spinner";
-import { Suspense } from "react";
+import { BULK_ACTION_RUN_LIMIT } from "~/consts";
+import { findProjectBySlug } from "~/models/project.server";
 
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const userId = await requireUserId(request);
@@ -37,19 +53,27 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
     environments: url.searchParams.getAll("environments"),
     tasks: url.searchParams.getAll("tasks"),
     period: url.searchParams.get("period") ?? undefined,
+    bulkId: url.searchParams.get("bulkId") ?? undefined,
   };
-  const { tasks, versions, statuses, environments, period, from, to, cursor, direction } =
+  const { tasks, versions, statuses, environments, period, bulkId, from, to, cursor, direction } =
     TaskRunListSearchFilters.parse(s);
 
+  const project = await findProjectBySlug(organizationSlug, projectParam, userId);
+
+  if (!project) {
+    throw new Error("Project not found");
+  }
+
   const presenter = new RunListPresenter();
   const list = presenter.call({
     userId,
-    projectSlug: projectParam,
+    projectId: project.id,
     tasks,
     versions,
     statuses,
     environments,
     period,
+    bulkId,
     from,
     to,
     direction: direction,
@@ -73,60 +97,246 @@ export default function Page() {
       <NavBar>
         <PageTitle title="Runs" />
       </NavBar>
-      <PageBody>
-        <Suspense
-          fallback={
-            <div className="flex items-center justify-center py-2">
-              <div className="mx-auto flex items-center gap-2">
-                <Spinner />
-                <Paragraph variant="small">Loading runs</Paragraph>
-              </div>
-            </div>
-          }
+      <PageBody scrollable={false}>
+        <SelectedItemsProvider
+          initialSelectedItems={[]}
+          maxSelectedItemCount={BULK_ACTION_RUN_LIMIT}
         >
-          <TypedAwait resolve={data}>
-            {(list) => (
-              <>
-                {list.runs.length === 0 && !list.hasFilters ? (
-                  list.possibleTasks.length === 0 ? (
-                    <CreateFirstTaskInstructions />
-                  ) : (
-                    <RunTaskInstructions />
-                  )
-                ) : (
-                  <div className={cn("grid h-fit grid-cols-1 gap-4")}>
-                    <div>
-                      <div className="mb-2 flex items-start justify-between gap-x-2">
-                        <RunsFilters
-                          possibleEnvironments={project.environments}
-                          possibleTasks={list.possibleTasks}
-                          hasFilters={list.hasFilters}
-                        />
-                        <div className="flex items-center justify-end gap-x-2">
-                          <ListPagination list={list} />
-                        </div>
+          {({ selectedItems }) => (
+            <div
+              className={cn(
+                "grid h-full max-h-full overflow-hidden",
+                selectedItems.size === 0 ? "grid-rows-1" : "grid-rows-[1fr_2.5rem]"
+              )}
+            >
+              <div className="overflow-y-auto p-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600">
+                <Suspense
+                  fallback={
+                    <div className="flex items-center justify-center py-2">
+                      <div className="mx-auto flex items-center gap-2">
+                        <Spinner />
+                        <Paragraph variant="small">Loading runs</Paragraph>
                       </div>
-
-                      <TaskRunsTable
-                        total={list.runs.length}
-                        hasFilters={list.hasFilters}
-                        filters={list.filters}
-                        runs={list.runs}
-                        isLoading={isLoading}
-                      />
-                      <ListPagination list={list} className="mt-2 justify-end" />
                     </div>
-                  </div>
-                )}
-              </>
-            )}
-          </TypedAwait>
-        </Suspense>
+                  }
+                >
+                  <TypedAwait resolve={data}>
+                    {(list) => (
+                      <>
+                        {list.runs.length === 0 && !list.hasFilters ? (
+                          list.possibleTasks.length === 0 ? (
+                            <CreateFirstTaskInstructions />
+                          ) : (
+                            <RunTaskInstructions />
+                          )
+                        ) : (
+                          <div className={cn("grid h-fit grid-cols-1 gap-4")}>
+                            <div>
+                              <div className="mb-2 flex items-start justify-between gap-x-2">
+                                <RunsFilters
+                                  possibleEnvironments={project.environments}
+                                  possibleTasks={list.possibleTasks}
+                                  bulkActions={list.bulkActions}
+                                  hasFilters={list.hasFilters}
+                                />
+                                <div className="flex items-center justify-end gap-x-2">
+                                  <ListPagination list={list} />
+                                </div>
+                              </div>
+
+                              <TaskRunsTable
+                                total={list.runs.length}
+                                hasFilters={list.hasFilters}
+                                filters={list.filters}
+                                runs={list.runs}
+                                isLoading={isLoading}
+                                allowSelection
+                              />
+                              <ListPagination list={list} className="mt-2 justify-end" />
+                            </div>
+                          </div>
+                        )}
+                      </>
+                    )}
+                  </TypedAwait>
+                </Suspense>
+              </div>
+              <BulkActionBar />
+            </div>
+          )}
+        </SelectedItemsProvider>
       </PageBody>
     </>
   );
 }
 
+function BulkActionBar() {
+  const { selectedItems, deselectAll } = useSelectedItems();
+  const [barState, setBarState] = useState<"none" | "replay" | "cancel">("none");
+
+  const hasSelectedMaximum = selectedItems.size >= BULK_ACTION_RUN_LIMIT;
+
+  return (
+    <AnimatePresence>
+      {selectedItems.size > 0 && (
+        <motion.div
+          initial={{ translateY: "100%" }}
+          animate={{ translateY: 0 }}
+          exit={{ translateY: "100%" }}
+          className="flex items-center justify-between gap-2 border-t border-grid-bright bg-background-bright px-3"
+        >
+          <div className="flex items-center gap-1.5 text-sm text-text-bright">
+            <span className="font-medium">Bulk actions:</span>
+            {hasSelectedMaximum ? (
+              <span className="text-warning">Maximum of {selectedItems.size} runs selected</span>
+            ) : (
+              <span>{selectedItems.size} runs selected</span>
+            )}
+          </div>
+          <div className="flex items-center gap-1 divide-x divide-charcoal-700">
+            <CancelRuns
+              onOpen={(o) => {
+                if (o) {
+                  setBarState("cancel");
+                } else {
+                  setBarState("none");
+                }
+              }}
+            />
+            <ReplayRuns
+              onOpen={(o) => {
+                if (o) {
+                  setBarState("replay");
+                } else {
+                  setBarState("none");
+                }
+              }}
+            />
+            <Button
+              variant="minimal/medium"
+              shortcut={{ key: "esc", enabledOnInputElements: true }}
+              onClick={() => {
+                if (barState !== "none") return;
+                deselectAll();
+              }}
+            >
+              Clear selection
+            </Button>
+          </div>
+        </motion.div>
+      )}
+    </AnimatePresence>
+  );
+}
+
+function CancelRuns({ onOpen }: { onOpen: (open: boolean) => void }) {
+  const { selectedItems } = useSelectedItems();
+
+  const organization = useOrganization();
+  const project = useProject();
+  const failedRedirect = v3RunsPath(organization, project);
+
+  const formAction = `/resources/taskruns/bulk/cancel`;
+
+  const navigation = useNavigation();
+  const isLoading = navigation.formAction === formAction;
+
+  return (
+    <Dialog onOpenChange={(o) => onOpen(o)}>
+      <DialogTrigger asChild>
+        <Button
+          type="button"
+          variant="minimal/medium"
+          shortcut={{ key: "c", enabledOnInputElements: true }}
+          LeadingIcon={StopCircleIcon}
+        >
+          Cancel runs
+        </Button>
+      </DialogTrigger>
+      <DialogContent key="replay">
+        <DialogHeader>Cancel {selectedItems.size} runs?</DialogHeader>
+        <DialogDescription>
+          Canceling these runs will stop them from running. Only runs that are not already finished
+          will be canceled, the others will remain in their existing state.
+        </DialogDescription>
+        <DialogFooter>
+          <Form action={formAction} method="post" reloadDocument>
+            <input type="hidden" name="failedRedirect" value={failedRedirect} />
+            <input type="hidden" name="organizationSlug" value={organization.slug} />
+            <input type="hidden" name="projectSlug" value={project.slug} />
+            {[...selectedItems].map((runId) => (
+              <input key={runId} type="hidden" name="runIds" value={runId} />
+            ))}
+            <Button
+              type="submit"
+              variant="danger/small"
+              LeadingIcon={isLoading ? "spinner-white" : StopCircleIcon}
+              disabled={isLoading}
+              shortcut={{ modifiers: ["meta"], key: "enter" }}
+            >
+              {isLoading ? "Canceling..." : `Cancel ${selectedItems.size} runs`}
+            </Button>
+          </Form>
+        </DialogFooter>
+      </DialogContent>
+    </Dialog>
+  );
+}
+
+function ReplayRuns({ onOpen }: { onOpen: (open: boolean) => void }) {
+  const { selectedItems } = useSelectedItems();
+
+  const organization = useOrganization();
+  const project = useProject();
+  const failedRedirect = v3RunsPath(organization, project);
+
+  const formAction = `/resources/taskruns/bulk/replay`;
+
+  const navigation = useNavigation();
+  const isLoading = navigation.formAction === formAction;
+
+  return (
+    <Dialog onOpenChange={(o) => onOpen(o)}>
+      <DialogTrigger asChild>
+        <Button
+          type="button"
+          variant="minimal/medium"
+          shortcut={{ key: "r", enabledOnInputElements: true }}
+          LeadingIcon={ArrowPathIcon}
+        >
+          Replay {selectedItems.size} runs
+        </Button>
+      </DialogTrigger>
+      <DialogContent key="replay">
+        <DialogHeader>Replay runs?</DialogHeader>
+        <DialogDescription>
+          Replaying these runs will create a new run for each with the same payload and environment
+          as the original. It will use the latest version of the code for each task.
+        </DialogDescription>
+        <DialogFooter>
+          <Form action={formAction} method="post" reloadDocument>
+            <input type="hidden" name="failedRedirect" value={failedRedirect} />
+            <input type="hidden" name="organizationSlug" value={organization.slug} />
+            <input type="hidden" name="projectSlug" value={project.slug} />
+            {[...selectedItems].map((runId) => (
+              <input key={runId} type="hidden" name="runIds" value={runId} />
+            ))}
+            <Button
+              type="submit"
+              variant="primary/small"
+              LeadingIcon={isLoading ? "spinner-white" : ArrowPathIcon}
+              disabled={isLoading}
+              shortcut={{ modifiers: ["meta"], key: "enter" }}
+            >
+              {isLoading ? "Replaying..." : `Replay ${selectedItems.size} runs`}
+            </Button>
+          </Form>
+        </DialogFooter>
+      </DialogContent>
+    </Dialog>
+  );
+}
+
 function CreateFirstTaskInstructions() {
   const organization = useOrganization();
   const project = useProject();

apps/webapp/app/routes/_app.orgs.$organizationSlug_.projects.new/route.tsx

@@ -16,13 +16,17 @@ import { FormTitle } from "~/components/primitives/FormTitle";
 import { Input } from "~/components/primitives/Input";
 import { InputGroup } from "~/components/primitives/InputGroup";
 import { Label } from "~/components/primitives/Label";
+import { Paragraph } from "~/components/primitives/Paragraph";
 import { Select, SelectItem } from "~/components/primitives/Select";
+import { TextLink } from "~/components/primitives/TextLink";
 import { prisma } from "~/db.server";
 import { useFeatures } from "~/hooks/useFeatures";
+import { useUser } from "~/hooks/useUser";
 import { redirectWithSuccessMessage } from "~/models/message.server";
 import { createProject } from "~/models/project.server";
 import { requireUserId } from "~/services/session.server";
 import { OrganizationParamsSchema, organizationPath, projectPath } from "~/utils/pathBuilder";
+import { RequestV3Access } from "../resources.orgs.$organizationSlug.v3-access";
 
 export async function loader({ params, request }: LoaderFunctionArgs) {
   const userId = await requireUserId(request);
@@ -34,10 +38,14 @@ export async function loader({ params, request }: LoaderFunctionArgs) {
       id: true,
       title: true,
       v3Enabled: true,
+      v2Enabled: true,
+      hasRequestedV3: true,
       _count: {
         select: {
           projects: {
-            where: { deletedAt: null },
+            where: {
+              deletedAt: null,
+            },
           },
         },
       },
@@ -57,6 +65,8 @@ export async function loader({ params, request }: LoaderFunctionArgs) {
       slug: organizationSlug,
       projectsCount: organization._count.projects,
       v3Enabled: organization.v3Enabled,
+      v2Enabled: organization.v2Enabled,
+      hasRequestedV3: organization.hasRequestedV3,
     },
     defaultVersion: url.searchParams.get("version") ?? "v2",
   });
@@ -98,11 +108,23 @@ export const action: ActionFunction = async ({ request, params }) => {
 };
 
 export default function NewOrganizationPage() {
-  const { organization, defaultVersion } = useTypedLoaderData<typeof loader>();
+  const { organization } = useTypedLoaderData<typeof loader>();
   const lastSubmission = useActionData();
-  const { v3Enabled } = useFeatures();
+  const { v3Enabled, isManagedCloud } = useFeatures();
 
   const canCreateV3Projects = organization.v3Enabled && v3Enabled;
+  const canCreateV2Projects = organization.v2Enabled || !isManagedCloud;
+  const canCreateProjects = canCreateV2Projects || canCreateV3Projects;
+
+  if (!canCreateProjects) {
+    return (
+      <RequestV3Access
+        hasRequestedV3={organization.hasRequestedV3}
+        organizationSlug={organization.slug}
+        projectsCount={organization.projectsCount}
+      />
+    );
+  }
 
   const [form, { projectName, projectVersion }] = useForm({
     id: "create-project",
@@ -119,7 +141,7 @@ export default function NewOrganizationPage() {
         <FormTitle
           LeadingIcon="folder"
           title="Create a new project"
-          description={`This will create a new project in your "${organization.title}" organization. `}
+          description={`This will create a new project in your "${organization.title}" organization.`}
         />
         <Form method="post" {...form.props}>
           {organization.projectsCount === 0 && (
@@ -138,7 +160,7 @@ export default function NewOrganizationPage() {
               />
               <FormError id={projectName.errorId}>{projectName.error}</FormError>
             </InputGroup>
-            {canCreateV3Projects ? (
+            {canCreateV2Projects && canCreateV3Projects ? (
               <InputGroup>
                 <Label htmlFor={projectVersion.id}>Project version</Label>
                 <Select
@@ -161,8 +183,16 @@ export default function NewOrganizationPage() {
                 </Select>
                 <FormError id={projectVersion.errorId}>{projectVersion.error}</FormError>
               </InputGroup>
+            ) : canCreateV3Projects ? (
+              <>
+                <Callout variant="info">This will be a v3 project</Callout>
+                <input {...conform.input(projectVersion, { type: "hidden" })} value={"v3"} />
+              </>
             ) : (
-              <input {...conform.input(projectVersion, { type: "hidden" })} value="v2" />
+              <>
+                <Callout variant="info">This will be a v2 project</Callout>
+                <input {...conform.input(projectVersion, { type: "hidden" })} value={"v2"} />
+              </>
             )}
             <FormButtons
               confirmButton={

apps/webapp/app/routes/_app.orgs.new/route.tsx

@@ -17,19 +17,14 @@ import { Input } from "~/components/primitives/Input";
 import { InputGroup } from "~/components/primitives/InputGroup";
 import { Label } from "~/components/primitives/Label";
 import { RadioGroupItem } from "~/components/primitives/RadioButton";
-import { Select, SelectItem } from "~/components/primitives/Select";
-import { featuresForRequest } from "~/features.server";
 import { useFeatures } from "~/hooks/useFeatures";
 import { createOrganization } from "~/models/organization.server";
 import { NewOrganizationPresenter } from "~/presenters/NewOrganizationPresenter.server";
-import { commitCurrentProjectSession, setCurrentProjectId } from "~/services/currentProject.server";
 import { requireUserId } from "~/services/session.server";
-import { projectPath, rootPath, selectPlanPath } from "~/utils/pathBuilder";
+import { organizationPath, rootPath } from "~/utils/pathBuilder";
 
 const schema = z.object({
   orgName: z.string().min(3).max(50),
-  projectName: z.string().min(3).max(50),
-  projectVersion: z.enum(["v2", "v3"]),
   companySize: z.string().optional(),
 });
 
@@ -57,29 +52,10 @@ export const action: ActionFunction = async ({ request }) => {
     const organization = await createOrganization({
       title: submission.value.orgName,
       userId,
-      projectName: submission.value.projectName,
       companySize: submission.value.companySize ?? null,
-      projectVersion: submission.value.projectVersion,
     });
 
-    const project = organization.projects[0];
-    const session = await setCurrentProjectId(project.id, request);
-
-    const { isManagedCloud } = featuresForRequest(request);
-
-    const headers = {
-      "Set-Cookie": await commitCurrentProjectSession(session),
-    };
-
-    if (isManagedCloud && submission.value.projectVersion === "v2") {
-      return redirect(selectPlanPath(organization), {
-        headers,
-      });
-    }
-
-    return redirect(projectPath(organization, project), {
-      headers,
-    });
+    return redirect(organizationPath(organization));
   } catch (error: any) {
     return json({ errors: { body: error.message } }, { status: 400 });
   }
@@ -91,10 +67,7 @@ export default function NewOrganizationPage() {
   const { isManagedCloud } = useFeatures();
   const navigation = useNavigation();
 
-  //this is temporary whilst v3 is invite-only. Switch to the useFeatures value when v3 is generally available.
-  const v3Enabled = false;
-
-  const [form, { orgName, projectName, projectVersion }] = useForm({
+  const [form, { orgName }] = useForm({
     id: "create-organization",
     // TODO: type this
     lastSubmission: lastSubmission as any,
@@ -123,45 +96,9 @@ export default function NewOrganizationPage() {
             <Hint>E.g. your company name or your workspace name.</Hint>
             <FormError id={orgName.errorId}>{orgName.error}</FormError>
           </InputGroup>
-          <InputGroup>
-            <Label htmlFor={projectName.id}>Project name</Label>
-            <Input
-              {...conform.input(projectName, { type: "text" })}
-              placeholder="Your Project name"
-              icon="folder"
-            />
-            <Hint>Your Jobs will live inside this Project.</Hint>
-            <FormError id={projectName.errorId}>{projectName.error}</FormError>
-          </InputGroup>
-          {v3Enabled ? (
-            <InputGroup>
-              <Label htmlFor={projectVersion.id}>Project version</Label>
-              <Select
-                {...conform.select(projectVersion)}
-                defaultValue={undefined}
-                variant="tertiary/medium"
-                placeholder="Select version"
-                dropdownIcon
-                text={(value) => {
-                  switch (value) {
-                    case "v2":
-                      return "Version 2";
-                    case "v3":
-                      return "Version 3";
-                  }
-                }}
-              >
-                <SelectItem value="v2">Version 2</SelectItem>
-                <SelectItem value="v3">Version 3 (Developer Preview)</SelectItem>
-              </Select>
-              <FormError id={projectVersion.errorId}>{projectVersion.error}</FormError>
-            </InputGroup>
-          ) : (
-            <input {...conform.input(projectVersion, { type: "hidden" })} value="v2" />
-          )}
           {isManagedCloud && (
             <InputGroup>
-              <Label htmlFor={projectName.id}>Number of employees</Label>
+              <Label htmlFor={"companySize"}>Number of employees</Label>
               <RadioGroup name="companySize" className="flex items-center justify-between gap-2">
                 <RadioGroupItem
                   id="employees-1-5"

apps/webapp/app/routes/account._index/route.tsx

@@ -6,7 +6,7 @@ import { z } from "zod";
 import { UserProfilePhoto } from "~/components/UserProfilePhoto";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { Button } from "~/components/primitives/Buttons";
-import { Checkbox } from "~/components/primitives/Checkbox";
+import { CheckboxWithLabel } from "~/components/primitives/Checkbox";
 import { Fieldset } from "~/components/primitives/Fieldset";
 import { FormButtons } from "~/components/primitives/FormButtons";
 import { FormError } from "~/components/primitives/FormError";
@@ -158,7 +158,7 @@ export default function Page() {
             </InputGroup>
             <InputGroup>
               <Label>Notifications</Label>
-              <Checkbox
+              <CheckboxWithLabel
                 id="marketingEmails"
                 {...conform.input(marketingEmails, { type: "checkbox" })}
                 label="Receive onboarding emails"

apps/webapp/app/routes/admin.api.v1.marqs.ts

@@ -0,0 +1,31 @@
+import { LoaderFunctionArgs, json } from "@remix-run/server-runtime";
+import { prisma } from "~/db.server";
+import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+import { marqs } from "~/v3/marqs/index.server";
+
+export async function loader({ request, params }: LoaderFunctionArgs) {
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const user = await prisma.user.findUnique({
+    where: {
+      id: authenticationResult.userId,
+    },
+  });
+
+  if (!user) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  if (!user.admin) {
+    return json({ error: "You must be an admin to perform this action" }, { status: 403 });
+  }
+
+  const details = await marqs?.getSharedQueueDetails();
+
+  return json(details);
+}

apps/webapp/app/routes/api.v1.accounts.$accountId.connections.$clientSlug/CreateExternalConnectionService.server.ts

@@ -0,0 +1,49 @@
+import { CreateExternalConnectionBody } from "@trigger.dev/core";
+import { PrismaClientOrTransaction, prisma } from "~/db.server";
+import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { integrationAuthRepository } from "~/services/externalApis/integrationAuthRepository.server";
+
+export class CreateExternalConnectionService {
+  #prismaClient: PrismaClientOrTransaction;
+
+  constructor(prismaClient: PrismaClientOrTransaction = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(
+    accountIdentifier: string,
+    clientSlug: string,
+    environment: AuthenticatedEnvironment,
+    payload: CreateExternalConnectionBody
+  ) {
+    const externalAccount = await this.#prismaClient.externalAccount.upsert({
+      where: {
+        environmentId_identifier: {
+          environmentId: environment.id,
+          identifier: accountIdentifier,
+        },
+      },
+      create: {
+        environmentId: environment.id,
+        organizationId: environment.organizationId,
+        identifier: accountIdentifier,
+      },
+      update: {},
+    });
+
+    const integration = await this.#prismaClient.integration.findUniqueOrThrow({
+      where: {
+        organizationId_slug: {
+          organizationId: environment.organizationId,
+          slug: clientSlug,
+        },
+      },
+    });
+
+    return await integrationAuthRepository.createConnectionFromToken({
+      externalAccount: externalAccount,
+      integration,
+      token: payload,
+    });
+  }
+}

apps/webapp/app/routes/api.v1.accounts.$accountId.connections.$clientSlug/route.ts

@@ -1,15 +1,10 @@
 import type { ActionFunctionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
-import {
-  CreateExternalConnectionBody,
-  CreateExternalConnectionBodySchema,
-  ErrorWithStackSchema,
-} from "@trigger.dev/core";
+import { CreateExternalConnectionBodySchema, ErrorWithStackSchema } from "@trigger.dev/core";
 import { z } from "zod";
 import { generateErrorMessage } from "zod-error";
-import { PrismaClientOrTransaction, prisma } from "~/db.server";
-import { AuthenticatedEnvironment, authenticateApiRequest } from "~/services/apiAuth.server";
-import { integrationAuthRepository } from "~/services/externalApis/integrationAuthRepository.server";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { CreateExternalConnectionService } from "./CreateExternalConnectionService.server";
 
 const ParamsSchema = z.object({
   accountId: z.string(),
@@ -67,48 +62,3 @@ export async function action({ request, params }: ActionFunctionArgs) {
     return json({ message: parsedError.data.message }, { status: 500 });
   }
 }
-
-class CreateExternalConnectionService {
-  #prismaClient: PrismaClientOrTransaction;
-
-  constructor(prismaClient: PrismaClientOrTransaction = prisma) {
-    this.#prismaClient = prismaClient;
-  }
-
-  public async call(
-    accountIdentifier: string,
-    clientSlug: string,
-    environment: AuthenticatedEnvironment,
-    payload: CreateExternalConnectionBody
-  ) {
-    const externalAccount = await this.#prismaClient.externalAccount.upsert({
-      where: {
-        environmentId_identifier: {
-          environmentId: environment.id,
-          identifier: accountIdentifier,
-        },
-      },
-      create: {
-        environmentId: environment.id,
-        organizationId: environment.organizationId,
-        identifier: accountIdentifier,
-      },
-      update: {},
-    });
-
-    const integration = await this.#prismaClient.integration.findUniqueOrThrow({
-      where: {
-        organizationId_slug: {
-          organizationId: environment.organizationId,
-          slug: clientSlug,
-        },
-      },
-    });
-
-    return await integrationAuthRepository.createConnectionFromToken({
-      externalAccount: externalAccount,
-      integration,
-      token: payload,
-    });
-  }
-}

apps/webapp/app/routes/api.v1.endpoints.$environmentId.$endpointSlug.index.$indexHookIdentifier/TriggerEndpointIndexHookService.server.ts

@@ -1,73 +1,10 @@
-import { ActionFunctionArgs, LoaderFunctionArgs, json } from "@remix-run/server-runtime";
-import { RuntimeEnvironmentType } from "@trigger.dev/database";
 import { z } from "zod";
-import { $transaction, PrismaClient, prisma } from "~/db.server";
+import type { PrismaClient } from "~/db.server";
+import { $transaction, prisma } from "~/db.server";
 import { logger } from "~/services/logger.server";
 import { workerQueue } from "~/services/worker.server";
-import { safeJsonParse } from "~/utils/json";
-
-const ParamsSchema = z.object({
-  environmentId: z.string(),
-  endpointSlug: z.string(),
-  indexHookIdentifier: z.string(),
-});
-
-export async function loader({ params }: LoaderFunctionArgs) {
-  const parsedParams = ParamsSchema.safeParse(params);
-
-  if (!parsedParams.success) {
-    return {
-      status: 400,
-      json: {
-        error: "Invalid params",
-      },
-    };
-  }
-
-  const { environmentId, endpointSlug, indexHookIdentifier } = parsedParams.data;
-
-  const service = new TriggerEndpointIndexHookService();
-
-  await service.call({
-    environmentId,
-    endpointSlug,
-    indexHookIdentifier,
-  });
-
-  return json({
-    ok: true,
-  });
-}
-
-export async function action({ request, params }: ActionFunctionArgs) {
-  const parsedParams = ParamsSchema.safeParse(params);
-
-  if (!parsedParams.success) {
-    return {
-      status: 400,
-      json: {
-        error: "Invalid params",
-      },
-    };
-  }
-
-  const { environmentId, endpointSlug, indexHookIdentifier } = parsedParams.data;
-
-  const body = await request.text();
-
-  const service = new TriggerEndpointIndexHookService();
-
-  await service.call({
-    environmentId,
-    endpointSlug,
-    indexHookIdentifier,
-    body: body ? safeJsonParse(body) : undefined,
-  });
-
-  return json({
-    ok: true,
-  });
-}
+import { RuntimeEnvironmentType } from "~/database-types";
+import type { ParamsSchema } from "./route";
 
 type TriggerEndpointDeployHookOptions = z.infer<typeof ParamsSchema> & {
   body?: any;
@@ -106,7 +43,7 @@ export class TriggerEndpointIndexHookService {
         },
       });
 
-      if (!endpoint) {
+      if (!endpoint || !endpoint.url) {
         throw new Error("Endpoint not found");
       }
 

apps/webapp/app/routes/api.v1.endpoints.$environmentId.$endpointSlug.index.$indexHookIdentifier/route.ts

@@ -0,0 +1,67 @@
+import { ActionFunctionArgs, LoaderFunctionArgs, json } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { safeJsonParse } from "~/utils/json";
+import { TriggerEndpointIndexHookService } from "./TriggerEndpointIndexHookService.server";
+
+export const ParamsSchema = z.object({
+  environmentId: z.string(),
+  endpointSlug: z.string(),
+  indexHookIdentifier: z.string(),
+});
+
+export async function loader({ params }: LoaderFunctionArgs) {
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return {
+      status: 400,
+      json: {
+        error: "Invalid params",
+      },
+    };
+  }
+
+  const { environmentId, endpointSlug, indexHookIdentifier } = parsedParams.data;
+
+  const service = new TriggerEndpointIndexHookService();
+
+  await service.call({
+    environmentId,
+    endpointSlug,
+    indexHookIdentifier,
+  });
+
+  return json({
+    ok: true,
+  });
+}
+
+export async function action({ request, params }: ActionFunctionArgs) {
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return {
+      status: 400,
+      json: {
+        error: "Invalid params",
+      },
+    };
+  }
+
+  const { environmentId, endpointSlug, indexHookIdentifier } = parsedParams.data;
+
+  const body = await request.text();
+
+  const service = new TriggerEndpointIndexHookService();
+
+  await service.call({
+    environmentId,
+    endpointSlug,
+    indexHookIdentifier,
+    body: body ? safeJsonParse(body) : undefined,
+  });
+
+  return json({
+    ok: true,
+  });
+}

apps/webapp/app/routes/api.v1.projects.$projectRef.alertChannels.ts

@@ -54,6 +54,7 @@ export async function action({ request, params }: ActionFunctionArgs) {
           email: body.data.channelData.email,
         },
         deduplicationKey: body.data.deduplicationKey,
+        environmentTypes: body.data.environmentTypes,
       });
 
       return json(await ApiAlertChannelPresenter.alertChannelToApi(alertChannel));
@@ -75,6 +76,7 @@ export async function action({ request, params }: ActionFunctionArgs) {
           secret: body.data.channelData.secret,
         },
         deduplicationKey: body.data.deduplicationKey,
+        environmentTypes: body.data.environmentTypes,
       });
 
       return json(await ApiAlertChannelPresenter.alertChannelToApi(alertChannel));

apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.$name.ts

@@ -0,0 +1,137 @@
+import { ActionFunctionArgs, LoaderFunctionArgs, json } from "@remix-run/server-runtime";
+import { UpdateEnvironmentVariableRequestBody } from "@trigger.dev/core/v3";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import {
+  authenticateProjectApiKeyOrPersonalAccessToken,
+  authenticatedEnvironmentForAuthentication,
+} from "~/services/apiAuth.server";
+import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server";
+
+const ParamsSchema = z.object({
+  projectRef: z.string(),
+  slug: z.string(),
+  name: z.string(),
+});
+
+export async function action({ params, request }: ActionFunctionArgs) {
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return json({ error: "Invalid params" }, { status: 400 });
+  }
+
+  const authenticationResult = await authenticateProjectApiKeyOrPersonalAccessToken(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const environment = await authenticatedEnvironmentForAuthentication(
+    authenticationResult,
+    parsedParams.data.projectRef,
+    parsedParams.data.slug
+  );
+
+  // Find the environment variable
+  const variable = await prisma.environmentVariable.findFirst({
+    where: {
+      key: parsedParams.data.name,
+      projectId: environment.project.id,
+    },
+  });
+
+  if (!variable) {
+    return json({ error: "Environment variable not found" }, { status: 404 });
+  }
+
+  const repository = new EnvironmentVariablesRepository();
+
+  switch (request.method.toUpperCase()) {
+    case "DELETE": {
+      const result = await repository.deleteValue(environment.project.id, {
+        id: variable.id,
+        environmentId: environment.id,
+      });
+
+      if (result.success) {
+        return json({ success: true });
+      } else {
+        return json({ error: result.error }, { status: 400 });
+      }
+    }
+    case "PUT":
+    case "POST": {
+      const jsonBody = await request.json();
+
+      const body = UpdateEnvironmentVariableRequestBody.safeParse(jsonBody);
+
+      if (!body.success) {
+        return json({ error: "Invalid request body", issues: body.error.issues }, { status: 400 });
+      }
+
+      const result = await repository.edit(environment.project.id, {
+        values: [
+          {
+            value: body.data.value,
+            environmentId: environment.id,
+          },
+        ],
+        id: variable.id,
+        keepEmptyValues: true,
+      });
+
+      if (result.success) {
+        return json({ success: true });
+      } else {
+        return json({ error: result.error }, { status: 400 });
+      }
+    }
+  }
+}
+
+export async function loader({ params, request }: LoaderFunctionArgs) {
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return json({ error: "Invalid params" }, { status: 400 });
+  }
+
+  const authenticationResult = await authenticateProjectApiKeyOrPersonalAccessToken(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const environment = await authenticatedEnvironmentForAuthentication(
+    authenticationResult,
+    parsedParams.data.projectRef,
+    parsedParams.data.slug
+  );
+
+  // Find the environment variable
+  const variable = await prisma.environmentVariable.findFirst({
+    where: {
+      key: parsedParams.data.name,
+      projectId: environment.project.id,
+    },
+  });
+
+  if (!variable) {
+    return json({ error: "Environment variable not found" }, { status: 404 });
+  }
+
+  const repository = new EnvironmentVariablesRepository();
+
+  const variables = await repository.getEnvironment(environment.project.id, environment.id, true);
+
+  const environmentVariable = variables.find((v) => v.key === parsedParams.data.name);
+
+  if (!environmentVariable) {
+    return json({ error: "Environment variable not found" }, { status: 404 });
+  }
+
+  return json({
+    value: environmentVariable.value,
+  });
+}

apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.import.ts

@@ -0,0 +1,84 @@
+import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
+import { ImportEnvironmentVariablesRequestBody } from "@trigger.dev/core/v3";
+import { parse } from "dotenv";
+import { z } from "zod";
+import {
+  authenticateProjectApiKeyOrPersonalAccessToken,
+  authenticatedEnvironmentForAuthentication,
+} from "~/services/apiAuth.server";
+import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server";
+
+const ParamsSchema = z.object({
+  projectRef: z.string(),
+  slug: z.string(),
+});
+
+export async function action({ params, request }: ActionFunctionArgs) {
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return json({ error: "Invalid params" }, { status: 400 });
+  }
+
+  const authenticationResult = await authenticateProjectApiKeyOrPersonalAccessToken(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const environment = await authenticatedEnvironmentForAuthentication(
+    authenticationResult,
+    parsedParams.data.projectRef,
+    parsedParams.data.slug
+  );
+
+  const repository = new EnvironmentVariablesRepository();
+
+  const body = await parseImportBody(request);
+
+  const result = await repository.create(environment.project.id, {
+    override: typeof body.override === "boolean" ? body.override : false,
+    environmentIds: [environment.id],
+    variables: Object.entries(body.variables).map(([key, value]) => ({
+      key,
+      value,
+    })),
+  });
+
+  if (result.success) {
+    return json({ success: true });
+  } else {
+    return json({ error: result.error, variableErrors: result.variableErrors }, { status: 400 });
+  }
+}
+
+async function parseImportBody(request: Request): Promise<ImportEnvironmentVariablesRequestBody> {
+  const contentType = request.headers.get("content-type") ?? "application/json";
+
+  if (contentType.includes("multipart/form-data")) {
+    const formData = await request.formData();
+
+    const file = formData.get("variables");
+    const override = formData.get("override") === "true";
+
+    if (file instanceof File) {
+      const buffer = await file.arrayBuffer();
+
+      const variables = parse(Buffer.from(buffer));
+
+      return { variables, override };
+    } else {
+      throw json({ error: "Invalid file" }, { status: 400 });
+    }
+  } else {
+    const rawBody = await request.json();
+
+    const body = ImportEnvironmentVariablesRequestBody.safeParse(rawBody);
+
+    if (!body.success) {
+      throw json({ error: "Invalid body" }, { status: 400 });
+    }
+
+    return body.data;
+  }
+}

apps/webapp/app/routes/api.v1.projects.$projectRef.envvars.$slug.ts

@@ -0,0 +1,86 @@
+import { ActionFunctionArgs, LoaderFunctionArgs, json } from "@remix-run/server-runtime";
+import { CreateEnvironmentVariableRequestBody } from "@trigger.dev/core/v3";
+import { z } from "zod";
+import {
+  authenticateProjectApiKeyOrPersonalAccessToken,
+  authenticatedEnvironmentForAuthentication,
+} from "~/services/apiAuth.server";
+import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server";
+
+const ParamsSchema = z.object({
+  projectRef: z.string(),
+  slug: z.string(),
+});
+
+export async function action({ params, request }: ActionFunctionArgs) {
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return json({ error: "Invalid params" }, { status: 400 });
+  }
+
+  const authenticationResult = await authenticateProjectApiKeyOrPersonalAccessToken(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const environment = await authenticatedEnvironmentForAuthentication(
+    authenticationResult,
+    parsedParams.data.projectRef,
+    parsedParams.data.slug
+  );
+
+  const jsonBody = await request.json();
+
+  const body = CreateEnvironmentVariableRequestBody.safeParse(jsonBody);
+
+  if (!body.success) {
+    return json({ error: "Invalid request body", issues: body.error.issues }, { status: 400 });
+  }
+
+  const repository = new EnvironmentVariablesRepository();
+
+  const result = await repository.create(environment.project.id, {
+    override: true,
+    environmentIds: [environment.id],
+    variables: [
+      {
+        key: body.data.name,
+        value: body.data.value,
+      },
+    ],
+  });
+
+  if (result.success) {
+    return json({ success: true });
+  } else {
+    return json({ error: result.error, variableErrors: result.variableErrors }, { status: 400 });
+  }
+}
+
+export async function loader({ params, request }: LoaderFunctionArgs) {
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return json({ error: "Invalid params" }, { status: 400 });
+  }
+
+  const authenticationResult = await authenticateProjectApiKeyOrPersonalAccessToken(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const environment = await authenticatedEnvironmentForAuthentication(
+    authenticationResult,
+    parsedParams.data.projectRef,
+    parsedParams.data.slug
+  );
+
+  const repository = new EnvironmentVariablesRepository();
+
+  const variables = await repository.getEnvironment(environment.project.id, environment.id, true);
+
+  return json(variables.map((variable) => ({ name: variable.key, value: variable.value })));
+}

apps/webapp/app/routes/api.v1.projects.$projectRef.runs.ts

@@ -0,0 +1,62 @@
+import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { ValidationError } from "zod-validation-error";
+import { findProjectByRef } from "~/models/project.server";
+import { ApiRunListPresenter } from "~/presenters/v3/ApiRunListPresenter.server";
+import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+import { apiCors } from "~/utils/apiCors";
+
+const ParamsSchema = z.object({
+  projectRef: z.string(),
+});
+
+export async function loader({ request, params }: LoaderFunctionArgs) {
+  if (request.method.toUpperCase() === "OPTIONS") {
+    return apiCors(request, json({}));
+  }
+
+  const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
+
+  if (!authenticationResult) {
+    return apiCors(request, json({ error: "Invalid or Missing API key" }, { status: 401 }));
+  }
+
+  const $params = ParamsSchema.safeParse(params);
+
+  if (!$params.success) {
+    return json({ error: "Invalid params" }, { status: 400 });
+  }
+
+  const project = await findProjectByRef($params.data.projectRef, authenticationResult.userId);
+
+  if (!project) {
+    return json({ error: "Project not found" }, { status: 404 });
+  }
+
+  const url = new URL(request.url);
+
+  const presenter = new ApiRunListPresenter();
+
+  try {
+    const result = await presenter.call(project, url.searchParams);
+
+    if (!result) {
+      return apiCors(request, json({ data: [] }));
+    }
+
+    return apiCors(request, json(result));
+  } catch (error) {
+    if (error instanceof ValidationError) {
+      return apiCors(
+        request,
+        json({ error: "Query Error", details: error.details }, { status: 400 })
+      );
+    } else {
+      return apiCors(
+        request,
+        json({ error: error instanceof Error ? error.message : String(error) }, { status: 400 })
+      );
+    }
+  }
+}

apps/webapp/app/routes/api.v1.runs.$runId.logs/CreateRunLogService.server.ts

@@ -0,0 +1,20 @@
+import type { LogMessage } from "@trigger.dev/core";
+import type { PrismaClient } from "@trigger.dev/database";
+import { prisma } from "~/db.server";
+import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { logger } from "~/services/logger.server";
+
+export class CreateRunLogService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(environment: AuthenticatedEnvironment, runId: string, logMessage: LogMessage) {
+    // @ts-ignore
+    logger.debug(logMessage.message, logMessage.data ?? {});
+
+    return logMessage;
+  }
+}

apps/webapp/app/routes/api.v1.runs.$runId.logs/route.ts

@@ -1,13 +1,9 @@
-import type { Organization, RuntimeEnvironment } from "@trigger.dev/database";
 import type { ActionFunctionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
-import type { LogMessage } from "@trigger.dev/core";
 import { LogMessageSchema } from "@trigger.dev/core";
 import { z } from "zod";
-import type { PrismaClient } from "~/db.server";
-import { prisma } from "~/db.server";
-import { authenticateApiRequest, AuthenticatedEnvironment } from "~/services/apiAuth.server";
-import { logger } from "~/services/logger.server";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { CreateRunLogService } from "./CreateRunLogService.server";
 
 const ParamsSchema = z.object({
   runId: z.string(),
@@ -53,18 +49,3 @@ export async function action({ request, params }: ActionFunctionArgs) {
     return json({ error: "Something went wrong" }, { status: 500 });
   }
 }
-
-export class CreateRunLogService {
-  #prismaClient: PrismaClient;
-
-  constructor(prismaClient: PrismaClient = prisma) {
-    this.#prismaClient = prismaClient;
-  }
-
-  public async call(environment: AuthenticatedEnvironment, runId: string, logMessage: LogMessage) {
-    // @ts-ignore
-    logger.debug(logMessage.message, logMessage.data ?? {});
-
-    return logMessage;
-  }
-}

apps/webapp/app/routes/api.v1.runs.$runId.statuses.$id/SetStatusService.server.ts

@@ -0,0 +1,71 @@
+import {
+  StatusUpdate,
+  StatusHistory,
+  StatusHistorySchema,
+  StatusUpdateState,
+  StatusUpdateData,
+} from "@trigger.dev/core";
+import { PrismaClient } from "@trigger.dev/database";
+import { prisma, $transaction } from "~/db.server";
+
+export class SetStatusService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call(runId: string, id: string, status: StatusUpdate) {
+    const statusRecord = await $transaction(this.#prismaClient, async (tx) => {
+      const existingStatus = await tx.jobRunStatusRecord.findUnique({
+        where: {
+          runId_key: {
+            runId,
+            key: id,
+          },
+        },
+      });
+
+      const history: StatusHistory = [];
+      const historyResult = StatusHistorySchema.safeParse(existingStatus?.history);
+      if (historyResult.success) {
+        history.push(...historyResult.data);
+      }
+      if (existingStatus) {
+        history.push({
+          label: existingStatus.label,
+          state: (existingStatus.state ?? undefined) as StatusUpdateState,
+          data: (existingStatus.data ?? undefined) as StatusUpdateData,
+        });
+      }
+
+      const updatedStatus = await tx.jobRunStatusRecord.upsert({
+        where: {
+          runId_key: {
+            runId,
+            key: id,
+          },
+        },
+        create: {
+          key: id,
+          runId,
+          //this shouldn't ever use the id in reality, as the SDK makess it compulsory on the first call
+          label: status.label ?? id,
+          state: status.state,
+          data: status.data as any,
+          history: [],
+        },
+        update: {
+          label: status.label,
+          state: status.state,
+          data: status.data as any,
+          history: history as any[],
+        },
+      });
+
+      return updatedStatus;
+    });
+
+    return statusRecord;
+  }
+}

apps/webapp/app/routes/api.v1.runs.$runId.statuses.$id/route.ts

@@ -1,18 +1,10 @@
 import type { ActionFunctionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
-import {
-  JobRunStatusRecordSchema,
-  StatusHistory,
-  StatusHistorySchema,
-  StatusUpdate,
-  StatusUpdateData,
-  StatusUpdateSchema,
-  StatusUpdateState,
-} from "@trigger.dev/core";
+import { JobRunStatusRecordSchema, StatusUpdateSchema } from "@trigger.dev/core";
 import { z } from "zod";
-import { $transaction, PrismaClient, prisma } from "~/db.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
+import { SetStatusService } from "./SetStatusService.server";
 
 const ParamsSchema = z.object({
   runId: z.string(),
@@ -80,65 +72,3 @@ export async function action({ request, params }: ActionFunctionArgs) {
     return json({ error: "Something went wrong" }, { status: 500 });
   }
 }
-
-export class SetStatusService {
-  #prismaClient: PrismaClient;
-
-  constructor(prismaClient: PrismaClient = prisma) {
-    this.#prismaClient = prismaClient;
-  }
-
-  public async call(runId: string, id: string, status: StatusUpdate) {
-    const statusRecord = await $transaction(this.#prismaClient, async (tx) => {
-      const existingStatus = await tx.jobRunStatusRecord.findUnique({
-        where: {
-          runId_key: {
-            runId,
-            key: id,
-          },
-        },
-      });
-
-      const history: StatusHistory = [];
-      const historyResult = StatusHistorySchema.safeParse(existingStatus?.history);
-      if (historyResult.success) {
-        history.push(...historyResult.data);
-      }
-      if (existingStatus) {
-        history.push({
-          label: existingStatus.label,
-          state: (existingStatus.state ?? undefined) as StatusUpdateState,
-          data: (existingStatus.data ?? undefined) as StatusUpdateData,
-        });
-      }
-
-      const updatedStatus = await tx.jobRunStatusRecord.upsert({
-        where: {
-          runId_key: {
-            runId,
-            key: id,
-          },
-        },
-        create: {
-          key: id,
-          runId,
-          //this shouldn't ever use the id in reality, as the SDK makess it compulsory on the first call
-          label: status.label ?? id,
-          state: status.state,
-          data: status.data as any,
-          history: [],
-        },
-        update: {
-          label: status.label,
-          state: status.state,
-          data: status.data as any,
-          history: history as any[],
-        },
-      });
-
-      return updatedStatus;
-    });
-
-    return statusRecord;
-  }
-}

apps/webapp/app/routes/api.v1.runs.$runId.tasks.$id.callback.$secret.ts

@@ -1,3 +1,3 @@
-import { action } from "./api.v1.tasks.$id.callback.$secret";
+import { action } from "./api.v1.tasks.$id.callback.$secret/route";
 
 export { action };