Merge branch 'main' into v3/worker-attempt-creation

.changeset/afraid-sheep-joke.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fixes an issue with scoped packages in additionalPackages option

.changeset/blue-pumas-whisper.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/shopify": patch
+---
+
+improved error messages when a shopify webhook fails to register

.changeset/famous-boats-tease.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Better handle issues with resolving dependency versions during deploy

.changeset/lazy-files-lay.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Increased the timeout when canceling a checkpoint to 31s (to match the timeout on the server)

.changeset/nice-bulldogs-turn.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Better handle uncaught exceptions

.changeset/pre.json

@@ -44,9 +44,11 @@
     "@trigger.dev/yalt": "2.3.18"
   },
   "changesets": [
+    "afraid-sheep-joke",
     "angry-eagles-trade",
     "beige-pens-dance",
     "big-tomatoes-deliver",
+    "blue-pumas-whisper",
     "breezy-gorillas-mate",
     "chilled-hornets-move",
     "clean-pianos-listen",
@@ -59,6 +61,7 @@
     "dull-mangos-press",
     "eight-pumas-float",
     "eleven-paws-join",
+    "famous-boats-tease",
     "few-students-share",
     "funny-swans-destroy",
     "gorgeous-gorillas-compete",
@@ -67,6 +70,7 @@
     "khaki-poems-lay",
     "late-icons-lie",
     "late-steaks-behave",
+    "lazy-files-lay",
     "lemon-jobs-repair",
     "light-bulldogs-press",
     "light-dragons-complain",
@@ -79,6 +83,7 @@
     "mighty-flowers-train",
     "nasty-jars-pump",
     "new-rivers-tell",
+    "nice-bulldogs-turn",
     "ninety-pets-travel",
     "odd-poets-own",
     "pink-pumas-rhyme",
@@ -113,8 +118,10 @@
     "tender-moose-tell",
     "tender-oranges-rhyme",
     "thin-parents-heal",
+    "thirty-islands-kiss",
     "tidy-balloons-suffer",
     "tidy-dryers-sleep",
+    "tidy-tomatoes-explain",
     "tiny-doors-type",
     "tiny-elephants-scream",
     "tricky-bulldogs-heal",

.changeset/thirty-islands-kiss.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix for typo in v3 CLI login command

.changeset/tidy-tomatoes-explain.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Fix 3rd party otel propagation from breaking our Task Events data from being properly correlated to the correct trace

.env.example

@@ -11,11 +11,16 @@ REMIX_APP_PORT=3030
 APP_ENV=development
 APP_ORIGIN=http://localhost:3030
 NODE_ENV=development
+CLOUD_ENV=development
+V3_ENABLED=true
 
-# Redis is used for concurrency control
-# REDIS_HOST="localhost"
-# REDIS_PORT="6379"
-# REDIS_TLS_DISABLED="true"
+# Redis is used for the v3 queuing and v2 concurrency control
+REDIS_HOST="localhost"
+REDIS_PORT="6379"
+REDIS_TLS_DISABLED="true"
+
+DEV_OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:3030/otel"
+DEV_OTEL_BATCH_PROCESSING_ENABLED="0"
 
 # OPTIONAL VARIABLES
 # This is used for validating emails that are allowed to log in. Every email that do not match this regex will be rejected.

CONTRIBUTING.md

@@ -10,12 +10,12 @@ Thank you for helping us make Trigger.dev even better! 🤩
 
 The development branch is `main`. This is the branch that all pull
 requests should be made against. The changes on the `main`
-branch are tagged into a release monthly.
+branch are tagged into a release periodically.
 
 ### Prerequisites
 
-- [Node.js](https://nodejs.org/en) version >=16.x
-- [pnpm package manager](https://pnpm.io/installation) version 7
+- [Node.js](https://nodejs.org/en) version 20.11.1
+- [pnpm package manager](https://pnpm.io/installation) version 8.15.5
 - [Docker](https://www.docker.com/get-started/)
 
 ### Setup
@@ -33,15 +33,19 @@ branch are tagged into a release monthly.
    ```
    cd trigger.dev
    ```
-3. Install the required packages using pnpm.
+3. Ensure you are on the correct version of Node.js (20.11.1). If you are using `nvm`, there is an `.nvmrc` file that will automatically select the correct version of Node.js when you navigate to the repository.
+
+4. Run `corepack enable` to use the correct version of pnpm (`8.15.5`) as specified in the root `package.json` file.
+
+5. Install the required packages using pnpm.
    ```
    pnpm i
    ```
-4. Create your `.env` file
+6. Create your `.env` file
    ```
    cp .env.example .env
    ```
-5. Open it and generate a new value for `ENCRYPTION_KEY`:
+7. Open it and generate a new value for `ENCRYPTION_KEY`:
 
    `ENCRYPTION_KEY` is used to two-way encrypt OAuth access tokens and so you'll probably want to actually generate a unique value, and it must be a random 16 byte hex string. You can generate one with the following command:
 
@@ -51,7 +55,7 @@ branch are tagged into a release monthly.
 
    Feel free to update `SESSION_SECRET` and `MAGIC_LINK_SECRET` as well using the same method.
 
-6. Start Docker. This starts the required services like Postgres. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)
+8. Start Docker. This starts the required services like Postgres & Redis. If this is your first time using Docker, consider going through this [guide](DOCKER_INSTALLATION.md)
 
    ```
    pnpm run docker
@@ -59,19 +63,15 @@ branch are tagged into a release monthly.
 
    This will also start and run a local instance of [pgAdmin](https://www.pgadmin.org/) on [localhost:5480](http://localhost:5480), preconfigured with email `admin@example.com` and pwd `admin`. Then use `postgres` as the password to the Trigger.dev server.
 
-7. Migrate the database
+9. Migrate the database
    ```
    pnpm run db:migrate
    ```
-8. Build the app
-   ```
-   pnpm run build --filter webapp
-   ```
-9. Run the seed script
-   ```
-   pnpm run db:seed
-   ```
-10. Run the app. See the section below.
+10. Build the server app
+    ```
+    pnpm run build --filter webapp
+    ```
+11. Run the app. See the section below.
 
 ## Running
 
@@ -83,99 +83,92 @@ branch are tagged into a release monthly.
 
    It should run on port `3030`: [http://localhost:3030](http://localhost:3030/)
 
-2. Once the app is running click the magic link button and enter your email.
-3. Check your terminal, the magic link email should have printed out as following:
+2. Once the app is running click the magic link button and enter your email. You will automatically be logged in, since you are running locally. Create an Org and your first project in the dashboard.
 
-   ```sh
-   webapp:dev: Log in to Trigger.dev
-   webapp:dev:
-   webapp:dev: Click here to log in with this magic link
-   webapp:dev: [http://localhost:3030/magic?token=U2FsdGVkX18OvB0JxgaswTLCSbaRz%2FY82TN0EZWhSzFyZYwgG%2BIzKVTkeiaOtWfotPw7F8RwFzCHh53aBpMEu%2B%2B%2FItb%2FcJYh89MSjc3Pz92bevoEjqxSQ%2Ff%2BZbks09JOpqlBbYC3FzGWC8vuSVFBlxqLXxteSDLthZSUaC%2BS2LaA%2BJgp%2BLO7hgjAaC2lXbCHrM7MTgTdXOFt7i0Dvvuwz6%2BWY25RnfomZOPqDsyH0xz8Q2rzPTz0Xu53WSXrZ1hd]
-   webapp:dev:
-   webapp:dev: If you didn't try to log in, you can safely ignore this email.
-   ```
+## Manual testing using v3-catalog
 
-   Paste the magic link shown in your terminal into your browser to login.
+We use the `<root>/references/v3-catalog` subdirectory as a staging ground for testing changes to the SDK (`@trigger.dev/sdk` at `<root>/packages/trigger-sdk`), the Core package (`@trigger.dev/core` at `<root>packages/core`), the CLI (`trigger.dev` at `<root>/packages/cli-v3`) and the platform (The remix app at `<root>/apps/webapp`). The instructions below will get you started on using the `v3-catalog` for local development of Trigger.dev (v3).
 
-## Adding and running migrations
+### First-time setup
 
-1. Modify packages/database/prisma/schema.prisma file
-2. Change directory to the packages/database folder
-   ```sh
-   cd packages/database
-   ```
-3. Generate the Prisma client
+First, make sure you are running the webapp according to the instructions above. Then:
 
-   ```sh
-   pnpm run generate
-   ```
+1. In Postgres go to the "Organizations" table and on your org set the `v3Enabled` column to `true`.
 
-   The above updates the prisma client generated into node_modules/.prisma/client folder. This helps with typing of relevant prisma models. It ensures typescript
-   recognizes fields added or removed from a model and type-checks appropriately.
+2. Visit http://localhost:3030 in your browser and create a new V3 project called "v3-catalog". If you don't see an option for V3, you haven't set the `v3Enabled` flag to true.
 
-4. Create and apply the migrations
+3. In Postgres go to the "Projects" table and for the project you create change the `externalRef` to `yubjwjsfkxnylobaqvqz`.
 
-   ```
-   pnpm run db:migrate:dev
-   ```
-
-   This creates a migration file and executes the migrations against your database and applies changes to the database schema(s)
-
-5. Commit generated migrations as well as changes to the schema.prisma file
-6. If you're using VSCode you may need to restart the Typescript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`.
-
-## Testing CLI changes
-
-To test CLI changes, follow the steps below:
-
-1. Build the CLI and watch for changes
+4. Build the CLI
 
 ```sh
-cd packages/cli
-pnpm run dev
+pnpm run build --filter trigger.dev
 ```
 
-2. Open a new Terminal window and run the webapp locally and then create a new project in the dashboard. Copy out the dev API key.
-
-3. Create a new temporary Next.js app in references directory
+5. Change into the `<root>/references/v3-catalog` directory and authorize the CLI to the local server:
 
 ```sh
-cd ./references
-pnpm create next-app@latest test-cli --ts --no-eslint --tailwind --app --src-dir --import-alias "@/*"
+cd references/v3-catalog
+pnpm exec triggerdev login -a http://localhost:3030
 ```
 
-4. Then once that's finished, add the `@trigger.dev/cli` to the `devDependencies` of the newly created Next.js app's `package.json` file, like so:
+This will open a new browser window and authorize the CLI against your local user account.
 
-```json
-{
-  // other package.json properties
-  "devDependencies": { "@trigger.dev/cli": "workspace:*" }
-}
-```
-
-5. Back in the terminal, navigate into the reference, and initialize the CLI. When prompted, select `self-hosted` and enter `localhost:3030` if you are testing against the local instance of Trigger.dev, or you can just use the Trigger.dev cloud. When asked for an API key, use the key you copied earlier.
+You can optionally pass a `--profile` flag to the `login` command, which will allow you to use the CLI with separate accounts/servers. We suggest using a profile called `local` for your local development:
 
 ```sh
-cd ./test-cli
-pnpm i
-pnpm exec trigger-cli init
+cd references/v3-catalog
+pnpm exec triggerdev login -a http://localhost:3030 --profile local
+# later when you run the dev or deploy command:
+pnpm exec triggerdev dev --profile local
+pnpm exec triggerdev deploy --profile local
 ```
 
-6. If you are just testing the `init` command, you can stop here. If you'd like to test the `dev` command, first start the Next.js app on port 3000:
+### Running
+
+The following steps should be followed any time you start working on a new feature you want to test in v3:
+
+1. Make sure the webapp is running on localhost:3030
+
+2. Open a terminal window and build the CLI and watch for changes
 
 ```sh
-pnpm run dev
+pnpm run dev --filter trigger.dev
 ```
 
-7. Open a new terminal window, and then run the `dev` command like so:
+2. Open a new terminal window, and anytime changes are made to the `@trigger.dev/core` package, you'll need to manually rebuild the CLI:
 
 ```sh
-pnpm exec trigger-cli dev
+pnpm run build --filter trigger.dev
 ```
 
-8. Please remember to delete the temporary project you created after you've tested the changes, and before you raise a PR.
+Note: You do not need to do the same for `@trigger.dev/sdk`, just core.
 
-## Running end-to-end webapp tests
+3. Open another terminal window, and change into the `<root>/references/v3-catalog` directory.
+
+4. Run the `dev` command, which will register all the local tasks with the platform and allow you to start testing task execution:
+
+```sh
+# in <root>/references/v3-catalog
+pnpm exec triggerdev dev
+```
+
+If you want additional debug logging, you can use the `--log-level debug` flag:
+
+```sh
+# in <root>/references/v3-catalog
+pnpm exec triggerdev dev --log-level debug
+```
+
+5. If you make any changes in the CLI/Core/SDK, you'll need to `CTRL+C` to exit the `dev` command and restart it to pickup changes. Any changes to the files inside of the `v3-catalog/src/trigger` dir will automatically be rebuilt by the `dev` command.
+
+6. Navigate to the `v3-catalog` project in your local dashboard at localhost:3030 and you should see the list of tasks.
+
+7. Go to the "Test" page in the sidebar and select a task. Then enter a payload and click "Run test". You can tell what the payloads should be by looking at the relevant task file inside the `/references/v3-catalog/src/trigger` folder. Many of them accept an empty payload.
+
+8. Feel free to add additional files in `v3-catalog/src/trigger` to test out specific aspects of the system, or add in edge cases.
+
+## Running end-to-end webapp tests (deprecated)
 
 To run the end-to-end tests, follow the steps below:
 
@@ -223,6 +216,26 @@ The end-to-end tests use a `setup` and `teardown` script to seed the database wi
 pnpm run db:studio
 ```
 
+## Adding and running migrations
+
+1. Modify packages/database/prisma/schema.prisma file
+2. Change directory to the packages/database folder
+
+   ```sh
+   cd packages/database
+   ```
+
+3. Create and apply the migrations
+
+   ```
+   pnpm run db:migrate:dev
+   ```
+
+   This creates a migration file and executes the migrations against your database and applies changes to the database schema(s)
+
+4. Commit generated migrations as well as changes to the schema.prisma file
+5. If you're using VSCode you may need to restart the Typescript server in the webapp to get updated type inference. Open a TypeScript file, then open the Command Palette (View > Command Palette) and run `TypeScript: Restart TS server`.
+
 ## Add sample jobs
 
 The [references/job-catalog](./references/job-catalog/) project defines simple jobs you can get started with.

apps/kubernetes-provider/src/index.ts

@@ -10,6 +10,7 @@ import {
 import { Machine, PostStartCauses, PreStopCauses, EnvironmentType } from "@trigger.dev/core/v3";
 import { randomUUID } from "crypto";
 import { TaskMonitor } from "./taskMonitor";
+import { PodCleaner } from "./podCleaner";
 
 const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
 const NODE_NAME = process.env.NODE_NAME || "local";
@@ -543,3 +544,11 @@ const taskMonitor = new TaskMonitor({
 });
 
 taskMonitor.start();
+
+const podCleaner = new PodCleaner({
+  runtimeEnv: RUNTIME_ENV,
+  namespace: "default",
+  intervalInSeconds: 300,
+});
+
+podCleaner.start();

apps/kubernetes-provider/src/podCleaner.ts

@@ -0,0 +1,264 @@
+import * as k8s from "@kubernetes/client-node";
+import { SimpleLogger } from "@trigger.dev/core-apps";
+
+type PodCleanerOptions = {
+  runtimeEnv: "local" | "kubernetes";
+  namespace?: string;
+  intervalInSeconds?: number;
+};
+
+export class PodCleaner {
+  private enabled = false;
+  private namespace = "default";
+  private intervalInSeconds = 300;
+
+  private logger = new SimpleLogger("[PodCleaner]");
+  private k8sClient: {
+    core: k8s.CoreV1Api;
+    kubeConfig: k8s.KubeConfig;
+  };
+
+  constructor(private opts: PodCleanerOptions) {
+    if (opts.namespace) {
+      this.namespace = opts.namespace;
+    }
+
+    if (opts.intervalInSeconds) {
+      this.intervalInSeconds = opts.intervalInSeconds;
+    }
+
+    this.k8sClient = this.#createK8sClient();
+  }
+
+  #createK8sClient() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (this.opts.runtimeEnv === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (this.opts.runtimeEnv === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      kubeConfig: kubeConfig,
+    };
+  }
+
+  #isRecord(candidate: unknown): candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  #logK8sError(err: unknown, debugOnly = false) {
+    if (debugOnly) {
+      this.logger.debug("K8s API Error", err);
+    } else {
+      this.logger.error("K8s API Error", err);
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
+      this.#logK8sError(err);
+      return;
+    }
+
+    this.#logK8sError(err, true);
+
+    if (typeof err.body.message === "string") {
+      this.#logK8sError({ message: err.body.message });
+      return;
+    }
+
+    this.#logK8sError({ body: err.body });
+  }
+
+  async #deletePods(opts: {
+    namespace: string;
+    dryRun?: boolean;
+    fieldSelector?: string;
+    labelSelector?: string;
+  }) {
+    return await this.k8sClient.core
+      .deleteCollectionNamespacedPod(
+        opts.namespace,
+        undefined, // pretty
+        undefined, // continue
+        opts.dryRun ? "All" : undefined,
+        opts.fieldSelector,
+        undefined, // gracePeriodSeconds
+        opts.labelSelector
+      )
+      .catch(this.#handleK8sError.bind(this));
+  }
+
+  async #deleteCompletedRuns() {
+    this.logger.log("Deleting completed runs");
+
+    const start = Date.now();
+
+    const result = await this.#deletePods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=Succeeded",
+      labelSelector: "app=task-run",
+    });
+
+    const elapsedMs = Date.now() - start;
+
+    if (!result) {
+      this.logger.log("Deleting completed runs: No delete result", { elapsedMs });
+      return;
+    }
+
+    const total = (result.response as any)?.body?.items?.length ?? 0;
+
+    this.logger.log("Deleting completed runs: Done", { total, elapsedMs });
+  }
+
+  async #deleteFailedRuns() {
+    this.logger.log("Deleting failed runs");
+
+    const start = Date.now();
+
+    const result = await this.#deletePods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=Failed",
+      labelSelector: "app=task-run",
+    });
+
+    const elapsedMs = Date.now() - start;
+
+    if (!result) {
+      this.logger.log("Deleting failed runs: No delete result", { elapsedMs });
+      return;
+    }
+
+    const total = (result.response as any)?.body?.items?.length ?? 0;
+
+    this.logger.log("Deleting failed runs: Done", { total, elapsedMs });
+  }
+
+  async #deleteUnrecoverableRuns() {
+    await this.#deletePods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=?",
+      labelSelector: "app=task-run",
+    });
+  }
+
+  async start() {
+    this.enabled = true;
+    this.logger.log("Starting");
+
+    const completedInterval = setInterval(async () => {
+      if (!this.enabled) {
+        clearInterval(completedInterval);
+        return;
+      }
+
+      try {
+        await this.#deleteCompletedRuns();
+      } catch (error) {
+        this.logger.error("Error deleting completed runs", error);
+      }
+    }, this.intervalInSeconds * 1000);
+
+    const failedInterval = setInterval(
+      async () => {
+        if (!this.enabled) {
+          clearInterval(failedInterval);
+          return;
+        }
+
+        try {
+          await this.#deleteFailedRuns();
+        } catch (error) {
+          this.logger.error("Error deleting completed runs", error);
+        }
+      },
+      // Use a longer interval for failed runs. This is only a backup in case the task monitor fails.
+      2 * this.intervalInSeconds * 1000
+    );
+
+    // this.#launchTests();
+  }
+
+  async stop() {
+    if (!this.enabled) {
+      return;
+    }
+
+    this.enabled = false;
+    this.logger.log("Shutting down..");
+  }
+
+  async #launchTests() {
+    const createPod = async (
+      container: k8s.V1Container,
+      name: string,
+      labels?: Record<string, string>
+    ) => {
+      this.logger.log("Creating pod:", name);
+
+      const pod = {
+        metadata: {
+          name,
+          labels,
+        },
+        spec: {
+          restartPolicy: "Never",
+          automountServiceAccountToken: false,
+          terminationGracePeriodSeconds: 1,
+          containers: [container],
+        },
+      } satisfies k8s.V1Pod;
+
+      await this.k8sClient.core
+        .createNamespacedPod(this.namespace, pod)
+        .catch(this.#handleK8sError.bind(this));
+    };
+
+    const createIdlePod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: ["-c", "sleep infinity"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createCompletedPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: ["-c", "true"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    const createFailedPod = async (name: string, labels?: Record<string, string>) => {
+      const container = {
+        name,
+        image: "docker.io/library/busybox",
+        command: ["sh"],
+        args: ["-c", "false"],
+      } satisfies k8s.V1Container;
+
+      await createPod(container, name, labels);
+    };
+
+    await createIdlePod("test-idle-1", { app: "task-run" });
+    await createFailedPod("test-failed-1", { app: "task-run" });
+    await createCompletedPod("test-completed-1", { app: "task-run" });
+  }
+}

apps/kubernetes-provider/src/taskMonitor.ts

@@ -30,10 +30,12 @@ type TaskMonitorOptions = {
 
 export class TaskMonitor {
   #enabled = false;
+
   #logger = new SimpleLogger("[TaskMonitor]");
   #taskInformer: ReturnType<typeof k8s.makeInformer<k8s.V1Pod>>;
   #processedPods = new Map<string, number>();
   #queue = new PQueue({ concurrency: 10 });
+
   #k8sClient: {
     core: k8s.CoreV1Api;
     kubeConfig: k8s.KubeConfig;
@@ -44,6 +46,10 @@ export class TaskMonitor {
   private labelSelector = "app in (task-index, task-run)";
 
   constructor(private opts: TaskMonitorOptions) {
+    if (opts.namespace) {
+      this.namespace = opts.namespace;
+    }
+
     this.#k8sClient = this.#createK8sClient();
 
     this.#taskInformer = this.#createTaskInformer();

apps/webapp/app/components/Feedback.tsx

@@ -115,14 +115,7 @@ export function Feedback({ button, defaultValue = "bug" }: FeedbackProps) {
               Docs
             </LinkButton>
             <LinkButton
-              to={docsPath("v3/introduction")}
-              variant="tertiary/medium"
-              LeadingIcon={BookOpenIcon}
-            >
-              v3 Docs (Developer preview)
-            </LinkButton>
-            <LinkButton
-              to={"https://trigger.openstatus.dev/"}
+              to={"https://status.trigger.dev/"}
               variant="tertiary/medium"
               LeadingIcon={ActivityIcon}
             >

apps/webapp/app/components/admin/debugTooltip.tsx

@@ -20,9 +20,11 @@ export function AdminDebugTooltip({ children }: { children: React.ReactNode }) {
     <TooltipProvider>
       <Tooltip>
         <TooltipTrigger>
-          <ShieldCheckIcon className="h-5 w-5" />
+          <ShieldCheckIcon className="size-5" />
         </TooltipTrigger>
-        <TooltipContent className="flex items-center gap-1">{children}</TooltipContent>
+        <TooltipContent className="flex max-h-[90vh] items-center gap-1 overflow-y-auto">
+          {children}
+        </TooltipContent>
       </Tooltip>
     </TooltipProvider>
   );

apps/webapp/app/components/billing/PricingTiers.tsx

@@ -290,6 +290,7 @@ export function TierPro({
           options={concurrencyTiers.map((c) => ({ label: `Up to ${c.upto}`, value: c.code }))}
           fullWidth
           value={concurrentBracketCode}
+          variant="primary"
           onChange={(v) => setConcurrentBracketCode(v)}
         />
         <div className="py-6">

apps/webapp/app/components/environments/EnvironmentLabel.tsx

@@ -1,5 +1,7 @@
-import { RuntimeEnvironment } from "~/models/runtimeEnvironment.server";
+import type { RuntimeEnvironment } from "~/models/runtimeEnvironment.server";
 import { cn } from "~/utils/cn";
+import { sortEnvironments } from "~/utils/environmentSort";
+import { SimpleTooltip } from "../primitives/Tooltip";
 
 type Environment = Pick<RuntimeEnvironment, "type">;
 const variants = {
@@ -7,6 +9,30 @@ const variants = {
   large: "h-6 text-xs px-1.5 rounded",
 };
 
+export function EnvironmentTypeLabel({
+  environment,
+  size = "small",
+  className,
+}: {
+  environment: Environment;
+  size?: keyof typeof variants;
+  className?: string;
+}) {
+  return (
+    <span
+      className={cn(
+        "text-midnight-900 inline-flex items-center justify-center whitespace-nowrap border font-medium uppercase tracking-wider",
+        environmentBorderClassName(environment),
+        environmentTextClassName(environment),
+        variants[size],
+        className
+      )}
+    >
+      {environmentTypeTitle(environment)}
+    </span>
+  );
+}
+
 export function EnvironmentLabel({
   environment,
   size = "small",
@@ -33,6 +59,74 @@ export function EnvironmentLabel({
   );
 }
 
+type EnvironmentWithUsername = Environment & { userName?: string };
+
+export function EnvironmentLabels({
+  environments,
+  size = "small",
+  className,
+}: {
+  environments: EnvironmentWithUsername[];
+  size?: keyof typeof variants;
+  className?: string;
+}) {
+  const devEnvironments = sortEnvironments(
+    environments.filter((env) => env.type === "DEVELOPMENT")
+  );
+  const firstDevEnvironment = devEnvironments[0];
+  const otherDevEnvironments = devEnvironments.slice(1);
+  const otherEnvironments = environments.filter((env) => env.type !== "DEVELOPMENT");
+
+  return (
+    <div className={cn("flex items-baseline gap-2", className)}>
+      {firstDevEnvironment && (
+        <EnvironmentLabel
+          environment={firstDevEnvironment}
+          userName={firstDevEnvironment.userName}
+          size={size}
+        />
+      )}
+      {otherDevEnvironments.length > 0 ? (
+        <SimpleTooltip
+          disableHoverableContent
+          button={
+            <span
+              className={cn(
+                "inline-flex items-center justify-center whitespace-nowrap border font-medium uppercase tracking-wider",
+                environmentBorderClassName({ type: "DEVELOPMENT" }),
+                environmentTextClassName({ type: "DEVELOPMENT" }),
+                variants[size]
+              )}
+            >
+              +{otherDevEnvironments.length}
+            </span>
+          }
+          content={
+            <div className="flex gap-1 py-1">
+              {otherDevEnvironments.map((environment, index) => (
+                <EnvironmentLabel
+                  key={index}
+                  environment={environment}
+                  userName={environment.userName}
+                  size={size}
+                />
+              ))}
+            </div>
+          }
+        />
+      ) : null}
+      {otherEnvironments.map((environment, index) => (
+        <EnvironmentLabel
+          key={index}
+          environment={environment}
+          userName={environment.userName}
+          size={size}
+        />
+      ))}
+    </div>
+  );
+}
+
 export function environmentTitle(environment: Environment, username?: string) {
   switch (environment.type) {
     case "PRODUCTION":
@@ -46,6 +140,19 @@ export function environmentTitle(environment: Environment, username?: string) {
   }
 }
 
+export function environmentTypeTitle(environment: Environment) {
+  switch (environment.type) {
+    case "PRODUCTION":
+      return "Prod";
+    case "STAGING":
+      return "Staging";
+    case "DEVELOPMENT":
+      return "Dev";
+    case "PREVIEW":
+      return "Preview";
+  }
+}
+
 export function environmentColorClassName(environment: Environment) {
   switch (environment.type) {
     case "PRODUCTION":

apps/webapp/app/components/integrations/ConnectToOAuthForm.tsx

@@ -12,7 +12,7 @@ import { ApiAuthenticationMethodOAuth2, Integration, Scope } from "~/services/ex
 import { cn } from "~/utils/cn";
 import { CodeBlock } from "../code/CodeBlock";
 import { Button } from "../primitives/Buttons";
-import { Checkbox } from "../primitives/Checkbox";
+import { CheckboxWithLabel } from "../primitives/Checkbox";
 import { Fieldset } from "../primitives/Fieldset";
 import { FormError } from "../primitives/FormError";
 import { Header2, Header3 } from "../primitives/Headers";
@@ -123,7 +123,7 @@ export function ConnectToOAuthForm({
           <Paragraph variant="small" className="mb-2">
             To use your own OAuth app, check the option below and insert the details.
           </Paragraph>
-          <Checkbox
+          <CheckboxWithLabel
             id="hasCustomClient"
             label="Use my OAuth App"
             variant="simple/small"
@@ -200,7 +200,7 @@ export function ConnectToOAuthForm({
               )}
               {authMethod.scopes.map((s) => {
                 return (
-                  <Checkbox
+                  <CheckboxWithLabel
                     key={s.name}
                     id={s.name}
                     value={s.name}

apps/webapp/app/components/integrations/UpdateOAuthForm.tsx

@@ -10,7 +10,7 @@ import { ApiAuthenticationMethodOAuth2, Integration, Scope } from "~/services/ex
 import { cn } from "~/utils/cn";
 import { CodeBlock } from "../code/CodeBlock";
 import { Button } from "../primitives/Buttons";
-import { Checkbox } from "../primitives/Checkbox";
+import { CheckboxWithLabel } from "../primitives/Checkbox";
 import { Fieldset } from "../primitives/Fieldset";
 import { FormError } from "../primitives/FormError";
 import { Header2, Header3 } from "../primitives/Headers";
@@ -113,7 +113,7 @@ export function UpdateOAuthForm({
           <Paragraph variant="small" className="mb-2">
             To use your own OAuth app, check the option below and insert the details.
           </Paragraph>
-          <Checkbox
+          <CheckboxWithLabel
             id="hasCustomClient"
             label="Use my OAuth App"
             variant="simple/small"
@@ -189,7 +189,7 @@ export function UpdateOAuthForm({
               )}
               {authMethod.scopes.map((s) => {
                 return (
-                  <Checkbox
+                  <CheckboxWithLabel
                     key={s.name}
                     id={s.name}
                     value={s.name}

apps/webapp/app/components/navigation/SideMenu.tsx

@@ -3,6 +3,7 @@ import {
   ArrowRightIcon,
   ArrowRightOnRectangleIcon,
   BeakerIcon,
+  BellAlertIcon,
   ChartBarIcon,
   ClockIcon,
   CursorArrowRaysIcon,
@@ -45,6 +46,7 @@ import {
   v3ApiKeysPath,
   v3DeploymentsPath,
   v3EnvironmentVariablesPath,
+  v3ProjectAlertsPath,
   v3ProjectPath,
   v3ProjectSettingsPath,
   v3RunsPath,
@@ -231,31 +233,22 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
                   </div>
                   <hr className="border-charcoal-800" />
                   <div>
-                    <StepNumber stepNumber="1" title="Create a new Slack channel" />
+                    <StepNumber stepNumber="1" title="Email us" />
                     <StepContentContainer>
                       <Paragraph>
-                        In your Slack app, create a new channel from the main menu by going to File{" "}
-                        <ArrowRightIcon className="inline h-4 w-4 text-text-dimmed" /> New Channel
-                      </Paragraph>
-                    </StepContentContainer>
-                    <StepNumber stepNumber="2" title="Setup your channel" />
-                    <StepContentContainer>
-                      <Paragraph>
-                        Name your channel, set its visibility and click 'Create'.
-                      </Paragraph>
-                    </StepContentContainer>
-                    <StepNumber stepNumber="3" title="Invite Trigger.dev" />
-                    <StepContentContainer>
-                      <Paragraph>
-                        Invite this email address to your channel:{" "}
+                        Send us an email to this address from your Trigger.dev account email
+                        address:
                         <ClipboardField
                           variant="primary/medium"
-                          value="james@trigger.dev"
+                          value="priority-support@trigger.dev"
                           className="my-2"
                         />
                       </Paragraph>
+                    </StepContentContainer>
+                    <StepNumber stepNumber="2" title="Look out for an invite from Slack" />
+                    <StepContentContainer>
                       <Paragraph>
-                        As soon as we can, we'll accept your invitation and say hello!
+                        As soon as we can, we'll setup a Slack Connect channel and say hello!
                       </Paragraph>
                     </StepContentContainer>
                   </div>
@@ -549,6 +542,8 @@ function V3ProjectSideMenu({
   project: SideMenuProject;
   organization: MatchedOrganization;
 }) {
+  const { alertsEnabled } = useFeatures();
+
   return (
     <>
       <SideMenuHeader title={"Project (v3)"} />
@@ -601,6 +596,15 @@ function V3ProjectSideMenu({
         to={v3DeploymentsPath(organization, project)}
         data-action="deployments"
       />
+      {alertsEnabled && (
+        <SideMenuItem
+          name="Alerts"
+          icon={BellAlertIcon}
+          iconColor="text-red-500"
+          to={v3ProjectAlertsPath(organization, project)}
+          data-action="alerts"
+        />
+      )}
       <SideMenuItem
         name="Project settings"
         icon="settings"

apps/webapp/app/components/primitives/Buttons.tsx

@@ -48,11 +48,11 @@ const theme = {
       "border-black/40 text-charcoal-900 group-hover:border-black/60 group-hover:text-charcoal-900",
   },
   secondary: {
-    textColor: "text-primary group-hover:text-apple-200 transition group-disabled:text-primary",
+    textColor: "text-secondary group-hover:text-secondary transition group-disabled:text-secondary",
     button:
-      "bg-transparent border border-primary group-hover:border-apple-200 group-hover:bg-apple-950 group-disabled:opacity-30 group-disabled:border-primary group-disabled:bg-transparent group-disabled:pointer-events-none",
+      "bg-transparent border border-secondary group-hover:border-secondary group-hover:bg-secondary/10 group-disabled:opacity-30 group-disabled:border-secondary group-disabled:bg-transparent group-disabled:pointer-events-none",
     shortcut:
-      "border-primary/30 text-apple-200 group-hover:text-text-bright/80 group-hover:border-dimmed/60",
+      "border-secondary/30 text-secondary group-hover:text-text-bright/80 group-hover:border-dimmed/60",
   },
   tertiary: {
     textColor: "text-text-bright transition group-disabled:text-text-dimmed/80",

apps/webapp/app/components/primitives/Checkbox.tsx

@@ -1,5 +1,5 @@
 import * as React from "react";
-import { useEffect, useState } from "react";
+import { forwardRef, useEffect, useState } from "react";
 import { cn } from "~/utils/cn";
 import { Badge } from "./Badge";
 import { Paragraph } from "./Paragraph";
@@ -53,18 +53,18 @@ export type CheckboxProps = Omit<
   React.InputHTMLAttributes<HTMLInputElement>,
   "checked" | "onChange"
 > & {
-  id: string;
+  id?: string;
   name?: string;
   value?: string;
   variant?: keyof typeof variants;
-  label?: React.ReactNode;
+  label: React.ReactNode;
   description?: string;
   badges?: string[];
   className?: string;
   onChange?: (isChecked: boolean) => void;
 };
 
-export const Checkbox = React.forwardRef<HTMLInputElement, CheckboxProps>(
+export const CheckboxWithLabel = React.forwardRef<HTMLInputElement, CheckboxProps>(
   (
     {
       id,
@@ -172,3 +172,21 @@ export const Checkbox = React.forwardRef<HTMLInputElement, CheckboxProps>(
     );
   }
 );
+
+type SimpleCheckboxProps = Omit<React.ComponentProps<"input">, "type">;
+
+export const Checkbox = forwardRef<HTMLInputElement, SimpleCheckboxProps>(
+  ({ className, ...props }: SimpleCheckboxProps, ref) => {
+    return (
+      <input
+        type="checkbox"
+        className={cn(
+          props.readOnly || props.disabled ? "cursor-default" : "cursor-pointer",
+          "read-only:border-charcoal-650 disabled:border-charcoal-650 rounded-sm border border-charcoal-600 bg-transparent transition checked:!bg-indigo-500 read-only:!bg-charcoal-700 group-hover:bg-charcoal-900 group-hover:checked:bg-indigo-500 group-focus:ring-1 focus:ring-indigo-500 focus:ring-offset-0 focus:ring-offset-transparent focus-visible:outline-none  focus-visible:ring-indigo-500 disabled:!bg-charcoal-700"
+        )}
+        {...props}
+        ref={ref}
+      />
+    );
+  }
+);

apps/webapp/app/components/primitives/DetailCell.tsx

@@ -49,12 +49,7 @@ export function DetailCell({
   const variation = variations[variant];
 
   return (
-    <div
-      className={cn(
-        "group flex h-11 w-full items-center gap-3 rounded-md p-1 pr-3 transition hover:bg-charcoal-900",
-        className
-      )}
-    >
+    <div className={cn("group flex h-11 w-full items-center gap-3 rounded-md p-1 pr-3", className)}>
       <IconInBox
         icon={leadingIcon}
         className={cn("flex-none transition group-hover:border-charcoal-750", leadingIconClassName)}
@@ -62,20 +57,14 @@ export function DetailCell({
       <div className="flex flex-1 flex-col">
         <Paragraph
           variant={variation.label.variant}
-          className={cn(
-            "flex-1 text-left transition group-hover:text-text-bright",
-            variation.label.className
-          )}
+          className={cn("flex-1 text-left", variation.label.className)}
         >
           {label}
         </Paragraph>
         {description && (
           <Paragraph
             variant={variation.description.variant}
-            className={cn(
-              "flex-1 text-left text-text-dimmed transition group-hover:text-text-bright",
-              variation.description.className
-            )}
+            className={cn("flex-1 text-left text-text-dimmed", variation.description.className)}
           >
             {description}
           </Paragraph>

apps/webapp/app/components/primitives/Dialog.tsx

@@ -43,7 +43,7 @@ const DialogContent = React.forwardRef<
     <DialogPrimitive.Content
       ref={ref}
       className={cn(
-        "fixed z-50 grid w-full gap-4 rounded-b-lg border bg-background-dimmed px-4 pb-4 pt-3.5 shadow-lg animate-in data-[state=open]:fade-in-90 data-[state=open]:slide-in-from-bottom-10 sm:max-w-lg sm:rounded-lg sm:zoom-in-90 data-[state=open]:sm:slide-in-from-bottom-0",
+        "fixed z-50 grid w-full gap-4 rounded-b-lg border bg-background-dimmed px-4 pb-4 pt-2.5 shadow-lg animate-in data-[state=open]:fade-in-90 data-[state=open]:slide-in-from-bottom-10 sm:max-w-lg sm:rounded-lg sm:zoom-in-90 data-[state=open]:sm:slide-in-from-bottom-0",
         className
       )}
       {...props}
@@ -74,7 +74,7 @@ DialogContent.displayName = DialogPrimitive.Content.displayName;
 
 const DialogHeader = ({ className, ...props }: React.HTMLAttributes<HTMLDivElement>) => (
   <div
-    className={cn("flex flex-col text-left font-medium text-text-dimmed", className)}
+    className={cn("flex flex-col text-left font-medium text-text-bright", className)}
     {...props}
   />
 );

apps/webapp/app/components/primitives/SegmentedControl.tsx

@@ -2,6 +2,19 @@ import { RadioGroup } from "@headlessui/react";
 import { motion } from "framer-motion";
 import { cn } from "~/utils/cn";
 
+const variants = {
+  primary: {
+    base: "bg-charcoal-700",
+    active: "text-text-bright hover:bg-charcoal-750/50",
+  },
+  secondary: {
+    base: "bg-charcoal-700/50",
+    active: "text-text-bright bg-charcoal-700 rounded-[2px] border border-charcoal-600/50",
+  },
+};
+
+type Variants = keyof typeof variants;
+
 type Options = {
   label: string;
   value: string;
@@ -12,6 +25,7 @@ type SegmentedControlProps = {
   value?: string;
   defaultValue?: string;
   options: Options[];
+  variant?: Variants;
   fullWidth?: boolean;
   onChange?: (value: string) => void;
 };
@@ -21,11 +35,18 @@ export default function SegmentedControl({
   value,
   defaultValue,
   options,
+  variant = "secondary",
   fullWidth,
   onChange,
 }: SegmentedControlProps) {
   return (
-    <div className={cn("flex h-10 rounded bg-charcoal-700", fullWidth ? "w-full" : "w-fit")}>
+    <div
+      className={cn(
+        "flex h-10 rounded text-text-bright",
+        variants[variant].base,
+        fullWidth ? "w-full" : "w-fit"
+      )}
+    >
       <RadioGroup
         value={value}
         defaultValue={defaultValue ?? options[0].value}
@@ -46,11 +67,11 @@ export default function SegmentedControl({
                 cn(
                   "relative flex h-full grow cursor-pointer text-center font-normal focus:outline-none",
                   active
-                    ? "ring-offset-2 focus-visible:ring focus-visible:ring-primary focus-visible:ring-opacity-60"
+                    ? "ring-offset-2 focus-visible:ring focus-visible:ring-secondary focus-visible:ring-opacity-60"
                     : "",
                   checked
-                    ? "text-text-bright"
-                    : "rounded-[2px] text-text-dimmed transition hover:bg-charcoal-750/50 hover:text-text-bright"
+                    ? variants[variant].active
+                    : "text-text-dimmed transition hover:text-text-bright"
                 )
               }
             >
@@ -60,12 +81,12 @@ export default function SegmentedControl({
                     <div className="z-10 flex h-full w-full items-center justify-center text-sm">
                       <RadioGroup.Label as="p">{option.label}</RadioGroup.Label>
                     </div>
-                    {checked && (
+                    {checked && variant === "primary" && (
                       <motion.div
                         layoutId={`segmented-control-${name}`}
                         transition={{ duration: 0.4, type: "spring" }}
                         className="absolute inset-0 rounded-[2px] shadow-md outline outline-3 outline-primary"
-                      ></motion.div>
+                      />
                     )}
                   </div>
                 </>

apps/webapp/app/components/primitives/SelectedItemsProvider.tsx

@@ -0,0 +1,146 @@
+"use client";
+
+import { createContext, useCallback, useContext, useReducer } from "react";
+
+type SelectedItemsContext = {
+  selectedItems: Set<string>;
+  select: (items: string | string[]) => void;
+  deselect: (items: string | string[]) => void;
+  toggle: (items: string | string[]) => void;
+  deselectAll: () => void;
+  has: (item: string) => boolean;
+  hasAll: (items: string[]) => boolean;
+};
+
+const SelectedItemsContext = createContext<SelectedItemsContext>({} as SelectedItemsContext);
+
+export function useSelectedItems(enabled = true) {
+  const context = useContext(SelectedItemsContext);
+  if (!context && enabled) {
+    throw new Error("useSelectedItems must be used within a SelectedItemsProvider");
+  }
+
+  return context;
+}
+
+export function SelectedItemsProvider({
+  initialSelectedItems,
+  maxSelectedItemCount,
+  children,
+}: {
+  initialSelectedItems: string[];
+  maxSelectedItemCount?: number;
+  children: React.ReactNode | ((context: SelectedItemsContext) => React.ReactNode);
+}) {
+  const [state, dispatch] = useReducer(selectedItemsReducer, {
+    items: new Set<string>(initialSelectedItems),
+    maxSelectedItemCount,
+  });
+
+  const select = useCallback((items: string | string[]) => {
+    dispatch({ type: "select", items: Array.isArray(items) ? items : [items] });
+  }, []);
+
+  const deselect = useCallback((items: string | string[]) => {
+    dispatch({ type: "deselect", items: Array.isArray(items) ? items : [items] });
+  }, []);
+
+  const toggle = useCallback((items: string | string[]) => {
+    dispatch({ type: "toggle", items: Array.isArray(items) ? items : [items] });
+  }, []);
+
+  const deselectAll = useCallback(() => {
+    dispatch({ type: "deselectAll" });
+  }, []);
+
+  const has = useCallback((item: string) => state.items.has(item), [state]);
+
+  const hasAll = useCallback(
+    (items: string[]) => items.every((item) => state.items.has(item)),
+    [state]
+  );
+
+  return (
+    <SelectedItemsContext.Provider
+      value={{ selectedItems: state.items, select, deselect, toggle, deselectAll, has, hasAll }}
+    >
+      {typeof children === "function"
+        ? children({
+            selectedItems: state.items,
+            select,
+            deselect,
+            toggle,
+            deselectAll,
+            has,
+            hasAll,
+          })
+        : children}
+    </SelectedItemsContext.Provider>
+  );
+}
+
+type SelectItemsAction = {
+  type: "select";
+  items: string[];
+};
+
+type DeSelectItemsAction = {
+  type: "deselect";
+  items: string[];
+};
+
+type DeselectAllItemsAction = {
+  type: "deselectAll";
+};
+
+type ToggleItemsAction = {
+  type: "toggle";
+  items: string[];
+};
+
+type Action = SelectItemsAction | DeSelectItemsAction | ToggleItemsAction | DeselectAllItemsAction;
+
+function selectedItemsReducer(
+  state: { items: Set<string>; maxSelectedItemCount?: number },
+  action: Action
+) {
+  switch (action.type) {
+    case "select":
+      const items = new Set([...state.items, ...action.items]);
+      return { ...state, items: cappedSet(items, state.maxSelectedItemCount) };
+    case "deselect":
+      const newItems = new Set(state.items);
+      action.items.forEach((item) => {
+        newItems.delete(item);
+      });
+      return { ...state, items: cappedSet(newItems, state.maxSelectedItemCount) };
+    case "toggle":
+      let newSet = new Set(state.items);
+      action.items.forEach((item) => {
+        if (newSet.has(item)) {
+          newSet.delete(item);
+        } else {
+          newSet.add(item);
+        }
+      });
+      return { ...state, items: cappedSet(newSet, state.maxSelectedItemCount) };
+    case "deselectAll":
+      return { ...state, items: new Set<string>() };
+    default:
+      return state;
+  }
+}
+
+function cappedSet(set: Set<string>, max?: number) {
+  if (!max) {
+    return set;
+  }
+
+  if (set.size <= max) {
+    return set;
+  }
+
+  console.warn(`Selected items exceeded the maximum count of ${max}.`);
+
+  return new Set([...set].slice(0, max));
+}

apps/webapp/app/components/primitives/Table.tsx

@@ -106,7 +106,7 @@ export const TableHeaderCell = forwardRef<HTMLTableCellElement, TableHeaderCellP
         ref={ref}
         scope="col"
         className={cn(
-          "px-4 py-2 align-middle text-xxs font-normal uppercase tracking-wider text-text-dimmed",
+          "px-3 py-2 align-middle text-xxs font-normal uppercase tracking-wider text-text-dimmed",
           alignmentClassName,
           className
         )}
@@ -155,7 +155,7 @@ export const TableCell = forwardRef<HTMLTableCellElement, TableCellProps>(
     }
 
     const flexClasses = cn(
-      "flex w-full whitespace-nowrap px-4 py-3 text-xs text-text-dimmed",
+      "flex w-full whitespace-nowrap px-3 py-3 text-xs text-text-dimmed",
       alignment === "left"
         ? "justify-start text-left"
         : alignment === "center"
@@ -170,7 +170,7 @@ export const TableCell = forwardRef<HTMLTableCellElement, TableCellProps>(
           "text-xs text-charcoal-400",
           to || onClick || hasAction
             ? "cursor-pointer group-hover/table-row:bg-charcoal-900"
-            : "px-4 py-3 align-middle",
+            : "px-3 py-3 align-middle",
           !to && !onClick && alignmentClassName,
           isSticky && stickyStyles,
           className

apps/webapp/app/components/runs/v3/BulkAction.tsx

@@ -0,0 +1,73 @@
+import { ArrowPathIcon, NoSymbolIcon } from "@heroicons/react/20/solid";
+import { BulkActionType } from "@trigger.dev/database";
+import assertNever from "assert-never";
+import { cn } from "~/utils/cn";
+
+export function BulkActionStatusCombo({
+  type,
+  className,
+  iconClassName,
+}: {
+  type: BulkActionType;
+  className?: string;
+  iconClassName?: string;
+}) {
+  return (
+    <span className={cn("flex items-center gap-1", className)}>
+      <BulkActionIcon type={type} className={cn("h-4 w-4", iconClassName)} />
+      <BulkActionLabel type={type} />
+    </span>
+  );
+}
+
+export function BulkActionLabel({ type }: { type: BulkActionType }) {
+  return <span className={bulkActionClassName(type)}>{bulkActionTitle(type)}</span>;
+}
+
+export function BulkActionIcon({ type, className }: { type: BulkActionType; className: string }) {
+  switch (type) {
+    case "REPLAY":
+      return <ArrowPathIcon className={cn(bulkActionClassName(type), className)} />;
+    case "CANCEL":
+      return <NoSymbolIcon className={cn(bulkActionClassName(type), className)} />;
+    default: {
+      assertNever(type);
+    }
+  }
+}
+
+export function bulkActionClassName(type: BulkActionType): string {
+  switch (type) {
+    case "REPLAY":
+      return "text-indigo-500";
+    case "CANCEL":
+      return "text-rose-500";
+    default: {
+      assertNever(type);
+    }
+  }
+}
+
+export function bulkActionTitle(type: BulkActionType): string {
+  switch (type) {
+    case "REPLAY":
+      return "Replay";
+    case "CANCEL":
+      return "Cancel";
+    default: {
+      assertNever(type);
+    }
+  }
+}
+
+export function bulkActionVerb(type: BulkActionType): string {
+  switch (type) {
+    case "REPLAY":
+      return "Replaying";
+    case "CANCEL":
+      return "Canceling";
+    default: {
+      assertNever(type);
+    }
+  }
+}

apps/webapp/app/components/runs/v3/EnabledStatus.tsx

@@ -1,18 +1,31 @@
 import { BoltSlashIcon, CheckCircleIcon } from "@heroicons/react/20/solid";
 
-export function EnabledStatus({ enabled }: { enabled: boolean }) {
+type EnabledStatusProps = {
+  enabled: boolean;
+  enabledIcon?: React.ComponentType<any>;
+  disabledIcon?: React.ComponentType<any>;
+};
+
+export function EnabledStatus({
+  enabled,
+  enabledIcon = CheckCircleIcon,
+  disabledIcon = BoltSlashIcon,
+}: EnabledStatusProps) {
+  const EnabledIcon = enabledIcon;
+  const DisabledIcon = disabledIcon;
+
   switch (enabled) {
     case true:
       return (
         <div className="flex items-center gap-1 text-xs text-success">
-          <CheckCircleIcon className="h-4 w-4" />
+          <EnabledIcon className="size-4" />
           Enabled
         </div>
       );
     case false:
       return (
         <div className="text-dimmed flex items-center gap-1 text-xs">
-          <BoltSlashIcon className="h-4 w-4" />
+          <DisabledIcon className="size-4" />
           Disabled
         </div>
       );

apps/webapp/app/components/runs/v3/RunFilters.tsx

@@ -1,7 +1,18 @@
 import * as Ariakit from "@ariakit/react";
-import { CalendarIcon, CpuChipIcon, XMarkIcon } from "@heroicons/react/20/solid";
+import {
+  ArrowPathIcon,
+  CalendarIcon,
+  CpuChipIcon,
+  InboxStackIcon,
+  XMarkIcon,
+} from "@heroicons/react/20/solid";
 import { Form } from "@remix-run/react";
-import { RuntimeEnvironment, TaskRunStatus, TaskTriggerSource } from "@trigger.dev/database";
+import {
+  BulkActionType,
+  RuntimeEnvironment,
+  TaskRunStatus,
+  TaskTriggerSource,
+} from "@trigger.dev/database";
 import { ListFilterIcon } from "lucide-react";
 import { ReactNode, startTransition, useCallback, useMemo, useState } from "react";
 import { z } from "zod";
@@ -36,6 +47,8 @@ import {
   runStatusTitle,
 } from "./TaskRunStatus";
 import { TaskTriggerSourceIcon } from "./TaskTriggerSource";
+import { DateTime } from "~/components/primitives/DateTime";
+import { BulkActionStatusCombo } from "./BulkAction";
 
 export const TaskAttemptStatus = z.nativeEnum(TaskRunStatus);
 
@@ -59,6 +72,7 @@ export const TaskRunListSearchFilters = z.object({
     TaskAttemptStatus.array().optional()
   ),
   period: z.preprocess((value) => (value === "all" ? undefined : value), z.string().optional()),
+  bulkId: z.string().optional(),
   from: z.coerce.number().optional(),
   to: z.coerce.number().optional(),
 });
@@ -72,6 +86,11 @@ type DisplayableEnvironment = Pick<RuntimeEnvironment, "type" | "id"> & {
 type RunFiltersProps = {
   possibleEnvironments: DisplayableEnvironment[];
   possibleTasks: { slug: string; triggerSource: TaskTriggerSource }[];
+  bulkActions: {
+    id: string;
+    type: BulkActionType;
+    createdAt: Date;
+  }[];
   hasFilters: boolean;
 };
 
@@ -82,7 +101,8 @@ export function RunsFilters(props: RunFiltersProps) {
     searchParams.has("statuses") ||
     searchParams.has("environments") ||
     searchParams.has("tasks") ||
-    searchParams.has("period");
+    searchParams.has("period") ||
+    searchParams.has("bulkId");
 
   return (
     <div className="flex flex-row flex-wrap items-center gap-1">
@@ -112,7 +132,8 @@ const filterTypes = [
   { name: "environments", title: "Environment", icon: <CpuChipIcon className="size-4" /> },
   { name: "tasks", title: "Tasks", icon: <TaskIcon className="size-4" /> },
   { name: "created", title: "Created", icon: <CalendarIcon className="size-4" /> },
-];
+  { name: "bulk", title: "Bulk action", icon: <InboxStackIcon className="size-4" /> },
+] as const;
 
 type FilterType = (typeof filterTypes)[number]["name"];
 
@@ -123,6 +144,7 @@ function FilterMenu(props: RunFiltersProps) {
 
   const filterTrigger = (
     <SelectTrigger
+      autoFocus
       icon={
         <div className="flex size-4 items-center justify-center">
           <ListFilterIcon className="size-3.5" />
@@ -180,13 +202,14 @@ function FilterMenuProvider({
   );
 }
 
-function AppliedFilters({ possibleEnvironments, possibleTasks }: RunFiltersProps) {
+function AppliedFilters({ possibleEnvironments, possibleTasks, bulkActions }: RunFiltersProps) {
   return (
     <>
       <AppliedStatusFilter />
       <AppliedEnvironmentFilter possibleEnvironments={possibleEnvironments} />
       <AppliedTaskFilter possibleTasks={possibleTasks} />
       <AppliedPeriodFilter />
+      <AppliedBulkActionsFilter bulkActions={bulkActions} />
     </>
   );
 }
@@ -211,6 +234,8 @@ function Menu(props: MenuProps) {
       return <TasksDropdown onClose={() => props.setFilterType(undefined)} {...props} />;
     case "created":
       return <CreatedDropdown onClose={() => props.setFilterType(undefined)} {...props} />;
+    case "bulk":
+      return <BulkActionsDropdown onClose={() => props.setFilterType(undefined)} {...props} />;
   }
 }
 
@@ -529,6 +554,99 @@ function AppliedTaskFilter({ possibleTasks }: Pick<RunFiltersProps, "possibleTas
   );
 }
 
+function BulkActionsDropdown({
+  trigger,
+  clearSearchValue,
+  searchValue,
+  onClose,
+  bulkActions,
+}: {
+  trigger: ReactNode;
+  clearSearchValue: () => void;
+  searchValue: string;
+  onClose?: () => void;
+  bulkActions: RunFiltersProps["bulkActions"];
+}) {
+  const { value, replace } = useSearchParams();
+
+  const handleChange = (value: string) => {
+    clearSearchValue();
+    replace({ bulkId: value, cursor: undefined, direction: undefined });
+  };
+
+  const filtered = useMemo(() => {
+    return bulkActions.filter((item) => {
+      return (
+        item.type.toLowerCase().includes(searchValue.toLowerCase()) ||
+        item.createdAt.toISOString().includes(searchValue)
+      );
+    });
+  }, [searchValue, bulkActions]);
+
+  return (
+    <SelectProvider value={value("bulkId")} setValue={handleChange} virtualFocus={true}>
+      {trigger}
+      <SelectPopover
+        className="min-w-0 max-w-[min(240px,var(--popover-available-width))]"
+        hideOnEscape={() => {
+          if (onClose) {
+            onClose();
+            return false;
+          }
+
+          return true;
+        }}
+      >
+        <ComboBox placeholder={"Filter by bulk action..."} value={searchValue} />
+        <SelectList>
+          <SelectItem value={""}>None</SelectItem>
+          {filtered.map((item, index) => (
+            <SelectItem key={item.id} value={item.id}>
+              <div className="flex gap-3">
+                <BulkActionStatusCombo type={item.type} iconClassName="size-4" />
+                <DateTime date={item.createdAt} />
+              </div>
+            </SelectItem>
+          ))}
+        </SelectList>
+      </SelectPopover>
+    </SelectProvider>
+  );
+}
+
+function AppliedBulkActionsFilter({ bulkActions }: Pick<RunFiltersProps, "bulkActions">) {
+  const { value, del } = useSearchParams();
+
+  const bulkId = value("bulkId");
+
+  if (!bulkId) {
+    return null;
+  }
+
+  const action = bulkActions.find((action) => action.id === bulkId);
+
+  return (
+    <FilterMenuProvider>
+      {(search, setSearch) => (
+        <BulkActionsDropdown
+          trigger={
+            <Ariakit.Select render={<div className="group cursor-pointer" />}>
+              <AppliedFilter
+                label="Bulk action"
+                value={bulkId}
+                onRemove={() => del(["bulkId", "cursor", "direction"])}
+              />
+            </Ariakit.Select>
+          }
+          searchValue={search}
+          clearSearchValue={() => setSearch("")}
+          bulkActions={bulkActions}
+        />
+      )}
+    </FilterMenuProvider>
+  );
+}
+
 const timePeriods = [
   {
     label: "All periods",

apps/webapp/app/components/runs/v3/TaskRunsTable.tsx

@@ -28,6 +28,11 @@ import { CancelRunDialog } from "./CancelRunDialog";
 import { ReplayRunDialog } from "./ReplayRunDialog";
 import { TaskRunStatusCombo } from "./TaskRunStatus";
 import { LiveTimer } from "./LiveTimer";
+import { useSelectedItems } from "~/components/primitives/SelectedItemsProvider";
+import { Checkbox } from "~/components/primitives/Checkbox";
+import { useCallback, useRef } from "react";
+import { run } from "@remix-run/dev/dist/cli/run";
+import { formatNumber } from "~/utils/numberFormatter";
 
 type RunsTableProps = {
   total: number;
@@ -36,6 +41,7 @@ type RunsTableProps = {
   showJob?: boolean;
   runs: RunListItem[];
   isLoading?: boolean;
+  allowSelection?: boolean;
 };
 
 export function TaskRunsTable({
@@ -44,15 +50,66 @@ export function TaskRunsTable({
   filters,
   runs,
   isLoading = false,
+  allowSelection = false,
 }: RunsTableProps) {
   const organization = useOrganization();
   const project = useProject();
+  const checkboxes = useRef<(HTMLInputElement | null)[]>([]);
+  const { selectedItems, has, hasAll, select, deselect, toggle } = useSelectedItems(allowSelection);
+
+  const navigateCheckboxes = useCallback(
+    (event: React.KeyboardEvent<HTMLInputElement>, index: number) => {
+      //indexes are out by one because of the header row
+      if (event.key === "ArrowUp" && index > 0) {
+        checkboxes.current[index - 1]?.focus();
+
+        if (event.shiftKey) {
+          const oldItem = runs.at(index - 1);
+          const newItem = runs.at(index - 2);
+          const itemsIds = [oldItem?.id, newItem?.id].filter(Boolean);
+          select(itemsIds);
+        }
+      } else if (event.key === "ArrowDown" && index < checkboxes.current.length - 1) {
+        checkboxes.current[index + 1]?.focus();
+
+        if (event.shiftKey) {
+          const oldItem = runs.at(index - 1);
+          const newItem = runs.at(index);
+          const itemsIds = [oldItem?.id, newItem?.id].filter(Boolean);
+          select(itemsIds);
+        }
+      }
+    },
+    [checkboxes, runs]
+  );
 
   return (
     <Table>
       <TableHeader>
         <TableRow>
-          <TableHeaderCell>Run</TableHeaderCell>
+          {allowSelection && (
+            <TableHeaderCell className="pl-2 pr-0">
+              {runs.length > 0 && (
+                <Checkbox
+                  checked={hasAll(runs.map((r) => r.id))}
+                  onChange={(element) => {
+                    const ids = runs.map((r) => r.id);
+                    const checked = element.currentTarget.checked;
+                    if (checked) {
+                      select(ids);
+                    } else {
+                      deselect(ids);
+                    }
+                  }}
+                  ref={(r) => {
+                    checkboxes.current[0] = r;
+                  }}
+                  onKeyDown={(event) => navigateCheckboxes(event, 0)}
+                />
+              )}
+            </TableHeaderCell>
+          )}
+          <TableHeaderCell alignment="right">Run #</TableHeaderCell>
           <TableHeaderCell>Task ID</TableHeaderCell>
           <TableHeaderCell>Version</TableHeaderCell>
           <TableHeaderCell>Env</TableHeaderCell>
@@ -74,11 +131,27 @@ export function TaskRunsTable({
         ) : runs.length === 0 ? (
           <BlankState isLoading={isLoading} filters={filters} />
         ) : (
-          runs.map((run) => {
+          runs.map((run, index) => {
             const path = v3RunSpanPath(organization, project, run, { spanId: run.spanId });
             return (
               <TableRow key={run.id}>
-                <TableCell to={path}>#{run.number}</TableCell>
+                {allowSelection && (
+                  <TableCell className="pl-2 pr-0">
+                    <Checkbox
+                      checked={has(run.id)}
+                      onChange={(element) => {
+                        toggle(run.id);
+                      }}
+                      ref={(r) => {
+                        checkboxes.current[index + 1] = r;
+                      }}
+                      onKeyDown={(event) => navigateCheckboxes(event, index + 1)}
+                    />
+                  </TableCell>
+                )}
+                <TableCell to={path} alignment="right">
+                  {formatNumber(run.number)}
+                </TableCell>
                 <TableCell to={path}>{run.taskIdentifier}</TableCell>
                 <TableCell to={path}>{run.version ?? "–"}</TableCell>
                 <TableCell to={path}>
@@ -194,7 +267,7 @@ function BlankState({ isLoading, filters }: Pick<RunsTableProps, "isLoading" | "
   ) {
     const environment = envs?.find((env) => env.id === filters.environments[0]);
     return (
-      <TableBlankRow colSpan={9}>
+      <TableBlankRow colSpan={10}>
         <div className="py-14">
           <Paragraph className="w-auto" variant="base/bright" spacing>
             There are no runs for {filters.tasks[0]}
@@ -235,8 +308,21 @@ function BlankState({ isLoading, filters }: Pick<RunsTableProps, "isLoading" | "
   }
 
   return (
-    <TableBlankRow colSpan={9}>
-      <NoRuns title="No runs match your filters" />
+    <TableBlankRow colSpan={10}>
+      <div className="flex flex-col items-center justify-center gap-2">
+        <Paragraph className="w-auto" variant="small">
+          No runs currently match your filters. Try refreshing or modifying your filters.
+        </Paragraph>
+        <Button
+          LeadingIcon={ArrowPathIcon}
+          variant="tertiary/small"
+          onClick={() => {
+            window.location.reload();
+          }}
+        >
+          Refresh
+        </Button>
+      </div>
     </TableBlankRow>
   );
 }

apps/webapp/app/consts.ts

@@ -12,3 +12,4 @@ export const MAX_RUN_CHUNK_EXECUTION_LIMIT = 120000; // 2 minutes
 export const VERCEL_RESPONSE_TIMEOUT_STATUS_CODES = [408, 504];
 export const MAX_BATCH_TRIGGER_ITEMS = 100;
 export const MAX_TASK_RUN_ATTEMPTS = 250;
+export const BULK_ACTION_RUN_LIMIT = 250;

apps/webapp/app/entry.server.tsx

@@ -174,6 +174,10 @@ Worker.init().catch((error) => {
 
 function logError(error: unknown, request?: Request) {
   console.error(error);
+
+  if (error instanceof Error && error.message.startsWith("There are locked jobs present")) {
+    console.log("⚠️  graphile-worker migration issue detected!");
+  }
 }
 
 const sqsEventConsumer = singleton("sqsEventConsumer", getSharedSqsEventConsumer);

apps/webapp/app/env.server.ts

@@ -47,6 +47,8 @@ const EnvironmentSchema = z.object({
   WORKER_SCHEMA: z.string().default("graphile_worker"),
   WORKER_CONCURRENCY: z.coerce.number().int().default(10),
   WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000),
+  /** The number of days a failed Graphile task should stay before getting cleaned up */
+  WORKER_CLEANUP_TTL_DAYS: z.coerce.number().int().default(3),
   EXECUTION_WORKER_CONCURRENCY: z.coerce.number().int().default(10),
   EXECUTION_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000),
   WORKER_ENABLED: z.string().default("true"),
@@ -153,6 +155,15 @@ const EnvironmentSchema = z.object({
   INTERNAL_OTEL_TRACE_SAMPLING_RATE: z.string().default("20"),
   INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED: z.string().default("0"),
   INTERNAL_OTEL_TRACE_DISABLED: z.string().default("0"),
+
+  ORG_SLACK_INTEGRATION_CLIENT_ID: z.string().optional(),
+  ORG_SLACK_INTEGRATION_CLIENT_SECRET: z.string().optional(),
+
+  /** These enable the alerts feature in v3 */
+  ALERT_FROM_EMAIL: z.string().optional(),
+  ALERT_RESEND_API_KEY: z.string().optional(),
+
+  MAX_SEQUENTIAL_INDEX_FAILURE_COUNT: z.coerce.number().default(96),
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/features.server.ts

@@ -4,6 +4,7 @@ import { requestUrl } from "./utils/requestUrl.server";
 export type TriggerFeatures = {
   isManagedCloud: boolean;
   v3Enabled: boolean;
+  alertsEnabled: boolean;
 };
 
 // If the request host is cloud.trigger.dev then we are on the managed cloud
@@ -20,5 +21,6 @@ export function featuresForRequest(request: Request): TriggerFeatures {
   return {
     isManagedCloud,
     v3Enabled: env.V3_ENABLED === "true",
+    alertsEnabled: env.ALERT_FROM_EMAIL !== undefined && env.ALERT_RESEND_API_KEY !== undefined,
   };
 }

apps/webapp/app/hooks/useFeatures.ts

@@ -5,5 +5,5 @@ import type { TriggerFeatures } from "~/features.server";
 export function useFeatures(): TriggerFeatures {
   const routeMatch = useTypedRouteLoaderData<typeof loader>("root");
 
-  return routeMatch?.features ?? { isManagedCloud: false, v3Enabled: false };
+  return routeMatch?.features ?? { isManagedCloud: false, v3Enabled: false, alertsEnabled: false };
 }

apps/webapp/app/models/orgIntegration.server.ts

@@ -0,0 +1,236 @@
+import { WebClient } from "@slack/web-api";
+import {
+  IntegrationService,
+  Organization,
+  OrganizationIntegration,
+  SecretReference,
+} from "@trigger.dev/database";
+import { z } from "zod";
+import { $transaction, prisma } from "~/db.server";
+import { env } from "~/env.server";
+import { logger } from "~/services/logger.server";
+import { getSecretStore } from "~/services/secrets/secretStore.server";
+import { commitSession, getUserSession } from "~/services/sessionStorage.server";
+import { generateFriendlyId } from "~/v3/friendlyIdentifiers";
+
+const SlackSecretSchema = z.object({
+  botAccessToken: z.string(),
+  userAccessToken: z.string().optional(),
+  expiresIn: z.number().optional(),
+  refreshToken: z.string().optional(),
+  botScopes: z.array(z.string()).optional(),
+  userScopes: z.array(z.string()).optional(),
+  raw: z.record(z.any()).optional(),
+});
+
+type SlackSecret = z.infer<typeof SlackSecretSchema>;
+
+const REDIRECT_AFTER_AUTH_KEY = "redirect-back-after-auth";
+
+export type OrganizationIntegrationForService<TService extends IntegrationService> = Omit<
+  AuthenticatableIntegration,
+  "service"
+> & {
+  service: TService;
+};
+
+type AuthenticatedClientOptions<TService extends IntegrationService> = TService extends "SLACK"
+  ? {
+      forceBotToken?: boolean;
+    }
+  : undefined;
+
+type AuthenticatedClientForIntegration<TService extends IntegrationService> =
+  TService extends "SLACK" ? InstanceType<typeof WebClient> : never;
+
+export type AuthenticatableIntegration = OrganizationIntegration & {
+  tokenReference: SecretReference;
+};
+
+export class OrgIntegrationRepository {
+  static async getAuthenticatedClientForIntegration<TService extends IntegrationService>(
+    integration: OrganizationIntegrationForService<TService>,
+    options?: AuthenticatedClientOptions<TService>
+  ): Promise<AuthenticatedClientForIntegration<TService>> {
+    const secretStore = getSecretStore(integration.tokenReference.provider);
+
+    switch (integration.service) {
+      case "SLACK": {
+        const secret = await secretStore.getSecret(
+          SlackSecretSchema,
+          integration.tokenReference.key
+        );
+
+        if (!secret) {
+          throw new Error("Failed to get access token");
+        }
+
+        // TODO refresh access token here
+        return new WebClient(
+          options?.forceBotToken
+            ? secret.botAccessToken
+            : secret.userAccessToken ?? secret.botAccessToken
+        ) as AuthenticatedClientForIntegration<TService>;
+      }
+      default: {
+        throw new Error(`Unsupported service ${integration.service}`);
+      }
+    }
+  }
+
+  static isSlackSupported =
+    !!env.ORG_SLACK_INTEGRATION_CLIENT_ID && !!env.ORG_SLACK_INTEGRATION_CLIENT_SECRET;
+
+  static slackAuthorizationUrl(
+    state: string,
+    scopes: string[] = [
+      "channels:read",
+      "groups:read",
+      "im:read",
+      "mpim:read",
+      "chat:write",
+      "chat:write.public",
+    ],
+    userScopes: string[] = ["channels:read", "groups:read", "im:read", "mpim:read", "chat:write"]
+  ) {
+    return `https://slack.com/oauth/v2/authorize?client_id=${
+      env.ORG_SLACK_INTEGRATION_CLIENT_ID
+    }&scope=${scopes.join(",")}&user_scope=${userScopes.join(",")}&state=${state}&redirect_uri=${
+      env.APP_ORIGIN
+    }/integrations/slack/callback`;
+  }
+
+  static async redirectToAuthService(
+    service: IntegrationService,
+    state: string,
+    request: Request,
+    redirectTo: string
+  ) {
+    const session = await getUserSession(request);
+    session.set(REDIRECT_AFTER_AUTH_KEY, redirectTo);
+
+    const authUrl = service === "SLACK" ? this.slackAuthorizationUrl(state) : undefined;
+
+    if (!authUrl) {
+      throw new Response("Unsupported service", { status: 400 });
+    }
+
+    logger.debug("Redirecting to auth service", {
+      service,
+      authUrl,
+      redirectTo,
+    });
+
+    return new Response(null, {
+      status: 302,
+      headers: {
+        location: authUrl,
+        "Set-Cookie": await commitSession(session),
+      },
+    });
+  }
+
+  static async redirectAfterAuth(request: Request) {
+    const session = await getUserSession(request);
+
+    logger.debug("Redirecting back after auth", {
+      sessionData: session.data,
+    });
+
+    const redirectTo = session.get(REDIRECT_AFTER_AUTH_KEY);
+
+    if (!redirectTo) {
+      throw new Response("Invalid redirect", { status: 400 });
+    }
+
+    session.unset(REDIRECT_AFTER_AUTH_KEY);
+
+    return new Response(null, {
+      status: 302,
+      headers: {
+        location: redirectTo,
+        "Set-Cookie": await commitSession(session),
+      },
+    });
+  }
+
+  static async createOrgIntegration(serviceName: string, code: string, org: Organization) {
+    switch (serviceName) {
+      case "slack": {
+        if (!env.ORG_SLACK_INTEGRATION_CLIENT_ID || !env.ORG_SLACK_INTEGRATION_CLIENT_SECRET) {
+          throw new Error("Slack integration not configured");
+        }
+
+        const client = new WebClient();
+
+        const result = await client.oauth.v2.access({
+          client_id: env.ORG_SLACK_INTEGRATION_CLIENT_ID,
+          client_secret: env.ORG_SLACK_INTEGRATION_CLIENT_SECRET,
+          code,
+          redirect_uri: `${env.APP_ORIGIN}/integrations/slack/callback`,
+        });
+
+        if (result.ok) {
+          logger.debug("Received slack access token", {
+            result,
+          });
+
+          if (!result.access_token) {
+            throw new Error("Failed to get access token");
+          }
+
+          return await $transaction(prisma, async (tx) => {
+            const secretStore = getSecretStore("DATABASE", {
+              prismaClient: tx,
+            });
+
+            const integrationFriendlyId = generateFriendlyId("org_integration");
+
+            const secretValue: SlackSecret = {
+              botAccessToken: result.access_token!,
+              userAccessToken: result.authed_user ? result.authed_user.access_token : undefined,
+              expiresIn: result.expires_in,
+              refreshToken: result.refresh_token,
+              botScopes: result.scope ? result.scope.split(",") : [],
+              userScopes: result.authed_user?.scope ? result.authed_user.scope.split(",") : [],
+              raw: result,
+            };
+
+            logger.debug("Setting secret", {
+              secretValue,
+            });
+
+            await secretStore.setSecret(integrationFriendlyId, secretValue);
+
+            const reference = await tx.secretReference.create({
+              data: {
+                provider: "DATABASE",
+                key: integrationFriendlyId,
+              },
+            });
+
+            return await tx.organizationIntegration.create({
+              data: {
+                friendlyId: integrationFriendlyId,
+                organizationId: org.id,
+                service: "SLACK",
+                tokenReferenceId: reference.id,
+                integrationData: {
+                  team: result.team,
+                  user: result.authed_user
+                    ? {
+                        id: result.authed_user.id,
+                      }
+                    : undefined,
+                } as any,
+              },
+            });
+          });
+        }
+      }
+      default: {
+        throw new Error(`Service ${serviceName} not supported`);
+      }
+    }
+  }
+}

apps/webapp/app/models/project.server.ts

@@ -110,3 +110,15 @@ export async function findProjectBySlug(orgSlug: string, projectSlug: string, us
     },
   });
 }
+
+export async function findProjectByRef(externalRef: string, userId: string) {
+  // Find the project scoped to the organization, making sure the user belongs to that org
+  return await prisma.project.findFirst({
+    where: {
+      externalRef,
+      organization: {
+        members: { some: { userId } },
+      },
+    },
+  });
+}

apps/webapp/app/models/projectAlert.server.ts

@@ -0,0 +1,33 @@
+import { z } from "zod";
+import { EncryptedSecretValueSchema } from "~/services/secrets/secretStore.server";
+
+export const ProjectAlertWebhookProperties = z.object({
+  secret: EncryptedSecretValueSchema,
+  url: z.string(),
+});
+
+export type ProjectAlertWebhookProperties = z.infer<typeof ProjectAlertWebhookProperties>;
+
+export const ProjectAlertEmailProperties = z.object({
+  email: z.string(),
+});
+
+export type ProjectAlertEmailProperties = z.infer<typeof ProjectAlertEmailProperties>;
+
+export const DeleteProjectAlertChannel = z.object({
+  id: z.string(),
+});
+
+export const ProjectAlertSlackProperties = z.object({
+  channelId: z.string(),
+  channelName: z.string(),
+  integrationId: z.string().nullish(),
+});
+
+export type ProjectAlertSlackProperties = z.infer<typeof ProjectAlertSlackProperties>;
+
+export const ProjectAlertSlackStorage = z.object({
+  message_ts: z.string(),
+});
+
+export type ProjectAlertSlackStorage = z.infer<typeof ProjectAlertSlackStorage>;

apps/webapp/app/platform/zodWorker.server.ts

@@ -1,20 +1,21 @@
 import type {
   CronItem,
   CronItemOptions,
-  Job as GraphileJob,
+  DbJob as GraphileJob,
   Runner as GraphileRunner,
   JobHelpers,
   RunnerOptions,
   Task,
   TaskList,
   TaskSpec,
+  WorkerUtils,
 } from "graphile-worker";
-import { run as graphileRun, parseCronItems } from "graphile-worker";
+import { run as graphileRun, makeWorkerUtils, parseCronItems } from "graphile-worker";
 import { SpanKind, trace } from "@opentelemetry/api";
 
 import omit from "lodash.omit";
 import { z } from "zod";
-import { PrismaClient, PrismaClientOrTransaction } from "~/db.server";
+import { $replica, PrismaClient, PrismaClientOrTransaction } from "~/db.server";
 import { PgListenService } from "~/services/db/pgListen.server";
 import { workerLogger as logger } from "~/services/logger.server";
 import { flattenAttributes } from "@trigger.dev/core/v3";
@@ -34,8 +35,8 @@ const RawCronPayloadSchema = z.object({
 
 const GraphileJobSchema = z.object({
   id: z.coerce.string(),
-  queue_name: z.string().nullable(),
-  task_identifier: z.string(),
+  job_queue_id: z.number().nullable(),
+  task_id: z.number(),
   payload: z.unknown(),
   priority: z.number(),
   run_at: z.coerce.date(),
@@ -72,7 +73,7 @@ type RecurringTaskPayload = {
 
 export type ZodRecurringTasks = {
   [key: string]: {
-    pattern: string;
+    match: string;
     options?: CronItemOptions;
     handler: (payload: RecurringTaskPayload, job: GraphileJob) => Promise<void>;
   };
@@ -129,6 +130,7 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
   #rateLimiter?: ZodWorkerRateLimiter;
   #shutdownTimeoutInMs?: number;
   #shuttingDown = false;
+  #workerUtils?: WorkerUtils;
 
   constructor(options: ZodWorkerOptions<TMessageCatalog>) {
     this.#name = options.name;
@@ -158,6 +160,8 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
 
     const parsedCronItems = parseCronItems(this.#createCronItemsFromRecurringTasks());
 
+    this.#workerUtils = await makeWorkerUtils(this.#runnerOptions);
+
     this.#runner = await graphileRun({
       ...this.#runnerOptions,
       noHandleSignals: true,
@@ -188,7 +192,7 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
         this.#logDebug("Detected incoming migration", { latestMigration });
 
         if (latestMigration > 10) {
-          // already migrated past v0.14 - nothing to do
+          this.#logDebug("Already migrated past v0.14 - nothing to do", { latestMigration });
           return;
         }
 
@@ -263,6 +267,7 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
 
   public async stop() {
     await this.#runner?.stop();
+    await this.#workerUtils?.release();
   }
 
   public async enqueue<K extends keyof TMessageCatalog>(
@@ -442,12 +447,29 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
     return taskList;
   }
 
+  async #getQueueName(queueId: number | null) {
+    if (queueId === null) {
+      return;
+    }
+
+    const schema = z.array(z.object({ queue_name: z.string() }));
+
+    const rawQueueNameResults = await $replica.$queryRawUnsafe(
+      `SELECT queue_name FROM ${this.graphileWorkerSchema}._private_job_queues WHERE id = $1`,
+      queueId
+    );
+
+    const queueNameResults = schema.parse(rawQueueNameResults);
+
+    return queueNameResults[0]?.queue_name;
+  }
+
   async #rescheduleTask(payload: unknown, helpers: JobHelpers) {
     this.#logDebug("Rescheduling task", { payload, job: helpers.job });
 
     await this.enqueue(helpers.job.task_identifier, payload, {
-      runAt: helpers.job.run_at,
-      queueName: helpers.job.queue_name ?? undefined,
+      runAt: new Date(Date.now() + 1000 * 10),
+      queueName: await this.#getQueueName(helpers.job.job_queue_id),
       priority: helpers.job.priority,
       jobKey: helpers.job.key ?? undefined,
       flags: Object.keys(helpers.job.flags ?? []),
@@ -460,7 +482,7 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
 
     if (this.#cleanup) {
       cronItems.push({
-        pattern: this.#cleanup.frequencyExpression,
+        match: this.#cleanup.frequencyExpression,
         identifier: CLEANUP_TASK_NAME,
         task: CLEANUP_TASK_NAME,
         options: this.#cleanup.taskOptions,
@@ -469,7 +491,7 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
 
     if (this.#reporter) {
       cronItems.push({
-        pattern: "50 * * * *", // Every hour at 50 minutes past the hour
+        match: "50 * * * *", // Every hour at 50 minutes past the hour
         identifier: REPORTER_TASK_NAME,
         task: REPORTER_TASK_NAME,
       });
@@ -481,7 +503,7 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
 
     for (const [key, task] of Object.entries(this.#recurringTasks)) {
       const cronItem: CronItem = {
-        pattern: task.pattern,
+        match: task.match,
         identifier: key,
         task: key,
         options: task.options,
@@ -529,7 +551,7 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
         attributes: {
           "job.task_identifier": job.task_identifier,
           "job.id": job.id,
-          ...(job.queue_name ? { "job.queue_name": job.queue_name } : {}),
+          ...(job.job_queue_id ? { "job.queue_id": job.job_queue_id } : {}),
           ...flattenAttributes(job.payload as Record<string, unknown>, "job.payload"),
           "job.priority": job.priority,
           "job.run_at": job.run_at.toISOString(),
@@ -555,6 +577,17 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
             span.recordException(new Error(String(error)));
           }
 
+          if (job.attempts >= job.max_attempts) {
+            logger.debug("Job failed after max attempts", {
+              job,
+              attempts: job.attempts,
+              max_attempts: job.max_attempts,
+              error: error instanceof Error ? error.message : error,
+            });
+
+            return;
+          }
+
           throw error;
         } finally {
           span.end();
@@ -599,7 +632,7 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
         attributes: {
           "job.task_identifier": job.task_identifier,
           "job.id": job.id,
-          ...(job.queue_name ? { "job.queue_name": job.queue_name } : {}),
+          ...(job.job_queue_id ? { "job.queue_id": job.job_queue_id } : {}),
           ...flattenAttributes(job.payload as Record<string, unknown>, "job.payload"),
           "job.priority": job.priority,
           "job.run_at": job.run_at.toISOString(),
@@ -638,6 +671,10 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
       return;
     }
 
+    if (!this.#workerUtils) {
+      throw new Error("WorkerUtils need to be initialized before running job cleanup.");
+    }
+
     const job = helpers.job;
 
     logger.debug("Received cleanup task", {
@@ -663,23 +700,38 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
       payload,
     });
 
-    const rawResults = await this.#prisma.$queryRawUnsafe(
-      `WITH rows AS (SELECT id FROM ${this.graphileWorkerSchema}.jobs WHERE run_at < $1 AND locked_at IS NULL AND max_attempts = attempts LIMIT $2 FOR UPDATE) DELETE FROM ${this.graphileWorkerSchema}.jobs WHERE id IN (SELECT id FROM rows) RETURNING id`,
+    const rawResults = await $replica.$queryRawUnsafe(
+      `SELECT id
+        FROM ${this.graphileWorkerSchema}.jobs
+        WHERE run_at < $1
+          AND locked_at IS NULL
+          AND max_attempts = attempts
+        LIMIT $2`,
       expirationDate,
       this.#cleanup.maxCount
     );
 
-    const results = Array.isArray(rawResults) ? rawResults : [];
+    const results = z
+      .array(
+        z.object({
+          id: z.coerce.string(),
+        })
+      )
+      .parse(rawResults);
+
+    const completedJobs = await this.#workerUtils.completeJobs(results.map((job) => job.id));
 
     logger.debug("Cleaned up old jobs", {
-      count: results.length,
+      found: results.length,
+      deleted: completedJobs.length,
       expirationDate,
       payload,
     });
 
     if (this.#reporter) {
       await this.#reporter("cleanup_stats", {
-        count: results.length,
+        found: results.length,
+        deleted: completedJobs.length,
         expirationDate,
         ts: payload._cron.ts,
       });
@@ -711,7 +763,7 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
     const schema = z.array(z.object({ count: z.coerce.number() }));
 
     // Count the number of jobs that have been added since the startAt date and before the payload._cron.ts date
-    const rawAddedResults = await this.#prisma.$queryRawUnsafe(
+    const rawAddedResults = await $replica.$queryRawUnsafe(
       `SELECT COUNT(*) FROM ${this.graphileWorkerSchema}.jobs WHERE created_at > $1 AND created_at < $2`,
       startAt,
       payload._cron.ts
@@ -720,7 +772,7 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
     const addedCountResults = schema.parse(rawAddedResults)[0];
 
     // Count the total number of jobs in the jobs table
-    const rawTotalResults = await this.#prisma.$queryRawUnsafe(
+    const rawTotalResults = await $replica.$queryRawUnsafe(
       `SELECT COUNT(*) FROM ${this.graphileWorkerSchema}.jobs`
     );
 

apps/webapp/app/presenters/EnvironmentsPresenter.server.ts

@@ -14,7 +14,7 @@ import {
   IndexEndpointStats,
   parseEndpointIndexStats,
 } from "@trigger.dev/core";
-import { sortEnvironments } from "~/services/environmentSort.server";
+import { sortEnvironments } from "~/utils/environmentSort";
 
 export type Client = {
   slug: string;

apps/webapp/app/presenters/HttpEndpointPresenter.server.ts

@@ -1,6 +1,6 @@
 import { z } from "zod";
 import { PrismaClient, prisma } from "~/db.server";
-import { sortEnvironments } from "~/services/environmentSort.server";
+import { sortEnvironments } from "~/utils/environmentSort";
 import { httpEndpointUrl } from "~/services/httpendpoint/HandleHttpEndpointService";
 import { getSecretStore } from "~/services/secrets/secretStore.server";
 import { projectPath } from "~/utils/pathBuilder";

apps/webapp/app/presenters/ProjectPresenter.server.ts

@@ -2,7 +2,7 @@ import { PrismaClient, prisma } from "~/db.server";
 import { Project } from "~/models/project.server";
 import { displayableEnvironments } from "~/models/runtimeEnvironment.server";
 import { User } from "~/models/user.server";
-import { sortEnvironments } from "~/services/environmentSort.server";
+import { sortEnvironments } from "~/utils/environmentSort";
 
 export class ProjectPresenter {
   #prismaClient: PrismaClient;

apps/webapp/app/presenters/v3/AlertChannelListPresenter.server.ts

@@ -0,0 +1,73 @@
+import { logger } from "~/services/logger.server";
+import { BasePresenter } from "./basePresenter.server";
+import { ProjectAlertChannel } from "@trigger.dev/database";
+import { decryptSecret } from "~/services/secrets/secretStore.server";
+import { env } from "~/env.server";
+import {
+  ProjectAlertEmailProperties,
+  ProjectAlertSlackProperties,
+  ProjectAlertWebhookProperties,
+} from "~/models/projectAlert.server";
+
+export type AlertChannelListPresenterData = Awaited<ReturnType<AlertChannelListPresenter["call"]>>;
+export type AlertChannelListPresenterRecord =
+  AlertChannelListPresenterData["alertChannels"][number];
+export type AlertChannelListPresenterAlertProperties = NonNullable<
+  AlertChannelListPresenterRecord["properties"]
+>;
+
+export class AlertChannelListPresenter extends BasePresenter {
+  public async call(projectId: string) {
+    logger.debug("AlertChannelListPresenter", { projectId });
+
+    const alertChannels = await this._prisma.projectAlertChannel.findMany({
+      where: {
+        projectId,
+      },
+      orderBy: {
+        createdAt: "desc",
+      },
+    });
+
+    return {
+      alertChannels: await Promise.all(
+        alertChannels.map(async (alertChannel) => ({
+          ...alertChannel,
+          properties: await this.#presentProperties(alertChannel),
+        }))
+      ),
+    };
+  }
+
+  async #presentProperties(alertChannel: ProjectAlertChannel) {
+    if (!alertChannel.properties) {
+      return;
+    }
+
+    switch (alertChannel.type) {
+      case "WEBHOOK":
+        const parsedProperties = ProjectAlertWebhookProperties.parse(alertChannel.properties);
+
+        const secret = await decryptSecret(env.ENCRYPTION_KEY, parsedProperties.secret);
+
+        return {
+          type: "WEBHOOK" as const,
+          url: parsedProperties.url,
+          secret,
+        };
+      case "EMAIL":
+        return {
+          type: "EMAIL" as const,
+          ...ProjectAlertEmailProperties.parse(alertChannel.properties),
+        };
+      case "SLACK": {
+        return {
+          type: "SLACK" as const,
+          ...ProjectAlertSlackProperties.parse(alertChannel.properties),
+        };
+      }
+      default:
+        throw new Error(`Unsupported alert channel type: ${alertChannel.type}`);
+    }
+  }
+}

apps/webapp/app/presenters/v3/ApiAlertChannelPresenter.server.ts

@@ -0,0 +1,136 @@
+import {
+  ProjectAlertChannel,
+  ProjectAlertChannelType,
+  ProjectAlertType,
+} from "@trigger.dev/database";
+import assertNever from "assert-never";
+import { z } from "zod";
+import { env } from "~/env.server";
+import {
+  ProjectAlertEmailProperties,
+  ProjectAlertWebhookProperties,
+} from "~/models/projectAlert.server";
+import { decryptSecret } from "~/services/secrets/secretStore.server";
+
+export const ApiAlertType = z.enum(["attempt_failure", "deployment_failure", "deployment_success"]);
+
+export type ApiAlertType = z.infer<typeof ApiAlertType>;
+
+export const ApiAlertEnvironmentType = z.enum(["STAGING", "PRODUCTION"]);
+
+export type ApiAlertEnvironmentType = z.infer<typeof ApiAlertEnvironmentType>;
+
+export const ApiAlertChannel = z.enum(["email", "webhook"]);
+
+export type ApiAlertChannel = z.infer<typeof ApiAlertChannel>;
+
+export const ApiAlertChannelData = z.object({
+  email: z.string().optional(),
+  url: z.string().optional(),
+  secret: z.string().optional(),
+});
+
+export type ApiAlertChannelData = z.infer<typeof ApiAlertChannelData>;
+
+export const ApiCreateAlertChannel = z.object({
+  alertTypes: ApiAlertType.array(),
+  name: z.string(),
+  channel: ApiAlertChannel,
+  channelData: ApiAlertChannelData,
+  deduplicationKey: z.string().optional(),
+  environmentTypes: ApiAlertEnvironmentType.array().default(["STAGING", "PRODUCTION"]),
+});
+
+export type ApiCreateAlertChannel = z.infer<typeof ApiCreateAlertChannel>;
+
+export const ApiAlertChannelObject = z.object({
+  id: z.string(),
+  name: z.string(),
+  alertTypes: ApiAlertType.array(),
+  channel: ApiAlertChannel,
+  channelData: ApiAlertChannelData,
+  deduplicationKey: z.string().optional(),
+});
+
+export type ApiAlertChannelObject = z.infer<typeof ApiAlertChannelObject>;
+
+export class ApiAlertChannelPresenter {
+  public static async alertChannelToApi(
+    alertChannel: ProjectAlertChannel
+  ): Promise<ApiAlertChannelObject> {
+    return {
+      id: alertChannel.friendlyId,
+      name: alertChannel.name,
+      alertTypes: alertChannel.alertTypes.map((type) => this.alertTypeToApi(type)),
+      channel: this.alertChannelTypeToApi(alertChannel.type),
+      channelData: await channelDataFromProperties(alertChannel.type, alertChannel.properties),
+      deduplicationKey: alertChannel.userProvidedDeduplicationKey
+        ? alertChannel.deduplicationKey
+        : undefined,
+    };
+  }
+
+  public static alertTypeToApi(alertType: ProjectAlertType): ApiAlertType {
+    switch (alertType) {
+      case "TASK_RUN_ATTEMPT":
+        return "attempt_failure";
+      case "DEPLOYMENT_FAILURE":
+        return "deployment_failure";
+      case "DEPLOYMENT_SUCCESS":
+        return "deployment_success";
+      default:
+        assertNever(alertType);
+    }
+  }
+
+  public static alertTypeFromApi(alertType: ApiAlertType): ProjectAlertType {
+    switch (alertType) {
+      case "attempt_failure":
+        return "TASK_RUN_ATTEMPT";
+      case "deployment_failure":
+        return "DEPLOYMENT_FAILURE";
+      case "deployment_success":
+        return "DEPLOYMENT_SUCCESS";
+      default:
+        assertNever(alertType);
+    }
+  }
+
+  public static alertChannelTypeToApi(type: ProjectAlertChannelType): ApiAlertChannel {
+    switch (type) {
+      case "EMAIL":
+        return "email";
+      case "WEBHOOK":
+        return "webhook";
+      case "SLACK":
+        throw new Error("Slack channels are not supported");
+      default:
+        assertNever(type);
+    }
+  }
+}
+
+async function channelDataFromProperties(
+  type: ProjectAlertChannelType,
+  properties: ProjectAlertChannel["properties"]
+): Promise<ApiAlertChannelData> {
+  if (!properties) {
+    return {};
+  }
+
+  switch (type) {
+    case "EMAIL":
+      return ProjectAlertEmailProperties.parse(properties);
+    case "WEBHOOK":
+      const { url, secret } = ProjectAlertWebhookProperties.parse(properties);
+
+      return {
+        url,
+        secret: await decryptSecret(env.ENCRYPTION_KEY, secret),
+      };
+    case "SLACK":
+      throw new Error("Slack channels are not supported");
+    default:
+      assertNever(type);
+  }
+}

apps/webapp/app/presenters/v3/ApiKeysPresenter.server.ts

@@ -1,7 +1,7 @@
 import { PrismaClient, prisma } from "~/db.server";
 import { Project } from "~/models/project.server";
 import { User } from "~/models/user.server";
-import { sortEnvironments } from "~/services/environmentSort.server";
+import { sortEnvironments } from "~/utils/environmentSort";
 
 export class ApiKeysPresenter {
   #prismaClient: PrismaClient;

apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts

@@ -144,18 +144,18 @@ export class DeploymentPresenter {
           userName: getUsername(deployment.environment.orgMember?.user),
         },
         deployedBy: deployment.triggeredBy,
-        errorData: this.#prepareErrorData(deployment.errorData),
         sdkVersion: deployment.worker?.sdkVersion,
         imageReference: deployment.imageReference,
         externalBuildData:
           externalBuildData && externalBuildData.success ? externalBuildData.data : undefined,
         projectId: deployment.projectId,
         organizationId: project.organizationId,
+        errorData: DeploymentPresenter.prepareErrorData(deployment.errorData),
       },
     };
   }
 
-  #prepareErrorData(errorData: WorkerDeployment["errorData"]): ErrorData | undefined {
+  public static prepareErrorData(errorData: WorkerDeployment["errorData"]): ErrorData | undefined {
     if (!errorData) {
       return;
     }

apps/webapp/app/presenters/v3/EnvironmentVariablesPresenter.server.ts

@@ -1,7 +1,7 @@
 import { PrismaClient, prisma } from "~/db.server";
 import { Project } from "~/models/project.server";
 import { User } from "~/models/user.server";
-import { sortEnvironments } from "~/services/environmentSort.server";
+import { sortEnvironments } from "~/utils/environmentSort";
 import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server";
 
 type Result = Awaited<ReturnType<EnvironmentVariablesPresenter["call"]>>;

apps/webapp/app/presenters/v3/NewAlertChannelPresenter.server.ts

@@ -0,0 +1,104 @@
+import {
+  AuthenticatableIntegration,
+  OrgIntegrationRepository,
+} from "~/models/orgIntegration.server";
+import { logger } from "~/services/logger.server";
+import { BasePresenter } from "./basePresenter.server";
+import { WebClient } from "@slack/web-api";
+
+export class NewAlertChannelPresenter extends BasePresenter {
+  public async call(projectId: string) {
+    const project = await this._prisma.project.findUniqueOrThrow({
+      where: {
+        id: projectId,
+      },
+    });
+
+    // Find the latest Slack integration
+    const slackIntegration = await this._prisma.organizationIntegration.findFirst({
+      where: {
+        service: "SLACK",
+        organizationId: project.organizationId,
+      },
+      orderBy: {
+        createdAt: "desc",
+      },
+      include: {
+        tokenReference: true,
+      },
+    });
+
+    // If there is a slack integration, then we need to get a list of Slack Channels
+    if (slackIntegration) {
+      const channels = await getSlackChannelsForToken(slackIntegration);
+
+      return {
+        slack: {
+          status: "READY" as const,
+          channels,
+          integrationId: slackIntegration.id,
+        },
+      };
+    } else {
+      if (OrgIntegrationRepository.isSlackSupported) {
+        return {
+          slack: {
+            status: "NOT_CONFIGURED" as const,
+          },
+        };
+      } else {
+        return {
+          slack: {
+            status: "NOT_AVAILABLE" as const,
+          },
+        };
+      }
+    }
+  }
+}
+
+async function getSlackChannelsForToken(integration: AuthenticatableIntegration) {
+  const client = await OrgIntegrationRepository.getAuthenticatedClientForIntegration(integration);
+
+  const channels = await getAllSlackConversations(client);
+
+  logger.debug("Received a list of slack conversations", {
+    channels,
+  });
+
+  return (channels ?? [])
+    .filter((channel) => !channel.is_archived)
+    .filter((channel) => channel.is_channel)
+    .filter((channel) => !channel.is_ext_shared)
+    .filter((channel) => channel.unlinked === 0)
+    .filter((channel) => channel.num_members)
+    .sort((a, b) => a.name!.localeCompare(b.name!));
+}
+
+type Channels = Awaited<ReturnType<WebClient["conversations"]["list"]>>["channels"];
+
+async function getSlackConversationsPage(client: WebClient, nextCursor?: string) {
+  return client.conversations.list({
+    types: "public_channel,private_channel",
+    exclude_archived: true,
+    cursor: nextCursor,
+  });
+}
+
+async function getAllSlackConversations(client: WebClient) {
+  let nextCursor: string | undefined = undefined;
+  let channels: Channels = [];
+
+  do {
+    const response = await getSlackConversationsPage(client, nextCursor);
+
+    if (!response.ok) {
+      throw new Error(`Failed to get channels: ${response.error}`);
+    }
+
+    channels = channels.concat(response.channels ?? []);
+    nextCursor = response.response_metadata?.next_cursor;
+  } while (nextCursor);
+
+  return channels;
+}

apps/webapp/app/presenters/v3/RunListPresenter.server.ts

@@ -17,6 +17,7 @@ type RunListOptions = {
   environments?: string[];
   scheduleId?: string;
   period?: string;
+  bulkId?: string;
   from?: number;
   to?: number;
   //pagination
@@ -25,7 +26,7 @@ type RunListOptions = {
   pageSize?: number;
 };
 
-const DEFAULT_PAGE_SIZE = 20;
+const DEFAULT_PAGE_SIZE = 25;
 
 export type RunList = Awaited<ReturnType<RunListPresenter["call"]>>;
 export type RunListItem = RunList["runs"][0];
@@ -41,6 +42,7 @@ export class RunListPresenter extends BasePresenter {
     environments,
     scheduleId,
     period,
+    bulkId,
     from,
     to,
     direction = "forward",
@@ -55,6 +57,7 @@ export class RunListPresenter extends BasePresenter {
       hasStatusFilters ||
       (environments !== undefined && environments.length > 0) ||
       (period !== undefined && period !== "all") ||
+      (bulkId !== undefined && bulkId !== "") ||
       from !== undefined ||
       to !== undefined;
 
@@ -87,13 +90,55 @@ export class RunListPresenter extends BasePresenter {
     });
 
     //get all possible tasks
-    const possibleTasks = await this._replica.backgroundWorkerTask.findMany({
+    const possibleTasksAsync = this._replica.backgroundWorkerTask.findMany({
       distinct: ["slug"],
       where: {
         projectId: project.id,
       },
     });
 
+    //get possible bulk actions
+    const bulkActionsAsync = this._replica.bulkActionGroup.findMany({
+      select: {
+        friendlyId: true,
+        type: true,
+        createdAt: true,
+      },
+      where: {
+        projectId: project.id,
+      },
+      orderBy: {
+        createdAt: "desc",
+      },
+      take: 20,
+    });
+
+    const [possibleTasks, bulkActions] = await Promise.all([possibleTasksAsync, bulkActionsAsync]);
+
+    //we can restrict to specific runs using bulkId, or batchId
+    let restrictToRunIds: undefined | string[] = undefined;
+
+    //bulk id
+    if (bulkId) {
+      const bulkAction = await this._replica.bulkActionGroup.findUnique({
+        select: {
+          items: {
+            select: {
+              destinationRunId: true,
+            },
+          },
+        },
+        where: {
+          friendlyId: bulkId,
+        },
+      });
+
+      if (bulkAction) {
+        const runIds = bulkAction.items.map((item) => item.destinationRunId).filter(Boolean);
+        restrictToRunIds = runIds;
+      }
+    }
+
     const periodMs = period ? parse(period) : undefined;
 
     //get the runs
@@ -142,6 +187,13 @@ export class RunListPresenter extends BasePresenter {
           : Prisma.empty
       }
       -- filters
+      ${
+        restrictToRunIds
+          ? restrictToRunIds.length === 0
+            ? Prisma.sql`AND tr.id = ''`
+            : Prisma.sql`AND tr.id IN (${Prisma.join(restrictToRunIds)})`
+          : Prisma.empty
+      }
       ${
         tasks && tasks.length > 0
           ? Prisma.sql`AND tr."taskIdentifier" IN (${Prisma.join(tasks)})`
@@ -240,6 +292,11 @@ export class RunListPresenter extends BasePresenter {
         .sort((a, b) => {
           return a.slug.localeCompare(b.slug);
         }),
+      bulkActions: bulkActions.map((bulkAction) => ({
+        id: bulkAction.friendlyId,
+        type: bulkAction.type,
+        createdAt: bulkAction.createdAt,
+      })),
       filters: {
         tasks: tasks || [],
         versions: versions || [],

apps/webapp/app/presenters/v3/TaskListPresenter.server.ts

@@ -10,7 +10,7 @@ import { Organization } from "~/models/organization.server";
 import { Project } from "~/models/project.server";
 import { displayableEnvironments } from "~/models/runtimeEnvironment.server";
 import { User } from "~/models/user.server";
-import { sortEnvironments } from "~/services/environmentSort.server";
+import { sortEnvironments } from "~/utils/environmentSort";
 import { logger } from "~/services/logger.server";
 import { BasePresenter } from "./basePresenter.server";
 

apps/webapp/app/presenters/v3/TestPresenter.server.ts

@@ -1,7 +1,7 @@
 import { TaskTriggerSource } from "@trigger.dev/database";
 import { sqlDatabaseSchema, PrismaClient, prisma } from "~/db.server";
 import { TestSearchParams } from "~/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.test/route";
-import { sortEnvironments } from "~/services/environmentSort.server";
+import { sortEnvironments } from "~/utils/environmentSort";
 import { createSearchParams } from "~/utils/searchParams";
 import { getUsername } from "~/utils/username";
 

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam._index/route.tsx

@@ -11,7 +11,7 @@ import { InitCommandV3, TriggerDevStepV3, TriggerLoginStepV3 } from "~/component
 import { StepContentContainer } from "~/components/StepContentContainer";
 import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
 import { InlineCode } from "~/components/code/InlineCode";
-import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
+import { EnvironmentLabels } from "~/components/environments/EnvironmentLabel";
 import { MainCenteredContainer, PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { Button } from "~/components/primitives/Buttons";
 import { Callout } from "~/components/primitives/Callout";
@@ -261,15 +261,7 @@ export default function Page() {
                               </Suspense>
                             </TableCell>
                             <TableCell to={path}>
-                              <div className="space-x-2">
-                                {task.environments.map((environment) => (
-                                  <EnvironmentLabel
-                                    key={environment.id}
-                                    environment={environment}
-                                    userName={environment.userName}
-                                  />
-                                ))}
-                              </div>
+                              <EnvironmentLabels environments={task.environments} />
                             </TableCell>
                             <TableCellChevron to={path} />
                           </TableRow>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.alerts.new.connect-to-slack.ts

@@ -0,0 +1,50 @@
+import { LoaderFunctionArgs, redirect } from "@remix-run/server-runtime";
+import { prisma } from "~/db.server";
+import {
+  redirectBackWithSuccessMessage,
+  redirectWithSuccessMessage,
+} from "~/models/message.server";
+import { OrgIntegrationRepository } from "~/models/orgIntegration.server";
+import { findProjectBySlug } from "~/models/project.server";
+import { requireUserId } from "~/services/session.server";
+import { getUserSession } from "~/services/sessionStorage.server";
+import {
+  ProjectParamSchema,
+  v3NewProjectAlertPath,
+  v3NewProjectAlertPathConnectToSlackPath,
+} from "~/utils/pathBuilder";
+
+export async function loader({ request, params }: LoaderFunctionArgs) {
+  const userId = await requireUserId(request);
+  const { organizationSlug, projectParam } = ProjectParamSchema.parse(params);
+
+  const project = await findProjectBySlug(organizationSlug, projectParam, userId);
+
+  if (!project) {
+    throw new Response("Project not found", { status: 404 });
+  }
+
+  // Find an integration for Slack for this org
+  const integration = await prisma.organizationIntegration.findFirst({
+    where: {
+      service: "SLACK",
+      organizationId: project.organizationId,
+    },
+  });
+
+  if (integration) {
+    return redirectWithSuccessMessage(
+      `${v3NewProjectAlertPath({ slug: organizationSlug }, project)}?option=slack`,
+      request,
+      "Successfully connected your Slack workspace"
+    );
+  } else {
+    // Redirect to Slack
+    return await OrgIntegrationRepository.redirectToAuthService(
+      "SLACK",
+      project.organizationId,
+      request,
+      v3NewProjectAlertPathConnectToSlackPath({ slug: organizationSlug }, project)
+    );
+  }
+}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.alerts.new/route.tsx

@@ -0,0 +1,456 @@
+import { conform, useForm } from "@conform-to/react";
+import { parse } from "@conform-to/zod";
+import { HashtagIcon, LockClosedIcon } from "@heroicons/react/20/solid";
+import { Form, useActionData, useNavigate, useNavigation } from "@remix-run/react";
+import { LoaderFunctionArgs } from "@remix-run/router";
+import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
+import { SlackIcon } from "@trigger.dev/companyicons";
+import { useEffect, useState } from "react";
+import { typedjson, useTypedLoaderData } from "remix-typedjson";
+import { z } from "zod";
+import { InlineCode } from "~/components/code/InlineCode";
+import { Button, LinkButton } from "~/components/primitives/Buttons";
+import { Callout, variantClasses } from "~/components/primitives/Callout";
+import { CheckboxWithLabel } from "~/components/primitives/Checkbox";
+import { Dialog, DialogContent, DialogHeader } from "~/components/primitives/Dialog";
+import { Fieldset } from "~/components/primitives/Fieldset";
+import { FormButtons } from "~/components/primitives/FormButtons";
+import { FormError } from "~/components/primitives/FormError";
+import { Hint } from "~/components/primitives/Hint";
+import { Input } from "~/components/primitives/Input";
+import { InputGroup } from "~/components/primitives/InputGroup";
+import { Label } from "~/components/primitives/Label";
+import { Paragraph } from "~/components/primitives/Paragraph";
+import SegmentedControl from "~/components/primitives/SegmentedControl";
+import { Select, SelectItem } from "~/components/primitives/Select";
+import { InfoIconTooltip } from "~/components/primitives/Tooltip";
+import { useOrganization } from "~/hooks/useOrganizations";
+import { useProject } from "~/hooks/useProject";
+import { redirectWithSuccessMessage } from "~/models/message.server";
+import { findProjectBySlug } from "~/models/project.server";
+import { NewAlertChannelPresenter } from "~/presenters/v3/NewAlertChannelPresenter.server";
+import { requireUserId } from "~/services/session.server";
+import { cn } from "~/utils/cn";
+import { ProjectParamSchema, v3ProjectAlertsPath } from "~/utils/pathBuilder";
+import {
+  CreateAlertChannelOptions,
+  CreateAlertChannelService,
+} from "~/v3/services/alerts/createAlertChannel.server";
+
+const FormSchema = z
+  .object({
+    alertTypes: z
+      .array(z.enum(["TASK_RUN_ATTEMPT", "DEPLOYMENT_FAILURE", "DEPLOYMENT_SUCCESS"]))
+      .min(1)
+      .or(z.enum(["TASK_RUN_ATTEMPT", "DEPLOYMENT_FAILURE", "DEPLOYMENT_SUCCESS"])),
+    environmentTypes: z
+      .array(z.enum(["STAGING", "PRODUCTION"]))
+      .min(1)
+      .or(z.enum(["STAGING", "PRODUCTION"])),
+    type: z.enum(["WEBHOOK", "SLACK", "EMAIL"]).default("EMAIL"),
+    channelValue: z.string().nonempty(),
+    integrationId: z.string().optional(),
+  })
+  .refine(
+    (value) =>
+      value.type === "EMAIL" ? z.string().email().safeParse(value.channelValue).success : true,
+    {
+      message: "Must be a valid email address",
+      path: ["channelValue"],
+    }
+  )
+  .refine(
+    (value) =>
+      value.type === "WEBHOOK" ? z.string().url().safeParse(value.channelValue).success : true,
+    {
+      message: "Must be a valid URL",
+      path: ["channelValue"],
+    }
+  )
+  .refine(
+    (value) =>
+      value.type === "SLACK"
+        ? typeof value.channelValue === "string" && value.channelValue.startsWith("C")
+        : true,
+    {
+      message: "Must select a Slack channel",
+      path: ["channelValue"],
+    }
+  );
+
+function formDataToCreateAlertChannelOptions(
+  formData: z.infer<typeof FormSchema>
+): CreateAlertChannelOptions {
+  switch (formData.type) {
+    case "WEBHOOK": {
+      return {
+        name: `Webhook to ${new URL(formData.channelValue).hostname}`,
+        alertTypes: Array.isArray(formData.alertTypes)
+          ? formData.alertTypes
+          : [formData.alertTypes],
+        environmentTypes: Array.isArray(formData.environmentTypes)
+          ? formData.environmentTypes
+          : [formData.environmentTypes],
+        channel: {
+          type: "WEBHOOK",
+          url: formData.channelValue,
+        },
+      };
+    }
+    case "EMAIL": {
+      return {
+        name: `Email to ${formData.channelValue}`,
+        alertTypes: Array.isArray(formData.alertTypes)
+          ? formData.alertTypes
+          : [formData.alertTypes],
+        environmentTypes: Array.isArray(formData.environmentTypes)
+          ? formData.environmentTypes
+          : [formData.environmentTypes],
+        channel: {
+          type: "EMAIL",
+          email: formData.channelValue,
+        },
+      };
+    }
+    case "SLACK": {
+      const [channelId, channelName] = formData.channelValue.split("/");
+
+      return {
+        name: `Slack message to ${channelName}`,
+        alertTypes: Array.isArray(formData.alertTypes)
+          ? formData.alertTypes
+          : [formData.alertTypes],
+        environmentTypes: Array.isArray(formData.environmentTypes)
+          ? formData.environmentTypes
+          : [formData.environmentTypes],
+        channel: {
+          type: "SLACK",
+          channelId,
+          channelName,
+          integrationId: formData.integrationId,
+        },
+      };
+    }
+  }
+}
+
+export async function loader({ request, params }: LoaderFunctionArgs) {
+  const userId = await requireUserId(request);
+  const { organizationSlug, projectParam } = ProjectParamSchema.parse(params);
+
+  const project = await findProjectBySlug(organizationSlug, projectParam, userId);
+
+  if (!project) {
+    throw new Response("Project not found", { status: 404 });
+  }
+
+  const presenter = new NewAlertChannelPresenter();
+
+  const results = await presenter.call(project.id);
+
+  const url = new URL(request.url);
+  const option = url.searchParams.get("option");
+
+  return typedjson({
+    ...results,
+    option: option === "slack" ? ("SLACK" as const) : undefined,
+  });
+}
+
+export const action = async ({ request, params }: ActionFunctionArgs) => {
+  const userId = await requireUserId(request);
+  const { organizationSlug, projectParam } = ProjectParamSchema.parse(params);
+
+  if (request.method.toUpperCase() !== "POST") {
+    return { status: 405, body: "Method Not Allowed" };
+  }
+
+  const formData = await request.formData();
+
+  const submission = parse(formData, { schema: FormSchema });
+
+  if (!submission.value) {
+    return json(submission);
+  }
+
+  const project = await findProjectBySlug(organizationSlug, projectParam, userId);
+
+  if (!project) {
+    submission.error.key = "Project not found";
+    return json(submission);
+  }
+
+  const service = new CreateAlertChannelService();
+  const alertChannel = await service.call(
+    project.externalRef,
+    userId,
+    formDataToCreateAlertChannelOptions(submission.value)
+  );
+
+  if (!alertChannel) {
+    submission.error.key = "Failed to create alert channel";
+    return json(submission);
+  }
+
+  return redirectWithSuccessMessage(
+    v3ProjectAlertsPath({ slug: organizationSlug }, { slug: projectParam }),
+    request,
+    `Created ${alertChannel.name} alert`
+  );
+};
+
+export default function Page() {
+  const [isOpen, setIsOpen] = useState(false);
+  const { slack, option } = useTypedLoaderData<typeof loader>();
+  const lastSubmission = useActionData();
+  const navigation = useNavigation();
+  const navigate = useNavigate();
+  const organization = useOrganization();
+  const project = useProject();
+  const [currentAlertChannel, setCurrentAlertChannel] = useState<string | null>(option ?? "EMAIL");
+
+  const [selectedSlackChannelValue, setSelectedSlackChannelValue] = useState<string | undefined>();
+
+  const selectedSlackChannel = slack.channels?.find(
+    (s) => selectedSlackChannelValue === `${s.id}/${s.name}`
+  );
+
+  const isLoading =
+    navigation.state !== "idle" &&
+    navigation.formMethod === "post" &&
+    navigation.formData?.get("action") === "create";
+
+  const [form, { channelValue: channelValue, alertTypes, environmentTypes, type, integrationId }] =
+    useForm({
+      id: "create-alert",
+      // TODO: type this
+      lastSubmission: lastSubmission as any,
+      onValidate({ formData }) {
+        return parse(formData, { schema: FormSchema });
+      },
+      shouldRevalidate: "onSubmit",
+    });
+
+  useEffect(() => {
+    setIsOpen(true);
+  }, []);
+
+  useEffect(() => {
+    if (navigation.state !== "idle") return;
+    if (lastSubmission !== undefined) return;
+
+    form.ref.current?.reset();
+  }, [navigation.state, lastSubmission]);
+
+  return (
+    <Dialog
+      open={isOpen}
+      onOpenChange={(o) => {
+        if (!o) {
+          navigate(v3ProjectAlertsPath(organization, project));
+        }
+      }}
+    >
+      <DialogContent>
+        <DialogHeader>New alert</DialogHeader>
+        <Form method="post" {...form.props}>
+          <Fieldset className="mt-2">
+            <InputGroup fullWidth>
+              <SegmentedControl
+                {...conform.input(type)}
+                options={[
+                  { label: "Email", value: "EMAIL" },
+                  { label: "Slack", value: "SLACK" },
+                  { label: "Webhook", value: "WEBHOOK" },
+                ]}
+                onChange={(value) => {
+                  setCurrentAlertChannel(value);
+                }}
+                fullWidth
+                defaultValue={currentAlertChannel ?? undefined}
+              />
+            </InputGroup>
+
+            {currentAlertChannel === "EMAIL" ? (
+              <InputGroup fullWidth>
+                <Label>Email</Label>
+                <Input
+                  {...conform.input(channelValue)}
+                  placeholder="email@youremail.com"
+                  type="email"
+                  autoFocus
+                />
+                <FormError id={channelValue.errorId}>{channelValue.error}</FormError>
+              </InputGroup>
+            ) : currentAlertChannel === "SLACK" ? (
+              <InputGroup fullWidth>
+                {slack.status === "READY" ? (
+                  <>
+                    <Select
+                      {...conform.select(channelValue)}
+                      placeholder="Select a Slack channel"
+                      heading="Filter channels…"
+                      defaultValue={undefined}
+                      dropdownIcon
+                      variant="tertiary/medium"
+                      items={slack.channels}
+                      setValue={(value) => {
+                        typeof value === "string" && setSelectedSlackChannelValue(value);
+                      }}
+                      filter={(channel, search) =>
+                        channel.name?.toLowerCase().includes(search.toLowerCase()) ?? false
+                      }
+                      text={(value) => {
+                        const channel = slack.channels.find((s) => value === `${s.id}/${s.name}`);
+                        if (!channel) return;
+                        return <SlackChannelTitle {...channel} />;
+                      }}
+                    >
+                      {(matches) => (
+                        <>
+                          {matches?.map((channel) => (
+                            <SelectItem key={channel.id} value={`${channel.id}/${channel.name}`}>
+                              <SlackChannelTitle {...channel} />
+                            </SelectItem>
+                          ))}
+                        </>
+                      )}
+                    </Select>
+                    {selectedSlackChannel && selectedSlackChannel.is_private && (
+                      <Callout
+                        variant="warning"
+                        className={cn("text-sm", variantClasses.warning.textColor)}
+                      >
+                        To receive alerts in the{" "}
+                        <InlineCode variant="extra-small">{selectedSlackChannel.name}</InlineCode>{" "}
+                        channel, you need to invite the @Trigger.dev Slack Bot. Go to the channel in
+                        Slack and type:{" "}
+                        <InlineCode variant="extra-small">/invite @Trigger.dev</InlineCode>.
+                      </Callout>
+                    )}
+
+                    <FormError id={channelValue.errorId}>{channelValue.error}</FormError>
+                    <input type="hidden" name="integrationId" value={slack.integrationId} />
+                  </>
+                ) : slack.status === "NOT_CONFIGURED" ? (
+                  <LinkButton variant="tertiary/large" to="connect-to-slack" fullWidth>
+                    <span className="flex items-center gap-2 text-text-bright">
+                      <SlackIcon className="size-5" /> Connect to Slack
+                    </span>
+                  </LinkButton>
+                ) : (
+                  <Callout variant="warning">
+                    Slack integration is not available. Please contact your organization
+                    administrator.
+                  </Callout>
+                )}
+              </InputGroup>
+            ) : (
+              <InputGroup fullWidth>
+                <Label>URL</Label>
+                <Input
+                  {...conform.input(channelValue)}
+                  placeholder="https://foobar.com/webhooks"
+                  type="url"
+                  autoFocus
+                />
+                <FormError id={channelValue.errorId}>{channelValue.error}</FormError>
+                <Hint>We'll issue POST requests to this URL with a JSON payload.</Hint>
+              </InputGroup>
+            )}
+
+            <InputGroup>
+              <Label>Alert me when</Label>
+
+              <div className="flex items-center gap-1">
+                <CheckboxWithLabel
+                  name={alertTypes.name}
+                  id="TASK_RUN_ATTEMPT"
+                  value="TASK_RUN_ATTEMPT"
+                  variant="simple/small"
+                  label="Task run attempts fail"
+                  defaultChecked
+                  className="pr-0"
+                />
+                <InfoIconTooltip content="You'll receive an alert every time an attempt fails on a run." />
+              </div>
+
+              <CheckboxWithLabel
+                name={alertTypes.name}
+                id="DEPLOYMENT_FAILURE"
+                value="DEPLOYMENT_FAILURE"
+                variant="simple/small"
+                label="Deployments fail"
+                defaultChecked
+              />
+
+              <CheckboxWithLabel
+                name={alertTypes.name}
+                id="DEPLOYMENT_SUCCESS"
+                value="DEPLOYMENT_SUCCESS"
+                variant="simple/small"
+                label="Deployments succeed"
+                defaultChecked
+              />
+
+              <FormError id={alertTypes.errorId}>{alertTypes.error}</FormError>
+            </InputGroup>
+            <InputGroup>
+              <Label>Environments</Label>
+              <CheckboxWithLabel
+                name={environmentTypes.name}
+                id="PRODUCTION"
+                value="PRODUCTION"
+                variant="simple/small"
+                label="PROD"
+                defaultChecked
+              />
+              <CheckboxWithLabel
+                name={environmentTypes.name}
+                id="STAGING"
+                value="STAGING"
+                variant="simple/small"
+                label="STAGING"
+                defaultChecked
+              />
+
+              <FormError id={environmentTypes.errorId}>{environmentTypes.error}</FormError>
+            </InputGroup>
+            <FormError>{form.error}</FormError>
+            <div className="border-t border-grid-bright pt-3">
+              <FormButtons
+                confirmButton={
+                  <Button
+                    variant="primary/medium"
+                    disabled={isLoading}
+                    name="action"
+                    value="create"
+                  >
+                    {isLoading ? "Saving…" : "Save"}
+                  </Button>
+                }
+                cancelButton={
+                  <LinkButton
+                    to={v3ProjectAlertsPath(organization, project)}
+                    variant="tertiary/medium"
+                  >
+                    Cancel
+                  </LinkButton>
+                }
+              />
+            </div>
+          </Fieldset>
+        </Form>
+      </DialogContent>
+    </Dialog>
+  );
+}
+
+function SlackChannelTitle({ name, is_private }: { name?: string; is_private?: boolean }) {
+  return (
+    <div className="flex items-center gap-1.5">
+      {is_private ? <LockClosedIcon className="size-4" /> : <HashtagIcon className="size-4" />}
+      <span>{name}</span>
+    </div>
+  );
+}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.alerts/route.tsx

@@ -0,0 +1,496 @@
+import { useForm } from "@conform-to/react";
+import { parse } from "@conform-to/zod";
+import {
+  ArrowUpRightIcon,
+  BellAlertIcon,
+  BellSlashIcon,
+  BookOpenIcon,
+  EnvelopeIcon,
+  GlobeAltIcon,
+  LockClosedIcon,
+  PlusIcon,
+  TrashIcon,
+} from "@heroicons/react/20/solid";
+import { Form, Outlet, useActionData, useNavigation } from "@remix-run/react";
+import { ActionFunctionArgs, LoaderFunctionArgs, json } from "@remix-run/server-runtime";
+import { SlackIcon } from "@trigger.dev/companyicons";
+import { ProjectAlertChannelType, ProjectAlertType } from "@trigger.dev/database";
+import assertNever from "assert-never";
+import { typedjson, useTypedLoaderData } from "remix-typedjson";
+import { z } from "zod";
+import { EnvironmentTypeLabel } from "~/components/environments/EnvironmentLabel";
+import { PageBody, PageContainer } from "~/components/layout/AppLayout";
+import { Button, LinkButton } from "~/components/primitives/Buttons";
+import { ClipboardField } from "~/components/primitives/ClipboardField";
+import { DetailCell } from "~/components/primitives/DetailCell";
+import { Header2 } from "~/components/primitives/Headers";
+import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
+import { Paragraph } from "~/components/primitives/Paragraph";
+import {
+  Table,
+  TableBody,
+  TableCell,
+  TableCellMenu,
+  TableHeader,
+  TableHeaderCell,
+  TableRow,
+} from "~/components/primitives/Table";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger,
+} from "~/components/primitives/Tooltip";
+import { EnabledStatus } from "~/components/runs/v3/EnabledStatus";
+import { prisma } from "~/db.server";
+import { useOrganization } from "~/hooks/useOrganizations";
+import { useProject } from "~/hooks/useProject";
+import { redirectWithSuccessMessage } from "~/models/message.server";
+import { findProjectBySlug } from "~/models/project.server";
+import {
+  AlertChannelListPresenter,
+  AlertChannelListPresenterRecord,
+} from "~/presenters/v3/AlertChannelListPresenter.server";
+import { requireUserId } from "~/services/session.server";
+import { cn } from "~/utils/cn";
+import {
+  ProjectParamSchema,
+  docsPath,
+  v3NewProjectAlertPath,
+  v3ProjectAlertsPath,
+} from "~/utils/pathBuilder";
+
+export const loader = async ({ request, params }: LoaderFunctionArgs) => {
+  const userId = await requireUserId(request);
+  const { projectParam, organizationSlug } = ProjectParamSchema.parse(params);
+
+  const project = await findProjectBySlug(organizationSlug, projectParam, userId);
+
+  if (!project) {
+    throw new Response(undefined, {
+      status: 404,
+      statusText: "Project not found",
+    });
+  }
+
+  const presenter = new AlertChannelListPresenter();
+  const data = await presenter.call(project.id);
+
+  return typedjson(data);
+};
+
+const schema = z.discriminatedUnion("action", [
+  z.object({ action: z.literal("delete"), id: z.string() }),
+  z.object({ action: z.literal("disable"), id: z.string() }),
+  z.object({ action: z.literal("enable"), id: z.string() }),
+]);
+
+export const action = async ({ request, params }: ActionFunctionArgs) => {
+  const userId = await requireUserId(request);
+  const { organizationSlug, projectParam } = ProjectParamSchema.parse(params);
+
+  if (request.method.toUpperCase() !== "POST") {
+    return { status: 405, body: "Method Not Allowed" };
+  }
+
+  const formData = await request.formData();
+  const submission = parse(formData, { schema });
+
+  if (!submission.value) {
+    return json(submission);
+  }
+
+  const project = await findProjectBySlug(organizationSlug, projectParam, userId);
+
+  if (!project) {
+    submission.error.key = "Project not found";
+    return json(submission);
+  }
+
+  switch (submission.value.action) {
+    case "delete": {
+      const alertChannel = await prisma.projectAlertChannel.delete({
+        where: { id: submission.value.id, projectId: project.id },
+      });
+
+      return redirectWithSuccessMessage(
+        v3ProjectAlertsPath({ slug: organizationSlug }, { slug: projectParam }),
+        request,
+        `Deleted ${alertChannel.name} alert`
+      );
+    }
+    case "disable": {
+      const alertChannel = await prisma.projectAlertChannel.update({
+        where: { id: submission.value.id, projectId: project.id },
+        data: { enabled: false },
+      });
+
+      return redirectWithSuccessMessage(
+        v3ProjectAlertsPath({ slug: organizationSlug }, { slug: projectParam }),
+        request,
+        `Disabled ${alertChannel.name} alert`
+      );
+    }
+    case "enable": {
+      const alertChannel = await prisma.projectAlertChannel.update({
+        where: { id: submission.value.id, projectId: project.id },
+        data: { enabled: true },
+      });
+
+      return redirectWithSuccessMessage(
+        v3ProjectAlertsPath({ slug: organizationSlug }, { slug: projectParam }),
+        request,
+        `Enabled ${alertChannel.name} alert`
+      );
+    }
+  }
+};
+
+export default function Page() {
+  const { alertChannels } = useTypedLoaderData<typeof loader>();
+  const project = useProject();
+  const organization = useOrganization();
+
+  return (
+    <PageContainer>
+      <NavBar>
+        <PageTitle title="Alerts" />
+        <PageAccessories>
+          <LinkButton
+            LeadingIcon={BookOpenIcon}
+            to={docsPath("v3/troubleshooting-alerts")}
+            variant="minimal/small"
+          >
+            Alerts docs
+          </LinkButton>
+        </PageAccessories>
+      </NavBar>
+      <PageBody>
+        <div className={cn("flex h-full flex-col gap-3")}>
+          {alertChannels.length > 0 && alertChannels.length < 10 && (
+            <div className="flex items-end justify-between">
+              <Header2 className="">Project alerts</Header2>
+              <LinkButton
+                to={v3NewProjectAlertPath(organization, project)}
+                variant="primary/small"
+                LeadingIcon={PlusIcon}
+                shortcut={{ key: "n" }}
+              >
+                New alert
+              </LinkButton>
+            </div>
+          )}
+          <Table>
+            <TableHeader>
+              <TableRow>
+                <TableHeaderCell>Name</TableHeaderCell>
+                <TableHeaderCell>Alert types</TableHeaderCell>
+                <TableHeaderCell>Environments</TableHeaderCell>
+                <TableHeaderCell>Channel</TableHeaderCell>
+                <TableHeaderCell>Enabled</TableHeaderCell>
+                <TableHeaderCell hiddenLabel>Actions</TableHeaderCell>
+              </TableRow>
+            </TableHeader>
+            <TableBody>
+              {alertChannels.length > 0 ? (
+                alertChannels.map((alertChannel) => (
+                  <TableRow key={alertChannel.id}>
+                    <TableCell className={alertChannel.enabled ? "" : "opacity-50"}>
+                      {alertChannel.name}
+                    </TableCell>
+                    <TableCell className={alertChannel.enabled ? "" : "opacity-50"}>
+                      {alertChannel.alertTypes.map((type) => alertTypeTitle(type)).join(", ")}
+                    </TableCell>
+                    <TableCell
+                      className={cn("space-x-2", alertChannel.enabled ? "" : "opacity-50")}
+                    >
+                      {alertChannel.environmentTypes.map((environmentType) => (
+                        <EnvironmentTypeLabel
+                          key={environmentType}
+                          environment={{ type: environmentType }}
+                        />
+                      ))}
+                    </TableCell>
+                    <TableCell className={alertChannel.enabled ? "" : "opacity-50"}>
+                      <AlertChannelDetails alertChannel={alertChannel} />
+                    </TableCell>
+                    <TableCell className={alertChannel.enabled ? "" : "opacity-50"}>
+                      <EnabledStatus
+                        enabled={alertChannel.enabled}
+                        enabledIcon={BellAlertIcon}
+                        disabledIcon={BellSlashIcon}
+                      />
+                    </TableCell>
+                    <TableCellMenu isSticky>
+                      {alertChannel.enabled ? (
+                        <DisableAlertChannelButton id={alertChannel.id} />
+                      ) : (
+                        <EnableAlertChannelButton id={alertChannel.id} />
+                      )}
+
+                      <DeleteAlertChannelButton id={alertChannel.id} />
+                    </TableCellMenu>
+                  </TableRow>
+                ))
+              ) : (
+                <TableRow>
+                  <TableCell colSpan={5}>
+                    <div className="flex flex-col items-center justify-center py-6">
+                      <Header2 spacing className="text-text-bright">
+                        You haven't created any project alerts yet
+                      </Header2>
+                      <Paragraph variant="small" className="mb-4">
+                        Get alerted when runs or deployments fail, or when deployments succeed in
+                        both Prod and Staging environments.
+                      </Paragraph>
+                      <LinkButton
+                        to={v3NewProjectAlertPath(organization, project)}
+                        variant="primary/medium"
+                        LeadingIcon={PlusIcon}
+                        shortcut={{ key: "n" }}
+                      >
+                        New alert
+                      </LinkButton>
+                    </div>
+                  </TableCell>
+                </TableRow>
+              )}
+            </TableBody>
+          </Table>
+          <div className="mt-4">
+            <Header2 className="mb-1">Platform alerts</Header2>
+            <Paragraph variant="small" className="mb-4">
+              Subscribe to get email notifications when Trigger.dev creates, updates or resolves a
+              platform incident.
+            </Paragraph>
+            <LinkButton
+              variant="tertiary/medium"
+              TrailingIcon={ArrowUpRightIcon}
+              to="https://status.trigger.dev/"
+              target="_blank"
+              className="inline-flex"
+            >
+              Subscribe
+            </LinkButton>
+          </div>
+        </div>
+        <Outlet />
+      </PageBody>
+    </PageContainer>
+  );
+}
+
+function DeleteAlertChannelButton(props: { id: string }) {
+  const lastSubmission = useActionData();
+  const navigation = useNavigation();
+
+  const isLoading =
+    navigation.state !== "idle" &&
+    navigation.formMethod === "post" &&
+    navigation.formData?.get("action") === "delete";
+
+  const [form, { id }] = useForm({
+    id: "delete-alert-channel",
+    // TODO: type this
+    lastSubmission: lastSubmission as any,
+    onValidate({ formData }) {
+      return parse(formData, { schema });
+    },
+    shouldRevalidate: "onSubmit",
+  });
+
+  return (
+    <Form method="post" {...form.props}>
+      <input type="hidden" name="id" value={props.id} />
+      <Button
+        name="action"
+        value="delete"
+        type="submit"
+        variant="small-menu-item"
+        LeadingIcon={TrashIcon}
+        leadingIconClassName="text-rose-500"
+        className="text-xs"
+      >
+        {isLoading ? "Deleting" : "Delete"}
+      </Button>
+    </Form>
+  );
+}
+
+function DisableAlertChannelButton(props: { id: string }) {
+  const lastSubmission = useActionData();
+  const navigation = useNavigation();
+
+  const isLoading =
+    navigation.state !== "idle" &&
+    navigation.formMethod === "post" &&
+    navigation.formData?.get("action") === "delete";
+
+  const [form, { id }] = useForm({
+    id: "disable-alert-channel",
+    // TODO: type this
+    lastSubmission: lastSubmission as any,
+    onValidate({ formData }) {
+      return parse(formData, { schema });
+    },
+    shouldRevalidate: "onSubmit",
+  });
+
+  return (
+    <Form method="post" {...form.props}>
+      <input type="hidden" name="id" value={props.id} />
+
+      <Button
+        name="action"
+        value="disable"
+        type="submit"
+        variant="small-menu-item"
+        LeadingIcon={BellSlashIcon}
+        leadingIconClassName="text-dimmed"
+        className="text-xs"
+      >
+        {isLoading ? "Disabling" : "Disable"}
+      </Button>
+    </Form>
+  );
+}
+
+function EnableAlertChannelButton(props: { id: string }) {
+  const lastSubmission = useActionData();
+  const navigation = useNavigation();
+
+  const isLoading =
+    navigation.state !== "idle" &&
+    navigation.formMethod === "post" &&
+    navigation.formData?.get("action") === "delete";
+
+  const [form, { id }] = useForm({
+    id: "enable-alert-channel",
+    // TODO: type this
+    lastSubmission: lastSubmission as any,
+    onValidate({ formData }) {
+      return parse(formData, { schema });
+    },
+    shouldRevalidate: "onSubmit",
+  });
+
+  return (
+    <Form method="post" {...form.props}>
+      <input type="hidden" name="id" value={props.id} />
+
+      <Button
+        name="action"
+        value="enable"
+        type="submit"
+        variant="small-menu-item"
+        LeadingIcon={BellAlertIcon}
+        leadingIconClassName="text-success"
+        className="text-xs"
+      >
+        {isLoading ? "Enabling" : "Enable"}
+      </Button>
+    </Form>
+  );
+}
+
+function AlertChannelDetails({ alertChannel }: { alertChannel: AlertChannelListPresenterRecord }) {
+  switch (alertChannel.properties?.type) {
+    case "EMAIL": {
+      return (
+        <DetailCell
+          leadingIcon={
+            <AlertChannelTypeIcon
+              channelType={alertChannel.type}
+              className="size-5 text-charcoal-400"
+            />
+          }
+          leadingIconClassName="text-charcoal-400"
+          label={"Email"}
+          description={alertChannel.properties.email}
+        />
+      );
+    }
+    case "WEBHOOK": {
+      return (
+        <DetailCell
+          leadingIcon={
+            <TooltipProvider>
+              <Tooltip>
+                <TooltipTrigger>
+                  <AlertChannelTypeIcon
+                    channelType={alertChannel.type}
+                    className="size-5 text-charcoal-400"
+                  />
+                </TooltipTrigger>
+                <TooltipContent className="flex items-center gap-1">Webhook</TooltipContent>
+              </Tooltip>
+            </TooltipProvider>
+          }
+          leadingIconClassName="text-charcoal-400"
+          label={alertChannel.properties.url}
+          description={
+            <ClipboardField
+              value={alertChannel.properties.secret}
+              variant="secondary/small"
+              icon={<LockClosedIcon className="size-4" />}
+              iconButton
+              secure={"•".repeat(alertChannel.properties.secret.length)}
+              className="mt-1 w-80"
+            />
+          }
+        />
+      );
+    }
+    case "SLACK": {
+      return (
+        <DetailCell
+          leadingIcon={
+            <AlertChannelTypeIcon
+              channelType={alertChannel.type}
+              className="size-5 text-charcoal-400"
+            />
+          }
+          leadingIconClassName="text-charcoal-400"
+          label={"Slack"}
+          description={`#${alertChannel.properties.channelName}`}
+        />
+      );
+    }
+  }
+
+  return null;
+}
+
+export function alertTypeTitle(alertType: ProjectAlertType): string {
+  switch (alertType) {
+    case "TASK_RUN_ATTEMPT":
+      return "Task attempt failure";
+    case "DEPLOYMENT_FAILURE":
+      return "Deployment failure";
+    case "DEPLOYMENT_SUCCESS":
+      return "Deployment success";
+    default: {
+      assertNever(alertType);
+    }
+  }
+}
+
+export function AlertChannelTypeIcon({
+  channelType,
+  className,
+}: {
+  channelType: ProjectAlertChannelType;
+  className: string;
+}) {
+  switch (channelType) {
+    case "EMAIL":
+      return <EnvelopeIcon className={className} />;
+    case "SLACK":
+      return <SlackIcon className={className} />;
+    case "WEBHOOK":
+      return <GlobeAltIcon className={className} />;
+    default: {
+      assertNever(channelType);
+    }
+  }
+}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.environment-variables.new/route.tsx

@@ -18,7 +18,7 @@ import {
   environmentTitle,
 } from "~/components/environments/EnvironmentLabel";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
-import { Checkbox } from "~/components/primitives/Checkbox";
+import { CheckboxWithLabel } from "~/components/primitives/Checkbox";
 import { Dialog, DialogContent, DialogHeader } from "~/components/primitives/Dialog";
 import { Fieldset } from "~/components/primitives/Fieldset";
 import { FormButtons } from "~/components/primitives/FormButtons";
@@ -46,13 +46,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
 
   try {
     const presenter = new EnvironmentVariablesPresenter();
-    const { environmentVariables, environments } = await presenter.call({
+    const { environments } = await presenter.call({
       userId,
       projectSlug: projectParam,
     });
 
     return typedjson({
-      environmentVariables,
       environments,
     });
   } catch (error) {
@@ -150,7 +149,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
 
 export default function Page() {
   const [isOpen, setIsOpen] = useState(false);
-  const { environmentVariables, environments } = useTypedLoaderData<typeof loader>();
+  const { environments } = useTypedLoaderData<typeof loader>();
   const lastSubmission = useActionData();
   const navigation = useNavigation();
   const navigate = useNavigate();
@@ -196,7 +195,7 @@ export default function Page() {
               <Label>Environments</Label>
               <div className="flex flex-wrap items-center gap-2">
                 {environments.map((environment) => (
-                  <Checkbox
+                  <CheckboxWithLabel
                     key={environment.id}
                     id={environment.id}
                     value={environment.id}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs._index/route.tsx

@@ -1,16 +1,33 @@
+import { ArrowPathIcon, StopCircleIcon } from "@heroicons/react/20/solid";
 import { BeakerIcon, BookOpenIcon } from "@heroicons/react/24/solid";
-import { useNavigation } from "@remix-run/react";
+import { Form, useNavigation } from "@remix-run/react";
 import { LoaderFunctionArgs } from "@remix-run/server-runtime";
-import { TypedAwait, typeddefer, typedjson, useTypedLoaderData } from "remix-typedjson";
+import { AnimatePresence, motion } from "framer-motion";
+import { Suspense, useState } from "react";
+import { TypedAwait, typeddefer, useTypedLoaderData } from "remix-typedjson";
 import { TaskIcon } from "~/assets/icons/TaskIcon";
 import { BlankstateInstructions } from "~/components/BlankstateInstructions";
 import { StepContentContainer } from "~/components/StepContentContainer";
 import { MainCenteredContainer, PageBody } from "~/components/layout/AppLayout";
-import { LinkButton } from "~/components/primitives/Buttons";
+import { Button, LinkButton } from "~/components/primitives/Buttons";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogFooter,
+  DialogHeader,
+  DialogTrigger,
+} from "~/components/primitives/Dialog";
 import { Header1 } from "~/components/primitives/Headers";
 import { NavBar, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
+import {
+  SelectedItemsProvider,
+  useSelectedItems,
+} from "~/components/primitives/SelectedItemsProvider";
+import { Spinner } from "~/components/primitives/Spinner";
 import { StepNumber } from "~/components/primitives/StepNumber";
+import { TextLink } from "~/components/primitives/TextLink";
 import { RunsFilters, TaskRunListSearchFilters } from "~/components/runs/v3/RunFilters";
 import { TaskRunsTable } from "~/components/runs/v3/TaskRunsTable";
 import { useOrganization } from "~/hooks/useOrganizations";
@@ -19,11 +36,9 @@ import { useUser } from "~/hooks/useUser";
 import { RunListPresenter } from "~/presenters/v3/RunListPresenter.server";
 import { requireUserId } from "~/services/session.server";
 import { cn } from "~/utils/cn";
-import { ProjectParamSchema, v3ProjectPath, v3TestPath } from "~/utils/pathBuilder";
+import { ProjectParamSchema, v3ProjectPath, v3RunsPath, v3TestPath } from "~/utils/pathBuilder";
 import { ListPagination } from "../../components/ListPagination";
-import { TextLink } from "~/components/primitives/TextLink";
-import { Spinner } from "~/components/primitives/Spinner";
-import { Suspense } from "react";
+import { BULK_ACTION_RUN_LIMIT } from "~/consts";
 
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const userId = await requireUserId(request);
@@ -37,8 +52,9 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
     environments: url.searchParams.getAll("environments"),
     tasks: url.searchParams.getAll("tasks"),
     period: url.searchParams.get("period") ?? undefined,
+    bulkId: url.searchParams.get("bulkId") ?? undefined,
   };
-  const { tasks, versions, statuses, environments, period, from, to, cursor, direction } =
+  const { tasks, versions, statuses, environments, period, bulkId, from, to, cursor, direction } =
     TaskRunListSearchFilters.parse(s);
 
   const presenter = new RunListPresenter();
@@ -50,6 +66,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
     statuses,
     environments,
     period,
+    bulkId,
     from,
     to,
     direction: direction,
@@ -73,60 +90,246 @@ export default function Page() {
       <NavBar>
         <PageTitle title="Runs" />
       </NavBar>
-      <PageBody>
-        <Suspense
-          fallback={
-            <div className="flex items-center justify-center py-2">
-              <div className="mx-auto flex items-center gap-2">
-                <Spinner />
-                <Paragraph variant="small">Loading runs</Paragraph>
-              </div>
-            </div>
-          }
+      <PageBody scrollable={false}>
+        <SelectedItemsProvider
+          initialSelectedItems={[]}
+          maxSelectedItemCount={BULK_ACTION_RUN_LIMIT}
         >
-          <TypedAwait resolve={data}>
-            {(list) => (
-              <>
-                {list.runs.length === 0 && !list.hasFilters ? (
-                  list.possibleTasks.length === 0 ? (
-                    <CreateFirstTaskInstructions />
-                  ) : (
-                    <RunTaskInstructions />
-                  )
-                ) : (
-                  <div className={cn("grid h-fit grid-cols-1 gap-4")}>
-                    <div>
-                      <div className="mb-2 flex items-start justify-between gap-x-2">
-                        <RunsFilters
-                          possibleEnvironments={project.environments}
-                          possibleTasks={list.possibleTasks}
-                          hasFilters={list.hasFilters}
-                        />
-                        <div className="flex items-center justify-end gap-x-2">
-                          <ListPagination list={list} />
-                        </div>
+          {({ selectedItems }) => (
+            <div
+              className={cn(
+                "grid h-full max-h-full overflow-hidden",
+                selectedItems.size === 0 ? "grid-rows-1" : "grid-rows-[1fr_2.5rem]"
+              )}
+            >
+              <div className="overflow-y-auto p-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600">
+                <Suspense
+                  fallback={
+                    <div className="flex items-center justify-center py-2">
+                      <div className="mx-auto flex items-center gap-2">
+                        <Spinner />
+                        <Paragraph variant="small">Loading runs</Paragraph>
                       </div>
-
-                      <TaskRunsTable
-                        total={list.runs.length}
-                        hasFilters={list.hasFilters}
-                        filters={list.filters}
-                        runs={list.runs}
-                        isLoading={isLoading}
-                      />
-                      <ListPagination list={list} className="mt-2 justify-end" />
                     </div>
-                  </div>
-                )}
-              </>
-            )}
-          </TypedAwait>
-        </Suspense>
+                  }
+                >
+                  <TypedAwait resolve={data}>
+                    {(list) => (
+                      <>
+                        {list.runs.length === 0 && !list.hasFilters ? (
+                          list.possibleTasks.length === 0 ? (
+                            <CreateFirstTaskInstructions />
+                          ) : (
+                            <RunTaskInstructions />
+                          )
+                        ) : (
+                          <div className={cn("grid h-fit grid-cols-1 gap-4")}>
+                            <div>
+                              <div className="mb-2 flex items-start justify-between gap-x-2">
+                                <RunsFilters
+                                  possibleEnvironments={project.environments}
+                                  possibleTasks={list.possibleTasks}
+                                  bulkActions={list.bulkActions}
+                                  hasFilters={list.hasFilters}
+                                />
+                                <div className="flex items-center justify-end gap-x-2">
+                                  <ListPagination list={list} />
+                                </div>
+                              </div>
+
+                              <TaskRunsTable
+                                total={list.runs.length}
+                                hasFilters={list.hasFilters}
+                                filters={list.filters}
+                                runs={list.runs}
+                                isLoading={isLoading}
+                                allowSelection
+                              />
+                              <ListPagination list={list} className="mt-2 justify-end" />
+                            </div>
+                          </div>
+                        )}
+                      </>
+                    )}
+                  </TypedAwait>
+                </Suspense>
+              </div>
+              <BulkActionBar />
+            </div>
+          )}
+        </SelectedItemsProvider>
       </PageBody>
     </>
   );
 }
 
+function BulkActionBar() {
+  const { selectedItems, deselectAll } = useSelectedItems();
+  const [barState, setBarState] = useState<"none" | "replay" | "cancel">("none");
+
+  const hasSelectedMaximum = selectedItems.size >= BULK_ACTION_RUN_LIMIT;
+
+  return (
+    <AnimatePresence>
+      {selectedItems.size > 0 && (
+        <motion.div
+          initial={{ translateY: "100%" }}
+          animate={{ translateY: 0 }}
+          exit={{ translateY: "100%" }}
+          className="flex items-center justify-between gap-2 border-t border-grid-bright bg-background-bright px-3"
+        >
+          <div className="flex items-center gap-1.5 text-sm text-text-bright">
+            <span className="font-medium">Bulk actions:</span>
+            {hasSelectedMaximum ? (
+              <span className="text-warning">Maximum of {selectedItems.size} runs selected</span>
+            ) : (
+              <span>{selectedItems.size} runs selected</span>
+            )}
+          </div>
+          <div className="flex items-center gap-1 divide-x divide-charcoal-700">
+            <CancelRuns
+              onOpen={(o) => {
+                if (o) {
+                  setBarState("cancel");
+                } else {
+                  setBarState("none");
+                }
+              }}
+            />
+            <ReplayRuns
+              onOpen={(o) => {
+                if (o) {
+                  setBarState("replay");
+                } else {
+                  setBarState("none");
+                }
+              }}
+            />
+            <Button
+              variant="minimal/medium"
+              shortcut={{ key: "esc", enabledOnInputElements: true }}
+              onClick={() => {
+                if (barState !== "none") return;
+                deselectAll();
+              }}
+            >
+              Clear selection
+            </Button>
+          </div>
+        </motion.div>
+      )}
+    </AnimatePresence>
+  );
+}
+
+function CancelRuns({ onOpen }: { onOpen: (open: boolean) => void }) {
+  const { selectedItems } = useSelectedItems();
+
+  const organization = useOrganization();
+  const project = useProject();
+  const failedRedirect = v3RunsPath(organization, project);
+
+  const formAction = `/resources/taskruns/bulk/cancel`;
+
+  const navigation = useNavigation();
+  const isLoading = navigation.formAction === formAction;
+
+  return (
+    <Dialog onOpenChange={(o) => onOpen(o)}>
+      <DialogTrigger asChild>
+        <Button
+          type="button"
+          variant="minimal/medium"
+          shortcut={{ key: "c", enabledOnInputElements: true }}
+          LeadingIcon={StopCircleIcon}
+        >
+          Cancel runs
+        </Button>
+      </DialogTrigger>
+      <DialogContent key="replay">
+        <DialogHeader>Cancel {selectedItems.size} runs?</DialogHeader>
+        <DialogDescription>
+          Canceling these runs will stop them from running. Only runs that are not already finished
+          will be canceled, the others will remain in their existing state.
+        </DialogDescription>
+        <DialogFooter>
+          <Form action={formAction} method="post" reloadDocument>
+            <input type="hidden" name="failedRedirect" value={failedRedirect} />
+            <input type="hidden" name="organizationSlug" value={organization.slug} />
+            <input type="hidden" name="projectSlug" value={project.slug} />
+            {[...selectedItems].map((runId) => (
+              <input key={runId} type="hidden" name="runIds" value={runId} />
+            ))}
+            <Button
+              type="submit"
+              variant="danger/small"
+              LeadingIcon={isLoading ? "spinner-white" : StopCircleIcon}
+              disabled={isLoading}
+              shortcut={{ modifiers: ["meta"], key: "enter" }}
+            >
+              {isLoading ? "Canceling..." : `Cancel ${selectedItems.size} runs`}
+            </Button>
+          </Form>
+        </DialogFooter>
+      </DialogContent>
+    </Dialog>
+  );
+}
+
+function ReplayRuns({ onOpen }: { onOpen: (open: boolean) => void }) {
+  const { selectedItems } = useSelectedItems();
+
+  const organization = useOrganization();
+  const project = useProject();
+  const failedRedirect = v3RunsPath(organization, project);
+
+  const formAction = `/resources/taskruns/bulk/replay`;
+
+  const navigation = useNavigation();
+  const isLoading = navigation.formAction === formAction;
+
+  return (
+    <Dialog onOpenChange={(o) => onOpen(o)}>
+      <DialogTrigger asChild>
+        <Button
+          type="button"
+          variant="minimal/medium"
+          shortcut={{ key: "r", enabledOnInputElements: true }}
+          LeadingIcon={ArrowPathIcon}
+        >
+          Replay {selectedItems.size} runs
+        </Button>
+      </DialogTrigger>
+      <DialogContent key="replay">
+        <DialogHeader>Replay runs?</DialogHeader>
+        <DialogDescription>
+          Replaying these runs will create a new run for each with the same payload and environment
+          as the original. It will use the latest version of the code for each task.
+        </DialogDescription>
+        <DialogFooter>
+          <Form action={formAction} method="post" reloadDocument>
+            <input type="hidden" name="failedRedirect" value={failedRedirect} />
+            <input type="hidden" name="organizationSlug" value={organization.slug} />
+            <input type="hidden" name="projectSlug" value={project.slug} />
+            {[...selectedItems].map((runId) => (
+              <input key={runId} type="hidden" name="runIds" value={runId} />
+            ))}
+            <Button
+              type="submit"
+              variant="primary/small"
+              LeadingIcon={isLoading ? "spinner-white" : ArrowPathIcon}
+              disabled={isLoading}
+              shortcut={{ modifiers: ["meta"], key: "enter" }}
+            >
+              {isLoading ? "Replaying..." : `Replay ${selectedItems.size} runs`}
+            </Button>
+          </Form>
+        </DialogFooter>
+      </DialogContent>
+    </Dialog>
+  );
+}
+
 function CreateFirstTaskInstructions() {
   const organization = useOrganization();
   const project = useProject();

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.schedules.$scheduleParam/route.tsx

@@ -8,7 +8,7 @@ import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { z } from "zod";
 import { ExitIcon } from "~/assets/icons/ExitIcon";
 import { InlineCode } from "~/components/code/InlineCode";
-import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
+import { EnvironmentLabel, EnvironmentLabels } from "~/components/environments/EnvironmentLabel";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
 import { DateTime } from "~/components/primitives/DateTime";
 import {
@@ -211,16 +211,7 @@ export default function Page() {
                 </div>
               </Property>
               <Property label="Environments">
-                <div className="flex flex-wrap gap-1">
-                  {schedule.environments.map((env) => (
-                    <EnvironmentLabel
-                      key={env.id}
-                      size="small"
-                      environment={env}
-                      userName={env.userName}
-                    />
-                  ))}
-                </div>
+                <EnvironmentLabels size="small" environments={schedule.environments} />
               </Property>
               <Property label="External ID">
                 {schedule.externalId ? schedule.externalId : "–"}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.schedules/route.tsx

@@ -6,7 +6,7 @@ import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { BlankstateInstructions } from "~/components/BlankstateInstructions";
 import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
 import { InlineCode } from "~/components/code/InlineCode";
-import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
+import { EnvironmentLabel, EnvironmentLabels } from "~/components/environments/EnvironmentLabel";
 import { MainCenteredContainer, PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { LinkButton } from "~/components/primitives/Buttons";
 import { DateTime } from "~/components/primitives/DateTime";
@@ -281,15 +281,7 @@ function SchedulesTable({
                   {schedule.lastRun ? <DateTime date={schedule.lastRun} timeZone="utc" /> : "–"}
                 </TableCell>
                 <TableCell to={path} className={cellClass}>
-                  <div className="flex gap-1">
-                    {schedule.environments.map((environment) => (
-                      <EnvironmentLabel
-                        key={environment.id}
-                        environment={environment}
-                        userName={environment.userName}
-                      />
-                    ))}
-                  </div>
+                  <EnvironmentLabels environments={schedule.environments} size="small" />
                 </TableCell>
                 <TableCell to={path}>
                   <EnabledStatus enabled={schedule.active} />

apps/webapp/app/routes/account._index/route.tsx

@@ -6,7 +6,7 @@ import { z } from "zod";
 import { UserProfilePhoto } from "~/components/UserProfilePhoto";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { Button } from "~/components/primitives/Buttons";
-import { Checkbox } from "~/components/primitives/Checkbox";
+import { CheckboxWithLabel } from "~/components/primitives/Checkbox";
 import { Fieldset } from "~/components/primitives/Fieldset";
 import { FormButtons } from "~/components/primitives/FormButtons";
 import { FormError } from "~/components/primitives/FormError";
@@ -158,7 +158,7 @@ export default function Page() {
             </InputGroup>
             <InputGroup>
               <Label>Notifications</Label>
-              <Checkbox
+              <CheckboxWithLabel
                 id="marketingEmails"
                 {...conform.input(marketingEmails, { type: "checkbox" })}
                 label="Receive onboarding emails"

apps/webapp/app/routes/api.v1.endpoints.$environmentId.$endpointSlug.index.$indexHookIdentifier.ts

@@ -106,7 +106,7 @@ export class TriggerEndpointIndexHookService {
         },
       });
 
-      if (!endpoint) {
+      if (!endpoint || !endpoint.url) {
         throw new Error("Endpoint not found");
       }
 

apps/webapp/app/routes/api.v1.projects.$projectRef.alertChannels.ts

@@ -0,0 +1,96 @@
+import type { ActionFunctionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import { z } from "zod";
+import {
+  ApiAlertChannelPresenter,
+  ApiCreateAlertChannel,
+} from "~/presenters/v3/ApiAlertChannelPresenter.server";
+import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+import { CreateAlertChannelService } from "~/v3/services/alerts/createAlertChannel.server";
+import { ServiceValidationError } from "~/v3/services/baseService.server";
+
+const ParamsSchema = z.object({
+  projectRef: z.string(),
+});
+
+export async function action({ request, params }: ActionFunctionArgs) {
+  const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing Access Token" }, { status: 401 });
+  }
+
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return json({ error: "Invalid Params" }, { status: 400 });
+  }
+
+  const { projectRef } = parsedParams.data;
+
+  const rawBody = await request.json();
+
+  const body = ApiCreateAlertChannel.safeParse(rawBody);
+
+  if (!body.success) {
+    return json({ error: "Invalid request body", issues: body.error.issues }, { status: 400 });
+  }
+
+  const service = new CreateAlertChannelService();
+
+  try {
+    if (body.data.channel === "email") {
+      if (!body.data.channelData.email) {
+        return json({ error: "Email is required" }, { status: 422 });
+      }
+
+      const alertChannel = await service.call(projectRef, authenticationResult.userId, {
+        name: body.data.name,
+        alertTypes: body.data.alertTypes.map((type) =>
+          ApiAlertChannelPresenter.alertTypeFromApi(type)
+        ),
+        channel: {
+          type: "EMAIL",
+          email: body.data.channelData.email,
+        },
+        deduplicationKey: body.data.deduplicationKey,
+        environmentTypes: body.data.environmentTypes,
+      });
+
+      return json(await ApiAlertChannelPresenter.alertChannelToApi(alertChannel));
+    }
+
+    if (body.data.channel === "webhook") {
+      if (!body.data.channelData.url) {
+        return json({ error: "webhook url is required" }, { status: 422 });
+      }
+
+      const alertChannel = await service.call(projectRef, authenticationResult.userId, {
+        name: body.data.name,
+        alertTypes: body.data.alertTypes.map((type) =>
+          ApiAlertChannelPresenter.alertTypeFromApi(type)
+        ),
+        channel: {
+          type: "WEBHOOK",
+          url: body.data.channelData.url,
+          secret: body.data.channelData.secret,
+        },
+        deduplicationKey: body.data.deduplicationKey,
+        environmentTypes: body.data.environmentTypes,
+      });
+
+      return json(await ApiAlertChannelPresenter.alertChannelToApi(alertChannel));
+    }
+
+    return json({ error: "Invalid channel type" }, { status: 422 });
+  } catch (error) {
+    if (error instanceof ServiceValidationError) {
+      return json({ error: error.message }, { status: 422 });
+    }
+
+    return json(
+      { error: error instanceof Error ? error.message : "Internal Server Error" },
+      { status: 500 }
+    );
+  }
+}

apps/webapp/app/routes/api.v1.tasks.$taskId.trigger.ts

@@ -11,9 +11,10 @@ const ParamsSchema = z.object({
 });
 
 export const HeadersSchema = z.object({
-  "idempotency-key": z.string().optional().nullable(),
-  "trigger-version": z.string().optional().nullable(),
-  "x-trigger-span-parent-as-link": z.coerce.number().optional().nullable(),
+  "idempotency-key": z.string().nullish(),
+  "trigger-version": z.string().nullish(),
+  "x-trigger-span-parent-as-link": z.coerce.number().nullish(),
+  "x-trigger-worker": z.string().nullish(),
   traceparent: z.string().optional(),
   tracestate: z.string().optional(),
 });
@@ -45,6 +46,7 @@ export async function action({ request, params }: ActionFunctionArgs) {
     "x-trigger-span-parent-as-link": spanParentAsLink,
     traceparent,
     tracestate,
+    "x-trigger-worker": isFromWorker,
   } = headers.data;
 
   const { taskId } = ParamsSchema.parse(params);
@@ -58,20 +60,31 @@ export async function action({ request, params }: ActionFunctionArgs) {
     return json({ error: "Invalid request body" }, { status: 400 });
   }
 
-  logger.debug("Triggering task", {
-    taskId,
-    idempotencyKey,
-    triggerVersion,
-    body: body.data,
-  });
-
   const service = new TriggerTaskService();
 
   try {
+    const traceContext = traceparent
+      ? !triggerVersion // If the trigger version is NOT set, we are in an older version of the SDK
+        ? { traceparent, tracestate }
+        : isFromWorker // If the trigger version is set, and the request is from a worker, we should pass the trace context
+        ? { traceparent, tracestate }
+        : undefined
+      : undefined;
+
+    logger.debug("Triggering task", {
+      taskId,
+      idempotencyKey,
+      triggerVersion,
+      headers: Object.fromEntries(request.headers),
+      body: body.data,
+      isFromWorker,
+      traceContext,
+    });
+
     const run = await service.call(taskId, authenticationResult.environment, body.data, {
       idempotencyKey: idempotencyKey ?? undefined,
       triggerVersion: triggerVersion ?? undefined,
-      traceContext: traceparent ? { traceparent, tracestate } : undefined,
+      traceContext,
       spanParentAsLink: spanParentAsLink === 1,
     });
 

apps/webapp/app/routes/integrations.$serviceName.callback.ts

@@ -0,0 +1,66 @@
+import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import z from "zod";
+import { redirectBackWithErrorMessage } from "~/models/message.server";
+import { OrgIntegrationRepository } from "~/models/orgIntegration.server";
+import { requireUserId } from "~/services/session.server";
+import { requestUrl } from "~/utils/requestUrl.server";
+import { CreateOrgIntegrationService } from "~/v3/services/createOrgIntegration.server";
+
+const URLSearchSchema = z
+  .object({
+    code: z.string().optional(),
+    state: z.string().optional(),
+    error: z.string().optional(),
+  })
+  .passthrough();
+
+const ParamsSchema = z.object({
+  serviceName: z.string(),
+});
+
+export async function loader({ request, params }: LoaderFunctionArgs) {
+  if (request.method.toUpperCase() !== "GET") {
+    return { status: 405, body: "Method Not Allowed" };
+  }
+
+  const userId = await requireUserId(request);
+
+  const url = requestUrl(request);
+
+  const parsedSearchParams = URLSearchSchema.safeParse(Object.fromEntries(url.searchParams));
+
+  if (!parsedSearchParams.success) {
+    // TODO: this needs to lookup the redirect url in the cookies
+    throw new Response("Invalid params", { status: 400 });
+  }
+
+  if (parsedSearchParams.data.error) {
+    // TODO: this needs to lookup the redirect url in the cookies
+    throw new Response(parsedSearchParams.data.error, { status: 400 });
+  }
+
+  if (!parsedSearchParams.data.code || !parsedSearchParams.data.state) {
+    throw new Response("Invalid params", { status: 400 });
+  }
+
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    throw new Response("Invalid params", { status: 400 });
+  }
+
+  const service = new CreateOrgIntegrationService();
+
+  const integration = await service.call(
+    userId,
+    parsedSearchParams.data.state,
+    parsedParams.data.serviceName,
+    parsedSearchParams.data.code
+  );
+
+  if (integration) {
+    return await OrgIntegrationRepository.redirectAfterAuth(request);
+  }
+
+  return redirectBackWithErrorMessage(request, "Failed to connect to the service");
+}

apps/webapp/app/routes/internal.webhooks.slack.interactivity.ts

@@ -0,0 +1,5 @@
+import { ActionFunctionArgs } from "@remix-run/server-runtime";
+
+export function action({ request }: ActionFunctionArgs) {
+  return new Response(null, { status: 200 });
+}

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.schedules.new/route.tsx

@@ -11,7 +11,7 @@ import {
   environmentTitle,
 } from "~/components/environments/EnvironmentLabel";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
-import { Checkbox } from "~/components/primitives/Checkbox";
+import { CheckboxWithLabel } from "~/components/primitives/Checkbox";
 import { DateTime } from "~/components/primitives/DateTime";
 import { Fieldset } from "~/components/primitives/Fieldset";
 import { FormError } from "~/components/primitives/FormError";
@@ -265,7 +265,7 @@ export function UpsertScheduleForm({
               <Label>Environments</Label>
               <div className="flex flex-wrap items-center gap-2">
                 {possibleEnvironments.map((environment) => (
-                  <Checkbox
+                  <CheckboxWithLabel
                     key={environment.id}
                     id={environment.id}
                     value={environment.id}

apps/webapp/app/routes/resources.taskruns.bulk.cancel.ts

@@ -0,0 +1,93 @@
+import { parse } from "@conform-to/zod";
+import { ActionFunctionArgs } from "@remix-run/router";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
+import { logger } from "~/services/logger.server";
+import { requireUserId } from "~/services/session.server";
+import { v3RunsPath } from "~/utils/pathBuilder";
+import { CreateBulkActionService } from "~/v3/services/bulk/createBulkAction.server";
+
+const FormSchema = z.object({
+  organizationSlug: z.string(),
+  projectSlug: z.string(),
+  failedRedirect: z.string(),
+  runIds: z.array(z.string()).or(z.string()),
+});
+
+export async function action({ request }: ActionFunctionArgs) {
+  const userId = await requireUserId(request);
+
+  if (request.method.toLowerCase() !== "post") {
+    return redirectWithErrorMessage("/", request, "Invalid method");
+  }
+
+  const formData = await request.formData();
+  const submission = parse(formData, { schema: FormSchema });
+
+  if (!submission.value) {
+    logger.error("Failed to parse resources/taskruns/bulk/cancel form data", { submission });
+    return redirectWithErrorMessage("/", request, "Failed to parse form data");
+  }
+
+  try {
+    const project = await prisma.project.findUnique({
+      where: {
+        slug: submission.value.projectSlug,
+        organization: {
+          members: {
+            some: {
+              userId,
+            },
+          },
+        },
+      },
+    });
+
+    if (!project) {
+      return redirectWithErrorMessage(
+        submission.value.failedRedirect,
+        request,
+        "Project not found"
+      );
+    }
+
+    const service = new CreateBulkActionService();
+    const result = await service.call({
+      projectId: project.id,
+      action: "CANCEL",
+      runIds:
+        typeof submission.value.runIds === "string"
+          ? [submission.value.runIds]
+          : submission.value.runIds,
+    });
+
+    const path = v3RunsPath(
+      { slug: submission.value.organizationSlug },
+      { slug: project.slug },
+      {
+        bulkId: result.friendlyId,
+      }
+    );
+
+    return redirectWithSuccessMessage(path, request, result.message);
+  } catch (error) {
+    if (error instanceof Error) {
+      logger.error("Failed to cancel runs", {
+        error: {
+          name: error.name,
+          message: error.message,
+          stack: error.stack,
+        },
+      });
+      return redirectWithErrorMessage(submission.value.failedRedirect, request, error.message);
+    } else {
+      logger.error("Failed to cancel runs", { error });
+      return redirectWithErrorMessage(
+        submission.value.failedRedirect,
+        request,
+        JSON.stringify(error)
+      );
+    }
+  }
+}

apps/webapp/app/routes/resources.taskruns.bulk.replay.ts

@@ -0,0 +1,93 @@
+import { parse } from "@conform-to/zod";
+import { ActionFunctionArgs } from "@remix-run/router";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
+import { logger } from "~/services/logger.server";
+import { requireUserId } from "~/services/session.server";
+import { v3RunsPath } from "~/utils/pathBuilder";
+import { CreateBulkActionService } from "~/v3/services/bulk/createBulkAction.server";
+
+const FormSchema = z.object({
+  organizationSlug: z.string(),
+  projectSlug: z.string(),
+  failedRedirect: z.string(),
+  runIds: z.array(z.string()).or(z.string()),
+});
+
+export async function action({ request }: ActionFunctionArgs) {
+  const userId = await requireUserId(request);
+
+  if (request.method.toLowerCase() !== "post") {
+    return redirectWithErrorMessage("/", request, "Invalid method");
+  }
+
+  const formData = await request.formData();
+  const submission = parse(formData, { schema: FormSchema });
+
+  if (!submission.value) {
+    logger.error("Failed to parse resources/taskruns/bulk/replay form data", { submission });
+    return redirectWithErrorMessage("/", request, "Failed to parse form data");
+  }
+
+  try {
+    const project = await prisma.project.findUnique({
+      where: {
+        slug: submission.value.projectSlug,
+        organization: {
+          members: {
+            some: {
+              userId,
+            },
+          },
+        },
+      },
+    });
+
+    if (!project) {
+      return redirectWithErrorMessage(
+        submission.value.failedRedirect,
+        request,
+        "Project not found"
+      );
+    }
+
+    const service = new CreateBulkActionService();
+    const result = await service.call({
+      projectId: project.id,
+      action: "REPLAY",
+      runIds:
+        typeof submission.value.runIds === "string"
+          ? [submission.value.runIds]
+          : submission.value.runIds,
+    });
+
+    const path = v3RunsPath(
+      { slug: submission.value.organizationSlug },
+      { slug: project.slug },
+      {
+        bulkId: result.friendlyId,
+      }
+    );
+
+    return redirectWithSuccessMessage(path, request, result.message);
+  } catch (error) {
+    if (error instanceof Error) {
+      logger.error("Failed to replay run", {
+        error: {
+          name: error.name,
+          message: error.message,
+          stack: error.stack,
+        },
+      });
+      return redirectWithErrorMessage(submission.value.failedRedirect, request, error.message);
+    } else {
+      logger.error("Failed to replay run", { error });
+      return redirectWithErrorMessage(
+        submission.value.failedRedirect,
+        request,
+        JSON.stringify(error)
+      );
+    }
+  }
+}

apps/webapp/app/routes/storybook.buttons/route.tsx

@@ -278,10 +278,8 @@ export default function Story() {
               <span className="text-charcoal-900">Continue with GitHub</span>
             </Button>
             <Button variant="secondary/large" fullWidth>
-              <EnvelopeIcon
-                className={"mr-1.5 h-5 w-5 text-primary transition group-hover:text-apple-200"}
-              />
-              <span className="text-primary group-hover:text-apple-200">Continue with Email</span>
+              <EnvelopeIcon className={"mr-1.5 h-5 w-5 text-secondary transition"} />
+              <span className="text-secondary">Continue with Email</span>
             </Button>
             <Button variant="tertiary/large" fullWidth>
               <GitHubLightIcon className={"mr-1.5 size-[1.2rem]"} />
@@ -308,10 +306,8 @@ export default function Story() {
               <span className="text-charcoal-900">Continue with GitHub</span>
             </Button>
             <Button variant="secondary/extra-large" fullWidth>
-              <EnvelopeIcon
-                className={"mr-1.5 h-5 w-5 text-primary transition group-hover:text-apple-200"}
-              />
-              <span className="text-primary group-hover:text-apple-200">Continue with Email</span>
+              <EnvelopeIcon className={"mr-1.5 h-5 w-5 text-secondary transition"} />
+              <span className="text-secondary">Continue with Email</span>
             </Button>
             <Button variant="tertiary/extra-large" fullWidth>
               <GitHubLightIcon className={"mr-1.5 h-5 w-5"} />

apps/webapp/app/routes/storybook.checkboxes/route.tsx

@@ -1,6 +1,6 @@
 import { useState } from "react";
 import { Button } from "~/components/primitives/Buttons";
-import { Checkbox } from "~/components/primitives/Checkbox";
+import { CheckboxWithLabel } from "~/components/primitives/Checkbox";
 
 export default function Story() {
   const [isDisabled, setIsDisabled] = useState(false);
@@ -14,21 +14,28 @@ export default function Story() {
       >
         {isDisabled ? "Enable checkboxes" : "Disable checkboxes"}
       </Button>
-      <Checkbox
+      <CheckboxWithLabel
+        name="Simple checkbox"
+        id="check1"
+        variant="simple/small"
+        label="This is a simple small checkbox"
+        disabled={isDisabled}
+      />
+      <CheckboxWithLabel
         name="Simple checkbox"
         id="check1"
         variant="simple"
         label="This is a simple checkbox"
         disabled={isDisabled}
       />
-      <Checkbox
+      <CheckboxWithLabel
         name="Button checkbox"
         id="check2"
         variant="button"
         label="This is a button checkbox"
         disabled={isDisabled}
       />
-      <Checkbox
+      <CheckboxWithLabel
         name="Button checkbox"
         id="check2"
         variant="button"
@@ -36,7 +43,7 @@ export default function Story() {
         badges={["This is a badge"]}
         disabled={isDisabled}
       />
-      <Checkbox
+      <CheckboxWithLabel
         name="Button checkbox"
         id="check2"
         variant="button"
@@ -45,7 +52,7 @@ export default function Story() {
         disabled={isDisabled}
       />
       <div className="flex flex-col gap-y-0.5 overflow-hidden rounded-md">
-        <Checkbox
+        <CheckboxWithLabel
           name="Description checkbox"
           id="check3"
           variant="description"
@@ -54,7 +61,7 @@ export default function Story() {
           description="This is a long checkbox description that goes full width. Grants full access to public and private repositories including read and write access to code, commit statuses, repository invitations, collaborators, deployment statuses, and repository webhooks. Note: In addition to repository related resources, the repo scope also grants access to manage organization-owned resources including projects, invitations, team memberships and webhooks. This scope also grants the ability to manage projects owned by users."
           disabled={isDisabled}
         />
-        <Checkbox
+        <CheckboxWithLabel
           name="Description checkbox"
           id="check4"
           variant="description"

apps/webapp/app/routes/storybook.segmented-control/route.tsx

@@ -1,4 +1,5 @@
 import { MainCenteredContainer } from "~/components/layout/AppLayout";
+import { Paragraph } from "~/components/primitives/Paragraph";
 import SegmentedControl from "~/components/primitives/SegmentedControl";
 
 const options = [
@@ -8,8 +9,15 @@ const options = [
 
 export default function Story() {
   return (
-    <MainCenteredContainer>
-      <SegmentedControl name="name" options={options} />
+    <MainCenteredContainer className="flex flex-col gap-8">
+      <div className="flex flex-col gap-2">
+        <Paragraph>Primary</Paragraph>
+        <SegmentedControl name="name" options={options} variant="primary" />
+      </div>
+      <div className="flex flex-col gap-2">
+        <Paragraph>Secondary</Paragraph>
+        <SegmentedControl name="name" options={options} variant="secondary" />
+      </div>
     </MainCenteredContainer>
   );
 }

apps/webapp/app/services/apiRateLimit.server.ts

@@ -191,6 +191,7 @@ export const apiRateLimiter = authorizationRateLimitMiddleware({
   pathMatchers: [/^\/api/],
   // Allow /api/v1/tasks/:id/callback/:secret
   pathWhiteList: [
+    "/api/internal/stripe_webhooks",
     "/api/v1/authorization-code",
     "/api/v1/token",
     /^\/api\/v1\/tasks\/[^\/]+\/callback\/[^\/]+$/, // /api/v1/tasks/$id/callback/$secret

apps/webapp/app/services/db/graphileMigrationHelper.server.ts

@@ -0,0 +1,95 @@
+import { runMigrations } from "graphile-worker";
+import { PrismaClient, prisma } from "~/db.server";
+import { env } from "~/env.server";
+import { logger } from "~/services/logger.server";
+import { PgNotifyService } from "./pgNotify.server";
+import { z } from "zod";
+
+export class GraphileMigrationHelperService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  public async call() {
+    this.#logDebug("GraphileMigrationHelperService.call");
+
+    await this.#detectAndPrepareForMigrations();
+
+    await runMigrations({
+      connectionString: env.DATABASE_URL,
+      schema: env.WORKER_SCHEMA,
+    });
+  }
+
+  #logDebug(message: string, args?: any) {
+    logger.debug(`[migrationHelper] ${message}`, args);
+  }
+
+  async #getLatestMigration() {
+    const migrationQueryResult = await this.#prismaClient.$queryRawUnsafe(`
+      SELECT id FROM ${env.WORKER_SCHEMA}.migrations
+      ORDER BY id DESC LIMIT 1
+    `);
+
+    const MigrationQueryResultSchema = z.array(z.object({ id: z.number() }));
+
+    const migrationResults = MigrationQueryResultSchema.parse(migrationQueryResult);
+
+    if (!migrationResults.length) {
+      // no migrations applied yet
+      return -1;
+    }
+
+    return migrationResults[0].id;
+  }
+
+  async #graphileSchemaExists() {
+    const schemaCount = await this.#prismaClient.$executeRaw`
+      SELECT schema_name FROM information_schema.schemata
+      WHERE schema_name = ${env.WORKER_SCHEMA}
+    `;
+
+    return schemaCount === 1;
+  }
+
+  /** Helper for graphile-worker v0.14.0 migration. No-op if already migrated. */
+  async #detectAndPrepareForMigrations() {
+    if (!(await this.#graphileSchemaExists())) {
+      // no schema yet, likely first start
+      return;
+    }
+
+    const latestMigration = await this.#getLatestMigration();
+
+    if (latestMigration < 0) {
+      // no migrations found
+      return;
+    }
+
+    // the first v0.14.0 migration has ID 11
+    if (latestMigration > 10) {
+      // already migrated
+      return;
+    }
+
+    // add 15s to graceful shutdown timeout, just to be safe
+    const migrationDelayInMs = env.GRACEFUL_SHUTDOWN_TIMEOUT + 15000;
+
+    this.#logDebug("Delaying worker startup due to pending migration", {
+      latestMigration,
+      migrationDelayInMs,
+    });
+
+    console.log(`⚠️  detected pending graphile migration`);
+    console.log(`⚠️  notifying running workers`);
+
+    const pgNotify = new PgNotifyService();
+    await pgNotify.call("trigger:graphile:migrate", { latestMigration });
+
+    console.log(`⚠️  delaying worker startup by ${migrationDelayInMs}ms`);
+
+    await new Promise((resolve) => setTimeout(resolve, migrationDelayInMs));
+  }
+}

apps/webapp/app/services/email.server.ts

@@ -20,6 +20,17 @@ const client = singleton(
     })
 );
 
+const alertsClient = singleton(
+  "alerts-email-client",
+  () =>
+    new EmailClient({
+      apikey: env.ALERT_RESEND_API_KEY,
+      imagesBaseUrl: env.APP_ORIGIN,
+      from: env.ALERT_FROM_EMAIL ?? "noreply@alerts.trigger.dev",
+      replyTo: env.REPLY_TO_EMAIL ?? "help@email.trigger.dev",
+    })
+);
+
 export async function sendMagicLinkEmail(options: SendEmailOptions<AuthUser>): Promise<void> {
   // Auto redirect when in development mode
   if (env.NODE_ENV === "development") {
@@ -67,3 +78,7 @@ export async function scheduleEmail(data: DeliverEmail, delay?: { seconds: numbe
 export async function sendEmail(data: DeliverEmail) {
   return client.send(data);
 }
+
+export async function sendAlertEmail(data: DeliverEmail) {
+  return alertsClient.send(data);
+}

apps/webapp/app/services/endpoints/performEndpointIndexService.ts

@@ -14,6 +14,9 @@ import { IndexEndpointStats } from "@trigger.dev/core";
 import { RegisterHttpEndpointService } from "../triggers/registerHttpEndpoint.server";
 import { RegisterWebhookService } from "../triggers/registerWebhook.server";
 import { EndpointIndex } from "@trigger.dev/database";
+import { env } from "~/env.server";
+
+const MAX_SEQUENTIAL_FAILURE_COUNT = env.MAX_SEQUENTIAL_INDEX_FAILURE_COUNT;
 
 export class PerformEndpointIndexService {
   #prismaClient: PrismaClient;
@@ -56,9 +59,16 @@ export class PerformEndpointIndexService {
 
     if (!endpointIndex.endpoint.url) {
       logger.debug("Endpoint URL is not set", endpointIndex);
-      return updateEndpointIndexWithError(this.#prismaClient, id, {
-        message: "Endpoint URL is not set",
-      });
+
+      return updateEndpointIndexWithError(
+        this.#prismaClient,
+        id,
+        endpointIndex.endpoint.id,
+        {
+          message: "Endpoint URL is not set",
+        },
+        false
+      );
     }
 
     // Make a request to the endpoint to fetch a list of jobs
@@ -69,9 +79,15 @@ export class PerformEndpointIndexService {
     const { response, parser, headerParser, errorParser } = await client.indexEndpoint();
 
     if (!response) {
-      return updateEndpointIndexWithError(this.#prismaClient, id, {
-        message: `Could not connect to endpoint ${endpointIndex.endpoint.url}`,
-      });
+      return updateEndpointIndexWithError(
+        this.#prismaClient,
+        id,
+        endpointIndex.endpoint.id,
+        {
+          message: `Could not connect to endpoint ${endpointIndex.endpoint.url}`,
+        },
+        endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
+      );
     }
 
     if (isRedirect(response.status)) {
@@ -83,15 +99,27 @@ export class PerformEndpointIndexService {
       const location = response.headers.get("location");
 
       if (!location) {
-        return updateEndpointIndexWithError(this.#prismaClient, id, {
-          message: `Endpoint ${endpointIndex.endpoint.url} is redirecting but no location header is present`,
-        });
+        return updateEndpointIndexWithError(
+          this.#prismaClient,
+          id,
+          endpointIndex.endpoint.id,
+          {
+            message: `Endpoint ${endpointIndex.endpoint.url} is redirecting but no location header is present`,
+          },
+          endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
+        );
       }
 
       if (redirectCount > 5) {
-        return updateEndpointIndexWithError(this.#prismaClient, id, {
-          message: `Endpoint ${endpointIndex.endpoint.url} is redirecting too many times`,
-        });
+        return updateEndpointIndexWithError(
+          this.#prismaClient,
+          id,
+          endpointIndex.endpoint.id,
+          {
+            message: `Endpoint ${endpointIndex.endpoint.url} is redirecting too many times`,
+          },
+          endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
+        );
       }
 
       await this.#prismaClient.endpoint.update({
@@ -111,20 +139,38 @@ export class PerformEndpointIndexService {
       const body = await safeBodyFromResponse(response, errorParser);
 
       if (body) {
-        return updateEndpointIndexWithError(this.#prismaClient, id, {
-          message: body.message,
-        });
+        return updateEndpointIndexWithError(
+          this.#prismaClient,
+          id,
+          endpointIndex.endpoint.id,
+          {
+            message: body.message,
+          },
+          endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
+        );
       }
 
-      return updateEndpointIndexWithError(this.#prismaClient, id, {
-        message: "Trigger API key is invalid",
-      });
+      return updateEndpointIndexWithError(
+        this.#prismaClient,
+        id,
+        endpointIndex.endpoint.id,
+        {
+          message: "Trigger API key is invalid",
+        },
+        endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
+      );
     }
 
     if (!response.ok) {
-      return updateEndpointIndexWithError(this.#prismaClient, id, {
-        message: `Could not connect to endpoint ${endpointIndex.endpoint.url}. Status code: ${response.status}`,
-      });
+      return updateEndpointIndexWithError(
+        this.#prismaClient,
+        id,
+        endpointIndex.endpoint.id,
+        {
+          message: `Could not connect to endpoint ${endpointIndex.endpoint.url}. Status code: ${response.status}`,
+        },
+        endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
+      );
     }
 
     const anyBody = await response.json();
@@ -152,10 +198,16 @@ export class PerformEndpointIndexService {
         }).message;
       }
 
-      return updateEndpointIndexWithError(this.#prismaClient, id, {
-        message: friendlyError,
-        raw: fromZodError(bodyResult.error).message,
-      });
+      return updateEndpointIndexWithError(
+        this.#prismaClient,
+        id,
+        endpointIndex.endpoint.id,
+        {
+          message: friendlyError,
+          raw: fromZodError(bodyResult.error).message,
+        },
+        endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
+      );
     }
 
     const headerResult = headerParser.safeParse(Object.fromEntries(response.headers.entries()));
@@ -163,10 +215,16 @@ export class PerformEndpointIndexService {
       const friendlyError = fromZodError(headerResult.error, {
         prefix: "Your headers are invalid",
       });
-      return updateEndpointIndexWithError(this.#prismaClient, id, {
-        message: friendlyError.message,
-        raw: headerResult.error.issues,
-      });
+      return updateEndpointIndexWithError(
+        this.#prismaClient,
+        id,
+        endpointIndex.endpoint.id,
+        {
+          message: friendlyError.message,
+          raw: headerResult.error.issues,
+        },
+        endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
+      );
     }
 
     const { jobs, sources, dynamicTriggers, dynamicSchedules, httpEndpoints, webhooks } =
@@ -407,8 +465,49 @@ export class PerformEndpointIndexService {
 async function updateEndpointIndexWithError(
   prismaClient: PrismaClient,
   id: string,
-  error: EndpointIndexError
+  endpointId: string,
+  error: EndpointIndexError,
+  checkDisabling = true
 ) {
+  // Check here to see if this endpoint has only failed for the last 50 times
+  // And if so, we disable the endpoint by setting the url to null
+  if (checkDisabling) {
+    const recentIndexes = await prismaClient.endpointIndex.findMany({
+      where: {
+        endpointId,
+        id: {
+          not: id,
+        },
+      },
+      orderBy: {
+        createdAt: "desc",
+      },
+      take: MAX_SEQUENTIAL_FAILURE_COUNT - 1,
+      select: {
+        status: true,
+      },
+    });
+
+    if (
+      recentIndexes.length === MAX_SEQUENTIAL_FAILURE_COUNT - 1 &&
+      recentIndexes.every((index) => index.status === "FAILURE")
+    ) {
+      logger.debug("Disabling endpoint", {
+        endpointId,
+        error,
+      });
+
+      await prismaClient.endpoint.update({
+        where: {
+          id: endpointId,
+        },
+        data: {
+          url: null,
+        },
+      });
+    }
+  }
+
   return await prismaClient.endpointIndex.update({
     where: {
       id,

apps/webapp/app/services/events/ingestSendEvent.server.ts

@@ -60,6 +60,11 @@ export class IngestSendEvent {
     try {
       const deliverAt = this.#calculateDeliverAt(options);
 
+      if (!environment.organization.runsEnabled) {
+        logger.debug("IngestSendEvent: Runs are disabled for this organization", environment);
+        return;
+      }
+
       return await $transaction(this.#prismaClient, async (tx) => {
         const externalAccount = options?.accountId
           ? await tx.externalAccount.upsert({

apps/webapp/app/services/runs/startRun.server.ts

@@ -8,6 +8,7 @@ import { $transaction, prisma } from "~/db.server";
 import { workerQueue } from "../worker.server";
 import { ResumeRunService } from "./resumeRun.server";
 import { createHash } from "node:crypto";
+import { logger } from "../logger.server";
 
 type FoundRun = NonNullable<Awaited<ReturnType<typeof findRun>>>;
 type RunConnectionsByKey = Awaited<ReturnType<typeof createRunConnections>>;
@@ -36,6 +37,13 @@ export class StartRunService {
   }
 
   #runIsStartable(run: FoundRun) {
+    if (!run.organization.runsEnabled) {
+      logger.debug("StartRunService: Runs are disabled for this organization", {
+        organizationId: run.organization.id,
+      });
+      return false;
+    }
+
     const startableStatuses = ["PENDING", "WAITING_ON_CONNECTIONS"] as const;
     return startableStatuses.includes(run.status);
   }
@@ -64,8 +72,6 @@ export class StartRunService {
     await $transaction(
       this.#prismaClient,
       async (tx) => {
-        await tx.$executeRaw`SELECT pg_advisory_xact_lock(${lockId})`;
-
         const counter = await tx.jobCounter.upsert({
           where: { jobId: run.jobId },
           update: { lastNumber: { increment: 1 } },
@@ -146,6 +152,7 @@ async function findRun(tx: PrismaClientOrTransaction, id: string) {
     include: {
       queue: true,
       environment: true,
+      organization: true,
       version: {
         include: {
           integrations: {

apps/webapp/app/services/schedules/deliverScheduledEvent.server.ts

@@ -4,6 +4,9 @@ import { NextScheduledEventService } from "./nextScheduledEvent.server";
 import { IngestSendEvent } from "../events/ingestSendEvent.server";
 import { InvokeDispatcherService } from "../events/invokeDispatcher.server";
 import { logger } from "../logger.server";
+import { workerQueue } from "../worker.server";
+
+const DEFERRED_ENQUEUE_THRESHOLD_IN_SECONDS = 60 * 60 * 2 - 1; // 2 hours - 1 second
 
 export class DeliverScheduledEventService {
   #prismaClient: PrismaClientOrTransaction;
@@ -12,6 +15,132 @@ export class DeliverScheduledEventService {
     this.#prismaClient = prismaClient;
   }
 
+  // This runs every 10 minutes
+  public static async scheduleImminentDeferredEvents() {
+    // Find all deferred events that are due to be enqueued in the next hour
+    const deferredEvents = await prisma.deferredScheduledEventService.findMany({
+      where: {
+        runAt: {
+          lte: new Date(Date.now() + (DEFERRED_ENQUEUE_THRESHOLD_IN_SECONDS / 2) * 1000),
+        },
+      },
+    });
+
+    for (const deferredEvent of deferredEvents) {
+      logger.debug("Enqueuing deferred scheduled event", {
+        scheduleSourceId: deferredEvent.scheduleSourceId,
+        runAt: deferredEvent.runAt,
+      });
+
+      try {
+        await DeliverScheduledEventService.enqueue(
+          deferredEvent.scheduleSourceId,
+          deferredEvent.runAt,
+          {
+            ts: deferredEvent.runAt,
+            lastTimestamp: deferredEvent.lastTimestamp ?? undefined,
+          }
+        );
+      } catch (error) {
+        logger.error("Error enqueuing deferred scheduled event", {
+          scheduleSourceId: deferredEvent.scheduleSourceId,
+          runAt: deferredEvent.runAt,
+          error,
+        });
+      }
+    }
+  }
+
+  public static async dequeue(id: string, tx: PrismaClientOrTransaction = prisma) {
+    await tx.deferredScheduledEventService.deleteMany({
+      where: {
+        scheduleSourceId: id,
+      },
+    });
+
+    await workerQueue.dequeue(`scheduled:${id}`);
+
+    await tx.scheduleSource.update({
+      where: {
+        id,
+      },
+      data: {
+        workerJobId: null,
+      },
+    });
+  }
+
+  public static async enqueue(
+    id: string,
+    runAt: Date,
+    payload: ScheduledPayload,
+    tx: PrismaClientOrTransaction = prisma
+  ) {
+    if (runAt.getTime() - Date.now() > DEFERRED_ENQUEUE_THRESHOLD_IN_SECONDS * 1000) {
+      logger.debug("Deferring enqueueing events.deliverScheduled", {
+        id,
+        runAt,
+        payload,
+      });
+
+      await tx.deferredScheduledEventService.upsert({
+        where: {
+          scheduleSourceId: id,
+        },
+        create: {
+          scheduleSourceId: id,
+          runAt,
+          lastTimestamp: payload.lastTimestamp,
+        },
+        update: {
+          runAt,
+          lastTimestamp: payload.lastTimestamp,
+        },
+      });
+
+      await tx.scheduleSource.update({
+        where: {
+          id,
+        },
+        data: {
+          workerJobId: null,
+          nextEventTimestamp: runAt,
+        },
+      });
+
+      await workerQueue.dequeue(`scheduled:${id}`);
+    } else {
+      await tx.deferredScheduledEventService.deleteMany({
+        where: {
+          scheduleSourceId: id,
+        },
+      });
+
+      const workerJob = await workerQueue.enqueue(
+        "events.deliverScheduled",
+        {
+          id: id,
+          payload,
+        },
+        {
+          runAt,
+          tx,
+          jobKey: `scheduled:${id}`,
+        }
+      );
+
+      await tx.scheduleSource.update({
+        where: {
+          id,
+        },
+        data: {
+          workerJobId: workerJob.id,
+          nextEventTimestamp: runAt,
+        },
+      });
+    }
+  }
+
   public async call(id: string, payload: ScheduledPayload) {
     return await $transaction(
       this.#prismaClient,

apps/webapp/app/services/schedules/disableScheduleSource.server.ts

@@ -1,6 +1,7 @@
 import type { EventDispatcher } from "@trigger.dev/database";
 import { $transaction, PrismaClientOrTransaction, prisma } from "~/db.server";
 import { workerQueue } from "../worker.server";
+import { DeliverScheduledEventService } from "./deliverScheduledEvent.server";
 
 export class DisableScheduleSourceService {
   #prismaClient: PrismaClientOrTransaction;
@@ -36,7 +37,7 @@ export class DisableScheduleSourceService {
         },
       });
 
-      await workerQueue.dequeue(`scheduled:${scheduleSource.id}`, { tx });
+      await DeliverScheduledEventService.dequeue(scheduleSource.id, tx);
 
       return scheduleSource;
     });

apps/webapp/app/services/schedules/nextScheduledEvent.server.ts

@@ -6,8 +6,8 @@ import {
 } from "@trigger.dev/core";
 import { $transaction, PrismaClientOrTransaction, prisma } from "~/db.server";
 import { parseExpression } from "cron-parser";
-import { workerQueue } from "../worker.server";
 import { logger } from "../logger.server";
+import { DeliverScheduledEventService } from "./deliverScheduledEvent.server";
 
 export class NextScheduledEventService {
   #prismaClient: PrismaClientOrTransaction;
@@ -45,32 +45,16 @@ export class NextScheduledEventService {
         lastTimestamp: scheduleSource.lastEventTimestamp,
       });
 
-      const workerJob = await workerQueue.enqueue(
-        "events.deliverScheduled",
+      await DeliverScheduledEventService.enqueue(
+        scheduleSource.id,
+        scheduleTime,
         {
-          id: scheduleSource.id,
-          payload: {
-            ts: scheduleTime,
-            lastTimestamp: scheduleSource.lastEventTimestamp ?? undefined,
-          },
+          ts: scheduleTime,
+          lastTimestamp: scheduleSource.lastEventTimestamp ?? undefined,
         },
-        {
-          runAt: scheduleTime,
-          tx,
-          jobKey: `scheduled:${scheduleSource.id}`,
-        }
+        tx
       );
 
-      await this.#prismaClient.scheduleSource.update({
-        where: {
-          id: scheduleSource.id,
-        },
-        data: {
-          workerJobId: workerJob.id,
-          nextEventTimestamp: scheduleTime,
-        },
-      });
-
       return scheduleSource;
     });
   }

apps/webapp/app/services/secrets/secretStore.server.ts

@@ -55,12 +55,14 @@ export class SecretStore {
   }
 }
 
-const EncryptedSecretValueSchema = z.object({
+export const EncryptedSecretValueSchema = z.object({
   nonce: z.string(),
   ciphertext: z.string(),
   tag: z.string(),
 });
 
+export type EncryptedSecretValue = z.infer<typeof EncryptedSecretValueSchema>;
+
 /** This stores secrets in the Postgres Database, encrypted using aes-256-gcm */
 class PrismaSecretStore implements SecretStoreProvider {
   #prismaClient: PrismaClientOrTransaction;
@@ -181,18 +183,11 @@ class PrismaSecretStore implements SecretStoreProvider {
   }
 
   async #decrypt(nonce: string, ciphertext: string, tag: string): Promise<string> {
-    const decipher = nodeCrypto.createDecipheriv(
-      "aes-256-gcm",
-      this.encryptionKey,
-      Buffer.from(nonce, "hex")
-    );
-
-    decipher.setAuthTag(Buffer.from(tag, "hex"));
-
-    let decrypted = decipher.update(ciphertext, "hex", "utf8");
-    decrypted += decipher.final("utf8");
-
-    return decrypted;
+    return await decryptSecret(this.encryptionKey, {
+      nonce,
+      ciphertext,
+      tag,
+    });
   }
 
   async #encrypt(value: string): Promise<{
@@ -200,19 +195,7 @@ class PrismaSecretStore implements SecretStoreProvider {
     ciphertext: string;
     tag: string;
   }> {
-    const nonce = nodeCrypto.randomBytes(12);
-    const cipher = nodeCrypto.createCipheriv("aes-256-gcm", this.encryptionKey, nonce);
-
-    let encrypted = cipher.update(value, "utf8", "hex");
-    encrypted += cipher.final("hex");
-
-    const tag = cipher.getAuthTag().toString("hex");
-
-    return {
-      nonce: nonce.toString("hex"),
-      ciphertext: encrypted,
-      tag,
-    };
+    return await encryptSecret(this.encryptionKey, value);
   }
 }
 
@@ -234,3 +217,40 @@ export function getSecretStore<
     }
   }
 }
+
+export async function decryptSecret(
+  encryptionKey: string,
+  secret: EncryptedSecretValue
+): Promise<string> {
+  const decipher = nodeCrypto.createDecipheriv(
+    "aes-256-gcm",
+    encryptionKey,
+    Buffer.from(secret.nonce, "hex")
+  );
+
+  decipher.setAuthTag(Buffer.from(secret.tag, "hex"));
+
+  let decrypted = decipher.update(secret.ciphertext, "hex", "utf8");
+  decrypted += decipher.final("utf8");
+
+  return decrypted;
+}
+
+export async function encryptSecret(
+  encryptionKey: string,
+  value: string
+): Promise<EncryptedSecretValue> {
+  const nonce = nodeCrypto.randomBytes(12);
+  const cipher = nodeCrypto.createCipheriv("aes-256-gcm", encryptionKey, nonce);
+
+  let encrypted = cipher.update(value, "utf8", "hex");
+  encrypted += cipher.final("hex");
+
+  const tag = cipher.getAuthTag().toString("hex");
+
+  return {
+    nonce: nonce.toString("hex"),
+    ciphertext: encrypted,
+    tag,
+  };
+}

apps/webapp/app/services/sources/handleHttpSource.server.ts

@@ -4,6 +4,7 @@ import { workerQueue } from "../worker.server";
 import { requestUrl } from "~/utils/requestUrl.server";
 import { RuntimeEnvironmentType } from "@trigger.dev/database";
 import { createHttpSourceRequest } from "~/utils/createHttpSourceRequest";
+import { logger } from "../logger.server";
 
 export class HandleHttpSourceService {
   #prismaClient: PrismaClient;
@@ -19,6 +20,7 @@ export class HandleHttpSourceService {
         endpoint: true,
         environment: true,
         secretReference: true,
+        organization: true,
       },
     });
 
@@ -30,6 +32,14 @@ export class HandleHttpSourceService {
       return { status: 200 };
     }
 
+    if (!triggerSource.endpoint.url) {
+      return { status: 404 };
+    }
+
+    if (!triggerSource.organization.runsEnabled) {
+      return { status: 404 };
+    }
+
     if (!triggerSource.interactive) {
       const sourceRequest = await createHttpSourceRequest(request);
 

apps/webapp/app/services/sources/handleWebhookRequest.server.ts

@@ -37,8 +37,6 @@ export class HandleWebhookRequestService {
     const lockId = webhookIdToLockId(webhookEnvironment.webhookId);
 
     await this.#prismaClient.$transaction(async (tx) => {
-      await tx.$executeRaw`SELECT pg_advisory_xact_lock(${lockId})`;
-
       const counter = await tx.webhookDeliveryCounter.upsert({
         where: { webhookId: webhookEnvironment.id },
         update: { lastNumber: { increment: 1 } },

apps/webapp/app/services/worker.server.ts

@@ -37,6 +37,13 @@ import { TimeoutDeploymentService } from "~/v3/services/timeoutDeployment.server
 import { eventRepository } from "~/v3/eventRepository.server";
 import { ExecuteTasksWaitingForDeployService } from "~/v3/services/executeTasksWaitingForDeploy";
 import { TriggerScheduledTaskService } from "~/v3/services/triggerScheduledTask.server";
+import { PerformTaskAttemptAlertsService } from "~/v3/services/alerts/performTaskAttemptAlerts.server";
+import { DeliverAlertService } from "~/v3/services/alerts/deliverAlert.server";
+import { PerformDeploymentAlertsService } from "~/v3/services/alerts/performDeploymentAlerts.server";
+import { GraphileMigrationHelperService } from "./db/graphileMigrationHelper.server";
+import { PerformBulkActionService } from "~/v3/services/bulk/performBulkAction.server";
+import { CancelTaskRunService } from "~/v3/services/cancelTaskRun.server";
+import { ReplayTaskRunService } from "~/v3/services/replayTaskRun.server";
 import { RequeueTaskRunService } from "~/v3/requeueTaskRun.server";
 import { RetryAttemptService } from "~/v3/services/retryAttempt.server";
 
@@ -138,6 +145,21 @@ const workerCatalog = {
   "v3.triggerScheduledTask": z.object({
     instanceId: z.string(),
   }),
+  "v3.performTaskAttemptAlerts": z.object({
+    attemptId: z.string(),
+  }),
+  "v3.deliverAlert": z.object({
+    alertId: z.string(),
+  }),
+  "v3.performDeploymentAlerts": z.object({
+    deploymentId: z.string(),
+  }),
+  "v3.performBulkAction": z.object({
+    bulkActionGroupId: z.string(),
+  }),
+  "v3.performBulkActionItem": z.object({
+    bulkActionItemId: z.string(),
+  }),
   "v3.requeueTaskRun": z.object({
     runId: z.string(),
   }),
@@ -207,9 +229,8 @@ if (env.NODE_ENV === "production") {
 }
 
 export async function init() {
-  // const pgNotify = new PgNotifyService();
-  // await pgNotify.call("trigger:graphile:migrate", { latestMigration: 10 });
-  // await new Promise((resolve) => setTimeout(resolve, 10000))
+  const migrationHelper = new GraphileMigrationHelperService();
+  await migrationHelper.call();
 
   if (env.WORKER_ENABLED === "true") {
     await workerQueue.initialize();
@@ -228,11 +249,6 @@ function getWorkerQueue() {
   return new ZodWorker({
     name: "workerQueue",
     prisma,
-    cleanup: {
-      frequencyExpression: "13,27,43 * * * *",
-      ttl: 7 * 24 * 60 * 60 * 1000, // 7 days
-      maxCount: 1000,
-    },
     runnerOptions: {
       connectionString: env.DATABASE_URL,
       concurrency: env.WORKER_CONCURRENCY,
@@ -246,16 +262,22 @@ function getWorkerQueue() {
     recurringTasks: {
       // Run this every 5 minutes
       autoIndexProductionEndpoints: {
-        pattern: "*/5 * * * *",
+        match: "*/5 * * * *",
         handler: async (payload, job) => {
           const service = new RecurringEndpointIndexService();
 
           await service.call(payload.ts);
         },
       },
+      scheduleImminentDeferredEvents: {
+        match: "*/10 * * * *",
+        handler: async (payload, job) => {
+          await DeliverScheduledEventService.scheduleImminentDeferredEvents();
+        },
+      },
       // Run this every hour
       purgeOldIndexings: {
-        pattern: "0 * * * *",
+        match: "0 * * * *",
         handler: async (payload, job) => {
           // Delete indexings that are older than 7 days
           await prisma.endpointIndex.deleteMany({
@@ -269,7 +291,7 @@ function getWorkerQueue() {
       },
       // Run this every hour at the 13 minute mark
       purgeOldTaskEvents: {
-        pattern: "47 * * * *",
+        match: "47 * * * *",
         handler: async (payload, job) => {
           await eventRepository.truncateEvents();
         },
@@ -541,6 +563,51 @@ function getWorkerQueue() {
           return await service.call(payload.instanceId);
         },
       },
+      "v3.performTaskAttemptAlerts": {
+        priority: 0,
+        maxAttempts: 3,
+        handler: async (payload, job) => {
+          const service = new PerformTaskAttemptAlertsService();
+
+          return await service.call(payload.attemptId);
+        },
+      },
+      "v3.deliverAlert": {
+        priority: 0,
+        maxAttempts: 8,
+        handler: async (payload, job) => {
+          const service = new DeliverAlertService();
+
+          return await service.call(payload.alertId);
+        },
+      },
+      "v3.performDeploymentAlerts": {
+        priority: 0,
+        maxAttempts: 3,
+        handler: async (payload, job) => {
+          const service = new PerformDeploymentAlertsService();
+
+          return await service.call(payload.deploymentId);
+        },
+      },
+      "v3.performBulkAction": {
+        priority: 0,
+        maxAttempts: 3,
+        handler: async (payload, job) => {
+          const service = new PerformBulkActionService();
+
+          return await service.call(payload.bulkActionGroupId);
+        },
+      },
+      "v3.performBulkActionItem": {
+        priority: 0,
+        maxAttempts: 3,
+        handler: async (payload, job) => {
+          const service = new PerformBulkActionService();
+
+          await service.performBulkActionItem(payload.bulkActionItemId);
+        },
+      },
       "v3.requeueTaskRun": {
         priority: 0,
         maxAttempts: 3,

apps/webapp/app/utils/environmentSort.ts

@@ -1,5 +1,4 @@
 import { RuntimeEnvironmentType } from "@trigger.dev/database";
-import { logger } from "./logger.server";
 
 const environmentSortOrder: RuntimeEnvironmentType[] = [
   "DEVELOPMENT",

apps/webapp/app/utils/formData.ts

@@ -1,10 +0,0 @@
-import type { z } from "zod";
-
-export function formDataAsObject<TValues>(
-  formData: FormData,
-  schema: z.ZodSchema<TValues>
-): TValues {
-  const object = Object.fromEntries(formData.entries());
-  const parsed = schema.parse(object);
-  return parsed;
-}

apps/webapp/app/utils/numberFormatter.ts

@@ -4,6 +4,12 @@ export const formatNumberCompact = (num: number): string => {
   return compactFormatter.format(num);
 };
 
+const formatter = Intl.NumberFormat("en");
+
+export const formatNumber = (num: number): string => {
+  return formatter.format(num);
+};
+
 const roundedCurrencyFormatter = Intl.NumberFormat("en-US", {
   style: "currency",
   currencyDisplay: "symbol",

apps/webapp/app/utils/pathBuilder.ts

@@ -325,6 +325,21 @@ export function v3NewEnvironmentVariablesPath(organization: OrgForPath, project:
   return `${v3EnvironmentVariablesPath(organization, project)}/new`;
 }
 
+export function v3ProjectAlertsPath(organization: OrgForPath, project: ProjectForPath) {
+  return `${v3ProjectPath(organization, project)}/alerts`;
+}
+
+export function v3NewProjectAlertPath(organization: OrgForPath, project: ProjectForPath) {
+  return `${v3ProjectAlertsPath(organization, project)}/new`;
+}
+
+export function v3NewProjectAlertPathConnectToSlackPath(
+  organization: OrgForPath,
+  project: ProjectForPath
+) {
+  return `${v3ProjectAlertsPath(organization, project)}/new/connect-to-slack`;
+}
+
 export function v3TestPath(
   organization: OrgForPath,
   project: ProjectForPath,

apps/webapp/app/v3/marqs/devQueueConsumer.server.ts

@@ -14,7 +14,7 @@ import { prisma } from "~/db.server";
 import { createNewSession, disconnectSession } from "~/models/runtimeEnvironment.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
-import { marqs } from "~/v3/marqs/index.server";
+import { marqs, sanitizeQueueName } from "~/v3/marqs/index.server";
 import { EnvironmentVariablesRepository } from "../environmentVariables/environmentVariablesRepository.server";
 import { CancelTaskRunService } from "../services/cancelTaskRun.server";
 import { CompleteAttemptService } from "../services/completeAttempt.server";
@@ -91,6 +91,10 @@ export class DevQueueConsumer {
       return;
     }
 
+    if (this._backgroundWorkers.has(backgroundWorker.id)) {
+      return;
+    }
+
     this._backgroundWorkers.set(backgroundWorker.id, backgroundWorker);
 
     logger.debug("Registered background worker", {
@@ -264,10 +268,10 @@ export class DevQueueConsumer {
       return;
     }
 
+    this._enabled = true;
     // Create the session
     await createNewSession(this.env, this._options.ipAddress ?? "unknown");
 
-    this._enabled = true;
     this._perTraceCountdown = this._options.maximumItemsPerTrace;
     this._lastNewTrace = new Date();
     this._taskFailures = 0;
@@ -439,6 +443,28 @@ export class DevQueueConsumer {
       return;
     }
 
+    const queue = await prisma.taskQueue.findUnique({
+      where: {
+        runtimeEnvironmentId_name: {
+          runtimeEnvironmentId: this.env.id,
+          name: sanitizeQueueName(lockedTaskRun.queue),
+        },
+      },
+    });
+
+    if (!queue) {
+      logger.debug("[DevQueueConsumer] Failed to find queue", {
+        queueName: lockedTaskRun.queue,
+        sanitizedName: sanitizeQueueName(lockedTaskRun.queue),
+        taskRun: lockedTaskRun.id,
+        messageId: message.messageId,
+      });
+
+      await marqs?.nackMessage(message.messageId);
+      setTimeout(() => this.#doWork(), 1000);
+      return;
+    }
+
     if (!this._enabled) {
       logger.debug("Dev queue consumer is disabled", { env: this.env, queueMessage: message });
 

apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts

@@ -22,7 +22,7 @@ import { z } from "zod";
 import { prisma } from "~/db.server";
 import { logger } from "~/services/logger.server";
 import { singleton } from "~/utils/singleton";
-import { marqs } from "~/v3/marqs/index.server";
+import { marqs, sanitizeQueueName } from "~/v3/marqs/index.server";
 import { EnvironmentVariablesRepository } from "../environmentVariables/environmentVariablesRepository.server";
 import { generateFriendlyId } from "../friendlyIdentifiers";
 import { socketIo } from "../handleSocketIo.server";
@@ -429,6 +429,26 @@ export class SharedQueueConsumer {
           return;
         }
 
+        const queue = await prisma.taskQueue.findUnique({
+          where: {
+            runtimeEnvironmentId_name: {
+              runtimeEnvironmentId: lockedTaskRun.runtimeEnvironmentId,
+              name: sanitizeQueueName(lockedTaskRun.queue),
+            },
+          },
+        });
+
+        if (!queue) {
+          logger.debug("SharedQueueConsumer queue not found, so nacking message", {
+            queueMessage: message,
+            taskRunQueue: lockedTaskRun.queue,
+            runtimeEnvironmentId: lockedTaskRun.runtimeEnvironmentId,
+          });
+
+          await this.#nackAndDoMoreWork(message.messageId, this._options.nextTickInterval);
+          return;
+        }
+
         if (!this._enabled) {
           logger.debug("SharedQueueConsumer not enabled, so nacking message", {
             queueMessage: message,
@@ -632,12 +652,17 @@ export class SharedQueueConsumer {
           where: {
             runtimeEnvironmentId_name: {
               runtimeEnvironmentId: resumableAttempt.runtimeEnvironmentId,
-              name: resumableRun.queue,
+              name: sanitizeQueueName(resumableRun.queue),
             },
           },
         });
 
         if (!queue) {
+          logger.debug("SharedQueueConsumer queue not found, so nacking message", {
+            queueName: sanitizeQueueName(resumableRun.queue),
+            attempt: resumableAttempt,
+          });
+
           await this.#nackAndDoMoreWork(message.messageId, this._options.nextTickInterval);
           return;
         }

apps/webapp/app/v3/services/alerts/createAlertChannel.server.ts

@@ -0,0 +1,114 @@
+import {
+  ProjectAlertChannel,
+  ProjectAlertType,
+  RuntimeEnvironmentType,
+} from "@trigger.dev/database";
+import { nanoid } from "nanoid";
+import { env } from "~/env.server";
+import { findProjectByRef } from "~/models/project.server";
+import { encryptSecret } from "~/services/secrets/secretStore.server";
+import { generateFriendlyId } from "~/v3/friendlyIdentifiers";
+import { BaseService, ServiceValidationError } from "../baseService.server";
+
+export type CreateAlertChannelOptions = {
+  name: string;
+  alertTypes: ProjectAlertType[];
+  environmentTypes: RuntimeEnvironmentType[];
+  deduplicationKey?: string;
+  channel:
+    | {
+        type: "EMAIL";
+        email: string;
+      }
+    | {
+        type: "WEBHOOK";
+        url: string;
+        secret?: string;
+      }
+    | {
+        type: "SLACK";
+        channelId: string;
+        channelName: string;
+        integrationId: string | undefined;
+      };
+};
+
+export class CreateAlertChannelService extends BaseService {
+  public async call(
+    projectRef: string,
+    userId: string,
+    options: CreateAlertChannelOptions
+  ): Promise<ProjectAlertChannel> {
+    const project = await findProjectByRef(projectRef, userId);
+
+    if (!project) {
+      throw new ServiceValidationError("Project not found");
+    }
+
+    const environmentTypes =
+      options.environmentTypes.length === 0
+        ? (["STAGING", "PRODUCTION"] satisfies RuntimeEnvironmentType[])
+        : options.environmentTypes;
+
+    const existingAlertChannel = options.deduplicationKey
+      ? await this._prisma.projectAlertChannel.findUnique({
+          where: {
+            projectId_deduplicationKey: {
+              projectId: project.id,
+              deduplicationKey: options.deduplicationKey,
+            },
+          },
+        })
+      : undefined;
+
+    if (existingAlertChannel) {
+      return await this._prisma.projectAlertChannel.update({
+        where: { id: existingAlertChannel.id },
+        data: {
+          name: options.name,
+          alertTypes: options.alertTypes,
+          type: options.channel.type,
+          properties: await this.#createProperties(options.channel),
+          environmentTypes,
+        },
+      });
+    }
+
+    const alertChannel = await this._prisma.projectAlertChannel.create({
+      data: {
+        friendlyId: generateFriendlyId("alert_channel"),
+        name: options.name,
+        alertTypes: options.alertTypes,
+        projectId: project.id,
+        type: options.channel.type,
+        properties: await this.#createProperties(options.channel),
+        enabled: true,
+        deduplicationKey: options.deduplicationKey,
+        userProvidedDeduplicationKey: options.deduplicationKey ? true : false,
+        environmentTypes,
+      },
+    });
+
+    return alertChannel;
+  }
+
+  async #createProperties(channel: CreateAlertChannelOptions["channel"]) {
+    switch (channel.type) {
+      case "EMAIL":
+        return {
+          email: channel.email,
+        };
+      case "WEBHOOK":
+        return {
+          url: channel.url,
+          secret: await encryptSecret(env.ENCRYPTION_KEY, channel.secret ?? nanoid()),
+        };
+      case "SLACK":
+        return {
+          channelId: channel.channelId,
+          channelName: channel.channelName,
+          integrationId: channel.integrationId,
+        };
+    }
+  }
+}

apps/webapp/app/v3/services/alerts/deliverAlert.server.ts

@@ -0,0 +1,831 @@
+import {
+  ChatPostMessageArguments,
+  ErrorCode,
+  WebAPIHTTPError,
+  WebAPIPlatformError,
+  WebAPIRateLimitedError,
+  WebAPIRequestError,
+} from "@slack/web-api";
+import { TaskRunError, createJsonErrorObject } from "@trigger.dev/core/v3";
+import assertNever from "assert-never";
+import { subtle } from "crypto";
+import { Prisma, PrismaClientOrTransaction, prisma } from "~/db.server";
+import { env } from "~/env.server";
+import {
+  OrgIntegrationRepository,
+  OrganizationIntegrationForService,
+} from "~/models/orgIntegration.server";
+import {
+  ProjectAlertEmailProperties,
+  ProjectAlertSlackProperties,
+  ProjectAlertSlackStorage,
+  ProjectAlertWebhookProperties,
+} from "~/models/projectAlert.server";
+import { DeploymentPresenter } from "~/presenters/v3/DeploymentPresenter.server";
+import { sendAlertEmail } from "~/services/email.server";
+import { logger } from "~/services/logger.server";
+import { decryptSecret } from "~/services/secrets/secretStore.server";
+import { workerQueue } from "~/services/worker.server";
+import { BaseService } from "../baseService.server";
+
+type FoundAlert = Prisma.Result<
+  typeof prisma.projectAlert,
+  {
+    include: {
+      channel: true;
+      project: {
+        include: {
+          organization: true;
+        };
+      };
+      environment: true;
+      taskRunAttempt: {
+        include: {
+          taskRun: true;
+          backgroundWorkerTask: true;
+          backgroundWorker: true;
+        };
+      };
+      workerDeployment: {
+        include: {
+          worker: {
+            include: {
+              tasks: true;
+            };
+          };
+        };
+      };
+    };
+  },
+  "findUniqueOrThrow"
+>;
+
+export class DeliverAlertService extends BaseService {
+  public async call(alertId: string) {
+    const alert = await this._prisma.projectAlert.findUnique({
+      where: { id: alertId },
+      include: {
+        channel: true,
+        project: {
+          include: {
+            organization: true,
+          },
+        },
+        environment: true,
+        taskRunAttempt: {
+          include: {
+            taskRun: true,
+            backgroundWorkerTask: true,
+            backgroundWorker: true,
+          },
+        },
+        workerDeployment: {
+          include: {
+            worker: {
+              include: {
+                tasks: true,
+              },
+            },
+          },
+        },
+      },
+    });
+
+    if (!alert) {
+      return;
+    }
+
+    if (alert.status !== "PENDING") {
+      return;
+    }
+
+    switch (alert.channel.type) {
+      case "EMAIL": {
+        await this.#sendEmail(alert);
+        break;
+      }
+      case "SLACK": {
+        await this.#sendSlack(alert);
+        break;
+      }
+      case "WEBHOOK": {
+        await this.#sendWebhook(alert);
+        break;
+      }
+      default: {
+        assertNever(alert.channel.type);
+      }
+    }
+
+    await this._prisma.projectAlert.update({
+      where: { id: alertId },
+      data: {
+        status: "SENT",
+      },
+    });
+  }
+
+  async #sendEmail(alert: FoundAlert) {
+    const emailProperties = ProjectAlertEmailProperties.safeParse(alert.channel.properties);
+
+    if (!emailProperties.success) {
+      logger.error("[DeliverAlert] Failed to parse email properties", {
+        issues: emailProperties.error.issues,
+        properties: alert.channel.properties,
+      });
+
+      return;
+    }
+
+    switch (alert.type) {
+      case "TASK_RUN_ATTEMPT": {
+        if (alert.taskRunAttempt) {
+          const taskRunError = TaskRunError.safeParse(alert.taskRunAttempt.error);
+
+          if (!taskRunError.success) {
+            logger.error("[DeliverAlert] Failed to parse task run error", {
+              issues: taskRunError.error.issues,
+              taskAttemptError: alert.taskRunAttempt.error,
+            });
+
+            return;
+          }
+
+          await sendAlertEmail({
+            email: "alert-attempt",
+            to: emailProperties.data.email,
+            taskIdentifier: alert.taskRunAttempt.taskRun.taskIdentifier,
+            fileName: alert.taskRunAttempt.backgroundWorkerTask.filePath,
+            exportName: alert.taskRunAttempt.backgroundWorkerTask.exportName,
+            version: alert.taskRunAttempt.backgroundWorker.version,
+            environment: alert.environment.slug,
+            error: createJsonErrorObject(taskRunError.data),
+            attemptLink: `${env.APP_ORIGIN}/projects/v3/${alert.project.externalRef}/runs/${alert.taskRunAttempt.taskRun.friendlyId}`,
+          });
+        } else {
+          logger.error("[DeliverAlert] Task run attempt not found", {
+            alert,
+          });
+        }
+
+        break;
+      }
+      case "DEPLOYMENT_FAILURE": {
+        if (alert.workerDeployment) {
+          const preparedError = DeploymentPresenter.prepareErrorData(
+            alert.workerDeployment.errorData
+          );
+
+          if (!preparedError) {
+            logger.error("[DeliverAlert] Failed to prepare deployment error data", {
+              errorData: alert.workerDeployment.errorData,
+            });
+
+            return;
+          }
+
+          await sendAlertEmail({
+            email: "alert-deployment-failure",
+            to: emailProperties.data.email,
+            version: alert.workerDeployment.version,
+            environment: alert.environment.slug,
+            shortCode: alert.workerDeployment.shortCode,
+            failedAt: alert.workerDeployment.failedAt ?? new Date(),
+            error: preparedError,
+            deploymentLink: `${env.APP_ORIGIN}/projects/v3/${alert.project.externalRef}/deployments/${alert.workerDeployment.shortCode}`,
+          });
+        } else {
+          logger.error("[DeliverAlert] Worker deployment not found", {
+            alert,
+          });
+        }
+
+        break;
+      }
+      case "DEPLOYMENT_SUCCESS": {
+        if (alert.workerDeployment) {
+          await sendAlertEmail({
+            email: "alert-deployment-success",
+            to: emailProperties.data.email,
+            version: alert.workerDeployment.version,
+            environment: alert.environment.slug,
+            shortCode: alert.workerDeployment.shortCode,
+            deployedAt: alert.workerDeployment.deployedAt ?? new Date(),
+            deploymentLink: `${env.APP_ORIGIN}/projects/v3/${alert.project.externalRef}/deployments/${alert.workerDeployment.shortCode}`,
+            taskCount: alert.workerDeployment.worker?.tasks.length ?? 0,
+          });
+        } else {
+          logger.error("[DeliverAlert] Worker deployment not found", {
+            alert,
+          });
+        }
+
+        break;
+      }
+      default: {
+        assertNever(alert.type);
+      }
+    }
+  }
+
+  async #sendWebhook(alert: FoundAlert) {
+    const webhookProperties = ProjectAlertWebhookProperties.safeParse(alert.channel.properties);
+
+    if (!webhookProperties.success) {
+      logger.error("[DeliverAlert] Failed to parse webhook properties", {
+        issues: webhookProperties.error.issues,
+        properties: alert.channel.properties,
+      });
+
+      return;
+    }
+
+    switch (alert.type) {
+      case "TASK_RUN_ATTEMPT": {
+        if (alert.taskRunAttempt) {
+          const taskRunError = TaskRunError.safeParse(alert.taskRunAttempt.error);
+
+          if (!taskRunError.success) {
+            logger.error("[DeliverAlert] Failed to parse task run error", {
+              issues: taskRunError.error.issues,
+              taskAttemptError: alert.taskRunAttempt.error,
+            });
+
+            return;
+          }
+
+          const error = createJsonErrorObject(taskRunError.data);
+
+          const payload = {
+            task: {
+              id: alert.taskRunAttempt.taskRun.taskIdentifier,
+              filePath: alert.taskRunAttempt.backgroundWorkerTask.filePath,
+              exportName: alert.taskRunAttempt.backgroundWorkerTask.exportName,
+            },
+            attempt: {
+              id: alert.taskRunAttempt.friendlyId,
+              number: alert.taskRunAttempt.number,
+              startedAt: alert.taskRunAttempt.startedAt,
+              status: alert.taskRunAttempt.status,
+            },
+            run: {
+              id: alert.taskRunAttempt.taskRun.friendlyId,
+              isTest: alert.taskRunAttempt.taskRun.isTest,
+              createdAt: alert.taskRunAttempt.taskRun.createdAt,
+              idempotencyKey: alert.taskRunAttempt.taskRun.idempotencyKey,
+            },
+            environment: {
+              id: alert.environment.id,
+              type: alert.environment.type,
+              slug: alert.environment.slug,
+            },
+            organization: {
+              id: alert.project.organizationId,
+              slug: alert.project.organization.slug,
+              name: alert.project.organization.title,
+            },
+            project: {
+              id: alert.project.id,
+              ref: alert.project.externalRef,
+              slug: alert.project.slug,
+              name: alert.project.name,
+            },
+            error,
+          };
+
+          await this.#deliverWebhook(payload, webhookProperties.data);
+        } else {
+          logger.error("[DeliverAlert] Task run attempt not found", {
+            alert,
+          });
+        }
+
+        break;
+      }
+      case "DEPLOYMENT_FAILURE": {
+        if (alert.workerDeployment) {
+          const preparedError = DeploymentPresenter.prepareErrorData(
+            alert.workerDeployment.errorData
+          );
+
+          if (!preparedError) {
+            logger.error("[DeliverAlert] Failed to prepare deployment error data", {
+              errorData: alert.workerDeployment.errorData,
+            });
+
+            return;
+          }
+
+          const payload = {
+            deployment: {
+              id: alert.workerDeployment.friendlyId,
+              status: alert.workerDeployment.status,
+              version: alert.workerDeployment.version,
+              shortCode: alert.workerDeployment.shortCode,
+              failedAt: alert.workerDeployment.failedAt ?? new Date(),
+            },
+            environment: {
+              id: alert.environment.id,
+              type: alert.environment.type,
+              slug: alert.environment.slug,
+            },
+            organization: {
+              id: alert.project.organizationId,
+              slug: alert.project.organization.slug,
+              name: alert.project.organization.title,
+            },
+            project: {
+              id: alert.project.id,
+              ref: alert.project.externalRef,
+              slug: alert.project.slug,
+              name: alert.project.name,
+            },
+            error: preparedError,
+          };
+
+          await this.#deliverWebhook(payload, webhookProperties.data);
+        } else {
+          logger.error("[DeliverAlert] Worker deployment not found", {
+            alert,
+          });
+        }
+
+        break;
+      }
+      case "DEPLOYMENT_SUCCESS": {
+        if (alert.workerDeployment) {
+          const payload = {
+            deployment: {
+              id: alert.workerDeployment.friendlyId,
+              status: alert.workerDeployment.status,
+              version: alert.workerDeployment.version,
+              shortCode: alert.workerDeployment.shortCode,
+              deployedAt: alert.workerDeployment.deployedAt ?? new Date(),
+            },
+            tasks:
+              alert.workerDeployment.worker?.tasks.map((task) => ({
+                id: task.slug,
+                filePath: task.filePath,
+                exportName: task.exportName,
+                triggerSource: task.triggerSource,
+              })) ?? [],
+            environment: {
+              id: alert.environment.id,
+              type: alert.environment.type,
+              slug: alert.environment.slug,
+            },
+            organization: {
+              id: alert.project.organizationId,
+              slug: alert.project.organization.slug,
+              name: alert.project.organization.title,
+            },
+            project: {
+              id: alert.project.id,
+              ref: alert.project.externalRef,
+              slug: alert.project.slug,
+              name: alert.project.name,
+            },
+          };
+
+          await this.#deliverWebhook(payload, webhookProperties.data);
+        } else {
+          logger.error("[DeliverAlert] Worker deployment not found", {
+            alert,
+          });
+        }
+
+        break;
+      }
+      default: {
+        assertNever(alert.type);
+      }
+    }
+  }
+
+  async #sendSlack(alert: FoundAlert) {
+    const slackProperties = ProjectAlertSlackProperties.safeParse(alert.channel.properties);
+
+    if (!slackProperties.success) {
+      logger.error("[DeliverAlert] Failed to parse slack properties", {
+        issues: slackProperties.error.issues,
+        properties: alert.channel.properties,
+      });
+
+      return;
+    }
+
+    // Get the org integration
+    const integration = slackProperties.data.integrationId
+      ? await this._prisma.organizationIntegration.findUnique({
+          where: {
+            id: slackProperties.data.integrationId,
+            organizationId: alert.project.organizationId,
+          },
+          include: {
+            tokenReference: true,
+          },
+        })
+      : await this._prisma.organizationIntegration.findFirst({
+          where: {
+            service: "SLACK",
+            organizationId: alert.project.organizationId,
+          },
+          orderBy: {
+            createdAt: "desc",
+          },
+          include: {
+            tokenReference: true,
+          },
+        });
+
+    if (!integration) {
+      logger.error("[DeliverAlert] Slack integration not found", {
+        alert,
+      });
+
+      return;
+    }
+
+    switch (alert.type) {
+      case "TASK_RUN_ATTEMPT": {
+        if (alert.taskRunAttempt) {
+          // Find existing storage by the run ID
+          const storage = await this._prisma.projectAlertStorage.findFirst({
+            where: {
+              alertChannelId: alert.channel.id,
+              alertType: alert.type,
+              storageId: alert.taskRunAttempt.taskRunId,
+            },
+          });
+
+          const storageData = storage
+            ? ProjectAlertSlackStorage.safeParse(storage.storageData)
+            : undefined;
+
+          const thread_ts =
+            storageData && storageData.success ? storageData.data.message_ts : undefined;
+
+          const taskRunError = TaskRunError.safeParse(alert.taskRunAttempt.error);
+
+          if (!taskRunError.success) {
+            logger.error("[DeliverAlert] Failed to parse task run error", {
+              issues: taskRunError.error.issues,
+              taskAttemptError: alert.taskRunAttempt.error,
+            });
+
+            return;
+          }
+
+          const error = createJsonErrorObject(taskRunError.data);
+
+          const exportName = alert.taskRunAttempt.backgroundWorkerTask.exportName;
+          const version = alert.taskRunAttempt.backgroundWorker.version;
+          const environment = alert.environment.slug;
+          const taskIdentifier = alert.taskRunAttempt.backgroundWorkerTask.slug;
+          const timestamp = alert.taskRunAttempt.completedAt ?? new Date();
+          const runId = alert.taskRunAttempt.taskRun.friendlyId;
+          const attemptNumber = alert.taskRunAttempt.number;
+
+          const message = await this.#postSlackMessage(integration, {
+            thread_ts,
+            channel: slackProperties.data.channelId,
+            text: `Task error in ${alert.taskRunAttempt.backgroundWorkerTask.exportName} [${alert.taskRunAttempt.backgroundWorker.version}.${alert.environment.slug}]`,
+            blocks: [
+              {
+                type: "section",
+                text: {
+                  type: "mrkdwn",
+                  text: `:rotating_light: Error in *${exportName}* _<!date^${Math.round(
+                    timestamp.getTime() / 1000
+                  )}^at {date_num} {time_secs}|${timestamp.toLocaleString()}>_`,
+                },
+              },
+              {
+                type: "section",
+                text: {
+                  type: "mrkdwn",
+                  text: `\`\`\`${error.stackTrace ?? error.message}\`\`\``,
+                },
+              },
+              {
+                type: "context",
+                elements: [
+                  {
+                    type: "mrkdwn",
+                    text: `${runId}.${attemptNumber} | ${taskIdentifier} | ${version}.${environment} | ${alert.project.name}`,
+                  },
+                ],
+              },
+              {
+                type: "divider",
+              },
+              {
+                type: "actions",
+                elements: [
+                  {
+                    type: "button",
+                    text: {
+                      type: "plain_text",
+                      text: "Investigate",
+                    },
+                    url: `${env.APP_ORIGIN}/projects/v3/${alert.project.externalRef}/runs/${alert.taskRunAttempt.taskRun.friendlyId}`,
+                  },
+                ],
+              },
+            ],
+          });
+
+          // Upsert the storage
+          if (message.ts) {
+            if (storage) {
+              await this._prisma.projectAlertStorage.update({
+                where: {
+                  id: storage.id,
+                },
+                data: {
+                  storageData: {
+                    message_ts: message.ts,
+                  },
+                },
+              });
+            } else {
+              await this._prisma.projectAlertStorage.create({
+                data: {
+                  alertChannelId: alert.channel.id,
+                  alertType: alert.type,
+                  storageId: alert.taskRunAttempt.taskRunId,
+                  storageData: {
+                    message_ts: message.ts,
+                  },
+                  projectId: alert.project.id,
+                },
+              });
+            }
+          }
+        } else {
+          logger.error("[DeliverAlert] Task run attempt not found", {
+            alert,
+          });
+        }
+
+        break;
+      }
+      case "DEPLOYMENT_FAILURE": {
+        if (alert.workerDeployment) {
+          const preparedError = DeploymentPresenter.prepareErrorData(
+            alert.workerDeployment.errorData
+          );
+
+          if (!preparedError) {
+            logger.error("[DeliverAlert] Failed to prepare deployment error data", {
+              errorData: alert.workerDeployment.errorData,
+            });
+
+            return;
+          }
+
+          const version = alert.workerDeployment.version;
+          const environment = alert.environment.slug;
+          const timestamp = alert.workerDeployment.failedAt ?? new Date();
+
+          await this.#postSlackMessage(integration, {
+            channel: slackProperties.data.channelId,
+            blocks: [
+              {
+                type: "section",
+                text: {
+                  type: "mrkdwn",
+                  text: `:rotating_light: Deployment failed *${version}.${environment}* _<!date^${Math.round(
+                    timestamp.getTime() / 1000
+                  )}^at {date_num} {time_secs}|${timestamp.toLocaleString()}>_`,
+                },
+              },
+              {
+                type: "section",
+                text: {
+                  type: "mrkdwn",
+                  text: `\`\`\`${preparedError.stack ?? preparedError.message}\`\`\``,
+                },
+              },
+              {
+                type: "context",
+                elements: [
+                  {
+                    type: "mrkdwn",
+                    text: `${alert.workerDeployment.shortCode} | ${version}.${environment} | ${alert.project.name}`,
+                  },
+                ],
+              },
+              {
+                type: "actions",
+                elements: [
+                  {
+                    type: "button",
+                    text: {
+                      type: "plain_text",
+                      text: "View Deployment",
+                    },
+                    url: `${env.APP_ORIGIN}/projects/v3/${alert.project.externalRef}/deployments/${alert.workerDeployment.shortCode}`,
+                  },
+                ],
+              },
+            ],
+          });
+        } else {
+          logger.error("[DeliverAlert] Worker deployment not found", {
+            alert,
+          });
+        }
+
+        break;
+      }
+      case "DEPLOYMENT_SUCCESS": {
+        if (alert.workerDeployment) {
+          const version = alert.workerDeployment.version;
+          const environment = alert.environment.slug;
+          const numberOfTasks = alert.workerDeployment.worker?.tasks.length ?? 0;
+          const timestamp = alert.workerDeployment.deployedAt ?? new Date();
+
+          await this.#postSlackMessage(integration, {
+            channel: slackProperties.data.channelId,
+            text: `Deployment ${alert.workerDeployment.version} [${alert.environment.slug}] succeeded`,
+            blocks: [
+              {
+                type: "section",
+                text: {
+                  type: "mrkdwn",
+                  text: `:rocket: Deployed *${version}.${environment}* successfully _<!date^${Math.round(
+                    timestamp.getTime() / 1000
+                  )}^at {date_num} {time_secs}|${timestamp.toLocaleString()}>_`,
+                },
+              },
+              {
+                type: "context",
+                elements: [
+                  {
+                    type: "mrkdwn",
+                    text: `${numberOfTasks} tasks | ${alert.workerDeployment.shortCode} | ${version}.${environment} | ${alert.project.name}`,
+                  },
+                ],
+              },
+              {
+                type: "actions",
+                elements: [
+                  {
+                    type: "button",
+                    text: {
+                      type: "plain_text",
+                      text: "View Deployment",
+                    },
+                    url: `${env.APP_ORIGIN}/projects/v3/${alert.project.externalRef}/deployments/${alert.workerDeployment.shortCode}`,
+                  },
+                ],
+              },
+            ],
+          });
+
+          return;
+        } else {
+          logger.error("[DeliverAlert] Worker deployment not found", {
+            alert,
+          });
+
+          return;
+        }
+      }
+    }
+  }
+
+  async #deliverWebhook(payload: any, webhook: ProjectAlertWebhookProperties) {
+    const rawPayload = JSON.stringify(payload);
+    const hashPayload = Buffer.from(rawPayload, "utf-8");
+
+    const secret = await decryptSecret(env.ENCRYPTION_KEY, webhook.secret);
+
+    const hmacSecret = Buffer.from(secret, "utf-8");
+    const key = await subtle.importKey(
+      "raw",
+      hmacSecret,
+      { name: "HMAC", hash: "SHA-256" },
+      false,
+      ["sign"]
+    );
+    const signature = await subtle.sign("HMAC", key, hashPayload);
+    const signatureHex = Buffer.from(signature).toString("hex");
+
+    // Send the webhook to the URL specified in webhook.url
+    const response = await fetch(webhook.url, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        "x-trigger-signature-hmacsha256": signatureHex,
+      },
+      body: rawPayload,
+      signal: AbortSignal.timeout(5000),
+    });
+
+    if (!response.ok) {
+      logger.error("[DeliverAlert] Failed to send alert webhook", {
+        status: response.status,
+        statusText: response.statusText,
+        url: webhook.url,
+        body: payload,
+        signature,
+      });
+
+      throw new Error(`Failed to send alert webhook to ${webhook.url}`);
+    }
+  }
+
+  async #postSlackMessage(
+    integration: OrganizationIntegrationForService<"SLACK">,
+    message: ChatPostMessageArguments
+  ) {
+    const client = await OrgIntegrationRepository.getAuthenticatedClientForIntegration(
+      integration,
+      { forceBotToken: true }
+    );
+
+    try {
+      return await client.chat.postMessage(message);
+    } catch (error) {
+      if (isWebAPIRateLimitedError(error)) {
+        logger.error("[DeliverAlert] Slack rate limited", {
+          error,
+          message,
+        });
+
+        throw new Error("Slack rate limited");
+      }
+
+      if (isWebAPIHTTPError(error)) {
+        logger.error("[DeliverAlert] Slack HTTP error", {
+          error,
+          message,
+        });
+
+        throw new Error("Slack HTTP error");
+      }
+
+      if (isWebAPIRequestError(error)) {
+        logger.error("[DeliverAlert] Slack request error", {
+          error,
+          message,
+        });
+
+        throw new Error("Slack request error");
+      }
+
+      if (isWebAPIPlatformError(error)) {
+        logger.error("[DeliverAlert] Slack platform error", {
+          error,
+          message,
+        });
+
+        throw new Error("Slack platform error");
+      }
+
+      logger.error("[DeliverAlert] Failed to send slack message", {
+        error,
+        message,
+      });
+
+      throw error;
+    }
+  }
+
+  static async enqueue(
+    alertId: string,
+    tx: PrismaClientOrTransaction,
+    options?: { runAt?: Date; queueName?: string }
+  ) {
+    return await workerQueue.enqueue(
+      "v3.deliverAlert",
+      {
+        alertId,
+      },
+      {
+        tx,
+        runAt: options?.runAt,
+        jobKey: `deliverAlert:${alertId}`,
+        queueName: options?.queueName,
+      }
+    );
+  }
+}
+
+function isWebAPIPlatformError(error: unknown): error is WebAPIPlatformError {
+  return (error as WebAPIPlatformError).code === ErrorCode.PlatformError;
+}
+
+function isWebAPIRequestError(error: unknown): error is WebAPIRequestError {
+  return (error as WebAPIRequestError).code === ErrorCode.RequestError;
+}
+
+function isWebAPIHTTPError(error: unknown): error is WebAPIHTTPError {
+  return (error as WebAPIHTTPError).code === ErrorCode.HTTPError;
+}
+
+function isWebAPIRateLimitedError(error: unknown): error is WebAPIRateLimitedError {
+  return (error as WebAPIRateLimitedError).code === ErrorCode.RateLimitedError;
+}

apps/webapp/app/v3/services/alerts/performDeploymentAlerts.server.ts

@@ -0,0 +1,80 @@
+import { ProjectAlertChannel, ProjectAlertType, WorkerDeployment } from "@trigger.dev/database";
+import { $transaction, PrismaClientOrTransaction } from "~/db.server";
+import { workerQueue } from "~/services/worker.server";
+import { generateFriendlyId } from "~/v3/friendlyIdentifiers";
+import { BaseService } from "../baseService.server";
+import { DeliverAlertService } from "./deliverAlert.server";
+
+export class PerformDeploymentAlertsService extends BaseService {
+  public async call(deploymentId: string) {
+    const deployment = await this._prisma.workerDeployment.findUnique({
+      where: { id: deploymentId },
+      include: {
+        environment: true,
+      },
+    });
+
+    if (!deployment) {
+      return;
+    }
+
+    const alertType =
+      deployment.status === "DEPLOYED" ? "DEPLOYMENT_SUCCESS" : "DEPLOYMENT_FAILURE";
+
+    // Find all the alert channels
+    const alertChannels = await this._prisma.projectAlertChannel.findMany({
+      where: {
+        projectId: deployment.projectId,
+        alertTypes: {
+          has: alertType,
+        },
+        environmentTypes: {
+          has: deployment.environment.type,
+        },
+        enabled: true,
+      },
+    });
+
+    for (const alertChannel of alertChannels) {
+      await this.#createAndSendAlert(alertChannel, deployment, alertType);
+    }
+  }
+
+  async #createAndSendAlert(
+    alertChannel: ProjectAlertChannel,
+    deployment: WorkerDeployment,
+    alertType: ProjectAlertType
+  ) {
+    await $transaction(this._prisma, async (tx) => {
+      const alert = await this._prisma.projectAlert.create({
+        data: {
+          friendlyId: generateFriendlyId("alert"),
+          channelId: alertChannel.id,
+          projectId: deployment.projectId,
+          environmentId: deployment.environmentId,
+          status: "PENDING",
+          type: alertType,
+          workerDeploymentId: deployment.id,
+        },
+      });
+
+      await DeliverAlertService.enqueue(alert.id, tx, {
+        queueName: `alert-channel:${alertChannel.id}`,
+      });
+    });
+  }
+
+  static async enqueue(deploymentId: string, tx: PrismaClientOrTransaction, runAt?: Date) {
+    return await workerQueue.enqueue(
+      "v3.performDeploymentAlerts",
+      {
+        deploymentId,
+      },
+      {
+        tx,
+        runAt,
+        jobKey: `performDeploymentAlerts:${deploymentId}`,
+      }
+    );
+  }
+}

apps/webapp/app/v3/services/alerts/performTaskAttemptAlerts.server.ts

@@ -0,0 +1,81 @@
+import { Prisma, ProjectAlertChannel } from "@trigger.dev/database";
+import { $transaction, PrismaClientOrTransaction, prisma } from "~/db.server";
+import { workerQueue } from "~/services/worker.server";
+import { generateFriendlyId } from "~/v3/friendlyIdentifiers";
+import { BaseService } from "../baseService.server";
+import { DeliverAlertService } from "./deliverAlert.server";
+
+type FoundTaskAttempt = Prisma.Result<
+  typeof prisma.taskRunAttempt,
+  { include: { taskRun: true; backgroundWorkerTask: true; runtimeEnvironment: true } },
+  "findUniqueOrThrow"
+>;
+
+export class PerformTaskAttemptAlertsService extends BaseService {
+  public async call(attemptId: string) {
+    const taskAttempt = await this._prisma.taskRunAttempt.findUnique({
+      where: { id: attemptId },
+      include: {
+        taskRun: true,
+        backgroundWorkerTask: true,
+        runtimeEnvironment: true,
+      },
+    });
+
+    if (!taskAttempt) {
+      return;
+    }
+
+    // Find all the alert channels
+    const alertChannels = await this._prisma.projectAlertChannel.findMany({
+      where: {
+        projectId: taskAttempt.taskRun.projectId,
+        alertTypes: {
+          has: "TASK_RUN_ATTEMPT",
+        },
+        environmentTypes: {
+          has: taskAttempt.runtimeEnvironment.type,
+        },
+        enabled: true,
+      },
+    });
+
+    for (const alertChannel of alertChannels) {
+      await this.#createAndSendAlert(alertChannel, taskAttempt);
+    }
+  }
+
+  async #createAndSendAlert(alertChannel: ProjectAlertChannel, taskAttempt: FoundTaskAttempt) {
+    await $transaction(this._prisma, async (tx) => {
+      const alert = await this._prisma.projectAlert.create({
+        data: {
+          friendlyId: generateFriendlyId("alert"),
+          channelId: alertChannel.id,
+          projectId: taskAttempt.taskRun.projectId,
+          environmentId: taskAttempt.runtimeEnvironmentId,
+          status: "PENDING",
+          type: "TASK_RUN_ATTEMPT",
+          taskRunAttemptId: taskAttempt.id,
+        },
+      });
+
+      await DeliverAlertService.enqueue(alert.id, tx, {
+        queueName: `alert-channel:${alertChannel.id}`,
+      });
+    });
+  }
+
+  static async enqueue(attemptId: string, tx: PrismaClientOrTransaction, runAt?: Date) {
+    return await workerQueue.enqueue(
+      "v3.performTaskAttemptAlerts",
+      {
+        attemptId,
+      },
+      {
+        tx,
+        runAt,
+        jobKey: `performTaskAttemptAlerts:${attemptId}`,
+      }
+    );
+  }
+}

apps/webapp/app/v3/services/bulk/createBulkAction.server.ts

@@ -0,0 +1,60 @@
+import { BulkActionType } from "@trigger.dev/database";
+import { bulkActionVerb } from "~/components/runs/v3/BulkAction";
+import { logger } from "~/services/logger.server";
+import { generateFriendlyId } from "../../friendlyIdentifiers";
+import { BaseService } from "../baseService.server";
+import { PerformBulkActionService } from "./performBulkAction.server";
+import { BULK_ACTION_RUN_LIMIT } from "~/consts";
+
+type BulkAction = {
+  projectId: string;
+  action: BulkActionType;
+  runIds: string[];
+};
+
+export class CreateBulkActionService extends BaseService {
+  public async call({ projectId, action, runIds }: BulkAction) {
+    const group = await this._prisma.bulkActionGroup.create({
+      data: {
+        friendlyId: generateFriendlyId("bulk"),
+        projectId,
+        type: action,
+      },
+    });
+
+    //limit to the first X runs
+    const passedTooManyRuns = runIds.length > BULK_ACTION_RUN_LIMIT;
+    runIds = runIds.slice(0, BULK_ACTION_RUN_LIMIT);
+
+    const items = await this._prisma.bulkActionItem.createMany({
+      data: runIds.map((runId) => ({
+        friendlyId: generateFriendlyId("bulkitem"),
+        type: action,
+        groupId: group.id,
+        sourceRunId: runId,
+      })),
+    });
+
+    logger.debug("Created bulk action group", {
+      groupId: group.id,
+      action,
+      runIds,
+    });
+
+    await PerformBulkActionService.enqueue(group.id, this._prisma);
+
+    let message = bulkActionVerb(action);
+    if (passedTooManyRuns) {
+      message += ` the first ${BULK_ACTION_RUN_LIMIT} runs`;
+    } else {
+      message += ` ${runIds.length} runs`;
+    }
+
+    return {
+      id: group.id,
+      friendlyId: group.friendlyId,
+      runCount: runIds.length,
+      message,
+    };
+  }
+}

apps/webapp/app/v3/services/bulk/performBulkAction.server.ts

@@ -0,0 +1,126 @@
+import assertNever from "assert-never";
+import { PrismaClientOrTransaction } from "~/db.server";
+import { workerQueue } from "~/services/worker.server";
+import { BaseService } from "../baseService.server";
+import { CancelTaskRunService } from "../cancelTaskRun.server";
+import { ReplayTaskRunService } from "../replayTaskRun.server";
+
+export class PerformBulkActionService extends BaseService {
+  public async performBulkActionItem(bulkActionItemId: string) {
+    const item = await this._prisma.bulkActionItem.findUnique({
+      where: { id: bulkActionItemId },
+      include: {
+        group: true,
+        sourceRun: true,
+        destinationRun: true,
+      },
+    });
+
+    if (!item) {
+      return;
+    }
+
+    if (item.status !== "PENDING") {
+      return;
+    }
+
+    switch (item.group.type) {
+      case "REPLAY": {
+        const service = new ReplayTaskRunService(this._prisma);
+        const result = await service.call(item.sourceRun);
+
+        await this._prisma.bulkActionItem.update({
+          where: { id: item.id },
+          data: {
+            destinationRunId: result?.id,
+            status: result ? "COMPLETED" : "FAILED",
+            error: result ? undefined : "Failed to replay task run",
+          },
+        });
+
+        break;
+      }
+      case "CANCEL": {
+        const service = new CancelTaskRunService(this._prisma);
+
+        const result = await service.call(item.sourceRun);
+
+        await this._prisma.bulkActionItem.update({
+          where: { id: item.id },
+          data: {
+            destinationRunId: item.sourceRun.id,
+            status: result ? "COMPLETED" : "FAILED",
+            error: result ? undefined : "Task wasn't cancelable",
+          },
+        });
+
+        break;
+      }
+      default: {
+        assertNever(item.group.type);
+      }
+    }
+
+    const groupItems = await this._prisma.bulkActionItem.findMany({
+      where: { groupId: item.groupId },
+      select: {
+        status: true,
+      },
+    });
+
+    const isGroupCompleted = groupItems.every((item) => item.status !== "PENDING");
+
+    if (isGroupCompleted) {
+      await this._prisma.bulkActionItem.update({
+        where: { id: item.id },
+        data: {
+          status: "COMPLETED",
+        },
+      });
+    }
+  }
+
+  public async enqueueBulkActionItem(bulkActionItemId: string, groupId: string) {
+    await workerQueue.enqueue(
+      "v3.performBulkActionItem",
+      {
+        bulkActionItemId,
+      },
+      {
+        jobKey: `performBulkActionItem:${bulkActionItemId}`,
+        queueName: `bulkActionItem:${groupId}`,
+      }
+    );
+  }
+
+  public async call(bulkActionGroupId: string) {
+    const actionGroup = await this._prisma.bulkActionGroup.findUnique({
+      include: {
+        items: true,
+      },
+      where: { id: bulkActionGroupId },
+    });
+
+    if (!actionGroup) {
+      return;
+    }
+
+    for (const item of actionGroup.items) {
+      await this.enqueueBulkActionItem(item.id, bulkActionGroupId);
+    }
+  }
+
+  static async enqueue(bulkActionGroupId: string, tx: PrismaClientOrTransaction, runAt?: Date) {
+    return await workerQueue.enqueue(
+      "v3.performBulkAction",
+      {
+        bulkActionGroupId,
+      },
+      {
+        tx,
+        runAt,
+        jobKey: `performBulkAction:${bulkActionGroupId}`,
+      }
+    );
+  }
+}

apps/webapp/app/v3/services/cancelTaskRun.server.ts

@@ -1,12 +1,12 @@
 import { Prisma, TaskRun, TaskRunAttemptStatus, TaskRunStatus } from "@trigger.dev/database";
-import { eventRepository } from "../eventRepository.server";
+import assertNever from "assert-never";
+import { logger } from "~/services/logger.server";
 import { marqs } from "~/v3/marqs/index.server";
+import { eventRepository } from "../eventRepository.server";
+import { socketIo } from "../handleSocketIo.server";
 import { devPubSub } from "../marqs/devPubSub.server";
 import { BaseService } from "./baseService.server";
-import { socketIo } from "../handleSocketIo.server";
 import { CancelAttemptService } from "./cancelAttempt.server";
-import { logger } from "~/services/logger.server";
-import assertNever from "assert-never";
 
 export const CANCELLABLE_STATUSES: Array<TaskRunStatus> = [
   "PENDING",
@@ -97,6 +97,10 @@ export class CancelTaskRunService extends BaseService {
     if (opts.cancelAttempts) {
       await this.#cancelPotentiallyRunningAttempts(cancelledTaskRun, cancelledTaskRun.attempts);
     }
+
+    return {
+      id: cancelledTaskRun.id,
+    };
   }
 
   async #cancelPotentiallyRunningAttempts(run: TaskRun, attempts: ExtendedTaskRunAttempt[]) {

apps/webapp/app/v3/services/completeAttempt.server.ts

@@ -19,6 +19,7 @@ import { ResumeTaskRunDependenciesService } from "./resumeTaskRunDependencies.se
 import { MAX_TASK_RUN_ATTEMPTS } from "~/consts";
 import { CreateCheckpointService } from "./createCheckpoint.server";
 import { TaskRun } from "@trigger.dev/database";
+import { PerformTaskAttemptAlertsService } from "./alerts/performTaskAttemptAlerts.server";
 import { RetryAttemptService } from "./retryAttempt.server";
 
 type FoundAttempt = Awaited<ReturnType<typeof findAttempt>>;
@@ -159,9 +160,13 @@ export class CompleteAttemptService extends BaseService {
       },
     });
 
-    if (completion.retry !== undefined && taskRunAttempt.number < MAX_TASK_RUN_ATTEMPTS) {
-      const environment = env ?? (await this.#getEnvironment(execution.environment.id));
+    const environment = env ?? (await this.#getEnvironment(execution.environment.id));
 
+    if (environment.type !== "DEVELOPMENT") {
+      await PerformTaskAttemptAlertsService.enqueue(taskRunAttempt.id, this._prisma);
+    }
+
+    if (completion.retry !== undefined && taskRunAttempt.number < MAX_TASK_RUN_ATTEMPTS) {
       const retryAt = new Date(completion.retry.timestamp);
 
       // Retry the task run

apps/webapp/app/v3/services/createDeployedBackgroundWorker.server.ts

@@ -9,6 +9,7 @@ import { projectPubSub } from "./projectPubSub.server";
 import { marqs } from "~/v3/marqs/index.server";
 import { logger } from "~/services/logger.server";
 import { ExecuteTasksWaitingForDeployService } from "./executeTasksWaitingForDeploy";
+import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server";
 
 export class CreateDeployedBackgroundWorkerService extends BaseService {
   public async call(
@@ -99,6 +100,7 @@ export class CreateDeployedBackgroundWorkerService extends BaseService {
       }
 
       await ExecuteTasksWaitingForDeployService.enqueue(backgroundWorker.id, this._prisma);
+      await PerformDeploymentAlertsService.enqueue(deployment.id, this._prisma);
 
       return backgroundWorker;
     });