Release 3.0.0-beta.47

chore: Update version for release (beta) (#1203 )
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
2024-07-09 12:25:02 +01:00 · 2024-07-09 12:24:09 +01:00 · 2024-07-09 10:27:35 +01:00 · 2024-07-09 09:25:08 +01:00 · 2024-07-08 19:33:15 +01:00 · 2024-07-08 18:48:39 +01:00
441 changed files with 238779 additions and 5277 deletions
@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Increase dev worker timeout
@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Add sox and audiowaveform binaries to worker images
@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+v3: recover from server rate limiting errors in a more reliable way
@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+v3: update @depot/cli to latest 0.0.1-cli.2.71.0
@@ -0,0 +1,30 @@
+---
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Tasks should now be much more robust and resilient to reconnects during crucial operations and other failure scenarios.
+
+Task runs now have to signal checkpointable state prior to ALL checkpoints. This ensures flushing always happens.
+
+All important socket.io RPCs will now be retried with backoff. Actions relying on checkpoints will be replayed if we haven't been checkpointed and restored as expected, e.g. after reconnect.
+
+Other changes:
+
+- Fix retry check in shared queue
+- Fix env var sync spinner
+- Heartbeat between retries
+- Fix retry prep
+- Fix prod worker no tasks detection
+- Fail runs above `MAX_TASK_RUN_ATTEMPTS`
+- Additional debug logs in all places
+- Prevent crashes due to failed socket schema parsing
+- Remove core-apps barrel
+- Upgrade socket.io-client to fix an ACK memleak
+- Additional index failure logs
+- Prevent message loss during reconnect
+- Prevent burst of heartbeats on reconnect
+- Prevent crash on failed cleanup
+- Handle at-least-once lazy execute message delivery
+- Handle uncaught entry point exceptions
@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"@trigger.dev/core": patch
+---
+
+v3: Trigger delayed runs and reschedule them
@@ -0,0 +1,9 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+- Improve non-zero exit code error messages
+- Detect OOM conditions within worker child processes
+- Internal errors can have optional stack traces
+- Docker provider can be set to enforce machine presets
@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Use global setTimeout to ensure cross-runtime support
@@ -0,0 +1,7 @@
+---
+"trigger.dev": patch
+---
+
+Add e2e fixtures corresponding to past issues
+Implement e2e suite parallelism
+Enhance log level for specific e2e suite messages
@@ -65,8 +65,10 @@
    "eight-pumas-float",
    "eleven-paws-join",
    "famous-boats-tease",
+    "fast-colts-relax",
    "few-students-share",
    "five-toes-destroy",
+    "friendly-walls-repair",
    "funny-swans-destroy",
    "gorgeous-gorillas-compete",
    "green-bags-wink",
@@ -80,17 +82,22 @@
    "late-steaks-behave",
    "lazy-files-lay",
    "lemon-jobs-repair",
+    "lemon-sloths-hide",
    "light-bulldogs-press",
    "light-dragons-complain",
    "little-crabs-cross",
    "long-fireants-search",
+    "long-hounds-wave",
    "loud-actors-remember",
    "lovely-drinks-flash",
    "many-ligers-pump",
    "mighty-camels-joke",
+    "mighty-eggs-grab",
    "mighty-flowers-train",
    "mighty-parrots-sin",
+    "modern-stingrays-end",
    "nasty-jars-pump",
+    "nervous-planets-sparkle",
    "new-pants-beg",
    "new-rivers-tell",
    "nice-bulldogs-turn",
@@ -99,9 +106,12 @@
    "pink-pumas-rhyme",
    "plenty-ducks-beam",
    "polite-ducks-switch",
+    "polite-pears-grow",
+    "polite-pots-walk",
    "polite-rockets-matter",
    "poor-flowers-cross",
    "purple-garlics-shop",
+    "purple-spiders-care",
    "rare-lamps-promise",
    "rare-roses-float",
    "real-planets-stare",
@@ -110,18 +120,26 @@
    "rotten-dryers-exercise",
    "rude-toys-compare",
    "selfish-ducks-sort",
+    "serious-hats-rest",
    "shaggy-spoons-taste",
+    "shaggy-weeks-live",
    "sharp-emus-compare",
    "sharp-zebras-serve",
    "shiny-coats-cry",
+    "silly-buses-obey",
+    "silly-forks-kiss",
    "silly-suits-switch",
    "silver-doors-juggle",
    "six-ligers-exist",
+    "six-rats-hunt",
    "sixty-insects-watch",
    "slow-buses-own",
+    "slow-kiwis-hide",
    "slow-sloths-retire",
    "smart-needles-move",
    "smart-olives-eat",
+    "sour-pugs-teach",
+    "spicy-frogs-remain",
    "spicy-lamps-smoke",
    "spicy-terms-bow",
    "strange-ghosts-matter",
@@ -131,13 +149,17 @@
    "strong-phones-smoke",
    "stupid-adults-sniff",
    "stupid-bulldogs-applaud",
+    "sweet-ducks-remember",
    "sweet-lizards-press",
    "swift-dragons-peel",
    "tall-bees-wave",
+    "tall-masks-repeat",
+    "tame-apricots-clap",
    "tame-guests-know",
    "tender-moose-tell",
    "tender-oranges-rhyme",
    "tender-turkeys-compete",
+    "thick-carrots-sneeze",
    "thin-parents-heal",
    "thirty-islands-kiss",
    "tidy-balloons-suffer",
@@ -148,7 +170,9 @@
    "tricky-bulldogs-heal",
    "tricky-keys-attack",
    "tricky-ladybugs-unite",
+    "twelve-knives-notice",
    "two-pumas-wait",
+    "violet-clocks-notice",
    "warm-olives-provide",
    "warm-planes-taste",
    "young-snails-sell"
@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Await file watcher cleanup in dev
@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+v3: [prod] force flush timeout should be 1s
@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+v3: sanitize errors with null unicode characters in some places
@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Add callback to checkpoint created message
@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+v3: Copy over more of the project's package.json keys into the deployed package.json (support for custom config like zenstack)
@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Improved ESM module require error detection logic
@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+v3: vercel edge runtime support
@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+v3: fix otel flushing causing CLEANUP ack timeout errors by always setting a forceFlushTimeoutMillis value
@@ -0,0 +1,10 @@
+---
+"trigger.dev": patch
+---
+
+- Prevent downgrades during update check and advise to upgrade CLI
+- Detect bun and use npm instead
+- During init, fail early and advise if not a TypeScript project
+- During init, allow specifying custom package manager args
+- Add links to dev worker started message
+- Fix links in unsupported terminals
@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+v3: Adding SDK functions for triggering tasks in a typesafe way, without importing task file
@@ -0,0 +1,9 @@
+---
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+- Fix artifact detection logs
+- Fix OOM detection and error messages
+- Add test link to cli deployment completion
@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+v3: postInstall config option now replaces the postinstall script found in package.json
@@ -0,0 +1,6 @@
+---
+"@trigger.dev/core": patch
+"@trigger.dev/sdk": patch
+---
+
+v3: Include presigned urls for downloading large payloads and outputs when using runs.retrieve
@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+v3: fix missing init output in task run function when no middleware is defined
@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix jsonc-parser import
@@ -71,7 +71,7 @@ COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl
 # OBJECT_STORE_BASE_URL="https://{bucket}.{accountId}.r2.cloudflarestorage.com"
 # OBJECT_STORE_ACCESS_KEY_ID=
 # OBJECT_STORE_SECRET_ACCESS_KEY=
-# RUNTIME_WAIT_THRESHOLD_IN_MS=10000
+# CHECKPOINT_THRESHOLD_IN_MS=10000

 # These control the server-side internal telemetry
 # INTERNAL_OTEL_TRACE_EXPORTER_URL=<URL to send traces to>
@@ -23,7 +23,7 @@ jobs:
          fetch-depth: 0

      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
        with:
          version: 8.15.5

@@ -63,7 +63,7 @@ jobs:
          submodules: recursive

      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
        with:
          version: 8.15.5

@@ -51,9 +51,9 @@ jobs:
          ref_without_tag=ghcr.io/triggerdotdev/trigger.dev
          image_tags=$ref_without_tag:${{ steps.get_version.outputs.version }}

-          # if it's a versioned tag, also tag it as latest
+          # if it's a versioned tag, also tag it as v3
          if [[ "${{ github.ref_name }}" == v.docker.* ]]; then
-            image_tags=$image_tags,$ref_without_tag:latest
+            image_tags=$image_tags,$ref_without_tag:v3
          fi
          
          echo "IMAGE_TAGS=${image_tags}" >> "$GITHUB_OUTPUT"
@@ -111,11 +111,11 @@ jobs:
          REPOSITORY: ${{ steps.prep.outputs.REPOSITORY }}
          IMAGE_TAG: ${{ steps.prep.outputs.IMAGE_TAG }}

-      - name: 🐙 Push 'latest' to GitHub Container Registry
+      - name: 🐙 Push 'v3' tag to GitHub Container Registry
        if: startsWith(github.ref_name, 'v.docker.')
        run: |
-          docker tag infra_image $REGISTRY/$REPOSITORY:latest
-          docker push $REGISTRY/$REPOSITORY:latest
+          docker tag infra_image $REGISTRY/$REPOSITORY:v3
+          docker push $REGISTRY/$REPOSITORY:v3
        env:
          REGISTRY: ghcr.io/triggerdotdev
          REPOSITORY: ${{ steps.prep.outputs.REPOSITORY }}
@@ -49,11 +49,11 @@ jobs:
    uses: ./.github/workflows/unit-tests.yml
    secrets: inherit

-  e2e:
-    uses: ./.github/workflows/e2e.yml
-    with:
-      package: cli-v3
-    secrets: inherit
+  # e2e:
+  #   uses: ./.github/workflows/e2e.yml
+  #   with:
+  #     package: cli-v3
+  #   secrets: inherit

  publish:
    needs: [typecheck, units]
@@ -28,7 +28,7 @@ jobs:
          fetch-depth: 0

      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
        with:
          version: 8.15.5

@@ -12,7 +12,7 @@ jobs:
          fetch-depth: 0

      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
        with:
          version: 8.15.5

@@ -12,7 +12,7 @@ jobs:
          fetch-depth: 0

      - name: ⎔ Setup pnpm
-        uses: pnpm/action-setup@v2.2.4
+        uses: pnpm/action-setup@v4
        with:
          version: 8.15.5

@@ -5,64 +5,71 @@
  <img alt="Trigger.dev logo" src="https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/a45d1fa2-0ae8-4a39-4409-f4f934bfae00/public">
 </picture>
  
-### The open source background jobs platform
+### Open source background jobs with no timeouts

 [Discord](https://trigger.dev/discord) | [Website](https://trigger.dev) | [Issues](https://github.com/triggerdotdev/trigger.dev/issues) | [Docs](https://trigger.dev/docs)

 [![Twitter](https://img.shields.io/twitter/url/https/twitter.com/triggerdotdev.svg?style=social&label=Follow%20%40trigger.dev)](https://twitter.com/triggerdotdev)
-[![GitHub Repo stars](https://img.shields.io/github/stars/triggerdotdev/trigger.dev?style=social)](https://github.com/triggerdotdev/trigger.dev)

 </div>

-> The Trigger.dev v3 developer preview is now open. For more information and to get early access, check out our [developer preview launch post](https://trigger.dev/blog/v3-developer-preview-launch/).
-
 ## About Trigger.dev

-Create long-running jobs directly in your codebase with features like API integrations, webhooks, scheduling and delays.
+Trigger.dev is an open source platform and SDK which allows you to create long-running background jobs with no timeouts. Write normal async code, deploy, and never hit a timeout.

-## Long running Jobs on serverless
+#### Features:

-Reliably run jobs and don’t worry about function timeouts, we handle those for you.
-
- Auto-resume after a function timeout
- Auto-resume after a server outage
- Add delays of up to a year
+- JavaScript and TypeScript SDK
+- Write reliable code by default
+- No infrastructure to manage
+- Works with your existing tech stack

 ## In your codebase

-Create Jobs where they belong: in your codebase. Version control, localhost, test, review, and deploy like you're already used to.
+Create tasks where they belong: in your codebase. Version control, localhost, test and review like you're already used to.

-## Secure by design
+```ts
+import { task } from "@trigger.dev/sdk/v3";

-We only receive Triggers and the data you choose to send to us. You can even completely self-host the entire platform.
+//1. You need to export each task
+export const helloWorld = task({
+  //2. Use a unique id for each task
+  id: "hello-world",
+  //3. The run function is the main function of the task
+  run: async (payload: { message: string }) => {
+    //4. You can write code that runs for a long time here, there are no timeouts
+    console.log(payload.message);
+  },
+});
+```

-## Don't worry about deployment
+## Deployment

-Just use our SDK to write Jobs in your codebase. There's nothing extra to deploy and no CI to configure, your Jobs just connect to our cloud. Or you can always self-host.
+Use our SDK to write tasks in your codebase. There's no infrastructure to manage, your tasks automatically scale and connect to our cloud. Or you can always [self-host](https://trigger.dev/docs/v3/open-source-self-hosting#overview).
+
+## Environments
+
+We support `Development`, `Staging`, and `Production` environments, allowing you to test your tasks before deploying them to production.

 ## Full visibility of every job run

-View every Task in every Run so you can tell exactly what happened.
+View every task in every run so you can tell exactly what happened. We provide a full trace view of every task run so you can see what happened at every step.

-![image](https://www.trigger.dev/build/_assets/web-app-2QFKXFLW.png)
-
-## Built-in integrations
-
-Easily integrate with hundreds of third-party APIs – including your own. Use API keys (which never leave your server) or let us handle OAuth for you. Install our integration packages and easily subscribe to webhooks and perform common tasks, or you can easily use your existing favorite Node.JS SDKs and get resumability and idempotency through our `runTask` function.
+![Trace view image](https://imagedelivery.net/3TbraffuDZ4aEf8KWOmI_w/7c1b347f-004c-4482-38a7-3f6fa9c00d00/public)

 # Getting started

-Visit our docs [here](https://trigger.dev/docs).
+Visit our docs [here](https://trigger.dev/docs/v3/introduction) for a full guide on how to get started with Trigger.dev.

 ## Self-host

-We provide an official trigger.dev docker image you can use to easily self-host the platform. We're working on more extensive guides but we currently provide a [Fly.io example repository](https://github.com/triggerdotdev/fly.io) with instructions in the README for deploying and using a self-hosted instance of Trigger.dev on Fly.io.
+If you prefer to self-host, you can follow our [self-hosting guide](https://trigger.dev/docs/v3/open-source-self-hosting#overview).

 ## Development

 To setup and develop locally or contribute to the open source project, follow our [development guide](./CONTRIBUTING.md).

-## Meet the Amazing People Behind This Project 🚀
+## Meet the Amazing People Behind This Project:

 <a href="https://github.com/triggerdotdev/trigger.dev/graphs/contributors">
  <img src="https://contrib.rocks/image?repo=triggerdotdev/trigger.dev" />
@@ -21,8 +21,7 @@
    "execa": "^8.0.1",
    "nanoid": "^5.0.6",
    "prom-client": "^15.1.0",
-    "socket.io": "4.7.4",
-    "socket.io-client": "4.7.4"
+    "socket.io": "4.7.4"
  },
  "devDependencies": {
    "@types/node": "^18",
@@ -0,0 +1,95 @@
+import type { Execa$ } from "execa";
+import { setTimeout as timeout } from "node:timers/promises";
+
+class ChaosMonkeyError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ChaosMonkeyError";
+  }
+}
+
+export class ChaosMonkey {
+  private chaosEventRate = 0.2;
+  private delayInSeconds = 45;
+
+  constructor(private enabled = false) {
+    if (this.enabled) {
+      console.log("🍌 Chaos monkey enabled");
+    }
+  }
+
+  static Error = ChaosMonkeyError;
+
+  enable() {
+    this.enabled = true;
+    console.log("🍌 Chaos monkey enabled");
+  }
+
+  disable() {
+    this.enabled = false;
+    console.log("🍌 Chaos monkey disabled");
+  }
+
+  async call({
+    $,
+    throwErrors = true,
+    addDelays = true,
+  }: {
+    $?: Execa$<string>;
+    throwErrors?: boolean;
+    addDelays?: boolean;
+  } = {}) {
+    if (!this.enabled) {
+      return;
+    }
+
+    const random = Math.random();
+
+    if (random > this.chaosEventRate) {
+      // Don't interfere with normal operation
+      return;
+    }
+
+    const chaosEvents: Array<() => Promise<any>> = [];
+
+    if (addDelays) {
+      chaosEvents.push(async () => {
+        console.log("🍌 Chaos monkey: Add delay");
+
+        if ($) {
+          await $`sleep ${this.delayInSeconds}`;
+        } else {
+          await timeout(this.delayInSeconds * 1000);
+        }
+      });
+    }
+
+    if (throwErrors) {
+      chaosEvents.push(async () => {
+        console.log("🍌 Chaos monkey: Throw error");
+
+        if ($) {
+          await $`false`;
+        } else {
+          throw new ChaosMonkey.Error("🍌 Chaos monkey: Throw error");
+        }
+      });
+    }
+
+    if (chaosEvents.length === 0) {
+      console.error("🍌 Chaos monkey: No events selected");
+      return;
+    }
+
+    const randomIndex = Math.floor(Math.random() * chaosEvents.length);
+
+    const chaosEvent = chaosEvents[randomIndex];
+
+    if (!chaosEvent) {
+      console.error("🍌 Chaos monkey: No event found");
+      return;
+    }
+
+    await chaosEvent();
+  }
+}
@@ -0,0 +1,587 @@
+import { ExponentialBackoff } from "@trigger.dev/core-apps/backoff";
+import { isExecaChildProcess, testDockerCheckpoint } from "@trigger.dev/core-apps/checkpoints";
+import { SimpleLogger } from "@trigger.dev/core-apps/logger";
+import { $ } from "execa";
+import { nanoid } from "nanoid";
+import fs from "node:fs/promises";
+import { ChaosMonkey } from "./chaosMonkey";
+
+type CheckpointerInitializeReturn = {
+  canCheckpoint: boolean;
+  willSimulate: boolean;
+};
+
+type CheckpointAndPushOptions = {
+  runId: string;
+  leaveRunning?: boolean;
+  projectRef: string;
+  deploymentVersion: string;
+  shouldHeartbeat?: boolean;
+};
+
+type CheckpointAndPushResult =
+  | { success: true; checkpoint: CheckpointData }
+  | {
+      success: false;
+      reason?: "CANCELED" | "DISABLED" | "ERROR" | "IN_PROGRESS" | "NO_SUPPORT" | "SKIP_RETRYING";
+    };
+
+type CheckpointData = {
+  location: string;
+  docker: boolean;
+};
+
+type CheckpointerOptions = {
+  dockerMode: boolean;
+  forceSimulate: boolean;
+  heartbeat: (runId: string) => void;
+  registryHost?: string;
+  registryNamespace?: string;
+  registryTlsVerify?: boolean;
+  disableCheckpointSupport?: boolean;
+  checkpointPath?: string;
+  simulateCheckpointFailure?: boolean;
+  simulateCheckpointFailureSeconds?: number;
+  simulatePushFailure?: boolean;
+  simulatePushFailureSeconds?: number;
+  chaosMonkey?: ChaosMonkey;
+};
+
+async function getFileSize(filePath: string): Promise<number> {
+  try {
+    const stats = await fs.stat(filePath);
+    return stats.size;
+  } catch (error) {
+    console.error("Error getting file size:", error);
+    return -1;
+  }
+}
+
+async function getParsedFileSize(filePath: string) {
+  const sizeInBytes = await getFileSize(filePath);
+
+  let message = `Size in bytes: ${sizeInBytes}`;
+
+  if (sizeInBytes > 1024 * 1024) {
+    const sizeInMB = (sizeInBytes / 1024 / 1024).toFixed(2);
+    message = `Size in MB (rounded): ${sizeInMB}`;
+  } else if (sizeInBytes > 1024) {
+    const sizeInKB = (sizeInBytes / 1024).toFixed(2);
+    message = `Size in KB (rounded): ${sizeInKB}`;
+  }
+
+  return {
+    path: filePath,
+    sizeInBytes,
+    message,
+  };
+}
+
+export class Checkpointer {
+  #initialized = false;
+  #canCheckpoint = false;
+  #dockerMode: boolean;
+
+  #logger = new SimpleLogger("[checkptr]");
+  #abortControllers = new Map<string, AbortController>();
+  #failedCheckpoints = new Map<string, unknown>();
+  #waitingForRetry = new Set<string>();
+
+  private registryHost: string;
+  private registryNamespace: string;
+  private registryTlsVerify: boolean;
+
+  private disableCheckpointSupport: boolean;
+  private checkpointPath: string;
+
+  private simulateCheckpointFailure: boolean;
+  private simulateCheckpointFailureSeconds: number;
+  private simulatePushFailure: boolean;
+  private simulatePushFailureSeconds: number;
+
+  private chaosMonkey: ChaosMonkey;
+
+  constructor(private opts: CheckpointerOptions) {
+    this.#dockerMode = opts.dockerMode;
+
+    this.registryHost = opts.registryHost ?? "localhost:5000";
+    this.registryNamespace = opts.registryNamespace ?? "trigger";
+    this.registryTlsVerify = opts.registryTlsVerify ?? true;
+
+    this.disableCheckpointSupport = opts.disableCheckpointSupport ?? false;
+    this.checkpointPath = opts.checkpointPath ?? "/checkpoints";
+
+    this.simulateCheckpointFailure = opts.simulateCheckpointFailure ?? false;
+    this.simulateCheckpointFailureSeconds = opts.simulateCheckpointFailureSeconds ?? 300;
+    this.simulatePushFailure = opts.simulatePushFailure ?? false;
+    this.simulatePushFailureSeconds = opts.simulatePushFailureSeconds ?? 300;
+
+    this.chaosMonkey = opts.chaosMonkey ?? new ChaosMonkey(!!process.env.CHAOS_MONKEY_ENABLED);
+  }
+
+  async init(): Promise<CheckpointerInitializeReturn> {
+    if (this.#initialized) {
+      return this.#getInitReturn(this.#canCheckpoint);
+    }
+
+    this.#logger.log(`${this.#dockerMode ? "Docker" : "Kubernetes"} mode`);
+
+    if (this.#dockerMode) {
+      const testCheckpoint = await testDockerCheckpoint();
+
+      if (testCheckpoint.ok) {
+        return this.#getInitReturn(true);
+      }
+
+      this.#logger.error(testCheckpoint.message, testCheckpoint.error ?? "");
+      return this.#getInitReturn(false);
+    } else {
+      try {
+        await $`buildah login --get-login ${this.registryHost}`;
+      } catch (error) {
+        this.#logger.error(`No checkpoint support: Not logged in to registry ${this.registryHost}`);
+        return this.#getInitReturn(false);
+      }
+    }
+
+    return this.#getInitReturn(true);
+  }
+
+  #getInitReturn(canCheckpoint: boolean): CheckpointerInitializeReturn {
+    this.#canCheckpoint = canCheckpoint;
+
+    if (canCheckpoint) {
+      if (!this.#initialized) {
+        this.#logger.log("Full checkpoint support!");
+      }
+    }
+
+    this.#initialized = true;
+
+    const willSimulate = this.#dockerMode && (!this.#canCheckpoint || this.opts.forceSimulate);
+
+    if (willSimulate) {
+      this.#logger.log("Simulation mode enabled. Containers will be paused, not checkpointed.", {
+        forceSimulate: this.opts.forceSimulate,
+      });
+    }
+
+    return {
+      canCheckpoint,
+      willSimulate,
+    };
+  }
+
+  #getImageRef(projectRef: string, deploymentVersion: string, shortCode: string) {
+    return `${this.registryHost}/${this.registryNamespace}/${projectRef}:${deploymentVersion}.prod-${shortCode}`;
+  }
+
+  #getExportLocation(projectRef: string, deploymentVersion: string, shortCode: string) {
+    const basename = `${projectRef}-${deploymentVersion}-${shortCode}`;
+
+    if (this.#dockerMode) {
+      return basename;
+    } else {
+      return `${this.checkpointPath}/${basename}.tar`;
+    }
+  }
+
+  async checkpointAndPush(opts: CheckpointAndPushOptions): Promise<CheckpointData | undefined> {
+    const start = performance.now();
+    this.#logger.log(`checkpointAndPush() start`, { start, opts });
+
+    let interval: NodeJS.Timer | undefined;
+
+    if (opts.shouldHeartbeat) {
+      interval = setInterval(() => {
+        this.#logger.log("Sending heartbeat", { runId: opts.runId });
+        this.opts.heartbeat(opts.runId);
+      }, 20_000);
+    }
+
+    try {
+      const result = await this.#checkpointAndPushWithBackoff(opts);
+
+      const end = performance.now();
+      this.#logger.log(`checkpointAndPush() end`, {
+        start,
+        end,
+        diff: end - start,
+        opts,
+        success: result.success,
+      });
+
+      if (!result.success) {
+        return;
+      }
+
+      return result.checkpoint;
+    } finally {
+      if (opts.shouldHeartbeat) {
+        clearInterval(interval);
+      }
+    }
+  }
+
+  isCheckpointing(runId: string) {
+    return this.#abortControllers.has(runId) || this.#waitingForRetry.has(runId);
+  }
+
+  cancelCheckpoint(runId: string): boolean {
+    // If the last checkpoint failed, pretend we canceled it
+    // This ensures tasks don't wait for external resume messages to continue
+    if (this.#hasFailedCheckpoint(runId)) {
+      this.#clearFailedCheckpoint(runId);
+      return true;
+    }
+
+    if (this.#waitingForRetry.has(runId)) {
+      this.#waitingForRetry.delete(runId);
+      return true;
+    }
+
+    const controller = this.#abortControllers.get(runId);
+
+    if (!controller) {
+      this.#logger.debug("Nothing to cancel", { runId });
+      return false;
+    }
+
+    controller.abort("cancelCheckpointing()");
+    this.#abortControllers.delete(runId);
+
+    return true;
+  }
+
+  async #checkpointAndPushWithBackoff({
+    runId,
+    leaveRunning = true, // This mirrors kubernetes behaviour more accurately
+    projectRef,
+    deploymentVersion,
+  }: CheckpointAndPushOptions): Promise<CheckpointAndPushResult> {
+    this.#logger.log("Checkpointing with backoff", {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+    });
+
+    const backoff = new ExponentialBackoff()
+      .type("EqualJitter")
+      .base(3)
+      .max(3 * 3600)
+      .maxElapsed(48 * 3600);
+
+    for await (const { delay, retry } of backoff) {
+      try {
+        if (retry > 0) {
+          this.#logger.error("Retrying checkpoint", {
+            runId,
+            retry,
+            delay,
+          });
+
+          this.#waitingForRetry.add(runId);
+          await new Promise((resolve) => setTimeout(resolve, delay.milliseconds));
+
+          if (!this.#waitingForRetry.has(runId)) {
+            this.#logger.log("Checkpoint canceled while waiting for retry", { runId });
+            return { success: false, reason: "CANCELED" };
+          } else {
+            this.#waitingForRetry.delete(runId);
+          }
+        }
+
+        const result = await this.#checkpointAndPush({
+          runId,
+          leaveRunning,
+          projectRef,
+          deploymentVersion,
+        });
+
+        if (result.success) {
+          return result;
+        }
+
+        if (result.reason === "CANCELED") {
+          this.#logger.log("Checkpoint canceled, won't retry", { runId });
+          // Don't fail the checkpoint, as it was canceled
+          return result;
+        }
+
+        if (result.reason === "IN_PROGRESS") {
+          this.#logger.log("Checkpoint already in progress, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        if (result.reason === "NO_SUPPORT") {
+          this.#logger.log("No checkpoint support, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        if (result.reason === "DISABLED") {
+          this.#logger.log("Checkpoint support disabled, won't retry", { runId });
+          this.#failCheckpoint(runId, result.reason);
+          return result;
+        }
+
+        if (result.reason === "SKIP_RETRYING") {
+          this.#logger.log("Skipping retrying", { runId });
+          return result;
+        }
+
+        continue;
+      } catch (error) {
+        this.#logger.error("Checkpoint error", {
+          retry,
+          runId,
+          delay,
+          error: error instanceof Error ? error.message : error,
+        });
+      }
+    }
+
+    this.#logger.error(`Checkpoint failed after exponential backoff`, {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+    });
+    this.#failCheckpoint(runId, "ERROR");
+
+    return { success: false, reason: "ERROR" };
+  }
+
+  async #checkpointAndPush({
+    runId,
+    leaveRunning = true, // This mirrors kubernetes behaviour more accurately
+    projectRef,
+    deploymentVersion,
+  }: CheckpointAndPushOptions): Promise<CheckpointAndPushResult> {
+    await this.init();
+
+    const options = {
+      runId,
+      leaveRunning,
+      projectRef,
+      deploymentVersion,
+    };
+
+    if (!this.#dockerMode && !this.#canCheckpoint) {
+      this.#logger.error("No checkpoint support. Simulation requires docker.");
+      return { success: false, reason: "NO_SUPPORT" };
+    }
+
+    if (this.isCheckpointing(runId)) {
+      this.#logger.error("Checkpoint procedure already in progress", { options });
+      return { success: false, reason: "IN_PROGRESS" };
+    }
+
+    // This is a new checkpoint, clear any last failure for this run
+    this.#clearFailedCheckpoint(runId);
+
+    if (this.disableCheckpointSupport) {
+      this.#logger.error("Checkpoint support disabled", { options });
+      return { success: false, reason: "DISABLED" };
+    }
+
+    const controller = new AbortController();
+    this.#abortControllers.set(runId, controller);
+
+    const $$ = $({ signal: controller.signal });
+
+    const shortCode = nanoid(8);
+    const imageRef = this.#getImageRef(projectRef, deploymentVersion, shortCode);
+    const exportLocation = this.#getExportLocation(projectRef, deploymentVersion, shortCode);
+
+    const cleanup = async () => {
+      if (this.#dockerMode) {
+        return;
+      }
+
+      try {
+        await $`rm ${exportLocation}`;
+        this.#logger.log("Deleted checkpoint archive", { exportLocation });
+
+        await $`buildah rmi ${imageRef}`;
+        this.#logger.log("Deleted checkpoint image", { imageRef });
+      } catch (error) {
+        this.#logger.error("Failure during checkpoint cleanup", { exportLocation, error });
+      }
+    };
+
+    try {
+      await this.chaosMonkey.call({ $: $$ });
+
+      this.#logger.log("Checkpointing:", { options });
+
+      const containterName = this.#getRunContainerName(runId);
+
+      // Create checkpoint (docker)
+      if (this.#dockerMode) {
+        try {
+          if (this.opts.forceSimulate || !this.#canCheckpoint) {
+            this.#logger.log("Simulating checkpoint");
+            this.#logger.debug(await $$`docker pause ${containterName}`);
+          } else {
+            if (this.simulateCheckpointFailure) {
+              if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) {
+                this.#logger.error("Simulating checkpoint failure", { options });
+                throw new Error("SIMULATE_CHECKPOINT_FAILURE");
+              }
+            }
+
+            if (leaveRunning) {
+              this.#logger.debug(
+                await $$`docker checkpoint create --leave-running ${containterName} ${exportLocation}`
+              );
+            } else {
+              this.#logger.debug(
+                await $$`docker checkpoint create ${containterName} ${exportLocation}`
+              );
+            }
+          }
+        } catch (error) {
+          this.#logger.error("Failed while creating docker checkpoint", { exportLocation });
+          throw error;
+        }
+
+        this.#logger.log("checkpoint created:", {
+          runId,
+          location: exportLocation,
+        });
+
+        return {
+          success: true,
+          checkpoint: {
+            location: exportLocation,
+            docker: true,
+          },
+        };
+      }
+
+      // Create checkpoint (CRI)
+      if (!this.#canCheckpoint) {
+        this.#logger.error("No checkpoint support in kubernetes mode.");
+        return { success: false, reason: "SKIP_RETRYING" };
+      }
+
+      const containerId = this.#logger.debug(
+        // @ts-expect-error
+        await $$`crictl ps`
+          .pipeStdout($$({ stdin: "pipe" })`grep ${containterName}`)
+          .pipeStdout($$({ stdin: "pipe" })`cut -f1 ${"-d "}`)
+      );
+
+      if (!containerId.stdout) {
+        this.#logger.error("could not find container id", { options, containterName });
+        return { success: false, reason: "SKIP_RETRYING" };
+      }
+
+      const start = performance.now();
+
+      if (this.simulateCheckpointFailure) {
+        if (performance.now() < this.simulateCheckpointFailureSeconds * 1000) {
+          this.#logger.error("Simulating checkpoint failure", { options });
+          throw new Error("SIMULATE_CHECKPOINT_FAILURE");
+        }
+      }
+
+      // Create checkpoint
+      this.#logger.debug(await $$`crictl checkpoint --export=${exportLocation} ${containerId}`);
+      const postCheckpoint = performance.now();
+
+      // Print checkpoint size
+      const size = await getParsedFileSize(exportLocation);
+      this.#logger.log("checkpoint archive created", { size, options });
+
+      // Create image from checkpoint
+      const container = this.#logger.debug(await $$`buildah from scratch`);
+      const postFrom = performance.now();
+
+      this.#logger.debug(await $$`buildah add ${container} ${exportLocation} /`);
+      const postAdd = performance.now();
+
+      this.#logger.debug(
+        await $$`buildah config --annotation=io.kubernetes.cri-o.annotations.checkpoint.name=counter ${container}`
+      );
+      const postConfig = performance.now();
+
+      this.#logger.debug(await $$`buildah commit ${container} ${imageRef}`);
+      const postCommit = performance.now();
+
+      this.#logger.debug(await $$`buildah rm ${container}`);
+      const postRm = performance.now();
+
+      if (this.simulatePushFailure) {
+        if (performance.now() < this.simulatePushFailureSeconds * 1000) {
+          this.#logger.error("Simulating push failure", { options });
+          throw new Error("SIMULATE_PUSH_FAILURE");
+        }
+      }
+
+      // Push checkpoint image
+      this.#logger.debug(
+        await $$`buildah push --tls-verify=${String(this.registryTlsVerify)} ${imageRef}`
+      );
+      const postPush = performance.now();
+
+      const perf = {
+        "crictl checkpoint": postCheckpoint - start,
+        "buildah from": postFrom - postCheckpoint,
+        "buildah add": postAdd - postFrom,
+        "buildah config": postConfig - postAdd,
+        "buildah commit": postCommit - postConfig,
+        "buildah rm": postRm - postCommit,
+        "buildah push": postPush - postRm,
+      };
+
+      this.#logger.log("Checkpointed and pushed image to:", { location: imageRef, perf });
+
+      return {
+        success: true,
+        checkpoint: {
+          location: imageRef,
+          docker: false,
+        },
+      };
+    } catch (error) {
+      if (isExecaChildProcess(error)) {
+        if (error.isCanceled) {
+          this.#logger.error("Checkpoint canceled", { options, error });
+
+          return { success: false, reason: "CANCELED" };
+        }
+
+        this.#logger.error("Checkpoint command error", { options, error });
+
+        return { success: false, reason: "ERROR" };
+      }
+
+      this.#logger.error("Unhandled checkpoint error", { options, error });
+
+      return { success: false, reason: "ERROR" };
+    } finally {
+      this.#abortControllers.delete(runId);
+      await cleanup();
+    }
+  }
+
+  #failCheckpoint(runId: string, error: unknown) {
+    this.#failedCheckpoints.set(runId, error);
+  }
+
+  #clearFailedCheckpoint(runId: string) {
+    this.#failedCheckpoints.delete(runId);
+  }
+
+  #hasFailedCheckpoint(runId: string) {
+    return this.#failedCheckpoints.has(runId);
+  }
+
+  #getRunContainerName(suffix: string) {
+    return `task-run-${suffix}`;
+  }
+}
@@ -18,8 +18,7 @@
  "dependencies": {
    "@trigger.dev/core": "workspace:*",
    "@trigger.dev/core-apps": "workspace:*",
-    "execa": "^8.0.1",
-    "socket.io-client": "^4.7.4"
+    "execa": "^8.0.1"
  },
  "devDependencies": {
    "@types/node": "^18.19.8",
@@ -1,14 +1,13 @@
 import { $, type ExecaChildProcess, execa } from "execa";
 import {
-  SimpleLogger,
-  TaskOperations,
  ProviderShell,
-  TaskOperationsRestoreOptions,
+  TaskOperations,
  TaskOperationsCreateOptions,
  TaskOperationsIndexOptions,
-  isExecaChildProcess,
-  testDockerCheckpoint,
-} from "@trigger.dev/core-apps";
+  TaskOperationsRestoreOptions,
+} from "@trigger.dev/core-apps/provider";
+import { SimpleLogger } from "@trigger.dev/core-apps/logger";
+import { isExecaChildProcess, testDockerCheckpoint } from "@trigger.dev/core-apps/checkpoints";
 import { setTimeout } from "node:timers/promises";
 import { PostStartCauses, PreStopCauses } from "@trigger.dev/core/v3";

@@ -54,13 +53,16 @@ class DockerTaskOperations implements TaskOperations {
  }

  #getInitReturn(canCheckpoint: boolean): TaskOperationsInitReturn {
-    this.#initialized = true;
    this.#canCheckpoint = canCheckpoint;

    if (canCheckpoint) {
-      logger.log("Full checkpoint support!");
+      if (!this.#initialized) {
+        logger.log("Full checkpoint support!");
+      }
    }

+    this.#initialized = true;
+
    const willSimulate = !canCheckpoint || this.opts.forceSimulate;

    if (willSimulate) {
@@ -109,22 +111,27 @@ class DockerTaskOperations implements TaskOperations {

    const containerName = this.#getRunContainerName(opts.runId);

+    const runArgs = [
+      "run",
+      "--network=host",
+      "--detach",
+      `--env=TRIGGER_ENV_ID=${opts.envId}`,
+      `--env=TRIGGER_RUN_ID=${opts.runId}`,
+      `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
+      `--env=POD_NAME=${containerName}`,
+      `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
+      `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
+      `--name=${containerName}`,
+    ];
+
+    if (process.env.ENFORCE_MACHINE_PRESETS) {
+      runArgs.push(`--cpus=${opts.machine.cpu}`, `--memory=${opts.machine.memory}G`);
+    }
+
+    runArgs.push(`${opts.image}`);
+
    try {
-      logger.debug(
-        await execa("docker", [
-          "run",
-          "--network=host",
-          "--detach",
-          `--env=TRIGGER_ENV_ID=${opts.envId}`,
-          `--env=TRIGGER_RUN_ID=${opts.runId}`,
-          `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
-          `--env=POD_NAME=${containerName}`,
-          `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
-          `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
-          `--name=${containerName}`,
-          `${opts.image}`,
-        ])
-      );
+      logger.debug(await execa("docker", runArgs));
    } catch (error) {
      if (!isExecaChildProcess(error)) {
        throw error;
@@ -19,8 +19,7 @@
    "@kubernetes/client-node": "^0.20.0",
    "@trigger.dev/core": "workspace:*",
    "@trigger.dev/core-apps": "workspace:*",
-    "p-queue": "^8.0.1",
-    "socket.io-client": "^4.7.4"
+    "p-queue": "^8.0.1"
  },
  "devDependencies": {
    "dotenv": "^16.4.2",
@@ -1,12 +1,12 @@
 import * as k8s from "@kubernetes/client-node";
 import {
  ProviderShell,
-  SimpleLogger,
  TaskOperations,
  TaskOperationsCreateOptions,
  TaskOperationsIndexOptions,
  TaskOperationsRestoreOptions,
-} from "@trigger.dev/core-apps";
+} from "@trigger.dev/core-apps/provider";
+import { SimpleLogger } from "@trigger.dev/core-apps/logger";
 import {
  MachinePreset,
  PostStartCauses,
@@ -16,13 +16,21 @@ import {
 import { randomUUID } from "crypto";
 import { TaskMonitor } from "./taskMonitor";
 import { PodCleaner } from "./podCleaner";
+import { UptimeHeartbeat } from "./uptimeHeartbeat";

 const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
 const NODE_NAME = process.env.NODE_NAME || "local";
 const OTEL_EXPORTER_OTLP_ENDPOINT =
  process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318";
+
 const POD_CLEANER_INTERVAL_SECONDS = Number(process.env.POD_CLEANER_INTERVAL_SECONDS || "300");

+const UPTIME_HEARTBEAT_URL = process.env.UPTIME_HEARTBEAT_URL;
+const UPTIME_INTERVAL_SECONDS = Number(process.env.UPTIME_INTERVAL_SECONDS || "60");
+const UPTIME_MAX_PENDING_RUNS = Number(process.env.UPTIME_MAX_PENDING_RUNS || "25");
+const UPTIME_MAX_PENDING_INDECES = Number(process.env.UPTIME_MAX_PENDING_INDECES || "10");
+const UPTIME_MAX_PENDING_ERRORS = Number(process.env.UPTIME_MAX_PENDING_ERRORS || "10");
+
 const logger = new SimpleLogger(`[${NODE_NAME}]`);
 logger.log(`running in ${RUNTIME_ENV} mode`);

@@ -529,27 +537,28 @@ provider.listen();

 const taskMonitor = new TaskMonitor({
  runtimeEnv: RUNTIME_ENV,
-  onIndexFailure: async (deploymentId, failureInfo) => {
-    logger.log("Indexing failed", { deploymentId, failureInfo });
+  onIndexFailure: async (deploymentId, details) => {
+    logger.log("Indexing failed", { deploymentId, details });

    try {
      provider.platformSocket.send("INDEXING_FAILED", {
        deploymentId,
        error: {
-          name: `Crashed with exit code ${failureInfo.exitCode}`,
-          message: failureInfo.reason,
-          stack: failureInfo.logs,
+          name: `Crashed with exit code ${details.exitCode}`,
+          message: details.reason,
+          stack: details.logs,
        },
+        overrideCompletion: details.overrideCompletion,
      });
    } catch (error) {
      logger.error(error);
    }
  },
-  onRunFailure: async (runId, failureInfo) => {
-    logger.log("Run failed:", { runId, failureInfo });
+  onRunFailure: async (runId, details) => {
+    logger.log("Run failed:", { runId, details });

    try {
-      provider.platformSocket.send("WORKER_CRASHED", { runId, ...failureInfo });
+      provider.platformSocket.send("WORKER_CRASHED", { runId, ...details });
    } catch (error) {
      logger.error(error);
    }
@@ -565,3 +574,19 @@ const podCleaner = new PodCleaner({
 });

 podCleaner.start();
+
+if (UPTIME_HEARTBEAT_URL) {
+  const uptimeHeartbeat = new UptimeHeartbeat({
+    runtimeEnv: RUNTIME_ENV,
+    namespace: "default",
+    intervalInSeconds: UPTIME_INTERVAL_SECONDS,
+    pingUrl: UPTIME_HEARTBEAT_URL,
+    maxPendingRuns: UPTIME_MAX_PENDING_RUNS,
+    maxPendingIndeces: UPTIME_MAX_PENDING_INDECES,
+    maxPendingErrors: UPTIME_MAX_PENDING_ERRORS,
+  });
+
+  uptimeHeartbeat.start();
+} else {
+  logger.log("Uptime heartbeat is disabled, set UPTIME_HEARTBEAT_URL to enable.");
+}
@@ -1,5 +1,5 @@
 import * as k8s from "@kubernetes/client-node";
-import { SimpleLogger } from "@trigger.dev/core-apps";
+import { SimpleLogger } from "@trigger.dev/core-apps/logger";

 type PodCleanerOptions = {
  runtimeEnv: "local" | "kubernetes";
@@ -1,25 +1,20 @@
 import * as k8s from "@kubernetes/client-node";
-import { SimpleLogger } from "@trigger.dev/core-apps";
+import { SimpleLogger } from "@trigger.dev/core-apps/logger";
+import { EXIT_CODE_ALREADY_HANDLED, EXIT_CODE_CHILD_NONZERO } from "@trigger.dev/core-apps/process";
 import { setTimeout } from "timers/promises";
 import PQueue from "p-queue";
+import type { Prettify } from "@trigger.dev/core/v3";

-type IndexFailureHandler = (
-  deploymentId: string,
-  failureInfo: {
-    exitCode: number;
-    reason: string;
-    logs: string;
-  }
-) => Promise<any>;
+type FailureDetails = Prettify<{
+  exitCode: number;
+  reason: string;
+  logs: string;
+  overrideCompletion: boolean;
+}>;

-type RunFailureHandler = (
-  runId: string,
-  failureInfo: {
-    exitCode: number;
-    reason: string;
-    logs: string;
-  }
-) => Promise<any>;
+type IndexFailureHandler = (deploymentId: string, details: FailureDetails) => Promise<any>;
+
+type RunFailureHandler = (runId: string, details: FailureDetails) => Promise<any>;

 type TaskMonitorOptions = {
  runtimeEnv: "local" | "kubernetes";
@@ -144,8 +139,10 @@ export class TaskMonitor {
    const containerState = this.#getContainerStateSummary(containerStatus.state);
    const exitCode = containerState.exitCode ?? -1;

-    // We use this special exit code to signal any errors were already handled elsewhere
-    if (exitCode === 111) {
+    if (exitCode === EXIT_CODE_ALREADY_HANDLED) {
+      this.#logger.debug("Ignoring pod failure, already handled by worker", {
+        podName,
+      });
      return;
    }

@@ -162,6 +159,7 @@ export class TaskMonitor {

    let reason = rawReason || "Unknown error";
    let logs = rawLogs || "";
+    let overrideCompletion = false;

    switch (rawReason) {
      case "Error":
@@ -181,7 +179,10 @@ export class TaskMonitor {
        }
        break;
      case "OOMKilled":
-        reason = "Out of memory! Try increasing the memory on this task.";
+        overrideCompletion = true;
+        reason = `${
+          exitCode === EXIT_CODE_CHILD_NONZERO ? "Child process" : "Parent process"
+        } ran out of memory! Try choosing a machine preset with more memory for this task.`;
        break;
      default:
        break;
@@ -191,7 +192,8 @@ export class TaskMonitor {
      exitCode,
      reason,
      logs,
-    };
+      overrideCompletion,
+    } satisfies FailureDetails;

    const app = pod.metadata?.labels?.app;

@@ -0,0 +1,272 @@
+import * as k8s from "@kubernetes/client-node";
+import { SimpleLogger } from "@trigger.dev/core-apps/logger";
+
+type UptimeHeartbeatOptions = {
+  runtimeEnv: "local" | "kubernetes";
+  pingUrl: string;
+  namespace?: string;
+  intervalInSeconds?: number;
+  maxPendingRuns?: number;
+  maxPendingIndeces?: number;
+  maxPendingErrors?: number;
+  leadingEdge?: boolean;
+};
+
+export class UptimeHeartbeat {
+  private enabled = false;
+  private namespace: string;
+
+  private intervalInSeconds: number;
+  private maxPendingRuns: number;
+  private maxPendingIndeces: number;
+  private maxPendingErrors: number;
+
+  private leadingEdge = true;
+
+  private logger = new SimpleLogger("[UptimeHeartbeat]");
+  private k8sClient: {
+    core: k8s.CoreV1Api;
+    kubeConfig: k8s.KubeConfig;
+  };
+
+  constructor(private opts: UptimeHeartbeatOptions) {
+    this.namespace = opts.namespace ?? "default";
+
+    this.intervalInSeconds = opts.intervalInSeconds ?? 60;
+    this.maxPendingRuns = opts.maxPendingRuns ?? 25;
+    this.maxPendingIndeces = opts.maxPendingIndeces ?? 10;
+    this.maxPendingErrors = opts.maxPendingErrors ?? 10;
+
+    this.k8sClient = this.#createK8sClient();
+  }
+
+  #createK8sClient() {
+    const kubeConfig = new k8s.KubeConfig();
+
+    if (this.opts.runtimeEnv === "local") {
+      kubeConfig.loadFromDefault();
+    } else if (this.opts.runtimeEnv === "kubernetes") {
+      kubeConfig.loadFromCluster();
+    } else {
+      throw new Error(`Unsupported runtime environment: ${this.opts.runtimeEnv}`);
+    }
+
+    return {
+      core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+      kubeConfig: kubeConfig,
+    };
+  }
+
+  #isRecord(candidate: unknown): candidate is Record<string, unknown> {
+    if (typeof candidate !== "object" || candidate === null) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  #logK8sError(err: unknown, debugOnly = false) {
+    if (debugOnly) {
+      this.logger.debug("K8s API Error", err);
+    } else {
+      this.logger.error("K8s API Error", err);
+    }
+  }
+
+  #handleK8sError(err: unknown) {
+    if (!this.#isRecord(err) || !this.#isRecord(err.body)) {
+      this.#logK8sError(err);
+      return;
+    }
+
+    this.#logK8sError(err, true);
+
+    if (typeof err.body.message === "string") {
+      this.#logK8sError({ message: err.body.message });
+      return;
+    }
+
+    this.#logK8sError({ body: err.body });
+  }
+
+  async #getPods(opts: {
+    namespace: string;
+    fieldSelector?: string;
+    labelSelector?: string;
+  }): Promise<Array<k8s.V1Pod> | undefined> {
+    const listReturn = await this.k8sClient.core
+      .listNamespacedPod(
+        opts.namespace,
+        undefined, // pretty
+        undefined, // allowWatchBookmarks
+        undefined, // _continue
+        opts.fieldSelector,
+        opts.labelSelector,
+        this.maxPendingRuns * 2, // limit
+        undefined, // resourceVersion
+        undefined, // resourceVersionMatch
+        undefined, // sendInitialEvents
+        this.intervalInSeconds, // timeoutSeconds,
+        undefined // watch
+      )
+      .catch(this.#handleK8sError.bind(this));
+
+    return listReturn?.body.items;
+  }
+
+  async #getPendingIndeces(): Promise<Array<k8s.V1Pod> | undefined> {
+    return await this.#getPods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=Pending",
+      labelSelector: "app=task-index",
+    });
+  }
+
+  async #getPendingTasks(): Promise<Array<k8s.V1Pod> | undefined> {
+    return await this.#getPods({
+      namespace: this.namespace,
+      fieldSelector: "status.phase=Pending",
+      labelSelector: "app=task-run",
+    });
+  }
+
+  #countPods(pods: Array<k8s.V1Pod>): number {
+    return pods.length;
+  }
+
+  #filterPendingPods(
+    pods: Array<k8s.V1Pod>,
+    waitingReason: "CreateContainerError" | "RunContainerError"
+  ): Array<k8s.V1Pod> {
+    return pods.filter((pod) => {
+      const containerStatus = pod.status?.containerStatuses?.[0];
+      return containerStatus?.state?.waiting?.reason === waitingReason;
+    });
+  }
+
+  async #sendPing() {
+    this.logger.log("Sending ping");
+
+    const start = Date.now();
+    const controller = new AbortController();
+
+    const timeoutMs = (this.intervalInSeconds * 1000) / 2;
+
+    const fetchTimeout = setTimeout(() => {
+      controller.abort();
+    }, timeoutMs);
+
+    try {
+      const response = await fetch(this.opts.pingUrl, {
+        signal: controller.signal,
+      });
+
+      if (!response.ok) {
+        this.logger.error("Failed to send ping, response not OK", {
+          status: response.status,
+        });
+        return;
+      }
+
+      const elapsedMs = Date.now() - start;
+      this.logger.log("Ping sent", { elapsedMs });
+    } catch (error) {
+      if (error instanceof DOMException && error.name === "AbortError") {
+        this.logger.log("Ping timeout", { timeoutSeconds: timeoutMs });
+        return;
+      }
+
+      this.logger.error("Failed to send ping", error);
+    } finally {
+      clearTimeout(fetchTimeout);
+    }
+  }
+
+  async #heartbeat() {
+    this.logger.log("Performing heartbeat");
+
+    const start = Date.now();
+
+    const pendingTasks = await this.#getPendingTasks();
+
+    if (!pendingTasks) {
+      this.logger.error("Failed to get pending tasks");
+      return;
+    }
+
+    const totalPendingTasks = this.#countPods(pendingTasks);
+
+    const pendingIndeces = await this.#getPendingIndeces();
+
+    if (!pendingIndeces) {
+      this.logger.error("Failed to get pending indeces");
+      return;
+    }
+
+    const totalPendingIndeces = this.#countPods(pendingIndeces);
+
+    const elapsedMs = Date.now() - start;
+
+    this.logger.log("Finished heartbeat checks", { elapsedMs });
+
+    if (totalPendingTasks > this.maxPendingRuns) {
+      this.logger.log("Too many pending tasks, skipping heartbeat", { totalPendingTasks });
+      return;
+    }
+
+    if (totalPendingIndeces > this.maxPendingIndeces) {
+      this.logger.log("Too many pending indeces, skipping heartbeat", { totalPendingIndeces });
+      return;
+    }
+
+    const totalCreateContainerErrors = this.#countPods(
+      this.#filterPendingPods(pendingTasks, "CreateContainerError")
+    );
+    const totalRunContainerErrors = this.#countPods(
+      this.#filterPendingPods(pendingTasks, "RunContainerError")
+    );
+
+    if (totalCreateContainerErrors + totalRunContainerErrors > this.maxPendingErrors) {
+      this.logger.log("Too many pending tasks with errors, skipping heartbeat", {
+        totalRunContainerErrors,
+        totalCreateContainerErrors,
+      });
+      return;
+    }
+
+    await this.#sendPing();
+
+    this.logger.log("Heartbeat done", { totalPendingTasks, elapsedMs });
+  }
+
+  async start() {
+    this.enabled = true;
+    this.logger.log("Starting");
+
+    if (this.leadingEdge) {
+      await this.#heartbeat();
+    }
+
+    const heartbeat = setInterval(async () => {
+      if (!this.enabled) {
+        clearInterval(heartbeat);
+        return;
+      }
+
+      try {
+        await this.#heartbeat();
+      } catch (error) {
+        this.logger.error("Error while heartbeating", error);
+      }
+    }, this.intervalInSeconds * 1000);
+  }
+
+  async stop() {
+    if (!this.enabled) {
+      return;
+    }
+
+    this.enabled = false;
+    this.logger.log("Shutting down..");
+  }
+}
@@ -15,11 +15,11 @@ export function DefinitionTip({
    <TooltipProvider>
      <Tooltip disableHoverableContent>
        <TooltipTrigger>
-          <span className="underline decoration-charcoal-600 decoration-dashed underline-offset-4 transition hover:decoration-charcoal-500">
+          <span className="cursor-default underline decoration-charcoal-500 decoration-dashed underline-offset-4 transition hover:decoration-charcoal-400">
            {children}
          </span>
        </TooltipTrigger>
-        <TooltipContent align="end" side="right" variant="dark" className="w-[16rem] min-w-[16rem]">
+        <TooltipContent align="end" side="right" className="w-[16rem] min-w-[16rem]">
          <Header3 className="mb-1">{title}</Header3>
          {typeof content === "string" ? (
            <Paragraph variant="small">{content}</Paragraph>
@@ -55,16 +55,6 @@ export function Feedback({ button, defaultValue = "bug" }: FeedbackProps) {
      <SheetTrigger asChild={true}>{button}</SheetTrigger>
      <SheetContent className="@container">
        <SheetBody className="flex h-full flex-col justify-between">
-          <Header2 className="mb-2.5 text-xl">Get help from the community</Header2>
-          <Paragraph className="mb-4">
-            The quickest way to get help and feedback or to provide advice to others is to join our
-            Discord.
-          </Paragraph>
-          <div className="flex flex-col gap-x-4 @[30rem]:flex-row">
-            <DiscordBanner />
-            <GitHubDiscussionsBanner />
-          </div>
-          <hr className="mb-4" />
          <Header2 className="mb-2.5 text-xl">Send us an email</Header2>
          <Paragraph className="mb-4">We read every message and respond quickly.</Paragraph>
          <Form method="post" action="/resources/feedback" {...form.props}>
@@ -106,6 +96,8 @@ export function Feedback({ button, defaultValue = "bug" }: FeedbackProps) {
            </Fieldset>
          </Form>
          <hr className="my-4" />
+          <DiscordBanner />
+          <hr className="mb-4" />
          <Header2 className="mb-2.5 text-xl">Troubleshooting</Header2>
          <Paragraph className="mb-4">
            If you're having trouble, check out our documentation or the Trigger.dev Status page.
@@ -134,44 +126,20 @@ function DiscordBanner({ className }: { className?: string }) {
      href="https://trigger.dev/discord"
      target="_blank"
      className={cn(
-        "group mb-4 flex w-full items-center justify-between rounded-md border border-charcoal-600 p-4 transition hover:border-text-link",
+        "group mb-4 flex w-full items-center justify-between rounded-md border border-grid-bright bg-charcoal-750 p-4 pl-6 transition hover:border-text-link",
        className
      )}
    >
-      <div className="flex flex-col gap-y-2">
-        <DiscordIcon className="h-8 w-8" />
-        <Header1 className="text-2xl font-semibold text-text-bright transition group-hover:text-white">
-          Join our Discord community
-        </Header1>
-        <Paragraph variant="small" className="mb-4">
-          Get help or answer questions from the Trigger.dev community.
-        </Paragraph>
+      <div className="flex items-center gap-6">
+        <DiscordIcon className="size-12" />
+        <div className="flex flex-col gap-2">
+          <Header1 className="text-2xl font-semibold text-text-bright transition group-hover:text-white">
+            Join our Discord community
+          </Header1>
+          <Paragraph>The quickest way to get answers from the Trigger.dev community.</Paragraph>
+        </div>
      </div>
      <ChevronRightIcon className="size-5 text-charcoal-500 transition group-hover:translate-x-1 group-hover:text-text-link" />
    </a>
  );
 }
-
-function GitHubDiscussionsBanner({ className }: { className?: string }) {
-  return (
-    <a
-      href="https://github.com/triggerdotdev/trigger.dev/discussions"
-      target="_blank"
-      className={cn(
-        "group mb-4 flex w-full items-center justify-between rounded-md border border-charcoal-600 p-4 transition hover:border-text-dimmed",
-        className
-      )}
-    >
-      <div className="flex flex-col gap-y-2">
-        <GitHubLightIcon className="mb-1 h-7 w-7" />
-        <Header1 className="text-2xl font-semibold text-text-bright transition group-hover:text-white">
-          View our GitHub Discussions
-        </Header1>
-        <Paragraph variant="small">
-          Post your questions, feedback, and feature requests on GitHub.
-        </Paragraph>
-      </div>
-      <ChevronRightIcon className="size-5 text-charcoal-500 transition group-hover:translate-x-1 group-hover:text-text-bright" />
-    </a>
-  );
-}
@@ -1,61 +0,0 @@
-import { ExclamationCircleIcon } from "@heroicons/react/20/solid";
-import { ArrowUpCircleIcon } from "@heroicons/react/24/outline";
-import tileBgPath from "~/assets/images/error-banner-tile@2x.png";
-import { MatchedOrganization } from "~/hooks/useOrganizations";
-import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route";
-import { formatNumberCompact } from "~/utils/numberFormatter";
-import { LinkButton } from "../primitives/Buttons";
-import { Icon } from "../primitives/Icon";
-import { Paragraph } from "../primitives/Paragraph";
-
-type UpgradePromptProps = {
-  runsEnabled: boolean;
-  runCountCap: number;
-  planPath: string;
-};
-
-export function UpgradePrompt({ runsEnabled, runCountCap, planPath }: UpgradePromptProps) {
-  return (
-    <div
-      className="flex h-10 items-center justify-between border border-error bg-repeat py-0 pl-3 pr-2"
-      style={{ backgroundImage: `url(${tileBgPath})`, backgroundSize: "8px 8px" }}
-    >
-      <div className="flex items-center gap-2">
-        <Icon icon={ExclamationCircleIcon} className="h-5 w-5 text-error" />
-        <Paragraph variant="small" className="text-error">
-          {runsEnabled
-            ? `You have exceeded the monthly ${formatNumberCompact(runCountCap)} runs
-          limit`
-            : `No runs are executing because you have exceeded the free limit`}
-        </Paragraph>
-      </div>
-      <LinkButton
-        variant={"primary/small"}
-        LeadingIcon={ArrowUpCircleIcon}
-        leadingIconClassName="px-0"
-        to={planPath}
-      >
-        Upgrade
-      </LinkButton>
-    </div>
-  );
-}
-
-export function useShowUpgradePrompt(organization?: MatchedOrganization) {
-  const currentPlan = useCurrentPlan();
-  const shouldShow =
-    organization !== undefined &&
-    currentPlan !== undefined &&
-    currentPlan.usage.exceededRunCount &&
-    currentPlan.usage.runCountCap !== undefined;
-
-  if (!shouldShow) {
-    return { shouldShow };
-  }
-
-  return {
-    shouldShow,
-    runCountCap: currentPlan.usage.runCountCap!,
-    runsEnabled: organization.runsEnabled,
-  };
-}
@@ -8,7 +8,7 @@ import {
  XAxis,
  YAxis,
 } from "recharts";
-import { Paragraph } from "../primitives/Paragraph";
+import { Paragraph } from "../../primitives/Paragraph";

 const tooltipStyle = {
  display: "flex",
@@ -1,5 +1,5 @@
 import { Label, Line, LineChart, ResponsiveContainer, Tooltip, XAxis, YAxis } from "recharts";
-import { Paragraph } from "../primitives/Paragraph";
+import { Paragraph } from "../../primitives/Paragraph";

 const tooltipStyle = {
  display: "flex",
@@ -1,6 +1,6 @@
 import { ArrowUpCircleIcon } from "@heroicons/react/24/outline";
 import { motion, useMotionValue, useTransform } from "framer-motion";
-import { Paragraph } from "../primitives/Paragraph";
+import { Paragraph } from "../../primitives/Paragraph";
 import { Link } from "@remix-run/react";
 import { cn } from "~/utils/cn";

@@ -28,7 +28,7 @@ export function FreePlanUsage({ to, percentage }: { to: string; percentage: numb
          <Paragraph className="text-2sm text-text-bright">Free Plan</Paragraph>
        </div>
        <Link to={to} className="text-2sm text-text-link">
-          Learn more
+          Upgrade
        </Link>
      </div>
      <div className="relative mt-3 h-1 rounded-full bg-background-dimmed">
@@ -1,9 +1,9 @@
 import * as Slider from "@radix-ui/react-slider";
-import { Plans, estimate } from "@trigger.dev/billing";
+import { Plans, estimate } from "@trigger.dev/platform/v2";
 import { useCallback, useState } from "react";
-import { DefinitionTip } from "../DefinitionTooltip";
-import { Header2 } from "../primitives/Headers";
-import { Paragraph } from "../primitives/Paragraph";
+import { DefinitionTip } from "../../DefinitionTooltip";
+import { Header2 } from "../../primitives/Headers";
+import { Paragraph } from "../../primitives/Paragraph";
 import { formatCurrency, formatNumberCompact } from "~/utils/numberFormatter";
 import { cn } from "~/utils/cn";

@@ -2,17 +2,17 @@ import { useForm } from "@conform-to/react";
 import { parse } from "@conform-to/zod";
 import { CheckIcon, XMarkIcon } from "@heroicons/react/24/solid";
 import { Form, useActionData, useNavigation } from "@remix-run/react";
-import { ActiveSubscription, Plan, Plans, SetPlanBodySchema } from "@trigger.dev/billing";
+import { ActiveSubscription, Plan, Plans, SetPlanBodySchema } from "@trigger.dev/platform/v2";
 import { useState } from "react";
 import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route";
 import { cn } from "~/utils/cn";
 import { formatNumberCompact } from "~/utils/numberFormatter";
-import { DefinitionTip } from "../DefinitionTooltip";
-import { Feedback } from "../Feedback";
-import { Button, LinkButton } from "../primitives/Buttons";
-import SegmentedControl from "../primitives/SegmentedControl";
+import { DefinitionTip } from "../../DefinitionTooltip";
+import { Feedback } from "../../Feedback";
+import { Button, LinkButton } from "../../primitives/Buttons";
+import SegmentedControl from "../../primitives/SegmentedControl";
 import { RunsVolumeDiscountTable } from "./RunsVolumeDiscountTable";
-import { Spinner } from "../primitives/Spinner";
+import { Spinner } from "../../primitives/Spinner";

 const pricingDefinitions = {
  concurrentRuns: {
@@ -1,6 +1,6 @@
-import { RunPriceBracket } from "@trigger.dev/billing";
-import { Header2 } from "../primitives/Headers";
-import { Paragraph } from "../primitives/Paragraph";
+import { RunPriceBracket } from "@trigger.dev/platform/v2";
+import { Header2 } from "../../primitives/Headers";
+import { Paragraph } from "../../primitives/Paragraph";
 import { formatNumberCompact } from "~/utils/numberFormatter";

 export function RunsVolumeDiscountTable({
@@ -1,7 +1,7 @@
 import { cn } from "~/utils/cn";
 import { formatNumberCompact } from "~/utils/numberFormatter";
-import { Paragraph } from "../primitives/Paragraph";
-import { SimpleTooltip } from "../primitives/Tooltip";
+import { Paragraph } from "../../primitives/Paragraph";
+import { SimpleTooltip } from "../../primitives/Tooltip";
 import { motion } from "framer-motion";

 type UsageBarProps = {
@@ -0,0 +1,53 @@
+import { ExclamationCircleIcon } from "@heroicons/react/20/solid";
+import tileBgPath from "~/assets/images/error-banner-tile@2x.png";
+import { MatchedOrganization, useOrganization } from "~/hooks/useOrganizations";
+import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route";
+import { v3BillingPath } from "~/utils/pathBuilder";
+import { LinkButton } from "../../primitives/Buttons";
+import { Icon } from "../../primitives/Icon";
+import { Paragraph } from "../../primitives/Paragraph";
+import { DateTime } from "~/components/primitives/DateTime";
+
+export function UpgradePrompt() {
+  const organization = useOrganization();
+  const plan = useCurrentPlan();
+
+  if (!plan || !plan.v3Usage.hasExceededFreeTier) {
+    return null;
+  }
+
+  const nextMonth = new Date();
+  nextMonth.setUTCMonth(nextMonth.getMonth() + 1);
+  nextMonth.setUTCDate(1);
+  nextMonth.setUTCHours(0, 0, 0, 0);
+
+  return (
+    <div
+      className="flex h-10 items-center justify-between border border-error bg-repeat py-0 pl-3 pr-2"
+      style={{ backgroundImage: `url(${tileBgPath})`, backgroundSize: "8px 8px" }}
+    >
+      <div className="flex items-center gap-2">
+        <Icon icon={ExclamationCircleIcon} className="h-5 w-5 text-error" />
+        <Paragraph variant="small" className="text-error">
+          You have exceeded the monthly $
+          {(plan.v3Subscription?.plan?.limits.includedUsage ?? 500) / 100} free credits. No runs
+          will execute in Prod until{" "}
+          <DateTime date={nextMonth} includeTime={false} timeZone="utc" />, or you upgrade.
+        </Paragraph>
+      </div>
+      <LinkButton
+        variant={"primary/small"}
+        leadingIconClassName="px-0"
+        to={v3BillingPath(organization)}
+      >
+        Upgrade
+      </LinkButton>
+    </div>
+  );
+}
+
+export function useShowUpgradePrompt(organization?: MatchedOrganization) {
+  const currentPlan = useCurrentPlan();
+  const shouldShow = currentPlan?.v3Usage.hasExceededFreeTier === true;
+  return { shouldShow };
+}
@@ -0,0 +1,170 @@
+import { cn } from "~/utils/cn";
+import { formatCurrency } from "~/utils/numberFormatter";
+import { Paragraph } from "../../primitives/Paragraph";
+import { SimpleTooltip } from "../../primitives/Tooltip";
+import { motion } from "framer-motion";
+
+type UsageBarProps = {
+  current: number;
+  billingLimit?: number;
+  tierLimit?: number;
+  projectedUsage?: number;
+  isPaying: boolean;
+};
+
+const startFactor = 4;
+
+export function UsageBar({
+  current,
+  billingLimit,
+  tierLimit,
+  projectedUsage,
+  isPaying,
+}: UsageBarProps) {
+  const getLargestNumber = Math.max(
+    current,
+    tierLimit ?? -Infinity,
+    projectedUsage ?? -Infinity,
+    billingLimit ?? -Infinity,
+    5
+  );
+  //creates a maximum range for the progress bar, add 10% to the largest number so the bar doesn't reach the end
+  const maxRange = Math.round(getLargestNumber * 1.1);
+  const tierRunLimitPercentage = tierLimit ? Math.round((tierLimit / maxRange) * 100) : 0;
+  const projectedRunsPercentage = projectedUsage
+    ? Math.round((projectedUsage / maxRange) * 100)
+    : 0;
+  const billingLimitPercentage =
+    billingLimit !== undefined ? Math.round((billingLimit / maxRange) * 100) : 0;
+  const usagePercentage = Math.round((current / maxRange) * 100);
+
+  //cap the usagePercentage to the freeRunLimitPercentage
+  const usageCappedToLimitPercentage = Math.min(usagePercentage, tierRunLimitPercentage);
+
+  return (
+    <div className="h-fit w-full py-12">
+      <div className="relative h-3 w-full rounded-sm bg-background-bright">
+        {billingLimit && (
+          <motion.div
+            initial={{ width: billingLimitPercentage / startFactor + "%" }}
+            animate={{ width: billingLimitPercentage + "%" }}
+            transition={{ duration: 1.5, type: "spring" }}
+            style={{ width: `${billingLimitPercentage}%` }}
+            className="absolute h-3 rounded-l-sm"
+          >
+            <Legend
+              text="Billing limit:"
+              value={formatCurrency(billingLimit, false)}
+              position="bottomRow2"
+              percentage={billingLimitPercentage}
+              tooltipContent={`Billing limit: ${formatCurrency(billingLimit, false)}`}
+            />
+          </motion.div>
+        )}
+        {tierLimit && (
+          <motion.div
+            initial={{ width: tierRunLimitPercentage / startFactor + "%" }}
+            animate={{ width: tierRunLimitPercentage + "%" }}
+            transition={{ duration: 1.5, type: "spring" }}
+            style={{ width: `${tierRunLimitPercentage}%` }}
+            className="absolute h-3 rounded-l-sm bg-green-900/50"
+          >
+            <Legend
+              text={isPaying ? `Included usage:` : `Tier limit:`}
+              value={formatCurrency(tierLimit, false)}
+              position="bottomRow1"
+              percentage={tierRunLimitPercentage}
+              tooltipContent={`${isPaying ? "Included usage" : "Tier limit"}: ${formatCurrency(
+                tierLimit,
+                false
+              )}`}
+            />
+          </motion.div>
+        )}
+        {projectedUsage && projectedUsage !== 0 && (
+          <motion.div
+            initial={{ width: projectedRunsPercentage / startFactor + "%" }}
+            animate={{ width: projectedRunsPercentage + "%" }}
+            transition={{ duration: 1.5, type: "spring" }}
+            style={{ width: `${projectedRunsPercentage}%` }}
+            className="absolute h-3 rounded-l-sm"
+          >
+            <Legend
+              text="Projected:"
+              value={formatCurrency(projectedUsage, false)}
+              position="topRow2"
+              percentage={projectedRunsPercentage}
+              tooltipContent={`Projected runs: ${formatCurrency(projectedUsage, false)}`}
+            />
+          </motion.div>
+        )}
+        <motion.div
+          initial={{ width: usagePercentage / startFactor + "%" }}
+          animate={{ width: usagePercentage + "%" }}
+          transition={{ duration: 1.5, type: "spring" }}
+          style={{ width: `${usagePercentage}%` }}
+          className={cn(
+            "absolute h-3 rounded-l-sm",
+            tierLimit && current > tierLimit ? "bg-rose-600" : "bg-green-600"
+          )}
+        >
+          <Legend
+            text="Used:"
+            value={formatCurrency(current, false)}
+            position="topRow1"
+            percentage={usagePercentage}
+            tooltipContent={`Used: ${formatCurrency(current, false)}`}
+          />
+        </motion.div>
+        <motion.div
+          initial={{ width: usageCappedToLimitPercentage / startFactor + "%" }}
+          animate={{ width: usageCappedToLimitPercentage + "%" }}
+          transition={{ duration: 1.5, type: "spring" }}
+          style={{ width: `${usageCappedToLimitPercentage}%` }}
+          className="absolute h-3 rounded-l-sm bg-green-600"
+        />
+      </div>
+    </div>
+  );
+}
+
+const positions = {
+  topRow1: "bottom-0 h-9",
+  topRow2: "bottom-0 h-14",
+  bottomRow1: "top-0 h-9 items-end",
+  bottomRow2: "top-0 h-14 items-end",
+};
+
+type LegendProps = {
+  text: string;
+  value: number | string;
+  percentage: number;
+  position: keyof typeof positions;
+  tooltipContent: string;
+};
+
+function Legend({ text, value, position, percentage, tooltipContent }: LegendProps) {
+  const flipLegendPositionValue = 80;
+  const flipLegendPosition = percentage > flipLegendPositionValue ? true : false;
+  return (
+    <div
+      className={cn(
+        "absolute left-full z-10 flex border-charcoal-500",
+        positions[position],
+        flipLegendPosition === true ? "-translate-x-full border-r" : "border-l"
+      )}
+    >
+      <SimpleTooltip
+        button={
+          <Paragraph className="mr-px h-fit whitespace-nowrap bg-background-bright px-1.5 text-xs text-text-bright">
+            {text}
+            <span className="ml-1 text-text-dimmed">{value}</span>
+          </Paragraph>
+        }
+        side="top"
+        content={tooltipContent}
+        className="z-50 h-fit"
+      />
+    </div>
+  );
+}
@@ -58,103 +58,116 @@ const extraLinesWhenClipping = 0.35;

 const defaultTheme: PrismTheme = {
  plain: {
-    color: "#9CDCFE",
+    color: "#9C9AF2",
    backgroundColor: "rgba(0, 0, 0, 0)",
  },
  styles: [
    {
-      types: ["prolog"],
+      types: ["comment", "prolog", "doctype", "cdata"],
      style: {
-        color: "rgb(0, 0, 128)",
+        color: "#5F6570",
      },
    },
    {
-      types: ["comment"],
-      style: {
-        color: "rgb(106, 153, 85)",
-      },
-    },
-    {
-      types: ["builtin", "changed", "keyword", "interpolation-punctuation"],
-      style: {
-        color: "rgb(86, 156, 214)",
-      },
-    },
-    {
-      types: ["number", "inserted"],
-      style: {
-        color: "rgb(181, 206, 168)",
-      },
-    },
-    {
-      types: ["constant"],
-      style: {
-        color: "rgb(100, 102, 149)",
-      },
-    },
-    {
-      types: ["attr-name", "variable"],
-      style: {
-        color: "rgb(156, 220, 254)",
-      },
-    },
-    {
-      types: ["deleted", "string", "attr-value", "template-punctuation"],
-      style: {
-        color: "rgb(206, 145, 120)",
-      },
-    },
-    {
-      types: ["selector"],
-      style: {
-        color: "rgb(215, 186, 125)",
-      },
-    },
-    {
-      // Fix tag color
-      types: ["tag"],
-      style: {
-        color: "rgb(78, 201, 176)",
-      },
-    },
-    {
-      // Fix tag color for HTML
-      types: ["tag"],
-      languages: ["markup"],
-      style: {
-        color: "rgb(86, 156, 214)",
-      },
-    },
-    {
-      types: ["punctuation", "operator"],
-      style: {
-        color: "rgb(212, 212, 212)",
-      },
-    },
-    {
-      // Fix punctuation color for HTML
      types: ["punctuation"],
-      languages: ["markup"],
      style: {
-        color: "#808080",
+        color: "#878C99",
      },
    },
    {
-      types: ["function"],
+      types: ["property", "tag", "boolean", "number", "constant", "symbol", "deleted"],
      style: {
-        color: "rgb(220, 220, 170)",
+        color: "#9B99FF",
      },
    },
    {
-      types: ["class-name"],
+      types: ["selector", "attr-name", "string", "char", "builtin", "inserted"],
      style: {
-        color: "rgb(78, 201, 176)",
+        color: "#AFEC73",
+      },
+    },
+    {
+      types: ["operator", "entity", "url"],
+      style: {
+        color: "#D4D4D4",
+      },
+    },
+    {
+      types: ["variable"],
+      style: {
+        color: "#CCCBFF",
+      },
+    },
+    {
+      types: ["atrule", "attr-value", "keyword"],
+      style: {
+        color: "#E888F8",
+      },
+    },
+    {
+      types: ["function", "class-name"],
+      style: {
+        color: "#D9F07C",
+      },
+    },
+    {
+      types: ["regex"],
+      style: {
+        color: "#d16969",
+      },
+    },
+    {
+      types: ["important", "bold"],
+      style: {
+        fontWeight: "bold",
+      },
+    },
+    {
+      types: ["italic"],
+      style: {
+        fontStyle: "italic",
+      },
+    },
+    {
+      types: ["namespace"],
+      style: {
+        opacity: 0.7,
+      },
+    },
+    {
+      types: ["deleted"],
+      style: {
+        color: "#F85149",
+      },
+    },
+    {
+      types: ["boolean"],
+      style: {
+        color: "#9B99FF",
      },
    },
    {
      types: ["char"],
      style: {
-        color: "rgb(209, 105, 105)",
+        color: "#b5cea8",
+      },
+    },
+    {
+      types: ["tag"],
+      style: {
+        color: "#D7BA7D",
+      },
+    },
+    {
+      types: ["keyword.operator"],
+      style: {
+        color: "#8271ED",
+      },
+    },
+    {
+      types: ["meta.template.expression"],
+      style: {
+        color: "#d4d4d4",
      },
    },
  ],
@@ -1,6 +1,6 @@
 import { useOptionalOrganization } from "~/hooks/useOrganizations";
 import { cn } from "~/utils/cn";
-import { useShowUpgradePrompt } from "../billing/UpgradePrompt";
+import { useShowUpgradePrompt } from "../billing/v3/UpgradePrompt";

 /** This container is used to surround the entire app, it correctly places the nav bar */
 export function AppContainer({ children }: { children: React.ReactNode }) {
@@ -60,7 +60,9 @@ export function MainCenteredContainer({
 }) {
  return (
    <div className="h-full w-full overflow-y-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600">
-      <div className={cn("mx-auto mt-[25vh] max-w-xs overflow-y-auto", className)}>{children}</div>
+      <div className={cn("mx-auto mt-6 max-w-xs overflow-y-auto p-1 md:mt-[22vh]", className)}>
+        {children}
+      </div>
    </div>
  );
 }
@@ -1,16 +1,17 @@
 import {
  AcademicCapIcon,
-  ArrowRightIcon,
  ArrowRightOnRectangleIcon,
  BeakerIcon,
  BellAlertIcon,
  ChartBarIcon,
  ClockIcon,
+  CreditCardIcon,
  CursorArrowRaysIcon,
  IdentificationIcon,
  KeyIcon,
  ServerStackIcon,
  ShieldCheckIcon,
+  SignalIcon,
 } from "@heroicons/react/20/solid";
 import { UserGroupIcon, UserPlusIcon } from "@heroicons/react/24/solid";
 import { useNavigation } from "@remix-run/react";
@@ -44,6 +45,7 @@ import {
  projectSetupPath,
  projectTriggersPath,
  v3ApiKeysPath,
+  v3BillingPath,
  v3DeploymentsPath,
  v3EnvironmentVariablesPath,
  v3ProjectAlertsPath,
@@ -52,13 +54,14 @@ import {
  v3RunsPath,
  v3SchedulesPath,
  v3TestPath,
+  v3UsagePath,
 } from "~/utils/pathBuilder";
 import { Feedback } from "../Feedback";
 import { ImpersonationBanner } from "../ImpersonationBanner";
 import { LogoIcon } from "../LogoIcon";
 import { StepContentContainer } from "../StepContentContainer";
 import { UserProfilePhoto } from "../UserProfilePhoto";
-import { FreePlanUsage } from "../billing/FreePlanUsage";
+import { FreePlanUsage } from "../billing/v2/FreePlanUsage";
 import { Badge } from "../primitives/Badge";
 import { Button } from "../primitives/Buttons";
 import { Callout } from "../primitives/Callout";
@@ -75,6 +78,7 @@ import {
  PopoverSectionHeader,
 } from "../primitives/Popover";
 import { StepNumber } from "../primitives/StepNumber";
+import { TextLink } from "../primitives/TextLink";
 import { SideMenuHeader } from "./SideMenuHeader";
 import { MenuCount, SideMenuItem } from "./SideMenuItem";

@@ -100,8 +104,8 @@ type SideMenuProps = {
 export function SideMenu({ user, project, organization, organizations }: SideMenuProps) {
  const borderRef = useRef<HTMLDivElement>(null);
  const [showHeaderDivider, setShowHeaderDivider] = useState(false);
-  const { isManagedCloud } = useFeatures();
  const currentPlan = useCurrentPlan();
+  const { isManagedCloud } = useFeatures();

  useEffect(() => {
    const handleScroll = () => {
@@ -173,16 +177,36 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
              name="Team"
              icon={UserGroupIcon}
              to={organizationTeamPath(organization)}
-              iconColor="text-sky-500"
+              iconColor="text-amber-500"
              data-action="team"
            />
-            <SideMenuItem
-              name={isManagedCloud ? "Usage & Billing" : "Usage"}
-              icon={ChartBarIcon}
-              to={organizationBillingPath(organization)}
-              iconColor="text-green-600"
-              data-action="usage & billing"
-            />
+            {organization.projects.some((proj) => proj.version === "V3") && isManagedCloud && (
+              <>
+                <SideMenuItem
+                  name="Usage"
+                  icon={ChartBarIcon}
+                  to={v3UsagePath(organization)}
+                  iconColor="text-green-600"
+                  data-action="usage"
+                />
+                <SideMenuItem
+                  name="Billing"
+                  icon={CreditCardIcon}
+                  to={v3BillingPath(organization)}
+                  iconColor="text-blue-600"
+                  data-action="billing"
+                />
+              </>
+            )}
+            {organization.projects.some((proj) => proj.version === "V2") && (
+              <SideMenuItem
+                name="Usage (v2)"
+                icon={ChartBarIcon}
+                to={organizationBillingPath(organization)}
+                iconColor="text-green-600"
+                data-action="usage & billing"
+              />
+            )}
            <SideMenuItem
              name="Organization settings"
              icon="settings"
@@ -193,22 +217,19 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
          </div>
        </div>
        <div className="m-2">
-          {project.version === "V2" ? (
-            <Callout variant={"info"}>This is a v2 project</Callout>
-          ) : (
-            <Callout variant={"idea"}>This is a v3 project in Developer Preview</Callout>
+          {project.version === "V2" && (
+            <Callout variant={"info"}>
+              <Paragraph variant="small">
+                This is a v2 project.{" "}
+                <TextLink href="https://trigger.dev/docs/v3/upgrading-from-v2">
+                  Upgrade to v3
+                </TextLink>
+              </Paragraph>
+            </Callout>
          )}
        </div>
        <div className="flex flex-col gap-1 border-t border-grid-bright p-1">
-          {project.version === "V2" && (
-            <SideMenuItem
-              to="https://trigger.dev/v3-early-access"
-              target="_blank"
-              name="Request access to v3"
-              icon={V3Icon}
-            />
-          )}
-          {currentPlan?.subscription?.isPaying === true && (
+          {currentPlan?.v3Subscription?.plan?.limits.support === "slack" && (
            <Dialog>
              <DialogTrigger asChild>
                <Button
@@ -263,23 +284,13 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
            data-action="join our discord"
            target="_blank"
          />
-          {project.version === "V2" ? (
-            <SideMenuItem
-              name="Documentation"
-              icon="docs"
-              to="https://trigger.dev/docs"
-              data-action="documentation"
-              target="_blank"
-            />
-          ) : (
-            <SideMenuItem
-              name="Documentation (v3)"
-              icon="docs"
-              to="https://trigger.dev/docs/v3"
-              data-action="documentation"
-              target="_blank"
-            />
-          )}
+          <SideMenuItem
+            name="Documentation"
+            icon="docs"
+            to="https://trigger.dev/docs"
+            data-action="documentation"
+            target="_blank"
+          />
          <SideMenuItem
            name="Changelog"
            icon="star"
@@ -287,41 +298,31 @@ export function SideMenu({ user, project, organization, organizations }: SideMen
            data-action="changelog"
            target="_blank"
          />
-          {project.version === "V2" ? (
-            <Feedback
-              button={
-                <Button
-                  variant="small-menu-item"
-                  LeadingIcon="log"
-                  data-action="help & feedback"
-                  fullWidth
-                  textAlignLeft
-                >
-                  Help & Feedback
-                </Button>
-              }
-            />
-          ) : (
-            <Feedback
-              defaultValue="developer preview"
-              button={
-                <Button
-                  variant="small-menu-item"
-                  LeadingIcon="log"
-                  leadingIconClassName="text-primary"
-                  data-action="help & feedback"
-                  fullWidth
-                  textAlignLeft
-                >
-                  <span className="text-primary">Give feedback on v3</span>
-                </Button>
-              }
-            />
-          )}
-          {currentPlan && !currentPlan.subscription?.isPaying && currentPlan.usage.runCountCap && (
+          <SideMenuItem
+            name="Status"
+            icon={SignalIcon}
+            to="https://status.trigger.dev/"
+            data-action="status"
+            target="_blank"
+          />
+          <Feedback
+            button={
+              <Button
+                variant="small-menu-item"
+                LeadingIcon="log"
+                leadingIconClassName="text-primary"
+                data-action="help & feedback"
+                fullWidth
+                textAlignLeft
+              >
+                <span className="text-primary">Help & Feedback</span>
+              </Button>
+            }
+          />
+          {currentPlan?.v3Subscription?.isPaying === false && (
            <FreePlanUsage
-              to={organizationBillingPath(organization)}
-              percentage={currentPlan.usage.currentRunCount / currentPlan.usage.runCountCap}
+              to={v3BillingPath(organization)}
+              percentage={currentPlan.v3Usage.usagePercentage}
            />
          )}
        </div>
@@ -474,7 +475,7 @@ function V2ProjectSideMenu({
 }) {
  return (
    <>
-      <SideMenuHeader title={"Project"}>
+      <SideMenuHeader title={"Project (v2)"}>
        <PopoverMenuItem
          to={projectSetupPath(organization, project)}
          title="Framework setup"
@@ -546,7 +547,7 @@ function V3ProjectSideMenu({

  return (
    <>
-      <SideMenuHeader title={"Project (v3)"} />
+      <SideMenuHeader title={"Project"} />
      <SideMenuItem
        name="Tasks"
        icon={TaskIcon}
@@ -615,15 +616,3 @@ function V3ProjectSideMenu({
    </>
  );
 }
-
-function V3Icon() {
-  return (
-    <svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
-      <rect width="16" height="16" rx="8" fill="#A8FF53" />
-      <path
-        d="M7.7488 6.17L5.5818 12H3.6678L1.5008 6.17H3.2058L4.6248 10.339L6.0328 6.17H7.7488ZM11.0527 7.292C11.6357 7.303 12.2297 7.006 12.2297 6.28C12.2297 5.774 11.7787 5.433 11.0527 5.433C10.4147 5.433 9.98567 5.741 9.91967 6.214L8.22567 6.126C8.36867 4.861 9.51267 4.014 11.0857 4.014C12.8457 4.014 13.9567 4.806 13.9567 6.049C13.9567 6.951 13.3847 7.534 12.3067 7.776C13.5387 8.04 14.2207 8.777 14.2207 9.855C14.2207 11.274 13.0107 12.176 11.0857 12.176C9.32567 12.176 8.12667 11.197 8.04967 9.712L9.75467 9.646C9.83167 10.405 10.4917 10.757 11.0967 10.757C11.8007 10.757 12.4937 10.394 12.4937 9.591C12.4937 8.81 11.7897 8.425 11.0527 8.447L10.3817 8.458V7.281L11.0527 7.292Z"
-        fill="#15171A"
-      />
-    </svg>
-  );
-}
@@ -21,7 +21,7 @@ const sizes = {
    shortcut: "-ml-0.5 -mr-1.5 rounded justify-self-center",
  },
  large: {
-    button: "h-10 px-2 text-sm font-medium",
+    button: "h-10 px-2 text-base font-medium",
    icon: "h-5",
    iconSpacing: "gap-x-0.5",
    shortcutVariant: "medium" as const,
@@ -0,0 +1,324 @@
+"use client";
+
+import * as React from "react";
+import * as RechartsPrimitive from "recharts";
+import { cn } from "~/utils/cn";
+
+// Format: { THEME_NAME: CSS_SELECTOR }
+const THEMES = { light: "", dark: ".dark" } as const;
+
+export type ChartConfig = {
+  [k in string]: {
+    label?: React.ReactNode;
+    icon?: React.ComponentType;
+  } & (
+    | { color?: string; theme?: never }
+    | { color?: never; theme: Record<keyof typeof THEMES, string> }
+  );
+};
+
+type ChartContextProps = {
+  config: ChartConfig;
+};
+
+const ChartContext = React.createContext<ChartContextProps | null>(null);
+
+function useChart() {
+  const context = React.useContext(ChartContext);
+
+  if (!context) {
+    throw new Error("useChart must be used within a <ChartContainer />");
+  }
+
+  return context;
+}
+
+const ChartContainer = React.forwardRef<
+  HTMLDivElement,
+  React.ComponentProps<"div"> & {
+    config: ChartConfig;
+    children: React.ComponentProps<typeof RechartsPrimitive.ResponsiveContainer>["children"];
+  }
+>(({ id, className, children, config, ...props }, ref) => {
+  const uniqueId = React.useId();
+  const chartId = `chart-${id || uniqueId.replace(/:/g, "")}`;
+
+  return (
+    <ChartContext.Provider value={{ config }}>
+      <div
+        data-chart={chartId}
+        ref={ref}
+        className={cn(
+          "flex aspect-video justify-center text-xs [&_.recharts-cartesian-axis-tick_text]:fill-text-dimmed/50 [&_.recharts-cartesian-grid_line]:stroke-grid-bright/50 [&_.recharts-curve.recharts-tooltip-cursor]:stroke-grid-dimmed [&_.recharts-dot[stroke='#fff']]:stroke-transparent [&_.recharts-layer]:outline-none [&_.recharts-polar-grid_[stroke='#ccc']]:stroke-grid-dimmed [&_.recharts-radial-bar-background-sector]:fill-grid-dimmed [&_.recharts-rectangle.recharts-tooltip-cursor]:fill-grid-dimmed [&_.recharts-reference-line-line]:stroke-grid-dimmed [&_.recharts-sector[stroke='#fff']]:stroke-transparent [&_.recharts-sector]:outline-none [&_.recharts-surface]:outline-none",
+          className
+        )}
+        {...props}
+      >
+        <ChartStyle id={chartId} config={config} />
+        <RechartsPrimitive.ResponsiveContainer>{children}</RechartsPrimitive.ResponsiveContainer>
+      </div>
+    </ChartContext.Provider>
+  );
+});
+ChartContainer.displayName = "Chart";
+
+const ChartStyle = ({ id, config }: { id: string; config: ChartConfig }) => {
+  const colorConfig = Object.entries(config).filter(([_, config]) => config.theme || config.color);
+
+  if (!colorConfig.length) {
+    return null;
+  }
+
+  return (
+    <style
+      dangerouslySetInnerHTML={{
+        __html: Object.entries(THEMES).map(
+          ([theme, prefix]) => `
+${prefix} [data-chart=${id}] {
+${colorConfig
+  .map(([key, itemConfig]) => {
+    const color = itemConfig.theme?.[theme as keyof typeof itemConfig.theme] || itemConfig.color;
+    return color ? `  --color-${key}: ${color};` : null;
+  })
+  .join("\n")}
+}
+`
+        ),
+      }}
+    />
+  );
+};
+
+const ChartTooltip = RechartsPrimitive.Tooltip;
+
+const ChartTooltipContent = React.forwardRef<
+  HTMLDivElement,
+  React.ComponentProps<typeof RechartsPrimitive.Tooltip> &
+    React.ComponentProps<"div"> & {
+      hideLabel?: boolean;
+      hideIndicator?: boolean;
+      indicator?: "line" | "dot" | "dashed";
+      nameKey?: string;
+      labelKey?: string;
+    }
+>(
+  (
+    {
+      active,
+      payload,
+      className,
+      indicator = "dot",
+      hideLabel = false,
+      hideIndicator = false,
+      label,
+      labelFormatter,
+      labelClassName,
+      formatter,
+      color,
+      nameKey,
+      labelKey,
+    },
+    ref
+  ) => {
+    const { config } = useChart();
+
+    const tooltipLabel = React.useMemo(() => {
+      if (hideLabel || !payload?.length) {
+        return null;
+      }
+
+      const [item] = payload;
+      const key = `${labelKey || item.dataKey || item.name || "value"}`;
+      const itemConfig = getPayloadConfigFromPayload(config, item, key);
+      const value =
+        !labelKey && typeof label === "string"
+          ? config[label as keyof typeof config]?.label || label
+          : itemConfig?.label;
+
+      if (labelFormatter) {
+        return (
+          <div className={cn("font-medium", labelClassName)}>{labelFormatter(value, payload)}</div>
+        );
+      }
+
+      if (!value) {
+        return null;
+      }
+
+      return <div className={cn("font-medium", labelClassName)}>{value}</div>;
+    }, [label, labelFormatter, payload, hideLabel, labelClassName, config, labelKey]);
+
+    if (!active || !payload?.length) {
+      return null;
+    }
+
+    const nestLabel = payload.length === 1 && indicator !== "dot";
+
+    return (
+      <div
+        ref={ref}
+        className={cn(
+          "grid min-w-[8rem] items-start gap-1.5 rounded-lg border border-grid-bright/50 bg-background-dimmed px-2.5 py-1.5 text-xs shadow-xl",
+          className
+        )}
+      >
+        {!nestLabel ? tooltipLabel : null}
+        <div className="grid gap-1.5">
+          {payload.map((item, index) => {
+            const key = `${nameKey || item.name || item.dataKey || "value"}`;
+            const itemConfig = getPayloadConfigFromPayload(config, item, key);
+            const indicatorColor = color || item.payload.fill || item.color;
+
+            return (
+              <div
+                key={item.dataKey}
+                className={cn(
+                  "flex w-full items-stretch gap-2 [&>svg]:h-2.5 [&>svg]:w-2.5 [&>svg]:text-text-dimmed",
+                  indicator === "dot" && "items-center"
+                )}
+              >
+                {formatter && item.value && item.name ? (
+                  formatter(item.value, item.name, item, index, item.payload)
+                ) : (
+                  <>
+                    {itemConfig?.icon ? (
+                      <itemConfig.icon />
+                    ) : (
+                      !hideIndicator && (
+                        <div
+                          className={cn(
+                            "shrink-0 rounded-[2px] border-[--color-border] bg-[--color-bg]",
+                            {
+                              "h-2.5 w-2.5": indicator === "dot",
+                              "w-1": indicator === "line",
+                              "w-0 border-[1.5px] border-dashed bg-transparent":
+                                indicator === "dashed",
+                              "my-0.5": nestLabel && indicator === "dashed",
+                            }
+                          )}
+                          style={
+                            {
+                              "--color-bg": indicatorColor,
+                              "--color-border": indicatorColor,
+                            } as React.CSSProperties
+                          }
+                        />
+                      )
+                    )}
+                    <div
+                      className={cn(
+                        "flex flex-1 justify-between gap-1.5 leading-none",
+                        nestLabel ? "items-end" : "items-center"
+                      )}
+                    >
+                      <div className="grid gap-1.5">
+                        {nestLabel ? tooltipLabel : null}
+                        <span className="text-text-dimmed">{itemConfig?.label || item.name}</span>
+                      </div>
+                      {item.value && (
+                        <span className="font-mono font-medium tabular-nums text-text-bright">
+                          {item.value.toLocaleString()}
+                        </span>
+                      )}
+                    </div>
+                  </>
+                )}
+              </div>
+            );
+          })}
+        </div>
+      </div>
+    );
+  }
+);
+ChartTooltipContent.displayName = "ChartTooltip";
+
+const ChartLegend = RechartsPrimitive.Legend;
+
+const ChartLegendContent = React.forwardRef<
+  HTMLDivElement,
+  React.ComponentProps<"div"> &
+    Pick<RechartsPrimitive.LegendProps, "payload" | "verticalAlign"> & {
+      hideIcon?: boolean;
+      nameKey?: string;
+    }
+>(({ className, hideIcon = false, payload, verticalAlign = "bottom", nameKey }, ref) => {
+  const { config } = useChart();
+
+  if (!payload?.length) {
+    return null;
+  }
+
+  return (
+    <div
+      ref={ref}
+      className={cn(
+        "flex items-center justify-center gap-4",
+        verticalAlign === "top" ? "pb-3" : "pt-3",
+        className
+      )}
+    >
+      {payload.map((item) => {
+        const key = `${nameKey || item.dataKey || "value"}`;
+        const itemConfig = getPayloadConfigFromPayload(config, item, key);
+
+        return (
+          <div
+            key={item.value}
+            className={cn(
+              "flex items-center gap-1.5 [&>svg]:h-3 [&>svg]:w-3 [&>svg]:text-text-dimmed"
+            )}
+          >
+            {itemConfig?.icon && !hideIcon ? (
+              <itemConfig.icon />
+            ) : (
+              <div
+                className="h-2 w-2 shrink-0 rounded-[2px]"
+                style={{
+                  backgroundColor: item.color,
+                }}
+              />
+            )}
+            {itemConfig?.label}
+          </div>
+        );
+      })}
+    </div>
+  );
+});
+ChartLegendContent.displayName = "ChartLegend";
+
+// Helper to extract item config from a payload.
+function getPayloadConfigFromPayload(config: ChartConfig, payload: unknown, key: string) {
+  if (typeof payload !== "object" || payload === null) {
+    return undefined;
+  }
+
+  const payloadPayload =
+    "payload" in payload && typeof payload.payload === "object" && payload.payload !== null
+      ? payload.payload
+      : undefined;
+
+  let configLabelKey: string = key;
+
+  if (key in payload && typeof payload[key as keyof typeof payload] === "string") {
+    configLabelKey = payload[key as keyof typeof payload] as string;
+  } else if (
+    payloadPayload &&
+    key in payloadPayload &&
+    typeof payloadPayload[key as keyof typeof payloadPayload] === "string"
+  ) {
+    configLabelKey = payloadPayload[key as keyof typeof payloadPayload] as string;
+  }
+
+  return configLabelKey in config ? config[configLabelKey] : config[key as keyof typeof config];
+}
+
+export {
+  ChartContainer,
+  ChartTooltip,
+  ChartTooltipContent,
+  ChartLegend,
+  ChartLegendContent,
+  ChartStyle,
+};
@@ -2,7 +2,7 @@ import { cn } from "~/utils/cn";

 const headerVariants = {
  header1: {
-    text: "font-sans text-2xl leading-5 md:leading-6 lg:leading-7 font-semibold",
+    text: "font-sans text-2xl leading-5 md:leading-6 lg:leading-7 font-medium",
    spacing: "mb-2",
  },
  header2: {
@@ -0,0 +1,71 @@
+import { LinkButton } from "./Buttons";
+import { Header3 } from "./Headers";
+import { Paragraph } from "./Paragraph";
+import { cn } from "~/utils/cn";
+
+const variants = {
+  info: {
+    panelStyle: "border-grid-bright bg-background-bright",
+  },
+  upgrade: {
+    panelStyle: "border-indigo-400/20 bg-indigo-800/10",
+  },
+};
+
+type InfoPanelVariant = keyof typeof variants;
+
+type Props = {
+  title?: string;
+  children: React.ReactNode;
+  to?: string;
+  buttonLabel?: string;
+  icon: React.ComponentType<any>;
+  iconClassName?: string;
+  variant?: InfoPanelVariant;
+  panelClassName?: string;
+};
+
+export function InfoPanel({
+  title,
+  children,
+  to,
+  buttonLabel,
+  icon,
+  iconClassName,
+  variant = "info",
+  panelClassName = "max-w-sm",
+}: Props) {
+  const Icon = icon;
+  const variantStyle = variants[variant];
+
+  return (
+    <div
+      className={cn(
+        variantStyle.panelStyle,
+        title ? "flex-col" : "",
+        "flex h-fit items-start gap-3 rounded-md border p-4",
+        panelClassName
+      )}
+    >
+      <div className={cn("flex items-center gap-2", to ? "w-full justify-between" : "")}>
+        <Icon className={cn("size-5", iconClassName)} />
+
+        {to && (
+          <LinkButton to={to} variant="secondary/small">
+            {buttonLabel}
+          </LinkButton>
+        )}
+      </div>
+      <div className="flex flex-col gap-1">
+        {title && <Header3 className="text-text-bright">{title}</Header3>}
+        {typeof children === "string" ? (
+          <Paragraph variant={"small"} className="text-text-dimmed">
+            {children}
+          </Paragraph>
+        ) : (
+          children
+        )}
+      </div>
+    </div>
+  );
+}
@@ -1,10 +1,8 @@
 import { ArrowUpRightIcon } from "@heroicons/react/20/solid";
 import { Link, useNavigation } from "@remix-run/react";
 import { useOptionalOrganization } from "~/hooks/useOrganizations";
-import { useOptionalProject } from "~/hooks/useProject";
 import { cn } from "~/utils/cn";
-import { plansPath } from "~/utils/pathBuilder";
-import { UpgradePrompt, useShowUpgradePrompt } from "../billing/UpgradePrompt";
+import { UpgradePrompt, useShowUpgradePrompt } from "../billing/v3/UpgradePrompt";
 import { BreadcrumbIcon } from "./BreadcrumbIcon";
 import { LinkButton } from "./Buttons";
 import { Header2 } from "./Headers";
@@ -23,21 +21,14 @@ export function NavBar({ children }: WithChildren) {
  const showUpgradePrompt = useShowUpgradePrompt(organization);
  const navigation = useNavigation();
  const isLoading = navigation.state === "loading" || navigation.state === "submitting";
-  const project = useOptionalProject();

  return (
    <div>
      <div className="grid h-10 w-full grid-rows-[auto_1px] bg-background-bright">
-        <div className="flex w-full items-center justify-between pl-3 pr-1">{children}</div>
+        <div className="flex w-full items-center justify-between pl-3 pr-2">{children}</div>
        <LoadingBarDivider isLoading={isLoading} />
      </div>
-      {showUpgradePrompt.shouldShow && organization && (
-        <UpgradePrompt
-          runsEnabled={showUpgradePrompt.runsEnabled}
-          runCountCap={showUpgradePrompt.runCountCap}
-          planPath={plansPath(organization)}
-        />
-      )}
+      {showUpgradePrompt.shouldShow && organization && <UpgradePrompt />}
    </div>
  );
 }
@@ -70,7 +61,7 @@ export function PageTitle({ title, backButton }: PageTitleProps) {
 }

 export function PageAccessories({ children }: WithChildren) {
-  return <div className="flex items-center gap-3">{children}</div>;
+  return <div className="flex items-center gap-2">{children}</div>;
 }

 export function PageInfoRow({ children, className }: WithChildren) {
@@ -3,6 +3,7 @@ import { Link } from "@remix-run/react";
 import { ReactNode, forwardRef, useState } from "react";
 import { cn } from "~/utils/cn";
 import { Popover, PopoverContent, PopoverVerticalEllipseTrigger } from "./Popover";
+import { InfoIconTooltip } from "./Tooltip";

 type TableProps = {
  containerClassName?: string;
@@ -87,10 +88,11 @@ type TableCellBasicProps = {

 type TableHeaderCellProps = TableCellBasicProps & {
  hiddenLabel?: boolean;
+  tooltip?: ReactNode;
 };

 export const TableHeaderCell = forwardRef<HTMLTableCellElement, TableHeaderCellProps>(
-  ({ className, alignment = "left", children, colSpan, hiddenLabel = false }, ref) => {
+  ({ className, alignment = "left", children, colSpan, hiddenLabel = false, tooltip }, ref) => {
    let alignmentClassName = "text-left";
    switch (alignment) {
      case "center":
@@ -112,7 +114,16 @@ export const TableHeaderCell = forwardRef<HTMLTableCellElement, TableHeaderCellP
        )}
        colSpan={colSpan}
      >
-        {hiddenLabel ? <span className="sr-only">{children}</span> : children}
+        {hiddenLabel ? (
+          <span className="sr-only">{children}</span>
+        ) : tooltip ? (
+          <div className="flex items-center gap-1">
+            {children}
+            <InfoIconTooltip content={tooltip} contentClassName="normal-case tracking-normal" />
+          </div>
+        ) : (
+          children
+        )}
      </th>
    );
  }
@@ -5,7 +5,7 @@ import { InformationCircleIcon } from "@heroicons/react/20/solid";

 const variantClasses = {
  basic:
-    "bg-background-dimmed border border-charcoal-700 rounded-md px-3 py-1.5 text-sm text-text-bright shadow-md fade-in-50",
+    "bg-background-bright border border-grid-bright rounded px-3 py-2 text-sm text-text-bright shadow-md fade-in-50",
  dark: "bg-background-dimmed border border-grid-bright rounded px-3 py-2 text-sm text-text-bright shadow-md fade-in-50",
 };

@@ -58,6 +58,7 @@ function SimpleTooltip({
  variant,
  disableHoverableContent = false,
  className,
+  buttonClassName,
 }: {
  button: React.ReactNode;
  content: React.ReactNode;
@@ -66,11 +67,12 @@ function SimpleTooltip({
  variant?: Variant;
  disableHoverableContent?: boolean;
  className?: string;
+  buttonClassName?: string;
 }) {
  return (
    <TooltipProvider disableHoverableContent={disableHoverableContent}>
      <Tooltip>
-        <TooltipTrigger className="h-fit">{button}</TooltipTrigger>
+        <TooltipTrigger className={cn("h-fit", buttonClassName)}>{button}</TooltipTrigger>
        <TooltipContent
          side={side}
          hidden={hidden}
@@ -87,17 +89,22 @@ function SimpleTooltip({
 export function InfoIconTooltip({
  content,
  buttonClassName,
+  contentClassName,
+  variant = "basic",
 }: {
  content: React.ReactNode;
  buttonClassName?: string;
+  contentClassName?: string;
+  variant?: Variant;
 }) {
  return (
    <SimpleTooltip
      button={
-        <InformationCircleIcon className={cn("h-3.5 w-3.5 text-text-dimmed", buttonClassName)} />
+        <InformationCircleIcon className={cn("size-3.5 text-text-dimmed", buttonClassName)} />
      }
      content={content}
-      variant="dark"
+      variant={variant}
+      className={contentClassName}
    />
  );
 }
@@ -528,7 +528,7 @@ export type Tree<TData> = {
 /** A tree but flattened so it can easily be used for DOM elements */
 export type FlatTreeItem<TData> = {
  id: string;
-  parentId: string | undefined;
+  parentId?: string | undefined;
  children: string[];
  hasChildren: boolean;
  /** The indentation level, the root is 0 */
@@ -23,8 +23,7 @@ export function CancelRunDialog({ runFriendlyId, redirectPath }: CancelRunDialog
    <DialogContent key="cancel">
      <DialogHeader>Cancel this run?</DialogHeader>
      <DialogDescription>
-        Canceling a run will stop execution. If you want to run this later you will have to replay
-        the entire run with the original payload.
+        Canceling a run will stop execution, along with any executing subtasks.
      </DialogDescription>
      <DialogFooter>
        <Form action={`/resources/taskruns/${runFriendlyId}/cancel`} method="post">
@@ -146,7 +146,6 @@ function FilterMenu(props: RunFiltersProps) {

  const filterTrigger = (
    <SelectTrigger
-      autoFocus
      icon={
        <div className="flex size-4 items-center justify-center">
          <ListFilterIcon className="size-3.5" />
@@ -3,10 +3,12 @@ import {
  BoltSlashIcon,
  BugAntIcon,
  CheckCircleIcon,
+  ClockIcon,
  FireIcon,
  NoSymbolIcon,
  PauseCircleIcon,
  RectangleStackIcon,
+  TrashIcon,
  XCircleIcon,
 } from "@heroicons/react/20/solid";
 import { TaskRunStatus } from "@trigger.dev/database";
@@ -16,6 +18,7 @@ import { Spinner } from "~/components/primitives/Spinner";
 import { cn } from "~/utils/cn";

 export const allTaskRunStatuses = [
+  "DELAYED",
  "WAITING_FOR_DEPLOY",
  "PENDING",
  "EXECUTING",
@@ -28,10 +31,12 @@ export const allTaskRunStatuses = [
  "PAUSED",
  "INTERRUPTED",
  "SYSTEM_FAILURE",
+  "EXPIRED",
 ] as const satisfies Readonly<Array<TaskRunStatus>>;

 export const filterableTaskRunStatuses = [
  "WAITING_FOR_DEPLOY",
+  "DELAYED",
  "PENDING",
  "EXECUTING",
  "RETRYING_AFTER_FAILURE",
@@ -42,9 +47,11 @@ export const filterableTaskRunStatuses = [
  "CRASHED",
  "INTERRUPTED",
  "SYSTEM_FAILURE",
+  "EXPIRED",
 ] as const satisfies Readonly<Array<TaskRunStatus>>;

 const taskRunStatusDescriptions: Record<TaskRunStatus, string> = {
+  DELAYED: "Task has been delayed and is waiting to be executed",
  PENDING: "Task is waiting to be executed",
  WAITING_FOR_DEPLOY: "Task needs to be deployed first to start executing",
  EXECUTING: "Task is currently being executed",
@@ -57,9 +64,10 @@ const taskRunStatusDescriptions: Record<TaskRunStatus, string> = {
  SYSTEM_FAILURE: "Task has failed due to a system failure",
  PAUSED: "Task has been paused by the user",
  CRASHED: "Task has crashed and won't be retried",
+  EXPIRED: "Task has surpassed its ttl and won't be executed",
 };

-export const QUEUED_STATUSES: TaskRunStatus[] = ["PENDING", "WAITING_FOR_DEPLOY"];
+export const QUEUED_STATUSES: TaskRunStatus[] = ["PENDING", "WAITING_FOR_DEPLOY", "DELAYED"];

 export const RUNNING_STATUSES: TaskRunStatus[] = [
  "EXECUTING",
@@ -74,6 +82,7 @@ export const FINISHED_STATUSES: TaskRunStatus[] = [
  "INTERRUPTED",
  "SYSTEM_FAILURE",
  "CRASHED",
+  "EXPIRED",
 ];

 export function descriptionForTaskRunStatus(status: TaskRunStatus): string {
@@ -109,6 +118,8 @@ export function TaskRunStatusIcon({
  className: string;
 }) {
  switch (status) {
+    case "DELAYED":
+      return <ClockIcon className={cn(runStatusClassNameColor(status), className)} />;
    case "PENDING":
      return <RectangleStackIcon className={cn(runStatusClassNameColor(status), className)} />;
    case "WAITING_FOR_DEPLOY":
@@ -133,6 +144,8 @@ export function TaskRunStatusIcon({
      return <BugAntIcon className={cn(runStatusClassNameColor(status), className)} />;
    case "CRASHED":
      return <FireIcon className={cn(runStatusClassNameColor(status), className)} />;
+    case "EXPIRED":
+      return <TrashIcon className={cn(runStatusClassNameColor(status), className)} />;

    default: {
      assertNever(status);
@@ -143,6 +156,7 @@ export function TaskRunStatusIcon({
 export function runStatusClassNameColor(status: TaskRunStatus): string {
  switch (status) {
    case "PENDING":
+    case "DELAYED":
      return "text-charcoal-500";
    case "WAITING_FOR_DEPLOY":
      return "text-amber-500";
@@ -154,6 +168,7 @@ export function runStatusClassNameColor(status: TaskRunStatus): string {
    case "PAUSED":
      return "text-amber-300";
    case "CANCELED":
+    case "EXPIRED":
      return "text-charcoal-500";
    case "INTERRUPTED":
      return "text-error";
@@ -173,6 +188,8 @@ export function runStatusClassNameColor(status: TaskRunStatus): string {

 export function runStatusTitle(status: TaskRunStatus): string {
  switch (status) {
+    case "DELAYED":
+      return "Delayed";
    case "PENDING":
      return "Queued";
    case "WAITING_FOR_DEPLOY":
@@ -197,6 +214,8 @@ export function runStatusTitle(status: TaskRunStatus): string {
      return "System failure";
    case "CRASHED":
      return "Crashed";
+    case "EXPIRED":
+      return "Expired";
    default: {
      assertNever(status);
    }
@@ -1,8 +1,14 @@
-import { ArrowPathIcon, StopCircleIcon } from "@heroicons/react/20/solid";
+import {
+  ArrowPathIcon,
+  ClockIcon,
+  CpuChipIcon,
+  RectangleStackIcon,
+  StopCircleIcon,
+} from "@heroicons/react/20/solid";
 import { StopIcon } from "@heroicons/react/24/outline";
 import { BeakerIcon, BookOpenIcon, CheckIcon } from "@heroicons/react/24/solid";
 import { useLocation } from "@remix-run/react";
-import { formatDuration } from "@trigger.dev/core/v3";
+import { formatDuration, formatDurationMilliseconds } from "@trigger.dev/core/v3";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
 import { Dialog, DialogTrigger } from "~/components/primitives/Dialog";
 import { useEnvironments } from "~/hooks/useEnvironments";
@@ -32,7 +38,11 @@ import { useSelectedItems } from "~/components/primitives/SelectedItemsProvider"
 import { Checkbox } from "~/components/primitives/Checkbox";
 import { useCallback, useRef } from "react";
 import { run } from "@remix-run/dev/dist/cli/run";
-import { formatNumber } from "~/utils/numberFormatter";
+import { formatCurrency, formatCurrencyAccurate, formatNumber } from "~/utils/numberFormatter";
+import { useFeatures } from "~/hooks/useFeatures";
+import { useUser } from "~/hooks/useUser";
+import { SimpleTooltip } from "~/components/primitives/Tooltip";
+import { Header3 } from "~/components/primitives/Headers";

 type RunsTableProps = {
  total: number;
@@ -52,10 +62,14 @@ export function TaskRunsTable({
  isLoading = false,
  allowSelection = false,
 }: RunsTableProps) {
+  const user = useUser();
  const organization = useOrganization();
  const project = useProject();
  const checkboxes = useRef<(HTMLInputElement | null)[]>([]);
  const { selectedItems, has, hasAll, select, deselect, toggle } = useSelectedItems(allowSelection);
+  const { isManagedCloud } = useFeatures();
+
+  const showCompute = user.admin && isManagedCloud;

  const navigateCheckboxes = useCallback(
    (event: React.KeyboardEvent<HTMLInputElement>, index: number) => {
@@ -110,14 +124,57 @@ export function TaskRunsTable({
            </TableHeaderCell>
          )}
          <TableHeaderCell alignment="right">Run #</TableHeaderCell>
+          <TableHeaderCell>Env</TableHeaderCell>
          <TableHeaderCell>Task ID</TableHeaderCell>
          <TableHeaderCell>Version</TableHeaderCell>
-          <TableHeaderCell>Env</TableHeaderCell>
          <TableHeaderCell>Status</TableHeaderCell>
          <TableHeaderCell>Started</TableHeaderCell>
-          <TableHeaderCell>Duration</TableHeaderCell>
+          <TableHeaderCell
+            colSpan={3}
+            tooltip={
+              <div className="flex max-w-xs flex-col gap-4 p-1">
+                <div>
+                  <div className="mb-0.5 flex items-center gap-1.5">
+                    <RectangleStackIcon className="size-4 text-text-dimmed" />
+                    <Header3>Queued duration</Header3>
+                  </div>
+                  <Paragraph variant="small" className="!text-wrap text-text-dimmed">
+                    The amount of time from when the run was created to it starting to run.
+                  </Paragraph>
+                </div>
+                <div>
+                  <div className="mb-0.5 flex items-center gap-1.5">
+                    <ClockIcon className="size-4 text-blue-500" /> <Header3>Run duration</Header3>
+                  </div>
+                  <Paragraph variant="small" className="!text-wrap text-text-dimmed">
+                    The total amount of time from the run starting to it finishing. This includes
+                    all time spent waiting.
+                  </Paragraph>
+                </div>
+                <div>
+                  <div className="mb-0.5 flex items-center gap-1.5">
+                    <CpuChipIcon className="size-4 text-success" />
+                    <Header3>Compute duration</Header3>
+                  </div>
+                  <Paragraph variant="small" className="!text-wrap text-text-dimmed">
+                    The amount of compute time used in the run. This does not include time spent
+                    waiting.
+                  </Paragraph>
+                </div>
+              </div>
+            }
+          >
+            Duration
+          </TableHeaderCell>
+          {showCompute && (
+            <>
+              <TableHeaderCell>Compute</TableHeaderCell>
+            </>
+          )}
          <TableHeaderCell>Test</TableHeaderCell>
          <TableHeaderCell>Created at</TableHeaderCell>
+          <TableHeaderCell>Delayed until</TableHeaderCell>
+          <TableHeaderCell>TTL</TableHeaderCell>
          <TableHeaderCell>
            <span className="sr-only">Go to page</span>
          </TableHeaderCell>
@@ -125,7 +182,7 @@ export function TaskRunsTable({
      </TableHeader>
      <TableBody>
        {total === 0 && !hasFilters ? (
-          <TableBlankRow colSpan={9}>
+          <TableBlankRow colSpan={14}>
            {!isLoading && <NoRuns title="No runs found" />}
          </TableBlankRow>
        ) : runs.length === 0 ? (
@@ -152,41 +209,71 @@ export function TaskRunsTable({
                <TableCell to={path} alignment="right">
                  {formatNumber(run.number)}
                </TableCell>
-                <TableCell to={path}>{run.taskIdentifier}</TableCell>
-                <TableCell to={path}>{run.version ?? "–"}</TableCell>
                <TableCell to={path}>
                  <EnvironmentLabel
                    environment={run.environment}
                    userName={run.environment.userName}
                  />
                </TableCell>
+                <TableCell to={path}>{run.taskIdentifier}</TableCell>
+                <TableCell to={path}>{run.version ?? "–"}</TableCell>
                <TableCell to={path}>
                  <TaskRunStatusCombo status={run.status} />
                </TableCell>
                <TableCell to={path}>
                  {run.startedAt ? <DateTime date={run.startedAt} /> : "–"}
                </TableCell>
-                <TableCell to={path}>
-                  {run.startedAt && run.finishedAt ? (
-                    formatDuration(new Date(run.startedAt), new Date(run.finishedAt), {
-                      style: "short",
-                    })
-                  ) : run.startedAt ? (
-                    <LiveTimer startTime={new Date(run.startedAt)} />
-                  ) : (
-                    "–"
-                  )}
+                <TableCell to={path} className="w-[1%]" actionClassName="pr-0 tabular-nums">
+                  <div className="flex items-center gap-1">
+                    <RectangleStackIcon className="size-4 text-text-dimmed" />
+                    {run.startedAt ? (
+                      formatDuration(new Date(run.createdAt), new Date(run.startedAt), {
+                        style: "short",
+                      })
+                    ) : (
+                      <LiveTimer startTime={new Date(run.createdAt)} />
+                    )}
+                  </div>
                </TableCell>
+                <TableCell to={path} className="w-[1%]" actionClassName="px-4 tabular-nums">
+                  <div className="flex items-center gap-1">
+                    <ClockIcon className="size-4 text-blue-500" />
+                    {run.startedAt && run.finishedAt ? (
+                      formatDuration(new Date(run.startedAt), new Date(run.finishedAt), {
+                        style: "short",
+                      })
+                    ) : run.startedAt ? (
+                      <LiveTimer startTime={new Date(run.startedAt)} />
+                    ) : (
+                      "–"
+                    )}
+                  </div>
+                </TableCell>
+                <TableCell to={path} actionClassName="pl-0 tabular-nums">
+                  <div className="flex items-center gap-1">
+                    <CpuChipIcon className="size-4 text-success" />
+                    {run.usageDurationMs > 0
+                      ? formatDurationMilliseconds(run.usageDurationMs, {
+                          style: "short",
+                        })
+                      : "–"}
+                  </div>
+                </TableCell>
+                {showCompute && (
+                  <TableCell to={path} className="tabular-nums">
+                    {run.costInCents > 0 ? formatCurrencyAccurate(run.costInCents / 100) : "–"}
+                  </TableCell>
+                )}
                <TableCell to={path}>
-                  {run.isTest ? (
-                    <CheckIcon className="h-4 w-4 text-charcoal-400" />
-                  ) : (
-                    <StopIcon className="h-4 w-4 text-charcoal-850" />
-                  )}
+                  {run.isTest ? <CheckIcon className="h-4 w-4 text-charcoal-400" /> : "–"}
                </TableCell>
                <TableCell to={path}>
                  {run.createdAt ? <DateTime date={run.createdAt} /> : "–"}
                </TableCell>
+                <TableCell to={path}>
+                  {run.delayUntil ? <DateTime date={run.delayUntil} /> : "–"}
+                </TableCell>
+                <TableCell to={path}>{run.ttl ?? "–"}</TableCell>
                <RunActionsCell run={run} path={path} />
              </TableRow>
            );
@@ -194,7 +281,7 @@ export function TaskRunsTable({
        )}
        {isLoading && (
          <TableBlankRow
-            colSpan={8}
+            colSpan={14}
            className="absolute left-0 top-0 flex h-full w-full items-center justify-center gap-2 bg-charcoal-900/90"
          >
            <Spinner /> <span className="text-text-dimmed">Loading…</span>
@@ -254,7 +341,7 @@ function BlankState({ isLoading, filters }: Pick<RunsTableProps, "isLoading" | "
  const organization = useOrganization();
  const project = useProject();
  const envs = useEnvironments();
-  if (isLoading) return <TableBlankRow colSpan={9}></TableBlankRow>;
+  if (isLoading) return <TableBlankRow colSpan={14}></TableBlankRow>;

  const { environments, tasks, from, to, ...otherFilters } = filters;

@@ -267,7 +354,7 @@ function BlankState({ isLoading, filters }: Pick<RunsTableProps, "isLoading" | "
  ) {
    const environment = envs?.find((env) => env.id === filters.environments[0]);
    return (
-      <TableBlankRow colSpan={10}>
+      <TableBlankRow colSpan={14}>
        <div className="py-14">
          <Paragraph className="w-auto" variant="base/bright" spacing>
            There are no runs for {filters.tasks[0]}
@@ -308,7 +395,7 @@ function BlankState({ isLoading, filters }: Pick<RunsTableProps, "isLoading" | "
  }

  return (
-    <TableBlankRow colSpan={10}>
+    <TableBlankRow colSpan={14}>
      <div className="flex flex-col items-center justify-center gap-2">
        <Paragraph className="w-auto" variant="small">
          No runs currently match your filters. Try refreshing or modifying your filters.
@@ -40,6 +40,8 @@ export const TaskRunStatus = {
  COMPLETED_WITH_ERRORS: "COMPLETED_WITH_ERRORS",
  SYSTEM_FAILURE: "SYSTEM_FAILURE",
  CRASHED: "CRASHED",
+  DELAYED: "DELAYED",
+  EXPIRED: "EXPIRED",
 } as const satisfies Record<TaskRunStatusType, TaskRunStatusType>;

 export const JobRunStatus = {
@@ -16,7 +16,6 @@ import {
 } from "./components/primitives/OperatingSystemProvider";
 import { getSharedSqsEventConsumer } from "./services/events/sqsEventConsumer";
 import { singleton } from "./utils/singleton";
-import { logger } from "./services/logger.server";

 const ABORT_DELAY = 30000;

@@ -180,9 +179,42 @@ function logError(error: unknown, request?: Request) {
  }
 }

+process.on("uncaughtException", (error, origin) => {
+  if (
+    error instanceof Prisma.PrismaClientKnownRequestError ||
+    error instanceof Prisma.PrismaClientUnknownRequestError
+  ) {
+    // Don't exit the process if the error is a Prisma error
+    logger.error("uncaughtException prisma error", {
+      error,
+      prismaMessage: error.message,
+      code: "code" in error ? error.code : undefined,
+      meta: "meta" in error ? error.meta : undefined,
+      stack: error.stack,
+      origin,
+    });
+  } else {
+    logger.error("uncaughtException", {
+      error: { name: error.name, message: error.message, stack: error.stack },
+      origin,
+    });
+  }
+
+  process.exit(1);
+});
+
 const sqsEventConsumer = singleton("sqsEventConsumer", getSharedSqsEventConsumer);

 export { apiRateLimiter } from "./services/apiRateLimit.server";
 export { socketIo } from "./v3/handleSocketIo.server";
 export { wss } from "./v3/handleWebsockets.server";
 export { registryProxy } from "./v3/registryProxy.server";
+export { runWithHttpContext } from "./services/httpAsyncStorage.server";
+import { eventLoopMonitor } from "./eventLoopMonitor.server";
+import { env } from "./env.server";
+import { logger } from "./services/logger.server";
+import { Prisma } from "./db.server";
+
+if (env.EVENT_LOOP_MONITOR_ENABLED === "1") {
+  eventLoopMonitor.enable();
+}
@@ -97,8 +97,9 @@ const EnvironmentSchema = z.object({
   * @example "1000ms"
   * @example "1000s"
   */
-  API_RATE_LIMIT_WINDOW: z.string().default("60s"),
-  API_RATE_LIMIT_MAX: z.coerce.number().int().default(600),
+  API_RATE_LIMIT_REFILL_INTERVAL: z.string().default("10s"), // refill 250 tokens every 10 seconds
+  API_RATE_LIMIT_MAX: z.coerce.number().int().default(750), // allow bursts of 750 requests
+  API_RATE_LIMIT_REFILL_RATE: z.coerce.number().int().default(250), // refix 250 tokens every 10 seconds
  API_RATE_LIMIT_REQUEST_LOGS_ENABLED: z.string().default("0"),
  API_RATE_LIMIT_REJECTION_LOGS_ENABLED: z.string().default("1"),

@@ -151,7 +152,7 @@ const EnvironmentSchema = z.object({
  PROD_OTEL_LOG_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"),
  PROD_OTEL_LOG_MAX_QUEUE_SIZE: z.string().default("512"),

-  RUNTIME_WAIT_THRESHOLD_IN_MS: z.coerce.number().int().default(30000),
+  CHECKPOINT_THRESHOLD_IN_MS: z.coerce.number().int().default(30000),

  // Internal OTEL environment variables
  INTERNAL_OTEL_TRACE_EXPORTER_URL: z.string().optional(),
@@ -193,17 +194,17 @@ const EnvironmentSchema = z.object({
  USAGE_EVENT_URL: z.string().optional(),
  PROD_USAGE_HEARTBEAT_INTERVAL_MS: z.coerce.number().int().optional(),

-  CENTS_PER_HOUR_MICRO: z.coerce.number().default(0),
-  CENTS_PER_HOUR_SMALL_1X: z.coerce.number().default(0),
-  CENTS_PER_HOUR_SMALL_2X: z.coerce.number().default(0),
-  CENTS_PER_HOUR_MEDIUM_1X: z.coerce.number().default(0),
-  CENTS_PER_HOUR_MEDIUM_2X: z.coerce.number().default(0),
-  CENTS_PER_HOUR_LARGE_1X: z.coerce.number().default(0),
-  CENTS_PER_HOUR_LARGE_2X: z.coerce.number().default(0),
-  BASE_RUN_COST_IN_CENTS: z.coerce.number().default(0),
+  CENTS_PER_VCPU_SECOND: z.coerce.number().default(0),
+  CENTS_PER_GB_RAM_SECOND: z.coerce.number().default(0),
+  CENTS_PER_RUN: z.coerce.number().default(0),

  USAGE_OPEN_METER_API_KEY: z.string().optional(),
  USAGE_OPEN_METER_BASE_URL: z.string().optional(),
+  EVENT_LOOP_MONITOR_ENABLED: z.string().default("1"),
+  MAXIMUM_LIVE_RELOADING_EVENTS: z.coerce.number().int().default(1000),
+  MAXIMUM_TRACE_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(25_000),
+  TASK_PAYLOAD_OFFLOAD_THRESHOLD: z.coerce.number().int().default(524_288), // 512KB
+  TASK_PAYLOAD_MAXIMUM_SIZE: z.coerce.number().int().default(3_145_728), // 3MB
 });

 export type Environment = z.infer<typeof EnvironmentSchema>;
@@ -0,0 +1,77 @@
+import { createHook } from "node:async_hooks";
+import { singleton } from "./utils/singleton";
+import { tracer } from "./v3/tracer.server";
+
+const THRESHOLD_NS = 1e8; // 100ms
+
+const cache = new Map<number, { type: string; start?: [number, number] }>();
+
+function init(asyncId: number, type: string, triggerAsyncId: number, resource: any) {
+  cache.set(asyncId, {
+    type,
+  });
+}
+
+function destroy(asyncId: number) {
+  cache.delete(asyncId);
+}
+
+function before(asyncId: number) {
+  const cached = cache.get(asyncId);
+
+  if (!cached) {
+    return;
+  }
+
+  cache.set(asyncId, {
+    ...cached,
+    start: process.hrtime(),
+  });
+}
+
+function after(asyncId: number) {
+  const cached = cache.get(asyncId);
+
+  if (!cached) {
+    return;
+  }
+
+  cache.delete(asyncId);
+
+  if (!cached.start) {
+    return;
+  }
+
+  const diff = process.hrtime(cached.start);
+  const diffNs = diff[0] * 1e9 + diff[1];
+  if (diffNs > THRESHOLD_NS) {
+    const time = diffNs / 1e6; // in ms
+
+    const newSpan = tracer.startSpan("event-loop-blocked", {
+      startTime: new Date(new Date().getTime() - time),
+      attributes: {
+        asyncType: cached.type,
+        label: "EventLoopMonitor",
+      },
+    });
+
+    newSpan.end();
+  }
+}
+
+export const eventLoopMonitor = singleton("eventLoopMonitor", () => {
+  const hook = createHook({ init, before, after, destroy });
+
+  return {
+    enable: () => {
+      console.log("🥸  Initializing event loop monitor");
+
+      hook.enable();
+    },
+    disable: () => {
+      console.log("🥸  Disabling event loop monitor");
+
+      hook.disable();
+    },
+  };
+});
@@ -26,7 +26,7 @@ export function useEventSource(
    const eventSource = new EventSource(url, init);
    eventSource.addEventListener(event ?? "message", handler);

-    // rest data if dependencies change
+    // reset data if dependencies change
    setData(null);

    function handler(event: MessageEvent) {
@@ -84,7 +84,9 @@ export function createPkApiKeyForEnv(envType: RuntimeEnvironment["type"]) {
  return `pk_${envSlug(envType)}_${apiKeyId(20)}`;
 }

-export function envSlug(environmentType: RuntimeEnvironment["type"]) {
+export type EnvSlug = "dev" | "stg" | "prod" | "prev";
+
+export function envSlug(environmentType: RuntimeEnvironment["type"]): EnvSlug {
  switch (environmentType) {
    case "DEVELOPMENT": {
      return "dev";
@@ -100,3 +102,7 @@ export function envSlug(environmentType: RuntimeEnvironment["type"]) {
    }
  }
 }
+
+export function isEnvSlug(maybeSlug: string): maybeSlug is EnvSlug {
+  return ["dev", "stg", "prod", "prev"].includes(maybeSlug);
+}
@@ -1,9 +1,15 @@
 import { prisma } from "~/db.server";
 import { createEnvironment } from "./organization.server";

-export async function getTeamMembersAndInvites({ userId, slug }: { userId: string; slug: string }) {
+export async function getTeamMembersAndInvites({
+  userId,
+  organizationId,
+}: {
+  userId: string;
+  organizationId: string;
+}) {
  const org = await prisma.organization.findFirst({
-    where: { slug, members: { some: { userId } } },
+    where: { id: organizationId, members: { some: { userId } } },
    select: {
      members: {
        select: {
@@ -77,7 +77,7 @@ export async function createOrganization(
 }

 export async function createEnvironment(
-  organization: Pick<Organization, "id">,
+  organization: Pick<Organization, "id" | "maximumConcurrencyLimit">,
  project: Pick<Project, "id">,
  type: RuntimeEnvironment["type"],
  member?: OrgMember,
@@ -95,7 +95,7 @@ export async function createEnvironment(
      pkApiKey,
      shortcode,
      autoEnableInternalSources: type !== "DEVELOPMENT",
-      maximumConcurrencyLimit: env.DEFAULT_ENV_EXECUTION_CONCURRENCY_LIMIT,
+      maximumConcurrencyLimit: organization.maximumConcurrencyLimit / 3,
      organization: {
        connect: {
          id: organization.id,
@@ -4,17 +4,20 @@ import { prisma } from "~/db.server";
 import type { Project } from "@trigger.dev/database";
 import { Organization, createEnvironment } from "./organization.server";
 import { env } from "~/env.server";
+import { projectCreated } from "~/services/platform.v3.server";
 export type { Project } from "@trigger.dev/database";

 const externalRefGenerator = customAlphabet("abcdefghijklmnopqrstuvwxyz", 20);

+type Options = {
+  organizationSlug: string;
+  name: string;
+  userId: string;
+  version: "v2" | "v3";
+};
+
 export async function createProject(
-  {
-    organizationSlug,
-    name,
-    userId,
-    version,
-  }: { organizationSlug: string; name: string; userId: string; version: "v2" | "v3" },
+  { organizationSlug, name, userId, version }: Options,
  attemptCount = 0
 ): Promise<Project & { organization: Organization }> {
  //check the user has permissions to do this
@@ -87,14 +90,12 @@ export async function createProject(
  // Create the dev and prod environments
  await createEnvironment(organization, project, "PRODUCTION");

-  if (version === "v2") {
-    await createEnvironment(organization, project, "STAGING");
-  }
-
  for (const member of project.organization.members) {
    await createEnvironment(organization, project, "DEVELOPMENT", member);
  }

+  await projectCreated(organization, project);
+
  return project;
 }

@@ -65,59 +65,55 @@ export async function findEnvironmentById(id: string) {
 }

 export async function createNewSession(environment: RuntimeEnvironment, ipAddress: string) {
-  return prisma.$transaction(async (tx) => {
-    const session = await tx.runtimeEnvironmentSession.create({
-      data: {
-        environmentId: environment.id,
-        ipAddress,
-      },
-    });
-
-    await tx.runtimeEnvironment.update({
-      where: {
-        id: environment.id,
-      },
-      data: {
-        currentSessionId: session.id,
-      },
-    });
-
-    return session;
+  const session = await prisma.runtimeEnvironmentSession.create({
+    data: {
+      environmentId: environment.id,
+      ipAddress,
+    },
  });
+
+  await prisma.runtimeEnvironment.update({
+    where: {
+      id: environment.id,
+    },
+    data: {
+      currentSessionId: session.id,
+    },
+  });
+
+  return session;
 }

 export async function disconnectSession(environmentId: string) {
-  return prisma.$transaction(async (tx) => {
-    const environment = await tx.runtimeEnvironment.findUnique({
-      where: {
-        id: environmentId,
-      },
-    });
-
-    if (!environment || !environment.currentSessionId) {
-      return null;
-    }
-
-    const session = await tx.runtimeEnvironmentSession.update({
-      where: {
-        id: environment.currentSessionId,
-      },
-      data: {
-        disconnectedAt: new Date(),
-      },
-    });
-
-    await tx.runtimeEnvironment.update({
-      where: {
-        id: environment.id,
-      },
-      data: {
-        currentSessionId: null,
-      },
-    });
-
-    return session;
+  const environment = await prisma.runtimeEnvironment.findUnique({
+    where: {
+      id: environmentId,
+    },
  });
+
+  if (!environment || !environment.currentSessionId) {
+    return null;
+  }
+
+  const session = await prisma.runtimeEnvironmentSession.update({
+    where: {
+      id: environment.currentSessionId,
+    },
+    data: {
+      disconnectedAt: new Date(),
+    },
+  });
+
+  await prisma.runtimeEnvironment.update({
+    where: {
+      id: environment.id,
+    },
+    data: {
+      currentSessionId: null,
+    },
+  });
+
+  return session;
 }

 type DisplayableInputEnvironment = Prisma.RuntimeEnvironmentGetPayload<{
@@ -118,6 +118,7 @@ export function batchTaskRunItemStatusForRunStatus(
    case TaskRunStatus.COMPLETED_WITH_ERRORS:
    case TaskRunStatus.SYSTEM_FAILURE:
    case TaskRunStatus.CRASHED:
+    case TaskRunStatus.EXPIRED:
      return BatchTaskRunItemStatus.FAILED;
    case TaskRunStatus.PENDING:
    case TaskRunStatus.WAITING_FOR_DEPLOY:
@@ -125,6 +126,7 @@ export function batchTaskRunItemStatusForRunStatus(
    case TaskRunStatus.RETRYING_AFTER_FAILURE:
    case TaskRunStatus.EXECUTING:
    case TaskRunStatus.PAUSED:
+    case TaskRunStatus.DELAYED:
      return BatchTaskRunItemStatus.PENDING;
    default:
      assertNever(status);
@@ -1,3 +1,4 @@
+import { SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
 import type {
  CronItem,
  CronItemOptions,
@@ -11,20 +12,19 @@ import type {
  WorkerUtils,
 } from "graphile-worker";
 import {
+  Logger as GraphileLogger,
  run as graphileRun,
  makeWorkerUtils,
  parseCronItems,
-  Logger as GraphileLogger,
 } from "graphile-worker";
-import { SpanKind, trace } from "@opentelemetry/api";

+import { flattenAttributes } from "@trigger.dev/core/v3";
 import omit from "lodash.omit";
 import { z } from "zod";
 import { $replica, PrismaClient, PrismaClientOrTransaction } from "~/db.server";
+import { env } from "~/env.server";
 import { PgListenService } from "~/services/db/pgListen.server";
 import { workerLogger as logger } from "~/services/logger.server";
-import { flattenAttributes } from "@trigger.dev/core/v3";
-import { env } from "~/env.server";

 const tracer = trace.getTracer("zodWorker", "3.0.0.dp.1");

@@ -338,11 +338,45 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
      }
    }

-    const { job, durationInMs } = await this.#addJob(
-      identifier as string,
-      payload,
-      spec,
-      options?.tx ?? this.#prisma
+    const { job, durationInMs } = await tracer.startActiveSpan(
+      `Enqueue ${identifier as string}`,
+      {
+        kind: SpanKind.PRODUCER,
+        attributes: {
+          "job.task_identifier": identifier as string,
+          "job.payload": payload,
+          "job.priority": spec.priority,
+          "job.run_at": spec.runAt?.toISOString(),
+          "job.jobKey": spec.jobKey,
+          "job.flags": spec.flags,
+          "job.max_attempts": spec.maxAttempts,
+          "worker.name": this.#name,
+        },
+      },
+      async (span) => {
+        try {
+          const results = await this.#addJob(
+            identifier as string,
+            payload,
+            spec,
+            options?.tx ?? this.#prisma
+          );
+
+          return results;
+        } catch (error) {
+          if (error instanceof Error) {
+            span.recordException(error);
+          } else {
+            span.recordException(new Error(String(error)));
+          }
+
+          span.setStatus({ code: SpanStatusCode.ERROR });
+
+          throw error;
+        } finally {
+          span.end();
+        }
+      }
    );

    logger.debug("Enqueued worker task", {
@@ -401,6 +435,12 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
    const rows = AddJobResultsSchema.safeParse(results);

    if (!rows.success) {
+      logger.debug("results returned from add_job could not be parsed", {
+        identifier,
+        payload,
+        spec,
+      });
+
      throw new Error(
        `Failed to add job to queue, zod parsing error: ${JSON.stringify(rows.error)}`
      );
@@ -422,9 +462,7 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
      const job = AddJobResultsSchema.safeParse(result);

      if (!job.success) {
-        logger.debug("results returned from remove_job could not be parsed", {
-          error: job.error.flatten(),
-          result,
+        logger.debug("could not remove job, job_key did not exist", {
          jobKey,
        });

@@ -1,4 +1,4 @@
-import { BillingService } from "../services/billing.server";
+import { BillingService } from "../services/billing.v2.server";
 import { BasePresenter } from "./v3/basePresenter.server";

 export class OrgBillingPlanPresenter extends BasePresenter {
@@ -1,7 +1,7 @@
-import { estimate } from "@trigger.dev/billing";
+import { estimate } from "@trigger.dev/platform/v2";
 import { sqlDatabaseSchema } from "~/db.server";
 import { featuresForRequest } from "~/features.server";
-import { BillingService } from "~/services/billing.server";
+import { BillingService } from "~/services/billing.v2.server";
 import { BasePresenter } from "./v3/basePresenter.server";

 export class OrgUsagePresenter extends BasePresenter {
@@ -1,11 +1,9 @@
-import { z } from "zod";
 import {
  Direction,
  FilterableEnvironment,
  FilterableStatus,
  filterableStatuses,
 } from "~/components/runs/RunStatuses";
-import { PrismaClient, prisma } from "~/db.server";
 import { getUsername } from "~/utils/username";
 import { BasePresenter } from "./v3/basePresenter.server";

@@ -29,8 +27,6 @@ const DEFAULT_PAGE_SIZE = 20;
 export type RunList = Awaited<ReturnType<RunListPresenter["call"]>>;

 export class RunListPresenter extends BasePresenter {
-  
-
  public async call({
    userId,
    eventId,
@@ -1,27 +1,52 @@
-import { User } from "@trigger.dev/database";
 import { ScheduleMetadataSchema } from "@trigger.dev/core";
-import { PrismaClient, prisma } from "~/db.server";
+import { User } from "@trigger.dev/database";
 import { Organization } from "~/models/organization.server";
 import { Project } from "~/models/project.server";
 import { calculateNextScheduledEvent } from "~/services/schedules/nextScheduledEvent.server";
+import { BasePresenter } from "./v3/basePresenter.server";

-export class ScheduledTriggersPresenter {
-  #prismaClient: PrismaClient;
-
-  constructor(prismaClient: PrismaClient = prisma) {
-    this.#prismaClient = prismaClient;
-  }
+const DEFAULT_PAGE_SIZE = 20;

+export class ScheduledTriggersPresenter extends BasePresenter {
  public async call({
    userId,
    projectSlug,
    organizationSlug,
+    direction = "forward",
+    pageSize = DEFAULT_PAGE_SIZE,
+    cursor,
  }: {
    userId: User["id"];
    projectSlug: Project["slug"];
    organizationSlug: Organization["slug"];
+    direction?: "forward" | "backward";
+    pageSize?: number;
+    cursor?: string;
  }) {
-    const scheduled = await this.#prismaClient.scheduleSource.findMany({
+    const organization = await this._replica.organization.findFirstOrThrow({
+      select: {
+        id: true,
+      },
+      where: {
+        slug: organizationSlug,
+        members: { some: { userId } },
+      },
+    });
+
+    // Find the project scoped to the organization
+    const project = await this._replica.project.findFirstOrThrow({
+      select: {
+        id: true,
+      },
+      where: {
+        slug: projectSlug,
+        organizationId: organization.id,
+      },
+    });
+
+    const directionMultiplier = direction === "forward" ? 1 : -1;
+
+    const scheduled = await this._replica.scheduleSource.findMany({
      select: {
        id: true,
        key: true,
@@ -50,23 +75,50 @@ export class ScheduledTriggersPresenter {
              },
            },
          ],
-          organization: {
-            slug: organizationSlug,
-            members: {
-              some: {
-                userId,
-              },
-            },
-          },
-          project: {
-            slug: projectSlug,
-          },
+          projectId: project.id,
        },
      },
+      orderBy: [{ id: "desc" }],
+      //take an extra record to tell if there are more
+      take: directionMultiplier * (pageSize + 1),
+      //skip the cursor if there is one
+      skip: cursor ? 1 : 0,
+      cursor: cursor
+        ? {
+            id: cursor,
+          }
+        : undefined,
    });

+    const hasMore = scheduled.length > pageSize;
+
+    //get cursors for next and previous pages
+    let next: string | undefined;
+    let previous: string | undefined;
+    switch (direction) {
+      case "forward":
+        previous = cursor ? scheduled.at(0)?.id : undefined;
+        if (hasMore) {
+          next = scheduled[pageSize - 1]?.id;
+        }
+        break;
+      case "backward":
+        if (hasMore) {
+          previous = scheduled[1]?.id;
+          next = scheduled[pageSize]?.id;
+        } else {
+          next = scheduled[pageSize - 1]?.id;
+        }
+        break;
+    }
+
+    const scheduledToReturn =
+      direction === "backward" && hasMore
+        ? scheduled.slice(1, pageSize + 1)
+        : scheduled.slice(0, pageSize);
+
    return {
-      scheduled: scheduled.map((s) => {
+      scheduled: scheduledToReturn.map((s) => {
        const schedule = ScheduleMetadataSchema.parse(s.schedule);
        const nextEventTimestamp = s.active
          ? calculateNextScheduledEvent(schedule, s.lastEventTimestamp)
@@ -78,6 +130,10 @@ export class ScheduledTriggersPresenter {
          nextEventTimestamp,
        };
      }),
+      pagination: {
+        next,
+        previous,
+      },
    };
  }
 }
@@ -0,0 +1,26 @@
+import { getTeamMembersAndInvites } from "~/models/member.server";
+import { BasePresenter } from "./v3/basePresenter.server";
+import { getLimit } from "~/services/platform.v3.server";
+
+export class TeamPresenter extends BasePresenter {
+  public async call({ userId, organizationId }: { userId: string; organizationId: string }) {
+    const result = await getTeamMembersAndInvites({
+      userId,
+      organizationId,
+    });
+
+    if (!result) {
+      return;
+    }
+
+    const limit = await getLimit(organizationId, "teamMembers", 25);
+
+    return {
+      ...result,
+      limits: {
+        used: result.members.length + result.invites.length,
+        limit,
+      },
+    };
+  }
+}
@@ -8,6 +8,7 @@ import {
  ProjectAlertSlackProperties,
  ProjectAlertWebhookProperties,
 } from "~/models/projectAlert.server";
+import { getLimit } from "~/services/platform.v3.server";

 export type AlertChannelListPresenterData = Awaited<ReturnType<AlertChannelListPresenter["call"]>>;
 export type AlertChannelListPresenterRecord =
@@ -29,6 +30,21 @@ export class AlertChannelListPresenter extends BasePresenter {
      },
    });

+    const organization = await this._replica.project.findFirst({
+      where: {
+        id: projectId,
+      },
+      select: {
+        organizationId: true,
+      },
+    });
+
+    if (!organization) {
+      throw new Error(`Project not found: ${projectId}`);
+    }
+
+    const limit = await getLimit(organization.organizationId, "alerts", 25);
+
    return {
      alertChannels: await Promise.all(
        alertChannels.map(async (alertChannel) => ({
@@ -36,6 +52,10 @@ export class AlertChannelListPresenter extends BasePresenter {
          properties: await this.#presentProperties(alertChannel),
        }))
      ),
+      limits: {
+        used: alertChannels.length,
+        limit,
+      },
    };
  }

@@ -12,6 +12,7 @@ import {
 import { Prisma, TaskRunAttemptStatus, TaskRunStatus } from "@trigger.dev/database";
 import assertNever from "assert-never";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { generatePresignedUrl } from "~/v3/r2.server";
 import { BasePresenter } from "./basePresenter.server";

 export class ApiRetrieveRunPresenter extends BasePresenter {
@@ -44,7 +45,9 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
      }

      let $payload: any;
+      let $payloadPresignedUrl: string | undefined;
      let $output: any;
+      let $outputPresignedUrl: string | undefined;

      if (showSecretDetails) {
        const payloadPacket = await conditionallyImportPacket({
@@ -52,7 +55,19 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
          dataType: taskRun.payloadType,
        });

-        $payload = await parsePacket(payloadPacket);
+        if (
+          payloadPacket.dataType === "application/store" &&
+          typeof payloadPacket.data === "string"
+        ) {
+          $payloadPresignedUrl = await generatePresignedUrl(
+            env.project.externalRef,
+            env.slug,
+            payloadPacket.data,
+            "GET"
+          );
+        } else {
+          $payload = await parsePacket(payloadPacket);
+        }

        if (taskRun.status === "COMPLETED_SUCCESSFULLY") {
          const completedAttempt = taskRun.attempts.find(
@@ -65,7 +80,19 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
              dataType: completedAttempt.outputType,
            });

-            $output = await parsePacket(outputPacket);
+            if (
+              outputPacket.dataType === "application/store" &&
+              typeof outputPacket.data === "string"
+            ) {
+              $outputPresignedUrl = await generatePresignedUrl(
+                env.project.externalRef,
+                env.slug,
+                outputPacket.data,
+                "GET"
+              );
+            } else {
+              $output = await parsePacket(outputPacket);
+            }
          }
        }
      }
@@ -84,9 +111,14 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
        finishedAt: ApiRetrieveRunPresenter.isStatusFinished(apiStatus)
          ? taskRun.updatedAt
          : undefined,
+        delayedUntil: taskRun.delayUntil ?? undefined,
        payload: $payload,
+        payloadPresignedUrl: $payloadPresignedUrl,
        output: $output,
+        outputPresignedUrl: $outputPresignedUrl,
        isTest: taskRun.isTest,
+        ttl: taskRun.ttl ?? undefined,
+        expiredAt: taskRun.expiredAt ?? undefined,
        schedule: taskRun.schedule
          ? {
              id: taskRun.schedule.friendlyId,
@@ -142,6 +174,9 @@ export class ApiRetrieveRunPresenter extends BasePresenter {

  static apiStatusFromRunStatus(status: TaskRunStatus): RunStatus {
    switch (status) {
+      case "DELAYED": {
+        return "DELAYED";
+      }
      case "WAITING_FOR_DEPLOY": {
        return "WAITING_FOR_DEPLOY";
      }
@@ -176,6 +211,9 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
      case "COMPLETED_WITH_ERRORS": {
        return "FAILED";
      }
+      case "EXPIRED": {
+        return "EXPIRED";
+      }
      default: {
        assertNever(status);
      }
@@ -183,7 +221,7 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
  }

  static apiBooleanHelpersFromRunStatus(status: RunStatus) {
-    const isQueued = status === "QUEUED" || status === "WAITING_FOR_DEPLOY";
+    const isQueued = status === "QUEUED" || status === "WAITING_FOR_DEPLOY" || status === "DELAYED";
    const isExecuting = status === "EXECUTING" || status === "REATTEMPTING" || status === "FROZEN";
    const isCompleted =
      status === "COMPLETED" ||
@@ -209,7 +209,10 @@ export class ApiRunListPresenter extends BasePresenter {
          updatedAt: new Date(run.updatedAt),
          startedAt: run.startedAt ? new Date(run.startedAt) : undefined,
          finishedAt: run.finishedAt ? new Date(run.finishedAt) : undefined,
+          delayedUntil: run.delayUntil ? new Date(run.delayUntil) : undefined,
          isTest: run.isTest,
+          ttl: run.ttl ?? undefined,
+          expiredAt: run.expiredAt ? new Date(run.expiredAt) : undefined,
          env: {
            id: run.environment.id,
            name: run.environment.slug,
@@ -233,6 +236,8 @@ export class ApiRunListPresenter extends BasePresenter {

  static apiStatusToRunStatuses(status: RunStatus): TaskRunStatus[] | TaskRunStatus {
    switch (status) {
+      case "DELAYED":
+        return "DELAYED";
      case "WAITING_FOR_DEPLOY": {
        return "WAITING_FOR_DEPLOY";
      }
@@ -266,6 +271,9 @@ export class ApiRunListPresenter extends BasePresenter {
      case "FAILED": {
        return "COMPLETED_WITH_ERRORS";
      }
+      case "EXPIRED": {
+        return "EXPIRED";
+      }
      default: {
        assertNever(status);
      }
@@ -123,6 +123,7 @@ export class EnvironmentVariablesPresenter {
        id: environment.id,
        type: environment.type,
      })),
+      hasStaging: environments.some((environment) => environment.type === "STAGING"),
    };
  }
 }
@@ -4,8 +4,8 @@ import { Direction } from "~/components/runs/RunStatuses";
 import { FINISHED_STATUSES } from "~/components/runs/v3/TaskRunStatus";
 import { sqlDatabaseSchema } from "~/db.server";
 import { displayableEnvironment } from "~/models/runtimeEnvironment.server";
-import { CANCELLABLE_STATUSES } from "~/v3/services/cancelTaskRun.server";
 import { BasePresenter } from "./basePresenter.server";
+import { isCancellableRunStatus } from "~/v3/taskStatus";

 export type RunListOptions = {
  userId?: string;
@@ -158,10 +158,15 @@ export class RunListPresenter extends BasePresenter {
        createdAt: Date;
        startedAt: Date | null;
        lockedAt: Date | null;
+        delayUntil: Date | null;
        updatedAt: Date;
        isTest: boolean;
        spanId: string;
        idempotencyKey: string | null;
+        ttl: string | null;
+        expiredAt: Date | null;
+        costInCents: number;
+        usageDurationMs: BigInt;
      }[]
    >`
    SELECT
@@ -174,11 +179,16 @@ export class RunListPresenter extends BasePresenter {
    tr.status AS status,
    tr."createdAt" AS "createdAt",
    tr."startedAt" AS "startedAt",
+    tr."delayUntil" AS "delayUntil",
    tr."lockedAt" AS "lockedAt",
    tr."updatedAt" AS "updatedAt",
    tr."isTest" AS "isTest",
    tr."spanId" AS "spanId",
-    tr."idempotencyKey" AS "idempotencyKey"
+    tr."idempotencyKey" AS "idempotencyKey",
+    tr."ttl" AS "ttl",
+    tr."expiredAt" AS "expiredAt",
+    tr."costInCents" AS "costInCents",
+    tr."usageDurationMs" AS "usageDurationMs"
  FROM
    ${sqlDatabaseSchema}."TaskRun" tr
  LEFT JOIN
@@ -283,6 +293,7 @@ export class RunListPresenter extends BasePresenter {
          createdAt: run.createdAt.toISOString(),
          updatedAt: run.updatedAt.toISOString(),
          startedAt: startedAt ? startedAt.toISOString() : undefined,
+          delayUntil: run.delayUntil ? run.delayUntil.toISOString() : undefined,
          hasFinished,
          finishedAt: hasFinished ? run.updatedAt.toISOString() : undefined,
          isTest: run.isTest,
@@ -291,9 +302,13 @@ export class RunListPresenter extends BasePresenter {
          taskIdentifier: run.taskIdentifier,
          spanId: run.spanId,
          isReplayable: true,
-          isCancellable: CANCELLABLE_STATUSES.includes(run.status),
+          isCancellable: isCancellableRunStatus(run.status),
          environment: displayableEnvironment(environment, userId),
          idempotencyKey: run.idempotencyKey ? run.idempotencyKey : undefined,
+          ttl: run.ttl ? run.ttl : undefined,
+          expiredAt: run.expiredAt ? run.expiredAt.toISOString() : undefined,
+          costInCents: run.costInCents,
+          usageDurationMs: Number(run.usageDurationMs),
        };
      }),
      pagination: {
@@ -94,17 +94,24 @@ export class RunStreamPresenter {

        eventEmitter.removeAllListeners();

-        unsubscribe().catch((error) => {
-          logger.error("RunStreamPresenter.abort.unsubscribe", {
-            runFriendlyId,
-            traceId: run.traceId,
-            error: {
-              name: error.name,
-              message: error.message,
-              stack: error.stack,
-            },
+        unsubscribe()
+          .then(() => {
+            logger.info("RunStreamPresenter.abort.unsubscribe succeeded", {
+              runFriendlyId,
+              traceId: run.traceId,
+            });
+          })
+          .catch((error) => {
+            logger.error("RunStreamPresenter.abort.unsubscribe failed", {
+              runFriendlyId,
+              traceId: run.traceId,
+              error: {
+                name: error.name,
+                message: error.message,
+                stack: error.stack,
+              },
+            });
          });
-        });
      };
    });
  }
@@ -1,8 +1,8 @@
 import { Prisma, RuntimeEnvironmentType } from "@trigger.dev/database";
 import { ScheduleListFilters } from "~/components/runs/v3/ScheduleFilters";
-import { PrismaClient, prisma, sqlDatabaseSchema } from "~/db.server";
+import { sqlDatabaseSchema } from "~/db.server";
 import { displayableEnvironment } from "~/models/runtimeEnvironment.server";
-import { getUsername } from "~/utils/username";
+import { getCurrentPlan, getLimit, getLimits } from "~/services/platform.v3.server";
 import { calculateNextScheduledTimestamp } from "~/v3/utils/calculateNextSchedule.server";
 import { BasePresenter } from "./basePresenter.server";

@@ -53,6 +53,7 @@ export class ScheduleListPresenter extends BasePresenter {
    const project = await this._replica.project.findFirstOrThrow({
      select: {
        id: true,
+        organizationId: true,
        environments: {
          select: {
            id: true,
@@ -71,11 +72,6 @@ export class ScheduleListPresenter extends BasePresenter {
            },
          },
        },
-        organization: {
-          select: {
-            maximumSchedulesLimit: true,
-          },
-        },
      },
      where: {
        id: projectId,
@@ -248,6 +244,8 @@ export class ScheduleListPresenter extends BasePresenter {
      };
    });

+    const limit = await getLimit(project.organizationId, "schedules", 500);
+
    return {
      currentPage: page,
      totalPages: Math.ceil(totalCount / pageSize),
@@ -260,7 +258,7 @@ export class ScheduleListPresenter extends BasePresenter {
      hasFilters,
      limits: {
        used: schedulesCount,
-        limit: project.organization.maximumSchedulesLimit,
+        limit,
      },
      filters: {
        tasks,
@@ -0,0 +1,115 @@
+import { sqlDatabaseSchema } from "~/db.server";
+import { env } from "~/env.server";
+import { getUsage, getUsageSeries } from "~/services/platform.v3.server";
+import { createTimeSeriesData } from "~/utils/graphs";
+import { BasePresenter } from "./basePresenter.server";
+import { start } from "@popperjs/core";
+
+type Options = {
+  organizationId: string;
+  startDate: Date;
+};
+
+export type TaskUsageItem = {
+  taskIdentifier: string;
+  runCount: number;
+  averageDuration: number;
+  averageCost: number;
+  totalDuration: number;
+  totalCost: number;
+  totalBaseCost: number;
+};
+
+export type UsageSeriesData = {
+  date: string;
+  dollars: number;
+}[];
+
+export class UsagePresenter extends BasePresenter {
+  public async call({ organizationId, startDate }: Options) {
+    //month period
+    const startOfMonth = new Date(startDate);
+    startOfMonth.setUTCDate(1);
+    startOfMonth.setUTCHours(0, 0, 0, 0);
+
+    const endOfMonth = new Date(
+      startOfMonth.getFullYear(),
+      startOfMonth.getMonth() + 1,
+      0,
+      23,
+      59,
+      59,
+      999
+    );
+
+    //usage data from the platform
+    const past30Days = getUsageSeries(organizationId, {
+      from: startOfMonth,
+      to: endOfMonth,
+      window: "DAY",
+    }).then((data) => {
+      return createTimeSeriesData({
+        startDate: startOfMonth,
+        endDate: endOfMonth,
+        window: "DAY",
+        data: data
+          ? data.data.map((period) => ({
+              date: new Date(period.windowStart),
+              value: period.value,
+            }))
+          : [],
+      }).map((period) => ({
+        date: period.date.toISOString(),
+        dollars: (period.value ?? 0) / 100,
+      }));
+    });
+
+    //usage by task
+    const tasks = this._replica.$queryRaw<TaskUsageItem[]>`
+    SELECT
+      tr."taskIdentifier",
+      COUNT(*) AS "runCount",
+      AVG(tr."usageDurationMs") AS "averageDuration",
+      SUM(tr."usageDurationMs") AS "totalDuration",
+      AVG(tr."costInCents") / 100.0 AS "averageCost",
+      SUM(tr."costInCents") / 100.0 AS "totalCost",
+      SUM(tr."baseCostInCents") / 100.0 AS "totalBaseCost"
+  FROM
+      ${sqlDatabaseSchema}."TaskRun" tr
+      JOIN ${sqlDatabaseSchema}."Project" pr ON pr.id = tr."projectId"
+      JOIN ${sqlDatabaseSchema}."Organization" org ON org.id = pr."organizationId"
+  WHERE
+      tr."createdAt" > ${startOfMonth}
+      AND tr."createdAt" < ${endOfMonth}
+      AND org.id = ${organizationId}
+  GROUP BY
+      tr."taskIdentifier";
+  `.then((data) => {
+      return data
+        .map((item) => ({
+          taskIdentifier: item.taskIdentifier,
+          runCount: Number(item.runCount),
+          averageDuration: Number(item.averageDuration),
+          averageCost: Number(item.averageCost) + env.CENTS_PER_RUN / 100,
+          totalDuration: Number(item.totalDuration),
+          totalCost: Number(item.totalCost + item.totalBaseCost),
+        }))
+        .sort((a, b) => b.totalCost - a.totalCost);
+    });
+
+    const usage = getUsage(organizationId, { from: startOfMonth, to: endOfMonth }).then((data) => {
+      const current = (data?.cents ?? 0) / 100;
+      const percentageThroughMonth = new Date().getDate() / endOfMonth.getDate();
+      return {
+        current: current,
+        projected: current / percentageThroughMonth,
+      };
+    });
+
+    return {
+      usageOverTime: past30Days,
+      usage,
+      tasks,
+    };
+  }
+}
@@ -0,0 +1,32 @@
+import { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { redirectWithErrorMessage } from "~/models/message.server";
+import { v3BillingPath } from "~/utils/pathBuilder";
+
+const ParamsSchema = z.object({
+  organizationId: z.string(),
+});
+
+export const loader = async ({ request, params }: LoaderFunctionArgs) => {
+  const { organizationId } = ParamsSchema.parse(params);
+
+  const org = await prisma.organization.findUnique({
+    select: {
+      slug: true,
+    },
+    where: {
+      id: organizationId,
+    },
+  });
+
+  if (!org) {
+    throw new Response(null, { status: 404 });
+  }
+
+  return redirectWithErrorMessage(
+    v3BillingPath({ slug: org.slug }),
+    request,
+    "You didn't complete your details on Stripe. Please try again."
+  );
+};
@@ -0,0 +1,46 @@
+import { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { redirect } from "remix-typedjson";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { redirectWithSuccessMessage } from "~/models/message.server";
+import { newProjectPath, v3BillingPath } from "~/utils/pathBuilder";
+
+const ParamsSchema = z.object({
+  organizationId: z.string(),
+});
+
+export const loader = async ({ request, params }: LoaderFunctionArgs) => {
+  const { organizationId } = ParamsSchema.parse(params);
+
+  const org = await prisma.organization.findUnique({
+    select: {
+      slug: true,
+      _count: {
+        select: {
+          projects: true,
+        },
+      },
+    },
+    where: {
+      id: organizationId,
+    },
+  });
+
+  if (!org) {
+    throw new Response(null, { status: 404 });
+  }
+
+  const hasProject = org._count.projects > 0;
+
+  if (hasProject) {
+    return redirectWithSuccessMessage(
+      v3BillingPath({ slug: org.slug }),
+      request,
+      "Your subscription has been successfully activated."
+    );
+  }
+
+  return redirect(
+    newProjectPath({ slug: org.slug }, "Your subscription has been successfully activated.")
+  );
+};
@@ -0,0 +1,34 @@
+import { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { redirectWithErrorMessage } from "~/models/message.server";
+import { v3BillingPath } from "~/utils/pathBuilder";
+
+const ParamsSchema = z.object({
+  organizationId: z.string(),
+});
+
+export const loader = async ({ request, params }: LoaderFunctionArgs) => {
+  const { organizationId } = ParamsSchema.parse(params);
+
+  const org = await prisma.organization.findUnique({
+    select: {
+      slug: true,
+    },
+    where: {
+      id: organizationId,
+    },
+  });
+
+  if (!org) {
+    throw new Response(null, { status: 404 });
+  }
+
+  const url = new URL(request.url);
+  const searchParams = new URLSearchParams(url.search);
+  const reason = searchParams.get("reason");
+
+  let errorMessage = reason ? decodeURIComponent(reason) : "Subscribing failed to complete";
+
+  return redirectWithErrorMessage(v3BillingPath({ slug: org.slug }), request, errorMessage);
+};
--- a/Show More
+++ b/Show More