Release 3.1.0

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.github/workflows/release.yml

@@ -17,7 +17,7 @@ concurrency:
 jobs:
   release:
     name: 🦋 Changesets Release
-    runs-on: buildjet-8vcpu-ubuntu-2204
+    runs-on: ubuntu-latest
     if: github.repository == 'triggerdotdev/trigger.dev'
     outputs:
       published: ${{ steps.changesets.outputs.published }}

.github/workflows/unit-tests.yml

@@ -6,7 +6,7 @@ on:
 jobs:
   unitTests:
     name: "🧪 Unit Tests"
-    runs-on: buildjet-8vcpu-ubuntu-2204
+    runs-on: buildjet-16vcpu-ubuntu-2204
     steps:
       - name: ⬇️ Checkout repo
         uses: actions/checkout@v4
@@ -30,5 +30,17 @@ jobs:
       - name: 📀 Generate Prisma Client
         run: pnpm run generate
 
-      - name: 🧪 Run Unit Tests
-        run: pnpm run test
+      - name: 🧪 Run Webapp Unit Tests
+        run: pnpm run test --filter webapp
+        env:
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
+          DIRECT_URL: postgresql://postgres:postgres@localhost:5432/postgres
+          SESSION_SECRET: "secret"
+          MAGIC_LINK_SECRET: "secret"
+          ENCRYPTION_KEY: "secret"
+      
+      - name: 🧪 Run Package Unit Tests
+        run: pnpm run test --filter "@trigger.dev/*"
+      
+      - name: 🧪 Run Internal Unit Tests
+        run: pnpm run test --filter "@internal/*"

.npmrc

@@ -1,2 +1,3 @@
 link-workspace-packages=false
-public-hoist-pattern[]=*prisma*
+public-hoist-pattern[]=*prisma*
+prefer-workspace-packages=true

.vscode/extensions.json

@@ -1,8 +1,4 @@
 {
-	"recommendations": [
-        "denoland.vscode-deno"
-    ],
-	"unwantedRecommendations": [
-		
-	]
+  "recommendations": ["bierner.comment-tagged-templates"],
+  "unwantedRecommendations": []
 }

apps/kubernetes-provider/src/index.ts

@@ -661,6 +661,7 @@ provider.listen();
 
 const taskMonitor = new TaskMonitor({
   runtimeEnv: RUNTIME_ENV,
+  namespace: KUBERNETES_NAMESPACE,
   onIndexFailure: async (deploymentId, details) => {
     logger.log("Indexing failed", { deploymentId, details });
 

apps/kubernetes-provider/src/taskMonitor.ts

@@ -160,7 +160,10 @@ export class TaskMonitor {
 
     let reason = rawReason || "Unknown error";
     let logs = rawLogs || "";
-    let overrideCompletion = false;
+
+    /** This will only override existing task errors. It will not crash the run.  */
+    let onlyOverrideExistingError = exitCode === EXIT_CODE_CHILD_NONZERO;
+
     let errorCode: TaskRunInternalError["code"] = TaskRunErrorCodes.POD_UNKNOWN_ERROR;
 
     switch (rawReason) {
@@ -185,10 +188,8 @@ export class TaskMonitor {
         }
         break;
       case "OOMKilled":
-        overrideCompletion = true;
-        reason = `${
-          exitCode === EXIT_CODE_CHILD_NONZERO ? "Child process" : "Parent process"
-        } ran out of memory! Try choosing a machine preset with more memory for this task.`;
+        reason =
+          "[TaskMonitor] Your task ran out of memory. Try increasing the machine specs. If this doesn't fix it there might be a memory leak.";
         errorCode = TaskRunErrorCodes.TASK_PROCESS_OOM_KILLED;
         break;
       default:
@@ -199,7 +200,7 @@ export class TaskMonitor {
       exitCode,
       reason,
       logs,
-      overrideCompletion,
+      overrideCompletion: onlyOverrideExistingError,
       errorCode,
     } satisfies FailureDetails;
 

apps/proxy/package.json

@@ -4,7 +4,8 @@
   "private": true,
   "scripts": {
     "deploy": "wrangler deploy",
-    "dev": "wrangler dev"
+    "dev": "wrangler dev",
+    "dry-run:staging": "wrangler deploy --dry-run --outdir=dist --env staging"
   },
   "devDependencies": {
     "@cloudflare/workers-types": "^4.20240512.0",

apps/proxy/src/index.ts

@@ -17,12 +17,9 @@ export interface Env {
 
 export default {
   async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
-    if (!env.REWRITE_HOSTNAME) throw new Error("Missing REWRITE_HOSTNAME");
-    console.log("url", request.url);
-
     if (!queueingIsEnabled(env)) {
       console.log("Missing AWS credentials. Passing through to the origin.");
-      return redirectToOrigin(request, env);
+      return fetch(request);
     }
 
     const url = new URL(request.url);
@@ -42,25 +39,10 @@ export default {
     }
 
     //the same request but with the hostname (and port) changed
-    return redirectToOrigin(request, env);
+    return fetch(request);
   },
 };
 
-function redirectToOrigin(request: Request, env: Env) {
-  const newUrl = new URL(request.url);
-  newUrl.hostname = env.REWRITE_HOSTNAME;
-  newUrl.port = env.REWRITE_PORT || newUrl.port;
-
-  const requestInit: RequestInit = {
-    method: request.method,
-    headers: request.headers,
-    body: request.body,
-  };
-
-  console.log("rewritten url", newUrl.toString());
-  return fetch(newUrl.toString(), requestInit);
-}
-
 function queueingIsEnabled(env: Env) {
   return (
     env.AWS_SQS_ACCESS_KEY_ID &&

apps/webapp/app/components/ErrorDisplay.tsx

@@ -1,7 +1,10 @@
+import { HomeIcon } from "@heroicons/react/20/solid";
 import { isRouteErrorResponse, useRouteError } from "@remix-run/react";
-import { LinkButton } from "./primitives/Buttons";
-import { Header1, Header3 } from "./primitives/Headers";
+import { motion } from "framer-motion";
 import { friendlyErrorDisplay } from "~/utils/httpErrors";
+import { LinkButton } from "./primitives/Buttons";
+import { Header1 } from "./primitives/Headers";
+import { Paragraph } from "./primitives/Paragraph";
 
 type ErrorDisplayOptions = {
   button?: {
@@ -39,12 +42,32 @@ type DisplayOptionsProps = {
 
 export function ErrorDisplay({ title, message, button }: DisplayOptionsProps) {
   return (
-    <div className="p-4">
-      <Header1 className="mb-4 border-b border-charcoal-800 pb-4">{title}</Header1>
-      {message && <Header3>{message}</Header3>}
-      <LinkButton to={button ? button.to : "/"} variant="primary/medium" className="mt-8">
-        {button ? button.title : "Home"}
-      </LinkButton>
+    <div className="relative flex min-h-screen flex-col items-center justify-center bg-[#16181C]">
+      <div className="z-10 mt-[30vh] flex flex-col items-center gap-8">
+        <Header1>{title}</Header1>
+        {message && <Paragraph>{message}</Paragraph>}
+        <LinkButton
+          to={button ? button.to : "/"}
+          shortcut={{ modifiers: ["meta"], key: "g" }}
+          variant="primary/medium"
+          LeadingIcon={HomeIcon}
+        >
+          {button ? button.title : "Go to homepage"}
+        </LinkButton>
+      </div>
+      <div className="pointer-events-none absolute bottom-4 right-4 z-10 h-[70px] w-[200px] bg-[rgb(24,26,30)]" />
+      <motion.div
+        className="pointer-events-none absolute inset-0 overflow-hidden"
+        initial={{ opacity: 0 }}
+        animate={{ opacity: 1 }}
+        transition={{ delay: 0.5, duration: 2, ease: "easeOut" }}
+      >
+        <iframe
+          src="https://my.spline.design/untitled-a6f70b5ebc46bdb2dcc0f21d5397e8ac/"
+          className="pointer-events-none absolute inset-0 h-full w-full object-cover"
+          style={{ border: "none" }}
+        />
+      </motion.div>
     </div>
   );
 }

apps/webapp/app/components/runs/v3/RunInspector.tsx

@@ -40,7 +40,7 @@ import {
 } from "~/utils/pathBuilder";
 import { TraceSpan } from "~/utils/taskEvent";
 import { SpanLink } from "~/v3/eventRepository.server";
-import { isFinalRunStatus } from "~/v3/taskStatus";
+import { isFailedRunStatus, isFinalRunStatus } from "~/v3/taskStatus";
 import { RunTimelineEvent, RunTimelineLine } from "./InspectorTimeline";
 import { RunTag } from "./RunTag";
 import { TaskRunStatusCombo } from "./TaskRunStatus";
@@ -479,6 +479,7 @@ function RunTimeline({ run }: { run: RawRun }) {
   const updatedAt = new Date(run.updatedAt);
 
   const isFinished = isFinalRunStatus(run.status);
+  const isError = isFailedRunStatus(run.status);
 
   return (
     <div className="min-w-fit max-w-80">
@@ -535,7 +536,7 @@ function RunTimeline({ run }: { run: RawRun }) {
               <RunTimelineEvent
                 title="Finished"
                 subtitle={<DateTimeAccurate date={updatedAt} />}
-                state="complete"
+                state={isError ? "error" : "complete"}
               />
             </>
           ) : (

apps/webapp/app/components/runs/v3/SpanEvents.tsx

@@ -1,3 +1,4 @@
+import { EnvelopeIcon } from "@heroicons/react/20/solid";
 import {
   exceptionEventEnhancer,
   isExceptionSpanEvent,
@@ -5,6 +6,8 @@ import {
   type SpanEvent as OtelSpanEvent,
 } from "@trigger.dev/core/v3";
 import { CodeBlock } from "~/components/code/CodeBlock";
+import { Feedback } from "~/components/Feedback";
+import { Button } from "~/components/primitives/Buttons";
 import { Callout } from "~/components/primitives/Callout";
 import { DateTimeAccurate } from "~/components/primitives/DateTime";
 import { Header2, Header3 } from "~/components/primitives/Headers";
@@ -75,11 +78,26 @@ export function SpanEventError({
         titleClassName="text-rose-500"
       />
       {enhancedException.message && <Callout variant="error">{enhancedException.message}</Callout>}
-      {enhancedException.link && (
-        <Callout variant="docs" to={enhancedException.link.href}>
-          {enhancedException.link.name}
-        </Callout>
-      )}
+      {enhancedException.link &&
+        (enhancedException.link.magic === "CONTACT_FORM" ? (
+          <Feedback
+            button={
+              <Button
+                variant="tertiary/medium"
+                LeadingIcon={EnvelopeIcon}
+                leadingIconClassName="text-blue-400"
+                fullWidth
+                textAlignLeft
+              >
+                {enhancedException.link.name}
+              </Button>
+            }
+          />
+        ) : (
+          <Callout variant="docs" to={enhancedException.link.href}>
+            {enhancedException.link.name}
+          </Callout>
+        ))}
       {enhancedException.stacktrace && (
         <CodeBlock
           showCopyButton={false}

apps/webapp/app/env.server.ts

@@ -31,7 +31,7 @@ const EnvironmentSchema = z.object({
   REMIX_APP_PORT: z.string().optional(),
   LOGIN_ORIGIN: z.string().default("http://localhost:3030"),
   APP_ORIGIN: z.string().default("http://localhost:3030"),
-  ELECTRIC_ORIGIN: z.string(),
+  ELECTRIC_ORIGIN: z.string().default("http://localhost:3060"),
   APP_ENV: z.string().default(process.env.NODE_ENV),
   SERVICE_NAME: z.string().default("trigger.dev webapp"),
   SECRET_STORE: SecretStoreOptionsSchema.default("DATABASE"),
@@ -103,6 +103,25 @@ const EnvironmentSchema = z.object({
   API_RATE_LIMIT_REFILL_RATE: z.coerce.number().int().default(250), // refix 250 tokens every 10 seconds
   API_RATE_LIMIT_REQUEST_LOGS_ENABLED: z.string().default("0"),
   API_RATE_LIMIT_REJECTION_LOGS_ENABLED: z.string().default("1"),
+  API_RATE_LIMIT_LIMITER_LOGS_ENABLED: z.string().default("0"),
+
+  API_RATE_LIMIT_JWT_WINDOW: z.string().default("1m"),
+  API_RATE_LIMIT_JWT_TOKENS: z.coerce.number().int().default(60),
+
+  //Realtime rate limiting
+  /**
+   * @example "60s"
+   * @example "1m"
+   * @example "1h"
+   * @example "1d"
+   * @example "1000ms"
+   * @example "1000s"
+   */
+  REALTIME_RATE_LIMIT_WINDOW: z.string().default("1m"),
+  REALTIME_RATE_LIMIT_TOKENS: z.coerce.number().int().default(100),
+  REALTIME_RATE_LIMIT_REQUEST_LOGS_ENABLED: z.string().default("0"),
+  REALTIME_RATE_LIMIT_REJECTION_LOGS_ENABLED: z.string().default("1"),
+  REALTIME_RATE_LIMIT_LIMITER_LOGS_ENABLED: z.string().default("0"),
 
   //Ingesting event rate limit
   INGEST_EVENT_RATE_LIMIT_WINDOW: z.string().default("60s"),

apps/webapp/app/models/taskRun.server.ts

@@ -3,7 +3,7 @@ import type {
   TaskRunFailedExecutionResult,
   TaskRunSuccessfulExecutionResult,
 } from "@trigger.dev/core/v3";
-import { TaskRunError } from "@trigger.dev/core/v3";
+import { TaskRunError, TaskRunErrorCodes } from "@trigger.dev/core/v3";
 
 import type {
   TaskRun,
@@ -62,7 +62,7 @@ export function executionResultForTaskRun(
         id: taskRun.friendlyId,
         error: {
           type: "INTERNAL_ERROR",
-          code: "TASK_RUN_CANCELLED",
+          code: TaskRunErrorCodes.TASK_RUN_CANCELLED,
         },
       } satisfies TaskRunFailedExecutionResult;
     }
@@ -94,7 +94,7 @@ export function executionResultForTaskRun(
         id: taskRun.friendlyId,
         error: {
           type: "INTERNAL_ERROR",
-          code: "CONFIGURED_INCORRECTLY",
+          code: TaskRunErrorCodes.CONFIGURED_INCORRECTLY,
         },
       } satisfies TaskRunFailedExecutionResult;
     }

apps/webapp/app/models/taskRunTag.server.ts

@@ -1,7 +1,7 @@
 import { prisma } from "~/db.server";
 import { generateFriendlyId } from "~/v3/friendlyIdentifiers";
 
-export const MAX_TAGS_PER_RUN = 5;
+export const MAX_TAGS_PER_RUN = 10;
 
 export async function createTag({ tag, projectId }: { tag: string; projectId: string }) {
   if (tag.trim().length === 0) return;

apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts

@@ -62,8 +62,7 @@ type CommonRelatedRun = Prisma.Result<
 export class ApiRetrieveRunPresenter extends BasePresenter {
   public async call(
     friendlyId: string,
-    env: AuthenticatedEnvironment,
-    showSecretDetails: boolean
+    env: AuthenticatedEnvironment
   ): Promise<RetrieveRunResponse | undefined> {
     return this.traceWithEnv("call", env, async (span) => {
       const taskRun = await this._replica.taskRun.findFirst({
@@ -72,11 +71,7 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
           runtimeEnvironmentId: env.id,
         },
         include: {
-          attempts: {
-            orderBy: {
-              createdAt: "desc",
-            },
-          },
+          attempts: true,
           lockedToVersion: true,
           schedule: true,
           tags: true,
@@ -111,50 +106,48 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
       let $output: any;
       let $outputPresignedUrl: string | undefined;
 
-      if (showSecretDetails) {
-        const payloadPacket = await conditionallyImportPacket({
-          data: taskRun.payload,
-          dataType: taskRun.payloadType,
-        });
+      const payloadPacket = await conditionallyImportPacket({
+        data: taskRun.payload,
+        dataType: taskRun.payloadType,
+      });
 
-        if (
-          payloadPacket.dataType === "application/store" &&
-          typeof payloadPacket.data === "string"
-        ) {
-          $payloadPresignedUrl = await generatePresignedUrl(
-            env.project.externalRef,
-            env.slug,
-            payloadPacket.data,
-            "GET"
-          );
-        } else {
-          $payload = await parsePacket(payloadPacket);
-        }
+      if (
+        payloadPacket.dataType === "application/store" &&
+        typeof payloadPacket.data === "string"
+      ) {
+        $payloadPresignedUrl = await generatePresignedUrl(
+          env.project.externalRef,
+          env.slug,
+          payloadPacket.data,
+          "GET"
+        );
+      } else {
+        $payload = await parsePacket(payloadPacket);
+      }
 
-        if (taskRun.status === "COMPLETED_SUCCESSFULLY") {
-          const completedAttempt = taskRun.attempts.find(
-            (a) => a.status === "COMPLETED" && typeof a.output !== null
-          );
+      if (taskRun.status === "COMPLETED_SUCCESSFULLY") {
+        const completedAttempt = taskRun.attempts.find(
+          (a) => a.status === "COMPLETED" && typeof a.output !== null
+        );
 
-          if (completedAttempt && completedAttempt.output) {
-            const outputPacket = await conditionallyImportPacket({
-              data: completedAttempt.output,
-              dataType: completedAttempt.outputType,
-            });
+        if (completedAttempt && completedAttempt.output) {
+          const outputPacket = await conditionallyImportPacket({
+            data: completedAttempt.output,
+            dataType: completedAttempt.outputType,
+          });
 
-            if (
-              outputPacket.dataType === "application/store" &&
-              typeof outputPacket.data === "string"
-            ) {
-              $outputPresignedUrl = await generatePresignedUrl(
-                env.project.externalRef,
-                env.slug,
-                outputPacket.data,
-                "GET"
-              );
-            } else {
-              $output = await parsePacket(outputPacket);
-            }
+          if (
+            outputPacket.dataType === "application/store" &&
+            typeof outputPacket.data === "string"
+          ) {
+            $outputPresignedUrl = await generatePresignedUrl(
+              env.project.externalRef,
+              env.slug,
+              outputPacket.data,
+              "GET"
+            );
+          } else {
+            $output = await parsePacket(outputPacket);
           }
         }
       }
@@ -165,6 +158,7 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
         payloadPresignedUrl: $payloadPresignedUrl,
         output: $output,
         outputPresignedUrl: $outputPresignedUrl,
+        error: ApiRetrieveRunPresenter.apiErrorFromError(taskRun.error),
         schedule: taskRun.schedule
           ? {
               id: taskRun.schedule.friendlyId,
@@ -179,17 +173,9 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
               },
             }
           : undefined,
-        attempts: !showSecretDetails
-          ? []
-          : taskRun.attempts.map((a) => ({
-              id: a.friendlyId,
-              status: ApiRetrieveRunPresenter.apiStatusFromAttemptStatus(a.status),
-              createdAt: a.createdAt ?? undefined,
-              updatedAt: a.updatedAt ?? undefined,
-              startedAt: a.startedAt ?? undefined,
-              completedAt: a.completedAt ?? undefined,
-              error: ApiRetrieveRunPresenter.apiErrorFromError(a.error),
-            })),
+        // We're removing attempts from the API
+        attemptCount: taskRun.attempts.length,
+        attempts: [],
         relatedRuns: {
           root: taskRun.rootTaskRun
             ? await createCommonRunStructure(taskRun.rootTaskRun)

apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts

@@ -29,7 +29,7 @@ const CoercedDate = z.preprocess((arg) => {
   return arg;
 }, z.date().optional());
 
-const SearchParamsSchema = z.object({
+export const ApiRunListSearchParams = z.object({
   "page[size]": z.coerce.number().int().positive().min(1).max(100).optional(),
   "page[after]": z.string().optional(),
   "page[before]": z.string().optional(),
@@ -121,45 +121,31 @@ const SearchParamsSchema = z.object({
   "filter[createdAt][period]": z.string().optional(),
 });
 
-type SearchParamsSchema = z.infer<typeof SearchParamsSchema>;
+type ApiRunListSearchParams = z.infer<typeof ApiRunListSearchParams>;
 
 export class ApiRunListPresenter extends BasePresenter {
   public async call(
     project: Project,
-    searchParams: URLSearchParams,
+    searchParams: ApiRunListSearchParams,
     environment?: RuntimeEnvironment
   ): Promise<ListRunResponse> {
     return this.trace("call", async (span) => {
-      const rawSearchParams = Object.fromEntries(searchParams.entries());
-      const $searchParams = SearchParamsSchema.safeParse(rawSearchParams);
-
-      if (!$searchParams.success) {
-        logger.error("Invalid search params", {
-          searchParams: rawSearchParams,
-          errors: $searchParams.error.errors,
-        });
-
-        throw fromZodError($searchParams.error);
-      }
-
-      logger.debug("Valid search params", { searchParams: $searchParams.data });
-
       const options: RunListOptions = {
         projectId: project.id,
       };
 
       // pagination
-      if ($searchParams.data["page[size]"]) {
-        options.pageSize = $searchParams.data["page[size]"];
+      if (searchParams["page[size]"]) {
+        options.pageSize = searchParams["page[size]"];
       }
 
-      if ($searchParams.data["page[after]"]) {
-        options.cursor = $searchParams.data["page[after]"];
+      if (searchParams["page[after]"]) {
+        options.cursor = searchParams["page[after]"];
         options.direction = "forward";
       }
 
-      if ($searchParams.data["page[before]"]) {
-        options.cursor = $searchParams.data["page[before]"];
+      if (searchParams["page[before]"]) {
+        options.cursor = searchParams["page[before]"];
         options.direction = "backward";
       }
 
@@ -167,12 +153,12 @@ export class ApiRunListPresenter extends BasePresenter {
       if (environment) {
         options.environments = [environment.id];
       } else {
-        if ($searchParams.data["filter[env]"]) {
+        if (searchParams["filter[env]"]) {
           const environments = await this._prisma.runtimeEnvironment.findMany({
             where: {
               projectId: project.id,
               slug: {
-                in: $searchParams.data["filter[env]"],
+                in: searchParams["filter[env]"],
               },
             },
           });
@@ -181,46 +167,46 @@ export class ApiRunListPresenter extends BasePresenter {
         }
       }
 
-      if ($searchParams.data["filter[status]"]) {
-        options.statuses = $searchParams.data["filter[status]"].flatMap((status) =>
+      if (searchParams["filter[status]"]) {
+        options.statuses = searchParams["filter[status]"].flatMap((status) =>
           ApiRunListPresenter.apiStatusToRunStatuses(status)
         );
       }
 
-      if ($searchParams.data["filter[taskIdentifier]"]) {
-        options.tasks = $searchParams.data["filter[taskIdentifier]"];
+      if (searchParams["filter[taskIdentifier]"]) {
+        options.tasks = searchParams["filter[taskIdentifier]"];
       }
 
-      if ($searchParams.data["filter[version]"]) {
-        options.versions = $searchParams.data["filter[version]"];
+      if (searchParams["filter[version]"]) {
+        options.versions = searchParams["filter[version]"];
       }
 
-      if ($searchParams.data["filter[tag]"]) {
-        options.tags = $searchParams.data["filter[tag]"];
+      if (searchParams["filter[tag]"]) {
+        options.tags = searchParams["filter[tag]"];
       }
 
-      if ($searchParams.data["filter[bulkAction]"]) {
-        options.bulkId = $searchParams.data["filter[bulkAction]"];
+      if (searchParams["filter[bulkAction]"]) {
+        options.bulkId = searchParams["filter[bulkAction]"];
       }
 
-      if ($searchParams.data["filter[schedule]"]) {
-        options.scheduleId = $searchParams.data["filter[schedule]"];
+      if (searchParams["filter[schedule]"]) {
+        options.scheduleId = searchParams["filter[schedule]"];
       }
 
-      if ($searchParams.data["filter[createdAt][from]"]) {
-        options.from = $searchParams.data["filter[createdAt][from]"].getTime();
+      if (searchParams["filter[createdAt][from]"]) {
+        options.from = searchParams["filter[createdAt][from]"].getTime();
       }
 
-      if ($searchParams.data["filter[createdAt][to]"]) {
-        options.to = $searchParams.data["filter[createdAt][to]"].getTime();
+      if (searchParams["filter[createdAt][to]"]) {
+        options.to = searchParams["filter[createdAt][to]"].getTime();
       }
 
-      if ($searchParams.data["filter[createdAt][period]"]) {
-        options.period = $searchParams.data["filter[createdAt][period]"];
+      if (searchParams["filter[createdAt][period]"]) {
+        options.period = searchParams["filter[createdAt][period]"];
       }
 
-      if (typeof $searchParams.data["filter[isTest]"] === "boolean") {
-        options.isTest = $searchParams.data["filter[isTest]"];
+      if (typeof searchParams["filter[isTest]"] === "boolean") {
+        options.isTest = searchParams["filter[isTest]"];
       }
 
       const presenter = new RunListPresenter();

apps/webapp/app/presenters/v3/SpanPresenter.server.ts

@@ -7,7 +7,7 @@ import {
 import { RUNNING_STATUSES } from "~/components/runs/v3/TaskRunStatus";
 import { eventRepository } from "~/v3/eventRepository.server";
 import { machinePresetFromName } from "~/v3/machinePresets.server";
-import { FINAL_ATTEMPT_STATUSES, isFinalRunStatus } from "~/v3/taskStatus";
+import { FINAL_ATTEMPT_STATUSES, isFailedRunStatus, isFinalRunStatus } from "~/v3/taskStatus";
 import { BasePresenter } from "./basePresenter.server";
 import { getMaxDuration } from "~/v3/utils/maxDuration";
 
@@ -294,6 +294,7 @@ export class SpanPresenter extends BasePresenter {
       usageDurationMs: run.usageDurationMs,
       isFinished,
       isRunning: RUNNING_STATUSES.includes(run.status),
+      isError: isFailedRunStatus(run.status),
       payload,
       payloadType: run.payloadType,
       output,

apps/webapp/app/routes/_app.orgs.$organizationSlug/route.tsx

@@ -10,7 +10,7 @@ import { useTypedMatchesData } from "~/hooks/useTypedMatchData";
 import { useUser } from "~/hooks/useUser";
 import { OrganizationsPresenter } from "~/presenters/OrganizationsPresenter.server";
 import { getImpersonationId } from "~/services/impersonation.server";
-import { getCurrentPlan, getUsage } from "~/services/platform.v3.server";
+import { getCachedUsage, getCurrentPlan, getUsage } from "~/services/platform.v3.server";
 import { requireUserId } from "~/services/session.server";
 import { telemetry } from "~/services/telemetry.server";
 import { organizationPath } from "~/utils/pathBuilder";
@@ -29,6 +29,27 @@ export function useCurrentPlan(matches?: UIMatch[]) {
   return data?.currentPlan;
 }
 
+export const shouldRevalidate: ShouldRevalidateFunction = (params) => {
+  const { currentParams, nextParams } = params;
+
+  const current = ParamsSchema.safeParse(currentParams);
+  const next = ParamsSchema.safeParse(nextParams);
+
+  if (current.success && next.success) {
+    if (current.data.organizationSlug !== next.data.organizationSlug) {
+      return true;
+    }
+    if (current.data.projectParam !== next.data.projectParam) {
+      return true;
+    }
+  }
+
+  // This prevents revalidation when there are search params changes
+  // IMPORTANT: If the loader function depends on search params, this should be updated
+  return params.currentUrl.pathname !== params.nextUrl.pathname;
+};
+
+// IMPORTANT: Make sure to update shouldRevalidate if this loader depends on search params
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const userId = await requireUserId(request);
   const impersonationId = await getImpersonationId(request);
@@ -50,11 +71,17 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const firstDayOfMonth = new Date();
   firstDayOfMonth.setUTCDate(1);
   firstDayOfMonth.setUTCHours(0, 0, 0, 0);
-  const tomorrow = new Date();
-  tomorrow.setUTCDate(tomorrow.getDate() + 1);
+
+  // Using the 1st day of next month means we get the usage for the current month
+  // and the cache key for getCachedUsage is stable over the month
+  const firstDayOfNextMonth = new Date();
+  firstDayOfNextMonth.setUTCMonth(firstDayOfNextMonth.getUTCMonth() + 1);
+  firstDayOfNextMonth.setUTCDate(1);
+  firstDayOfNextMonth.setUTCHours(0, 0, 0, 0);
+
   const [plan, usage] = await Promise.all([
     getCurrentPlan(organization.id),
-    getUsage(organization.id, { from: firstDayOfMonth, to: tomorrow }),
+    getCachedUsage(organization.id, { from: firstDayOfMonth, to: firstDayOfNextMonth }),
   ]);
 
   let hasExceededFreeTier = false;
@@ -100,26 +127,6 @@ export function ErrorBoundary() {
   return org ? (
     <RouteErrorDisplay button={{ title: org.title, to: organizationPath(org) }} />
   ) : (
-    <RouteErrorDisplay button={{ title: "Home", to: "/" }} />
+    <RouteErrorDisplay button={{ title: "Go to homepage", to: "/" }} />
   );
 }
-
-export const shouldRevalidate: ShouldRevalidateFunction = ({
-  defaultShouldRevalidate,
-  currentParams,
-  nextParams,
-}) => {
-  const current = ParamsSchema.safeParse(currentParams);
-  const next = ParamsSchema.safeParse(nextParams);
-
-  if (current.success && next.success) {
-    if (current.data.organizationSlug !== next.data.organizationSlug) {
-      return true;
-    }
-    if (current.data.projectParam !== next.data.projectParam) {
-      return true;
-    }
-  }
-
-  return defaultShouldRevalidate;
-};

apps/webapp/app/routes/api.v1.auth.jwt.claims.ts

@@ -0,0 +1,19 @@
+import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+
+export async function action({ request }: LoaderFunctionArgs) {
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const claims = {
+    sub: authenticationResult.environment.id,
+    pub: true,
+  };
+
+  return json(claims);
+}

apps/webapp/app/routes/api.v1.auth.jwt.ts

@@ -0,0 +1,46 @@
+import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { json } from "@remix-run/server-runtime";
+import { authenticateApiRequest } from "~/services/apiAuth.server";
+import { z } from "zod";
+import { generateJWT as internal_generateJWT } from "@trigger.dev/core/v3";
+
+const RequestBodySchema = z.object({
+  claims: z
+    .object({
+      scopes: z.array(z.string()).default([]),
+    })
+    .optional(),
+  expirationTime: z.union([z.number(), z.string()]).optional(),
+});
+
+export async function action({ request }: LoaderFunctionArgs) {
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequest(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const parsedBody = RequestBodySchema.safeParse(await request.json());
+
+  if (!parsedBody.success) {
+    return json(
+      { error: "Invalid request body", issues: parsedBody.error.issues },
+      { status: 400 }
+    );
+  }
+
+  const claims = {
+    sub: authenticationResult.environment.id,
+    pub: true,
+    ...parsedBody.data.claims,
+  };
+
+  const jwt = await internal_generateJWT({
+    secretKey: authenticationResult.apiKey,
+    payload: claims,
+    expirationTime: parsedBody.data.expirationTime ?? "1h",
+  });
+
+  return json({ token: jwt });
+}

apps/webapp/app/routes/api.v1.integrations.$integrationSlug.connections.$connectionId.ts

@@ -33,7 +33,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
       id: parsedParams.data.connectionId,
       integration: {
         slug: parsedParams.data.integrationSlug,
-        organization: authenticatedEnv.organization,
+        organizationId: authenticatedEnv.organization.id,
       },
     },
     include: {

apps/webapp/app/routes/api.v1.projects.$projectRef.runs.ts

@@ -1,62 +1,36 @@
-import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
 import { z } from "zod";
-import { ValidationError } from "zod-validation-error";
 import { findProjectByRef } from "~/models/project.server";
-import { ApiRunListPresenter } from "~/presenters/v3/ApiRunListPresenter.server";
-import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
-import { apiCors } from "~/utils/apiCors";
+import {
+  ApiRunListPresenter,
+  ApiRunListSearchParams,
+} from "~/presenters/v3/ApiRunListPresenter.server";
+import { createLoaderPATApiRoute } from "~/services/routeBuiilders/apiBuilder.server";
 
 const ParamsSchema = z.object({
   projectRef: z.string(),
 });
 
-export async function loader({ request, params }: LoaderFunctionArgs) {
-  if (request.method.toUpperCase() === "OPTIONS") {
-    return apiCors(request, json({}));
-  }
+export const loader = createLoaderPATApiRoute(
+  {
+    params: ParamsSchema,
+    searchParams: ApiRunListSearchParams,
+    corsStrategy: "all",
+  },
+  async ({ searchParams, params, authentication }) => {
+    const project = await findProjectByRef(params.projectRef, authentication.userId);
 
-  const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
+    if (!project) {
+      return json({ error: "Project not found" }, { status: 404 });
+    }
 
-  if (!authenticationResult) {
-    return apiCors(request, json({ error: "Invalid or Missing API key" }, { status: 401 }));
-  }
-
-  const $params = ParamsSchema.safeParse(params);
-
-  if (!$params.success) {
-    return json({ error: "Invalid params" }, { status: 400 });
-  }
-
-  const project = await findProjectByRef($params.data.projectRef, authenticationResult.userId);
-
-  if (!project) {
-    return json({ error: "Project not found" }, { status: 404 });
-  }
-
-  const url = new URL(request.url);
-
-  const presenter = new ApiRunListPresenter();
-
-  try {
-    const result = await presenter.call(project, url.searchParams);
+    const presenter = new ApiRunListPresenter();
+    const result = await presenter.call(project, searchParams);
 
     if (!result) {
-      return apiCors(request, json({ data: [] }));
+      return json({ data: [] });
     }
 
-    return apiCors(request, json(result));
-  } catch (error) {
-    if (error instanceof ValidationError) {
-      return apiCors(
-        request,
-        json({ error: "Query Error", details: error.details }, { status: 400 })
-      );
-    } else {
-      return apiCors(
-        request,
-        json({ error: error instanceof Error ? error.message : String(error) }, { status: 400 })
-      );
-    }
+    return json(result);
   }
-}
+);

apps/webapp/app/routes/api.v1.runs.$runId.tags.ts

@@ -89,6 +89,9 @@ export async function action({ request, params }: ActionFunctionArgs) {
         tags: {
           connect: tagIds.map((id) => ({ id })),
         },
+        runTags: {
+          push: newTags,
+        },
       },
     });
 

apps/webapp/app/routes/api.v1.runs.$runParam.attempts.ts

@@ -29,7 +29,10 @@ export async function action({ request, params }: ActionFunctionArgs) {
   const service = new CreateTaskRunAttemptService();
 
   try {
-    const { execution } = await service.call(runParam, authenticationResult.environment);
+    const { execution } = await service.call({
+      runId: runParam,
+      authenticatedEnv: authenticationResult.environment,
+    });
 
     return json(execution, { status: 200 });
   } catch (error) {

apps/webapp/app/routes/api.v1.runs.$runParam.reschedule.ts

@@ -62,11 +62,7 @@ export async function action({ request, params }: ActionFunctionArgs) {
     }
 
     const presenter = new ApiRetrieveRunPresenter();
-    const result = await presenter.call(
-      updatedRun.friendlyId,
-      authenticationResult.environment,
-      true
-    );
+    const result = await presenter.call(updatedRun.friendlyId, authenticationResult.environment);
 
     if (!result) {
       return json({ error: "Run not found" }, { status: 404 });

apps/webapp/app/routes/api.v1.runs.ts

@@ -1,52 +1,29 @@
-import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
-import { ValidationError } from "zod-validation-error";
-import { ApiRunListPresenter } from "~/presenters/v3/ApiRunListPresenter.server";
-import { authenticateApiRequest } from "~/services/apiAuth.server";
-import { apiCors } from "~/utils/apiCors";
+import {
+  ApiRunListPresenter,
+  ApiRunListSearchParams,
+} from "~/presenters/v3/ApiRunListPresenter.server";
+import { createLoaderApiRoute } from "~/services/routeBuiilders/apiBuilder.server";
 
-export async function loader({ request, params }: LoaderFunctionArgs) {
-  if (request.method.toUpperCase() === "OPTIONS") {
-    return apiCors(request, json({}));
-  }
-
-  const authenticationResult = await authenticateApiRequest(request, {
-    allowPublicKey: false,
-  });
-
-  if (!authenticationResult) {
-    return apiCors(request, json({ error: "Invalid or Missing API key" }, { status: 401 }));
-  }
-
-  const authenticatedEnv = authenticationResult.environment;
-
-  const url = new URL(request.url);
-
-  const presenter = new ApiRunListPresenter();
-
-  try {
+export const loader = createLoaderApiRoute(
+  {
+    searchParams: ApiRunListSearchParams,
+    allowJWT: true,
+    corsStrategy: "all",
+    authorization: {
+      action: "read",
+      resource: (_, searchParams) => ({ tasks: searchParams["filter[taskIdentifier]"] }),
+      superScopes: ["read:runs", "read:all", "admin"],
+    },
+  },
+  async ({ searchParams, authentication }) => {
+    const presenter = new ApiRunListPresenter();
     const result = await presenter.call(
-      authenticatedEnv.project,
-      url.searchParams,
-      authenticatedEnv
+      authentication.environment.project,
+      searchParams,
+      authentication.environment
     );
 
-    if (!result) {
-      return apiCors(request, json({ data: [] }));
-    }
-
-    return apiCors(request, json(result));
-  } catch (error) {
-    if (error instanceof ValidationError) {
-      return apiCors(
-        request,
-        json({ error: "Query Error", details: error.details }, { status: 400 })
-      );
-    } else {
-      return apiCors(
-        request,
-        json({ error: error instanceof Error ? error.message : String(error) }, { status: 400 })
-      );
-    }
+    return json(result);
   }
-}
+);

apps/webapp/app/routes/api.v1.tasks.$taskId.batch.ts

@@ -104,10 +104,20 @@ export async function action({ request, params }: ActionFunctionArgs) {
       return json({ error: "Task not found" }, { status: 404 });
     }
 
-    return json({
-      batchId: result.batch.friendlyId,
-      runs: result.runs,
-    });
+    return json(
+      {
+        batchId: result.batch.friendlyId,
+        runs: result.runs,
+      },
+      {
+        headers: {
+          "x-trigger-jwt-claims": JSON.stringify({
+            sub: authenticationResult.environment.id,
+            pub: true,
+          }),
+        },
+      }
+    );
   } catch (error) {
     if (error instanceof Error) {
       return json({ error: error.message }, { status: 400 });

apps/webapp/app/routes/api.v1.tasks.$taskId.trigger.ts

@@ -30,6 +30,8 @@ export async function action({ request, params }: ActionFunctionArgs) {
     return { status: 405, body: "Method Not Allowed" };
   }
 
+  logger.debug("TriggerTask action", { headers: Object.fromEntries(request.headers) });
+
   // Next authenticate the request
   const authenticationResult = await authenticateApiRequest(request);
 
@@ -105,9 +107,19 @@ export async function action({ request, params }: ActionFunctionArgs) {
       return json({ error: "Task not found" }, { status: 404 });
     }
 
-    return json({
-      id: run.friendlyId,
-    });
+    return json(
+      {
+        id: run.friendlyId,
+      },
+      {
+        headers: {
+          "x-trigger-jwt-claims": JSON.stringify({
+            sub: authenticationResult.environment.id,
+            pub: true,
+          }),
+        },
+      }
+    );
   } catch (error) {
     if (error instanceof ServiceValidationError) {
       return json({ error: error.message }, { status: 422 });

apps/webapp/app/routes/api.v3.runs.$runId.ts

@@ -1,44 +1,31 @@
-import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
 import { z } from "zod";
 import { ApiRetrieveRunPresenter } from "~/presenters/v3/ApiRetrieveRunPresenter.server";
-import { authenticateApiRequest } from "~/services/apiAuth.server";
-import { apiCors } from "~/utils/apiCors";
+import { createLoaderApiRoute } from "~/services/routeBuiilders/apiBuilder.server";
 
 const ParamsSchema = z.object({
   runId: z.string(),
 });
 
-export async function loader({ request, params }: LoaderFunctionArgs) {
-  if (request.method.toUpperCase() === "OPTIONS") {
-    return apiCors(request, json({}));
+export const loader = createLoaderApiRoute(
+  {
+    params: ParamsSchema,
+    allowJWT: true,
+    corsStrategy: "all",
+    authorization: {
+      action: "read",
+      resource: (params) => ({ runs: params.runId }),
+      superScopes: ["read:runs", "read:all", "admin"],
+    },
+  },
+  async ({ params, authentication }) => {
+    const presenter = new ApiRetrieveRunPresenter();
+    const result = await presenter.call(params.runId, authentication.environment);
+
+    if (!result) {
+      return json({ error: "Run not found" }, { status: 404 });
+    }
+
+    return json(result);
   }
-
-  const authenticationResult = await authenticateApiRequest(request, {
-    allowPublicKey: true,
-  });
-  if (!authenticationResult) {
-    return apiCors(request, json({ error: "Invalid or Missing API key" }, { status: 401 }));
-  }
-
-  const authenticatedEnv = authenticationResult.environment;
-
-  const parsed = ParamsSchema.safeParse(params);
-
-  if (!parsed.success) {
-    return apiCors(request, json({ error: "Invalid or missing runId" }, { status: 400 }));
-  }
-
-  const { runId } = parsed.data;
-
-  const showSecretDetails = authenticationResult.type === "PRIVATE";
-
-  const presenter = new ApiRetrieveRunPresenter();
-  const result = await presenter.call(runId, authenticatedEnv, showSecretDetails);
-
-  if (!result) {
-    return apiCors(request, json({ error: "Run not found" }, { status: 404 }));
-  }
-
-  return apiCors(request, json(result));
-}
+);

apps/webapp/app/routes/realtime.v1.batches.$batchId.ts

@@ -0,0 +1,36 @@
+import { json } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { $replica } from "~/db.server";
+import { realtimeClient } from "~/services/realtimeClientGlobal.server";
+import { createLoaderApiRoute } from "~/services/routeBuiilders/apiBuilder.server";
+
+const ParamsSchema = z.object({
+  batchId: z.string(),
+});
+
+export const loader = createLoaderApiRoute(
+  {
+    params: ParamsSchema,
+    allowJWT: true,
+    corsStrategy: "all",
+    authorization: {
+      action: "read",
+      resource: (params) => ({ batch: params.batchId }),
+      superScopes: ["read:runs", "read:all", "admin"],
+    },
+  },
+  async ({ params, authentication, request }) => {
+    const batchRun = await $replica.batchTaskRun.findFirst({
+      where: {
+        friendlyId: params.batchId,
+        runtimeEnvironmentId: authentication.environment.id,
+      },
+    });
+
+    if (!batchRun) {
+      return json({ error: "Batch not found" }, { status: 404 });
+    }
+
+    return realtimeClient.streamBatch(request.url, authentication.environment, batchRun.id);
+  }
+);

apps/webapp/app/routes/realtime.v1.runs.$runId.ts

@@ -0,0 +1,36 @@
+import { json } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { $replica } from "~/db.server";
+import { realtimeClient } from "~/services/realtimeClientGlobal.server";
+import { createLoaderApiRoute } from "~/services/routeBuiilders/apiBuilder.server";
+
+const ParamsSchema = z.object({
+  runId: z.string(),
+});
+
+export const loader = createLoaderApiRoute(
+  {
+    params: ParamsSchema,
+    allowJWT: true,
+    corsStrategy: "all",
+    authorization: {
+      action: "read",
+      resource: (params) => ({ runs: params.runId }),
+      superScopes: ["read:runs", "read:all", "admin"],
+    },
+  },
+  async ({ params, authentication, request }) => {
+    const run = await $replica.taskRun.findFirst({
+      where: {
+        friendlyId: params.runId,
+        runtimeEnvironmentId: authentication.environment.id,
+      },
+    });
+
+    if (!run) {
+      return json({ error: "Run not found" }, { status: 404 });
+    }
+
+    return realtimeClient.streamRun(request.url, authentication.environment, run.id);
+  }
+);

apps/webapp/app/routes/realtime.v1.runs.ts

@@ -0,0 +1,28 @@
+import { z } from "zod";
+import { realtimeClient } from "~/services/realtimeClientGlobal.server";
+import { createLoaderApiRoute } from "~/services/routeBuiilders/apiBuilder.server";
+
+const SearchParamsSchema = z.object({
+  tags: z
+    .string()
+    .optional()
+    .transform((value) => {
+      return value ? value.split(",") : undefined;
+    }),
+});
+
+export const loader = createLoaderApiRoute(
+  {
+    searchParams: SearchParamsSchema,
+    allowJWT: true,
+    corsStrategy: "all",
+    authorization: {
+      action: "read",
+      resource: (_, searchParams) => searchParams,
+      superScopes: ["read:runs", "read:all", "admin"],
+    },
+  },
+  async ({ searchParams, authentication, request }) => {
+    return realtimeClient.streamRuns(request.url, authentication.environment, searchParams);
+  }
+);

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.schedules.new/route.tsx

@@ -44,6 +44,7 @@ import { CronPattern, UpsertSchedule } from "~/v3/schedules";
 import { UpsertTaskScheduleService } from "~/v3/services/upsertTaskSchedule.server";
 import { AIGeneratedCronField } from "../resources.orgs.$organizationSlug.projects.$projectParam.schedules.new.natural-language";
 import { TimezoneList } from "~/components/scheduled/timezones";
+import { logger } from "~/services/logger.server";
 
 const cronFormat = `*    *    *    *    *
 ┬    ┬    ┬    ┬    ┬
@@ -94,9 +95,9 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
       submission.value?.friendlyId === result.id ? "Schedule updated" : "Schedule created"
     );
   } catch (error: any) {
-    const errorMessage = `Failed: ${
-      error instanceof Error ? error.message : JSON.stringify(error)
-    }`;
+    logger.error("Failed to create schedule", error);
+
+    const errorMessage = `Something went wrong. Please try again.`;
     return redirectWithErrorMessage(
       v3SchedulesPath({ slug: organizationSlug }, { slug: projectParam }),
       request,

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx

@@ -857,7 +857,7 @@ function RunTimeline({ run }: { run: SpanRun }) {
               <RunTimelineEvent
                 title="Finished"
                 subtitle={<DateTimeAccurate date={run.updatedAt} />}
-                state="complete"
+                state={run.isError ? "error" : "complete"}
               />
             </>
           ) : (

apps/webapp/app/services/apiAuth.server.ts

@@ -1,52 +1,56 @@
+import { json } from "@remix-run/server-runtime";
 import { Prettify } from "@trigger.dev/core";
+import { SignJWT, errors, jwtVerify } from "jose";
 import { z } from "zod";
+import { prisma } from "~/db.server";
+import { env } from "~/env.server";
+import { findProjectByRef } from "~/models/project.server";
 import {
   RuntimeEnvironment,
   findEnvironmentByApiKey,
   findEnvironmentByPublicApiKey,
 } from "~/models/runtimeEnvironment.server";
+import { logger } from "./logger.server";
 import {
   PersonalAccessTokenAuthenticationResult,
   authenticateApiRequestWithPersonalAccessToken,
   isPersonalAccessToken,
 } from "./personalAccessToken.server";
-import { prisma } from "~/db.server";
-import { json } from "@remix-run/server-runtime";
-import { findProjectByRef } from "~/models/project.server";
-import { SignJWT, jwtVerify, errors } from "jose";
-import { env } from "~/env.server";
-import { logger } from "./logger.server";
+import { isPublicJWT, validatePublicJwtKey } from "./realtime/jwtAuth.server";
+
+const ClaimsSchema = z.object({
+  scopes: z.array(z.string()).optional(),
+});
 
 type Optional<T, K extends keyof T> = Prettify<Omit<T, K> & Partial<Pick<T, K>>>;
 
-const AuthorizationHeaderSchema = z.string().regex(/^Bearer .+$/);
-
 export type AuthenticatedEnvironment = Optional<
   NonNullable<Awaited<ReturnType<typeof findEnvironmentByApiKey>>>,
   "orgMember"
 >;
 
-type ApiAuthenticationResult = {
+export type ApiAuthenticationResult = {
   apiKey: string;
-  type: "PUBLIC" | "PRIVATE";
+  type: "PUBLIC" | "PRIVATE" | "PUBLIC_JWT";
   environment: AuthenticatedEnvironment;
+  scopes?: string[];
 };
 
 export async function authenticateApiRequest(
   request: Request,
-  { allowPublicKey = false }: { allowPublicKey?: boolean } = {}
+  options: { allowPublicKey?: boolean; allowJWT?: boolean } = {}
 ): Promise<ApiAuthenticationResult | undefined> {
   const apiKey = getApiKeyFromRequest(request);
   if (!apiKey) {
     return;
   }
 
-  return authenticateApiKey(apiKey, { allowPublicKey });
+  return authenticateApiKey(apiKey, options);
 }
 
 export async function authenticateApiKey(
   apiKey: string,
-  { allowPublicKey = false }: { allowPublicKey?: boolean } = {}
+  options: { allowPublicKey?: boolean; allowJWT?: boolean } = {}
 ): Promise<ApiAuthenticationResult | undefined> {
   const result = getApiKeyResult(apiKey);
 
@@ -54,14 +58,12 @@ export async function authenticateApiKey(
     return;
   }
 
-  //if it's a public API key and we don't allow public keys, return
-  if (!allowPublicKey) {
-    const environment = await findEnvironmentByApiKey(result.apiKey);
-    if (!environment) return;
-    return {
-      ...result,
-      environment,
-    };
+  if (!options.allowPublicKey && result.type === "PUBLIC") {
+    return;
+  }
+
+  if (!options.allowJWT && result.type === "PUBLIC_JWT") {
+    return;
   }
 
   switch (result.type) {
@@ -81,27 +83,72 @@ export async function authenticateApiKey(
         environment,
       };
     }
+    case "PUBLIC_JWT": {
+      const validationResults = await validatePublicJwtKey(result.apiKey);
+
+      if (!validationResults) {
+        return;
+      }
+
+      const parsedClaims = ClaimsSchema.safeParse(validationResults.claims);
+
+      return {
+        ...result,
+        environment: validationResults.environment,
+        scopes: parsedClaims.success ? parsedClaims.data.scopes : [],
+      };
+    }
   }
 }
 
+export async function authenticateAuthorizationHeader(
+  authorization: string,
+  {
+    allowPublicKey = false,
+    allowJWT = false,
+  }: { allowPublicKey?: boolean; allowJWT?: boolean } = {}
+): Promise<ApiAuthenticationResult | undefined> {
+  const apiKey = getApiKeyFromHeader(authorization);
+
+  if (!apiKey) {
+    return;
+  }
+
+  return authenticateApiKey(apiKey, { allowPublicKey, allowJWT });
+}
+
 export function isPublicApiKey(key: string) {
   return key.startsWith("pk_");
 }
 
-export function getApiKeyFromRequest(request: Request) {
-  const rawAuthorization = request.headers.get("Authorization");
+export function isSecretApiKey(key: string) {
+  return key.startsWith("tr_");
+}
 
-  const authorization = AuthorizationHeaderSchema.safeParse(rawAuthorization);
-  if (!authorization.success) {
+export function getApiKeyFromRequest(request: Request) {
+  return getApiKeyFromHeader(request.headers.get("Authorization"));
+}
+
+export function getApiKeyFromHeader(authorization?: string | null) {
+  if (typeof authorization !== "string" || !authorization) {
     return;
   }
 
-  const apiKey = authorization.data.replace(/^Bearer /, "");
+  const apiKey = authorization.replace(/^Bearer /, "");
   return apiKey;
 }
 
-export function getApiKeyResult(apiKey: string) {
-  const type = isPublicApiKey(apiKey) ? ("PUBLIC" as const) : ("PRIVATE" as const);
+export function getApiKeyResult(apiKey: string): {
+  apiKey: string;
+  type: "PUBLIC" | "PRIVATE" | "PUBLIC_JWT";
+} {
+  const type = isPublicApiKey(apiKey)
+    ? "PUBLIC"
+    : isSecretApiKey(apiKey)
+    ? "PRIVATE"
+    : isPublicJWT(apiKey)
+    ? "PUBLIC_JWT"
+    : "PRIVATE"; // Fallback to private key
   return { apiKey, type };
 }
 

apps/webapp/app/services/apiRateLimit.server.ts

@@ -1,150 +1,48 @@
-import { Ratelimit } from "@upstash/ratelimit";
-import { Request as ExpressRequest, Response as ExpressResponse, NextFunction } from "express";
-import { RedisOptions } from "ioredis";
-import { createHash } from "node:crypto";
 import { env } from "~/env.server";
-import { logger } from "./logger.server";
-import { Duration, Limiter, RateLimiter, createRedisRateLimitClient } from "./rateLimiter.server";
-
-type Options = {
-  redis?: RedisOptions;
-  keyPrefix: string;
-  pathMatchers: (RegExp | string)[];
-  pathWhiteList?: (RegExp | string)[];
-  limiter: Limiter;
-  log?: {
-    requests?: boolean;
-    rejections?: boolean;
-  };
-};
-
-//returns an Express middleware that rate limits using the Bearer token in the Authorization header
-export function authorizationRateLimitMiddleware({
-  redis,
-  keyPrefix,
-  limiter,
-  pathMatchers,
-  pathWhiteList = [],
-  log = {
-    rejections: true,
-    requests: true,
-  },
-}: Options) {
-  const rateLimiter = new RateLimiter({
-    redis,
-    keyPrefix,
-    limiter,
-    logSuccess: log.requests,
-    logFailure: log.rejections,
-  });
-
-  return async (req: ExpressRequest, res: ExpressResponse, next: NextFunction) => {
-    if (log.requests) {
-      logger.info(`RateLimiter (${keyPrefix}): request to ${req.path}`);
-    }
-
-    // allow OPTIONS requests
-    if (req.method.toUpperCase() === "OPTIONS") {
-      return next();
-    }
-
-    //first check if any of the pathMatchers match the request path
-    const path = req.path;
-    if (
-      !pathMatchers.some((matcher) =>
-        matcher instanceof RegExp ? matcher.test(path) : path === matcher
-      )
-    ) {
-      if (log.requests) {
-        logger.info(`RateLimiter (${keyPrefix}): didn't match ${req.path}`);
-      }
-      return next();
-    }
-
-    // Check if the path matches any of the whitelisted paths
-    if (
-      pathWhiteList.some((matcher) =>
-        matcher instanceof RegExp ? matcher.test(path) : path === matcher
-      )
-    ) {
-      if (log.requests) {
-        logger.info(`RateLimiter (${keyPrefix}): whitelisted ${req.path}`);
-      }
-      return next();
-    }
-
-    if (log.requests) {
-      logger.info(`RateLimiter (${keyPrefix}): matched ${req.path}`);
-    }
-
-    const authorizationValue = req.headers.authorization;
-    if (!authorizationValue) {
-      if (log.requests) {
-        logger.info(`RateLimiter (${keyPrefix}): no key`, { headers: req.headers, url: req.url });
-      }
-      res.setHeader("Content-Type", "application/problem+json");
-      return res.status(401).send(
-        JSON.stringify(
-          {
-            title: "Unauthorized",
-            status: 401,
-            type: "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/401",
-            detail: "No authorization header provided",
-            error: "No authorization header provided",
-          },
-          null,
-          2
-        )
-      );
-    }
-
-    const hash = createHash("sha256");
-    hash.update(authorizationValue);
-    const hashedAuthorizationValue = hash.digest("hex");
-
-    const { success, pending, limit, reset, remaining } = await rateLimiter.limit(
-      hashedAuthorizationValue
-    );
-
-    const $remaining = Math.max(0, remaining); // remaining can be negative if the user has exceeded the limit, so clamp it to 0
-
-    res.set("x-ratelimit-limit", limit.toString());
-    res.set("x-ratelimit-remaining", $remaining.toString());
-    res.set("x-ratelimit-reset", reset.toString());
-
-    if (success) {
-      return next();
-    }
-
-    res.setHeader("Content-Type", "application/problem+json");
-    const secondsUntilReset = Math.max(0, (reset - new Date().getTime()) / 1000);
-    return res.status(429).send(
-      JSON.stringify(
-        {
-          title: "Rate Limit Exceeded",
-          status: 429,
-          type: "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429",
-          detail: `Rate limit exceeded ${$remaining}/${limit} requests remaining. Retry in ${secondsUntilReset} seconds.`,
-          reset,
-          limit,
-          remaining,
-          secondsUntilReset,
-          error: `Rate limit exceeded ${$remaining}/${limit} requests remaining. Retry in ${secondsUntilReset} seconds.`,
-        },
-        null,
-        2
-      )
-    );
-  };
-}
+import { authenticateAuthorizationHeader } from "./apiAuth.server";
+import { authorizationRateLimitMiddleware } from "./authorizationRateLimitMiddleware.server";
+import { Duration } from "./rateLimiter.server";
 
 export const apiRateLimiter = authorizationRateLimitMiddleware({
+  redis: {
+    port: env.REDIS_PORT,
+    host: env.REDIS_HOST,
+    username: env.REDIS_USERNAME,
+    password: env.REDIS_PASSWORD,
+    enableAutoPipelining: true,
+    ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+  },
   keyPrefix: "api",
-  limiter: Ratelimit.tokenBucket(
-    env.API_RATE_LIMIT_REFILL_RATE,
-    env.API_RATE_LIMIT_REFILL_INTERVAL as Duration,
-    env.API_RATE_LIMIT_MAX
-  ),
+  defaultLimiter: {
+    type: "tokenBucket",
+    refillRate: env.API_RATE_LIMIT_REFILL_RATE,
+    interval: env.API_RATE_LIMIT_REFILL_INTERVAL as Duration,
+    maxTokens: env.API_RATE_LIMIT_MAX,
+  },
+  limiterCache: {
+    fresh: 60_000 * 10, // Data is fresh for 10 minutes
+    stale: 60_000 * 20, // Date is stale after 20 minutes
+  },
+  limiterConfigOverride: async (authorizationValue) => {
+    const authenticatedEnv = await authenticateAuthorizationHeader(authorizationValue, {
+      allowPublicKey: true,
+      allowJWT: true,
+    });
+
+    if (!authenticatedEnv) {
+      return;
+    }
+
+    if (authenticatedEnv.type === "PUBLIC_JWT") {
+      return {
+        type: "fixedWindow",
+        window: env.API_RATE_LIMIT_JWT_WINDOW,
+        tokens: env.API_RATE_LIMIT_JWT_TOKENS,
+      };
+    } else {
+      return authenticatedEnv.environment.organization.apiRateLimiterConfig;
+    }
+  },
   pathMatchers: [/^\/api/],
   // Allow /api/v1/tasks/:id/callback/:secret
   pathWhiteList: [
@@ -159,11 +57,13 @@ export const apiRateLimiter = authorizationRateLimitMiddleware({
     /^\/api\/v1\/endpoints\/[^\/]+\/[^\/]+\/index\/[^\/]+$/, // /api/v1/endpoints/$environmentId/$endpointSlug/index/$indexHookIdentifier
     "/api/v1/timezones",
     "/api/v1/usage/ingest",
+    "/api/v1/auth/jwt/claims",
     /^\/api\/v1\/runs\/[^\/]+\/attempts$/, // /api/v1/runs/$runFriendlyId/attempts
   ],
   log: {
     rejections: env.API_RATE_LIMIT_REJECTION_LOGS_ENABLED === "1",
     requests: env.API_RATE_LIMIT_REQUEST_LOGS_ENABLED === "1",
+    limiter: env.API_RATE_LIMIT_LIMITER_LOGS_ENABLED === "1",
   },
 });
 

apps/webapp/app/services/authorization.server.ts

@@ -0,0 +1,103 @@
+export type AuthorizationAction = "read"; // Add more actions as needed
+
+const ResourceTypes = ["tasks", "tags", "runs", "batch"] as const;
+
+export type AuthorizationResources = {
+  [key in (typeof ResourceTypes)[number]]?: string | string[];
+};
+
+export type AuthorizationEntity = {
+  type: "PUBLIC" | "PRIVATE" | "PUBLIC_JWT";
+  scopes?: string[];
+};
+
+/**
+ * Checks if the given entity is authorized to perform a specific action on a resource.
+ *
+ * @param entity - The entity requesting authorization.
+ * @param action - The action the entity wants to perform.
+ * @param resource - The resource on which the action is to be performed.
+ * @param superScopes - An array of super scopes that can bypass the normal authorization checks.
+ *
+ * @example
+ *
+ * ```typescript
+ * import { checkAuthorization } from "./authorization.server";
+ *
+ * const entity = {
+ *  type: "PUBLIC",
+ *  scope: ["read:runs:run_1234", "read:tasks"]
+ * };
+ *
+ * checkAuthorization(entity, "read", { runs: "run_1234" }); // Returns true
+ * checkAuthorization(entity, "read", { runs: "run_5678" }); // Returns false
+ * checkAuthorization(entity, "read", { tasks: "task_1234" }); // Returns true
+ * checkAuthorization(entity, "read", { tasks: ["task_5678"] }); // Returns true
+ * ```
+ */
+export function checkAuthorization(
+  entity: AuthorizationEntity,
+  action: AuthorizationAction,
+  resource: AuthorizationResources,
+  superScopes?: string[]
+) {
+  // "PRIVATE" is a secret key and has access to everything
+  if (entity.type === "PRIVATE") {
+    return true;
+  }
+
+  // "PUBLIC" is a deprecated key and has no access
+  if (entity.type === "PUBLIC") {
+    return false;
+  }
+
+  // If the entity has no permissions, deny access
+  if (!entity.scopes || entity.scopes.length === 0) {
+    return false;
+  }
+
+  // If the resource object is empty, deny access
+  if (Object.keys(resource).length === 0) {
+    return false;
+  }
+
+  // Check for any of the super scopes
+  if (superScopes && superScopes.length > 0) {
+    if (superScopes.some((permission) => entity.scopes?.includes(permission))) {
+      return true;
+    }
+  }
+
+  const filteredResource = Object.keys(resource).reduce((acc, key) => {
+    if (ResourceTypes.includes(key)) {
+      acc[key as keyof AuthorizationResources] = resource[key as keyof AuthorizationResources];
+    }
+    return acc;
+  }, {} as AuthorizationResources);
+
+  // Check each resource type
+  for (const [resourceType, resourceValue] of Object.entries(filteredResource)) {
+    const resourceValues = Array.isArray(resourceValue) ? resourceValue : [resourceValue];
+
+    let resourceAuthorized = false;
+    for (const value of resourceValues) {
+      // Check for specific resource permission
+      const specificPermission = `${action}:${resourceType}:${value}`;
+      // Check for general resource type permission
+      const generalPermission = `${action}:${resourceType}`;
+
+      if (entity.scopes.includes(specificPermission) || entity.scopes.includes(generalPermission)) {
+        resourceAuthorized = true;
+        break;
+      }
+    }
+
+    // If any resource is not authorized, return false
+    if (!resourceAuthorized) {
+      return false;
+    }
+  }
+
+  // All resources are authorized
+  return true;
+}

apps/webapp/app/services/authorizationRateLimitMiddleware.server.ts

@@ -0,0 +1,301 @@
+import { createCache, DefaultStatefulContext, Namespace, Cache as UnkeyCache } from "@unkey/cache";
+import { MemoryStore } from "@unkey/cache/stores";
+import { Ratelimit } from "@upstash/ratelimit";
+import { Request as ExpressRequest, Response as ExpressResponse, NextFunction } from "express";
+import { RedisOptions } from "ioredis";
+import { createHash } from "node:crypto";
+import { z } from "zod";
+import { env } from "~/env.server";
+import { logger } from "./logger.server";
+import { createRedisRateLimitClient, Duration, RateLimiter } from "./rateLimiter.server";
+import { RedisCacheStore } from "./unkey/redisCacheStore.server";
+
+const DurationSchema = z.custom<Duration>((value) => {
+  if (typeof value !== "string") {
+    throw new Error("Duration must be a string");
+  }
+
+  return value as Duration;
+});
+
+export const RateLimitFixedWindowConfig = z.object({
+  type: z.literal("fixedWindow"),
+  window: DurationSchema,
+  tokens: z.number(),
+});
+
+export type RateLimitFixedWindowConfig = z.infer<typeof RateLimitFixedWindowConfig>;
+
+export const RateLimitSlidingWindowConfig = z.object({
+  type: z.literal("slidingWindow"),
+  window: DurationSchema,
+  tokens: z.number(),
+});
+
+export type RateLimitSlidingWindowConfig = z.infer<typeof RateLimitSlidingWindowConfig>;
+
+export const RateLimitTokenBucketConfig = z.object({
+  type: z.literal("tokenBucket"),
+  refillRate: z.number(),
+  interval: DurationSchema,
+  maxTokens: z.number(),
+});
+
+export type RateLimitTokenBucketConfig = z.infer<typeof RateLimitTokenBucketConfig>;
+
+export const RateLimiterConfig = z.discriminatedUnion("type", [
+  RateLimitFixedWindowConfig,
+  RateLimitSlidingWindowConfig,
+  RateLimitTokenBucketConfig,
+]);
+
+export type RateLimiterConfig = z.infer<typeof RateLimiterConfig>;
+
+type LimitConfigOverrideFunction = (authorizationValue: string) => Promise<unknown>;
+
+type Options = {
+  redis?: RedisOptions;
+  keyPrefix: string;
+  pathMatchers: (RegExp | string)[];
+  pathWhiteList?: (RegExp | string)[];
+  defaultLimiter: RateLimiterConfig;
+  limiterConfigOverride?: LimitConfigOverrideFunction;
+  limiterCache?: {
+    fresh: number;
+    stale: number;
+  };
+  log?: {
+    requests?: boolean;
+    rejections?: boolean;
+    limiter?: boolean;
+  };
+};
+
+async function resolveLimitConfig(
+  authorizationValue: string,
+  hashedAuthorizationValue: string,
+  defaultLimiter: RateLimiterConfig,
+  cache: UnkeyCache<{ limiter: RateLimiterConfig }>,
+  logsEnabled: boolean,
+  limiterConfigOverride?: LimitConfigOverrideFunction
+): Promise<RateLimiterConfig> {
+  if (!limiterConfigOverride) {
+    return defaultLimiter;
+  }
+
+  if (logsEnabled) {
+    logger.info("RateLimiter: checking for override", {
+      authorizationValue: hashedAuthorizationValue,
+      defaultLimiter,
+    });
+  }
+
+  const cacheResult = await cache.limiter.swr(hashedAuthorizationValue, async (key) => {
+    const override = await limiterConfigOverride(authorizationValue);
+
+    if (!override) {
+      if (logsEnabled) {
+        logger.info("RateLimiter: no override found", {
+          authorizationValue,
+          defaultLimiter,
+        });
+      }
+
+      return defaultLimiter;
+    }
+
+    const parsedOverride = RateLimiterConfig.safeParse(override);
+
+    if (!parsedOverride.success) {
+      logger.error("Error parsing rate limiter override", {
+        override,
+        errors: parsedOverride.error.errors,
+      });
+
+      return defaultLimiter;
+    }
+
+    if (logsEnabled && parsedOverride.data) {
+      logger.info("RateLimiter: override found", {
+        authorizationValue,
+        defaultLimiter,
+        override: parsedOverride.data,
+      });
+    }
+
+    return parsedOverride.data;
+  });
+
+  return cacheResult.val ?? defaultLimiter;
+}
+
+//returns an Express middleware that rate limits using the Bearer token in the Authorization header
+export function authorizationRateLimitMiddleware({
+  redis,
+  keyPrefix,
+  defaultLimiter,
+  pathMatchers,
+  pathWhiteList = [],
+  log = {
+    rejections: true,
+    requests: true,
+  },
+  limiterCache,
+  limiterConfigOverride,
+}: Options) {
+  const ctx = new DefaultStatefulContext();
+  const memory = new MemoryStore({ persistentMap: new Map() });
+  const redisCacheStore = new RedisCacheStore({
+    connection: {
+      keyPrefix: `cache:${keyPrefix}:rate-limit-cache:`,
+      ...redis,
+    },
+  });
+
+  // This cache holds the rate limit configuration for each org, so we don't have to fetch it every request
+  const cache = createCache({
+    limiter: new Namespace<RateLimiterConfig>(ctx, {
+      stores: [memory, redisCacheStore],
+      fresh: limiterCache?.fresh ?? 30_000,
+      stale: limiterCache?.stale ?? 60_000,
+    }),
+  });
+
+  const redisClient = createRedisRateLimitClient(
+    redis ?? {
+      port: env.REDIS_PORT,
+      host: env.REDIS_HOST,
+      username: env.REDIS_USERNAME,
+      password: env.REDIS_PASSWORD,
+      enableAutoPipelining: true,
+      ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+    }
+  );
+
+  return async (req: ExpressRequest, res: ExpressResponse, next: NextFunction) => {
+    if (log.requests) {
+      logger.info(`RateLimiter (${keyPrefix}): request to ${req.path}`);
+    }
+
+    // allow OPTIONS requests
+    if (req.method.toUpperCase() === "OPTIONS") {
+      return next();
+    }
+
+    //first check if any of the pathMatchers match the request path
+    const path = req.path;
+    if (
+      !pathMatchers.some((matcher) =>
+        matcher instanceof RegExp ? matcher.test(path) : path === matcher
+      )
+    ) {
+      if (log.requests) {
+        logger.info(`RateLimiter (${keyPrefix}): didn't match ${req.path}`);
+      }
+      return next();
+    }
+
+    // Check if the path matches any of the whitelisted paths
+    if (
+      pathWhiteList.some((matcher) =>
+        matcher instanceof RegExp ? matcher.test(path) : path === matcher
+      )
+    ) {
+      if (log.requests) {
+        logger.info(`RateLimiter (${keyPrefix}): whitelisted ${req.path}`);
+      }
+      return next();
+    }
+
+    if (log.requests) {
+      logger.info(`RateLimiter (${keyPrefix}): matched ${req.path}`);
+    }
+
+    const authorizationValue = req.headers.authorization;
+    if (!authorizationValue) {
+      if (log.requests) {
+        logger.info(`RateLimiter (${keyPrefix}): no key`, { headers: req.headers, url: req.url });
+      }
+      res.setHeader("Content-Type", "application/problem+json");
+      return res.status(401).send(
+        JSON.stringify(
+          {
+            title: "Unauthorized",
+            status: 401,
+            type: "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/401",
+            detail: "No authorization header provided",
+            error: "No authorization header provided",
+          },
+          null,
+          2
+        )
+      );
+    }
+
+    const hash = createHash("sha256");
+    hash.update(authorizationValue);
+    const hashedAuthorizationValue = hash.digest("hex");
+
+    const limiterConfig = await resolveLimitConfig(
+      authorizationValue,
+      hashedAuthorizationValue,
+      defaultLimiter,
+      cache,
+      typeof log.limiter === "boolean" ? log.limiter : false,
+      limiterConfigOverride
+    );
+
+    const limiter =
+      limiterConfig.type === "fixedWindow"
+        ? Ratelimit.fixedWindow(limiterConfig.tokens, limiterConfig.window)
+        : limiterConfig.type === "tokenBucket"
+        ? Ratelimit.tokenBucket(
+            limiterConfig.refillRate,
+            limiterConfig.interval,
+            limiterConfig.maxTokens
+          )
+        : Ratelimit.slidingWindow(limiterConfig.tokens, limiterConfig.window);
+
+    const rateLimiter = new RateLimiter({
+      redisClient,
+      keyPrefix,
+      limiter,
+      logSuccess: log.requests,
+      logFailure: log.rejections,
+    });
+
+    const { success, limit, reset, remaining } = await rateLimiter.limit(hashedAuthorizationValue);
+
+    const $remaining = Math.max(0, remaining); // remaining can be negative if the user has exceeded the limit, so clamp it to 0
+
+    res.set("x-ratelimit-limit", limit.toString());
+    res.set("x-ratelimit-remaining", $remaining.toString());
+    res.set("x-ratelimit-reset", reset.toString());
+
+    if (success) {
+      return next();
+    }
+
+    res.setHeader("Content-Type", "application/problem+json");
+    const secondsUntilReset = Math.max(0, (reset - new Date().getTime()) / 1000);
+    return res.status(429).send(
+      JSON.stringify(
+        {
+          title: "Rate Limit Exceeded",
+          status: 429,
+          type: "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429",
+          detail: `Rate limit exceeded ${$remaining}/${limit} requests remaining. Retry in ${secondsUntilReset} seconds.`,
+          reset,
+          limit,
+          remaining,
+          secondsUntilReset,
+          error: `Rate limit exceeded ${$remaining}/${limit} requests remaining. Retry in ${secondsUntilReset} seconds.`,
+        },
+        null,
+        2
+      )
+    );
+  };
+}
+
+export type RateLimitMiddleware = ReturnType<typeof authorizationRateLimitMiddleware>;

apps/webapp/app/services/platform.v3.server.ts

@@ -1,5 +1,13 @@
-import { BillingClient, Limits, SetPlanBody, UsageSeriesParams } from "@trigger.dev/platform/v3";
 import { Organization, Project } from "@trigger.dev/database";
+import {
+  BillingClient,
+  Limits,
+  SetPlanBody,
+  UsageSeriesParams,
+  UsageResult,
+} from "@trigger.dev/platform/v3";
+import { createCache, DefaultStatefulContext, Namespace } from "@unkey/cache";
+import { MemoryStore } from "@unkey/cache/stores";
 import { redirect } from "remix-typedjson";
 import { $replica } from "~/db.server";
 import { env } from "~/env.server";
@@ -7,10 +15,61 @@ import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/m
 import { createEnvironment } from "~/models/organization.server";
 import { logger } from "~/services/logger.server";
 import { newProjectPath, organizationBillingPath } from "~/utils/pathBuilder";
+import { singleton } from "~/utils/singleton";
+import { RedisCacheStore } from "./unkey/redisCacheStore.server";
+
+function initializeClient() {
+  if (isCloud() && process.env.BILLING_API_URL && process.env.BILLING_API_KEY) {
+    const client = new BillingClient({
+      url: process.env.BILLING_API_URL,
+      apiKey: process.env.BILLING_API_KEY,
+    });
+    console.log(`🤑 Billing client initialized: ${process.env.BILLING_API_URL}`);
+    return client;
+  } else {
+    console.log(`🤑 Billing client not initialized`);
+  }
+}
+
+const client = singleton("billingClient", initializeClient);
+
+function initializePlatformCache() {
+  const ctx = new DefaultStatefulContext();
+  const memory = new MemoryStore({ persistentMap: new Map() });
+  const redisCacheStore = new RedisCacheStore({
+    connection: {
+      keyPrefix: "tr:cache:platform:v3",
+      port: env.REDIS_PORT,
+      host: env.REDIS_HOST,
+      username: env.REDIS_USERNAME,
+      password: env.REDIS_PASSWORD,
+      enableAutoPipelining: true,
+      ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+    },
+  });
+
+  // This cache holds the limits fetched from the platform service
+  const cache = createCache({
+    limits: new Namespace<number>(ctx, {
+      stores: [memory, redisCacheStore],
+      fresh: 60_000 * 5, // 5 minutes
+      stale: 60_000 * 10, // 10 minutes
+    }),
+    usage: new Namespace<UsageResult>(ctx, {
+      stores: [memory, redisCacheStore],
+      fresh: 60_000 * 5, // 5 minutes
+      stale: 60_000 * 10, // 10 minutes
+    }),
+  });
+
+  return cache;
+}
+
+const platformCache = singleton("platformCache", initializePlatformCache);
 
 export async function getCurrentPlan(orgId: string) {
-  const client = getClient();
   if (!client) return undefined;
+
   try {
     const result = await client.currentPlan(orgId);
 
@@ -60,8 +119,8 @@ export async function getCurrentPlan(orgId: string) {
 }
 
 export async function getLimits(orgId: string) {
-  const client = getClient();
   if (!client) return undefined;
+
   try {
     const result = await client.currentPlan(orgId);
     if (!result.success) {
@@ -87,9 +146,15 @@ export async function getLimit(orgId: string, limit: keyof Limits, fallback: num
   return fallback;
 }
 
+export async function getCachedLimit(orgId: string, limit: keyof Limits, fallback: number) {
+  return platformCache.limits.swr(`${orgId}:${limit}`, async () => {
+    return getLimit(orgId, limit, fallback);
+  });
+}
+
 export async function customerPortalUrl(orgId: string, orgSlug: string) {
-  const client = getClient();
   if (!client) return undefined;
+
   try {
     return client.createPortalSession(orgId, {
       returnUrl: `${env.APP_ORIGIN}${organizationBillingPath({ slug: orgSlug })}`,
@@ -101,8 +166,8 @@ export async function customerPortalUrl(orgId: string, orgSlug: string) {
 }
 
 export async function getPlans() {
-  const client = getClient();
   if (!client) return undefined;
+
   try {
     const result = await client.plans();
     if (!result.success) {
@@ -122,7 +187,6 @@ export async function setPlan(
   callerPath: string,
   plan: SetPlanBody
 ) {
-  const client = getClient();
   if (!client) {
     throw redirectWithErrorMessage(callerPath, request, "Error setting plan");
   }
@@ -178,8 +242,8 @@ export async function setPlan(
 }
 
 export async function getUsage(organizationId: string, { from, to }: { from: Date; to: Date }) {
-  const client = getClient();
   if (!client) return undefined;
+
   try {
     const result = await client.usage(organizationId, { from, to });
     if (!result.success) {
@@ -193,9 +257,27 @@ export async function getUsage(organizationId: string, { from, to }: { from: Dat
   }
 }
 
-export async function getUsageSeries(organizationId: string, params: UsageSeriesParams) {
-  const client = getClient();
+export async function getCachedUsage(
+  organizationId: string,
+  { from, to }: { from: Date; to: Date }
+) {
   if (!client) return undefined;
+
+  const result = await platformCache.usage.swr(
+    `${organizationId}:${from.toISOString()}:${to.toISOString()}`,
+    async () => {
+      const usageResponse = await getUsage(organizationId, { from, to });
+
+      return usageResponse;
+    }
+  );
+
+  return result.val;
+}
+
+export async function getUsageSeries(organizationId: string, params: UsageSeriesParams) {
+  if (!client) return undefined;
+
   try {
     const result = await client.usageSeries(organizationId, params);
     if (!result.success) {
@@ -214,8 +296,8 @@ export async function reportInvocationUsage(
   costInCents: number,
   additionalData?: Record<string, any>
 ) {
-  const client = getClient();
   if (!client) return undefined;
+
   try {
     const result = await client.reportInvocationUsage({
       organizationId,
@@ -234,8 +316,8 @@ export async function reportInvocationUsage(
 }
 
 export async function reportComputeUsage(request: Request) {
-  const client = getClient();
   if (!client) return undefined;
+
   return fetch(`${process.env.BILLING_API_URL}/api/v1/usage/ingest/compute`, {
     method: "POST",
     headers: request.headers,
@@ -244,8 +326,8 @@ export async function reportComputeUsage(request: Request) {
 }
 
 export async function getEntitlement(organizationId: string) {
-  const client = getClient();
   if (!client) return undefined;
+
   try {
     const result = await client.getEntitlement(organizationId);
     if (!result.success) {
@@ -275,19 +357,6 @@ export async function projectCreated(organization: Organization, project: Projec
   }
 }
 
-function getClient() {
-  if (isCloud() && process.env.BILLING_API_URL && process.env.BILLING_API_KEY) {
-    const client = new BillingClient({
-      url: process.env.BILLING_API_URL,
-      apiKey: process.env.BILLING_API_KEY,
-    });
-    console.log(`Billing client initialized: ${process.env.BILLING_API_URL}`);
-    return client;
-  } else {
-    console.log(`Billing client not initialized`);
-  }
-}
-
 function isCloud(): boolean {
   const acceptableHosts = [
     "https://cloud.trigger.dev",

apps/webapp/app/services/rateLimiter.server.ts

@@ -5,6 +5,7 @@ import { logger } from "./logger.server";
 
 type Options = {
   redis?: RedisOptions;
+  redisClient?: RateLimiterRedisClient;
   keyPrefix: string;
   limiter: Limiter;
   logSuccess?: boolean;
@@ -14,34 +15,32 @@ type Options = {
 export type Limiter = ConstructorParameters<typeof Ratelimit>[0]["limiter"];
 export type Duration = Parameters<typeof Ratelimit.slidingWindow>[1];
 export type RateLimitResponse = Awaited<ReturnType<Ratelimit["limit"]>>;
+export type RateLimiterRedisClient = ConstructorParameters<typeof Ratelimit>[0]["redis"];
 
 export class RateLimiter {
   #ratelimit: Ratelimit;
 
   constructor(private readonly options: Options) {
-    const { redis, keyPrefix, limiter } = options;
+    const { redis, redisClient, keyPrefix, limiter } = options;
     const prefix = `ratelimit:${keyPrefix}`;
     this.#ratelimit = new Ratelimit({
-      redis: createRedisRateLimitClient(
-        redis ?? {
-          port: env.REDIS_PORT,
-          host: env.REDIS_HOST,
-          username: env.REDIS_USERNAME,
-          password: env.REDIS_PASSWORD,
-          enableAutoPipelining: true,
-          ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
-        }
-      ),
+      redis:
+        redisClient ??
+        createRedisRateLimitClient(
+          redis ?? {
+            port: env.REDIS_PORT,
+            host: env.REDIS_HOST,
+            username: env.REDIS_USERNAME,
+            password: env.REDIS_PASSWORD,
+            enableAutoPipelining: true,
+            ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+          }
+        ),
       limiter,
       ephemeralCache: new Map(),
       analytics: false,
       prefix,
     });
-
-    logger.info(`RateLimiter (${keyPrefix}): initialized`, {
-      keyPrefix,
-      redisKeyspace: prefix,
-    });
   }
 
   async limit(identifier: string, rate = 1): Promise<RateLimitResponse> {
@@ -71,9 +70,7 @@ export class RateLimiter {
   }
 }
 
-export function createRedisRateLimitClient(
-  redisOptions: RedisOptions
-): ConstructorParameters<typeof Ratelimit>[0]["redis"] {
+export function createRedisRateLimitClient(redisOptions: RedisOptions): RateLimiterRedisClient {
   const redis = new Redis(redisOptions);
 
   return {

apps/webapp/app/services/realtime/jwtAuth.server.ts

@@ -0,0 +1,85 @@
+import { validateJWT } from "@trigger.dev/core/v3/jwt";
+import { findEnvironmentById } from "~/models/runtimeEnvironment.server";
+
+export async function validatePublicJwtKey(token: string) {
+  // Get the sub claim from the token
+  // Use the sub claim to find the environment
+  // Validate the token against the environment.apiKey
+  // Once that's done, return the environment and the claims
+  const sub = extractJWTSub(token);
+
+  if (!sub) {
+    return;
+  }
+
+  const environment = await findEnvironmentById(sub);
+
+  if (!environment) {
+    return;
+  }
+
+  const claims = await validateJWT(token, environment.apiKey);
+
+  if (!claims) {
+    return;
+  }
+
+  return {
+    environment,
+    claims,
+  };
+}
+
+export function isPublicJWT(token: string): boolean {
+  // Split the token
+  const parts = token.split(".");
+  if (parts.length !== 3) return false;
+
+  try {
+    // Decode the payload (second part)
+    const payload = JSON.parse(decodeBase64Url(parts[1]));
+
+    if (payload === null || typeof payload !== "object") return false;
+
+    // Check for the pub: true claim
+    return "pub" in payload && payload.pub === true;
+  } catch (error) {
+    // If there's any error in decoding or parsing, it's not a valid JWT
+    return false;
+  }
+}
+
+function extractJWTSub(token: string): string | undefined {
+  // Split the token
+  const parts = token.split(".");
+  if (parts.length !== 3) return;
+
+  try {
+    // Decode the payload (second part)
+    const payload = JSON.parse(decodeBase64Url(parts[1]));
+
+    if (payload === null || typeof payload !== "object") return;
+
+    // Check for the pub: true claim
+    return "sub" in payload && typeof payload.sub === "string" ? payload.sub : undefined;
+  } catch (error) {
+    // If there's any error in decoding or parsing, it's not a valid JWT
+    return;
+  }
+}
+
+function decodeBase64Url(str: string): string {
+  // Replace URL-safe characters and add padding
+  str = str.replace(/-/g, "+").replace(/_/g, "/");
+  switch (str.length % 4) {
+    case 2:
+      str += "==";
+      break;
+    case 3:
+      str += "=";
+      break;
+  }
+
+  // Decode using Node.js Buffer
+  return Buffer.from(str, "base64").toString("utf8");
+}

apps/webapp/app/services/realtimeClient.server.ts

@@ -0,0 +1,253 @@
+import { json } from "@remix-run/server-runtime";
+import Redis, { Callback, Result, type RedisOptions } from "ioredis";
+import { randomUUID } from "node:crypto";
+import { longPollingFetch } from "~/utils/longPollingFetch";
+import { logger } from "./logger.server";
+
+export interface CachedLimitProvider {
+  getCachedLimit: (organizationId: string, defaultValue: number) => Promise<number | undefined>;
+}
+
+export type RealtimeClientOptions = {
+  electricOrigin: string;
+  redis: RedisOptions;
+  cachedLimitProvider: CachedLimitProvider;
+  keyPrefix: string;
+  expiryTimeInSeconds?: number;
+};
+
+export type RealtimeEnvironment = {
+  id: string;
+  organizationId: string;
+};
+
+export type RealtimeRunsParams = {
+  tags?: string[];
+};
+
+export class RealtimeClient {
+  private redis: Redis;
+  private expiryTimeInSeconds: number;
+  private cachedLimitProvider: CachedLimitProvider;
+
+  constructor(private options: RealtimeClientOptions) {
+    this.redis = new Redis(options.redis);
+    this.expiryTimeInSeconds = options.expiryTimeInSeconds ?? 60 * 5; // default to 5 minutes
+    this.cachedLimitProvider = options.cachedLimitProvider;
+    this.#registerCommands();
+  }
+
+  async streamRun(url: URL | string, environment: RealtimeEnvironment, runId: string) {
+    return this.#streamRunsWhere(url, environment, `id='${runId}'`);
+  }
+
+  async streamBatch(url: URL | string, environment: RealtimeEnvironment, batchId: string) {
+    return this.#streamRunsWhere(url, environment, `"batchId"='${batchId}'`);
+  }
+
+  async streamRuns(
+    url: URL | string,
+    environment: RealtimeEnvironment,
+    params: RealtimeRunsParams
+  ) {
+    const whereClauses: string[] = [`"runtimeEnvironmentId"='${environment.id}'`];
+
+    if (params.tags) {
+      whereClauses.push(`"runTags" @> ARRAY[${params.tags.map((t) => `'${t}'`).join(",")}]`);
+    }
+
+    const whereClause = whereClauses.join(" AND ");
+
+    return this.#streamRunsWhere(url, environment, whereClause);
+  }
+
+  async #streamRunsWhere(url: URL | string, environment: RealtimeEnvironment, whereClause: string) {
+    const electricUrl = this.#constructElectricUrl(url, whereClause);
+
+    return this.#performElectricRequest(electricUrl, environment);
+  }
+
+  #constructElectricUrl(url: URL | string, whereClause: string): URL {
+    const $url = new URL(url.toString());
+
+    const electricUrl = new URL(`${this.options.electricOrigin}/v1/shape/public."TaskRun"`);
+
+    // Copy over all the url search params to the electric url
+    $url.searchParams.forEach((value, key) => {
+      electricUrl.searchParams.set(key, value);
+    });
+
+    // const electricParams = ["shape_id", "live", "offset", "columns", "cursor"];
+
+    // electricParams.forEach((param) => {
+    //   if ($url.searchParams.has(param) && $url.searchParams.get(param)) {
+    //     electricUrl.searchParams.set(param, $url.searchParams.get(param)!);
+    //   }
+    // });
+
+    electricUrl.searchParams.set("where", whereClause);
+
+    return electricUrl;
+  }
+
+  async #performElectricRequest(url: URL, environment: RealtimeEnvironment) {
+    const shapeId = extractShapeId(url);
+
+    logger.debug("[realtimeClient] request", {
+      url: url.toString(),
+    });
+
+    if (!shapeId) {
+      // If the shapeId is not present, we're just getting the initial value
+      return longPollingFetch(url.toString());
+    }
+
+    const isLive = isLiveRequestUrl(url);
+
+    if (!isLive) {
+      return longPollingFetch(url.toString());
+    }
+
+    const requestId = randomUUID();
+
+    // We now need to wrap the longPollingFetch in a concurrency tracker
+    const concurrencyLimit = await this.cachedLimitProvider.getCachedLimit(
+      environment.organizationId,
+      100_000
+    );
+
+    if (!concurrencyLimit) {
+      logger.error("Failed to get concurrency limit", {
+        organizationId: environment.organizationId,
+      });
+
+      return json({ error: "Failed to get concurrency limit" }, { status: 500 });
+    }
+
+    logger.debug("[realtimeClient] increment and check", {
+      concurrencyLimit,
+      shapeId,
+      requestId,
+      environment: {
+        id: environment.id,
+        organizationId: environment.organizationId,
+      },
+    });
+
+    const canProceed = await this.#incrementAndCheck(environment.id, requestId, concurrencyLimit);
+
+    if (!canProceed) {
+      logger.debug("[realtimeClient] too many concurrent requests", {
+        requestId,
+        environmentId: environment.id,
+      });
+
+      return json({ error: "Too many concurrent requests" }, { status: 429 });
+    }
+
+    try {
+      // ... (rest of your existing code for the long polling request)
+      const response = await longPollingFetch(url.toString());
+
+      // Decrement the counter after the long polling request is complete
+      await this.#decrementConcurrency(environment.id, requestId);
+
+      return response;
+    } catch (error) {
+      // Decrement the counter if the request fails
+      await this.#decrementConcurrency(environment.id, requestId);
+
+      throw error;
+    }
+  }
+
+  async #incrementAndCheck(environmentId: string, requestId: string, limit: number) {
+    const key = this.#getKey(environmentId);
+    const now = Date.now();
+
+    const result = await this.redis.incrementAndCheckConcurrency(
+      key,
+      now.toString(),
+      requestId,
+      this.expiryTimeInSeconds.toString(), // expiry time
+      (now - this.expiryTimeInSeconds * 1000).toString(), // cutoff time
+      limit.toString()
+    );
+
+    return result === 1;
+  }
+
+  async #decrementConcurrency(environmentId: string, requestId: string) {
+    logger.debug("[realtimeClient] decrement", {
+      requestId,
+      environmentId,
+    });
+
+    const key = this.#getKey(environmentId);
+
+    await this.redis.zrem(key, requestId);
+  }
+
+  #getKey(environmentId: string): string {
+    return `${this.options.keyPrefix}:${environmentId}`;
+  }
+
+  #registerCommands() {
+    this.redis.defineCommand("incrementAndCheckConcurrency", {
+      numberOfKeys: 1,
+      lua: /* lua */ `
+        local concurrencyKey = KEYS[1]
+
+        local timestamp = tonumber(ARGV[1])
+        local requestId = ARGV[2]
+        local expiryTime = tonumber(ARGV[3])
+        local cutoffTime = tonumber(ARGV[4])
+        local limit = tonumber(ARGV[5])
+
+        -- Remove expired entries
+        redis.call('ZREMRANGEBYSCORE', concurrencyKey, '-inf', cutoffTime)
+
+        -- Add the new request to the sorted set
+        redis.call('ZADD', concurrencyKey, timestamp, requestId)
+
+        -- Set the expiry time on the key
+        redis.call('EXPIRE', concurrencyKey, expiryTime)
+
+        -- Get the total number of concurrent requests
+        local totalRequests = redis.call('ZCARD', concurrencyKey)
+
+        -- Check if the limit has been exceeded
+        if totalRequests > limit then
+            -- Remove the request we just added
+            redis.call('ZREM', concurrencyKey, requestId)
+            return 0
+        end
+
+        -- Return 1 to indicate success
+        return 1
+      `,
+    });
+  }
+}
+
+function extractShapeId(url: URL) {
+  return url.searchParams.get("shape_id");
+}
+
+function isLiveRequestUrl(url: URL) {
+  return url.searchParams.has("live") && url.searchParams.get("live") === "true";
+}
+
+declare module "ioredis" {
+  interface RedisCommander<Context> {
+    incrementAndCheckConcurrency(
+      key: string,
+      timestamp: string,
+      requestId: string,
+      expiryTime: string,
+      cutoffTime: string,
+      limit: string,
+      callback?: Callback<number>
+    ): Result<number, Context>;
+  }
+}

apps/webapp/app/services/realtimeClientGlobal.server.ts

@@ -0,0 +1,32 @@
+import { env } from "~/env.server";
+import { singleton } from "~/utils/singleton";
+import { RealtimeClient } from "./realtimeClient.server";
+import { getCachedLimit } from "./platform.v3.server";
+
+function initializeRealtimeClient() {
+  return new RealtimeClient({
+    electricOrigin: env.ELECTRIC_ORIGIN,
+    keyPrefix: "tr:realtime:concurrency",
+    redis: {
+      port: env.REDIS_PORT,
+      host: env.REDIS_HOST,
+      username: env.REDIS_USERNAME,
+      password: env.REDIS_PASSWORD,
+      enableAutoPipelining: true,
+      ...(env.REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+    },
+    cachedLimitProvider: {
+      async getCachedLimit(organizationId, defaultValue) {
+        const result = await getCachedLimit(
+          organizationId,
+          "realtimeConcurrentConnections",
+          defaultValue
+        );
+
+        return result.val;
+      },
+    },
+  });
+}
+
+export const realtimeClient = singleton("realtimeClient", initializeRealtimeClient);

apps/webapp/app/services/routeBuiilders/apiBuilder.server.ts

@@ -0,0 +1,260 @@
+import { z } from "zod";
+import { ApiAuthenticationResult, authenticateApiRequest } from "../apiAuth.server";
+import { json, LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { fromZodError } from "zod-validation-error";
+import { apiCors } from "~/utils/apiCors";
+import {
+  AuthorizationAction,
+  AuthorizationResources,
+  checkAuthorization,
+} from "../authorization.server";
+import { logger } from "../logger.server";
+import {
+  authenticateApiRequestWithPersonalAccessToken,
+  PersonalAccessTokenAuthenticationResult,
+} from "../personalAccessToken.server";
+
+type ApiKeyRouteBuilderOptions<
+  TParamsSchema extends z.AnyZodObject | undefined = undefined,
+  TSearchParamsSchema extends z.AnyZodObject | undefined = undefined
+> = {
+  params?: TParamsSchema;
+  searchParams?: TSearchParamsSchema;
+  allowJWT?: boolean;
+  corsStrategy?: "all" | "none";
+  authorization?: {
+    action: AuthorizationAction;
+    resource: (
+      params: TParamsSchema extends z.AnyZodObject ? z.infer<TParamsSchema> : undefined,
+      searchParams: TSearchParamsSchema extends z.AnyZodObject
+        ? z.infer<TSearchParamsSchema>
+        : undefined
+    ) => AuthorizationResources;
+    superScopes?: string[];
+  };
+};
+
+type ApiKeyHandlerFunction<
+  TParamsSchema extends z.AnyZodObject | undefined,
+  TSearchParamsSchema extends z.AnyZodObject | undefined
+> = (args: {
+  params: TParamsSchema extends z.AnyZodObject ? z.infer<TParamsSchema> : undefined;
+  searchParams: TSearchParamsSchema extends z.AnyZodObject
+    ? z.infer<TSearchParamsSchema>
+    : undefined;
+  authentication: ApiAuthenticationResult;
+  request: Request;
+}) => Promise<Response>;
+
+export function createLoaderApiRoute<
+  TParamsSchema extends z.AnyZodObject | undefined = undefined,
+  TSearchParamsSchema extends z.AnyZodObject | undefined = undefined
+>(
+  options: ApiKeyRouteBuilderOptions<TParamsSchema, TSearchParamsSchema>,
+  handler: ApiKeyHandlerFunction<TParamsSchema, TSearchParamsSchema>
+) {
+  return async function loader({ request, params }: LoaderFunctionArgs) {
+    const {
+      params: paramsSchema,
+      searchParams: searchParamsSchema,
+      allowJWT = false,
+      corsStrategy = "none",
+      authorization,
+    } = options;
+
+    if (corsStrategy !== "none" && request.method.toUpperCase() === "OPTIONS") {
+      return apiCors(request, json({}));
+    }
+
+    const authenticationResult = await authenticateApiRequest(request, { allowJWT });
+
+    if (!authenticationResult) {
+      return wrapResponse(
+        request,
+        json({ error: "Invalid or Missing API key" }, { status: 401 }),
+        corsStrategy !== "none"
+      );
+    }
+
+    let parsedParams: any = undefined;
+    if (paramsSchema) {
+      const parsed = paramsSchema.safeParse(params);
+      if (!parsed.success) {
+        return wrapResponse(
+          request,
+          json(
+            { error: "Params Error", details: fromZodError(parsed.error).details },
+            { status: 400 }
+          ),
+          corsStrategy !== "none"
+        );
+      }
+      parsedParams = parsed.data;
+    }
+
+    let parsedSearchParams: any = undefined;
+    if (searchParamsSchema) {
+      const searchParams = Object.fromEntries(new URL(request.url).searchParams);
+      const parsed = searchParamsSchema.safeParse(searchParams);
+      if (!parsed.success) {
+        return wrapResponse(
+          request,
+          json(
+            { error: "Query Error", details: fromZodError(parsed.error).details },
+            { status: 400 }
+          ),
+          corsStrategy !== "none"
+        );
+      }
+      parsedSearchParams = parsed.data;
+    }
+
+    if (authorization) {
+      const { action, resource, superScopes } = authorization;
+      const $resource = resource(parsedParams, parsedSearchParams);
+
+      logger.debug("Checking authorization", {
+        action,
+        resource: $resource,
+        superScopes,
+        scopes: authenticationResult.scopes,
+      });
+
+      if (!checkAuthorization(authenticationResult, action, $resource, superScopes)) {
+        return wrapResponse(
+          request,
+          json({ error: "Unauthorized" }, { status: 403 }),
+          corsStrategy !== "none"
+        );
+      }
+    }
+
+    try {
+      const result = await handler({
+        params: parsedParams,
+        searchParams: parsedSearchParams,
+        authentication: authenticationResult,
+        request,
+      });
+      return wrapResponse(request, result, corsStrategy !== "none");
+    } catch (error) {
+      console.error("Error in API route:", error);
+      if (error instanceof Response) {
+        return wrapResponse(request, error, corsStrategy !== "none");
+      }
+      return wrapResponse(
+        request,
+        json({ error: "Internal Server Error" }, { status: 500 }),
+        corsStrategy !== "none"
+      );
+    }
+  };
+}
+
+type PATRouteBuilderOptions<
+  TParamsSchema extends z.AnyZodObject | undefined = undefined,
+  TSearchParamsSchema extends z.AnyZodObject | undefined = undefined
+> = {
+  params?: TParamsSchema;
+  searchParams?: TSearchParamsSchema;
+  corsStrategy?: "all" | "none";
+};
+
+type PATHandlerFunction<
+  TParamsSchema extends z.AnyZodObject | undefined,
+  TSearchParamsSchema extends z.AnyZodObject | undefined
+> = (args: {
+  params: TParamsSchema extends z.AnyZodObject ? z.infer<TParamsSchema> : undefined;
+  searchParams: TSearchParamsSchema extends z.AnyZodObject
+    ? z.infer<TSearchParamsSchema>
+    : undefined;
+  authentication: PersonalAccessTokenAuthenticationResult;
+  request: Request;
+}) => Promise<Response>;
+
+export function createLoaderPATApiRoute<
+  TParamsSchema extends z.AnyZodObject | undefined = undefined,
+  TSearchParamsSchema extends z.AnyZodObject | undefined = undefined
+>(
+  options: PATRouteBuilderOptions<TParamsSchema, TSearchParamsSchema>,
+  handler: PATHandlerFunction<TParamsSchema, TSearchParamsSchema>
+) {
+  return async function loader({ request, params }: LoaderFunctionArgs) {
+    const {
+      params: paramsSchema,
+      searchParams: searchParamsSchema,
+      corsStrategy = "none",
+    } = options;
+
+    if (corsStrategy !== "none" && request.method.toUpperCase() === "OPTIONS") {
+      return apiCors(request, json({}));
+    }
+
+    const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
+
+    if (!authenticationResult) {
+      return wrapResponse(
+        request,
+        json({ error: "Invalid or Missing API key" }, { status: 401 }),
+        corsStrategy !== "none"
+      );
+    }
+
+    let parsedParams: any = undefined;
+    if (paramsSchema) {
+      const parsed = paramsSchema.safeParse(params);
+      if (!parsed.success) {
+        return wrapResponse(
+          request,
+          json(
+            { error: "Params Error", details: fromZodError(parsed.error).details },
+            { status: 400 }
+          ),
+          corsStrategy !== "none"
+        );
+      }
+      parsedParams = parsed.data;
+    }
+
+    let parsedSearchParams: any = undefined;
+    if (searchParamsSchema) {
+      const searchParams = Object.fromEntries(new URL(request.url).searchParams);
+      const parsed = searchParamsSchema.safeParse(searchParams);
+      if (!parsed.success) {
+        return wrapResponse(
+          request,
+          json(
+            { error: "Query Error", details: fromZodError(parsed.error).details },
+            { status: 400 }
+          ),
+          corsStrategy !== "none"
+        );
+      }
+      parsedSearchParams = parsed.data;
+    }
+
+    try {
+      const result = await handler({
+        params: parsedParams,
+        searchParams: parsedSearchParams,
+        authentication: authenticationResult,
+        request,
+      });
+      return wrapResponse(request, result, corsStrategy !== "none");
+    } catch (error) {
+      console.error("Error in API route:", error);
+      if (error instanceof Response) {
+        return wrapResponse(request, error, corsStrategy !== "none");
+      }
+      return wrapResponse(
+        request,
+        json({ error: "Internal Server Error" }, { status: 500 }),
+        corsStrategy !== "none"
+      );
+    }
+  };
+}
+
+function wrapResponse(request: Request, response: Response, useCors: boolean) {
+  return useCors ? apiCors(request, response) : response;
+}

apps/webapp/app/services/unkey/redisCacheStore.server.ts

@@ -0,0 +1,97 @@
+import { Err, Ok, type Result } from "@unkey/error";
+import type { Entry, Store } from "@unkey/cache/stores";
+import type { RedisOptions } from "ioredis";
+import { Redis } from "ioredis";
+import { CacheError } from "@unkey/cache";
+
+export type RedisCacheStoreConfig = {
+  connection: RedisOptions;
+};
+
+export class RedisCacheStore<TNamespace extends string, TValue = any>
+  implements Store<TNamespace, TValue>
+{
+  public readonly name = "redis";
+  private readonly redis: Redis;
+
+  constructor(config: RedisCacheStoreConfig) {
+    this.redis = new Redis(config.connection);
+  }
+
+  private buildCacheKey(namespace: TNamespace, key: string): string {
+    return [namespace, key].join("::");
+  }
+
+  public async get(
+    namespace: TNamespace,
+    key: string
+  ): Promise<Result<Entry<TValue> | undefined, CacheError>> {
+    let raw: string | null;
+    try {
+      raw = await this.redis.get(this.buildCacheKey(namespace, key));
+    } catch (err) {
+      return Err(
+        new CacheError({
+          tier: this.name,
+          key,
+          message: (err as Error).message,
+        })
+      );
+    }
+
+    if (!raw) {
+      return Promise.resolve(Ok(undefined));
+    }
+
+    try {
+      const superjson = await import("superjson");
+      const entry = superjson.parse(raw) as Entry<TValue>;
+      return Ok(entry);
+    } catch (err) {
+      return Err(
+        new CacheError({
+          tier: this.name,
+          key,
+          message: (err as Error).message,
+        })
+      );
+    }
+  }
+
+  public async set(
+    namespace: TNamespace,
+    key: string,
+    entry: Entry<TValue>
+  ): Promise<Result<void, CacheError>> {
+    const cacheKey = this.buildCacheKey(namespace, key);
+    try {
+      const superjson = await import("superjson");
+      await this.redis.set(cacheKey, superjson.stringify(entry), "PXAT", entry.staleUntil);
+      return Ok();
+    } catch (err) {
+      return Err(
+        new CacheError({
+          tier: this.name,
+          key,
+          message: (err as Error).message,
+        })
+      );
+    }
+  }
+
+  public async remove(namespace: TNamespace, key: string): Promise<Result<void, CacheError>> {
+    try {
+      const cacheKey = this.buildCacheKey(namespace, key);
+      await this.redis.del(cacheKey);
+      return Promise.resolve(Ok());
+    } catch (err) {
+      return Err(
+        new CacheError({
+          tier: this.name,
+          key,
+          message: (err as Error).message,
+        })
+      );
+    }
+  }
+}

apps/webapp/app/utils/apiCors.ts

@@ -10,10 +10,25 @@ type CorsOptions = {
   credentials?: boolean;
 };
 
-export function apiCors(
+export async function apiCors(
   request: Request,
   response: Response,
   options: CorsOptions = { maxAge: 5 * 60 }
 ): Promise<Response> {
+  if (hasCorsHeaders(response)) {
+    return response;
+  }
+
   return cors(request, response, options);
 }
+
+export function makeApiCors(
+  request: Request,
+  options: CorsOptions = { maxAge: 5 * 60 }
+): (response: Response) => Promise<Response> {
+  return (response: Response) => apiCors(request, response, options);
+}
+
+function hasCorsHeaders(response: Response) {
+  return response.headers.has("access-control-allow-origin");
+}

apps/webapp/app/utils/longPollingFetch.ts

@@ -10,23 +10,16 @@ export async function longPollingFetch(url: string, options?: RequestInit) {
   try {
     let response = await fetch(url, options);
 
-    // Check if the response is ok (status in the range 200-299)
-    if (!response.ok) {
-      const body = await response.text();
-      throw new Error(`HTTP error! status: ${response.status}. ${body}`);
-    }
-
-    if (response.headers.get(`content-encoding`)) {
+    if (response.headers.get("content-encoding")) {
       const headers = new Headers(response.headers);
-      headers.delete(`content-encoding`);
-      headers.delete(`content-length`);
+      headers.delete("content-encoding");
+      headers.delete("content-length");
       response = new Response(response.body, {
+        headers,
         status: response.status,
         statusText: response.statusText,
-        headers,
       });
     }
-
     return response;
   } catch (error) {
     if (error instanceof TypeError) {

apps/webapp/app/v3/failedTaskRun.server.ts

@@ -1,15 +1,41 @@
-import { sanitizeError, TaskRunFailedExecutionResult } from "@trigger.dev/core/v3";
+import {
+  calculateNextRetryDelay,
+  RetryOptions,
+  TaskRunExecution,
+  TaskRunExecutionRetry,
+  TaskRunFailedExecutionResult,
+} from "@trigger.dev/core/v3";
 import { logger } from "~/services/logger.server";
-import { createExceptionPropertiesFromError, eventRepository } from "./eventRepository.server";
 import { BaseService } from "./services/baseService.server";
-import { FinalizeTaskRunService } from "./services/finalizeTaskRun.server";
-import { FAILABLE_RUN_STATUSES } from "./taskStatus";
+import { isFailableRunStatus, isFinalAttemptStatus } from "./taskStatus";
+import type { Prisma, TaskRun } from "@trigger.dev/database";
+import { CompleteAttemptService } from "./services/completeAttempt.server";
+import { CreateTaskRunAttemptService } from "./services/createTaskRunAttempt.server";
+import { sharedQueueTasks } from "./marqs/sharedQueueConsumer.server";
+import * as semver from "semver";
+
+const includeAttempts = {
+  attempts: {
+    orderBy: {
+      createdAt: "desc",
+    },
+    take: 1,
+  },
+  lockedBy: true, // task
+  lockedToVersion: true, // worker
+} satisfies Prisma.TaskRunInclude;
+
+type TaskRunWithAttempts = Prisma.TaskRunGetPayload<{
+  include: typeof includeAttempts;
+}>;
 
 export class FailedTaskRunService extends BaseService {
   public async call(anyRunId: string, completion: TaskRunFailedExecutionResult) {
+    logger.debug("[FailedTaskRunService] Handling failed task run", { anyRunId, completion });
+
     const isFriendlyId = anyRunId.startsWith("run_");
 
-    const taskRun = await this._prisma.taskRun.findUnique({
+    const taskRun = await this._prisma.taskRun.findFirst({
       where: {
         friendlyId: isFriendlyId ? anyRunId : undefined,
         id: !isFriendlyId ? anyRunId : undefined,
@@ -25,7 +51,7 @@ export class FailedTaskRunService extends BaseService {
       return;
     }
 
-    if (!FAILABLE_RUN_STATUSES.includes(taskRun.status)) {
+    if (!isFailableRunStatus(taskRun.status)) {
       logger.error("[FailedTaskRunService] Task run is not in a failable state", {
         taskRun,
         completion,
@@ -34,33 +60,217 @@ export class FailedTaskRunService extends BaseService {
       return;
     }
 
-    // No more retries, we need to fail the task run
-    logger.debug("[FailedTaskRunService] Failing task run", { taskRun, completion });
-
-    const finalizeService = new FinalizeTaskRunService();
-    await finalizeService.call({
-      id: taskRun.id,
-      status: "SYSTEM_FAILURE",
-      completedAt: new Date(),
-      attemptStatus: "FAILED",
-      error: sanitizeError(completion.error),
+    const retryHelper = new FailedTaskRunRetryHelper(this._prisma);
+    const retryResult = await retryHelper.call({
+      runId: taskRun.id,
+      completion,
     });
 
-    // Now we need to "complete" the task run event/span
-    await eventRepository.completeEvent(taskRun.spanId, {
-      endTime: new Date(),
-      attributes: {
-        isError: true,
-      },
-      events: [
-        {
-          name: "exception",
-          time: new Date(),
-          properties: {
-            exception: createExceptionPropertiesFromError(completion.error),
-          },
-        },
-      ],
+    logger.debug("[FailedTaskRunService] Completion result", {
+      runId: taskRun.id,
+      result: retryResult,
     });
   }
 }
+
+interface TaskRunWithWorker extends TaskRun {
+  lockedBy: { retryConfig: Prisma.JsonValue } | null;
+  lockedToVersion: { sdkVersion: string } | null;
+}
+
+export class FailedTaskRunRetryHelper extends BaseService {
+  async call({
+    runId,
+    completion,
+    isCrash,
+  }: {
+    runId: string;
+    completion: TaskRunFailedExecutionResult;
+    isCrash?: boolean;
+  }) {
+    const taskRun = await this._prisma.taskRun.findFirst({
+      where: {
+        id: runId,
+      },
+      include: includeAttempts,
+    });
+
+    if (!taskRun) {
+      logger.error("[FailedTaskRunRetryHelper] Task run not found", {
+        runId,
+        completion,
+      });
+
+      return "NO_TASK_RUN";
+    }
+
+    const retriableExecution = await this.#getRetriableAttemptExecution(taskRun, completion);
+
+    if (!retriableExecution) {
+      return "NO_EXECUTION";
+    }
+
+    logger.debug("[FailedTaskRunRetryHelper] Completing attempt", { taskRun, completion });
+
+    const completeAttempt = new CompleteAttemptService({
+      prisma: this._prisma,
+      isSystemFailure: !isCrash,
+      isCrash,
+    });
+    const completeResult = await completeAttempt.call({
+      completion,
+      execution: retriableExecution,
+    });
+
+    return completeResult;
+  }
+
+  async #getRetriableAttemptExecution(
+    run: TaskRunWithAttempts,
+    completion: TaskRunFailedExecutionResult
+  ): Promise<TaskRunExecution | undefined> {
+    let attempt = run.attempts[0];
+
+    // We need to create an attempt if:
+    // - None exists yet
+    // - The last attempt has a final status, e.g. we failed between attempts
+    if (!attempt || isFinalAttemptStatus(attempt.status)) {
+      logger.debug("[FailedTaskRunRetryHelper] No attempts found", {
+        run,
+        completion,
+      });
+
+      const createAttempt = new CreateTaskRunAttemptService(this._prisma);
+
+      try {
+        const { execution } = await createAttempt.call({
+          runId: run.id,
+          // This ensures we correctly respect `maxAttempts = 1` when failing before the first attempt was created
+          startAtZero: true,
+        });
+        return execution;
+      } catch (error) {
+        logger.error("[FailedTaskRunRetryHelper] Failed to create attempt", {
+          run,
+          completion,
+          error,
+        });
+
+        return;
+      }
+    }
+
+    // We already have an attempt with non-final status, let's use it
+    try {
+      const executionPayload = await sharedQueueTasks.getExecutionPayloadFromAttempt({
+        id: attempt.id,
+        skipStatusChecks: true,
+      });
+
+      return executionPayload?.execution;
+    } catch (error) {
+      logger.error("[FailedTaskRunRetryHelper] Failed to get execution payload", {
+        run,
+        completion,
+        error,
+      });
+
+      return;
+    }
+  }
+
+  static async getExecutionRetry({
+    run,
+    execution,
+  }: {
+    run: TaskRunWithWorker;
+    execution: TaskRunExecution;
+  }): Promise<TaskRunExecutionRetry | undefined> {
+    try {
+      const retryConfig = run.lockedBy?.retryConfig;
+
+      if (!retryConfig) {
+        if (!run.lockedToVersion) {
+          logger.error("[FailedTaskRunRetryHelper] Run not locked to version", {
+            run,
+            execution,
+          });
+
+          return;
+        }
+
+        const sdkVersion = run.lockedToVersion.sdkVersion ?? "0.0.0";
+        const isValid = semver.valid(sdkVersion);
+
+        if (!isValid) {
+          logger.error("[FailedTaskRunRetryHelper] Invalid SDK version", {
+            run,
+            execution,
+          });
+
+          return;
+        }
+
+        // With older SDK versions, tasks only have a retry config stored in the DB if it's explicitly defined on the task itself
+        // It won't get populated with retry.default in trigger.config.ts
+        if (semver.lt(sdkVersion, FailedTaskRunRetryHelper.DEFAULT_RETRY_CONFIG_SINCE_VERSION)) {
+          logger.warn(
+            "[FailedTaskRunRetryHelper] SDK version not recent enough to determine retry config",
+            {
+              run,
+              execution,
+            }
+          );
+
+          return;
+        }
+      }
+
+      const parsedRetryConfig = RetryOptions.nullable().safeParse(retryConfig);
+
+      if (!parsedRetryConfig.success) {
+        logger.error("[FailedTaskRunRetryHelper] Invalid retry config", {
+          run,
+          execution,
+        });
+
+        return;
+      }
+
+      if (!parsedRetryConfig.data) {
+        logger.debug("[FailedTaskRunRetryHelper] No retry config", {
+          run,
+          execution,
+        });
+
+        return;
+      }
+
+      const delay = calculateNextRetryDelay(parsedRetryConfig.data, execution.attempt.number);
+
+      if (!delay) {
+        logger.debug("[FailedTaskRunRetryHelper] No more retries", {
+          run,
+          execution,
+        });
+
+        return;
+      }
+
+      return {
+        timestamp: Date.now() + delay,
+        delay,
+      };
+    } catch (error) {
+      logger.error("[FailedTaskRunRetryHelper] Failed to get execution retry", {
+        run,
+        execution,
+        error,
+      });
+
+      return;
+    }
+  }
+
+  static DEFAULT_RETRY_CONFIG_SINCE_VERSION = "3.1.0";
+}

apps/webapp/app/v3/handleSocketIo.server.ts

@@ -24,6 +24,7 @@ import { Redis } from "ioredis";
 import { createAdapter } from "@socket.io/redis-adapter";
 import { CrashTaskRunService } from "./services/crashTaskRun.server";
 import { CreateTaskRunAttemptService } from "./services/createTaskRunAttempt.server";
+import { UpdateFatalRunErrorService } from "./services/updateFatalRunError.server";
 
 export const socketIo = singleton("socketIo", initalizeIoServer);
 
@@ -123,12 +124,13 @@ function createCoordinatorNamespace(io: Server) {
         await resumeAttempt.call(message);
       },
       TASK_RUN_COMPLETED: async (message) => {
-        const completeAttempt = new CompleteAttemptService();
+        const completeAttempt = new CompleteAttemptService({
+          supportsRetryCheckpoints: message.version === "v1",
+        });
         await completeAttempt.call({
           completion: message.completion,
           execution: message.execution,
           checkpoint: message.checkpoint,
-          supportsRetryCheckpoints: message.version === "v1",
         });
       },
       TASK_RUN_FAILED_TO_RUN: async (message) => {
@@ -193,7 +195,11 @@ function createCoordinatorNamespace(io: Server) {
           }
 
           const service = new CreateTaskRunAttemptService();
-          const { attempt } = await service.call(message.runId, environment, false);
+          const { attempt } = await service.call({
+            runId: message.runId,
+            authenticatedEnv: environment,
+            setToExecuting: false,
+          });
 
           const payload = await sharedQueueTasks.getExecutionPayloadFromAttempt({
             id: attempt.id,
@@ -297,11 +303,13 @@ function createProviderNamespace(io: Server) {
     handlers: {
       WORKER_CRASHED: async (message) => {
         try {
-          const service = new CrashTaskRunService();
-
-          await service.call(message.runId, {
-            ...message,
-          });
+          if (message.overrideCompletion) {
+            const updateErrorService = new UpdateFatalRunErrorService();
+            await updateErrorService.call(message.runId, { ...message });
+          } else {
+            const crashRunService = new CrashTaskRunService();
+            await crashRunService.call(message.runId, { ...message });
+          }
         } catch (error) {
           logger.error("Error while handling crashed worker", { error });
         }

apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts

@@ -509,7 +509,10 @@ export class SharedQueueConsumer {
           if (!deployment.worker.supportsLazyAttempts) {
             try {
               const service = new CreateTaskRunAttemptService();
-              await service.call(lockedTaskRun.friendlyId, undefined, false);
+              await service.call({
+                runId: lockedTaskRun.id,
+                setToExecuting: false,
+              });
             } catch (error) {
               logger.error("Failed to create task run attempt for outdate worker", {
                 error,

apps/webapp/app/v3/requeueTaskRun.server.ts

@@ -7,6 +7,7 @@ import { BaseService } from "./services/baseService.server";
 import { PrismaClientOrTransaction } from "~/db.server";
 import { workerQueue } from "~/services/worker.server";
 import { socketIo } from "./handleSocketIo.server";
+import { TaskRunErrorCodes } from "@trigger.dev/core/v3";
 
 export class RequeueTaskRunService extends BaseService {
   public async call(runId: string) {
@@ -59,7 +60,7 @@ export class RequeueTaskRunService extends BaseService {
           retry: undefined,
           error: {
             type: "INTERNAL_ERROR",
-            code: "TASK_RUN_HEARTBEAT_TIMEOUT",
+            code: TaskRunErrorCodes.TASK_RUN_HEARTBEAT_TIMEOUT,
             message: "Did not receive a heartbeat from the worker in time",
           },
         });

apps/webapp/app/v3/services/completeAttempt.server.ts

@@ -1,6 +1,7 @@
 import { Attributes } from "@opentelemetry/api";
 import {
   TaskRunContext,
+  TaskRunErrorCodes,
   TaskRunExecution,
   TaskRunExecutionResult,
   TaskRunExecutionRetry,
@@ -8,6 +9,8 @@ import {
   TaskRunSuccessfulExecutionResult,
   flattenAttributes,
   sanitizeError,
+  shouldRetryError,
+  taskRunErrorEnhancer,
 } from "@trigger.dev/core/v3";
 import { $transaction, PrismaClientOrTransaction } from "~/db.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
@@ -21,9 +24,10 @@ import { MAX_TASK_RUN_ATTEMPTS } from "~/consts";
 import { CreateCheckpointService } from "./createCheckpoint.server";
 import { TaskRun } from "@trigger.dev/database";
 import { RetryAttemptService } from "./retryAttempt.server";
-import { isFinalAttemptStatus, isFinalRunStatus } from "../taskStatus";
+import { FAILED_RUN_STATUSES, isFinalAttemptStatus, isFinalRunStatus } from "../taskStatus";
 import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
 import { env } from "~/env.server";
+import { FailedTaskRunRetryHelper } from "../failedTaskRun.server";
 
 type FoundAttempt = Awaited<ReturnType<typeof findAttempt>>;
 
@@ -32,19 +36,28 @@ type CheckpointData = {
   location: string;
 };
 
+type CompleteAttemptServiceOptions = {
+  prisma?: PrismaClientOrTransaction;
+  supportsRetryCheckpoints?: boolean;
+  isSystemFailure?: boolean;
+  isCrash?: boolean;
+};
+
 export class CompleteAttemptService extends BaseService {
+  constructor(private opts: CompleteAttemptServiceOptions = {}) {
+    super(opts.prisma);
+  }
+
   public async call({
     completion,
     execution,
     env,
     checkpoint,
-    supportsRetryCheckpoints,
   }: {
     completion: TaskRunExecutionResult;
     execution: TaskRunExecution;
     env?: AuthenticatedEnvironment;
     checkpoint?: CheckpointData;
-    supportsRetryCheckpoints?: boolean;
   }): Promise<"COMPLETED" | "RETRIED"> {
     const taskRunAttempt = await findAttempt(this._prisma, execution.attempt.id);
 
@@ -78,7 +91,7 @@ export class CompleteAttemptService extends BaseService {
         attemptStatus: "FAILED",
         error: {
           type: "INTERNAL_ERROR",
-          code: "TASK_EXECUTION_FAILED",
+          code: TaskRunErrorCodes.TASK_EXECUTION_FAILED,
           message: "Tried to complete attempt but it doesn't exist",
         },
       });
@@ -109,7 +122,6 @@ export class CompleteAttemptService extends BaseService {
         taskRunAttempt,
         env,
         checkpoint,
-        supportsRetryCheckpoints,
       });
     }
   }
@@ -169,14 +181,12 @@ export class CompleteAttemptService extends BaseService {
     taskRunAttempt,
     env,
     checkpoint,
-    supportsRetryCheckpoints,
   }: {
     completion: TaskRunFailedExecutionResult;
     execution: TaskRunExecution;
     taskRunAttempt: NonNullable<FoundAttempt>;
     env?: AuthenticatedEnvironment;
     checkpoint?: CheckpointData;
-    supportsRetryCheckpoints?: boolean;
   }): Promise<"COMPLETED" | "RETRIED"> {
     if (
       completion.error.type === "INTERNAL_ERROR" &&
@@ -194,18 +204,17 @@ export class CompleteAttemptService extends BaseService {
         env
       );
 
-      // The cancel service handles ACK
-
       return "COMPLETED";
     }
 
+    const failedAt = new Date();
     const sanitizedError = sanitizeError(completion.error);
 
     await this._prisma.taskRunAttempt.update({
       where: { id: taskRunAttempt.id },
       data: {
         status: "FAILED",
-        completedAt: new Date(),
+        completedAt: failedAt,
         error: sanitizedError,
         usageDurationMs: completion.usage?.durationMs,
       },
@@ -213,226 +222,230 @@ export class CompleteAttemptService extends BaseService {
 
     const environment = env ?? (await this.#getEnvironment(execution.environment.id));
 
-    if (completion.retry !== undefined && taskRunAttempt.number < MAX_TASK_RUN_ATTEMPTS) {
-      const retryAt = new Date(completion.retry.timestamp);
+    // This means that tasks won't know they are being retried
+    let executionRetryInferred = false;
+    let executionRetry = completion.retry;
 
-      // Retry the task run
-      await eventRepository.recordEvent(`Retry #${execution.attempt.number} delay`, {
-        taskSlug: taskRunAttempt.taskRun.taskIdentifier,
+    const shouldInfer = this.opts.isCrash || this.opts.isSystemFailure;
+
+    if (!executionRetry && shouldInfer) {
+      executionRetryInferred = true;
+      executionRetry = await FailedTaskRunRetryHelper.getExecutionRetry({
+        run: {
+          ...taskRunAttempt.taskRun,
+          lockedBy: taskRunAttempt.backgroundWorkerTask,
+          lockedToVersion: taskRunAttempt.backgroundWorker,
+        },
+        execution,
+      });
+    }
+
+    const retriableError = shouldRetryError(taskRunErrorEnhancer(completion.error));
+
+    if (
+      retriableError &&
+      executionRetry !== undefined &&
+      taskRunAttempt.number < MAX_TASK_RUN_ATTEMPTS
+    ) {
+      return await this.#retryAttempt({
+        execution,
+        executionRetry,
+        executionRetryInferred,
+        taskRunAttempt,
         environment,
-        attributes: {
-          metadata: this.#generateMetadataAttributesForNextAttempt(execution),
+        checkpoint,
+      });
+    }
+
+    // The attempt has failed and we won't retry
+
+    // Now we need to "complete" the task run event/span
+    await eventRepository.completeEvent(taskRunAttempt.taskRun.spanId, {
+      endTime: failedAt,
+      attributes: {
+        isError: true,
+      },
+      events: [
+        {
+          name: "exception",
+          time: failedAt,
           properties: {
-            retryAt: retryAt.toISOString(),
+            exception: createExceptionPropertiesFromError(sanitizedError),
           },
-          runId: taskRunAttempt.taskRun.friendlyId,
-          style: {
-            icon: "schedule-attempt",
-          },
-          queueId: taskRunAttempt.queueId,
-          queueName: taskRunAttempt.taskRun.queue,
         },
-        context: taskRunAttempt.taskRun.traceContext as Record<string, string | undefined>,
-        spanIdSeed: `retry-${taskRunAttempt.number + 1}`,
-        endTime: retryAt,
-      });
+      ],
+    });
 
-      logger.debug("Retrying", {
-        taskRun: taskRunAttempt.taskRun.friendlyId,
-        retry: completion.retry,
-      });
+    await this._prisma.taskRun.update({
+      where: {
+        id: taskRunAttempt.taskRunId,
+      },
+      data: {
+        error: sanitizedError,
+      },
+    });
 
-      await this._prisma.taskRun.update({
-        where: {
-          id: taskRunAttempt.taskRunId,
-        },
-        data: {
-          status: "RETRYING_AFTER_FAILURE",
-        },
-      });
+    let status: FAILED_RUN_STATUSES;
 
-      if (environment.type === "DEVELOPMENT") {
-        // This is already an EXECUTE message so we can just NACK
-        await marqs?.nackMessage(taskRunAttempt.taskRunId, completion.retry.timestamp);
-        return "RETRIED";
-      }
-
-      if (!checkpoint) {
-        await this.#retryAttempt({
-          run: taskRunAttempt.taskRun,
-          retry: completion.retry,
-          supportsLazyAttempts: taskRunAttempt.backgroundWorker.supportsLazyAttempts,
-          supportsRetryCheckpoints,
-        });
-
-        return "RETRIED";
-      }
-
-      const createCheckpoint = new CreateCheckpointService(this._prisma);
-      const checkpointCreateResult = await createCheckpoint.call({
-        attemptFriendlyId: execution.attempt.id,
-        docker: checkpoint.docker,
-        location: checkpoint.location,
-        reason: {
-          type: "RETRYING_AFTER_FAILURE",
-          attemptNumber: execution.attempt.number,
-        },
-      });
-
-      if (!checkpointCreateResult.success) {
-        logger.error("Failed to create checkpoint", { checkpoint, execution: execution.run.id });
-
-        const finalizeService = new FinalizeTaskRunService();
-        await finalizeService.call({
-          id: taskRunAttempt.taskRunId,
-          status: "SYSTEM_FAILURE",
-          completedAt: new Date(),
-        });
-
-        return "COMPLETED";
-      }
-
-      await this.#retryAttempt({
-        run: taskRunAttempt.taskRun,
-        retry: completion.retry,
-        checkpointEventId: checkpointCreateResult.event.id,
-        supportsLazyAttempts: taskRunAttempt.backgroundWorker.supportsLazyAttempts,
-        supportsRetryCheckpoints,
-      });
-
-      return "RETRIED";
+    // Set the correct task run status
+    if (this.opts.isSystemFailure) {
+      status = "SYSTEM_FAILURE";
+    } else if (this.opts.isCrash) {
+      status = "CRASHED";
+    } else if (
+      sanitizedError.type === "INTERNAL_ERROR" &&
+      sanitizedError.code === "MAX_DURATION_EXCEEDED"
+    ) {
+      status = "TIMED_OUT";
+    } else if (sanitizedError.type === "INTERNAL_ERROR") {
+      status = "CRASHED";
     } else {
-      // Now we need to "complete" the task run event/span
-      await eventRepository.completeEvent(taskRunAttempt.taskRun.spanId, {
-        endTime: new Date(),
-        attributes: {
-          isError: true,
-        },
-        events: [
-          {
-            name: "exception",
-            time: new Date(),
-            properties: {
-              exception: createExceptionPropertiesFromError(sanitizedError),
-            },
-          },
-        ],
-      });
+      status = "COMPLETED_WITH_ERRORS";
+    }
 
-      if (
-        sanitizedError.type === "INTERNAL_ERROR" &&
-        sanitizedError.code === "GRACEFUL_EXIT_TIMEOUT"
-      ) {
-        const finalizeService = new FinalizeTaskRunService();
-        await finalizeService.call({
-          id: taskRunAttempt.taskRunId,
-          status: "SYSTEM_FAILURE",
-          completedAt: new Date(),
-        });
+    const finalizeService = new FinalizeTaskRunService();
+    await finalizeService.call({
+      id: taskRunAttempt.taskRunId,
+      status,
+      completedAt: failedAt,
+    });
 
-        // We need to fail all incomplete spans
-        const inProgressEvents = await eventRepository.queryIncompleteEvents({
-          attemptId: execution.attempt.id,
-        });
+    if (status !== "CRASHED" && status !== "SYSTEM_FAILURE") {
+      return "COMPLETED";
+    }
 
-        logger.debug("Failing in-progress events", {
+    const inProgressEvents = await eventRepository.queryIncompleteEvents({
+      runId: taskRunAttempt.taskRun.friendlyId,
+    });
+
+    // Handle in-progress events
+    switch (status) {
+      case "CRASHED": {
+        logger.debug("[CompleteAttemptService] Crashing in-progress events", {
           inProgressEvents: inProgressEvents.map((event) => event.id),
         });
 
-        const exception = {
-          type: "Graceful exit timeout",
-          message: sanitizedError.message,
-        };
-
         await Promise.all(
           inProgressEvents.map((event) => {
             return eventRepository.crashEvent({
-              event: event,
-              crashedAt: new Date(),
-              exception,
+              event,
+              crashedAt: failedAt,
+              exception: createExceptionPropertiesFromError(sanitizedError),
             });
           })
         );
-      } else {
-        await this._prisma.taskRun.update({
-          where: {
-            id: taskRunAttempt.taskRunId,
-          },
-          data: {
-            error: sanitizedError,
-          },
-        });
 
-        const status =
-          sanitizedError.type === "INTERNAL_ERROR" &&
-          sanitizedError.code === "MAX_DURATION_EXCEEDED"
-            ? "TIMED_OUT"
-            : "COMPLETED_WITH_ERRORS";
-
-        const finalizeService = new FinalizeTaskRunService();
-        await finalizeService.call({
-          id: taskRunAttempt.taskRunId,
-          status,
-          completedAt: new Date(),
-        });
+        break;
       }
+      case "SYSTEM_FAILURE": {
+        logger.debug("[CompleteAttemptService] Failing in-progress events", {
+          inProgressEvents: inProgressEvents.map((event) => event.id),
+        });
 
-      return "COMPLETED";
+        await Promise.all(
+          inProgressEvents.map((event) => {
+            return eventRepository.completeEvent(event.spanId, {
+              endTime: failedAt,
+              attributes: {
+                isError: true,
+              },
+              events: [
+                {
+                  name: "exception",
+                  time: failedAt,
+                  properties: {
+                    exception: createExceptionPropertiesFromError(sanitizedError),
+                  },
+                },
+              ],
+            });
+          })
+        );
+      }
     }
+
+    return "COMPLETED";
   }
 
-  async #retryAttempt({
+  async #enqueueReattempt({
     run,
-    retry,
+    executionRetry,
+    executionRetryInferred,
     checkpointEventId,
     supportsLazyAttempts,
-    supportsRetryCheckpoints,
   }: {
     run: TaskRun;
-    retry: TaskRunExecutionRetry;
+    executionRetry: TaskRunExecutionRetry;
+    executionRetryInferred: boolean;
     checkpointEventId?: string;
     supportsLazyAttempts: boolean;
-    supportsRetryCheckpoints?: boolean;
   }) {
     const retryViaQueue = () => {
+      logger.debug("[CompleteAttemptService] Enqueuing retry attempt", { runId: run.id });
+
       // We have to replace a potential RESUME with EXECUTE to correctly retry the attempt
       return marqs?.replaceMessage(
         run.id,
         {
           type: "EXECUTE",
           taskIdentifier: run.taskIdentifier,
-          checkpointEventId: supportsRetryCheckpoints ? checkpointEventId : undefined,
-          retryCheckpointsDisabled: !supportsRetryCheckpoints,
+          checkpointEventId: this.opts.supportsRetryCheckpoints ? checkpointEventId : undefined,
+          retryCheckpointsDisabled: !this.opts.supportsRetryCheckpoints,
         },
-        retry.timestamp
+        executionRetry.timestamp
       );
     };
 
     const retryDirectly = () => {
-      return RetryAttemptService.enqueue(run.id, this._prisma, new Date(retry.timestamp));
+      logger.debug("[CompleteAttemptService] Retrying attempt directly", { runId: run.id });
+      return RetryAttemptService.enqueue(run.id, this._prisma, new Date(executionRetry.timestamp));
     };
 
     // There's a checkpoint, so we need to go through the queue
     if (checkpointEventId) {
-      if (!supportsRetryCheckpoints) {
-        logger.error("Worker does not support retry checkpoints, but a checkpoint was created", {
-          runId: run.id,
-          checkpointEventId,
-        });
+      if (!this.opts.supportsRetryCheckpoints) {
+        logger.error(
+          "[CompleteAttemptService] Worker does not support retry checkpoints, but a checkpoint was created",
+          {
+            runId: run.id,
+            checkpointEventId,
+          }
+        );
       }
 
+      logger.debug("[CompleteAttemptService] Enqueuing retry attempt with checkpoint", {
+        runId: run.id,
+      });
       await retryViaQueue();
       return;
     }
 
     // Workers without lazy attempt support always need to go through the queue, which is where the attempt is created
     if (!supportsLazyAttempts) {
+      logger.debug("[CompleteAttemptService] Worker does not support lazy attempts", {
+        runId: run.id,
+      });
       await retryViaQueue();
       return;
     }
 
     // Workers that never checkpoint between attempts will exit after completing their current attempt if the retry delay exceeds the threshold
-    if (!supportsRetryCheckpoints && retry.delay >= env.CHECKPOINT_THRESHOLD_IN_MS) {
+    if (
+      !this.opts.supportsRetryCheckpoints &&
+      executionRetry.delay >= env.CHECKPOINT_THRESHOLD_IN_MS
+    ) {
+      logger.debug(
+        "[CompleteAttemptService] Worker does not support retry checkpoints and the delay exceeds the threshold",
+        { runId: run.id }
+      );
+      await retryViaQueue();
+      return;
+    }
+
+    if (executionRetryInferred) {
+      logger.debug("[CompleteAttemptService] Execution retry inferred, forcing retry via queue", {
+        runId: run.id,
+      });
       await retryViaQueue();
       return;
     }
@@ -441,6 +454,141 @@ export class CompleteAttemptService extends BaseService {
     await retryDirectly();
   }
 
+  async #retryAttempt({
+    execution,
+    executionRetry,
+    executionRetryInferred,
+    taskRunAttempt,
+    environment,
+    checkpoint,
+  }: {
+    execution: TaskRunExecution;
+    executionRetry: TaskRunExecutionRetry;
+    executionRetryInferred: boolean;
+    taskRunAttempt: NonNullable<FoundAttempt>;
+    environment: AuthenticatedEnvironment;
+    checkpoint?: CheckpointData;
+  }) {
+    const retryAt = new Date(executionRetry.timestamp);
+
+    // Retry the task run
+    await eventRepository.recordEvent(`Retry #${execution.attempt.number} delay`, {
+      taskSlug: taskRunAttempt.taskRun.taskIdentifier,
+      environment,
+      attributes: {
+        metadata: this.#generateMetadataAttributesForNextAttempt(execution),
+        properties: {
+          retryAt: retryAt.toISOString(),
+        },
+        runId: taskRunAttempt.taskRun.friendlyId,
+        style: {
+          icon: "schedule-attempt",
+        },
+        queueId: taskRunAttempt.queueId,
+        queueName: taskRunAttempt.taskRun.queue,
+      },
+      context: taskRunAttempt.taskRun.traceContext as Record<string, string | undefined>,
+      spanIdSeed: `retry-${taskRunAttempt.number + 1}`,
+      endTime: retryAt,
+    });
+
+    logger.debug("[CompleteAttemptService] Retrying", {
+      taskRun: taskRunAttempt.taskRun.friendlyId,
+      retry: executionRetry,
+    });
+
+    await this._prisma.taskRun.update({
+      where: {
+        id: taskRunAttempt.taskRunId,
+      },
+      data: {
+        status: "RETRYING_AFTER_FAILURE",
+      },
+    });
+
+    if (environment.type === "DEVELOPMENT") {
+      // This is already an EXECUTE message so we can just NACK
+      await marqs?.nackMessage(taskRunAttempt.taskRunId, executionRetry.timestamp);
+      return "RETRIED";
+    }
+
+    if (checkpoint) {
+      // This is only here for backwards compat - we don't checkpoint between attempts anymore
+      return await this.#retryAttemptWithCheckpoint({
+        execution,
+        taskRunAttempt,
+        executionRetry,
+        executionRetryInferred,
+        checkpoint,
+      });
+    }
+
+    await this.#enqueueReattempt({
+      run: taskRunAttempt.taskRun,
+      executionRetry,
+      supportsLazyAttempts: taskRunAttempt.backgroundWorker.supportsLazyAttempts,
+      executionRetryInferred,
+    });
+
+    return "RETRIED";
+  }
+
+  async #retryAttemptWithCheckpoint({
+    execution,
+    taskRunAttempt,
+    executionRetry,
+    executionRetryInferred,
+    checkpoint,
+  }: {
+    execution: TaskRunExecution;
+    taskRunAttempt: NonNullable<FoundAttempt>;
+    executionRetry: TaskRunExecutionRetry;
+    executionRetryInferred: boolean;
+    checkpoint: CheckpointData;
+  }) {
+    const createCheckpoint = new CreateCheckpointService(this._prisma);
+    const checkpointCreateResult = await createCheckpoint.call({
+      attemptFriendlyId: execution.attempt.id,
+      docker: checkpoint.docker,
+      location: checkpoint.location,
+      reason: {
+        type: "RETRYING_AFTER_FAILURE",
+        attemptNumber: execution.attempt.number,
+      },
+    });
+
+    if (!checkpointCreateResult.success) {
+      logger.error("[CompleteAttemptService] Failed to create reattempt checkpoint", {
+        checkpoint,
+        runId: execution.run.id,
+        attemptId: execution.attempt.id,
+      });
+
+      const finalizeService = new FinalizeTaskRunService();
+      await finalizeService.call({
+        id: taskRunAttempt.taskRunId,
+        status: "SYSTEM_FAILURE",
+        completedAt: new Date(),
+        error: {
+          type: "STRING_ERROR",
+          raw: "Failed to create reattempt checkpoint",
+        },
+      });
+
+      return "COMPLETED" as const;
+    }
+
+    await this.#enqueueReattempt({
+      run: taskRunAttempt.taskRun,
+      executionRetry,
+      checkpointEventId: checkpointCreateResult.event.id,
+      supportsLazyAttempts: taskRunAttempt.backgroundWorker.supportsLazyAttempts,
+      executionRetryInferred,
+    });
+
+    return "RETRIED" as const;
+  }
+
   #generateMetadataAttributesForNextAttempt(execution: TaskRunExecution) {
     const context = TaskRunContext.parse(execution);
 
@@ -475,6 +623,7 @@ async function findAttempt(prismaClient: PrismaClientOrTransaction, friendlyId:
         select: {
           id: true,
           supportsLazyAttempts: true,
+          sdkVersion: true,
         },
       },
     },

apps/webapp/app/v3/services/crashTaskRun.server.ts

@@ -1,12 +1,12 @@
 import { TaskRun, TaskRunAttempt } from "@trigger.dev/database";
 import { eventRepository } from "../eventRepository.server";
-import { marqs } from "~/v3/marqs/index.server";
 import { BaseService } from "./baseService.server";
 import { logger } from "~/services/logger.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { CRASHABLE_ATTEMPT_STATUSES, isCrashableRunStatus } from "../taskStatus";
-import { sanitizeError, TaskRunInternalError } from "@trigger.dev/core/v3";
+import { sanitizeError, TaskRunErrorCodes, TaskRunInternalError } from "@trigger.dev/core/v3";
 import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
+import { FailedTaskRunRetryHelper } from "../failedTaskRun.server";
 
 export type CrashTaskRunServiceOptions = {
   reason?: string;
@@ -29,6 +29,11 @@ export class CrashTaskRunService extends BaseService {
 
     logger.debug("CrashTaskRunService.call", { runId, opts });
 
+    if (options?.overrideCompletion) {
+      logger.error("CrashTaskRunService.call: overrideCompletion is deprecated", { runId });
+      return;
+    }
+
     const taskRun = await this._prisma.taskRun.findFirst({
       where: {
         id: runId,
@@ -36,16 +41,50 @@ export class CrashTaskRunService extends BaseService {
     });
 
     if (!taskRun) {
-      logger.error("Task run not found", { runId });
+      logger.error("[CrashTaskRunService] Task run not found", { runId });
       return;
     }
 
     // Make sure the task run is in a crashable state
     if (!opts.overrideCompletion && !isCrashableRunStatus(taskRun.status)) {
-      logger.error("Task run is not in a crashable state", { runId, status: taskRun.status });
+      logger.error("[CrashTaskRunService] Task run is not in a crashable state", {
+        runId,
+        status: taskRun.status,
+      });
       return;
     }
 
+    logger.debug("[CrashTaskRunService] Completing attempt", { runId, options });
+
+    const retryHelper = new FailedTaskRunRetryHelper(this._prisma);
+    const retryResult = await retryHelper.call({
+      runId,
+      completion: {
+        ok: false,
+        id: runId,
+        error: {
+          type: "INTERNAL_ERROR",
+          code: opts.errorCode ?? TaskRunErrorCodes.TASK_RUN_CRASHED,
+          message: opts.reason,
+          stackTrace: opts.logs,
+        },
+      },
+      isCrash: true,
+    });
+
+    logger.debug("[CrashTaskRunService] Completion result", { runId, retryResult });
+
+    if (retryResult === "RETRIED") {
+      logger.debug("[CrashTaskRunService] Retried task run", { runId });
+      return;
+    }
+
+    if (!opts.overrideCompletion) {
+      return;
+    }
+
+    logger.debug("[CrashTaskRunService] Overriding completion", { runId, options });
+
     const finalizeService = new FinalizeTaskRunService();
     const crashedTaskRun = await finalizeService.call({
       id: taskRun.id,
@@ -74,7 +113,7 @@ export class CrashTaskRunService extends BaseService {
       attemptStatus: "FAILED",
       error: {
         type: "INTERNAL_ERROR",
-        code: opts.errorCode ?? "TASK_RUN_CRASHED",
+        code: opts.errorCode ?? TaskRunErrorCodes.TASK_RUN_CRASHED,
         message: opts.reason,
         stackTrace: opts.logs,
       },
@@ -87,7 +126,7 @@ export class CrashTaskRunService extends BaseService {
       options?.overrideCompletion
     );
 
-    logger.debug("Crashing in-progress events", {
+    logger.debug("[CrashTaskRunService] Crashing in-progress events", {
       inProgressEvents: inProgressEvents.map((event) => event.id),
     });
 
@@ -97,7 +136,7 @@ export class CrashTaskRunService extends BaseService {
           event: event,
           crashedAt: opts.crashedAt,
           exception: {
-            type: opts.errorCode ?? "TASK_RUN_CRASHED",
+            type: opts.errorCode ?? TaskRunErrorCodes.TASK_RUN_CRASHED,
             message: opts.reason,
             stacktrace: opts.logs,
           },
@@ -136,27 +175,29 @@ export class CrashTaskRunService extends BaseService {
       code?: TaskRunInternalError["code"];
     }
   ) {
-    return await this.traceWithEnv("failAttempt()", environment, async (span) => {
-      span.setAttribute("taskRunId", run.id);
-      span.setAttribute("attemptId", attempt.id);
+    return await this.traceWithEnv(
+      "[CrashTaskRunService] failAttempt()",
+      environment,
+      async (span) => {
+        span.setAttribute("taskRunId", run.id);
+        span.setAttribute("attemptId", attempt.id);
 
-      await marqs?.acknowledgeMessage(run.id);
-
-      await this._prisma.taskRunAttempt.update({
-        where: {
-          id: attempt.id,
-        },
-        data: {
-          status: "FAILED",
-          completedAt: failedAt,
-          error: sanitizeError({
-            type: "INTERNAL_ERROR",
-            code: error.code ?? "TASK_RUN_CRASHED",
-            message: error.reason,
-            stackTrace: error.logs,
-          }),
-        },
-      });
-    });
+        await this._prisma.taskRunAttempt.update({
+          where: {
+            id: attempt.id,
+          },
+          data: {
+            status: "FAILED",
+            completedAt: failedAt,
+            error: sanitizeError({
+              type: "INTERNAL_ERROR",
+              code: error.code ?? TaskRunErrorCodes.TASK_RUN_CRASHED,
+              message: error.reason,
+              stackTrace: error.logs,
+            }),
+          },
+        });
+      }
+    );
   }
 }

apps/webapp/app/v3/services/createTaskRunAttempt.server.ts

@@ -12,11 +12,17 @@ import { CrashTaskRunService } from "./crashTaskRun.server";
 import { ExpireEnqueuedRunService } from "./expireEnqueuedRun.server";
 
 export class CreateTaskRunAttemptService extends BaseService {
-  public async call(
-    runId: string,
-    authenticatedEnv?: AuthenticatedEnvironment,
-    setToExecuting = true
-  ): Promise<{
+  public async call({
+    runId,
+    authenticatedEnv,
+    setToExecuting = true,
+    startAtZero = false,
+  }: {
+    runId: string;
+    authenticatedEnv?: AuthenticatedEnvironment;
+    setToExecuting?: boolean;
+    startAtZero?: boolean;
+  }): Promise<{
     execution: TaskRunExecution;
     run: TaskRun;
     attempt: TaskRunAttempt;
@@ -102,7 +108,11 @@ export class CreateTaskRunAttemptService extends BaseService {
         throw new ServiceValidationError("Queue not found", 404);
       }
 
-      const nextAttemptNumber = taskRun.attempts[0] ? taskRun.attempts[0].number + 1 : 1;
+      const nextAttemptNumber = taskRun.attempts[0]
+        ? taskRun.attempts[0].number + 1
+        : startAtZero
+        ? 0
+        : 1;
 
       if (nextAttemptNumber > MAX_TASK_RUN_ATTEMPTS) {
         const service = new CrashTaskRunService(this._prisma);

apps/webapp/app/v3/services/finalizeTaskRun.server.ts

@@ -3,11 +3,17 @@ import { type Prisma, type TaskRun } from "@trigger.dev/database";
 import { logger } from "~/services/logger.server";
 import { marqs, sanitizeQueueName } from "~/v3/marqs/index.server";
 import { generateFriendlyId } from "../friendlyIdentifiers";
-import { FINAL_ATTEMPT_STATUSES, isFailedRunStatus, type FINAL_RUN_STATUSES } from "../taskStatus";
+import {
+  FINAL_ATTEMPT_STATUSES,
+  isFailedRunStatus,
+  isFatalRunStatus,
+  type FINAL_RUN_STATUSES,
+} from "../taskStatus";
 import { PerformTaskRunAlertsService } from "./alerts/performTaskRunAlerts.server";
 import { BaseService } from "./baseService.server";
 import { ResumeDependentParentsService } from "./resumeDependentParents.server";
 import { ExpireEnqueuedRunService } from "./expireEnqueuedRun.server";
+import { socketIo } from "../handleSocketIo.server";
 
 type BaseInput = {
   id: string;
@@ -90,6 +96,42 @@ export class FinalizeTaskRunService extends BaseService {
       await PerformTaskRunAlertsService.enqueue(run.id, this._prisma);
     }
 
+    if (isFatalRunStatus(run.status)) {
+      logger.error("FinalizeTaskRunService: Fatal status", { runId: run.id, status: run.status });
+
+      const extendedRun = await this._prisma.taskRun.findFirst({
+        where: { id: run.id },
+        select: {
+          id: true,
+          lockedToVersion: {
+            select: {
+              supportsLazyAttempts: true,
+            },
+          },
+          runtimeEnvironment: {
+            select: {
+              type: true,
+            },
+          },
+        },
+      });
+
+      if (extendedRun && extendedRun.runtimeEnvironment.type !== "DEVELOPMENT") {
+        logger.error("FinalizeTaskRunService: Fatal status, requesting worker exit", {
+          runId: run.id,
+          status: run.status,
+        });
+
+        // Signal to exit any leftover containers
+        socketIo.coordinatorNamespace.emit("REQUEST_RUN_CANCELLATION", {
+          version: "v1",
+          runId: run.id,
+          // Give the run a few seconds to exit to complete any flushing etc
+          delayInMs: extendedRun.lockedToVersion?.supportsLazyAttempts ? 5_000 : undefined,
+        });
+      }
+    }
+
     return run as Output<T>;
   }
 
@@ -111,83 +153,90 @@ export class FinalizeTaskRunService extends BaseService {
     error?: TaskRunError;
     run: TaskRun;
   }) {
-    if (attemptStatus || error) {
-      const latestAttempt = await this._prisma.taskRunAttempt.findFirst({
-        where: { taskRunId: run.id },
-        orderBy: { id: "desc" },
-        take: 1,
+    if (!attemptStatus && !error) {
+      logger.error("FinalizeTaskRunService: No attemptStatus or error provided", { runId: run.id });
+      return;
+    }
+
+    const latestAttempt = await this._prisma.taskRunAttempt.findFirst({
+      where: { taskRunId: run.id },
+      orderBy: { id: "desc" },
+      take: 1,
+    });
+
+    if (latestAttempt) {
+      logger.debug("Finalizing run attempt", {
+        id: latestAttempt.id,
+        status: attemptStatus,
+        error,
       });
 
-      if (latestAttempt) {
-        logger.debug("Finalizing run attempt", {
-          id: latestAttempt.id,
-          status: attemptStatus,
-          error,
-        });
+      await this._prisma.taskRunAttempt.update({
+        where: { id: latestAttempt.id },
+        data: { status: attemptStatus, error: error ? sanitizeError(error) : undefined },
+      });
 
-        await this._prisma.taskRunAttempt.update({
-          where: { id: latestAttempt.id },
-          data: { status: attemptStatus, error: error ? sanitizeError(error) : undefined },
-        });
-      } else {
-        logger.debug("Finalizing run no attempt found", {
-          runId: run.id,
-          attemptStatus,
-          error,
-        });
-
-        if (!run.lockedById) {
-          logger.error(
-            "FinalizeTaskRunService: No lockedById, so can't get the BackgroundWorkerTask. Not creating an attempt.",
-            { runId: run.id }
-          );
-          return;
-        }
-
-        const workerTask = await this._prisma.backgroundWorkerTask.findFirst({
-          select: {
-            id: true,
-            workerId: true,
-            runtimeEnvironmentId: true,
-          },
-          where: {
-            id: run.lockedById,
-          },
-        });
-
-        if (!workerTask) {
-          logger.error("FinalizeTaskRunService: No worker task found", { runId: run.id });
-          return;
-        }
-
-        const queue = await this._prisma.taskQueue.findUnique({
-          where: {
-            runtimeEnvironmentId_name: {
-              runtimeEnvironmentId: workerTask.runtimeEnvironmentId,
-              name: sanitizeQueueName(run.queue),
-            },
-          },
-        });
-
-        if (!queue) {
-          logger.error("FinalizeTaskRunService: No queue found", { runId: run.id });
-          return;
-        }
-
-        await this._prisma.taskRunAttempt.create({
-          data: {
-            number: 1,
-            friendlyId: generateFriendlyId("attempt"),
-            taskRunId: run.id,
-            backgroundWorkerId: workerTask?.workerId,
-            backgroundWorkerTaskId: workerTask?.id,
-            queueId: queue.id,
-            runtimeEnvironmentId: workerTask.runtimeEnvironmentId,
-            status: attemptStatus,
-            error: error ? sanitizeError(error) : undefined,
-          },
-        });
-      }
+      return;
     }
+
+    // There's no attempt, so create one
+
+    logger.debug("Finalizing run no attempt found", {
+      runId: run.id,
+      attemptStatus,
+      error,
+    });
+
+    if (!run.lockedById) {
+      logger.error(
+        "FinalizeTaskRunService: No lockedById, so can't get the BackgroundWorkerTask. Not creating an attempt.",
+        { runId: run.id }
+      );
+      return;
+    }
+
+    const workerTask = await this._prisma.backgroundWorkerTask.findFirst({
+      select: {
+        id: true,
+        workerId: true,
+        runtimeEnvironmentId: true,
+      },
+      where: {
+        id: run.lockedById,
+      },
+    });
+
+    if (!workerTask) {
+      logger.error("FinalizeTaskRunService: No worker task found", { runId: run.id });
+      return;
+    }
+
+    const queue = await this._prisma.taskQueue.findUnique({
+      where: {
+        runtimeEnvironmentId_name: {
+          runtimeEnvironmentId: workerTask.runtimeEnvironmentId,
+          name: sanitizeQueueName(run.queue),
+        },
+      },
+    });
+
+    if (!queue) {
+      logger.error("FinalizeTaskRunService: No queue found", { runId: run.id });
+      return;
+    }
+
+    await this._prisma.taskRunAttempt.create({
+      data: {
+        number: 1,
+        friendlyId: generateFriendlyId("attempt"),
+        taskRunId: run.id,
+        backgroundWorkerId: workerTask?.workerId,
+        backgroundWorkerTaskId: workerTask?.id,
+        queueId: queue.id,
+        runtimeEnvironmentId: workerTask.runtimeEnvironmentId,
+        status: attemptStatus,
+        error: error ? sanitizeError(error) : undefined,
+      },
+    });
   }
 }

apps/webapp/app/v3/services/triggerTask.server.ts

@@ -378,6 +378,7 @@ export class TriggerTaskService extends BaseService {
                   maxDurationInSeconds: body.options?.maxDuration
                     ? clampMaxDuration(body.options.maxDuration)
                     : undefined,
+                  runTags: bodyTags,
                 },
               });
 

apps/webapp/app/v3/services/updateFatalRunError.server.ts

@@ -0,0 +1,57 @@
+import { BaseService } from "./baseService.server";
+import { logger } from "~/services/logger.server";
+import { isFatalRunStatus } from "../taskStatus";
+import { TaskRunErrorCodes, TaskRunInternalError } from "@trigger.dev/core/v3";
+import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
+
+export type UpdateFatalRunErrorServiceOptions = {
+  reason?: string;
+  exitCode?: number;
+  logs?: string;
+  errorCode?: TaskRunInternalError["code"];
+};
+
+export class UpdateFatalRunErrorService extends BaseService {
+  public async call(runId: string, options?: UpdateFatalRunErrorServiceOptions) {
+    const opts = {
+      reason: "Worker crashed",
+      ...options,
+    };
+
+    logger.debug("UpdateFatalRunErrorService.call", { runId, opts });
+
+    const taskRun = await this._prisma.taskRun.findFirst({
+      where: {
+        id: runId,
+      },
+    });
+
+    if (!taskRun) {
+      logger.error("[UpdateFatalRunErrorService] Task run not found", { runId });
+      return;
+    }
+
+    if (!isFatalRunStatus(taskRun.status)) {
+      logger.warn("[UpdateFatalRunErrorService] Task run is not in a fatal state", {
+        runId,
+        status: taskRun.status,
+      });
+
+      return;
+    }
+
+    logger.debug("[UpdateFatalRunErrorService] Updating crash error", { runId, options });
+
+    const finalizeService = new FinalizeTaskRunService();
+    await finalizeService.call({
+      id: taskRun.id,
+      status: "CRASHED",
+      error: {
+        type: "INTERNAL_ERROR",
+        code: opts.errorCode ?? TaskRunErrorCodes.TASK_RUN_CRASHED,
+        message: opts.reason,
+        stackTrace: opts.logs,
+      },
+    });
+  }
+}

apps/webapp/app/v3/services/upsertTaskSchedule.server.ts

@@ -1,7 +1,7 @@
 import { Prisma, TaskSchedule } from "@trigger.dev/database";
 import cronstrue from "cronstrue";
 import { nanoid } from "nanoid";
-import { $transaction, PrismaClientOrTransaction } from "~/db.server";
+import { $transaction } from "~/db.server";
 import { generateFriendlyId } from "../friendlyIdentifiers";
 import { UpsertSchedule } from "../schedules";
 import { calculateNextScheduledTimestamp } from "../utils/calculateNextSchedule.server";
@@ -31,124 +31,45 @@ export class UpsertTaskScheduleService extends BaseService {
     const checkSchedule = new CheckScheduleService(this._prisma);
     await checkSchedule.call(projectId, schedule);
 
-    const result = await $transaction(this._prisma, async (tx) => {
-      const deduplicationKey =
-        typeof schedule.deduplicationKey === "string" && schedule.deduplicationKey !== ""
-          ? schedule.deduplicationKey
-          : nanoid(24);
+    const deduplicationKey =
+      typeof schedule.deduplicationKey === "string" && schedule.deduplicationKey !== ""
+        ? schedule.deduplicationKey
+        : nanoid(24);
 
-      const existingSchedule = schedule.friendlyId
-        ? await tx.taskSchedule.findUnique({
-            where: {
-              friendlyId: schedule.friendlyId,
+    const existingSchedule = schedule.friendlyId
+      ? await this._prisma.taskSchedule.findUnique({
+          where: {
+            friendlyId: schedule.friendlyId,
+          },
+        })
+      : await this._prisma.taskSchedule.findUnique({
+          where: {
+            projectId_deduplicationKey: {
+              projectId,
+              deduplicationKey,
             },
-          })
-        : await tx.taskSchedule.findUnique({
-            where: {
-              projectId_deduplicationKey: {
-                projectId,
-                deduplicationKey,
-              },
-            },
-          });
+          },
+        });
 
+    const result = await (async (tx) => {
       if (existingSchedule) {
         if (existingSchedule.type === "DECLARATIVE") {
           throw new ServiceValidationError("Cannot update a declarative schedule");
         }
 
-        return await this.#updateExistingSchedule(tx, existingSchedule, schedule, projectId);
+        return await this.#updateExistingSchedule(existingSchedule, schedule);
       } else {
-        return await this.#createNewSchedule(tx, schedule, projectId, deduplicationKey);
+        return await this.#createNewSchedule(schedule, projectId, deduplicationKey);
       }
-    });
+    })();
 
     if (!result) {
-      throw new Error("Failed to create or update the schedule");
+      throw new ServiceValidationError("Failed to create or update schedule");
     }
 
-    const { scheduleRecord, instances } = result;
+    const { scheduleRecord } = result;
 
-    return this.#createReturnObject(scheduleRecord, instances);
-  }
-
-  async #createNewSchedule(
-    tx: PrismaClientOrTransaction,
-    options: UpsertTaskScheduleServiceOptions,
-    projectId: string,
-    deduplicationKey: string
-  ) {
-    const scheduleRecord = await tx.taskSchedule.create({
-      data: {
-        projectId,
-        friendlyId: generateFriendlyId("sched"),
-        taskIdentifier: options.taskIdentifier,
-        deduplicationKey,
-        userProvidedDeduplicationKey:
-          options.deduplicationKey !== undefined && options.deduplicationKey !== "",
-        generatorExpression: options.cron,
-        generatorDescription: cronstrue.toString(options.cron),
-        timezone: options.timezone ?? "UTC",
-        externalId: options.externalId ? options.externalId : undefined,
-      },
-    });
-
-    const registerNextService = new RegisterNextTaskScheduleInstanceService(tx);
-
-    //create the instances (links to environments)
-    let instances: InstanceWithEnvironment[] = [];
-    for (const environmentId of options.environments) {
-      const instance = await tx.taskScheduleInstance.create({
-        data: {
-          taskScheduleId: scheduleRecord.id,
-          environmentId,
-        },
-        include: {
-          environment: {
-            include: {
-              orgMember: {
-                include: {
-                  user: true,
-                },
-              },
-            },
-          },
-        },
-      });
-
-      await registerNextService.call(instance.id);
-
-      instances.push(instance);
-    }
-
-    return { scheduleRecord, instances };
-  }
-
-  async #updateExistingSchedule(
-    tx: PrismaClientOrTransaction,
-    existingSchedule: TaskSchedule,
-    options: UpsertTaskScheduleServiceOptions,
-    projectId: string
-  ) {
-    //update the schedule
-    const scheduleRecord = await tx.taskSchedule.update({
-      where: {
-        id: existingSchedule.id,
-      },
-      data: {
-        generatorExpression: options.cron,
-        generatorDescription: cronstrue.toString(options.cron),
-        timezone: options.timezone ?? "UTC",
-        externalId: options.externalId ? options.externalId : null,
-      },
-    });
-
-    const scheduleHasChanged =
-      scheduleRecord.generatorExpression !== existingSchedule.generatorExpression ||
-      scheduleRecord.timezone !== existingSchedule.timezone;
-
-    // find the existing instances
-    const existingInstances = await tx.taskScheduleInstance.findMany({
+    const instances = await this._prisma.taskScheduleInstance.findMany({
       where: {
         taskScheduleId: scheduleRecord.id,
       },
@@ -165,18 +86,35 @@ export class UpsertTaskScheduleService extends BaseService {
       },
     });
 
-    // create the new instances
-    const newInstances: InstanceWithEnvironment[] = [];
-    const updatingInstances: InstanceWithEnvironment[] = [];
+    return this.#createReturnObject(scheduleRecord, instances);
+  }
 
-    for (const environmentId of options.environments) {
-      const existingInstance = existingInstances.find((i) => i.environmentId === environmentId);
+  async #createNewSchedule(
+    options: UpsertTaskScheduleServiceOptions,
+    projectId: string,
+    deduplicationKey: string
+  ) {
+    return await $transaction(this._prisma, async (tx) => {
+      const scheduleRecord = await tx.taskSchedule.create({
+        data: {
+          projectId,
+          friendlyId: generateFriendlyId("sched"),
+          taskIdentifier: options.taskIdentifier,
+          deduplicationKey,
+          userProvidedDeduplicationKey:
+            options.deduplicationKey !== undefined && options.deduplicationKey !== "",
+          generatorExpression: options.cron,
+          generatorDescription: cronstrue.toString(options.cron),
+          timezone: options.timezone ?? "UTC",
+          externalId: options.externalId ? options.externalId : undefined,
+        },
+      });
 
-      if (existingInstance) {
-        // Update the existing instance
-        updatingInstances.push(existingInstance);
-      } else {
-        // Create a new instance
+      const registerNextService = new RegisterNextTaskScheduleInstanceService(tx);
+
+      //create the instances (links to environments)
+
+      for (const environmentId of options.environments) {
         const instance = await tx.taskScheduleInstance.create({
           data: {
             taskScheduleId: scheduleRecord.id,
@@ -195,39 +133,21 @@ export class UpsertTaskScheduleService extends BaseService {
           },
         });
 
-        newInstances.push(instance);
+        await registerNextService.call(instance.id);
       }
-    }
 
-    // find the instances that need to be removed
-    const instancesToDeleted = existingInstances.filter(
-      (i) => !options.environments.includes(i.environmentId)
-    );
+      return { scheduleRecord };
+    });
+  }
 
-    // delete the instances no longer selected
-    for (const instance of instancesToDeleted) {
-      await tx.taskScheduleInstance.delete({
-        where: {
-          id: instance.id,
-        },
-      });
-    }
-
-    const registerService = new RegisterNextTaskScheduleInstanceService(tx);
-
-    for (const instance of newInstances) {
-      await registerService.call(instance.id);
-    }
-
-    if (scheduleHasChanged) {
-      for (const instance of updatingInstances) {
-        await registerService.call(instance.id);
-      }
-    }
-
-    const instances = await tx.taskScheduleInstance.findMany({
+  async #updateExistingSchedule(
+    existingSchedule: TaskSchedule,
+    options: UpsertTaskScheduleServiceOptions
+  ) {
+    // find the existing instances
+    const existingInstances = await this._prisma.taskScheduleInstance.findMany({
       where: {
-        taskScheduleId: scheduleRecord.id,
+        taskScheduleId: existingSchedule.id,
       },
       include: {
         environment: {
@@ -242,7 +162,89 @@ export class UpsertTaskScheduleService extends BaseService {
       },
     });
 
-    return { scheduleRecord, instances };
+    return await $transaction(
+      this._prisma,
+      async (tx) => {
+        const scheduleRecord = await tx.taskSchedule.update({
+          where: {
+            id: existingSchedule.id,
+          },
+          data: {
+            generatorExpression: options.cron,
+            generatorDescription: cronstrue.toString(options.cron),
+            timezone: options.timezone ?? "UTC",
+            externalId: options.externalId ? options.externalId : null,
+          },
+        });
+
+        const scheduleHasChanged =
+          scheduleRecord.generatorExpression !== existingSchedule.generatorExpression ||
+          scheduleRecord.timezone !== existingSchedule.timezone;
+
+        // create the new instances
+        const newInstances: InstanceWithEnvironment[] = [];
+        const updatingInstances: InstanceWithEnvironment[] = [];
+
+        for (const environmentId of options.environments) {
+          const existingInstance = existingInstances.find((i) => i.environmentId === environmentId);
+
+          if (existingInstance) {
+            // Update the existing instance
+            updatingInstances.push(existingInstance);
+          } else {
+            // Create a new instance
+            const instance = await tx.taskScheduleInstance.create({
+              data: {
+                taskScheduleId: scheduleRecord.id,
+                environmentId,
+              },
+              include: {
+                environment: {
+                  include: {
+                    orgMember: {
+                      include: {
+                        user: true,
+                      },
+                    },
+                  },
+                },
+              },
+            });
+
+            newInstances.push(instance);
+          }
+        }
+
+        // find the instances that need to be removed
+        const instancesToDeleted = existingInstances.filter(
+          (i) => !options.environments.includes(i.environmentId)
+        );
+
+        // delete the instances no longer selected
+        for (const instance of instancesToDeleted) {
+          await tx.taskScheduleInstance.delete({
+            where: {
+              id: instance.id,
+            },
+          });
+        }
+
+        const registerService = new RegisterNextTaskScheduleInstanceService(tx);
+
+        for (const instance of newInstances) {
+          await registerService.call(instance.id);
+        }
+
+        if (scheduleHasChanged) {
+          for (const instance of updatingInstances) {
+            await registerService.call(instance.id);
+          }
+        }
+
+        return { scheduleRecord };
+      },
+      { timeout: 10_000 }
+    );
   }
 
   #createReturnObject(taskSchedule: TaskSchedule, instances: InstanceWithEnvironment[]) {

apps/webapp/app/v3/taskStatus.ts

@@ -1,96 +1,45 @@
 import type { TaskRunAttemptStatus, TaskRunStatus } from "@trigger.dev/database";
 
-export const CANCELLABLE_RUN_STATUSES: TaskRunStatus[] = [
-  "DELAYED",
-  "PENDING",
-  "WAITING_FOR_DEPLOY",
-  "EXECUTING",
-  "PAUSED",
-  "WAITING_TO_RESUME",
-  "PAUSED",
-  "RETRYING_AFTER_FAILURE",
-];
-export const CANCELLABLE_ATTEMPT_STATUSES: TaskRunAttemptStatus[] = [
-  "EXECUTING",
-  "PAUSED",
-  "PENDING",
-];
-
-export function isCancellableRunStatus(status: TaskRunStatus): boolean {
-  return CANCELLABLE_RUN_STATUSES.includes(status);
-}
-export function isCancellableAttemptStatus(status: TaskRunAttemptStatus): boolean {
-  return CANCELLABLE_ATTEMPT_STATUSES.includes(status);
-}
-
-export const CRASHABLE_RUN_STATUSES: TaskRunStatus[] = CANCELLABLE_RUN_STATUSES;
-export const CRASHABLE_ATTEMPT_STATUSES: TaskRunAttemptStatus[] = CANCELLABLE_ATTEMPT_STATUSES;
-
-export function isCrashableRunStatus(status: TaskRunStatus): boolean {
-  return CRASHABLE_RUN_STATUSES.includes(status);
-}
-export function isCrashableAttemptStatus(status: TaskRunAttemptStatus): boolean {
-  return CRASHABLE_ATTEMPT_STATUSES.includes(status);
-}
-
 export const FINAL_RUN_STATUSES = [
   "CANCELED",
+  "INTERRUPTED",
   "COMPLETED_SUCCESSFULLY",
   "COMPLETED_WITH_ERRORS",
-  "INTERRUPTED",
   "SYSTEM_FAILURE",
-  "EXPIRED",
   "CRASHED",
+  "EXPIRED",
   "TIMED_OUT",
 ] satisfies TaskRunStatus[];
 
 export type FINAL_RUN_STATUSES = (typeof FINAL_RUN_STATUSES)[number];
 
+export const NON_FINAL_RUN_STATUSES = [
+  "DELAYED",
+  "PENDING",
+  "WAITING_FOR_DEPLOY",
+  "EXECUTING",
+  "WAITING_TO_RESUME",
+  "RETRYING_AFTER_FAILURE",
+  "PAUSED",
+] satisfies TaskRunStatus[];
+
+export type NON_FINAL_RUN_STATUSES = (typeof NON_FINAL_RUN_STATUSES)[number];
+
 export const FINAL_ATTEMPT_STATUSES = [
+  "FAILED",
   "CANCELED",
   "COMPLETED",
-  "FAILED",
 ] satisfies TaskRunAttemptStatus[];
 
 export type FINAL_ATTEMPT_STATUSES = (typeof FINAL_ATTEMPT_STATUSES)[number];
 
-export const FAILED_ATTEMPT_STATUSES = ["FAILED", "CANCELED"] satisfies TaskRunAttemptStatus[];
-
-export type FAILED_ATTEMPT_STATUSES = (typeof FAILED_ATTEMPT_STATUSES)[number];
-
-export const FREEZABLE_RUN_STATUSES: TaskRunStatus[] = ["EXECUTING", "RETRYING_AFTER_FAILURE"];
-export const FREEZABLE_ATTEMPT_STATUSES: TaskRunAttemptStatus[] = ["EXECUTING", "FAILED"];
-
-export function isFreezableRunStatus(status: TaskRunStatus): boolean {
-  return FREEZABLE_RUN_STATUSES.includes(status);
-}
-export function isFreezableAttemptStatus(status: TaskRunAttemptStatus): boolean {
-  return FREEZABLE_ATTEMPT_STATUSES.includes(status);
-}
-
-export function isFinalRunStatus(status: TaskRunStatus): boolean {
-  return FINAL_RUN_STATUSES.includes(status);
-}
-export function isFinalAttemptStatus(status: TaskRunAttemptStatus): boolean {
-  return FINAL_ATTEMPT_STATUSES.includes(status);
-}
-
-export const RESTORABLE_RUN_STATUSES: TaskRunStatus[] = ["WAITING_TO_RESUME"];
-export const RESTORABLE_ATTEMPT_STATUSES: TaskRunAttemptStatus[] = ["PAUSED"];
-
-export function isRestorableRunStatus(status: TaskRunStatus): boolean {
-  return RESTORABLE_RUN_STATUSES.includes(status);
-}
-export function isRestorableAttemptStatus(status: TaskRunAttemptStatus): boolean {
-  return RESTORABLE_ATTEMPT_STATUSES.includes(status);
-}
-
-export const FAILABLE_RUN_STATUSES = [
-  "EXECUTING",
+export const NON_FINAL_ATTEMPT_STATUSES = [
   "PENDING",
-  "WAITING_FOR_DEPLOY",
-  "RETRYING_AFTER_FAILURE",
-] satisfies TaskRunStatus[];
+  "EXECUTING",
+  "PAUSED",
+] satisfies TaskRunAttemptStatus[];
+
+export type NON_FINAL_ATTEMPT_STATUSES = (typeof NON_FINAL_ATTEMPT_STATUSES)[number];
 
 export const FAILED_RUN_STATUSES = [
   "INTERRUPTED",
@@ -100,6 +49,69 @@ export const FAILED_RUN_STATUSES = [
   "TIMED_OUT",
 ] satisfies TaskRunStatus[];
 
+export type FAILED_RUN_STATUSES = (typeof FAILED_RUN_STATUSES)[number];
+
+export const FATAL_RUN_STATUSES = ["SYSTEM_FAILURE", "CRASHED"] satisfies TaskRunStatus[];
+
+export type FATAL_RUN_STATUSES = (typeof FAILED_RUN_STATUSES)[number];
+
+export const CANCELLABLE_RUN_STATUSES = NON_FINAL_RUN_STATUSES;
+export const CANCELLABLE_ATTEMPT_STATUSES = NON_FINAL_ATTEMPT_STATUSES;
+
+export const CRASHABLE_RUN_STATUSES = NON_FINAL_RUN_STATUSES;
+export const CRASHABLE_ATTEMPT_STATUSES = NON_FINAL_ATTEMPT_STATUSES;
+
+export const FAILABLE_RUN_STATUSES = NON_FINAL_RUN_STATUSES;
+
+export const FREEZABLE_RUN_STATUSES: TaskRunStatus[] = ["EXECUTING", "RETRYING_AFTER_FAILURE"];
+export const FREEZABLE_ATTEMPT_STATUSES: TaskRunAttemptStatus[] = ["EXECUTING", "FAILED"];
+
+export const RESTORABLE_RUN_STATUSES: TaskRunStatus[] = ["WAITING_TO_RESUME"];
+export const RESTORABLE_ATTEMPT_STATUSES: TaskRunAttemptStatus[] = ["PAUSED"];
+
+export function isFinalRunStatus(status: TaskRunStatus): boolean {
+  return FINAL_RUN_STATUSES.includes(status);
+}
+export function isFinalAttemptStatus(status: TaskRunAttemptStatus): boolean {
+  return FINAL_ATTEMPT_STATUSES.includes(status);
+}
+
 export function isFailedRunStatus(status: TaskRunStatus): boolean {
   return FAILED_RUN_STATUSES.includes(status);
 }
+
+export function isFatalRunStatus(status: TaskRunStatus): boolean {
+  return FATAL_RUN_STATUSES.includes(status);
+}
+
+export function isCancellableRunStatus(status: TaskRunStatus): boolean {
+  return CANCELLABLE_RUN_STATUSES.includes(status);
+}
+export function isCancellableAttemptStatus(status: TaskRunAttemptStatus): boolean {
+  return CANCELLABLE_ATTEMPT_STATUSES.includes(status);
+}
+
+export function isCrashableRunStatus(status: TaskRunStatus): boolean {
+  return CRASHABLE_RUN_STATUSES.includes(status);
+}
+export function isCrashableAttemptStatus(status: TaskRunAttemptStatus): boolean {
+  return CRASHABLE_ATTEMPT_STATUSES.includes(status);
+}
+
+export function isFailableRunStatus(status: TaskRunStatus): boolean {
+  return FAILABLE_RUN_STATUSES.includes(status);
+}
+
+export function isFreezableRunStatus(status: TaskRunStatus): boolean {
+  return FREEZABLE_RUN_STATUSES.includes(status);
+}
+export function isFreezableAttemptStatus(status: TaskRunAttemptStatus): boolean {
+  return FREEZABLE_ATTEMPT_STATUSES.includes(status);
+}
+
+export function isRestorableRunStatus(status: TaskRunStatus): boolean {
+  return RESTORABLE_RUN_STATUSES.includes(status);
+}
+export function isRestorableAttemptStatus(status: TaskRunAttemptStatus): boolean {
+  return RESTORABLE_ATTEMPT_STATUSES.includes(status);
+}

apps/webapp/package.json

@@ -23,7 +23,7 @@
     "clean:sourcemaps": "run-s clean:sourcemaps:*",
     "clean:sourcemaps:public": "rimraf ./build/**/*.map",
     "clean:sourcemaps:build": "rimraf ./public/build/**/*.map",
-    "test": "vitest"
+    "test": "vitest --no-file-parallelism"
   },
   "eslintIgnore": [
     "/node_modules",
@@ -97,11 +97,13 @@
     "@trigger.dev/core": "workspace:*",
     "@trigger.dev/database": "workspace:*",
     "@trigger.dev/otlp-importer": "workspace:*",
-    "@trigger.dev/platform": "1.0.12",
+    "@trigger.dev/platform": "1.0.13",
     "@trigger.dev/sdk": "workspace:*",
     "@trigger.dev/yalt": "npm:@trigger.dev/yalt",
     "@types/pg": "8.6.6",
     "@uiw/react-codemirror": "^4.19.5",
+    "@unkey/cache": "^1.5.0",
+    "@unkey/error": "^0.2.0",
     "@upstash/ratelimit": "^1.1.3",
     "@whatwg-node/fetch": "^0.9.14",
     "assert-never": "^1.2.1",
@@ -162,6 +164,7 @@
     "remix-typedjson": "0.3.1",
     "remix-utils": "^7.1.0",
     "seedrandom": "^3.0.5",
+    "semver": "^7.5.0",
     "simple-oauth2": "^5.0.0",
     "simplur": "^3.0.1",
     "slug": "^6.0.0",
@@ -183,6 +186,7 @@
     "zod-validation-error": "^1.5.0"
   },
   "devDependencies": {
+    "@internal/testcontainers": "workspace:*",
     "@remix-run/dev": "2.1.0",
     "@remix-run/eslint-config": "2.1.0",
     "@remix-run/testing": "^2.1.0",
@@ -210,8 +214,10 @@
     "@types/react-dom": "18.2.7",
     "@types/regression": "^2.0.6",
     "@types/seedrandom": "^3.0.8",
+    "@types/semver": "^7.5.0",
     "@types/simple-oauth2": "^5.0.4",
     "@types/slug": "^5.0.3",
+    "@types/supertest": "^6.0.2",
     "@types/tar": "^6.1.4",
     "@types/ws": "^8.5.3",
     "@typescript-eslint/eslint-plugin": "^5.59.6",
@@ -236,14 +242,16 @@
     "prop-types": "^15.8.1",
     "rimraf": "^3.0.2",
     "style-loader": "^3.3.4",
+    "supertest": "^7.0.0",
     "tailwind-scrollbar": "^3.0.1",
     "tailwindcss": "3.4.1",
     "ts-node": "^10.7.0",
     "tsconfig-paths": "^3.14.1",
     "typescript": "^5.1.6",
+    "vite-tsconfig-paths": "^4.0.5",
     "vitest": "^1.4.0"
   },
   "engines": {
     "node": ">=16.0.0"
   }
-}
+}

apps/webapp/remix.config.js

@@ -15,6 +15,8 @@ module.exports = {
     "@trigger.dev/sdk",
     "@trigger.dev/platform",
     "@trigger.dev/yalt",
+    "@unkey/cache",
+    "@unkey/cache/stores",
     "emails",
     "highlight.run",
     "random-words",

apps/webapp/server.ts

@@ -1,16 +1,16 @@
-import path from "path";
-import express from "express";
-import compression from "compression";
-import morgan from "morgan";
 import { createRequestHandler } from "@remix-run/express";
-import { WebSocketServer } from "ws";
 import { broadcastDevReady, logDevReady } from "@remix-run/server-runtime";
-import type { Server as IoServer } from "socket.io";
+import compression from "compression";
 import type { Server as EngineServer } from "engine.io";
-import { RegistryProxy } from "~/v3/registryProxy.server";
-import { RateLimitMiddleware, apiRateLimiter } from "~/services/apiRateLimit.server";
-import { type RunWithHttpContextFunction } from "~/services/httpAsyncStorage.server";
+import express from "express";
+import morgan from "morgan";
 import { nanoid } from "nanoid";
+import path from "path";
+import type { Server as IoServer } from "socket.io";
+import { WebSocketServer } from "ws";
+import { RateLimitMiddleware } from "~/services/apiRateLimit.server";
+import { type RunWithHttpContextFunction } from "~/services/httpAsyncStorage.server";
+import { RegistryProxy } from "~/v3/registryProxy.server";
 
 const app = express();
 

apps/webapp/test/authorization.test.ts

@@ -0,0 +1,219 @@
+import { describe, it, expect } from "vitest";
+import { checkAuthorization, AuthorizationEntity } from "../app/services/authorization.server";
+
+describe("checkAuthorization", () => {
+  // Test entities
+  const privateEntity: AuthorizationEntity = { type: "PRIVATE" };
+  const publicEntity: AuthorizationEntity = { type: "PUBLIC" };
+  const publicJwtEntityWithPermissions: AuthorizationEntity = {
+    type: "PUBLIC_JWT",
+    scopes: ["read:runs:run_1234", "read:tasks", "read:tags:tag_5678"],
+  };
+  const publicJwtEntityNoPermissions: AuthorizationEntity = { type: "PUBLIC_JWT" };
+
+  describe("PRIVATE entity", () => {
+    it("should always return true regardless of action or resource", () => {
+      expect(checkAuthorization(privateEntity, "read", { runs: "run_1234" })).toBe(true);
+      expect(checkAuthorization(privateEntity, "read", { tasks: ["task_1", "task_2"] })).toBe(true);
+      expect(checkAuthorization(privateEntity, "read", { tags: "nonexistent_tag" })).toBe(true);
+    });
+  });
+
+  describe("PUBLIC entity", () => {
+    it("should always return false regardless of action or resource", () => {
+      expect(checkAuthorization(publicEntity, "read", { runs: "run_1234" })).toBe(false);
+      expect(checkAuthorization(publicEntity, "read", { tasks: ["task_1", "task_2"] })).toBe(false);
+      expect(checkAuthorization(publicEntity, "read", { tags: "tag_5678" })).toBe(false);
+    });
+  });
+
+  describe("PUBLIC_JWT entity with scope", () => {
+    it("should return true for specific resource scope", () => {
+      expect(checkAuthorization(publicJwtEntityWithPermissions, "read", { runs: "run_1234" })).toBe(
+        true
+      );
+    });
+
+    it("should return false for unauthorized specific resources", () => {
+      expect(checkAuthorization(publicJwtEntityWithPermissions, "read", { runs: "run_5678" })).toBe(
+        false
+      );
+    });
+
+    it("should return true for general resource type scope", () => {
+      expect(
+        checkAuthorization(publicJwtEntityWithPermissions, "read", { tasks: "task_1234" })
+      ).toBe(true);
+      expect(
+        checkAuthorization(publicJwtEntityWithPermissions, "read", {
+          tasks: ["task_5678", "task_9012"],
+        })
+      ).toBe(true);
+    });
+
+    it("should return true if any resource in an array is authorized", () => {
+      expect(
+        checkAuthorization(publicJwtEntityWithPermissions, "read", {
+          tags: ["tag_1234", "tag_5678"],
+        })
+      ).toBe(true);
+    });
+
+    it("should return true for nonexistent resource types", () => {
+      expect(
+        // @ts-expect-error
+        checkAuthorization(publicJwtEntityWithPermissions, "read", { nonexistent: "resource" })
+      ).toBe(true);
+    });
+  });
+
+  describe("PUBLIC_JWT entity without scope", () => {
+    it("should always return false regardless of action or resource", () => {
+      expect(checkAuthorization(publicJwtEntityNoPermissions, "read", { runs: "run_1234" })).toBe(
+        false
+      );
+      expect(
+        checkAuthorization(publicJwtEntityNoPermissions, "read", { tasks: ["task_1", "task_2"] })
+      ).toBe(false);
+      expect(checkAuthorization(publicJwtEntityNoPermissions, "read", { tags: "tag_5678" })).toBe(
+        false
+      );
+    });
+  });
+
+  describe("Edge cases", () => {
+    it("should handle empty resource objects", () => {
+      expect(checkAuthorization(publicJwtEntityWithPermissions, "read", {})).toBe(false);
+    });
+
+    it("should handle undefined scope", () => {
+      const entityUndefinedPermissions: AuthorizationEntity = { type: "PUBLIC_JWT" };
+      expect(checkAuthorization(entityUndefinedPermissions, "read", { runs: "run_1234" })).toBe(
+        false
+      );
+    });
+
+    it("should handle empty scope array", () => {
+      const entityEmptyPermissions: AuthorizationEntity = { type: "PUBLIC_JWT", scopes: [] };
+      expect(checkAuthorization(entityEmptyPermissions, "read", { runs: "run_1234" })).toBe(false);
+    });
+
+    it("should return false if any resource is not authorized", () => {
+      expect(
+        checkAuthorization(publicJwtEntityWithPermissions, "read", {
+          runs: "run_1234", // This is authorized
+          tasks: "task_5678", // This is authorized (general permission)
+          tags: "tag_3456", // This is not authorized
+        })
+      ).toBe(false);
+    });
+
+    it("should return true only if all resources are authorized", () => {
+      expect(
+        checkAuthorization(publicJwtEntityWithPermissions, "read", {
+          runs: "run_1234", // This is authorized
+          tasks: "task_5678", // This is authorized (general permission)
+          tags: "tag_5678", // This is authorized
+        })
+      ).toBe(true);
+    });
+  });
+
+  describe("Super scope", () => {
+    const entityWithSuperPermissions: AuthorizationEntity = {
+      type: "PUBLIC_JWT",
+      scopes: ["read:all", "admin"],
+    };
+
+    const entityWithOneSuperPermission: AuthorizationEntity = {
+      type: "PUBLIC_JWT",
+      scopes: ["read:all"],
+    };
+
+    it("should grant access with any of the super scope", () => {
+      expect(
+        checkAuthorization(entityWithSuperPermissions, "read", { tasks: "task_1234" }, [
+          "read:all",
+          "admin",
+        ])
+      ).toBe(true);
+      expect(
+        checkAuthorization(entityWithSuperPermissions, "read", { tags: ["tag_1", "tag_2"] }, [
+          "write:all",
+          "admin",
+        ])
+      ).toBe(true);
+    });
+
+    it("should grant access with one matching super permission", () => {
+      expect(
+        checkAuthorization(entityWithOneSuperPermission, "read", { runs: "run_5678" }, [
+          "read:all",
+          "admin",
+        ])
+      ).toBe(true);
+    });
+
+    it("should not grant access when no super scope match", () => {
+      expect(
+        checkAuthorization(entityWithOneSuperPermission, "read", { tasks: "task_1234" }, [
+          "write:all",
+          "admin",
+        ])
+      ).toBe(false);
+    });
+
+    it("should grant access to multiple resources with super scope", () => {
+      expect(
+        checkAuthorization(
+          entityWithSuperPermissions,
+          "read",
+          {
+            tasks: "task_1234",
+            tags: ["tag_1", "tag_2"],
+            runs: "run_5678",
+          },
+          ["read:all"]
+        )
+      ).toBe(true);
+    });
+
+    it("should fall back to specific scope when super scope are not provided", () => {
+      const entityWithSpecificPermissions: AuthorizationEntity = {
+        type: "PUBLIC_JWT",
+        scopes: ["read:tasks", "read:tags"],
+      };
+      expect(
+        checkAuthorization(entityWithSpecificPermissions, "read", { tasks: "task_1234" })
+      ).toBe(true);
+      expect(checkAuthorization(entityWithSpecificPermissions, "read", { runs: "run_5678" })).toBe(
+        false
+      );
+    });
+  });
+
+  describe("Without super scope", () => {
+    const entityWithoutSuperPermissions: AuthorizationEntity = {
+      type: "PUBLIC_JWT",
+      scopes: ["read:tasks"],
+    };
+
+    it("should still grant access based on specific scope", () => {
+      expect(
+        checkAuthorization(entityWithoutSuperPermissions, "read", { tasks: "task_1234" }, [
+          "read:all",
+          "admin",
+        ])
+      ).toBe(true);
+    });
+
+    it("should deny access to resources not in scope", () => {
+      expect(
+        checkAuthorization(entityWithoutSuperPermissions, "read", { runs: "run_5678" }, [
+          "read:all",
+          "admin",
+        ])
+      ).toBe(false);
+    });
+  });
+});

apps/webapp/test/authorizationRateLimitMiddleware.test.ts

@@ -0,0 +1,416 @@
+import { redisTest } from "@internal/testcontainers";
+import { describe, expect, vi, beforeEach } from "vitest";
+
+vi.setConfig({ testTimeout: 30_000 }); // 30 seconds timeout
+
+// Mock the logger
+vi.mock("./logger.server", () => ({
+  logger: {
+    info: vi.fn(),
+    error: vi.fn(),
+  },
+}));
+
+import express, { Express } from "express";
+import request from "supertest";
+import { authorizationRateLimitMiddleware } from "../app/services/authorizationRateLimitMiddleware.server.js";
+
+describe("authorizationRateLimitMiddleware", () => {
+  let app: Express;
+
+  beforeEach(() => {
+    app = express();
+  });
+
+  redisTest("should allow requests within the rate limit", async ({ redis }) => {
+    const rateLimitMiddleware = authorizationRateLimitMiddleware({
+      redis: redis.options,
+      keyPrefix: "test",
+      defaultLimiter: {
+        type: "tokenBucket",
+        refillRate: 10,
+        interval: "1m",
+        maxTokens: 100,
+      },
+      pathMatchers: [/^\/api/],
+      log: {
+        rejections: false,
+        requests: false,
+      },
+    });
+
+    app.use(rateLimitMiddleware);
+    app.get("/api/test", (req, res) => {
+      res.status(200).json({ message: "Success" });
+    });
+
+    const response = await request(app).get("/api/test").set("Authorization", "Bearer test-token");
+
+    expect(response.status).toBe(200);
+    expect(response.body).toEqual({ message: "Success" });
+    expect(response.headers["x-ratelimit-limit"]).toBeDefined();
+    expect(response.headers["x-ratelimit-remaining"]).toBeDefined();
+    expect(response.headers["x-ratelimit-reset"]).toBeDefined();
+  });
+
+  redisTest("should reject requests without an Authorization header", async ({ redis }) => {
+    const rateLimitMiddleware = authorizationRateLimitMiddleware({
+      redis: redis.options,
+      keyPrefix: "test",
+      defaultLimiter: {
+        type: "tokenBucket",
+        refillRate: 10,
+        interval: "1m",
+        maxTokens: 100,
+      },
+      pathMatchers: [/^\/api/],
+    });
+
+    app.use(rateLimitMiddleware);
+    app.get("/api/test", (req, res) => {
+      res.status(200).json({ message: "Success" });
+    });
+
+    const response = await request(app).get("/api/test");
+
+    expect(response.status).toBe(401);
+    expect(response.body).toHaveProperty("title", "Unauthorized");
+  });
+
+  redisTest("should reject requests that exceed the rate limit", async ({ redis }) => {
+    const rateLimitMiddleware = authorizationRateLimitMiddleware({
+      redis: redis.options,
+      keyPrefix: "test",
+      defaultLimiter: {
+        type: "tokenBucket",
+        refillRate: 1,
+        interval: "1m",
+        maxTokens: 1,
+      },
+      pathMatchers: [/^\/api/],
+    });
+
+    app.use(rateLimitMiddleware);
+    app.get("/api/test", (req, res) => {
+      res.status(200).json({ message: "Success" });
+    });
+
+    // First request should succeed
+    await request(app).get("/api/test").set("Authorization", "Bearer test-token");
+
+    // Second request should be rate limited
+    const response = await request(app).get("/api/test").set("Authorization", "Bearer test-token");
+
+    expect(response.status).toBe(429);
+    expect(response.body).toHaveProperty("title", "Rate Limit Exceeded");
+  });
+
+  redisTest("should not apply rate limiting to whitelisted paths", async ({ redis }) => {
+    const rateLimitMiddleware = authorizationRateLimitMiddleware({
+      redis: redis.options,
+      keyPrefix: "test",
+      defaultLimiter: {
+        type: "tokenBucket",
+        refillRate: 10,
+        interval: "1m",
+        maxTokens: 100,
+      },
+      pathMatchers: [/^\/api/],
+      pathWhiteList: ["/api/whitelist"],
+    });
+
+    app.use(rateLimitMiddleware);
+    app.get("/api/whitelist", (req, res) => {
+      res.status(200).json({ message: "Whitelisted" });
+    });
+
+    const response = await request(app)
+      .get("/api/whitelist")
+      .set("Authorization", "Bearer test-token");
+
+    expect(response.status).toBe(200);
+    expect(response.body).toEqual({ message: "Whitelisted" });
+    expect(response.headers["x-ratelimit-limit"]).toBeUndefined();
+  });
+
+  redisTest(
+    "should apply different rate limits based on limiterConfigOverride",
+    async ({ redis }) => {
+      const rateLimitMiddleware = authorizationRateLimitMiddleware({
+        redis: redis.options,
+        keyPrefix: "test",
+        defaultLimiter: {
+          type: "tokenBucket",
+          refillRate: 1,
+          interval: "1m",
+          maxTokens: 1,
+        },
+        pathMatchers: [/^\/api/],
+        limiterConfigOverride: async (authorizationValue) => {
+          if (authorizationValue === "Bearer premium-token") {
+            return {
+              type: "tokenBucket",
+              refillRate: 10,
+              interval: "1m",
+              maxTokens: 100,
+            };
+          }
+          return undefined;
+        },
+      });
+
+      app.use(rateLimitMiddleware);
+      app.get("/api/test", (req, res) => {
+        res.status(200).json({ message: "Success" });
+      });
+
+      // Regular user should be rate limited after 1 request
+      await request(app).get("/api/test").set("Authorization", "Bearer regular-token");
+      const regularResponse = await request(app)
+        .get("/api/test")
+        .set("Authorization", "Bearer regular-token");
+      expect(regularResponse.status).toBe(429);
+
+      // Premium user should be able to make multiple requests
+      const premiumResponse1 = await request(app)
+        .get("/api/test")
+        .set("Authorization", "Bearer premium-token");
+      expect(premiumResponse1.status).toBe(200);
+      const premiumResponse2 = await request(app)
+        .get("/api/test")
+        .set("Authorization", "Bearer premium-token");
+      expect(premiumResponse2.status).toBe(200);
+    }
+  );
+
+  describe("Advanced Cases", () => {
+    // 1. Test different rate limit configurations
+    redisTest("should enforce fixed window rate limiting", async ({ redis }) => {
+      const rateLimitMiddleware = authorizationRateLimitMiddleware({
+        redis: redis.options,
+        keyPrefix: "test-fixed",
+        defaultLimiter: {
+          type: "fixedWindow",
+          window: "10s",
+          tokens: 3,
+        },
+        pathMatchers: [/^\/api/],
+      });
+
+      app.use(rateLimitMiddleware);
+      app.get("/api/test", (req, res) => res.status(200).json({ message: "Success" }));
+
+      const makeRequest = () =>
+        request(app).get("/api/test").set("Authorization", "Bearer test-token");
+
+      // Should allow 3 requests
+      for (let i = 0; i < 3; i++) {
+        const response = await makeRequest();
+        expect(response.status).toBe(200);
+      }
+
+      // 4th request should be rate limited
+      const limitedResponse = await makeRequest();
+      expect(limitedResponse.status).toBe(429);
+
+      // Wait for the window to reset
+      await new Promise((resolve) => setTimeout(resolve, 10000));
+
+      // Should allow requests again
+      const newResponse = await makeRequest();
+      expect(newResponse.status).toBe(200);
+    });
+
+    redisTest("should enforce sliding window rate limiting", async ({ redis }) => {
+      const rateLimitMiddleware = authorizationRateLimitMiddleware({
+        redis: redis.options,
+        keyPrefix: "test-sliding",
+        defaultLimiter: {
+          type: "slidingWindow",
+          window: "10s",
+          tokens: 3,
+        },
+        pathMatchers: [/^\/api/],
+      });
+
+      app.use(rateLimitMiddleware);
+      app.get("/api/test", (req, res) => res.status(200).json({ message: "Success" }));
+
+      const makeRequest = () =>
+        request(app).get("/api/test").set("Authorization", "Bearer test-token");
+
+      // Should allow 3 requests
+      for (let i = 0; i < 3; i++) {
+        const response = await makeRequest();
+        expect(response.status).toBe(200);
+      }
+
+      // 4th request should be rate limited
+      const limitedResponse = await makeRequest();
+      expect(limitedResponse.status).toBe(429);
+
+      // Wait for part of the window to pass
+      await new Promise((resolve) => setTimeout(resolve, 1000));
+
+      // Should still be limited
+      const stillLimitedResponse = await makeRequest();
+      expect(stillLimitedResponse.status).toBe(429);
+
+      // Wait for the full window to pass
+      await new Promise((resolve) => setTimeout(resolve, 10000));
+
+      // Should allow requests again
+      const newResponse = await makeRequest();
+      expect(newResponse.status).toBe(200);
+    });
+
+    // 2. Test edge cases around rate limit calculations
+    redisTest("should handle token refill correctly", async ({ redis }) => {
+      const rateLimitMiddleware = authorizationRateLimitMiddleware({
+        redis: redis.options,
+        keyPrefix: "test-refill",
+        defaultLimiter: {
+          type: "tokenBucket",
+          refillRate: 1,
+          interval: "5s",
+          maxTokens: 3,
+        },
+        pathMatchers: [/^\/api/],
+      });
+
+      app.use(rateLimitMiddleware);
+      app.get("/api/test", (req, res) => res.status(200).json({ message: "Success" }));
+
+      const makeRequest = () =>
+        request(app).get("/api/test").set("Authorization", "Bearer test-token");
+
+      // Use up all tokens
+      for (let i = 0; i < 3; i++) {
+        const response = await makeRequest();
+        expect(response.status).toBe(200);
+      }
+
+      // Next request should be limited
+      const limitedResponse = await makeRequest();
+      expect(limitedResponse.status).toBe(429);
+
+      // Wait for one token to be refilled
+      await new Promise((resolve) => setTimeout(resolve, 5000));
+
+      // Should allow one request
+      const newResponse = await makeRequest();
+      expect(newResponse.status).toBe(200);
+
+      // But the next one should be limited again
+      const limitedAgainResponse = await makeRequest();
+      expect(limitedAgainResponse.status).toBe(429);
+    });
+
+    redisTest("should handle near-zero remaining tokens correctly", async ({ redis }) => {
+      const rateLimitMiddleware = authorizationRateLimitMiddleware({
+        redis: redis.options,
+        keyPrefix: "test-near-zero",
+        defaultLimiter: {
+          type: "tokenBucket",
+          refillRate: 1, // 1 token every 5 seconds
+          interval: "5s",
+          maxTokens: 1,
+        },
+        pathMatchers: [/^\/api/],
+      });
+
+      app.use(rateLimitMiddleware);
+      app.get("/api/test", (req, res) => res.status(200).json({ message: "Success" }));
+
+      const makeRequest = () =>
+        request(app).get("/api/test").set("Authorization", "Bearer test-token");
+
+      // First request should succeed
+      const firstResponse = await makeRequest();
+      expect(firstResponse.status).toBe(200);
+
+      // Immediate second request should fail
+      const secondResponse = await makeRequest();
+      expect(secondResponse.status).toBe(429);
+
+      // Wait for almost one token to be refilled (4.9 seconds)
+      await new Promise((resolve) => setTimeout(resolve, 4900));
+
+      // This request should still fail as we're just shy of a full token
+      const thirdResponse = await makeRequest();
+      expect(thirdResponse.status).toBe(429);
+
+      // Wait for the full token to be refilled (additional 200ms)
+      await new Promise((resolve) => setTimeout(resolve, 200));
+
+      // This request should now succeed
+      const fourthResponse = await makeRequest();
+      expect(fourthResponse.status).toBe(200);
+
+      // Immediate next request should fail again
+      const fifthResponse = await makeRequest();
+      expect(fifthResponse.status).toBe(429);
+    });
+
+    // 3. Test the limiterCache functionality
+    redisTest("should use cached limiter configurations", async ({ redis }) => {
+      let configOverrideCalls = 0;
+      const rateLimitMiddleware = authorizationRateLimitMiddleware({
+        redis: redis.options,
+        keyPrefix: "test-cache",
+        defaultLimiter: {
+          type: "tokenBucket",
+          refillRate: 1,
+          interval: "1m",
+          maxTokens: 10,
+        },
+        pathMatchers: [/^\/api/],
+        limiterCache: {
+          fresh: 1000, // 1 second
+          stale: 2000, // 2 seconds
+        },
+        limiterConfigOverride: async (authorizationValue) => {
+          configOverrideCalls++;
+          if (authorizationValue === "Bearer premium-token") {
+            return {
+              type: "tokenBucket",
+              refillRate: 10,
+              interval: "1m",
+              maxTokens: 100,
+            };
+          }
+          return undefined;
+        },
+      });
+
+      app.use(rateLimitMiddleware);
+      app.get("/api/test", (req, res) => res.status(200).json({ message: "Success" }));
+
+      const makeRequest = () =>
+        request(app).get("/api/test").set("Authorization", "Bearer premium-token");
+
+      // First request should call the override
+      await makeRequest();
+      expect(configOverrideCalls).toBe(1);
+
+      // Subsequent requests within 1 second should use the cache
+      await makeRequest();
+      await makeRequest();
+      expect(configOverrideCalls).toBe(1);
+
+      // Wait for the cache to become stale
+      await new Promise((resolve) => setTimeout(resolve, 1100));
+
+      // This should still use the cache, but also trigger a refresh
+      await makeRequest();
+      expect(configOverrideCalls).toBe(2);
+
+      // Wait for the cache to expire completely
+      await new Promise((resolve) => setTimeout(resolve, 1000));
+
+      // This should trigger a new override call
+      await makeRequest();
+      expect(configOverrideCalls).toBe(3);
+    });
+  });
+});

apps/webapp/test/placeholder.test.ts

@@ -1,5 +0,0 @@
-describe("Placeholder", () => {
-  it("should pass", () => {
-    expect(true).toBe(true);
-  });
-});

apps/webapp/test/realtimeClient.test.ts

@@ -0,0 +1,211 @@
+import { containerWithElectricTest } from "@internal/testcontainers";
+import { expect, describe } from "vitest";
+import { RealtimeClient } from "../app/services/realtimeClient.server.js";
+
+describe("RealtimeClient", () => {
+  containerWithElectricTest(
+    "Should only track concurrency for live requests",
+    { timeout: 30_000 },
+    async ({ redis, electricOrigin, prisma }) => {
+      const client = new RealtimeClient({
+        electricOrigin,
+        keyPrefix: "test:realtime",
+        redis: redis.options,
+        expiryTimeInSeconds: 5,
+        cachedLimitProvider: {
+          async getCachedLimit() {
+            return 1;
+          },
+        },
+      });
+
+      const organization = await prisma.organization.create({
+        data: {
+          title: "test-org",
+          slug: "test-org",
+        },
+      });
+
+      const project = await prisma.project.create({
+        data: {
+          name: "test-project",
+          slug: "test-project",
+          organizationId: organization.id,
+          externalRef: "test-project",
+        },
+      });
+
+      const environment = await prisma.runtimeEnvironment.create({
+        data: {
+          projectId: project.id,
+          organizationId: organization.id,
+          slug: "test",
+          type: "DEVELOPMENT",
+          shortcode: "1234",
+          apiKey: "tr_dev_1234",
+          pkApiKey: "pk_test_1234",
+        },
+      });
+
+      const run = await prisma.taskRun.create({
+        data: {
+          taskIdentifier: "test-task",
+          friendlyId: "run_1234",
+          payload: "{}",
+          payloadType: "application/json",
+          traceId: "trace_1234",
+          spanId: "span_1234",
+          queue: "test-queue",
+          projectId: project.id,
+          runtimeEnvironmentId: environment.id,
+        },
+      });
+
+      const initialResponsePromise = client.streamRun(
+        "http://localhost:3000?offset=-1",
+        environment,
+        run.id
+      );
+
+      const initializeResponsePromise2 = new Promise<Response>((resolve) => {
+        setTimeout(async () => {
+          const response = await client.streamRun(
+            "http://localhost:3000?offset=-1",
+            environment,
+            run.id
+          );
+
+          resolve(response);
+        }, 1);
+      });
+
+      const [response, response2] = await Promise.all([
+        initialResponsePromise,
+        initializeResponsePromise2,
+      ]);
+
+      const headers = Object.fromEntries(response.headers.entries());
+
+      const shapeId = headers["electric-shape-id"];
+      const chunkOffset = headers["electric-chunk-last-offset"];
+
+      expect(response.status).toBe(200);
+      expect(response2.status).toBe(200);
+      expect(shapeId).toBeDefined();
+      expect(chunkOffset).toBe("0_0");
+
+      // Okay, now we will do two live requests, and the second one should fail because of the concurrency limit
+      const liveResponsePromise = client.streamRun(
+        `http://localhost:3000?offset=0_0&live=true&shape_id=${shapeId}`,
+        environment,
+        run.id
+      );
+
+      const liveResponsePromise2 = new Promise<Response>((resolve) => {
+        setTimeout(async () => {
+          const response = await client.streamRun(
+            `http://localhost:3000?offset=0_0&live=true&shape_id=${shapeId}`,
+            environment,
+            run.id
+          );
+
+          resolve(response);
+        }, 1);
+      });
+
+      const updateRunAfter1SecondPromise = new Promise<void>((resolve) => {
+        setTimeout(async () => {
+          await prisma.taskRun.update({
+            where: { id: run.id },
+            data: { metadata: "{}" },
+          });
+
+          resolve();
+        }, 1000);
+      });
+
+      const [liveResponse, liveResponse2] = await Promise.all([
+        liveResponsePromise,
+        liveResponsePromise2,
+        updateRunAfter1SecondPromise,
+      ]);
+
+      expect(liveResponse.status).toBe(200);
+      expect(liveResponse2.status).toBe(429);
+    }
+  );
+
+  containerWithElectricTest(
+    "Should support subscribing to a run tag",
+    { timeout: 30_000 },
+    async ({ redis, electricOrigin, prisma }) => {
+      const client = new RealtimeClient({
+        electricOrigin,
+        keyPrefix: "test:realtime",
+        redis: redis.options,
+        expiryTimeInSeconds: 5,
+        cachedLimitProvider: {
+          async getCachedLimit() {
+            return 1;
+          },
+        },
+      });
+
+      const organization = await prisma.organization.create({
+        data: {
+          title: "test-org",
+          slug: "test-org",
+        },
+      });
+
+      const project = await prisma.project.create({
+        data: {
+          name: "test-project",
+          slug: "test-project",
+          organizationId: organization.id,
+          externalRef: "test-project",
+        },
+      });
+
+      const environment = await prisma.runtimeEnvironment.create({
+        data: {
+          projectId: project.id,
+          organizationId: organization.id,
+          slug: "test",
+          type: "DEVELOPMENT",
+          shortcode: "1234",
+          apiKey: "tr_dev_1234",
+          pkApiKey: "pk_test_1234",
+        },
+      });
+
+      const run = await prisma.taskRun.create({
+        data: {
+          taskIdentifier: "test-task",
+          friendlyId: "run_1234",
+          payload: "{}",
+          payloadType: "application/json",
+          traceId: "trace_1234",
+          spanId: "span_1234",
+          queue: "test-queue",
+          projectId: project.id,
+          runtimeEnvironmentId: environment.id,
+          runTags: ["test:tag:1234", "test:tag:5678"],
+        },
+      });
+
+      const response = await client.streamRuns("http://localhost:3000?offset=-1", environment, {
+        tags: ["test:tag:1234"],
+      });
+
+      const headers = Object.fromEntries(response.headers.entries());
+
+      const shapeId = headers["electric-shape-id"];
+      const chunkOffset = headers["electric-chunk-last-offset"];
+
+      expect(response.status).toBe(200);
+      expect(shapeId).toBeDefined();
+      expect(chunkOffset).toBe("0_0");
+    }
+  );
+});

apps/webapp/tsconfig.json

@@ -7,6 +7,7 @@
     "isolatedModules": true,
     "esModuleInterop": true,
     "jsx": "react-jsx",
+    "module": "esnext",
     "moduleResolution": "Bundler",
     "resolveJsonModule": true,
     "target": "ES2019",

apps/webapp/vitest.config.ts

@@ -1,8 +1,12 @@
 import { defineConfig } from "vitest/config";
+import tsconfigPaths from "vite-tsconfig-paths";
 
 export default defineConfig({
   test: {
     include: ["test/**/*.test.ts"],
     globals: true,
+    pool: "forks",
   },
+  // @ts-ignore
+  plugins: [tsconfigPaths({ projects: ["./tsconfig.json"] })],
 });

docker/docker-compose.yml

@@ -60,10 +60,10 @@ services:
       - 6379:6379
 
   electric:
-    image: electricsql/electric
+    image: electricsql/electric:0.7.5
     restart: always
     environment:
-      DATABASE_URL: postgresql://postgres:postgres@database:5432/postgres
+      DATABASE_URL: postgresql://postgres:postgres@database:5432/postgres?sslmode=disable
     networks:
       - app_network
     ports:

docs/guides/examples/scrape-hacker-news.mdx

@@ -7,6 +7,10 @@ description: "This example demonstrates how to scrape the top 3 articles from Ha
 import LocalDevelopment from "/snippets/local-development-extensions.mdx";
 import ScrapingWarning from "/snippets/web-scraping-warning.mdx";
 
+<div className="w-full h-full aspect-video">
+<iframe width="100%" height="100%" src="https://www.youtube.com/embed/6azvzrZITKY?si=muKtsBiS9TJGGKWg" title="YouTube video player" frameborder="0" allow="accelerometer; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen/>
+</div>
+
 ## Overview
 
 In this example we'll be using a number of different tools and features to:

docs/guides/examples/sentry-error-tracking.mdx

@@ -0,0 +1,120 @@
+---
+title: "Track errors with Sentry"
+sidebarTitle: "Sentry error tracking"
+description: "This example demonstrates how to track errors with Sentry using Trigger.dev."
+---
+
+## Overview
+
+Automatically send errors and source maps to your Sentry project from your Trigger.dev tasks. Sending source maps to Sentry allows for more detailed stack traces when errors occur, as Sentry can map the minified code back to the original source code.
+
+## Prerequisites
+
+- A [Sentry](https://sentry.io) account and project
+- A [Trigger.dev](https://trigger.dev) account and project
+
+## Build configuration
+
+To send errors to Sentry when there are errors in your tasks, you'll need to add this build configuration to your `trigger.config.ts` file. This will then run every time you deploy your project.
+
+<Note>
+  You will need to set the `SENTRY_AUTH_TOKEN` and `SENTRY_DSN` environment variables. You can find
+  the `SENTRY_AUTH_TOKEN` in your Sentry dashboard, in settings -> developer settings -> auth tokens
+  and the `SENTRY_DSN` in your Sentry dashboard, in settings -> projects -> your project -> client
+  keys (DSN). Add these to your `.env` file, and in your [Trigger.dev
+  dashboard](https://cloud.trigger.dev), under environment variables in your project's sidebar.
+</Note>
+
+```ts trigger.config.ts
+import { defineConfig } from "@trigger.dev/sdk/v3";
+import { esbuildPlugin } from "@trigger.dev/build/extensions";
+import { sentryEsbuildPlugin } from "@sentry/esbuild-plugin";
+import * as Sentry from "@sentry/node";
+
+export default defineConfig({
+  project: "<project ref>",
+  // Your other config settings...
+  build: {
+    extensions: [
+      esbuildPlugin(
+        sentryEsbuildPlugin({
+          org: "<your-sentry-org>",
+          project: "<your-sentry-project>",
+          // Find this auth token in settings -> developer settings -> auth tokens
+          authToken: process.env.SENTRY_AUTH_TOKEN,
+        }),
+        { placement: "last", target: "deploy" }
+      ),
+    ],
+  },
+  init: async () => {
+    Sentry.init({
+      // The Data Source Name (DSN) is a unique identifier for your Sentry project.
+      dsn: process.env.SENTRY_DSN,
+      // Update this to match the environment you want to track errors for
+      environment: process.env.NODE_ENV === "production" ? "production" : "development",
+    });
+  },
+  onFailure: async (payload, error, { ctx }) => {
+    Sentry.captureException(error, {
+      extra: {
+        payload,
+        ctx,
+      },
+    });
+  },
+});
+```
+
+<Note>
+  [Build extensions](/config/config-file#extensions) allow you to hook into the build system and
+  customize the build process or the resulting bundle and container image (in the case of
+  deploying). You can use pre-built extensions or create your own.
+</Note>
+
+## Testing that errors are being sent to Sentry
+
+To test that errors are being sent to Sentry, you need to create a task that will fail.
+
+This task takes no payload, and will throw an error.
+
+```ts trigger/sentry-error-test.ts
+import { task } from "@trigger.dev/sdk/v3";
+
+export const sentryErrorTest = task({
+  id: "sentry-error-test",
+  retry: {
+    // Only retry once
+    maxAttempts: 1,
+  },
+  run: async () => {
+    const error = new Error("This is a custom error that Sentry will capture");
+    error.cause = { additionalContext: "This is additional context" };
+    throw error;
+  },
+});
+```
+
+After creating the task, deploy your project.
+
+ <CodeGroup>
+
+    ```bash npm
+    npx trigger.dev@latest deploy
+    ```
+
+    ```bash pnpm
+    pnpm dlx trigger.dev@latest deploy
+    ```
+
+    ```bash yarn
+    yarn dlx trigger.dev@latest deploy
+    ```
+
+    </CodeGroup>
+
+Once deployed, navigate to the `test` page in the sidebar of your [Trigger.dev dashboard](https://cloud.trigger.dev), click on your `prod` environment, and select the `sentryErrorTest` task.
+
+Run a test task with an empty payload by clicking the `Run test` button.
+
+Your run should then fail, and if everything is set up correctly, you will see an error in the Sentry project dashboard shortly after.

docs/guides/introduction.mdx

@@ -47,6 +47,7 @@ Tasks you can copy and paste to get started with Trigger.dev. They can all be ex
 | [React to PDF](/guides/examples/react-pdf)                                    | Use `react-pdf` to generate a PDF and save it to Cloudflare R2.                |
 | [Puppeteer](/guides/examples/puppeteer)                                       | Use Puppeteer to generate a PDF or scrape a webpage.                           |
 | [Resend email sequence](/guides/examples/resend-email-sequence)               | Send a sequence of emails over several days using Resend with Trigger.dev.     |
+| [Sentry error tracking](/guides/examples/sentry-error-tracking)               | Automatically send errors to Sentry from your tasks.                           |
 | [Sharp image processing](/guides/examples/sharp-image-processing)             | Use Sharp to process an image and save it to Cloudflare R2.                    |
 | [Supabase database operations](/guides/examples/supabase-database-operations) | Run basic CRUD operations on a table in a Supabase database using Trigger.dev. |
 | [Supabase Storage upload](/guides/examples/supabase-storage-upload)           | Download a video from a URL and upload it to Supabase Storage using S3.        |

docs/guides/use-cases/upgrading-from-v2.mdx

@@ -12,6 +12,177 @@ The main difference is that things in v3 are far simpler. That's because in v3 y
 3. Just use official SDKs, not integrations.
 4. `task`s are the new primitive, not `job`s.
 
+## Convert your v2 job using an AI prompt
+
+The prompt in the accordion below gives good results when using Anthropic Claude 3.5 Sonnet. You’ll need a relatively large token limit.
+
+<Note>Don't forget to paste your own v2 code in a markdown codeblock at the bottom of the prompt before running it.</Note>
+
+<Accordion title="Copy and paste this prompt in full:">
+
+I would like you to help me convert from Trigger.dev v2 to Trigger.dev v3.
+The important differences:
+1. The syntax for creating "background jobs" has changed. In v2 it looked like this:
+
+```ts
+import { eventTrigger } from "@trigger.dev/sdk";
+import { client } from "@/trigger";
+import { db } from "@/lib/db";
+client.defineJob({
+  enabled: true,
+  id: "my-job-id",
+  name: "My job name",
+  version: "0.0.1",
+  // This is triggered by an event using eventTrigger. You can also trigger Jobs with webhooks, on schedules, and more: https://trigger.dev/docs/documentation/concepts/triggers/introduction
+  trigger: eventTrigger({
+    name: "theevent.name",
+    schema: z.object({
+      phoneNumber: z.string(),
+      verified: z.boolean(),
+    }),
+  }),
+  run: async (payload, io) => {
+    
+    //everything needed to be wrapped in io.runTask in v2, to make it possible for long-running code to work
+    const result = await io.runTask("get-stuff-from-db", async () => {
+      const socials = await db.query.Socials.findMany({
+        where: eq(Socials.service, "tiktok"),
+      });
+      return socials;
+    });
+    
+    io.logger.info("Completed fetch successfully");
+  },
+});
+```
+
+In v3 it looks like this:
+
+```ts
+import { task } from "@trigger.dev/sdk/v3";
+import { db } from "@/lib/db";
+export const getCreatorVideosFromTikTok = task({
+  id: "my-job-id",
+  run: async (payload: { phoneNumber: string, verified: boolean }) => {
+    //in v3 there are no timeouts, so you can just use the code as is, no need to wrap in `io.runTask`
+    const socials = await db.query.Socials.findMany({
+      where: eq(Socials.service, "tiktok"),
+    });
+    
+    //use `logger` instead of `io.logger`
+    logger.info("Completed fetch successfully");
+  },
+});
+```
+
+Notice that the schema on v2 `eventTrigger` defines the payload type. In v3 that needs to be done on the TypeScript type of the `run` payload param.
+2. v2 had integrations with some APIs. Any package that isn't `@trigger.dev/sdk` can be replaced with an official SDK. The syntax may need to be adapted.
+For example:
+v2:
+
+```ts
+import { OpenAI } from "@trigger.dev/openai";
+const openai = new OpenAI({
+  id: "openai",
+  apiKey: process.env.OPENAI_API_KEY!,
+});
+client.defineJob({
+  id: "openai-job",
+  name: "OpenAI Job",
+  version: "1.0.0",
+  trigger: invokeTrigger(),
+  integrations: {
+    openai, // Add the OpenAI client as an integration
+  },
+  run: async (payload, io, ctx) => {
+    // Now you can access it through the io object
+    const completion = await io.openai.chat.completions.create("completion", {
+      model: "gpt-3.5-turbo",
+      messages: [
+        {
+          role: "user",
+          content: "Create a good programming joke about background jobs",
+        },
+      ],
+    });
+  },
+});
+```
+
+Would become in v3:
+
+```ts
+import OpenAI from "openai";
+const openai = new OpenAI({
+  apiKey: process.env.OPENAI_API_KEY,
+});
+export const openaiJob = task({
+  id: "openai-job",
+  run: async (payload) => {
+    const completion = await openai.chat.completions.create(
+      {
+        model: "gpt-3.5-turbo",
+        messages: [
+          {
+            role: "user",
+            content: "Create a good programming joke about background jobs",
+          },
+        ],
+    });
+  },
+});
+```
+
+So don't use the `@trigger.dev/openai` package in v3, use the official OpenAI SDK.
+Bear in mind that the syntax for the latest official SDK will probably be different from the @trigger.dev integration SDK. You will need to adapt the code accordingly.
+3. The most critical difference is that inside the `run` function you do NOT need to wrap everything in `io.runTask`. So anything inside there can be extracted out and be used in the main body of the function without wrapping it.
+4. The import for `task` in v3 is `import { task } from "@trigger.dev/sdk/v3";`
+5. You can trigger jobs from other jobs. In v2 this was typically done by either calling `io.sendEvent()` or by calling `yourOtherTask.invoke()`. In v3 you call `.trigger()` on the other task, there are no events in v3.
+v2:
+
+```ts
+export const parentJob = client.defineJob({
+  id: "parent-job",
+  run: async (payload, io) => {
+    //send event
+    await client.sendEvent({
+      name: "user.created",
+      payload: { name: "John Doe", email: "john@doe.com", paidPlan: true },
+    });
+    
+    //invoke
+    await exampleJob.invoke({ foo: "bar" }, {
+      idempotencyKey: `some_string_here_${
+        payload.someValue
+      }_${new Date().toDateString()}`,
+    });
+  },
+});
+```
+
+v3:
+
+```ts
+export const parentJob = task({
+  id: "parent-job",
+  run: async (payload) => {
+    //trigger
+    await userCreated.trigger({ name: "John Doe", email: "john@doe.com", paidPlan: true });
+    
+    //trigger, you can pass in an idempotency key
+    await exampleJob.trigger({ foo: "bar" }, {
+      idempotencyKey: `some_string_here_${
+        payload.someValue
+      }_${new Date().toDateString()}`,
+    });
+  }
+});
+```
+
+Can you help me convert the following code from v2 to v3? Please include the full converted code in the answer, do not truncate it anywhere.
+
+</Accordion>
+
 ## OpenAI example comparison
 
 This is a (very contrived) example that does a long OpenAI API call (>10s), stores the result in a database, waits for 5 mins, and then returns the result.

docs/mint.json

@@ -311,6 +311,7 @@
         "guides/examples/pdf-to-image",
         "guides/examples/puppeteer",
         "guides/examples/scrape-hacker-news",
+        "guides/examples/sentry-error-tracking",
         "guides/examples/sharp-image-processing",
         "guides/examples/supabase-database-operations",
         "guides/examples/supabase-storage-upload",

internal-packages/database/README.md

@@ -5,7 +5,7 @@ This is the internal database package for the Trigger.dev project. It exports a
 ### How to add a new index on a large table
 
 1. Modify the Prisma.schema with a single index change (no other changes, just one index at a time)
-2. Create a Prisma migration using `cd packages/database && pnpm run db:migrate:dev --create-only`
+2. Create a Prisma migration using `cd internal-packages/database && pnpm run db:migrate:dev --create-only`
 3. Modify the SQL file: add IF NOT EXISTS to it and CONCURRENTLY:
 
 ```sql

internal-packages/database/prisma/migrations/20240929115226_add_run_tags_to_task_run/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "TaskRun" ADD COLUMN     "runTags" TEXT[];

internal-packages/database/prisma/migrations/20241001190022_add_rate_limiter_config_to_orgs/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "Organization" ADD COLUMN     "apiRateLimiterConfig" JSONB,
+ADD COLUMN     "realtimeRateLimiterConfig" JSONB;

internal-packages/database/prisma/migrations/20241023154826_add_schedule_instance_id_index_to_task_run/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "TaskRun_scheduleInstanceId_idx" ON "TaskRun"("scheduleInstanceId");

internal-packages/database/prisma/migrations/20241028133512_add_attempt_id_index_to_checkpoint/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "Checkpoint_attemptId_idx" ON "Checkpoint"("attemptId");

internal-packages/database/prisma/migrations/20241028133719_add_run_id_index_to_checkpoint/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "Checkpoint_runId_idx" ON "Checkpoint"("runId");

internal-packages/database/prisma/schema.prisma

@@ -138,6 +138,9 @@ model Organization {
   events       EventRecord[]
   jobRuns      JobRun[]
 
+  apiRateLimiterConfig      Json?
+  realtimeRateLimiterConfig Json?
+
   projects                 Project[]
   members                  OrgMember[]
   invites                  OrgMemberInvite[]
@@ -1683,6 +1686,9 @@ model TaskRun {
   attempts TaskRunAttempt[] @relation("attempts")
   tags     TaskRunTag[]
 
+  /// Denormized column that holds the raw tags
+  runTags String[]
+
   checkpoints Checkpoint[]
 
   startedAt     DateTime?
@@ -1787,6 +1793,7 @@ model TaskRun {
   @@index([projectId, taskIdentifier, status])
   //Schedules
   @@index([scheduleId])
+  @@index([scheduleInstanceId])
   // Run page inspector
   @@index([spanId])
   @@index([parentSpanId])
@@ -1795,6 +1802,10 @@ model TaskRun {
 }
 
 enum TaskRunStatus {
+  ///
+  /// NON-FINAL STATUSES
+  ///
+
   /// Task has been scheduled to run in the future
   DELAYED
   /// Task is waiting to be executed by a worker
@@ -1815,6 +1826,10 @@ enum TaskRunStatus {
   /// Task has been paused by the user, and can be resumed by the user
   PAUSED
 
+  ///
+  /// FINAL STATUSES
+  ///
+
   /// Task has been canceled by the user
   CANCELED
 
@@ -1949,9 +1964,11 @@ model TaskRunAttempt {
 }
 
 enum TaskRunAttemptStatus {
+  /// NON-FINAL
   PENDING
   EXECUTING
   PAUSED
+  /// FINAL
   FAILED
   CANCELED
   COMPLETED
@@ -2229,6 +2246,9 @@ model Checkpoint {
 
   createdAt DateTime @default(now())
   updatedAt DateTime @updatedAt
+
+  @@index([attemptId])
+  @@index([runId])
 }
 
 enum CheckpointType {

internal-packages/testcontainers/package.json

@@ -14,6 +14,7 @@
     "@testcontainers/postgresql": "^10.13.1",
     "@testcontainers/redis": "^10.13.1",
     "testcontainers": "^10.13.1",
+    "tinyexec": "^0.3.0",
     "vitest": "^1.4.0"
   },
   "scripts": {

internal-packages/testcontainers/src/index.ts

@@ -3,20 +3,37 @@ import { StartedRedisContainer } from "@testcontainers/redis";
 import { Redis } from "ioredis";
 import { test } from "vitest";
 import { PrismaClient } from "@trigger.dev/database";
-import { createPostgresContainer, createRedisContainer } from "./utils";
+import { createPostgresContainer, createRedisContainer, createElectricContainer } from "./utils";
+import { Network, type StartedNetwork, type StartedTestContainer } from "testcontainers";
 
-type PostgresContext = {
+type NetworkContext = { network: StartedNetwork };
+
+type PostgresContext = NetworkContext & {
   postgresContainer: StartedPostgreSqlContainer;
   prisma: PrismaClient;
 };
 
 type RedisContext = { redisContainer: StartedRedisContainer; redis: Redis };
-type ContainerContext = PostgresContext & RedisContext;
+
+type ElectricContext = {
+  electricOrigin: string;
+};
+
+type ContainerContext = NetworkContext & PostgresContext & RedisContext;
+type ContainerWithElectricContext = ContainerContext & ElectricContext;
 
 type Use<T> = (value: T) => Promise<void>;
 
-const postgresContainer = async ({}, use: Use<StartedPostgreSqlContainer>) => {
-  const { container } = await createPostgresContainer();
+const network = async ({}, use: Use<StartedNetwork>) => {
+  const network = await new Network().start();
+  await use(network);
+};
+
+const postgresContainer = async (
+  { network }: { network: StartedNetwork },
+  use: Use<StartedPostgreSqlContainer>
+) => {
+  const { container } = await createPostgresContainer(network);
   await use(container);
   await container.stop();
 };
@@ -36,7 +53,7 @@ const prisma = async (
   await prisma.$disconnect();
 };
 
-export const postgresTest = test.extend<PostgresContext>({ postgresContainer, prisma });
+export const postgresTest = test.extend<PostgresContext>({ network, postgresContainer, prisma });
 
 const redisContainer = async ({}, use: Use<StartedRedisContainer>) => {
   const { container } = await createRedisContainer();
@@ -59,9 +76,31 @@ const redis = async (
 
 export const redisTest = test.extend<RedisContext>({ redisContainer, redis });
 
+const electricOrigin = async (
+  {
+    postgresContainer,
+    network,
+  }: { postgresContainer: StartedPostgreSqlContainer; network: StartedNetwork },
+  use: Use<string>
+) => {
+  const { origin, container } = await createElectricContainer(postgresContainer, network);
+  await use(origin);
+  await container.stop();
+};
+
 export const containerTest = test.extend<ContainerContext>({
+  network,
   postgresContainer,
   prisma,
   redisContainer,
   redis,
 });
+
+export const containerWithElectricTest = test.extend<ContainerWithElectricContext>({
+  network,
+  postgresContainer,
+  prisma,
+  redisContainer,
+  redis,
+  electricOrigin,
+});

internal-packages/testcontainers/src/utils.ts

@@ -1,35 +1,70 @@
-import { PostgreSqlContainer } from "@testcontainers/postgresql";
+import { PostgreSqlContainer, StartedPostgreSqlContainer } from "@testcontainers/postgresql";
 import { RedisContainer } from "@testcontainers/redis";
-import { execSync } from "child_process";
 import path from "path";
+import { GenericContainer, StartedNetwork } from "testcontainers";
+import { x } from "tinyexec";
 
-export async function createPostgresContainer() {
-  const container = await new PostgreSqlContainer().start();
+export async function createPostgresContainer(network: StartedNetwork) {
+  const container = await new PostgreSqlContainer("docker.io/postgres:14")
+    .withNetwork(network)
+    .withNetworkAliases("database")
+    .withCommand(["-c", "listen_addresses=*", "-c", "wal_level=logical"])
+    .start();
 
   // Run migrations
   const databasePath = path.resolve(__dirname, "../../database");
 
-  execSync(`npx prisma@5.4.1 db push --schema ${databasePath}/prisma/schema.prisma`, {
-    env: {
-      ...process.env,
-      DATABASE_URL: container.getConnectionUri(),
-      DIRECT_URL: container.getConnectionUri(),
-    },
-  });
+  await x(
+    `${databasePath}/node_modules/.bin/prisma`,
+    [
+      "db",
+      "push",
+      "--force-reset",
+      "--accept-data-loss",
+      "--skip-generate",
+      "--schema",
+      `${databasePath}/prisma/schema.prisma`,
+    ],
+    {
+      nodeOptions: {
+        env: {
+          ...process.env,
+          DATABASE_URL: container.getConnectionUri(),
+          DIRECT_URL: container.getConnectionUri(),
+        },
+      },
+    }
+  );
 
-  // console.log(container.getConnectionUri());
-
-  return { url: container.getConnectionUri(), container };
+  return { url: container.getConnectionUri(), container, network };
 }
 
 export async function createRedisContainer() {
   const container = await new RedisContainer().start();
-  try {
-    return {
-      container,
-    };
-  } catch (e) {
-    console.error(e);
-    throw e;
-  }
+
+  return {
+    container,
+  };
+}
+
+export async function createElectricContainer(
+  postgresContainer: StartedPostgreSqlContainer,
+  network: StartedNetwork
+) {
+  const databaseUrl = `postgresql://${postgresContainer.getUsername()}:${postgresContainer.getPassword()}@${postgresContainer.getIpAddress(
+    network.getName()
+  )}:5432/${postgresContainer.getDatabase()}?sslmode=disable`;
+
+  const container = await new GenericContainer("electricsql/electric:0.7.5")
+    .withExposedPorts(3000)
+    .withNetwork(network)
+    .withEnvironment({
+      DATABASE_URL: databaseUrl,
+    })
+    .start();
+
+  return {
+    container,
+    origin: `http://${container.getHost()}:${container.getMappedPort(3000)}`,
+  };
 }

packages/build/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @trigger.dev/build
 
+## 3.1.0
+
+### Patch Changes
+
+- Added a Vercel sync env vars extension. Given a Vercel projectId and access token it will sync Vercel env vars when deploying Trigger.dev tasks. ([#1425](https://github.com/triggerdotdev/trigger.dev/pull/1425))
+- Updated dependencies:
+  - `@trigger.dev/core@3.1.0`
+
 ## 3.0.13
 
 ### Patch Changes

packages/build/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/build",
-  "version": "3.0.13",
+  "version": "3.1.0",
   "description": "trigger.dev build extensions",
   "license": "MIT",
   "publishConfig": {
@@ -65,7 +65,7 @@
     "check-exports": "attw --pack ."
   },
   "dependencies": {
-    "@trigger.dev/core": "workspace:3.0.13",
+    "@trigger.dev/core": "workspace:3.1.0",
     "pkg-types": "^1.1.3",
     "tinyglobby": "^0.2.2",
     "tsconfck": "3.1.3"

packages/build/src/extensions/core.ts

@@ -3,3 +3,4 @@ export * from "./core/additionalPackages.js";
 export * from "./core/syncEnvVars.js";
 export * from "./core/aptGet.js";
 export * from "./core/ffmpeg.js";
+export * from "./core/vercelSyncEnvVars.js";

packages/build/src/extensions/core/vercelSyncEnvVars.ts

@@ -0,0 +1,83 @@
+import { BuildExtension } from "@trigger.dev/core/v3/build";
+import { syncEnvVars } from "../core.js";
+
+export function syncVercelEnvVars(
+  options?: { projectId?: string; vercelAccessToken?: string },
+): BuildExtension {
+  const sync = syncEnvVars(async (ctx) => {
+    const projectId = options?.projectId ?? process.env.VERCEL_PROJECT_ID ??
+      ctx.env.VERCEL_PROJECT_ID;
+    const vercelAccessToken = options?.vercelAccessToken ??
+      process.env.VERCEL_ACCESS_TOKEN ??
+      ctx.env.VERCEL_ACCESS_TOKEN;
+
+    if (!projectId) {
+      throw new Error(
+        "vercelSyncEnvVars: you did not pass in a projectId or set the VERCEL_PROJECT_ID env var.",
+      );
+    }
+
+    if (!vercelAccessToken) {
+      throw new Error(
+        "vercelSyncEnvVars: you did not pass in a vercelAccessToken or set the VERCEL_ACCESS_TOKEN env var.",
+      );
+    }
+
+    const environmentMap = {
+      prod: "production",
+      staging: "preview",
+      dev: "development",
+    } as const;
+
+    const vercelEnvironment =
+      environmentMap[ctx.environment as keyof typeof environmentMap];
+
+    if (!vercelEnvironment) {
+      throw new Error(
+        `Invalid environment '${ctx.environment}'. Expected 'prod', 'staging', or 'dev'.`,
+      );
+    }
+    const vercelApiUrl =
+      `https://api.vercel.com/v8/projects/${projectId}/env?decrypt=true`;
+
+    try {
+      const response = await fetch(vercelApiUrl, {
+        headers: {
+          Authorization: `Bearer ${vercelAccessToken}`,
+        },
+      });
+
+      if (!response.ok) {
+        throw new Error(`HTTP error! status: ${response.status}`);
+      }
+
+      const data = await response.json();
+
+      const filteredEnvs = data.envs
+        .filter(
+          (env: { type: string; value: string; target: string[] }) =>
+            env.value &&
+            env.target.includes(vercelEnvironment),
+        )
+        .map((env: { key: string; value: string }) => ({
+          name: env.key,
+          value: env.value,
+        }));
+
+      return filteredEnvs;
+    } catch (error) {
+      console.error(
+        "Error fetching or processing Vercel environment variables:",
+        error,
+      );
+      throw error; // Re-throw the error to be handled by the caller
+    }
+  });
+
+  return {
+    name: "SyncVercelEnvVarsExtension",
+    async onBuildComplete(context, manifest) {
+      await sync.onBuildComplete?.(context, manifest);
+    },
+  };
+}

packages/cli-v3/CHANGELOG.md

@@ -1,5 +1,18 @@
 # trigger.dev
 
+## 3.1.0
+
+### Patch Changes
+
+- Fix issue with prisma extension breaking deploy builds ([#1429](https://github.com/triggerdotdev/trigger.dev/pull/1429))
+- - Include retries.default in task retry config when indexing ([#1424](https://github.com/triggerdotdev/trigger.dev/pull/1424))
+  - New helpers for internal error retry mechanics
+  - Detection for segfaults and ffmpeg OOM errors
+  - Retries for packet import and export
+- Updated dependencies:
+  - `@trigger.dev/core@3.1.0`
+  - `@trigger.dev/build@3.1.0`
+
 ## 3.0.13
 
 ### Patch Changes

packages/cli-v3/package.json

@@ -1,6 +1,6 @@
 {
   "name": "trigger.dev",
-  "version": "3.0.13",
+  "version": "3.1.0",
   "description": "A Command-Line Interface for Trigger.dev (v3) projects",
   "type": "module",
   "license": "MIT",
@@ -87,8 +87,8 @@
     "@opentelemetry/sdk-trace-base": "1.25.1",
     "@opentelemetry/sdk-trace-node": "1.25.1",
     "@opentelemetry/semantic-conventions": "1.25.1",
-    "@trigger.dev/build": "workspace:3.0.13",
-    "@trigger.dev/core": "workspace:3.0.13",
+    "@trigger.dev/build": "workspace:3.1.0",
+    "@trigger.dev/core": "workspace:3.1.0",
     "c12": "^1.11.1",
     "chalk": "^5.2.0",
     "cli-table3": "^0.6.3",

packages/cli-v3/src/deploy/buildImage.ts

@@ -602,6 +602,9 @@ COPY --chown=node:node . .
 
 ${postInstallCommands}
 
+# IMPORTANT: Doing this again to fix an issue with prisma generate removing the files in node_modules/trigger.dev for some reason...
+COPY --chown=node:node . .
+
 FROM build AS indexer
 
 USER node

packages/cli-v3/src/entryPoints/deploy-index-worker.ts

@@ -3,6 +3,7 @@ import {
   type HandleErrorFunction,
   indexerToWorkerMessages,
   taskCatalog,
+  type TaskManifest,
   TriggerConfig,
 } from "@trigger.dev/core/v3";
 import {
@@ -99,6 +100,20 @@ const { buildManifest, importErrors, config } = await bootstrap();
 
 let tasks = taskCatalog.listTaskManifests();
 
+// If the config has retry defaults, we need to apply them to all tasks that don't have any retry settings
+if (config.retries?.default) {
+  tasks = tasks.map((task) => {
+    if (!task.retry) {
+      return {
+        ...task,
+        retry: config.retries?.default,
+      } satisfies TaskManifest;
+    }
+
+    return task;
+  });
+}
+
 // If the config has a machine preset, we need to apply it to all tasks that don't have a machine preset
 if (typeof config.machine === "string") {
   tasks = tasks.map((task) => {
@@ -108,7 +123,7 @@ if (typeof config.machine === "string") {
         machine: {
           preset: config.machine,
         },
-      };
+      } satisfies TaskManifest;
     }
 
     return task;
@@ -122,7 +137,7 @@ if (typeof config.maxDuration === "number") {
       return {
         ...task,
         maxDuration: config.maxDuration,
-      };
+      } satisfies TaskManifest;
     }
 
     return task;

packages/cli-v3/src/entryPoints/deploy-run-controller.ts

@@ -519,12 +519,12 @@ class ProdWorker {
 
     logger.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
 
-    const exitCode =
+    const isNonZeroExitError =
       !completion.ok &&
       completion.error.type === "INTERNAL_ERROR" &&
-      completion.error.code === TaskRunErrorCodes.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE
-        ? EXIT_CODE_CHILD_NONZERO
-        : 0;
+      completion.error.code === TaskRunErrorCodes.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE;
+
+    const exitCode = isNonZeroExitError ? EXIT_CODE_CHILD_NONZERO : 0;
 
     if (shouldExit) {
       // Exit after completion, without any retrying

packages/cli-v3/src/entryPoints/deploy-run-worker.ts

@@ -11,17 +11,19 @@ import {
   TaskRunExecution,
   WorkerToExecutorMessageCatalog,
   TriggerConfig,
-  TriggerTracer,
   WorkerManifest,
   ExecutorToWorkerMessageCatalog,
   timeout,
+  runMetadata,
 } from "@trigger.dev/core/v3";
+import { TriggerTracer } from "@trigger.dev/core/v3/tracer";
 import { ProdRuntimeManager } from "@trigger.dev/core/v3/prod";
 import {
   ConsoleInterceptor,
   DevUsageManager,
   DurableClock,
   getEnvVar,
+  getNumberEnvVar,
   logLevels,
   OtelTaskLogger,
   ProdUsageManager,
@@ -31,6 +33,7 @@ import {
   TracingSDK,
   usage,
   UsageTimeoutManager,
+  StandardMetadataManager,
 } from "@trigger.dev/core/v3/workers";
 import { ZodIpcConnection } from "@trigger.dev/core/v3/zodIpc";
 import { readFile } from "node:fs/promises";
@@ -97,6 +100,8 @@ timeout.setGlobalManager(new UsageTimeoutManager(devUsageManager));
 taskCatalog.setGlobalTaskCatalog(new StandardTaskCatalog());
 const durableClock = new DurableClock();
 clock.setGlobalClock(durableClock);
+const runMetadataManager = new StandardMetadataManager();
+runMetadata.setGlobalManager(runMetadataManager);
 
 const triggerLogLevel = getEnvVar("TRIGGER_LOG_LEVEL");
 
@@ -303,6 +308,10 @@ const zodIpc = new ZodIpcConnection({
           _execution = execution;
           _isRunning = true;
 
+          runMetadataManager.startPeriodicFlush(
+            getNumberEnvVar("TRIGGER_RUN_METADATA_FLUSH_INTERVAL", 1000)
+          );
+
           const measurement = usage.start();
 
           // This lives outside of the executor because this will eventually be moved to the controller level
@@ -397,7 +406,11 @@ const zodIpc = new ZodIpcConnection({
 async function flushAll(timeoutInMs: number = 10_000) {
   const now = performance.now();
 
-  await Promise.all([flushUsage(timeoutInMs), flushTracingSDK(timeoutInMs)]);
+  await Promise.all([
+    flushUsage(timeoutInMs),
+    flushTracingSDK(timeoutInMs),
+    flushMetadata(timeoutInMs),
+  ]);
 
   const duration = performance.now() - now;
 
@@ -424,6 +437,16 @@ async function flushTracingSDK(timeoutInMs: number = 10_000) {
   console.log(`Flushed tracingSDK in ${duration}ms`);
 }
 
+async function flushMetadata(timeoutInMs: number = 10_000) {
+  const now = performance.now();
+
+  await Promise.race([runMetadataManager.flush(), setTimeout(timeoutInMs)]);
+
+  const duration = performance.now() - now;
+
+  console.log(`Flushed runMetadata in ${duration}ms`);
+}
+
 const prodRuntimeManager = new ProdRuntimeManager(zodIpc, {
   waitThresholdInMs: parseInt(env.TRIGGER_RUNTIME_WAIT_THRESHOLD_IN_MS ?? "30000", 10),
 });

packages/cli-v3/src/entryPoints/dev-index-worker.ts

@@ -3,6 +3,7 @@ import {
   type HandleErrorFunction,
   indexerToWorkerMessages,
   taskCatalog,
+  type TaskManifest,
   TriggerConfig,
 } from "@trigger.dev/core/v3";
 import {
@@ -99,6 +100,20 @@ const { buildManifest, importErrors, config } = await bootstrap();
 
 let tasks = taskCatalog.listTaskManifests();
 
+// If the config has retry defaults, we need to apply them to all tasks that don't have any retry settings
+if (config.retries?.default) {
+  tasks = tasks.map((task) => {
+    if (!task.retry) {
+      return {
+        ...task,
+        retry: config.retries?.default,
+      } satisfies TaskManifest;
+    }
+
+    return task;
+  });
+}
+
 // If the config has a maxDuration, we need to apply it to all tasks that don't have a maxDuration
 if (typeof config.maxDuration === "number") {
   tasks = tasks.map((task) => {
@@ -106,7 +121,7 @@ if (typeof config.maxDuration === "number") {
       return {
         ...task,
         maxDuration: config.maxDuration,
-      };
+      } satisfies TaskManifest;
     }
 
     return task;

packages/cli-v3/src/entryPoints/dev-run-worker.ts

@@ -11,11 +11,12 @@ import {
   TaskRunExecution,
   WorkerToExecutorMessageCatalog,
   TriggerConfig,
-  TriggerTracer,
   WorkerManifest,
   ExecutorToWorkerMessageCatalog,
   timeout,
+  runMetadata,
 } from "@trigger.dev/core/v3";
+import { TriggerTracer } from "@trigger.dev/core/v3/tracer";
 import { DevRuntimeManager } from "@trigger.dev/core/v3/dev";
 import {
   ConsoleInterceptor,
@@ -30,6 +31,8 @@ import {
   TracingDiagnosticLogLevel,
   TracingSDK,
   usage,
+  getNumberEnvVar,
+  StandardMetadataManager,
 } from "@trigger.dev/core/v3/workers";
 import { ZodIpcConnection } from "@trigger.dev/core/v3/zodIpc";
 import { readFile } from "node:fs/promises";
@@ -79,6 +82,8 @@ usage.setGlobalUsageManager(devUsageManager);
 const devRuntimeManager = new DevRuntimeManager();
 runtime.setGlobalRuntimeManager(devRuntimeManager);
 timeout.setGlobalManager(new UsageTimeoutManager(devUsageManager));
+const runMetadataManager = new StandardMetadataManager();
+runMetadata.setGlobalManager(runMetadataManager);
 
 const triggerLogLevel = getEnvVar("TRIGGER_LOG_LEVEL");
 
@@ -273,6 +278,9 @@ const zodIpc = new ZodIpcConnection({
         _execution = execution;
         _isRunning = true;
 
+        runMetadataManager.startPeriodicFlush(
+          getNumberEnvVar("TRIGGER_RUN_METADATA_FLUSH_INTERVAL", 1000)
+        );
         const measurement = usage.start();
 
         // This lives outside of the executor because this will eventually be moved to the controller level
@@ -345,7 +353,7 @@ const zodIpc = new ZodIpcConnection({
       }
     },
     FLUSH: async ({ timeoutInMs }, sender) => {
-      await _tracingSDK?.flush();
+      await Promise.allSettled([_tracingSDK?.flush(), runMetadataManager.flush()]);
     },
   },
 });

packages/core/CHANGELOG.md

@@ -1,5 +1,19 @@
 # internal-platform
 
+## 3.1.0
+
+### Minor Changes
+
+- Access run status updates in realtime, from your server or from your frontend ([#1402](https://github.com/triggerdotdev/trigger.dev/pull/1402))
+
+### Patch Changes
+
+- Fix: Handle circular references in flattenAttributes function ([#1433](https://github.com/triggerdotdev/trigger.dev/pull/1433))
+- - Include retries.default in task retry config when indexing ([#1424](https://github.com/triggerdotdev/trigger.dev/pull/1424))
+  - New helpers for internal error retry mechanics
+  - Detection for segfaults and ffmpeg OOM errors
+  - Retries for packet import and export
+
 ## 3.0.13
 
 ### Patch Changes

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/core",
-  "version": "3.0.13",
+  "version": "3.1.0",
   "description": "Core code used across the Trigger.dev SDK and platform",
   "license": "MIT",
   "publishConfig": {
@@ -33,8 +33,10 @@
       "./types": "./src/types.ts",
       "./versions": "./src/versions.ts",
       "./v3": "./src/v3/index.ts",
+      "./v3/tracer": "./src/v3/tracer.ts",
       "./v3/build": "./src/v3/build/index.ts",
       "./v3/apps": "./src/v3/apps/index.ts",
+      "./v3/jwt": "./src/v3/jwt.ts",
       "./v3/errors": "./src/v3/errors.ts",
       "./v3/logger-api": "./src/v3/logger-api.ts",
       "./v3/otel": "./src/v3/otel/index.ts",
@@ -95,6 +97,9 @@
       "v3": [
         "dist/commonjs/v3/index.d.ts"
       ],
+      "v3/tracer": [
+        "dist/commonjs/v3/tracer.d.ts"
+      ],
       "v3/build": [
         "dist/commonjs/v3/build/index.d.ts"
       ],
@@ -160,6 +165,9 @@
       ],
       "v3/schemas": [
         "dist/commonjs/v3/schemas/index.d.ts"
+      ],
+      "v3/jwt": [
+        "dist/commonjs/v3/jwt.d.ts"
       ]
     }
   },
@@ -174,7 +182,9 @@
     "check-exports": "attw --pack ."
   },
   "dependencies": {
+    "@electric-sql/client": "0.6.3",
     "@google-cloud/precise-date": "^4.0.0",
+    "@jsonhero/path": "^1.0.21",
     "@opentelemetry/api": "1.9.0",
     "@opentelemetry/api-logs": "0.52.1",
     "@opentelemetry/exporter-logs-otlp-http": "0.52.1",
@@ -186,8 +196,10 @@
     "@opentelemetry/sdk-trace-base": "1.25.1",
     "@opentelemetry/sdk-trace-node": "1.25.1",
     "@opentelemetry/semantic-conventions": "1.25.1",
+    "dequal": "^2.0.3",
     "execa": "^8.0.1",
     "humanize-duration": "^3.27.3",
+    "jose": "^5.4.0",
     "nanoid": "^3.3.4",
     "socket.io-client": "4.7.5",
     "superjson": "^2.2.1",
@@ -347,6 +359,17 @@
         "default": "./dist/commonjs/v3/index.js"
       }
     },
+    "./v3/tracer": {
+      "import": {
+        "@triggerdotdev/source": "./src/v3/tracer.ts",
+        "types": "./dist/esm/v3/tracer.d.ts",
+        "default": "./dist/esm/v3/tracer.js"
+      },
+      "require": {
+        "types": "./dist/commonjs/v3/tracer.d.ts",
+        "default": "./dist/commonjs/v3/tracer.js"
+      }
+    },
     "./v3/build": {
       "import": {
         "@triggerdotdev/source": "./src/v3/build/index.ts",
@@ -369,6 +392,17 @@
         "default": "./dist/commonjs/v3/apps/index.js"
       }
     },
+    "./v3/jwt": {
+      "import": {
+        "@triggerdotdev/source": "./src/v3/jwt.ts",
+        "types": "./dist/esm/v3/jwt.d.ts",
+        "default": "./dist/esm/v3/jwt.js"
+      },
+      "require": {
+        "types": "./dist/commonjs/v3/jwt.d.ts",
+        "default": "./dist/commonjs/v3/jwt.js"
+      }
+    },
     "./v3/errors": {
       "import": {
         "@triggerdotdev/source": "./src/v3/errors.ts",

packages/core/src/v3/apiClient/core.ts

@@ -6,7 +6,7 @@ import { ApiConnectionError, ApiError, ApiSchemaValidationError } from "./errors
 
 import { Attributes, Span, context, propagation } from "@opentelemetry/api";
 import { SemanticInternalAttributes } from "../semanticInternalAttributes.js";
-import { TriggerTracer } from "../tracer.js";
+import type { TriggerTracer } from "../tracer.js";
 import { accessoryAttributes } from "../utils/styleAttributes.js";
 import {
   CursorPage,
@@ -16,6 +16,7 @@ import {
   OffsetLimitPageParams,
   OffsetLimitPageResponse,
 } from "./pagination.js";
+import { TriggerJwtOptions } from "../types/tasks.js";
 
 export const defaultRetryOptions = {
   maxAttempts: 3,
@@ -35,6 +36,7 @@ export type ZodFetchOptions = {
 };
 
 export type ApiRequestOptions = Pick<ZodFetchOptions, "retry">;
+
 type KeysEnum<T> = { [P in keyof Required<T>]: true };
 
 // This is required so that we can determine if a given object matches the ApiRequestOptions
@@ -227,7 +229,7 @@ async function _doZodFetchWithRetries<TResponseBodySchema extends z.ZodTypeAny>(
       }
     }
 
-    const jsonBody = await response.json();
+    const jsonBody = await safeJsonFromResponse(response);
     const parsedResult = schema.safeParse(jsonBody);
 
     if (parsedResult.success) {
@@ -267,6 +269,14 @@ async function _doZodFetchWithRetries<TResponseBodySchema extends z.ZodTypeAny>(
   }
 }
 
+async function safeJsonFromResponse(response: Response): Promise<any> {
+  try {
+    return await response.clone().json();
+  } catch (error) {
+    return;
+  }
+}
+
 function castToError(err: any): Error {
   if (err instanceof Error) return err;
   return new Error(err);

packages/core/src/v3/apiClient/index.ts

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { VERSION } from "../../version.js";
 import {
   AddTagsRequestBody,
   BatchTaskRunExecutionResult,
@@ -37,25 +38,44 @@ import {
   zodfetchOffsetLimitPage,
 } from "./core.js";
 import { ApiError } from "./errors.js";
+import {
+  RunShape,
+  AnyRunShape,
+  runShapeStream,
+  RunStreamCallback,
+  RunSubscription,
+  TaskRunShape,
+} from "./runStream.js";
 import {
   CreateEnvironmentVariableParams,
   ImportEnvironmentVariablesParams,
   ListProjectRunsQueryParams,
   ListRunsQueryParams,
+  SubscribeToRunsQueryParams,
   UpdateEnvironmentVariableParams,
 } from "./types.js";
-import { VERSION } from "../../version.js";
+import { generateJWT } from "../jwt.js";
+import { AnyRunTypes, TriggerJwtOptions } from "../types/tasks.js";
 
 export type {
   CreateEnvironmentVariableParams,
   ImportEnvironmentVariablesParams,
   UpdateEnvironmentVariableParams,
+  SubscribeToRunsQueryParams,
 };
 
 export type TriggerOptions = {
   spanParentAsLink?: boolean;
 };
 
+export type TriggerRequestOptions = ZodFetchOptions & {
+  publicAccessToken?: TriggerJwtOptions;
+};
+
+export type TriggerApiRequestOptions = ApiRequestOptions & {
+  publicAccessToken?: TriggerJwtOptions;
+};
+
 const DEFAULT_ZOD_FETCH_OPTIONS: ZodFetchOptions = {
   retry: {
     maxAttempts: 3,
@@ -68,23 +88,40 @@ const DEFAULT_ZOD_FETCH_OPTIONS: ZodFetchOptions = {
 
 export { isRequestOptions };
 export type { ApiRequestOptions };
+export type { RunShape, AnyRunShape, TaskRunShape, RunStreamCallback, RunSubscription };
 
 /**
  * Trigger.dev v3 API client
  */
 export class ApiClient {
-  private readonly baseUrl: string;
+  public readonly baseUrl: string;
+  public readonly accessToken: string;
   private readonly defaultRequestOptions: ZodFetchOptions;
 
-  constructor(
-    baseUrl: string,
-    private readonly accessToken: string,
-    requestOptions: ApiRequestOptions = {}
-  ) {
+  constructor(baseUrl: string, accessToken: string, requestOptions: ApiRequestOptions = {}) {
+    this.accessToken = accessToken;
     this.baseUrl = baseUrl.replace(/\/$/, "");
     this.defaultRequestOptions = mergeRequestOptions(DEFAULT_ZOD_FETCH_OPTIONS, requestOptions);
   }
 
+  get fetchClient(): typeof fetch {
+    const headers = this.#getHeaders(false);
+
+    const fetchClient: typeof fetch = (input, requestInit) => {
+      const $requestInit: RequestInit = {
+        ...requestInit,
+        headers: {
+          ...requestInit?.headers,
+          ...headers,
+        },
+      };
+
+      return fetch(input, $requestInit);
+    };
+
+    return fetchClient;
+  }
+
   async getRunResult(
     runId: string,
     requestOptions?: ZodFetchOptions
@@ -129,7 +166,7 @@ export class ApiClient {
     taskId: string,
     body: TriggerTaskRequestBody,
     options?: TriggerOptions,
-    requestOptions?: ZodFetchOptions
+    requestOptions?: TriggerRequestOptions
   ) {
     const encodedTaskId = encodeURIComponent(taskId);
 
@@ -142,14 +179,35 @@ export class ApiClient {
         body: JSON.stringify(body),
       },
       mergeRequestOptions(this.defaultRequestOptions, requestOptions)
-    );
+    )
+      .withResponse()
+      .then(async ({ response, data }) => {
+        const claimsHeader = response.headers.get("x-trigger-jwt-claims");
+        const claims = claimsHeader ? JSON.parse(claimsHeader) : undefined;
+
+        const jwt = await generateJWT({
+          secretKey: this.accessToken,
+          payload: {
+            ...claims,
+            scopes: [`read:runs:${data.id}`].concat(
+              body.options?.tags ? Array.from(body.options?.tags).map((t) => `read:tags:${t}`) : []
+            ),
+          },
+          expirationTime: requestOptions?.publicAccessToken?.expirationTime ?? "1h",
+        });
+
+        return {
+          ...data,
+          publicAccessToken: jwt,
+        };
+      });
   }
 
   batchTriggerTask(
     taskId: string,
     body: BatchTriggerTaskRequestBody,
     options?: TriggerOptions,
-    requestOptions?: ZodFetchOptions
+    requestOptions?: TriggerRequestOptions
   ) {
     const encodedTaskId = encodeURIComponent(taskId);
 
@@ -162,7 +220,26 @@ export class ApiClient {
         body: JSON.stringify(body),
       },
       mergeRequestOptions(this.defaultRequestOptions, requestOptions)
-    );
+    )
+      .withResponse()
+      .then(async ({ response, data }) => {
+        const claimsHeader = response.headers.get("x-trigger-jwt-claims");
+        const claims = claimsHeader ? JSON.parse(claimsHeader) : undefined;
+
+        const jwt = await generateJWT({
+          secretKey: this.accessToken,
+          payload: {
+            ...claims,
+            scopes: [`read:batch:${data.batchId}`],
+          },
+          expirationTime: requestOptions?.publicAccessToken?.expirationTime ?? "1h",
+        });
+
+        return {
+          ...data,
+          publicAccessToken: jwt,
+        };
+      });
   }
 
   createUploadPayloadUrl(filename: string, requestOptions?: ZodFetchOptions) {
@@ -517,6 +594,46 @@ export class ApiClient {
     );
   }
 
+  subscribeToRun<TRunTypes extends AnyRunTypes>(runId: string) {
+    return runShapeStream<TRunTypes>(`${this.baseUrl}/realtime/v1/runs/${runId}`, {
+      closeOnComplete: true,
+      headers: this.#getRealtimeHeaders(),
+    });
+  }
+
+  subscribeToRunsWithTag<TRunTypes extends AnyRunTypes>(tag: string | string[]) {
+    const searchParams = createSearchQueryForSubscribeToRuns({
+      tags: tag,
+    });
+
+    return runShapeStream<TRunTypes>(
+      `${this.baseUrl}/realtime/v1/runs${searchParams ? `?${searchParams}` : ""}`,
+      {
+        closeOnComplete: false,
+        headers: this.#getRealtimeHeaders(),
+      }
+    );
+  }
+
+  subscribeToBatch<TRunTypes extends AnyRunTypes>(batchId: string) {
+    return runShapeStream<TRunTypes>(`${this.baseUrl}/realtime/v1/batches/${batchId}`, {
+      closeOnComplete: false,
+      headers: this.#getRealtimeHeaders(),
+    });
+  }
+
+  async generateJWTClaims(requestOptions?: ZodFetchOptions): Promise<Record<string, any>> {
+    return zodfetch(
+      z.record(z.any()),
+      `${this.baseUrl}/api/v1/auth/jwt/claims`,
+      {
+        method: "POST",
+        headers: this.#getHeaders(false),
+      },
+      mergeRequestOptions(this.defaultRequestOptions, requestOptions)
+    );
+  }
+
   #getHeaders(spanParentAsLink: boolean) {
     const headers: Record<string, string> = {
       "Content-Type": "application/json",
@@ -535,6 +652,34 @@ export class ApiClient {
 
     return headers;
   }
+
+  #getRealtimeHeaders() {
+    const headers: Record<string, string> = {
+      Authorization: `Bearer ${this.accessToken}`,
+      "trigger-version": VERSION,
+    };
+
+    return headers;
+  }
+}
+
+function createSearchQueryForSubscribeToRuns(query?: SubscribeToRunsQueryParams): URLSearchParams {
+  const searchParams = new URLSearchParams();
+
+  if (query) {
+    if (query.tasks) {
+      searchParams.append(
+        "tasks",
+        Array.isArray(query.tasks) ? query.tasks.join(",") : query.tasks
+      );
+    }
+
+    if (query.tags) {
+      searchParams.append("tags", Array.isArray(query.tags) ? query.tags.join(",") : query.tags);
+    }
+  }
+
+  return searchParams;
 }
 
 function createSearchQueryForListRuns(query?: ListRunsQueryParams): URLSearchParams {