Merge branch 'main' into nicer-app-emails

.changeset/angry-trainers-perform.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Stop failing attempt spans when a run is cancelled

.changeset/kind-kids-teach.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Added INSTALLING status to the deployment status enum.

CHANGESETS.md

@@ -30,14 +30,16 @@ Please follow the best-practice of adding changesets in the same commit as the c
 
 ## Snapshot instructions
 
-1. Delete the `.changeset/pre.json` file (if it exists)
+1. Update the `.changeset/config.json` file to set the `"changelog"` field to this:
+
+```json
+"changelog": "@changesets/cli/changelog",
+```
 
 2. Do a temporary commit (do NOT push this, you should undo it after)
 
-3. Copy the `GITHUB_TOKEN` line from the .env file
+3. Run `./scripts/publish-prerelease.sh prerelease`
 
-4. Run `GITHUB_TOKEN=github_pat_12345 ./scripts/publish-prerelease.sh re2`
+You can choose a different tag if you want, but usually `prerelease` is fine.
 
-Make sure to replace the token with yours. `re2` is the tag that will be used for the pre-release.
-
-5. Undo the commit where you deleted the pre.json file.
+5. Undo the commit where you updated the config.json file.

apps/supervisor/src/env.ts

@@ -89,6 +89,7 @@ const Env = z.object({
   KUBERNETES_CPU_REQUEST_RATIO: z.coerce.number().min(0).max(1).default(0.75), // Ratio of CPU limit, so 0.75 = 75% of CPU limit
   KUBERNETES_MEMORY_REQUEST_MIN_GB: z.coerce.number().min(0).default(0),
   KUBERNETES_MEMORY_REQUEST_RATIO: z.coerce.number().min(0).max(1).default(1), // Ratio of memory limit, so 1 = 100% of memory limit
+  KUBERNETES_MEMORY_OVERHEAD_GB: z.coerce.number().min(0).optional(), // Optional memory overhead to add to the limit in GB
 
   // Placement tags settings
   PLACEMENT_TAGS_ENABLED: BoolEnv.default(false),

apps/supervisor/src/workloadManager/kubernetes.ts

@@ -25,6 +25,7 @@ export class KubernetesWorkloadManager implements WorkloadManager {
   private readonly cpuRequestRatio = env.KUBERNETES_CPU_REQUEST_RATIO;
   private readonly memoryRequestMinGb = env.KUBERNETES_MEMORY_REQUEST_MIN_GB;
   private readonly memoryRequestRatio = env.KUBERNETES_MEMORY_REQUEST_RATIO;
+  private readonly memoryOverheadGb = env.KUBERNETES_MEMORY_OVERHEAD_GB;
 
   constructor(private opts: WorkloadManagerOptions) {
     this.k8s = createK8sApi();
@@ -319,9 +320,13 @@ export class KubernetesWorkloadManager implements WorkloadManager {
   }
 
   #getResourceLimitsForMachine(preset: MachinePreset): ResourceQuantities {
+    const memoryLimit = this.memoryOverheadGb
+      ? preset.memory + this.memoryOverheadGb
+      : preset.memory;
+
     return {
       cpu: `${preset.cpu}`,
-      memory: `${preset.memory}G`,
+      memory: `${memoryLimit}G`,
     };
   }
 

apps/supervisor/src/workloadServer/index.ts

@@ -16,7 +16,6 @@ import {
   type WorkloadRunAttemptCompleteResponseBody,
   WorkloadRunAttemptStartRequestBody,
   type WorkloadRunAttemptStartResponseBody,
-  type WorkloadRunLatestSnapshotResponseBody,
   WorkloadRunSnapshotsSinceResponseBody,
   type WorkloadServerToClientEvents,
   type WorkloadSuspendRunResponseBody,
@@ -126,7 +125,7 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
   }
 
   private createHttpServer({ host, port }: { host: string; port: number }) {
-    return new HttpServer({
+    const httpServer = new HttpServer({
       port,
       host,
       metrics: {
@@ -322,28 +321,6 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
           },
         }
       )
-      .route("/api/v1/workload-actions/runs/:runFriendlyId/snapshots/latest", "GET", {
-        paramsSchema: WorkloadActionParams.pick({ runFriendlyId: true }),
-        handler: async ({ req, reply, params }) => {
-          const latestSnapshotResponse = await this.workerClient.getLatestSnapshot(
-            params.runFriendlyId,
-            this.runnerIdFromRequest(req)
-          );
-
-          if (!latestSnapshotResponse.success) {
-            this.logger.error("Failed to get latest snapshot", {
-              runId: params.runFriendlyId,
-              error: latestSnapshotResponse.error,
-            });
-            reply.empty(500);
-            return;
-          }
-
-          reply.json({
-            execution: latestSnapshotResponse.data.execution,
-          } satisfies WorkloadRunLatestSnapshotResponseBody);
-        },
-      })
       .route(
         "/api/v1/workload-actions/runs/:runFriendlyId/snapshots/since/:snapshotFriendlyId",
         "GET",
@@ -369,23 +346,6 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
           },
         }
       )
-      .route("/api/v1/workload-actions/runs/:runFriendlyId/logs/debug", "POST", {
-        paramsSchema: WorkloadActionParams.pick({ runFriendlyId: true }),
-        bodySchema: WorkloadDebugLogRequestBody,
-        handler: async ({ req, reply, params, body }) => {
-          reply.empty(204);
-
-          if (!env.SEND_RUN_DEBUG_LOGS) {
-            return;
-          }
-
-          await this.workerClient.sendDebugLog(
-            params.runFriendlyId,
-            body,
-            this.runnerIdFromRequest(req)
-          );
-        },
-      })
       .route("/api/v1/workload-actions/deployments/:deploymentId/dequeue", "GET", {
         paramsSchema: z.object({
           deploymentId: z.string(),
@@ -410,6 +370,31 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
           reply.json(dequeueResponse.data satisfies WorkloadDequeueFromVersionResponseBody);
         },
       });
+
+    if (env.SEND_RUN_DEBUG_LOGS) {
+      httpServer.route("/api/v1/workload-actions/runs/:runFriendlyId/logs/debug", "POST", {
+        paramsSchema: WorkloadActionParams.pick({ runFriendlyId: true }),
+        bodySchema: WorkloadDebugLogRequestBody,
+        handler: async ({ req, reply, params, body }) => {
+          reply.empty(204);
+
+          await this.workerClient.sendDebugLog(
+            params.runFriendlyId,
+            body,
+            this.runnerIdFromRequest(req)
+          );
+        },
+      });
+    } else {
+      // Lightweight mock route without schemas
+      httpServer.route("/api/v1/workload-actions/runs/:runFriendlyId/logs/debug", "POST", {
+        handler: async ({ reply }) => {
+          reply.empty(204);
+        },
+      });
+    }
+
+    return httpServer;
   }
 
   private createWebsocketServer() {

apps/webapp/app/assets/icons/MoveToTopIcon.tsx

@@ -0,0 +1,34 @@
+export function MoveToTopIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <g clipPath="url(#clip0_17186_103975)">
+        <path
+          d="M12 21L12 9"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M3 3L21 3"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M16.5 11.5L12 7L7.5 11.5"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+      </g>
+      <defs>
+        <clipPath id="clip0_17186_103975">
+          <rect width="24" height="24" fill="currentColor" />
+        </clipPath>
+      </defs>
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/MoveUpIcon.tsx

@@ -0,0 +1,41 @@
+export function MoveUpIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <g clipPath="url(#clip0_17177_110851)">
+        <path
+          d="M12 21L12 13"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M3 3L21 3"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M3 7L21 7"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M16.5 15.5L12 11L7.5 15.5"
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeLinecap="round"
+          strokeLinejoin="round"
+        />
+      </g>
+      <defs>
+        <clipPath id="clip0_17177_110851">
+          <rect width="24" height="24" fill="currentColor" />
+        </clipPath>
+      </defs>
+    </svg>
+  );
+}

apps/webapp/app/components/DefinitionTooltip.tsx

@@ -14,7 +14,7 @@ export function DefinitionTip({
   return (
     <TooltipProvider>
       <Tooltip disableHoverableContent>
-        <TooltipTrigger>
+        <TooltipTrigger className="text-left">
           <span className="cursor-default underline decoration-charcoal-500 decoration-dashed underline-offset-4 transition hover:decoration-charcoal-400">
             {children}
           </span>

apps/webapp/app/components/Shortcuts.tsx

@@ -147,6 +147,12 @@ function ShortcutContent() {
               </Paragraph>
               <ShortcutKey shortcut={{ key: "9" }} variant="medium/bright" />
             </Shortcut>
+            <Shortcut name="Jump to root run">
+              <ShortcutKey shortcut={{ key: "t" }} variant="medium/bright" />
+            </Shortcut>
+            <Shortcut name="Jump to parent run">
+              <ShortcutKey shortcut={{ key: "p" }} variant="medium/bright" />
+            </Shortcut>
           </div>
           <div className="space-y-3">
             <Header3>Schedules page</Header3>

apps/webapp/app/components/primitives/CopyableText.tsx

@@ -8,10 +8,12 @@ export function CopyableText({
   value,
   copyValue,
   className,
+  asChild,
 }: {
   value: string;
   copyValue?: string;
   className?: string;
+  asChild?: boolean;
 }) {
   const [isHovered, setIsHovered] = useState(false);
   const { copy, copied } = useCopy(copyValue ?? value);
@@ -35,6 +37,7 @@ export function CopyableText({
             <span
               className={cn(
                 "ml-1 flex size-6 items-center justify-center rounded border border-charcoal-650 bg-charcoal-750",
+                asChild && "p-1",
                 copied
                   ? "text-green-500"
                   : "text-text-dimmed hover:border-charcoal-600 hover:bg-charcoal-700 hover:text-text-bright"
@@ -50,6 +53,7 @@ export function CopyableText({
           content={copied ? "Copied!" : "Copy"}
           className="font-sans"
           disableHoverableContent
+          asChild={asChild}
         />
       </span>
     </span>

apps/webapp/app/components/primitives/TextLink.tsx

@@ -1,6 +1,10 @@
 import { Link } from "@remix-run/react";
 import { cn } from "~/utils/cn";
 import { Icon, type RenderIcon } from "./Icon";
+import { useRef } from "react";
+import { type ShortcutDefinition, useShortcutKeys } from "~/hooks/useShortcutKeys";
+import { ShortcutKey } from "./ShortcutKey";
+import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "./Tooltip";
 
 const variations = {
   primary:
@@ -17,6 +21,9 @@ type TextLinkProps = {
   trailingIconClassName?: string;
   variant?: keyof typeof variations;
   children: React.ReactNode;
+  shortcut?: ShortcutDefinition;
+  hideShortcutKey?: boolean;
+  tooltip?: React.ReactNode;
 } & React.AnchorHTMLAttributes<HTMLAnchorElement>;
 
 export function TextLink({
@@ -27,20 +34,61 @@ export function TextLink({
   trailingIcon,
   trailingIconClassName,
   variant = "primary",
+  shortcut,
+  hideShortcutKey,
+  tooltip,
   ...props
 }: TextLinkProps) {
+  const innerRef = useRef<HTMLAnchorElement>(null);
   const classes = variations[variant];
-  return to ? (
-    <Link to={to} className={cn(classes, className)} {...props}>
+
+  if (shortcut) {
+    useShortcutKeys({
+      shortcut: shortcut,
+      action: () => {
+        if (innerRef.current) {
+          innerRef.current.click();
+        }
+      },
+    });
+  }
+
+  const renderShortcutKey = () =>
+    shortcut &&
+    !hideShortcutKey && <ShortcutKey className="ml-1.5" shortcut={shortcut} variant="small" />;
+
+  const linkContent = (
+    <>
       {children}{" "}
       {trailingIcon && <Icon icon={trailingIcon} className={cn("size-4", trailingIconClassName)} />}
+      {shortcut && !tooltip && renderShortcutKey()}
+    </>
+  );
+
+  const linkElement = to ? (
+    <Link ref={innerRef} to={to} className={cn(classes, className)} {...props}>
+      {linkContent}
     </Link>
   ) : href ? (
-    <a href={href} className={cn(classes, className)} {...props}>
-      {children}{" "}
-      {trailingIcon && <Icon icon={trailingIcon} className={cn("size-4", trailingIconClassName)} />}
+    <a ref={innerRef} href={href} className={cn(classes, className)} {...props}>
+      {linkContent}
     </a>
   ) : (
     <span>Need to define a path or href</span>
   );
+
+  if (tooltip) {
+    return (
+      <TooltipProvider>
+        <Tooltip>
+          <TooltipTrigger asChild>{linkElement}</TooltipTrigger>
+          <TooltipContent className="text-dimmed flex items-center gap-3 py-1.5 pl-2.5 pr-3 text-xs">
+            {tooltip} {shortcut && renderShortcutKey()}
+          </TooltipContent>
+        </Tooltip>
+      </TooltipProvider>
+    );
+  }
+
+  return linkElement;
 }

apps/webapp/app/components/runs/v3/DeploymentStatus.tsx

@@ -2,6 +2,7 @@ import {
   CheckCircleIcon,
   ExclamationTriangleIcon,
   NoSymbolIcon,
+  RectangleStackIcon,
   XCircleIcon,
 } from "@heroicons/react/20/solid";
 import type { WorkerDeploymentStatus } from "@trigger.dev/database";
@@ -49,6 +50,10 @@ export function DeploymentStatusIcon({
 }) {
   switch (status) {
     case "PENDING":
+      return (
+        <RectangleStackIcon className={cn(deploymentStatusClassNameColor(status), className)} />
+      );
+    case "INSTALLING":
     case "BUILDING":
     case "DEPLOYING":
       return <Spinner className={cn(deploymentStatusClassNameColor(status), className)} />;
@@ -73,6 +78,8 @@ export function DeploymentStatusIcon({
 export function deploymentStatusClassNameColor(status: WorkerDeploymentStatus): string {
   switch (status) {
     case "PENDING":
+      return "text-charcoal-500";
+    case "INSTALLING":
     case "BUILDING":
     case "DEPLOYING":
       return "text-pending";
@@ -92,7 +99,9 @@ export function deploymentStatusClassNameColor(status: WorkerDeploymentStatus):
 export function deploymentStatusTitle(status: WorkerDeploymentStatus, isBuilt: boolean): string {
   switch (status) {
     case "PENDING":
-      return "Pending…";
+      return "Queued…";
+    case "INSTALLING":
+      return "Installing…";
     case "BUILDING":
       return "Building…";
     case "DEPLOYING":
@@ -121,17 +130,22 @@ export function deploymentStatusTitle(status: WorkerDeploymentStatus, isBuilt: b
 
 // PENDING and CANCELED are not used so are ommited from the UI
 export const deploymentStatuses: WorkerDeploymentStatus[] = [
+  "PENDING",
+  "INSTALLING",
   "BUILDING",
   "DEPLOYING",
   "DEPLOYED",
   "FAILED",
   "TIMED_OUT",
+  "CANCELED",
 ];
 
 export function deploymentStatusDescription(status: WorkerDeploymentStatus): string {
   switch (status) {
     case "PENDING":
       return "The deployment is queued and waiting to be processed.";
+    case "INSTALLING":
+      return "The project dependencies are being installed.";
     case "BUILDING":
       return "The code is being built and prepared for deployment.";
     case "DEPLOYING":

apps/webapp/app/components/runs/v3/RollbackDeploymentDialog.tsx

@@ -1,102 +0,0 @@
-import { ArrowPathIcon } from "@heroicons/react/20/solid";
-import { DialogClose } from "@radix-ui/react-dialog";
-import { Form, useNavigation } from "@remix-run/react";
-import { Button } from "~/components/primitives/Buttons";
-import {
-  DialogContent,
-  DialogDescription,
-  DialogFooter,
-  DialogHeader,
-} from "~/components/primitives/Dialog";
-import { SpinnerWhite } from "~/components/primitives/Spinner";
-
-type RollbackDeploymentDialogProps = {
-  projectId: string;
-  deploymentShortCode: string;
-  redirectPath: string;
-};
-
-export function RollbackDeploymentDialog({
-  projectId,
-  deploymentShortCode,
-  redirectPath,
-}: RollbackDeploymentDialogProps) {
-  const navigation = useNavigation();
-
-  const formAction = `/resources/${projectId}/deployments/${deploymentShortCode}/rollback`;
-  const isLoading = navigation.formAction === formAction;
-
-  return (
-    <DialogContent key="rollback">
-      <DialogHeader>Rollback to this deployment?</DialogHeader>
-      <DialogDescription>
-        This deployment will become the default for all future runs. Tasks triggered but not
-        included in this deploy will remain queued until you roll back to or create a new deployment
-        with these tasks included.
-      </DialogDescription>
-      <DialogFooter>
-        <DialogClose asChild>
-          <Button variant="tertiary/medium">Cancel</Button>
-        </DialogClose>
-        <Form
-          action={`/resources/${projectId}/deployments/${deploymentShortCode}/rollback`}
-          method="post"
-        >
-          <Button
-            type="submit"
-            name="redirectUrl"
-            value={redirectPath}
-            variant="primary/medium"
-            LeadingIcon={isLoading ? SpinnerWhite : ArrowPathIcon}
-            disabled={isLoading}
-            shortcut={{ modifiers: ["mod"], key: "enter" }}
-          >
-            {isLoading ? "Rolling back..." : "Rollback deployment"}
-          </Button>
-        </Form>
-      </DialogFooter>
-    </DialogContent>
-  );
-}
-
-export function PromoteDeploymentDialog({
-  projectId,
-  deploymentShortCode,
-  redirectPath,
-}: RollbackDeploymentDialogProps) {
-  const navigation = useNavigation();
-
-  const formAction = `/resources/${projectId}/deployments/${deploymentShortCode}/promote`;
-  const isLoading = navigation.formAction === formAction;
-
-  return (
-    <DialogContent key="promote">
-      <DialogHeader>Promote this deployment?</DialogHeader>
-      <DialogDescription>
-        This deployment will become the default for all future runs not explicitly tied to a
-        specific deployment.
-      </DialogDescription>
-      <DialogFooter>
-        <DialogClose asChild>
-          <Button variant="tertiary/medium">Cancel</Button>
-        </DialogClose>
-        <Form
-          action={`/resources/${projectId}/deployments/${deploymentShortCode}/promote`}
-          method="post"
-        >
-          <Button
-            type="submit"
-            name="redirectUrl"
-            value={redirectPath}
-            variant="primary/medium"
-            LeadingIcon={isLoading ? SpinnerWhite : ArrowPathIcon}
-            disabled={isLoading}
-            shortcut={{ modifiers: ["mod"], key: "enter" }}
-          >
-            {isLoading ? "Promoting..." : "Promote deployment"}
-          </Button>
-        </Form>
-      </DialogFooter>
-    </DialogContent>
-  );
-}

apps/webapp/app/env.server.ts

@@ -312,6 +312,10 @@ const EnvironmentSchema = z
       .number()
       .int()
       .default(60 * 1000 * 8), // 8 minutes
+    DEPLOY_QUEUE_TIMEOUT_MS: z.coerce
+      .number()
+      .int()
+      .default(60 * 1000 * 15), // 15 minutes
 
     OBJECT_STORE_BASE_URL: z.string().optional(),
     OBJECT_STORE_ACCESS_KEY_ID: z.string().optional(),
@@ -519,8 +523,8 @@ const EnvironmentSchema = z
     RUN_ENGINE_WORKER_IMMEDIATE_POLL_INTERVAL: z.coerce.number().int().default(100),
     RUN_ENGINE_TIMEOUT_PENDING_EXECUTING: z.coerce.number().int().default(60_000),
     RUN_ENGINE_TIMEOUT_PENDING_CANCEL: z.coerce.number().int().default(60_000),
-    RUN_ENGINE_TIMEOUT_EXECUTING: z.coerce.number().int().default(60_000),
-    RUN_ENGINE_TIMEOUT_EXECUTING_WITH_WAITPOINTS: z.coerce.number().int().default(60_000),
+    RUN_ENGINE_TIMEOUT_EXECUTING: z.coerce.number().int().default(300_000), // 5 minutes
+    RUN_ENGINE_TIMEOUT_EXECUTING_WITH_WAITPOINTS: z.coerce.number().int().default(300_000), // 5 minutes
     RUN_ENGINE_TIMEOUT_SUSPENDED: z.coerce
       .number()
       .int()
@@ -735,6 +739,7 @@ const EnvironmentSchema = z
     RUN_ENGINE_RUN_QUEUE_LOG_LEVEL: z
       .enum(["log", "error", "warn", "info", "debug"])
       .default("info"),
+    RUN_ENGINE_TREAT_PRODUCTION_EXECUTION_STALLS_AS_OOM: z.string().default("0"),
 
     /** How long should the presence ttl last */
     DEV_PRESENCE_SSE_TIMEOUT: z.coerce.number().int().default(30_000),
@@ -1023,8 +1028,9 @@ const EnvironmentSchema = z
     TASK_EVENT_PARTITIONING_ENABLED: z.string().default("0"),
     TASK_EVENT_PARTITIONED_WINDOW_IN_SECONDS: z.coerce.number().int().default(60), // 1 minute
 
-    QUEUE_SSE_AUTORELOAD_INTERVAL_MS: z.coerce.number().int().default(5_000),
-    QUEUE_SSE_AUTORELOAD_TIMEOUT_MS: z.coerce.number().int().default(60_000),
+    DEPLOYMENTS_AUTORELOAD_POLL_INTERVAL_MS: z.coerce.number().int().default(5_000),
+    BULK_ACTION_AUTORELOAD_POLL_INTERVAL_MS: z.coerce.number().int().default(1_000),
+    QUEUES_AUTORELOAD_POLL_INTERVAL_MS: z.coerce.number().int().default(5_000),
 
     SLACK_BOT_TOKEN: z.string().optional(),
     SLACK_SIGNUP_REASON_CHANNEL_ID: z.string().optional(),

apps/webapp/app/hooks/useAutoRevalidate.ts

@@ -0,0 +1,48 @@
+import { useRevalidator } from "@remix-run/react";
+import { useEffect } from "react";
+
+type UseAutoRevalidateOptions = {
+  interval?: number; // in milliseconds
+  onFocus?: boolean;
+  disabled?: boolean;
+};
+
+export function useAutoRevalidate(options: UseAutoRevalidateOptions = {}) {
+  const { interval = 5000, onFocus = true, disabled = false } = options;
+  const revalidator = useRevalidator();
+
+  useEffect(() => {
+    if (!interval || interval <= 0 || disabled) return;
+
+    const intervalId = setInterval(() => {
+      if (revalidator.state === "loading") {
+        return;
+      }
+      revalidator.revalidate();
+    }, interval);
+
+    return () => clearInterval(intervalId);
+  }, [interval, disabled]);
+
+  useEffect(() => {
+    if (!onFocus || disabled) return;
+
+    const handleFocus = () => {
+      if (document.visibilityState === "visible" && revalidator.state !== "loading") {
+        revalidator.revalidate();
+      }
+    };
+
+    // Revalidate when the page becomes visible
+    document.addEventListener("visibilitychange", handleFocus);
+    // Revalidate when the window gains focus
+    window.addEventListener("focus", handleFocus);
+
+    return () => {
+      document.removeEventListener("visibilitychange", handleFocus);
+      window.removeEventListener("focus", handleFocus);
+    };
+  }, [onFocus, disabled]);
+
+  return revalidator;
+}

apps/webapp/app/presenters/v3/DeploymentListPresenter.server.ts

@@ -1,5 +1,5 @@
 import {
-  Prisma,
+  type Prisma,
   type WorkerDeploymentStatus,
   type WorkerInstanceGroupType,
 } from "@trigger.dev/database";
@@ -9,6 +9,7 @@ import { type Project } from "~/models/project.server";
 import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
 import { type User } from "~/models/user.server";
 import { processGitMetadata } from "./BranchesPresenter.server";
+import { BranchTrackingConfigSchema, getTrackedBranchForEnvironment } from "~/v3/github";
 
 const pageSize = 20;
 
@@ -56,6 +57,18 @@ export class DeploymentListPresenter {
             },
           },
         },
+        connectedGithubRepository: {
+          select: {
+            branchTracking: true,
+            previewDeploymentsEnabled: true,
+            repository: {
+              select: {
+                htmlUrl: true,
+                fullName: true,
+              },
+            },
+          },
+        },
       },
       where: {
         slug: projectSlug,
@@ -131,7 +144,7 @@ export class DeploymentListPresenter {
   wd."git"
 FROM
   ${sqlDatabaseSchema}."WorkerDeployment" as wd
-INNER JOIN
+LEFT JOIN
   ${sqlDatabaseSchema}."User" as u ON wd."triggeredById" = u."id"
 WHERE
   wd."projectId" = ${project.id}
@@ -140,9 +153,28 @@ ORDER BY
   string_to_array(wd."version", '.')::int[] DESC
 LIMIT ${pageSize} OFFSET ${pageSize * (page - 1)};`;
 
+    const { connectedGithubRepository } = project;
+
+    const branchTrackingOrError =
+      connectedGithubRepository &&
+      BranchTrackingConfigSchema.safeParse(connectedGithubRepository.branchTracking);
+    const environmentGitHubBranch =
+      branchTrackingOrError && branchTrackingOrError.success
+        ? getTrackedBranchForEnvironment(
+            branchTrackingOrError.data,
+            connectedGithubRepository.previewDeploymentsEnabled,
+            {
+              type: environment.type,
+              branchName: environment.branchName ?? undefined,
+            }
+          )
+        : undefined;
+
     return {
       currentPage: page,
       totalPages: Math.ceil(totalCount / pageSize),
+      connectedGithubRepository: project.connectedGithubRepository ?? undefined,
+      environmentGitHubBranch,
       deployments: deployments.map((deployment, index) => {
         const label = labeledDeployments.find(
           (labeledDeployment) => labeledDeployment.deploymentId === deployment.id

apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts

@@ -102,6 +102,10 @@ export class DeploymentPresenter {
         builtAt: true,
         deployedAt: true,
         createdAt: true,
+        startedAt: true,
+        installedAt: true,
+        canceledAt: true,
+        canceledReason: true,
         git: true,
         promotions: {
           select: {
@@ -145,8 +149,12 @@ export class DeploymentPresenter {
         version: deployment.version,
         status: deployment.status,
         createdAt: deployment.createdAt,
+        startedAt: deployment.startedAt,
+        installedAt: deployment.installedAt,
         builtAt: deployment.builtAt,
         deployedAt: deployment.deployedAt,
+        canceledAt: deployment.canceledAt,
+        canceledReason: deployment.canceledReason,
         tasks: deployment.worker?.tasks,
         label: deployment.promotions?.[0]?.label,
         environment: {

apps/webapp/app/presenters/v3/RunPresenter.server.ts

@@ -1,6 +1,6 @@
 import { millisecondsToNanoseconds } from "@trigger.dev/core/v3";
 import { createTreeFromFlatItems, flattenTree } from "~/components/primitives/TreeView/TreeView";
-import { prisma, PrismaClient } from "~/db.server";
+import { prisma, type PrismaClient } from "~/db.server";
 import { createTimelineSpanEventsFromSpanEvents } from "~/utils/timelineSpanEvents";
 import { getUsername } from "~/utils/username";
 import { eventRepository } from "~/v3/eventRepository.server";
@@ -58,7 +58,13 @@ export class RunPresenter {
         rootTaskRun: {
           select: {
             friendlyId: true,
-            taskIdentifier: true,
+            spanId: true,
+            createdAt: true,
+          },
+        },
+        parentTaskRun: {
+          select: {
+            friendlyId: true,
             spanId: true,
             createdAt: true,
           },
@@ -111,6 +117,7 @@ export class RunPresenter {
       completedAt: run.completedAt,
       logsDeletedAt: showDeletedLogs ? null : run.logsDeletedAt,
       rootTaskRun: run.rootTaskRun,
+      parentTaskRun: run.parentTaskRun,
       environment: {
         id: run.runtimeEnvironment.id,
         organizationId: run.runtimeEnvironment.organizationId,
@@ -202,8 +209,6 @@ export class RunPresenter {
       trace: {
         rootSpanStatus,
         events: events,
-        parentRunFriendlyId:
-          tree?.id === traceSummary.rootSpan.id ? undefined : traceSummary.rootSpan.runId,
         duration: totalDuration,
         rootStartedAt: tree?.data.startTime,
         startedAt: run.startedAt,

apps/webapp/app/presenters/v3/SpanPresenter.server.ts

@@ -251,6 +251,7 @@ export class SpanPresenter extends BasePresenter {
       engine: run.engine,
       region,
       workerQueue: run.workerQueue,
+      traceId: run.traceId,
       spanId: run.spanId,
       isCached: !!span.originalRun,
       machinePreset: machine?.name,

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.branches/route.tsx

@@ -130,7 +130,10 @@ export async function action({ request }: ActionFunctionArgs) {
   }
 
   const upsertBranchService = new UpsertBranchService();
-  const result = await upsertBranchService.call(userId, submission.value);
+  const result = await upsertBranchService.call(
+    { type: "userMembership", userId },
+    submission.value
+  );
 
   if (result.success) {
     if (result.alreadyExisted) {

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.bulk-actions.$bulkActionParam/route.tsx

@@ -1,10 +1,9 @@
 import { ArrowPathIcon } from "@heroicons/react/20/solid";
-import { Form, useRevalidator } from "@remix-run/react";
+import { Form } from "@remix-run/react";
 import { type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { tryCatch } from "@trigger.dev/core";
 import type { BulkActionType } from "@trigger.dev/database";
 import { motion } from "framer-motion";
-import { useEffect } from "react";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { z } from "zod";
 import { ExitIcon } from "~/assets/icons/ExitIcon";
@@ -18,8 +17,9 @@ import { Paragraph } from "~/components/primitives/Paragraph";
 import * as Property from "~/components/primitives/PropertyTable";
 import { BulkActionStatusCombo, BulkActionTypeCombo } from "~/components/runs/v3/BulkAction";
 import { UserAvatar } from "~/components/UserProfilePhoto";
+import { env } from "~/env.server";
+import { useAutoRevalidate } from "~/hooks/useAutoRevalidate";
 import { useEnvironment } from "~/hooks/useEnvironment";
-import { useEventSource } from "~/hooks/useEventSource";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
 import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
@@ -72,7 +72,9 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
       throw new Error(error.message);
     }
 
-    return typedjson({ bulkAction: data });
+    const autoReloadPollIntervalMs = env.BULK_ACTION_AUTORELOAD_POLL_INTERVAL_MS;
+
+    return typedjson({ bulkAction: data, autoReloadPollIntervalMs });
   } catch (error) {
     console.error(error);
     throw new Response(undefined, {
@@ -130,30 +132,16 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
 };
 
 export default function Page() {
-  const { bulkAction } = useTypedLoaderData<typeof loader>();
+  const { bulkAction, autoReloadPollIntervalMs } = useTypedLoaderData<typeof loader>();
   const organization = useOrganization();
   const project = useProject();
   const environment = useEnvironment();
 
-  const disabled = bulkAction.status !== "PENDING";
-
-  const streamedEvents = useEventSource(
-    `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${environment.id}/runs/bulkaction/${bulkAction.friendlyId}/stream`,
-    {
-      event: "progress",
-      disabled,
-    }
-  );
-
-  const revalidation = useRevalidator();
-
-  useEffect(() => {
-    if (disabled || streamedEvents === null) {
-      return;
-    }
-
-    revalidation.revalidate();
-  }, [streamedEvents, disabled]);
+  useAutoRevalidate({
+    interval: autoReloadPollIntervalMs,
+    onFocus: true,
+    disabled: bulkAction.status !== "PENDING",
+  });
 
   return (
     <div className="grid h-full max-h-full grid-rows-[2.5rem_2.5rem_1fr_3.25rem] overflow-hidden bg-background-bright">

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.deployments.$deploymentParam/route.tsx

@@ -4,7 +4,6 @@ import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { ExitIcon } from "~/assets/icons/ExitIcon";
 import { GitMetadata } from "~/components/GitMetadata";
 import { RuntimeIcon } from "~/components/RuntimeIcon";
-import { UserAvatar } from "~/components/UserProfilePhoto";
 import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
 import { EnvironmentCombo } from "~/components/environments/EnvironmentLabel";
 import { Badge } from "~/components/primitives/Badge";
@@ -32,6 +31,7 @@ import { requireUserId } from "~/services/session.server";
 import { cn } from "~/utils/cn";
 import { v3DeploymentParams, v3DeploymentsPath, v3RunsPath } from "~/utils/pathBuilder";
 import { capitalizeWord } from "~/utils/string";
+import { UserTag } from "../_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.deployments/route";
 
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const userId = await requireUserId(request);
@@ -131,7 +131,11 @@ export default function Page() {
                 <Property.Label>Deploy</Property.Label>
                 <Property.Value className="flex items-center gap-2">
                   <span>{deployment.shortCode}</span>
-                  {deployment.label && <Badge variant="outline-rounded">{deployment.label}</Badge>}
+                  {deployment.label && (
+                    <Badge variant="extra-small" className="capitalize">
+                      {deployment.label}
+                    </Badge>
+                  )}
                 </Property.Value>
               </Property.Item>
               <Property.Item>
@@ -154,6 +158,22 @@ export default function Page() {
                   />
                 </Property.Value>
               </Property.Item>
+              {deployment.canceledAt && (
+                <Property.Item>
+                  <Property.Label>Canceled at</Property.Label>
+                  <Property.Value>
+                    <>
+                      <DateTimeAccurate date={deployment.canceledAt} /> UTC
+                    </>
+                  </Property.Value>
+                </Property.Item>
+              )}
+              {deployment.canceledReason && (
+                <Property.Item>
+                  <Property.Label>Cancelation reason</Property.Label>
+                  <Property.Value>{deployment.canceledReason}</Property.Value>
+                </Property.Item>
+              )}
               <Property.Item>
                 <Property.Label>Tasks</Property.Label>
                 <Property.Value>{deployment.tasks ? deployment.tasks.length : "–"}</Property.Value>
@@ -187,7 +207,25 @@ export default function Page() {
               <Property.Item>
                 <Property.Label>Started at</Property.Label>
                 <Property.Value>
-                  <DateTimeAccurate date={deployment.createdAt} /> UTC
+                  {deployment.startedAt ? (
+                    <>
+                      <DateTimeAccurate date={deployment.startedAt} /> UTC
+                    </>
+                  ) : (
+                    "–"
+                  )}
+                </Property.Value>
+              </Property.Item>
+              <Property.Item>
+                <Property.Label>Installed at</Property.Label>
+                <Property.Value>
+                  {deployment.installedAt ? (
+                    <>
+                      <DateTimeAccurate date={deployment.installedAt} /> UTC
+                    </>
+                  ) : (
+                    "–"
+                  )}
                 </Property.Value>
               </Property.Item>
               <Property.Item>
@@ -226,17 +264,16 @@ export default function Page() {
               <Property.Item>
                 <Property.Label>Deployed by</Property.Label>
                 <Property.Value>
-                  {deployment.deployedBy ? (
-                    <div className="flex items-center gap-1">
-                      <UserAvatar
-                        avatarUrl={deployment.deployedBy.avatarUrl}
-                        name={deployment.deployedBy.name ?? deployment.deployedBy.displayName}
-                        className="h-4 w-4"
-                      />
-                      <Paragraph variant="small">
-                        {deployment.deployedBy.name ?? deployment.deployedBy.displayName}
-                      </Paragraph>
-                    </div>
+                  {deployment.git?.source === "trigger_github_app" ? (
+                    <UserTag
+                      name={deployment.git.ghUsername ?? "GitHub Integration"}
+                      avatarUrl={deployment.git.ghUserAvatarUrl}
+                    />
+                  ) : deployment.deployedBy ? (
+                    <UserTag
+                      name={deployment.deployedBy.name ?? deployment.deployedBy.displayName ?? ""}
+                      avatarUrl={deployment.deployedBy.avatarUrl ?? undefined}
+                    />
                   ) : (
                     "–"
                   )}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.deployments/route.tsx

@@ -1,11 +1,26 @@
-import { ArrowUturnLeftIcon, BookOpenIcon } from "@heroicons/react/20/solid";
-import { type MetaFunction, Outlet, useLocation, useNavigate, useParams } from "@remix-run/react";
+import {
+  ArrowPathIcon,
+  ArrowUturnLeftIcon,
+  BookOpenIcon,
+  NoSymbolIcon,
+} from "@heroicons/react/20/solid";
+import {
+  Form,
+  type MetaFunction,
+  Outlet,
+  useLocation,
+  useNavigate,
+  useNavigation,
+  useParams,
+} from "@remix-run/react";
 import { type LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { CogIcon, GitBranchIcon } from "lucide-react";
 import { useEffect } from "react";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { z } from "zod";
 import { PromoteIcon } from "~/assets/icons/PromoteIcon";
 import { DeploymentsNone, DeploymentsNoneDev } from "~/components/BlankStatePanels";
+import { OctoKitty } from "~/components/GitHubLoginButton";
 import { GitMetadata } from "~/components/GitMetadata";
 import { RuntimeIcon } from "~/components/RuntimeIcon";
 import { UserAvatar } from "~/components/UserProfilePhoto";
@@ -13,7 +28,15 @@ import { MainCenteredContainer, PageBody, PageContainer } from "~/components/lay
 import { Badge } from "~/components/primitives/Badge";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
 import { DateTime } from "~/components/primitives/DateTime";
-import { Dialog, DialogTrigger } from "~/components/primitives/Dialog";
+import { SpinnerWhite } from "~/components/primitives/Spinner";
+import {
+  Dialog,
+  DialogDescription,
+  DialogContent,
+  DialogHeader,
+  DialogTrigger,
+  DialogFooter,
+} from "~/components/primitives/Dialog";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { PaginationControls } from "~/components/primitives/Pagination";
 import { Paragraph } from "~/components/primitives/Paragraph";
@@ -37,10 +60,6 @@ import {
   deploymentStatusDescription,
   deploymentStatuses,
 } from "~/components/runs/v3/DeploymentStatus";
-import {
-  PromoteDeploymentDialog,
-  RollbackDeploymentDialog,
-} from "~/components/runs/v3/RollbackDeploymentDialog";
 import { useEnvironment } from "~/hooks/useEnvironment";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
@@ -50,9 +69,18 @@ import {
 } from "~/presenters/v3/DeploymentListPresenter.server";
 import { requireUserId } from "~/services/session.server";
 import { titleCase } from "~/utils";
-import { EnvironmentParamSchema, docsPath, v3DeploymentPath } from "~/utils/pathBuilder";
+import { cn } from "~/utils/cn";
+import {
+  EnvironmentParamSchema,
+  docsPath,
+  v3DeploymentPath,
+  v3ProjectSettingsPath,
+} from "~/utils/pathBuilder";
 import { createSearchParams } from "~/utils/searchParams";
 import { compareDeploymentVersions } from "~/v3/utils/deploymentVersions";
+import { useAutoRevalidate } from "~/hooks/useAutoRevalidate";
+import { env } from "~/env.server";
+import { DialogClose } from "@radix-ui/react-dialog";
 
 export const meta: MetaFunction = () => {
   return [
@@ -108,7 +136,9 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
       ? result.deployments.find((d) => d.version === version)
       : undefined;
 
-    return typedjson({ ...result, selectedDeployment });
+    const autoReloadPollIntervalMs = env.DEPLOYMENTS_AUTORELOAD_POLL_INTERVAL_MS;
+
+    return typedjson({ ...result, selectedDeployment, autoReloadPollIntervalMs });
   } catch (error) {
     console.error(error);
     throw new Response(undefined, {
@@ -122,14 +152,23 @@ export default function Page() {
   const organization = useOrganization();
   const project = useProject();
   const environment = useEnvironment();
-  const { deployments, currentPage, totalPages, selectedDeployment } =
-    useTypedLoaderData<typeof loader>();
+  const {
+    deployments,
+    currentPage,
+    totalPages,
+    selectedDeployment,
+    connectedGithubRepository,
+    environmentGitHubBranch,
+    autoReloadPollIntervalMs,
+  } = useTypedLoaderData<typeof loader>();
   const hasDeployments = totalPages > 0;
 
   const { deploymentParam } = useParams();
   const location = useLocation();
   const navigate = useNavigate();
 
+  useAutoRevalidate({ interval: autoReloadPollIntervalMs, onFocus: true });
+
   // If we have a selected deployment from the version param, show it
   useEffect(() => {
     if (selectedDeployment && !deploymentParam) {
@@ -160,8 +199,8 @@ export default function Page() {
         <ResizablePanelGroup orientation="horizontal" className="h-full max-h-full">
           <ResizablePanel id="deployments-main" min="100px" className="max-h-full">
             {hasDeployments ? (
-              <div className="grid max-h-full grid-rows-[1fr_auto]">
-                <Table containerClassName="border-t-0">
+              <div className="flex h-full max-h-full flex-col">
+                <Table containerClassName="border-t-0 grow">
                   <TableHeader>
                     <TableRow>
                       <TableHeaderCell>Deploy</TableHeaderCell>
@@ -286,11 +325,38 @@ export default function Page() {
                     )}
                   </TableBody>
                 </Table>
-                {totalPages > 1 && (
-                  <div className="-mt-px flex justify-end border-t border-grid-dimmed py-2 pr-2">
-                    <PaginationControls currentPage={currentPage} totalPages={totalPages} />
-                  </div>
-                )}
+                <div
+                  className={cn(
+                    "-mt-px flex flex-wrap justify-end gap-2 border-t border-grid-dimmed px-3 pb-[7px] pt-[6px]",
+                    connectedGithubRepository && environmentGitHubBranch && "justify-between"
+                  )}
+                >
+                  {connectedGithubRepository && environmentGitHubBranch && (
+                    <div className="flex flex-nowrap items-center gap-2 whitespace-nowrap text-sm">
+                      <OctoKitty className="size-4" />
+                      Automatically triggered by pushes to{" "}
+                      <div className="flex max-w-32 items-center gap-1 truncate rounded bg-grid-dimmed px-1 font-mono">
+                        <GitBranchIcon className="size-3 shrink-0" />
+                        <span className="max-w-28 truncate">{environmentGitHubBranch}</span>
+                      </div>{" "}
+                      in
+                      <a
+                        href={connectedGithubRepository.repository.htmlUrl}
+                        target="_blank"
+                        rel="noreferrer noopener"
+                        className="max-w-52 truncate text-sm text-text-dimmed underline transition-colors hover:text-text-bright"
+                      >
+                        {connectedGithubRepository.repository.fullName}
+                      </a>
+                      <LinkButton
+                        variant="minimal/small"
+                        LeadingIcon={CogIcon}
+                        to={v3ProjectSettingsPath(organization, project, environment)}
+                      />
+                    </div>
+                  )}
+                  <PaginationControls currentPage={currentPage} totalPages={totalPages} />
+                </div>
               </div>
             ) : environment.type === "DEVELOPMENT" ? (
               <MainCenteredContainer className="max-w-md">
@@ -317,7 +383,7 @@ export default function Page() {
   );
 }
 
-function UserTag({ name, avatarUrl }: { name: string; avatarUrl?: string }) {
+export function UserTag({ name, avatarUrl }: { name: string; avatarUrl?: string }) {
   return (
     <div className="flex items-center gap-1">
       <UserAvatar avatarUrl={avatarUrl} name={name} className="h-4 w-4" />
@@ -347,7 +413,10 @@ function DeploymentActionsCell({
     compareDeploymentVersions(deployment.version, currentDeployment.version) === -1;
   const canBePromoted = canBeMadeCurrent && !canBeRolledBack;
 
-  if (!canBeRolledBack && !canBePromoted) {
+  const finalStatuses = ["CANCELED", "DEPLOYED", "FAILED", "TIMED_OUT"];
+  const canBeCanceled = !finalStatuses.includes(deployment.status);
+
+  if (!canBeRolledBack && !canBePromoted && !canBeCanceled) {
     return (
       <TableCell to={path} isSelected={isSelected}>
         {""}
@@ -371,7 +440,7 @@ function DeploymentActionsCell({
                   fullWidth
                   textAlignLeft
                 >
-                  Rollback…
+                  Rollback
                 </Button>
               </DialogTrigger>
               <RollbackDeploymentDialog
@@ -391,7 +460,7 @@ function DeploymentActionsCell({
                   fullWidth
                   textAlignLeft
                 >
-                  Promote…
+                  Promote
                 </Button>
               </DialogTrigger>
               <PromoteDeploymentDialog
@@ -401,8 +470,155 @@ function DeploymentActionsCell({
               />
             </Dialog>
           )}
+          {canBeCanceled && (
+            <Dialog>
+              <DialogTrigger asChild>
+                <Button
+                  variant="small-menu-item"
+                  LeadingIcon={NoSymbolIcon}
+                  leadingIconClassName="text-error"
+                  fullWidth
+                  textAlignLeft
+                >
+                  Cancel
+                </Button>
+              </DialogTrigger>
+              <CancelDeploymentDialog
+                projectId={project.id}
+                deploymentShortCode={deployment.shortCode}
+                redirectPath={`${location.pathname}${location.search}`}
+              />
+            </Dialog>
+          )}
         </>
       }
     />
   );
 }
+
+type RollbackDeploymentDialogProps = {
+  projectId: string;
+  deploymentShortCode: string;
+  redirectPath: string;
+};
+
+function RollbackDeploymentDialog({
+  projectId,
+  deploymentShortCode,
+  redirectPath,
+}: RollbackDeploymentDialogProps) {
+  const navigation = useNavigation();
+
+  const formAction = `/resources/${projectId}/deployments/${deploymentShortCode}/rollback`;
+  const isLoading = navigation.formAction === formAction;
+
+  return (
+    <DialogContent key="rollback">
+      <DialogHeader>Rollback to this deployment?</DialogHeader>
+      <DialogDescription>
+        This deployment will become the default for all future runs. Tasks triggered but not
+        included in this deploy will remain queued until you roll back to or create a new deployment
+        with these tasks included.
+      </DialogDescription>
+      <DialogFooter>
+        <DialogClose asChild>
+          <Button variant="tertiary/medium">Cancel</Button>
+        </DialogClose>
+        <Form
+          action={`/resources/${projectId}/deployments/${deploymentShortCode}/rollback`}
+          method="post"
+        >
+          <Button
+            type="submit"
+            name="redirectUrl"
+            value={redirectPath}
+            variant="primary/medium"
+            LeadingIcon={isLoading ? SpinnerWhite : ArrowPathIcon}
+            disabled={isLoading}
+            shortcut={{ modifiers: ["mod"], key: "enter" }}
+          >
+            {isLoading ? "Rolling back..." : "Rollback deployment"}
+          </Button>
+        </Form>
+      </DialogFooter>
+    </DialogContent>
+  );
+}
+
+function PromoteDeploymentDialog({
+  projectId,
+  deploymentShortCode,
+  redirectPath,
+}: RollbackDeploymentDialogProps) {
+  const navigation = useNavigation();
+
+  const formAction = `/resources/${projectId}/deployments/${deploymentShortCode}/promote`;
+  const isLoading = navigation.formAction === formAction;
+
+  return (
+    <DialogContent key="promote">
+      <DialogHeader>Promote this deployment?</DialogHeader>
+      <DialogDescription>
+        This deployment will become the default for all future runs not explicitly tied to a
+        specific deployment.
+      </DialogDescription>
+      <DialogFooter>
+        <DialogClose asChild>
+          <Button variant="tertiary/medium">Cancel</Button>
+        </DialogClose>
+        <Form
+          action={`/resources/${projectId}/deployments/${deploymentShortCode}/promote`}
+          method="post"
+        >
+          <Button
+            type="submit"
+            name="redirectUrl"
+            value={redirectPath}
+            variant="primary/medium"
+            LeadingIcon={isLoading ? SpinnerWhite : ArrowPathIcon}
+            disabled={isLoading}
+            shortcut={{ modifiers: ["mod"], key: "enter" }}
+          >
+            {isLoading ? "Promoting..." : "Promote deployment"}
+          </Button>
+        </Form>
+      </DialogFooter>
+    </DialogContent>
+  );
+}
+
+function CancelDeploymentDialog({
+  projectId,
+  deploymentShortCode,
+  redirectPath,
+}: RollbackDeploymentDialogProps) {
+  const navigation = useNavigation();
+
+  const formAction = `/resources/${projectId}/deployments/${deploymentShortCode}/cancel`;
+  const isLoading = navigation.formAction === formAction;
+
+  return (
+    <DialogContent key="cancel">
+      <DialogHeader>Cancel this deployment?</DialogHeader>
+      <DialogDescription>Canceling a deployment cannot be undone. Are you sure?</DialogDescription>
+      <DialogFooter>
+        <DialogClose asChild>
+          <Button variant="tertiary/medium">Back</Button>
+        </DialogClose>
+        <Form action={formAction} method="post">
+          <Button
+            type="submit"
+            name="redirectUrl"
+            value={redirectPath}
+            variant="danger/medium"
+            LeadingIcon={isLoading ? SpinnerWhite : NoSymbolIcon}
+            disabled={isLoading}
+            shortcut={{ modifiers: ["mod"], key: "enter" }}
+          >
+            {isLoading ? "Canceling..." : "Cancel deployment"}
+          </Button>
+        </Form>
+      </DialogFooter>
+    </DialogContent>
+  );
+}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx

@@ -8,14 +8,7 @@ import {
   RectangleStackIcon,
 } from "@heroicons/react/20/solid";
 import { DialogClose } from "@radix-ui/react-dialog";
-import {
-  Form,
-  useNavigate,
-  useNavigation,
-  useRevalidator,
-  useSearchParams,
-  type MetaFunction,
-} from "@remix-run/react";
+import { Form, useNavigation, useSearchParams, type MetaFunction } from "@remix-run/react";
 import { type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime";
 import type { RuntimeEnvironmentType } from "@trigger.dev/database";
 import { useEffect, useState } from "react";
@@ -30,7 +23,7 @@ import { Feedback } from "~/components/Feedback";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { BigNumber } from "~/components/metrics/BigNumber";
 import { Badge } from "~/components/primitives/Badge";
-import { Button, ButtonVariant, LinkButton } from "~/components/primitives/Buttons";
+import { Button, type ButtonVariant, LinkButton } from "~/components/primitives/Buttons";
 import { Callout } from "~/components/primitives/Callout";
 import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components/primitives/Dialog";
 import { FormButtons } from "~/components/primitives/FormButtons";
@@ -56,7 +49,6 @@ import {
   TooltipTrigger,
 } from "~/components/primitives/Tooltip";
 import { useEnvironment } from "~/hooks/useEnvironment";
-import { useEventSource } from "~/hooks/useEventSource";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
 import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
@@ -74,6 +66,8 @@ import { Header3 } from "~/components/primitives/Headers";
 import { Input } from "~/components/primitives/Input";
 import { useThrottle } from "~/hooks/useThrottle";
 import { RunsIcon } from "~/assets/icons/RunsIcon";
+import { useAutoRevalidate } from "~/hooks/useAutoRevalidate";
+import { env } from "~/env.server";
 
 const SearchParamsSchema = z.object({
   query: z.string().optional(),
@@ -121,9 +115,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
 
     const environmentQueuePresenter = new EnvironmentQueuePresenter();
 
+    const autoReloadPollIntervalMs = env.QUEUES_AUTORELOAD_POLL_INTERVAL_MS;
+
     return typedjson({
       ...queues,
       environment: await environmentQueuePresenter.call(environment),
+      autoReloadPollIntervalMs,
     });
   } catch (error) {
     console.error(error);
@@ -217,28 +214,23 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
 };
 
 export default function Page() {
-  const { environment, queues, success, pagination, code, totalQueues, hasFilters } =
-    useTypedLoaderData<typeof loader>();
+  const {
+    environment,
+    queues,
+    success,
+    pagination,
+    code,
+    totalQueues,
+    hasFilters,
+    autoReloadPollIntervalMs,
+  } = useTypedLoaderData<typeof loader>();
 
   const organization = useOrganization();
   const project = useProject();
   const env = useEnvironment();
   const plan = useCurrentPlan();
 
-  // Reload the page periodically
-  const streamedEvents = useEventSource(
-    `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${env.slug}/queues/stream`,
-    {
-      event: "update",
-    }
-  );
-
-  const revalidation = useRevalidator();
-  useEffect(() => {
-    if (streamedEvents) {
-      revalidation.revalidate();
-    }
-  }, [streamedEvents]);
+  useAutoRevalidate({ interval: autoReloadPollIntervalMs, onFocus: true });
 
   const limitStatus =
     environment.running === environment.concurrencyLimit * environment.burstFactor

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx

@@ -11,7 +11,7 @@ import {
   MagnifyingGlassPlusIcon,
   StopCircleIcon,
 } from "@heroicons/react/20/solid";
-import { useLoaderData, useParams, useRevalidator } from "@remix-run/react";
+import { useLoaderData, useRevalidator } from "@remix-run/react";
 import { type LoaderFunctionArgs, type SerializeFrom, json } from "@remix-run/server-runtime";
 import { type Virtualizer } from "@tanstack/react-virtual";
 import {
@@ -25,7 +25,8 @@ import { motion } from "framer-motion";
 import { useCallback, useEffect, useRef, useState } from "react";
 import { useHotkeys } from "react-hotkeys-hook";
 import { redirect } from "remix-typedjson";
-import { ShowParentIcon, ShowParentIconSelected } from "~/assets/icons/ShowParentIcon";
+import { MoveToTopIcon } from "~/assets/icons/MoveToTopIcon";
+import { MoveUpIcon } from "~/assets/icons/MoveUpIcon";
 import tileBgPath from "~/assets/images/error-banner-tile@2x.png";
 import { DevDisconnectedBanner, useCrossEngineIsConnected } from "~/components/DevPresence";
 import { WarmStartIconWithTooltip } from "~/components/WarmStarts";
@@ -87,7 +88,6 @@ import {
   docsPath,
   v3BillingPath,
   v3RunParamsSchema,
-  v3RunPath,
   v3RunRedirectPath,
   v3RunSpanPath,
   v3RunStreamingPath,
@@ -302,8 +302,7 @@ function TraceView({ run, trace, maximumLiveReloadingSetting, resizable }: Loade
     return <></>;
   }
 
-  const { events, parentRunFriendlyId, duration, rootSpanStatus, rootStartedAt, queuedDuration } =
-    trace;
+  const { events, duration, rootSpanStatus, rootStartedAt, queuedDuration } = trace;
   const shouldLiveReload = events.length <= maximumLiveReloadingSetting;
 
   const changeToSpan = useDebounce((selectedSpan: string) => {
@@ -340,7 +339,6 @@ function TraceView({ run, trace, maximumLiveReloadingSetting, resizable }: Loade
             selectedId={selectedSpanId}
             key={events[0]?.id ?? "-"}
             events={events}
-            parentRunFriendlyId={parentRunFriendlyId}
             onSelectedIdChanged={(selectedSpan) => {
               //instantly close the panel if no span is selected
               if (!selectedSpan) {
@@ -358,6 +356,7 @@ function TraceView({ run, trace, maximumLiveReloadingSetting, resizable }: Loade
             shouldLiveReload={shouldLiveReload}
             maximumLiveReloadingSetting={maximumLiveReloadingSetting}
             rootRun={run.rootTaskRun}
+            parentRun={run.parentTaskRun}
             isCompleted={run.completedAt !== null}
           />
         </ResizablePanel>
@@ -476,7 +475,6 @@ function NoLogsView({ run, resizable }: LoaderData) {
 type TasksTreeViewProps = {
   events: TraceEvent[];
   selectedId?: string;
-  parentRunFriendlyId?: string;
   onSelectedIdChanged: (selectedId: string | undefined) => void;
   totalDuration: number;
   rootSpanStatus: "executing" | "completed" | "failed";
@@ -487,7 +485,10 @@ type TasksTreeViewProps = {
   maximumLiveReloadingSetting: number;
   rootRun: {
     friendlyId: string;
-    taskIdentifier: string;
+    spanId: string;
+  } | null;
+  parentRun: {
+    friendlyId: string;
     spanId: string;
   } | null;
   isCompleted: boolean;
@@ -496,7 +497,6 @@ type TasksTreeViewProps = {
 function TasksTreeView({
   events,
   selectedId,
-  parentRunFriendlyId,
   onSelectedIdChanged,
   totalDuration,
   rootSpanStatus,
@@ -506,6 +506,7 @@ function TasksTreeView({
   shouldLiveReload,
   maximumLiveReloadingSetting,
   rootRun,
+  parentRun,
   isCompleted,
 }: TasksTreeViewProps) {
   const isAdmin = useHasAdminAccess();
@@ -596,20 +597,30 @@ function TasksTreeView({
           id={resizableSettings.tree.tree.id}
           default={resizableSettings.tree.tree.default}
           min={resizableSettings.tree.tree.min}
-          className="pl-3"
         >
           <div className="grid h-full grid-rows-[2rem_1fr] overflow-hidden">
-            <div className="flex items-center pr-2">
-              {rootRun ? (
-                <ShowParentLink
-                  runFriendlyId={rootRun.friendlyId}
-                  isRoot={true}
-                  spanId={rootRun.spanId}
+            <div className="flex items-center justify-between pl-1 pr-2">
+              {rootRun || parentRun ? (
+                <ShowParentOrRootLinks
+                  relationships={{
+                    root: rootRun
+                      ? {
+                          friendlyId: rootRun.friendlyId,
+                          spanId: rootRun.spanId,
+                          isParent: parentRun ? rootRun.friendlyId === parentRun.friendlyId : true,
+                        }
+                      : undefined,
+                    parent:
+                      parentRun && rootRun?.friendlyId !== parentRun.friendlyId
+                        ? {
+                            friendlyId: parentRun.friendlyId,
+                            spanId: "",
+                          }
+                        : undefined,
+                  }}
                 />
-              ) : parentRunFriendlyId ? (
-                <ShowParentLink runFriendlyId={parentRunFriendlyId} isRoot={false} />
               ) : (
-                <Paragraph variant="small" className="flex-1 text-charcoal-500">
+                <Paragraph variant="extra-small" className="flex-1 pl-3 text-charcoal-500">
                   This is the root task
                 </Paragraph>
               )}
@@ -628,6 +639,7 @@ function TasksTreeView({
               nodes={nodes}
               getNodeProps={getNodeProps}
               getTreeProps={getTreeProps}
+              parentClassName="pl-3"
               renderNode={({ node, state, index }) => (
                 <>
                   <div
@@ -1139,60 +1151,108 @@ function TaskLine({ isError, isSelected }: { isError: boolean; isSelected: boole
   return <div className={cn("h-8 w-2 border-r border-grid-bright")} />;
 }
 
-function ShowParentLink({
-  runFriendlyId,
-  spanId,
-  isRoot,
+function ShowParentOrRootLinks({
+  relationships,
 }: {
-  runFriendlyId: string;
-  spanId?: string;
-  isRoot: boolean;
+  relationships: {
+    root?: {
+      friendlyId: string;
+      spanId: string;
+      isParent?: boolean;
+    };
+    parent?: {
+      friendlyId: string;
+      spanId: string;
+    };
+  };
 }) {
-  const [mouseOver, setMouseOver] = useState(false);
   const organization = useOrganization();
   const project = useProject();
   const environment = useEnvironment();
-  const { spanParam } = useParams();
 
-  const span = spanId ? spanId : spanParam;
-
-  return (
-    <LinkButton
-      variant="minimal/medium"
-      to={
-        span
-          ? v3RunSpanPath(
-              organization,
-              project,
-              environment,
-              {
-                friendlyId: runFriendlyId,
-              },
-              { spanId: span }
-            )
-          : v3RunPath(organization, project, environment, {
-              friendlyId: runFriendlyId,
-            })
-      }
-      onMouseEnter={() => setMouseOver(true)}
-      onMouseLeave={() => setMouseOver(false)}
-      fullWidth
-      textAlignLeft
-      shortcut={{ key: "p" }}
-      className="flex-1"
-    >
-      {mouseOver ? (
-        <ShowParentIconSelected className="h-4 w-4 text-indigo-500" />
-      ) : (
-        <ShowParentIcon className="h-4 w-4 text-charcoal-650" />
-      )}
-      <Paragraph
-        variant="small"
-        className={cn(mouseOver ? "text-indigo-500" : "text-charcoal-500")}
+  // Case 1: Root is also the parent
+  if (relationships.root?.isParent === true) {
+    return (
+      <LinkButton
+        variant="minimal/small"
+        to={v3RunSpanPath(
+          organization,
+          project,
+          environment,
+          { friendlyId: relationships.root.friendlyId },
+          { spanId: relationships.root.spanId }
+        )}
+        LeadingIcon={MoveToTopIcon}
+        leadingIconClassName="gap-x-2"
+        shortcut={{ key: "p" }}
+        hideShortcutKey
+        tooltip={
+          <div className="-mr-1 flex items-center gap-1">
+            <Paragraph variant="extra-small">Jump to root and parent run</Paragraph>
+            <ShortcutKey shortcut={{ key: "p" }} variant="small" />
+          </div>
+        }
+        className="text-xs"
       >
-        {isRoot ? "Show root run" : "Show parent run"}
-      </Paragraph>
-    </LinkButton>
+        Root/parent
+      </LinkButton>
+    );
+  }
+
+  // Case 2: Root and Parent are different runs
+  return (
+    <div className="flex items-center">
+      {relationships.root && (
+        <LinkButton
+          variant="minimal/small"
+          to={v3RunSpanPath(
+            organization,
+            project,
+            environment,
+            { friendlyId: relationships.root.friendlyId },
+            { spanId: relationships.root.spanId }
+          )}
+          LeadingIcon={MoveToTopIcon}
+          leadingIconClassName="gap-x-2"
+          shortcut={{ key: "t" }}
+          hideShortcutKey
+          tooltip={
+            <div className="-mr-1 flex items-center gap-1">
+              <Paragraph variant="extra-small">Jump to root run</Paragraph>
+              <ShortcutKey shortcut={{ key: "t" }} variant="small" />
+            </div>
+          }
+          className="text-xs"
+        >
+          Root
+        </LinkButton>
+      )}
+      {relationships.parent && (
+        <LinkButton
+          variant="minimal/small"
+          to={v3RunSpanPath(
+            organization,
+            project,
+            environment,
+            { friendlyId: relationships.parent.friendlyId },
+            { spanId: relationships.parent.spanId }
+          )}
+          LeadingIcon={MoveUpIcon}
+          leadingIconClassName="gap-x-2"
+          shortcut={{ key: "p" }}
+          hideShortcutKey
+          tooltip={
+            <div className="-mr-1 flex items-center gap-1">
+              <Paragraph variant="extra-small">Jump to parent run</Paragraph>
+              <ShortcutKey shortcut={{ key: "p" }} variant="small" />
+            </div>
+          }
+          className="text-xs"
+        >
+          Parent
+        </LinkButton>
+      )}
+    </div>
   );
 }
 

apps/webapp/app/routes/_app.orgs.$organizationSlug_.select-plan/route.tsx

@@ -1,7 +1,7 @@
-import { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { type LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { redirect, typedjson, useTypedLoaderData } from "remix-typedjson";
 import { BackgroundWrapper } from "~/components/BackgroundWrapper";
-import { AppContainer } from "~/components/layout/AppLayout";
+import { AppContainer, MainBody, PageBody } from "~/components/layout/AppLayout";
 import { Header1 } from "~/components/primitives/Headers";
 import { prisma } from "~/db.server";
 import { featuresForRequest } from "~/features.server";
@@ -49,22 +49,24 @@ export default function ChoosePlanPage() {
     useTypedLoaderData<typeof loader>();
 
   return (
-    <AppContainer className="bg-charcoal-900">
-      <BackgroundWrapper>
-        <div className="mx-auto flex h-full w-full max-w-[80rem] flex-col items-center justify-center gap-8 p-3">
-          <Header1 className="text-center">Subscribe for full access</Header1>
-          <div className="w-full rounded-lg border border-grid-bright bg-background-dimmed p-5 shadow-lg">
-            <PricingPlans
-              plans={plans}
-              subscription={v3Subscription}
-              organizationSlug={organizationSlug}
-              hasPromotedPlan
-              showGithubVerificationBadge
-              periodEnd={periodEnd}
-            />
+    <AppContainer>
+      <PageBody className="bg-charcoal-900">
+        <BackgroundWrapper>
+          <div className="mx-auto mt-4 flex h-fit min-h-full max-w-[80rem] flex-col items-center justify-center gap-8 lg:mt-0">
+            <Header1 className="text-center">Subscribe for full access</Header1>
+            <div className="w-full rounded-lg border border-grid-bright bg-background-dimmed p-5 shadow-lg">
+              <PricingPlans
+                plans={plans}
+                subscription={v3Subscription}
+                organizationSlug={organizationSlug}
+                hasPromotedPlan
+                showGithubVerificationBadge
+                periodEnd={periodEnd}
+              />
+            </div>
           </div>
-        </div>
-      </BackgroundWrapper>
+        </BackgroundWrapper>
+      </PageBody>
     </AppContainer>
   );
 }

apps/webapp/app/routes/api.v1.deployments.$deploymentId.cancel.ts

@@ -0,0 +1,72 @@
+import { type ActionFunctionArgs, json } from "@remix-run/server-runtime";
+import { CancelDeploymentRequestBody, tryCatch } from "@trigger.dev/core/v3";
+import { z } from "zod";
+import { authenticateRequest } from "~/services/apiAuth.server";
+import { logger } from "~/services/logger.server";
+import { DeploymentService } from "~/v3/services/deployment.server";
+
+const ParamsSchema = z.object({
+  deploymentId: z.string(),
+});
+
+export async function action({ request, params }: ActionFunctionArgs) {
+  if (request.method.toUpperCase() !== "POST") {
+    return json({ error: "Method Not Allowed" }, { status: 405 });
+  }
+
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return json({ error: "Invalid params" }, { status: 400 });
+  }
+
+  const authenticationResult = await authenticateRequest(request, {
+    apiKey: true,
+    organizationAccessToken: false,
+    personalAccessToken: false,
+  });
+
+  if (!authenticationResult || !authenticationResult.result.ok) {
+    logger.info("Invalid or missing api key", { url: request.url });
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const { environment: authenticatedEnv } = authenticationResult.result;
+  const { deploymentId } = parsedParams.data;
+
+  const [, rawBody] = await tryCatch(request.json());
+  const body = CancelDeploymentRequestBody.safeParse(rawBody ?? {});
+
+  if (!body.success) {
+    return json({ error: "Invalid request body", issues: body.error.issues }, { status: 400 });
+  }
+
+  const deploymentService = new DeploymentService();
+
+  return await deploymentService
+    .cancelDeployment(authenticatedEnv, deploymentId, {
+      canceledReason: body.data.reason,
+    })
+    .match(
+      () => {
+        return new Response(null, { status: 204 });
+      },
+      (error) => {
+        switch (error.type) {
+          case "deployment_not_found":
+            return json({ error: "Deployment not found" }, { status: 404 });
+          case "failed_to_delete_deployment_timeout":
+            return new Response(null, { status: 204 }); // not a critical error, ignore
+          case "deployment_cannot_be_cancelled":
+            return json(
+              { error: "Deployment is already in a final state and cannot be canceled" },
+              { status: 409 }
+            );
+          case "other":
+          default:
+            error.type satisfies "other";
+            return json({ error: "Internal server error" }, { status: 500 });
+        }
+      }
+    );
+}

apps/webapp/app/routes/api.v1.deployments.$deploymentId.progress.ts

@@ -0,0 +1,76 @@
+import { type ActionFunctionArgs, json } from "@remix-run/server-runtime";
+import { ProgressDeploymentRequestBody, tryCatch } from "@trigger.dev/core/v3";
+import { z } from "zod";
+import { authenticateRequest } from "~/services/apiAuth.server";
+import { logger } from "~/services/logger.server";
+import { DeploymentService } from "~/v3/services/deployment.server";
+
+const ParamsSchema = z.object({
+  deploymentId: z.string(),
+});
+
+export async function action({ request, params }: ActionFunctionArgs) {
+  if (request.method.toUpperCase() !== "POST") {
+    return json({ error: "Method Not Allowed" }, { status: 405 });
+  }
+
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return json({ error: "Invalid params" }, { status: 400 });
+  }
+
+  const authenticationResult = await authenticateRequest(request, {
+    apiKey: true,
+    organizationAccessToken: false,
+    personalAccessToken: false,
+  });
+
+  if (!authenticationResult || !authenticationResult.result.ok) {
+    logger.info("Invalid or missing api key", { url: request.url });
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const { environment: authenticatedEnv } = authenticationResult.result;
+  const { deploymentId } = parsedParams.data;
+
+  const [, rawBody] = await tryCatch(request.json());
+  const body = ProgressDeploymentRequestBody.safeParse(rawBody ?? {});
+
+  if (!body.success) {
+    return json({ error: "Invalid request body", issues: body.error.issues }, { status: 400 });
+  }
+
+  const deploymentService = new DeploymentService();
+
+  return await deploymentService
+    .progressDeployment(authenticatedEnv, deploymentId, {
+      contentHash: body.data.contentHash,
+      git: body.data.gitMeta,
+      runtime: body.data.runtime,
+    })
+    .match(
+      () => {
+        return new Response(null, { status: 204 });
+      },
+      (error) => {
+        switch (error.type) {
+          case "failed_to_extend_deployment_timeout":
+            return new Response(null, { status: 204 }); // ignore these errors for now
+          case "deployment_not_found":
+            return json({ error: "Deployment not found" }, { status: 404 });
+          case "deployment_cannot_be_progressed":
+            return json(
+              { error: "Deployment is not in a progressable state (PENDING or INSTALLING)" },
+              { status: 409 }
+            );
+          case "failed_to_create_remote_build":
+            return json({ error: "Failed to create remote build" }, { status: 500 });
+          case "other":
+          default:
+            error.type satisfies "other";
+            return json({ error: "Internal server error" }, { status: 500 });
+        }
+      }
+    );
+}

apps/webapp/app/routes/api.v1.deployments.$deploymentId.ts

@@ -1,4 +1,5 @@
-import { LoaderFunctionArgs, json } from "@remix-run/server-runtime";
+import { type LoaderFunctionArgs, json } from "@remix-run/server-runtime";
+import { type GetDeploymentResponseBody } from "@trigger.dev/core/v3";
 import { z } from "zod";
 import { prisma } from "~/db.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
@@ -52,7 +53,10 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
     shortCode: deployment.shortCode,
     version: deployment.version,
     imageReference: deployment.imageReference,
-    errorData: deployment.errorData,
+    imagePlatform: deployment.imagePlatform,
+    externalBuildData:
+      deployment.externalBuildData as GetDeploymentResponseBody["externalBuildData"],
+    errorData: deployment.errorData as GetDeploymentResponseBody["errorData"],
     worker: deployment.worker
       ? {
           id: deployment.worker.friendlyId,
@@ -65,5 +69,5 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
           })),
         }
       : undefined,
-  });
+  } satisfies GetDeploymentResponseBody);
 }

apps/webapp/app/routes/api.v1.deployments.ts

@@ -1,8 +1,8 @@
-import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
+import { type ActionFunctionArgs, json } from "@remix-run/server-runtime";
 import {
   ApiDeploymentListSearchParams,
   InitializeDeploymentRequestBody,
-  InitializeDeploymentResponseBody,
+  type InitializeDeploymentResponseBody,
 } from "@trigger.dev/core/v3";
 import { $replica } from "~/db.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";

apps/webapp/app/routes/api.v1.projects.$projectRef.branches.archive.ts

@@ -2,9 +2,9 @@ import { type ActionFunctionArgs, json } from "@remix-run/server-runtime";
 import { tryCatch } from "@trigger.dev/core";
 import { z } from "zod";
 import { prisma } from "~/db.server";
+import { authenticateRequest } from "~/services/apiAuth.server";
 import { ArchiveBranchService } from "~/services/archiveBranch.server";
 import { logger } from "~/services/logger.server";
-import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
 
 const ParamsSchema = z.object({
   projectRef: z.string(),
@@ -21,7 +21,12 @@ export async function action({ request, params }: ActionFunctionArgs) {
 
   logger.info("Archive branch", { url: request.url, params });
 
-  const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
+  const authenticationResult = await authenticateRequest(request, {
+    personalAccessToken: true,
+    organizationAccessToken: true,
+    apiKey: false,
+  });
+
   if (!authenticationResult) {
     return json({ error: "Invalid or Missing Access Token" }, { status: 401 });
   }
@@ -50,13 +55,16 @@ export async function action({ request, params }: ActionFunctionArgs) {
       archivedAt: true,
     },
     where: {
-      organization: {
-        members: {
-          some: {
-            userId: authenticationResult.userId,
-          },
-        },
-      },
+      organization:
+        authenticationResult.type === "organizationAccessToken"
+          ? { id: authenticationResult.result.organizationId }
+          : {
+              members: {
+                some: {
+                  userId: authenticationResult.result.userId,
+                },
+              },
+            },
       project: {
         externalRef: projectRef,
       },
@@ -74,9 +82,14 @@ export async function action({ request, params }: ActionFunctionArgs) {
   }
 
   const service = new ArchiveBranchService();
-  const result = await service.call(authenticationResult.userId, {
-    environmentId: environment.id,
-  });
+  const result = await service.call(
+    authenticationResult.type === "organizationAccessToken"
+      ? { type: "orgId", organizationId: authenticationResult.result.organizationId }
+      : { type: "userMembership", userId: authenticationResult.result.userId },
+    {
+      environmentId: environment.id,
+    }
+  );
 
   if (result.success) {
     return json(result);

apps/webapp/app/routes/api.v1.projects.$projectRef.branches.ts

@@ -1,7 +1,8 @@
-import { json, LoaderFunctionArgs, type ActionFunctionArgs } from "@remix-run/server-runtime";
+import { json, type LoaderFunctionArgs, type ActionFunctionArgs } from "@remix-run/server-runtime";
 import { tryCatch, UpsertBranchRequestBody } from "@trigger.dev/core/v3";
 import { z } from "zod";
 import { prisma } from "~/db.server";
+import { authenticateRequest } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
 import { UpsertBranchService } from "~/services/upsertBranch.server";
@@ -19,7 +20,11 @@ export async function action({ request, params }: ActionFunctionArgs) {
 
   logger.info("project upsert branch", { url: request.url });
 
-  const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
+  const authenticationResult = await authenticateRequest(request, {
+    personalAccessToken: true,
+    organizationAccessToken: true,
+    apiKey: false,
+  });
   if (!authenticationResult) {
     return json({ error: "Invalid or Missing Access Token" }, { status: 401 });
   }
@@ -38,13 +43,16 @@ export async function action({ request, params }: ActionFunctionArgs) {
     },
     where: {
       externalRef: projectRef,
-      organization: {
-        members: {
-          some: {
-            userId: authenticationResult.userId,
-          },
-        },
-      },
+      organization:
+        authenticationResult.type === "organizationAccessToken"
+          ? { id: authenticationResult.result.organizationId }
+          : {
+              members: {
+                some: {
+                  userId: authenticationResult.result.userId,
+                },
+              },
+            },
     },
   });
   if (!project) {
@@ -81,11 +89,16 @@ export async function action({ request, params }: ActionFunctionArgs) {
   const { branch, env, git } = parsed.data;
 
   const service = new UpsertBranchService();
-  const result = await service.call(authenticationResult.userId, {
-    branchName: branch,
-    parentEnvironmentId: previewEnvironment.id,
-    git,
-  });
+  const result = await service.call(
+    authenticationResult.type === "organizationAccessToken"
+      ? { type: "orgId", organizationId: authenticationResult.result.organizationId }
+      : { type: "userMembership", userId: authenticationResult.result.userId },
+    {
+      branchName: branch,
+      parentEnvironmentId: previewEnvironment.id,
+      git,
+    }
+  );
 
   if (!result.success) {
     return json({ error: result.error }, { status: 400 });

apps/webapp/app/routes/api.v1.tasks.$taskId.trigger.ts

@@ -1,3 +1,4 @@
+import { EngineServiceValidationError } from "@internal/run-engine";
 import { json } from "@remix-run/server-runtime";
 import {
   generateJWT as internal_generateJWT,
@@ -8,7 +9,6 @@ import { TaskRun } from "@trigger.dev/database";
 import { z } from "zod";
 import { prisma } from "~/db.server";
 import { env } from "~/env.server";
-import { EngineServiceValidationError } from "~/runEngine/concerns/errors";
 import { ApiAuthenticationResultSuccess, getOneTimeUseToken } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";

apps/webapp/app/routes/resources.$projectId.deployments.$deploymentShortCode.cancel.ts

@@ -0,0 +1,133 @@
+import { parse } from "@conform-to/zod";
+import { type ActionFunction, json } from "@remix-run/node";
+import { errAsync, fromPromise, okAsync } from "neverthrow";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
+import { logger } from "~/services/logger.server";
+import { requireUserId } from "~/services/session.server";
+import { DeploymentService } from "~/v3/services/deployment.server";
+
+export const cancelSchema = z.object({
+  redirectUrl: z.string(),
+});
+
+const ParamSchema = z.object({
+  projectId: z.string(),
+  deploymentShortCode: z.string(),
+});
+
+export const action: ActionFunction = async ({ request, params }) => {
+  const userId = await requireUserId(request);
+  const { projectId, deploymentShortCode } = ParamSchema.parse(params);
+
+  const formData = await request.formData();
+  const submission = parse(formData, { schema: cancelSchema });
+
+  if (!submission.value) {
+    return json(submission);
+  }
+
+  const verifyProjectMembership = () =>
+    fromPromise(
+      prisma.project.findFirst({
+        where: {
+          id: projectId,
+          organization: {
+            members: {
+              some: {
+                userId,
+              },
+            },
+          },
+        },
+        select: {
+          id: true,
+        },
+      }),
+      (error) => ({ type: "other" as const, cause: error })
+    ).andThen((project) => {
+      if (!project) {
+        return errAsync({ type: "project_not_found" as const });
+      }
+      return okAsync(project);
+    });
+
+  const findDeploymentFriendlyId = ({ id }: { id: string }) =>
+    fromPromise(
+      prisma.workerDeployment.findUnique({
+        select: {
+          friendlyId: true,
+          projectId: true,
+        },
+        where: {
+          projectId_shortCode: {
+            projectId: id,
+            shortCode: deploymentShortCode,
+          },
+        },
+      }),
+      (error) => ({ type: "other" as const, cause: error })
+    ).andThen((deployment) => {
+      if (!deployment) {
+        return errAsync({ type: "deployment_not_found" as const });
+      }
+      return okAsync(deployment);
+    });
+
+  const deploymentService = new DeploymentService();
+  const result = await verifyProjectMembership()
+    .andThen(findDeploymentFriendlyId)
+    .andThen((deployment) =>
+      deploymentService.cancelDeployment({ projectId: deployment.projectId }, deployment.friendlyId)
+    );
+
+  if (result.isErr()) {
+    logger.error(
+      `Failed to cancel deployment: ${result.error.type}`,
+      result.error.type === "other"
+        ? {
+            cause: result.error.cause,
+          }
+        : undefined
+    );
+
+    switch (result.error.type) {
+      case "project_not_found":
+        return redirectWithErrorMessage(submission.value.redirectUrl, request, "Project not found");
+      case "deployment_not_found":
+        return redirectWithErrorMessage(
+          submission.value.redirectUrl,
+          request,
+          "Deployment not found"
+        );
+      case "deployment_cannot_be_cancelled":
+        return redirectWithErrorMessage(
+          submission.value.redirectUrl,
+          request,
+          "Deployment is already in a final state and cannot be canceled"
+        );
+      case "failed_to_delete_deployment_timeout":
+        // not a critical error, ignore
+        return redirectWithSuccessMessage(
+          submission.value.redirectUrl,
+          request,
+          `Canceled deployment ${deploymentShortCode}.`
+        );
+      case "other":
+      default:
+        result.error.type satisfies "other";
+        return redirectWithErrorMessage(
+          submission.value.redirectUrl,
+          request,
+          "Internal server error"
+        );
+    }
+  }
+
+  return redirectWithSuccessMessage(
+    submission.value.redirectUrl,
+    request,
+    `Canceled deployment ${deploymentShortCode}.`
+  );
+};

apps/webapp/app/routes/resources.branches.archive.tsx

@@ -37,7 +37,12 @@ export async function action({ request }: ActionFunctionArgs) {
 
   const archiveBranchService = new ArchiveBranchService();
 
-  const result = await archiveBranchService.call(userId, submission.value);
+  const result = await archiveBranchService.call(
+    { type: "userMembership", userId },
+    {
+      environmentId: submission.value.environmentId,
+    }
+  );
 
   if (result.success) {
     return redirectWithSuccessMessage(

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues.stream.tsx

@@ -1,64 +0,0 @@
-import { $replica } from "~/db.server";
-import { env } from "~/env.server";
-import { logger } from "~/services/logger.server";
-import { requireUserId } from "~/services/session.server";
-import { EnvironmentParamSchema } from "~/utils/pathBuilder";
-import { createSSELoader } from "~/utils/sse";
-
-export const loader = createSSELoader({
-  timeout: env.QUEUE_SSE_AUTORELOAD_TIMEOUT_MS,
-  interval: env.QUEUE_SSE_AUTORELOAD_INTERVAL_MS,
-  debug: false,
-  handler: async ({ request, params }) => {
-    const userId = await requireUserId(request);
-    const { projectParam, envParam } = EnvironmentParamSchema.parse(params);
-
-    const environment = await $replica.runtimeEnvironment.findFirst({
-      where: {
-        slug: envParam,
-        OR: [
-          {
-            type: {
-              in: ["PREVIEW", "STAGING", "PRODUCTION"],
-            },
-          },
-          {
-            type: "DEVELOPMENT",
-            orgMember: {
-              userId,
-            },
-          },
-        ],
-        project: {
-          slug: projectParam,
-        },
-      },
-    });
-
-    if (!environment) {
-      throw new Response("Not Found", { status: 404 });
-    }
-
-    return {
-      beforeStream: async () => {
-        logger.debug("Start queue page SSE session", {
-          environmentId: environment.id,
-        });
-      },
-      initStream: async ({ send }) => {
-        send({ event: "time", data: new Date().toISOString() });
-      },
-      iterator: async ({ send }) => {
-        send({
-          event: "update",
-          data: new Date().toISOString(),
-        });
-      },
-      cleanup: async () => {
-        logger.debug("End queue page SSE session", {
-          environmentId: environment.id,
-        });
-      },
-    };
-  },
-});

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx

@@ -14,6 +14,7 @@ import { assertNever } from "assert-never";
 import { useEffect } from "react";
 import { typedjson, useTypedFetcher } from "remix-typedjson";
 import { ExitIcon } from "~/assets/icons/ExitIcon";
+import { FlagIcon } from "~/assets/icons/RegionIcons";
 import { AdminDebugRun } from "~/components/admin/debugRun";
 import { CodeBlock } from "~/components/code/CodeBlock";
 import { EnvironmentCombo } from "~/components/environments/EnvironmentLabel";
@@ -22,6 +23,7 @@ import { MachineLabelCombo } from "~/components/MachineLabelCombo";
 import { MachineTooltipInfo } from "~/components/MachineTooltipInfo";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
 import { Callout } from "~/components/primitives/Callout";
+import { CopyableText } from "~/components/primitives/CopyableText";
 import { DateTime, DateTimeAccurate } from "~/components/primitives/DateTime";
 import { Header2, Header3 } from "~/components/primitives/Headers";
 import { Paragraph } from "~/components/primitives/Paragraph";
@@ -76,7 +78,6 @@ import {
 } from "~/utils/pathBuilder";
 import { createTimelineSpanEventsFromSpanEvents } from "~/utils/timelineSpanEvents";
 import { CompleteWaitpointForm } from "../resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.$waitpointFriendlyId.complete/route";
-import { FlagIcon } from "~/assets/icons/RegionIcons";
 
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const { projectParam, organizationSlug, envParam, runParam, spanParam } =
@@ -409,6 +410,7 @@ function RunBody({
                     <SimpleTooltip
                       button={<TaskRunStatusCombo status={run.status} />}
                       content={descriptionForTaskRunStatus(run.status)}
+                      disableHoverableContent
                     />
                   </Property.Value>
                 </Property.Item>
@@ -422,82 +424,129 @@ function RunBody({
                             tasks: [run.taskIdentifier],
                           })}
                         >
-                          {run.taskIdentifier}
+                          <CopyableText
+                            value={run.taskIdentifier}
+                            copyValue={run.taskIdentifier}
+                            asChild
+                          />
                         </TextLink>
                       }
-                      content={`Filter runs by ${run.taskIdentifier}`}
+                      content={`View runs filtered by ${run.taskIdentifier}`}
+                      disableHoverableContent
                     />
                   </Property.Value>
                 </Property.Item>
                 {run.relationships.root ? (
                   run.relationships.root.isParent ? (
                     <Property.Item>
-                      <Property.Label>Root & Parent</Property.Label>
+                      <Property.Label>Root & Parent run</Property.Label>
                       <Property.Value>
-                        <TextLink
-                          to={v3RunSpanPath(
-                            organization,
-                            project,
-                            environment,
-                            {
-                              friendlyId: run.relationships.root.friendlyId,
-                            },
-                            { spanId: run.relationships.root.spanId }
-                          )}
-                          className="group flex flex-wrap items-center gap-x-1 gap-y-0"
-                        >
-                          {run.relationships.root.taskIdentifier}
-                          <span className="break-all text-text-dimmed transition-colors group-hover:text-text-bright/80">
-                            ({run.relationships.root.friendlyId})
-                          </span>
-                        </TextLink>
-                      </Property.Value>
-                    </Property.Item>
-                  ) : (
-                    <>
-                      <Property.Item>
-                        <Property.Label>Root</Property.Label>
-                        <Property.Value>
-                          <TextLink
-                            to={v3RunSpanPath(
-                              organization,
-                              project,
-                              environment,
-                              {
-                                friendlyId: run.relationships.root.friendlyId,
-                              },
-                              { spanId: run.relationships.root.spanId }
-                            )}
-                            className="group flex flex-wrap items-center gap-x-1 gap-y-0"
-                          >
-                            {run.relationships.root.taskIdentifier}
-                            <span className="break-all text-text-dimmed transition-colors group-hover:text-text-bright/80">
-                              ({run.relationships.root.friendlyId})
-                            </span>
-                          </TextLink>
-                        </Property.Value>
-                      </Property.Item>
-                      {run.relationships.parent ? (
-                        <Property.Item>
-                          <Property.Label>Parent</Property.Label>
-                          <Property.Value>
+                        <SimpleTooltip
+                          button={
                             <TextLink
                               to={v3RunSpanPath(
                                 organization,
                                 project,
                                 environment,
                                 {
-                                  friendlyId: run.relationships.parent.friendlyId,
+                                  friendlyId: run.relationships.root.friendlyId,
                                 },
-                                { spanId: run.relationships.parent.spanId }
+                                { spanId: run.relationships.root.spanId }
                               )}
                               className="group flex flex-wrap items-center gap-x-1 gap-y-0"
                             >
-                              {run.relationships.parent.taskIdentifier}
+                              <CopyableText
+                                value={run.relationships.root.taskIdentifier}
+                                copyValue={run.relationships.root.taskIdentifier}
+                                asChild
+                              />
                               <span className="break-all text-text-dimmed transition-colors group-hover:text-text-bright/80">
-                                ({run.relationships.parent.friendlyId})
+                                <CopyableText
+                                  value={run.relationships.root.friendlyId}
+                                  copyValue={run.relationships.root.friendlyId}
+                                  asChild
+                                />
                               </span>
                             </TextLink>
+                          }
+                          content={`Jump to root/parent run`}
+                          disableHoverableContent
+                        />
+                      </Property.Value>
+                    </Property.Item>
+                  ) : (
+                    <>
+                      <Property.Item>
+                        <Property.Label>Root run</Property.Label>
+                        <Property.Value>
+                          <SimpleTooltip
+                            button={
+                              <TextLink
+                                to={v3RunSpanPath(
+                                  organization,
+                                  project,
+                                  environment,
+                                  {
+                                    friendlyId: run.relationships.root.friendlyId,
+                                  },
+                                  { spanId: run.relationships.root.spanId }
+                                )}
+                                className="group flex flex-wrap items-center gap-x-1 gap-y-0"
+                              >
+                                <CopyableText
+                                  value={run.relationships.root.taskIdentifier}
+                                  copyValue={run.relationships.root.taskIdentifier}
+                                  asChild
+                                />
+                                <span className="break-all text-text-dimmed transition-colors group-hover:text-text-bright/80">
+                                  <CopyableText
+                                    value={run.relationships.root.friendlyId}
+                                    copyValue={run.relationships.root.friendlyId}
+                                    asChild
+                                  />
+                                </span>
+                              </TextLink>
+                            }
+                            content={`Jump to root run`}
+                            disableHoverableContent
+                          />
+                        </Property.Value>
+                      </Property.Item>
+                      {run.relationships.parent ? (
+                        <Property.Item>
+                          <Property.Label>Parent run</Property.Label>
+                          <Property.Value>
+                            <SimpleTooltip
+                              button={
+                                <TextLink
+                                  to={v3RunSpanPath(
+                                    organization,
+                                    project,
+                                    environment,
+                                    {
+                                      friendlyId: run.relationships.parent.friendlyId,
+                                    },
+                                    { spanId: run.relationships.parent.spanId }
+                                  )}
+                                  className="group flex flex-wrap items-center gap-x-1 gap-y-0"
+                                >
+                                  <CopyableText
+                                    value={run.relationships.parent.taskIdentifier}
+                                    copyValue={run.relationships.parent.taskIdentifier}
+                                    asChild
+                                  />
+                                  <span className="break-all text-text-dimmed transition-colors group-hover:text-text-bright/80">
+                                    <CopyableText
+                                      value={run.relationships.parent.friendlyId}
+                                      copyValue={run.relationships.parent.friendlyId}
+                                      asChild
+                                    />
+                                  </span>
+                                </TextLink>
+                              }
+                              content={`Jump to parent run`}
+                              disableHoverableContent
+                            />
                           </Property.Value>
                         </Property.Item>
                       ) : null}
@@ -511,10 +560,15 @@ function RunBody({
                       <SimpleTooltip
                         button={
                           <TextLink to={v3BatchPath(organization, project, environment, run.batch)}>
-                            {run.batch.friendlyId}
+                            <CopyableText
+                              value={run.batch.friendlyId}
+                              copyValue={run.batch.friendlyId}
+                              asChild
+                            />
                           </TextLink>
                         }
-                        content={`Jump to ${run.batch.friendlyId}`}
+                        content={`View batches filtered by ${run.batch.friendlyId}`}
+                        disableHoverableContent
                       />
                     </Property.Value>
                   </Property.Item>
@@ -540,7 +594,7 @@ function RunBody({
                   <Property.Value>
                     {run.version ? (
                       environment.type === "DEVELOPMENT" ? (
-                        run.version
+                        <CopyableText value={run.version} copyValue={run.version} asChild />
                       ) : (
                         <SimpleTooltip
                           button={
@@ -553,7 +607,7 @@ function RunBody({
                               )}
                               className="group flex flex-wrap items-center gap-x-1 gap-y-0"
                             >
-                              {run.version}
+                              <CopyableText value={run.version} copyValue={run.version} asChild />
                             </TextLink>
                           }
                           content={"Jump to deployment"}
@@ -606,13 +660,23 @@ function RunBody({
                   <Property.Item>
                     <Property.Label>Replayed from</Property.Label>
                     <Property.Value>
-                      <TextLink
-                        to={v3RunRedirectPath(organization, project, {
-                          friendlyId: run.replayedFromTaskRunFriendlyId,
-                        })}
-                      >
-                        {run.replayedFromTaskRunFriendlyId}
-                      </TextLink>
+                      <SimpleTooltip
+                        button={
+                          <TextLink
+                            to={v3RunRedirectPath(organization, project, {
+                              friendlyId: run.replayedFromTaskRunFriendlyId,
+                            })}
+                          >
+                            <CopyableText
+                              value={run.replayedFromTaskRunFriendlyId}
+                              copyValue={run.replayedFromTaskRunFriendlyId}
+                              asChild
+                            />
+                          </TextLink>
+                        }
+                        content={`Jump to replayed run`}
+                        disableHoverableContent
+                      />
                     </Property.Value>
                   </Property.Item>
                 )}
@@ -747,11 +811,15 @@ function RunBody({
                 </Property.Item>
                 <Property.Item>
                   <Property.Label>Run ID</Property.Label>
-                  <Property.Value>{run.friendlyId}</Property.Value>
+                  <Property.Value>
+                    <CopyableText value={run.friendlyId} copyValue={run.friendlyId} asChild />
+                  </Property.Value>
                 </Property.Item>
                 <Property.Item>
                   <Property.Label>Internal ID</Property.Label>
-                  <Property.Value>{run.id}</Property.Value>
+                  <Property.Value>
+                    <CopyableText value={run.id} copyValue={run.id} asChild />
+                  </Property.Value>
                 </Property.Item>
                 <Property.Item>
                   <Property.Label>Run Engine</Property.Label>
@@ -772,6 +840,14 @@ function RunBody({
                       <Property.Label>Worker queue</Property.Label>
                       <Property.Value>{run.workerQueue}</Property.Value>
                     </Property.Item>
+                    <Property.Item>
+                      <Property.Label>Trace ID</Property.Label>
+                      <Property.Value>{run.traceId}</Property.Value>
+                    </Property.Item>
+                    <Property.Item>
+                      <Property.Label>Span ID</Property.Label>
+                      <Property.Value>{run.spanId}</Property.Value>
+                    </Property.Item>
                   </div>
                 )}
               </Property.Table>

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.bulkaction.$bulkActionParam.stream.tsx

@@ -1,103 +0,0 @@
-import { z } from "zod";
-import { $replica } from "~/db.server";
-import { env } from "~/env.server";
-import { logger } from "~/services/logger.server";
-import { requireUserId } from "~/services/session.server";
-import { EnvironmentParamSchema } from "~/utils/pathBuilder";
-import { createSSELoader, type SendFunction } from "~/utils/sse";
-
-const Params = EnvironmentParamSchema.extend({
-  bulkActionParam: z.string(),
-});
-
-export const loader = createSSELoader({
-  timeout: env.DEV_PRESENCE_SSE_TIMEOUT,
-  interval: env.DEV_PRESENCE_POLL_MS,
-  debug: false,
-  handler: async ({ id, controller, debug, request, params }) => {
-    const userId = await requireUserId(request);
-    const { organizationSlug, projectParam, envParam, bulkActionParam } = Params.parse(params);
-
-    const environment = await $replica.runtimeEnvironment.findFirst({
-      where: {
-        id: envParam,
-        project: {
-          slug: projectParam,
-          organization: {
-            members: {
-              some: {
-                userId,
-              },
-            },
-          },
-        },
-      },
-    });
-
-    if (!environment) {
-      throw new Response("Not Found", { status: 404 });
-    }
-
-    const getBulkActionProgress = async (send: SendFunction) => {
-      try {
-        const bulkAction = await $replica.bulkActionGroup.findFirst({
-          select: {
-            status: true,
-            successCount: true,
-            failureCount: true,
-          },
-          where: {
-            friendlyId: bulkActionParam,
-            environmentId: environment.id,
-          },
-        });
-
-        send({
-          event: "progress",
-          data: JSON.stringify({
-            status: bulkAction?.status,
-            successCount: bulkAction?.successCount,
-            failureCount: bulkAction?.failureCount,
-          }),
-        });
-
-        return bulkAction;
-      } catch (error) {
-        // Handle the case where the controller is closed
-        logger.debug("Failed to send bulk action progress data, stream might be closed", { error });
-        return null;
-      }
-    };
-
-    return {
-      beforeStream: async () => {
-        logger.debug("Start dev presence listening SSE session", {
-          environmentId: environment.id,
-        });
-      },
-      initStream: async ({ send }) => {
-        const bulkAction = await getBulkActionProgress(send);
-
-        send({ event: "time", data: new Date().toISOString() });
-
-        if (bulkAction?.status !== "PENDING") {
-          return false;
-        }
-
-        return true;
-      },
-      iterator: async ({ send, date }) => {
-        const bulkAction = await getBulkActionProgress(send);
-
-        if (bulkAction?.status !== "PENDING") {
-          return false;
-        }
-
-        return true;
-      },
-      cleanup: async ({ send }) => {
-        await getBulkActionProgress(send);
-      },
-    };
-  },
-});

apps/webapp/app/routes/resources.orgs.$organizationSlug.select-plan.tsx

@@ -171,6 +171,10 @@ const pricingDefinitions = {
     title: "Concurrent runs",
     content: "The number of runs that can be executed at the same time.",
   },
+  additionalConcurrency: {
+    title: "Additional concurrency",
+    content: "Then $50/month per 50",
+  },
   taskRun: {
     title: "Task runs",
     content: "A single execution of a task.",
@@ -188,6 +192,10 @@ const pricingDefinitions = {
     title: "Schedules",
     content: "You can attach recurring schedules to tasks using cron syntax.",
   },
+  additionalSchedules: {
+    title: "Additional schedules",
+    content: "Then $10/month per 1,000",
+  },
   alerts: {
     title: "Alert destination",
     content:
@@ -198,9 +206,22 @@ const pricingDefinitions = {
     content:
       "Realtime allows you to send the live status and data from your runs to your frontend. This is the number of simultaneous Realtime connections that can be made.",
   },
+  additionalRealtimeConnections: {
+    title: "Additional Realtime connections",
+    content: "Then $10/month per 100",
+  },
+  additionalSeats: {
+    title: "Additional seats",
+    content: "Then $20/month per seat",
+  },
   branches: {
     title: "Branches",
-    content: "The number of preview branches that can be active (you can archive old ones).",
+    content:
+      "Preview branches allow you to test changes before deploying to production. You can have a limited number active at once (but can archive old ones).",
+  },
+  additionalBranches: {
+    title: "Additional branches",
+    content: "Then $10/month per branch",
   },
 };
 
@@ -338,7 +359,7 @@ export function TierFree({
                 <div className="my-6">
                   <Button
                     type="button"
-                    variant="tertiary/large"
+                    variant="secondary/large"
                     fullWidth
                     className="text-md font-medium"
                     disabled={isLoading}
@@ -384,7 +405,7 @@ export function TierFree({
             <Dialog open={isDialogOpen} onOpenChange={setIsDialogOpen} key="cancel">
               <DialogTrigger asChild>
                 <div className="my-6">
-                  <Button variant="tertiary/large" fullWidth className="text-md font-medium">
+                  <Button variant="secondary/large" fullWidth className="text-md font-medium">
                     {`Downgrade to ${plan.title}`}
                   </Button>
                 </div>
@@ -411,6 +432,7 @@ export function TierFree({
                       <Header2 className="mb-1">Why are you thinking of downgrading?</Header2>
                       <ul className="space-y-1">
                         {[
+                          "The Free plan is all I need",
                           "Subscription or usage costs too expensive",
                           "Bugs or technical issues",
                           "No longer need the service",
@@ -445,7 +467,7 @@ export function TierFree({
                     </div>
                   </div>
                   <DialogFooter className="mt-2">
-                    <Button variant="tertiary/medium" onClick={() => setIsDialogOpen(false)}>
+                    <Button variant="secondary/medium" onClick={() => setIsDialogOpen(false)}>
                       Dismiss
                     </Button>
                     <Button
@@ -465,7 +487,7 @@ export function TierFree({
               <input type="hidden" name="type" value="free" />
               <input type="hidden" name="callerPath" value={location.pathname} />
               <Button
-                variant="tertiary/large"
+                variant="secondary/large"
                 type="submit"
                 form="subscribe-verified"
                 fullWidth
@@ -507,7 +529,7 @@ export function TierFree({
             <LogRetention limits={plan.limits} />
             <SupportLevel limits={plan.limits} />
             <Alerts limits={plan.limits} />
-            <RealtimeConnecurrency limits={plan.limits} />
+            <RealtimeConcurrency limits={plan.limits} />
           </ul>
         </>
       )}
@@ -552,7 +574,7 @@ export function TierHobby({
         subscription.plan.code !== plan.code ? (
           <Dialog open={isDialogOpen} onOpenChange={setIsDialogOpen} key="downgrade">
             <DialogTrigger asChild>
-              <Button variant="tertiary/large" fullWidth className="text-md font-medium">
+              <Button variant="secondary/large" fullWidth className="text-md font-medium">
                 {`Downgrade to ${plan.title}`}
               </Button>
             </DialogTrigger>
@@ -568,11 +590,11 @@ export function TierHobby({
                 </Paragraph>
               </div>
               <DialogFooter>
-                <Button variant="tertiary/medium" onClick={() => setIsDialogOpen(false)}>
+                <Button variant="secondary/medium" onClick={() => setIsDialogOpen(false)}>
                   Dismiss
                 </Button>
                 <Button
-                  variant="tertiary/medium"
+                  variant="secondary/medium"
                   disabled={isLoading}
                   LeadingIcon={isLoading ? () => <Spinner color="white" /> : undefined}
                   form="subscribe-hobby"
@@ -584,7 +606,7 @@ export function TierHobby({
           </Dialog>
         ) : (
           <Button
-            variant={isHighlighted ? "primary/large" : "tertiary/large"}
+            variant={isHighlighted ? "primary/large" : "secondary/large"}
             fullWidth
             className="text-md font-medium"
             form="subscribe-hobby"
@@ -624,7 +646,7 @@ export function TierHobby({
         <LogRetention limits={plan.limits} />
         <SupportLevel limits={plan.limits} />
         <Alerts limits={plan.limits} />
-        <RealtimeConnecurrency limits={plan.limits} />
+        <RealtimeConcurrency limits={plan.limits} />
       </ul>
     </TierContainer>
   );
@@ -666,7 +688,7 @@ export function TierPro({
           subscription.canceledAt === undefined ? (
             <Dialog open={isDialogOpen} onOpenChange={setIsDialogOpen} key="upgrade">
               <DialogTrigger asChild>
-                <Button variant="tertiary/large" fullWidth className="text-md font-medium">
+                <Button variant="secondary/large" fullWidth className="text-md font-medium">
                   {`Upgrade to ${plan.title}`}
                 </Button>
               </DialogTrigger>
@@ -682,7 +704,7 @@ export function TierPro({
                   </Paragraph>
                 </div>
                 <DialogFooter>
-                  <Button variant="tertiary/medium" onClick={() => setIsDialogOpen(false)}>
+                  <Button variant="secondary/medium" onClick={() => setIsDialogOpen(false)}>
                     Dismiss
                   </Button>
                   <Button
@@ -698,7 +720,7 @@ export function TierPro({
             </Dialog>
           ) : (
             <Button
-              variant="tertiary/large"
+              variant="secondary/large"
               fullWidth
               form="subscribe-pro"
               className="text-md font-medium"
@@ -724,7 +746,9 @@ export function TierPro({
         </div>
       </Form>
       <ul className="flex flex-col gap-2.5">
-        <ConcurrentRuns limits={plan.limits} />
+        <ConcurrentRuns limits={plan.limits}>
+          {pricingDefinitions.additionalConcurrency.content}
+        </ConcurrentRuns>
         <FeatureItem checked>
           Unlimited{" "}
           <DefinitionTip
@@ -734,14 +758,16 @@ export function TierPro({
             tasks
           </DefinitionTip>
         </FeatureItem>
-        <TeamMembers limits={plan.limits} />
+        <TeamMembers limits={plan.limits}>{pricingDefinitions.additionalSeats.content}</TeamMembers>
         <Environments limits={plan.limits} />
-        <Branches limits={plan.limits} />
-        <Schedules limits={plan.limits} />
+        <Branches limits={plan.limits}>{pricingDefinitions.additionalBranches.content}</Branches>
+        <Schedules limits={plan.limits}>{pricingDefinitions.additionalSchedules.content}</Schedules>
         <LogRetention limits={plan.limits} />
         <SupportLevel limits={plan.limits} />
         <Alerts limits={plan.limits} />
-        <RealtimeConnecurrency limits={plan.limits} />
+        <RealtimeConcurrency limits={plan.limits}>
+          {pricingDefinitions.additionalRealtimeConnections.content}
+        </RealtimeConcurrency>
       </ul>
     </TierContainer>
   );
@@ -788,11 +814,11 @@ export function TierEnterprise() {
           <Feedback
             defaultValue="enterprise"
             button={
-              <div className="flex h-10 w-full cursor-pointer items-center justify-center rounded bg-tertiary px-8 text-base font-medium transition hover:bg-charcoal-600">
+              <div className="flex h-10 w-full cursor-pointer items-center justify-center rounded border border-charcoal-600 bg-tertiary px-8 text-base font-medium transition hover:border-charcoal-550 hover:bg-charcoal-600">
                 <span className="text-center text-text-bright">Contact us</span>
               </div>
             }
-          ></Feedback>
+          />
         </div>
       </div>
     </TierContainer>
@@ -812,7 +838,7 @@ function TierContainer({
     <div
       className={cn(
         "flex w-full min-w-[16rem] flex-col p-6",
-        isHighlighted ? "border border-primary" : "border border-grid-dimmed",
+        isHighlighted ? "border border-indigo-500" : "border border-grid-dimmed",
         className
       )}
     >
@@ -843,7 +869,10 @@ function PricingHeader({
   return (
     <div className="flex flex-col gap-2">
       <h2
-        className={cn("text-xl font-medium", isHighlighted ? "text-primary" : "text-text-dimmed")}
+        className={cn(
+          "text-xl font-medium",
+          isHighlighted ? "text-indigo-500" : "text-text-dimmed"
+        )}
       >
         {title}
       </h2>
@@ -899,16 +928,16 @@ function FeatureItem({
   children: React.ReactNode;
 }) {
   return (
-    <li className="flex items-center gap-2">
+    <li className="flex items-start gap-2">
       {checked ? (
         <CheckIcon
           className={cn(
-            "size-4 min-w-4",
+            "mt-0.5 size-4 min-w-4",
             checkedColor === "primary" ? "text-primary" : "text-text-bright"
           )}
         />
       ) : (
-        <XMarkIcon className="size-4 min-w-4 text-charcoal-500" />
+        <XMarkIcon className="mt-0.5 size-4 min-w-4 text-charcoal-500" />
       )}
       <div
         className={cn(
@@ -922,26 +951,42 @@ function FeatureItem({
   );
 }
 
-function ConcurrentRuns({ limits }: { limits: Limits }) {
+function ConcurrentRuns({ limits, children }: { limits: Limits; children?: React.ReactNode }) {
   return (
     <FeatureItem checked>
-      {limits.concurrentRuns.number}
-      {limits.concurrentRuns.canExceed ? "+" : ""}{" "}
-      <DefinitionTip
-        title={pricingDefinitions.concurrentRuns.title}
-        content={pricingDefinitions.concurrentRuns.content}
-      >
-        concurrent runs
-      </DefinitionTip>
+      <div className="flex flex-col gap-y-0.5">
+        <div className="flex items-center gap-1">
+          {limits.concurrentRuns.canExceed ? (
+            <>
+              {limits.concurrentRuns.number}
+              {"+"}
+            </>
+          ) : (
+            <>{limits.concurrentRuns.number} </>
+          )}{" "}
+          <DefinitionTip
+            title={pricingDefinitions.concurrentRuns.title}
+            content={pricingDefinitions.concurrentRuns.content}
+          >
+            concurrent runs
+          </DefinitionTip>
+        </div>
+        {children && <span className="text-xs text-text-dimmed">{children}</span>}
+      </div>
     </FeatureItem>
   );
 }
 
-function TeamMembers({ limits }: { limits: Limits }) {
+function TeamMembers({ limits, children }: { limits: Limits; children?: React.ReactNode }) {
   return (
     <FeatureItem checked>
-      {limits.teamMembers.number}
-      {limits.concurrentRuns.canExceed ? "+" : ""} team members
+      <div className="flex flex-col gap-y-0.5">
+        <div className="flex items-center gap-1">
+          {limits.teamMembers.number}
+          {limits.teamMembers.canExceed ? "+" : ""} team members
+        </div>
+        {children && <span className="text-xs text-text-dimmed">{children}</span>}
+      </div>
     </FeatureItem>
   );
 }
@@ -960,17 +1005,22 @@ function Environments({ limits }: { limits: Limits }) {
   );
 }
 
-function Schedules({ limits }: { limits: Limits }) {
+function Schedules({ limits, children }: { limits: Limits; children?: React.ReactNode }) {
   return (
     <FeatureItem checked>
-      {limits.schedules.number}
-      {limits.schedules.canExceed ? "+" : ""}{" "}
-      <DefinitionTip
-        title={pricingDefinitions.schedules.title}
-        content={pricingDefinitions.schedules.content}
-      >
-        schedules
-      </DefinitionTip>
+      <div className="flex flex-col gap-y-0.5">
+        <div className="flex items-center gap-1">
+          {limits.schedules.number}
+          {limits.schedules.canExceed ? "+" : ""}{" "}
+          <DefinitionTip
+            title={pricingDefinitions.schedules.title}
+            content={pricingDefinitions.schedules.content}
+          >
+            schedules
+          </DefinitionTip>
+        </div>
+        {children && <span className="text-xs text-text-dimmed">{children}</span>}
+      </div>
     </FeatureItem>
   );
 }
@@ -1015,32 +1065,52 @@ function Alerts({ limits }: { limits: Limits }) {
   );
 }
 
-function RealtimeConnecurrency({ limits }: { limits: Limits }) {
+function RealtimeConcurrency({ limits, children }: { limits: Limits; children?: React.ReactNode }) {
   return (
     <FeatureItem checked>
-      {limits.realtimeConcurrentConnections.number}
-      {limits.realtimeConcurrentConnections.canExceed ? "+" : ""}{" "}
-      <DefinitionTip
-        title={pricingDefinitions.realtime.title}
-        content={pricingDefinitions.realtime.content}
-      >
-        concurrent Realtime connections
-      </DefinitionTip>
+      <div className="flex flex-col gap-y-0.5">
+        <div className="flex items-start gap-1">
+          {limits.realtimeConcurrentConnections.canExceed ? (
+            <>
+              {limits.realtimeConcurrentConnections.number}
+              {"+"}
+            </>
+          ) : (
+            <>{limits.realtimeConcurrentConnections.number} </>
+          )}{" "}
+          <DefinitionTip
+            title={pricingDefinitions.realtime.title}
+            content={pricingDefinitions.realtime.content}
+          >
+            concurrent Realtime connections
+          </DefinitionTip>
+        </div>
+        {children && <span className="text-xs text-text-dimmed">{children}</span>}
+      </div>
     </FeatureItem>
   );
 }
 
-function Branches({ limits }: { limits: Limits }) {
+function Branches({ limits, children }: { limits: Limits; children?: React.ReactNode }) {
   return (
     <FeatureItem checked={limits.branches.number > 0}>
-      {limits.branches.number}
-      {limits.branches.canExceed ? "+ " : " "}
-      <DefinitionTip
-        title={pricingDefinitions.branches.title}
-        content={pricingDefinitions.branches.content}
-      >
-        preview branches
-      </DefinitionTip>
+      <div className="flex flex-col gap-y-0.5">
+        <div className="flex items-center gap-1">
+          {limits.branches.number > 0 && (
+            <>
+              {limits.branches.number}
+              {limits.branches.canExceed ? "+ " : " "}
+            </>
+          )}
+          <DefinitionTip
+            title={pricingDefinitions.branches.title}
+            content={pricingDefinitions.branches.content}
+          >
+            {limits.branches.number > 0 ? "preview" : "Preview"} branches
+          </DefinitionTip>
+        </div>
+        {children && <span className="text-xs text-text-dimmed">{children}</span>}
+      </div>
     </FeatureItem>
   );
 }

apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts

@@ -108,6 +108,8 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
   const disableVersionSelection = environment.type === "DEVELOPMENT";
   const allowArbitraryQueues = backgroundWorkers.at(0)?.engine === "V1";
 
+  const payload = await prettyPrintPacket(run.payload, run.payloadType);
+
   return typedjson({
     concurrencyKey: run.concurrencyKey,
     maxAttempts: run.maxAttempts,
@@ -116,7 +118,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
     ttlSeconds: run.ttl ? parseDuration(run.ttl, "s") ?? undefined : undefined,
     idempotencyKey: run.idempotencyKey,
     runTags: run.runTags,
-    payload: await prettyPrintPacket(run.payload, run.payloadType),
+    payload,
     payloadType: run.payloadType,
     queue: run.queue,
     metadata: run.seedMetadata

apps/webapp/app/runEngine/concerns/errors.ts

@@ -1,6 +0,0 @@
-export class EngineServiceValidationError extends Error {
-  constructor(message: string, public status?: number) {
-    super(message);
-    this.name = "EngineServiceValidationError";
-  }
-}

apps/webapp/app/runEngine/concerns/idempotencyKeys.server.ts

@@ -90,11 +90,18 @@ export class IdempotencyKeyConcern {
             isError: associatedWaitpoint.outputIsError,
           },
           async (event) => {
+            const spanId =
+              request.options?.parentAsLinkType === "replay"
+                ? event.spanId
+                : event.traceparent?.spanId
+                ? `${event.traceparent.spanId}:${event.spanId}`
+                : event.spanId;
+
             //block run with waitpoint
             await this.engine.blockRunWithWaitpoint({
               runId: RunId.fromFriendlyId(parentRunId),
               waitpoints: associatedWaitpoint.id,
-              spanIdToComplete: event.spanId,
+              spanIdToComplete: spanId,
               batch: request.options?.batchId
                 ? {
                     id: request.options.batchId,

apps/webapp/app/runEngine/concerns/payloads.server.ts

@@ -3,7 +3,7 @@ import { PayloadProcessor, TriggerTaskRequest } from "../types";
 import { env } from "~/env.server";
 import { startActiveSpan } from "~/v3/tracer.server";
 import { uploadPacketToObjectStore } from "~/v3/r2.server";
-import { EngineServiceValidationError } from "./errors";
+import { ServiceValidationError } from "~/v3/services/common.server";
 
 export class DefaultPayloadProcessor implements PayloadProcessor {
   async process(request: TriggerTaskRequest): Promise<IOPacket> {
@@ -36,10 +36,7 @@ export class DefaultPayloadProcessor implements PayloadProcessor {
       );
 
       if (uploadError) {
-        throw new EngineServiceValidationError(
-          "Failed to upload large payload to object store",
-          500
-        ); // This is retryable
+        throw new ServiceValidationError("Failed to upload large payload to object store", 500); // This is retryable
       }
 
       return {

apps/webapp/app/runEngine/concerns/queues.server.ts

@@ -13,8 +13,8 @@ import {
 import { WorkerGroupService } from "~/v3/services/worker/workerGroupService.server";
 import type { RunEngine } from "~/v3/runEngine.server";
 import { env } from "~/env.server";
-import { EngineServiceValidationError } from "./errors";
 import { tryCatch } from "@trigger.dev/core/v3";
+import { ServiceValidationError } from "~/v3/services/common.server";
 
 export class DefaultQueueManager implements QueueManager {
   constructor(
@@ -45,7 +45,7 @@ export class DefaultQueueManager implements QueueManager {
         });
 
         if (!specifiedQueue) {
-          throw new EngineServiceValidationError(
+          throw new ServiceValidationError(
             `Specified queue '${specifiedQueueName}' not found or not associated with locked version '${
               lockedBackgroundWorker.version ?? "<unknown>"
             }'.`
@@ -68,7 +68,7 @@ export class DefaultQueueManager implements QueueManager {
         });
 
         if (!lockedTask) {
-          throw new EngineServiceValidationError(
+          throw new ServiceValidationError(
             `Task '${request.taskId}' not found on locked version '${
               lockedBackgroundWorker.version ?? "<unknown>"
             }'.`
@@ -83,7 +83,7 @@ export class DefaultQueueManager implements QueueManager {
             workerId: lockedBackgroundWorker.id,
             version: lockedBackgroundWorker.version,
           });
-          throw new EngineServiceValidationError(
+          throw new ServiceValidationError(
             `Default queue configuration for task '${request.taskId}' missing on locked version '${
               lockedBackgroundWorker.version ?? "<unknown>"
             }'.`
@@ -97,7 +97,7 @@ export class DefaultQueueManager implements QueueManager {
       // Task is not locked to a specific version, use regular logic
       if (request.body.options?.lockToVersion) {
         // This should only happen if the findFirst failed, indicating the version doesn't exist
-        throw new EngineServiceValidationError(
+        throw new ServiceValidationError(
           `Task locked to version '${request.body.options.lockToVersion}', but no worker found with that version.`
         );
       }
@@ -221,11 +221,11 @@ export class DefaultQueueManager implements QueueManager {
     );
 
     if (error) {
-      throw new EngineServiceValidationError(error.message);
+      throw new ServiceValidationError(error.message);
     }
 
     if (!workerGroup) {
-      throw new EngineServiceValidationError("No worker group found");
+      throw new ServiceValidationError("No worker group found");
     }
 
     return workerGroup.masterQueue;

apps/webapp/app/runEngine/services/triggerTask.server.ts

@@ -31,16 +31,24 @@ import type {
 } from "../../v3/services/triggerTask.server";
 import { getTaskEventStore } from "../../v3/taskEventStore.server";
 import { clampMaxDuration } from "../../v3/utils/maxDuration";
-import { EngineServiceValidationError } from "../concerns/errors";
 import { IdempotencyKeyConcern } from "../concerns/idempotencyKeys.server";
 import type {
   PayloadProcessor,
   QueueManager,
   RunNumberIncrementer,
   TraceEventConcern,
+  TriggerRacepoints,
+  TriggerRacepointSystem,
   TriggerTaskRequest,
   TriggerTaskValidator,
 } from "../types";
+import { ServiceValidationError } from "~/v3/services/common.server";
+
+class NoopTriggerRacepointSystem implements TriggerRacepointSystem {
+  async waitForRacepoint(options: { racepoint: TriggerRacepoints; id: string }): Promise<void> {
+    return;
+  }
+}
 
 export class RunEngineTriggerTaskService {
   private readonly queueConcern: QueueManager;
@@ -52,6 +60,7 @@ export class RunEngineTriggerTaskService {
   private readonly engine: RunEngine;
   private readonly tracer: Tracer;
   private readonly traceEventConcern: TraceEventConcern;
+  private readonly triggerRacepointSystem: TriggerRacepointSystem;
   private readonly metadataMaximumSize: number;
 
   constructor(opts: {
@@ -65,6 +74,7 @@ export class RunEngineTriggerTaskService {
     traceEventConcern: TraceEventConcern;
     tracer: Tracer;
     metadataMaximumSize: number;
+    triggerRacepointSystem?: TriggerRacepointSystem;
   }) {
     this.prisma = opts.prisma;
     this.engine = opts.engine;
@@ -76,6 +86,7 @@ export class RunEngineTriggerTaskService {
     this.tracer = opts.tracer;
     this.traceEventConcern = opts.traceEventConcern;
     this.metadataMaximumSize = opts.metadataMaximumSize;
+    this.triggerRacepointSystem = opts.triggerRacepointSystem ?? new NoopTriggerRacepointSystem();
   }
 
   public async call({
@@ -157,7 +168,7 @@ export class RunEngineTriggerTaskService {
       const [parseDelayError, delayUntil] = await tryCatch(parseDelay(body.options?.delay));
 
       if (parseDelayError) {
-        throw new EngineServiceValidationError(`Invalid delay ${body.options?.delay}`);
+        throw new ServiceValidationError(`Invalid delay ${body.options?.delay}`);
       }
 
       const ttl =
@@ -196,21 +207,18 @@ export class RunEngineTriggerTaskService {
 
       const { idempotencyKey, idempotencyKeyExpiresAt } = idempotencyKeyConcernResult;
 
+      if (idempotencyKey) {
+        await this.triggerRacepointSystem.waitForRacepoint({
+          racepoint: "idempotencyKey",
+          id: idempotencyKey,
+        });
+      }
+
       if (!options.skipChecks) {
         const queueSizeGuard = await this.queueConcern.validateQueueLimits(environment);
 
-        logger.debug("Queue size guard result", {
-          queueSizeGuard,
-          environment: {
-            id: environment.id,
-            type: environment.type,
-            organization: environment.organization,
-            project: environment.project,
-          },
-        });
-
         if (!queueSizeGuard.ok) {
-          throw new EngineServiceValidationError(
+          throw new ServiceValidationError(
             `Cannot trigger ${taskId} as the queue size limit for this environment has been reached. The maximum size is ${queueSizeGuard.maximumSize}`
           );
         }
@@ -351,7 +359,7 @@ export class RunEngineTriggerTaskService {
           );
 
           if (result?.error) {
-            throw new EngineServiceValidationError(
+            throw new ServiceValidationError(
               taskRunErrorToString(taskRunErrorEnhancer(result.error))
             );
           }
@@ -365,7 +373,7 @@ export class RunEngineTriggerTaskService {
         }
 
         if (error instanceof RunOneTimeUseTokenError) {
-          throw new EngineServiceValidationError(
+          throw new ServiceValidationError(
             `Cannot trigger ${taskId} with a one-time use token as it has already been used.`
           );
         }

apps/webapp/app/runEngine/types.ts

@@ -156,3 +156,9 @@ export interface TraceEventConcern {
     callback: (span: TracedEventSpan) => Promise<T>
   ): Promise<T>;
 }
+
+export type TriggerRacepoints = "idempotencyKey";
+
+export interface TriggerRacepointSystem {
+  waitForRacepoint(options: { racepoint: TriggerRacepoints; id: string }): Promise<void>;
+}

apps/webapp/app/runEngine/validators/triggerTaskValidator.ts

@@ -3,7 +3,6 @@ import { logger } from "~/services/logger.server";
 import { getEntitlement } from "~/services/platform.v3.server";
 import { MAX_ATTEMPTS, OutOfEntitlementError } from "~/v3/services/triggerTask.server";
 import { isFinalRunStatus } from "~/v3/taskStatus";
-import { EngineServiceValidationError } from "../concerns/errors";
 import type {
   EntitlementValidationParams,
   EntitlementValidationResult,
@@ -13,6 +12,7 @@ import type {
   TriggerTaskValidator,
   ValidationResult,
 } from "../types";
+import { ServiceValidationError } from "~/v3/services/common.server";
 
 export class DefaultTriggerTaskValidator implements TriggerTaskValidator {
   validateTags(params: TagValidationParams): ValidationResult {
@@ -29,7 +29,7 @@ export class DefaultTriggerTaskValidator implements TriggerTaskValidator {
     if (tags.length > MAX_TAGS_PER_RUN) {
       return {
         ok: false,
-        error: new EngineServiceValidationError(
+        error: new ServiceValidationError(
           `Runs can only have ${MAX_TAGS_PER_RUN} tags, you're trying to set ${tags.length}.`
         ),
       };
@@ -65,7 +65,7 @@ export class DefaultTriggerTaskValidator implements TriggerTaskValidator {
     if (attempt > MAX_ATTEMPTS) {
       return {
         ok: false,
-        error: new EngineServiceValidationError(
+        error: new ServiceValidationError(
           `Failed to trigger ${taskId} after ${MAX_ATTEMPTS} attempts.`
         ),
       };
@@ -95,7 +95,7 @@ export class DefaultTriggerTaskValidator implements TriggerTaskValidator {
 
       return {
         ok: false,
-        error: new EngineServiceValidationError(
+        error: new ServiceValidationError(
           `Cannot trigger ${taskId} as the parent run has a status of ${parentRun.status}`
         ),
       };

apps/webapp/app/services/archiveBranch.server.ts

@@ -10,18 +10,34 @@ export class ArchiveBranchService {
     this.#prismaClient = prismaClient;
   }
 
-  public async call(userId: string, { environmentId }: { environmentId: string }) {
+  public async call(
+    // The orgFilter approach is not ideal but we need to keep it this way for now because of how the service is used in routes and api endpoints.
+    // Currently authorization checks are spread across the controller/route layer and the service layer. Often we check in multiple places for org/project membership.
+    // Ideally we would take care of both the authentication and authorization checks in the controllers and routes.
+    // That would unify how we handle authorization and org/project membership checks. Also it would make the service layer queries simpler.
+    orgFilter:
+      | { type: "userMembership"; userId: string }
+      | { type: "orgId"; organizationId: string },
+    {
+      environmentId,
+    }: {
+      environmentId: string;
+    }
+  ) {
     try {
       const environment = await this.#prismaClient.runtimeEnvironment.findFirstOrThrow({
         where: {
           id: environmentId,
-          organization: {
-            members: {
-              some: {
-                userId: userId,
-              },
-            },
-          },
+          organization:
+            orgFilter.type === "userMembership"
+              ? {
+                  members: {
+                    some: {
+                      userId: orgFilter.userId,
+                    },
+                  },
+                }
+              : { id: orgFilter.organizationId },
         },
         include: {
           organization: {

apps/webapp/app/services/gitHub.server.ts

@@ -2,6 +2,7 @@ import { App, type Octokit } from "octokit";
 import { env } from "../env.server";
 import { prisma } from "~/db.server";
 import { logger } from "./logger.server";
+import { errAsync, fromPromise, okAsync, type ResultAsync } from "neverthrow";
 
 export const githubApp =
   env.GITHUB_APP_ENABLED === "1"
@@ -133,3 +134,57 @@ async function fetchInstallationRepositories(octokit: Octokit, installationId: n
     defaultBranch: repo.default_branch,
   }));
 }
+
+/**
+ * Checks if a branch exists in a GitHub repository
+ */
+export function checkGitHubBranchExists(
+  installationId: number,
+  fullRepoName: string,
+  branch: string
+): ResultAsync<boolean, { type: "other" | "github_app_not_enabled"; cause?: unknown }> {
+  if (!githubApp) {
+    return errAsync({ type: "github_app_not_enabled" as const });
+  }
+
+  if (!branch || branch.trim() === "") {
+    return okAsync(false);
+  }
+
+  const [owner, repo] = fullRepoName.split("/");
+
+  const getOctokit = () =>
+    fromPromise(githubApp.getInstallationOctokit(installationId), (error) => ({
+      type: "other" as const,
+      cause: error,
+    }));
+
+  const getBranch = (octokit: Octokit) =>
+    fromPromise(
+      octokit.rest.repos.getBranch({
+        owner,
+        repo,
+        branch,
+      }),
+      (error) => ({
+        type: "other" as const,
+        cause: error,
+      })
+    );
+
+  return getOctokit()
+    .andThen((octokit) => getBranch(octokit))
+    .map(() => true)
+    .orElse((error) => {
+      if (
+        error.cause &&
+        error.cause instanceof Error &&
+        "status" in error.cause &&
+        error.cause.status === 404
+      ) {
+        return okAsync(false);
+      }
+
+      return errAsync(error);
+    });
+}

apps/webapp/app/services/projectSettings.server.ts

@@ -0,0 +1,299 @@
+import { type PrismaClient } from "@trigger.dev/database";
+import { prisma } from "~/db.server";
+import { DeleteProjectService } from "~/services/deleteProject.server";
+import { BranchTrackingConfigSchema, type BranchTrackingConfig } from "~/v3/github";
+import { checkGitHubBranchExists } from "~/services/gitHub.server";
+import { errAsync, fromPromise, okAsync, ResultAsync } from "neverthrow";
+import { BuildSettings } from "~/v3/buildSettings";
+
+export class ProjectSettingsService {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  renameProject(projectId: string, newName: string) {
+    return fromPromise(
+      this.#prismaClient.project.update({
+        where: {
+          id: projectId,
+        },
+        data: {
+          name: newName,
+        },
+      }),
+      (error) => ({
+        type: "other" as const,
+        cause: error,
+      })
+    );
+  }
+
+  deleteProject(projectSlug: string, userId: string) {
+    const deleteProjectService = new DeleteProjectService(this.#prismaClient);
+
+    return fromPromise(deleteProjectService.call({ projectSlug, userId }), (error) => ({
+      type: "other" as const,
+      cause: error,
+    }));
+  }
+
+  connectGitHubRepo(
+    projectId: string,
+    organizationId: string,
+    repositoryId: string,
+    installationId: string
+  ) {
+    const getRepository = () =>
+      fromPromise(
+        this.#prismaClient.githubRepository.findFirst({
+          where: {
+            id: repositoryId,
+            installationId,
+            installation: {
+              organizationId: organizationId,
+            },
+          },
+          select: {
+            id: true,
+            name: true,
+            defaultBranch: true,
+          },
+        }),
+        (error) => ({
+          type: "other" as const,
+          cause: error,
+        })
+      ).andThen((repository) => {
+        if (!repository) {
+          return errAsync({ type: "gh_repository_not_found" as const });
+        }
+        return okAsync(repository);
+      });
+
+    const findExistingConnection = () =>
+      fromPromise(
+        this.#prismaClient.connectedGithubRepository.findFirst({
+          where: {
+            projectId: projectId,
+          },
+        }),
+        (error) => ({ type: "other" as const, cause: error })
+      );
+
+    const createConnectedRepo = (defaultBranch: string) =>
+      fromPromise(
+        this.#prismaClient.connectedGithubRepository.create({
+          data: {
+            projectId: projectId,
+            repositoryId: repositoryId,
+            branchTracking: {
+              prod: { branch: defaultBranch },
+              staging: {},
+            } satisfies BranchTrackingConfig,
+            previewDeploymentsEnabled: true,
+          },
+        }),
+        (error) => ({ type: "other" as const, cause: error })
+      );
+
+    return ResultAsync.combine([getRepository(), findExistingConnection()]).andThen(
+      ([repository, existingConnection]) => {
+        if (existingConnection) {
+          return errAsync({ type: "project_already_has_connected_repository" as const });
+        }
+
+        return createConnectedRepo(repository.defaultBranch);
+      }
+    );
+  }
+
+  disconnectGitHubRepo(projectId: string) {
+    return fromPromise(
+      this.#prismaClient.connectedGithubRepository.delete({
+        where: {
+          projectId: projectId,
+        },
+      }),
+      (error) => ({ type: "other" as const, cause: error })
+    );
+  }
+
+  updateGitSettings(
+    projectId: string,
+    productionBranch?: string,
+    stagingBranch?: string,
+    previewDeploymentsEnabled?: boolean
+  ) {
+    const getExistingConnectedRepo = () =>
+      fromPromise(
+        this.#prismaClient.connectedGithubRepository.findFirst({
+          where: {
+            projectId: projectId,
+          },
+          include: {
+            repository: {
+              include: {
+                installation: true,
+              },
+            },
+          },
+        }),
+        (error) => ({ type: "other" as const, cause: error })
+      )
+        .andThen((connectedRepo) => {
+          if (!connectedRepo) {
+            return errAsync({ type: "connected_gh_repository_not_found" as const });
+          }
+          return okAsync(connectedRepo);
+        })
+        .map((connectedRepo) => {
+          const branchTrackingOrFailure = BranchTrackingConfigSchema.safeParse(
+            connectedRepo.branchTracking
+          );
+          const branchTracking = branchTrackingOrFailure.success
+            ? branchTrackingOrFailure.data
+            : undefined;
+
+          return {
+            ...connectedRepo,
+            branchTracking,
+          };
+        });
+
+    const validateProductionBranch = ({
+      installationId,
+      fullRepoName,
+      oldProductionBranch,
+    }: {
+      installationId: number;
+      fullRepoName: string;
+      oldProductionBranch?: string;
+    }) => {
+      if (productionBranch && oldProductionBranch !== productionBranch) {
+        return checkGitHubBranchExists(installationId, fullRepoName, productionBranch).andThen(
+          (exists) => {
+            if (!exists) {
+              return errAsync({ type: "production_tracking_branch_not_found" as const });
+            }
+            return okAsync(productionBranch);
+          }
+        );
+      }
+
+      return okAsync(productionBranch);
+    };
+
+    const validateStagingBranch = ({
+      installationId,
+      fullRepoName,
+      oldStagingBranch,
+    }: {
+      installationId: number;
+      fullRepoName: string;
+      oldStagingBranch?: string;
+    }) => {
+      if (stagingBranch && oldStagingBranch !== stagingBranch) {
+        return checkGitHubBranchExists(installationId, fullRepoName, stagingBranch).andThen(
+          (exists) => {
+            if (!exists) {
+              return errAsync({ type: "staging_tracking_branch_not_found" as const });
+            }
+            return okAsync(stagingBranch);
+          }
+        );
+      }
+
+      return okAsync(stagingBranch);
+    };
+
+    const updateConnectedRepo = () =>
+      fromPromise(
+        this.#prismaClient.connectedGithubRepository.update({
+          where: {
+            projectId: projectId,
+          },
+          data: {
+            branchTracking: {
+              prod: productionBranch ? { branch: productionBranch } : {},
+              staging: stagingBranch ? { branch: stagingBranch } : {},
+            } satisfies BranchTrackingConfig,
+            previewDeploymentsEnabled: previewDeploymentsEnabled,
+          },
+        }),
+        (error) => ({ type: "other" as const, cause: error })
+      );
+
+    return getExistingConnectedRepo()
+      .andThen((connectedRepo) => {
+        const installationId = Number(connectedRepo.repository.installation.appInstallationId);
+
+        return ResultAsync.combine([
+          validateProductionBranch({
+            installationId,
+            fullRepoName: connectedRepo.repository.fullName,
+            oldProductionBranch: connectedRepo.branchTracking?.prod?.branch,
+          }),
+          validateStagingBranch({
+            installationId,
+            fullRepoName: connectedRepo.repository.fullName,
+            oldStagingBranch: connectedRepo.branchTracking?.staging?.branch,
+          }),
+        ]);
+      })
+      .andThen(updateConnectedRepo);
+  }
+
+  updateBuildSettings(projectId: string, buildSettings: BuildSettings) {
+    return fromPromise(
+      this.#prismaClient.project.update({
+        where: {
+          id: projectId,
+        },
+        data: {
+          buildSettings: buildSettings,
+        },
+      }),
+      (error) => ({
+        type: "other" as const,
+        cause: error,
+      })
+    );
+  }
+
+  verifyProjectMembership(organizationSlug: string, projectSlug: string, userId: string) {
+    const findProject = () =>
+      fromPromise(
+        this.#prismaClient.project.findFirst({
+          where: {
+            slug: projectSlug,
+            organization: {
+              slug: organizationSlug,
+              members: {
+                some: {
+                  userId,
+                },
+              },
+            },
+          },
+          select: {
+            id: true,
+            organizationId: true,
+          },
+        }),
+        (error) => ({ type: "other" as const, cause: error })
+      );
+
+    return findProject().andThen((project) => {
+      if (!project) {
+        return errAsync({ type: "user_not_in_project" as const });
+      }
+
+      return okAsync({
+        projectId: project.id,
+        organizationId: project.organizationId,
+      });
+    });
+  }
+}

apps/webapp/app/services/projectSettingsPresenter.server.ts

@@ -0,0 +1,172 @@
+import { type PrismaClient } from "@trigger.dev/database";
+import { prisma } from "~/db.server";
+import { BranchTrackingConfigSchema } from "~/v3/github";
+import { env } from "~/env.server";
+import { findProjectBySlug } from "~/models/project.server";
+import { err, fromPromise, ok, okAsync } from "neverthrow";
+import { BuildSettingsSchema } from "~/v3/buildSettings";
+
+export class ProjectSettingsPresenter {
+  #prismaClient: PrismaClient;
+
+  constructor(prismaClient: PrismaClient = prisma) {
+    this.#prismaClient = prismaClient;
+  }
+
+  getProjectSettings(organizationSlug: string, projectSlug: string, userId: string) {
+    const githubAppEnabled = env.GITHUB_APP_ENABLED === "1";
+
+    const getProject = () =>
+      fromPromise(findProjectBySlug(organizationSlug, projectSlug, userId), (error) => ({
+        type: "other" as const,
+        cause: error,
+      })).andThen((project) => {
+        if (!project) {
+          return err({ type: "project_not_found" as const });
+        }
+        return ok(project);
+      });
+
+    if (!githubAppEnabled) {
+      return getProject().andThen((project) => {
+        if (!project) {
+          return err({ type: "project_not_found" as const });
+        }
+
+        const buildSettingsOrFailure = BuildSettingsSchema.safeParse(project.buildSettings);
+        const buildSettings = buildSettingsOrFailure.success
+          ? buildSettingsOrFailure.data
+          : undefined;
+
+        return ok({
+          gitHubApp: {
+            enabled: false,
+            connectedRepository: undefined,
+            installations: undefined,
+          },
+          buildSettings,
+        });
+      });
+    }
+
+    const findConnectedGithubRepository = (projectId: string) =>
+      fromPromise(
+        this.#prismaClient.connectedGithubRepository.findFirst({
+          where: {
+            projectId,
+            repository: {
+              installation: {
+                deletedAt: null,
+                suspendedAt: null,
+              },
+            },
+          },
+          select: {
+            branchTracking: true,
+            previewDeploymentsEnabled: true,
+            createdAt: true,
+            repository: {
+              select: {
+                id: true,
+                name: true,
+                fullName: true,
+                htmlUrl: true,
+                private: true,
+              },
+            },
+          },
+        }),
+        (error) => ({
+          type: "other" as const,
+          cause: error,
+        })
+      ).map((connectedGithubRepository) => {
+        if (!connectedGithubRepository) {
+          return undefined;
+        }
+
+        const branchTrackingOrFailure = BranchTrackingConfigSchema.safeParse(
+          connectedGithubRepository.branchTracking
+        );
+        const branchTracking = branchTrackingOrFailure.success
+          ? branchTrackingOrFailure.data
+          : undefined;
+
+        return {
+          ...connectedGithubRepository,
+          branchTracking,
+        };
+      });
+
+    const listGithubAppInstallations = (organizationId: string) =>
+      fromPromise(
+        this.#prismaClient.githubAppInstallation.findMany({
+          where: {
+            organizationId,
+            deletedAt: null,
+            suspendedAt: null,
+          },
+          select: {
+            id: true,
+            accountHandle: true,
+            targetType: true,
+            appInstallationId: true,
+            repositories: {
+              select: {
+                id: true,
+                name: true,
+                fullName: true,
+                htmlUrl: true,
+                private: true,
+              },
+              // Most installations will only have a couple of repos so loading them here should be fine.
+              // However, there might be outlier organizations so it's best to expose the installation repos
+              // via a resource endpoint and filter on user input.
+              take: 200,
+            },
+          },
+          take: 20,
+          orderBy: {
+            createdAt: "desc",
+          },
+        }),
+        (error) => ({
+          type: "other" as const,
+          cause: error,
+        })
+      );
+
+    return getProject().andThen((project) =>
+      findConnectedGithubRepository(project.id).andThen((connectedGithubRepository) => {
+        const buildSettingsOrFailure = BuildSettingsSchema.safeParse(project.buildSettings);
+        const buildSettings = buildSettingsOrFailure.success
+          ? buildSettingsOrFailure.data
+          : undefined;
+
+        if (connectedGithubRepository) {
+          return okAsync({
+            gitHubApp: {
+              enabled: true,
+              connectedRepository: connectedGithubRepository,
+              // skip loading installations if there is a connected repository
+              // a project can have only a single connected repository
+              installations: undefined,
+            },
+            buildSettings,
+          });
+        }
+
+        return listGithubAppInstallations(project.organizationId).map((githubAppInstallations) => {
+          return {
+            gitHubApp: {
+              enabled: true,
+              connectedRepository: undefined,
+              installations: githubAppInstallations,
+            },
+            buildSettings,
+          };
+        });
+      })
+    );
+  }
+}

apps/webapp/app/services/routeBuilders/apiBuilder.server.ts

@@ -23,6 +23,8 @@ import {
 } from "~/v3/services/worker/workerGroupTokenService.server";
 import { API_VERSIONS, getApiVersion } from "~/api/versions";
 import { WORKER_HEADERS } from "@trigger.dev/core/v3/runEngineWorker";
+import { ServiceValidationError } from "~/v3/services/common.server";
+import { EngineServiceValidationError } from "@internal/run-engine";
 
 type AnyZodSchema = z.ZodFirstPartySchemaTypes | z.ZodDiscriminatedUnion<any, any>;
 
@@ -1040,11 +1042,18 @@ export function createActionWorkerApiRoute<
       });
       return result;
     } catch (error) {
-      console.error("Error in API route:", error);
       if (error instanceof Response) {
         return error;
       }
 
+      if (error instanceof EngineServiceValidationError) {
+        return json({ error: error.message }, { status: error.status ?? 422 });
+      }
+
+      if (error instanceof ServiceValidationError) {
+        return json({ error: error.message }, { status: error.status ?? 422 });
+      }
+
       logger.error("Error in action", {
         error:
           error instanceof Error

apps/webapp/app/services/runsReplicationService.server.ts

@@ -148,6 +148,11 @@ export class RunsReplicationService {
         }
 
         for (const item of newBatch) {
+          if (!item?.run?.id) {
+            this.logger.warn("Skipping replication event with null run", { event: item });
+            continue;
+          }
+
           const key = `${item.event}_${item.run.id}`;
           const existingItem = merged.get(key);
 

apps/webapp/app/services/upsertBranch.server.ts

@@ -14,7 +14,16 @@ export class UpsertBranchService {
     this.#prismaClient = prismaClient;
   }
 
-  public async call(userId: string, { parentEnvironmentId, branchName, git }: CreateBranchOptions) {
+  public async call(
+    // The orgFilter approach is not ideal but we need to keep it this way for now because of how the service is used in routes and api endpoints.
+    // Currently authorization checks are spread across the controller/route layer and the service layer. Often we check in multiple places for org/project membership.
+    // Ideally we would take care of both the authentication and authorization checks in the controllers and routes.
+    // That would unify how we handle authorization and org/project membership checks. Also it would make the service layer queries simpler.
+    orgFilter:
+      | { type: "userMembership"; userId: string }
+      | { type: "orgId"; organizationId: string },
+    { parentEnvironmentId, branchName, git }: CreateBranchOptions
+  ) {
     const sanitizedBranchName = sanitizeBranchName(branchName);
     if (!sanitizedBranchName) {
       return {
@@ -34,13 +43,16 @@ export class UpsertBranchService {
       const parentEnvironment = await this.#prismaClient.runtimeEnvironment.findFirst({
         where: {
           id: parentEnvironmentId,
-          organization: {
-            members: {
-              some: {
-                userId: userId,
-              },
-            },
-          },
+          organization:
+            orgFilter.type === "userMembership"
+              ? {
+                  members: {
+                    some: {
+                      userId: orgFilter.userId,
+                    },
+                  },
+                }
+              : { id: orgFilter.organizationId },
         },
         include: {
           organization: {

apps/webapp/app/utils/pathBuilder.ts

@@ -141,6 +141,12 @@ export function v3ProjectPath(organization: OrgForPath, project: ProjectForPath)
   return `/orgs/${organizationParam(organization)}/projects/${projectParam(project)}`;
 }
 
+export function githubAppInstallPath(organizationSlug: string, redirectTo: string) {
+  return `/github/install?org_slug=${organizationSlug}&redirect_to=${encodeURIComponent(
+    redirectTo
+  )}`;
+}
+
 export function v3EnvironmentPath(
   organization: OrgForPath,
   project: ProjectForPath,

apps/webapp/app/v3/buildSettings.ts

@@ -0,0 +1,9 @@
+import { z } from "zod";
+
+export const BuildSettingsSchema = z.object({
+  triggerConfigFilePath: z.string().optional(),
+  installDirectory: z.string().optional(),
+  installCommand: z.string().optional(),
+});
+
+export type BuildSettings = z.infer<typeof BuildSettingsSchema>;

apps/webapp/app/v3/github.ts

@@ -0,0 +1,35 @@
+import { z } from "zod";
+
+export const BranchTrackingConfigSchema = z.object({
+  prod: z.object({
+    branch: z.string().optional(),
+  }),
+  staging: z.object({
+    branch: z.string().optional(),
+  }),
+});
+
+export type BranchTrackingConfig = z.infer<typeof BranchTrackingConfigSchema>;
+
+export function getTrackedBranchForEnvironment(
+  branchTracking: BranchTrackingConfig | undefined,
+  previewDeploymentsEnabled: boolean,
+  environment: {
+    type: "PRODUCTION" | "STAGING" | "DEVELOPMENT" | "PREVIEW";
+    branchName?: string;
+  }
+): string | undefined {
+  switch (environment.type) {
+    case "PRODUCTION":
+      return branchTracking?.prod?.branch;
+    case "STAGING":
+      return branchTracking?.staging?.branch;
+    case "PREVIEW":
+      return previewDeploymentsEnabled ? environment.branchName : undefined;
+    case "DEVELOPMENT":
+      return undefined;
+    default:
+      environment.type satisfies never;
+      return undefined;
+  }
+}

apps/webapp/app/v3/remoteImageBuilder.server.ts

@@ -1,9 +1,12 @@
 import { depot } from "@depot/sdk-node";
-import { Project } from "@trigger.dev/database";
+import { type ExternalBuildData } from "@trigger.dev/core/v3";
+import { type Project } from "@trigger.dev/database";
 import { prisma } from "~/db.server";
 import { env } from "~/env.server";
 
-export async function createRemoteImageBuild(project: Project) {
+export async function createRemoteImageBuild(
+  project: Project
+): Promise<ExternalBuildData | undefined> {
   if (!remoteBuildsEnabled()) {
     return;
   }

apps/webapp/app/v3/runEngine.server.ts

@@ -15,6 +15,8 @@ function createRunEngine() {
     prisma,
     readOnlyPrisma: $replica,
     logLevel: env.RUN_ENGINE_WORKER_LOG_LEVEL,
+    treatProductionExecutionStallsAsOOM:
+      env.RUN_ENGINE_TREAT_PRODUCTION_EXECUTION_STALLS_AS_OOM === "1",
     worker: {
       disabled: env.RUN_ENGINE_WORKER_ENABLED === "0",
       workers: env.RUN_ENGINE_WORKER_COUNT,

apps/webapp/app/v3/runEngineHandlers.server.ts

@@ -1,59 +1,66 @@
-import { $replica, prisma } from "~/db.server";
+import { tryCatch } from "@trigger.dev/core/utils";
+import { createJsonErrorObject, sanitizeError } from "@trigger.dev/core/v3";
+import { RunId } from "@trigger.dev/core/v3/isomorphic";
+import { $replica } from "~/db.server";
+import { env } from "~/env.server";
+import { findEnvironmentFromRun } from "~/models/runtimeEnvironment.server";
+import { logger } from "~/services/logger.server";
+import { updateMetadataService } from "~/services/metadata/updateMetadataInstance.server";
+import { reportInvocationUsage } from "~/services/platform.v3.server";
+import { MetadataTooLargeError } from "~/utils/packets";
 import {
   createExceptionPropertiesFromError,
   eventRepository,
   recordRunDebugLog,
 } from "./eventRepository.server";
-import { createJsonErrorObject, sanitizeError } from "@trigger.dev/core/v3";
-import { logger } from "~/services/logger.server";
-import { safeJsonParse } from "~/utils/json";
-import type { Attributes } from "@opentelemetry/api";
-import { reportInvocationUsage } from "~/services/platform.v3.server";
 import { roomFromFriendlyRunId, socketIo } from "./handleSocketIo.server";
 import { engine } from "./runEngine.server";
 import { PerformTaskRunAlertsService } from "./services/alerts/performTaskRunAlerts.server";
-import { RunId } from "@trigger.dev/core/v3/isomorphic";
-import { updateMetadataService } from "~/services/metadata/updateMetadataInstance.server";
-import { findEnvironmentFromRun } from "~/models/runtimeEnvironment.server";
-import { env } from "~/env.server";
-import { getTaskEventStoreTableForRun } from "./taskEventStore.server";
-import { MetadataTooLargeError } from "~/utils/packets";
 
 export function registerRunEngineEventBusHandlers() {
   engine.eventBus.on("runSucceeded", async ({ time, run }) => {
-    try {
-      const completedEvent = await eventRepository.completeEvent(
-        getTaskEventStoreTableForRun(run),
-        run.spanId,
-        run.createdAt,
-        run.completedAt ?? undefined,
-        {
-          endTime: time,
-          attributes: {
-            isError: false,
-            output:
-              run.outputType === "application/store" || run.outputType === "text/plain"
-                ? run.output
-                : run.output
-                ? (safeJsonParse(run.output) as Attributes)
-                : undefined,
-            outputType: run.outputType,
-          },
-        }
-      );
+    const [taskRunError, taskRun] = await tryCatch(
+      $replica.taskRun.findFirstOrThrow({
+        where: {
+          id: run.id,
+        },
+        select: {
+          id: true,
+          friendlyId: true,
+          traceId: true,
+          spanId: true,
+          parentSpanId: true,
+          createdAt: true,
+          completedAt: true,
+          taskIdentifier: true,
+          projectId: true,
+          runtimeEnvironmentId: true,
+          environmentType: true,
+          isTest: true,
+          organizationId: true,
+        },
+      })
+    );
 
-      if (!completedEvent) {
-        logger.error("[runSucceeded] Failed to complete event for unknown reason", {
-          runId: run.id,
-          spanId: run.spanId,
-        });
-        return;
-      }
-    } catch (error) {
-      logger.error("[runSucceeded] Failed to complete event", {
-        error: error instanceof Error ? error.message : error,
+    if (taskRunError) {
+      logger.error("[runSucceeded] Failed to find task run", {
+        error: taskRunError,
+        runId: run.id,
+      });
+      return;
+    }
+
+    const [completeSuccessfulRunEventError] = await tryCatch(
+      eventRepository.completeSuccessfulRunEvent({
+        run: taskRun,
+        endTime: time,
+      })
+    );
+
+    if (completeSuccessfulRunEventError) {
+      logger.error("[runSucceeded] Failed to complete successful run event", {
+        error: completeSuccessfulRunEventError,
         runId: run.id,
-        spanId: run.spanId,
       });
     }
   });
@@ -73,149 +80,181 @@ export function registerRunEngineEventBusHandlers() {
 
   // Handle events
   engine.eventBus.on("runFailed", async ({ time, run }) => {
-    try {
-      const sanitizedError = sanitizeError(run.error);
-      const exception = createExceptionPropertiesFromError(sanitizedError);
+    const sanitizedError = sanitizeError(run.error);
+    const exception = createExceptionPropertiesFromError(sanitizedError);
 
-      const eventStore = getTaskEventStoreTableForRun(run);
-
-      const completedEvent = await eventRepository.completeEvent(
-        eventStore,
-        run.spanId,
-        run.createdAt,
-        run.completedAt ?? undefined,
-        {
-          endTime: time,
-          attributes: {
-            isError: true,
-          },
-          events: [
-            {
-              name: "exception",
-              time,
-              properties: {
-                exception,
-              },
-            },
-          ],
-        }
-      );
-
-      if (!completedEvent) {
-        logger.error("[runFailed] Failed to complete event for unknown reason", {
-          runId: run.id,
-          spanId: run.spanId,
-        });
-        return;
-      }
-
-      const inProgressEvents = await eventRepository.queryIncompleteEvents(
-        eventStore,
-        {
-          runId: completedEvent?.runId,
+    const [taskRunError, taskRun] = await tryCatch(
+      $replica.taskRun.findFirstOrThrow({
+        where: {
+          id: run.id,
         },
-        run.createdAt,
-        run.completedAt ?? undefined
-      );
+        select: {
+          id: true,
+          friendlyId: true,
+          traceId: true,
+          spanId: true,
+          parentSpanId: true,
+          createdAt: true,
+          completedAt: true,
+          taskIdentifier: true,
+          projectId: true,
+          runtimeEnvironmentId: true,
+          environmentType: true,
+          isTest: true,
+          organizationId: true,
+        },
+      })
+    );
 
-      await Promise.all(
-        inProgressEvents.map((event) => {
-          try {
-            const completedEvent = eventRepository.completeEvent(
-              eventStore,
-              run.spanId,
-              run.createdAt,
-              run.completedAt ?? undefined,
-              {
-                endTime: time,
-                attributes: {
-                  isError: true,
-                },
-                events: [
-                  {
-                    name: "exception",
-                    time,
-                    properties: {
-                      exception,
-                    },
-                  },
-                ],
-              }
-            );
-
-            if (!completedEvent) {
-              logger.error("[runFailed] Failed to complete in-progress event for unknown reason", {
-                runId: run.id,
-                spanId: run.spanId,
-                eventId: event.id,
-              });
-              return;
-            }
-          } catch (error) {
-            logger.error("[runFailed] Failed to complete in-progress event", {
-              error: error instanceof Error ? error.message : error,
-              runId: run.id,
-              spanId: run.spanId,
-              eventId: event.id,
-            });
-          }
-        })
-      );
-    } catch (error) {
-      logger.error("[runFailed] Failed to complete event", {
-        error: error instanceof Error ? error.message : error,
+    if (taskRunError) {
+      logger.error("[runFailed] Failed to find task run", {
+        error: taskRunError,
+        runId: run.id,
+      });
+      return;
+    }
+
+    const [completeFailedRunEventError] = await tryCatch(
+      eventRepository.completeFailedRunEvent({
+        run: taskRun,
+        endTime: time,
+        exception,
+      })
+    );
+
+    if (completeFailedRunEventError) {
+      logger.error("[runFailed] Failed to complete failed run event", {
+        error: completeFailedRunEventError,
         runId: run.id,
-        spanId: run.spanId,
       });
     }
   });
 
   engine.eventBus.on("runAttemptFailed", async ({ time, run }) => {
-    try {
-      const sanitizedError = sanitizeError(run.error);
-      const exception = createExceptionPropertiesFromError(sanitizedError);
-      const eventStore = getTaskEventStoreTableForRun(run);
+    const sanitizedError = sanitizeError(run.error);
+    const exception = createExceptionPropertiesFromError(sanitizedError);
 
-      const inProgressEvents = await eventRepository.queryIncompleteEvents(
-        eventStore,
-        {
-          runId: RunId.toFriendlyId(run.id),
-          spanId: {
-            not: run.spanId,
-          },
+    const [taskRunError, taskRun] = await tryCatch(
+      $replica.taskRun.findFirstOrThrow({
+        where: {
+          id: run.id,
         },
-        run.createdAt,
-        run.completedAt ?? undefined
-      );
+        select: {
+          id: true,
+          friendlyId: true,
+          traceId: true,
+          spanId: true,
+          parentSpanId: true,
+          createdAt: true,
+          completedAt: true,
+          taskIdentifier: true,
+          projectId: true,
+          runtimeEnvironmentId: true,
+          environmentType: true,
+          isTest: true,
+          organizationId: true,
+        },
+      })
+    );
 
-      await Promise.all(
-        inProgressEvents.map((event) => {
-          return eventRepository.crashEvent({
-            event: event,
-            crashedAt: time,
-            exception,
-          });
-        })
-      );
-    } catch (error) {
-      logger.error("[runAttemptFailed] Failed to complete event", {
-        error: error instanceof Error ? error.message : error,
+    if (taskRunError) {
+      logger.error("[runAttemptFailed] Failed to find task run", {
+        error: taskRunError,
+        runId: run.id,
+      });
+      return;
+    }
+
+    const [createAttemptFailedRunEventError] = await tryCatch(
+      eventRepository.createAttemptFailedRunEvent({
+        run: taskRun,
+        endTime: time,
+        attemptNumber: run.attemptNumber,
+        exception,
+      })
+    );
+
+    if (createAttemptFailedRunEventError) {
+      logger.error("[runAttemptFailed] Failed to create attempt failed run event", {
+        error: createAttemptFailedRunEventError,
         runId: run.id,
-        spanId: run.spanId,
       });
     }
   });
 
-  engine.eventBus.on("cachedRunCompleted", async ({ time, span, blockedRunId, hasError }) => {
-    try {
-      const blockedRun = await $replica.taskRun.findFirst({
-        select: {
-          taskEventStore: true,
-        },
-        where: {
-          id: blockedRunId,
-        },
-      });
+  engine.eventBus.on(
+    "cachedRunCompleted",
+    async ({ time, span, blockedRunId, hasError, cachedRunId }) => {
+      const [parentSpanId, spanId] = span.id.split(":");
+
+      if (!spanId || !parentSpanId) {
+        logger.debug("[cachedRunCompleted] Invalid span id", {
+          spanId,
+          parentSpanId,
+        });
+        return;
+      }
+
+      const [cachedRunError, cachedRun] = await tryCatch(
+        $replica.taskRun.findFirstOrThrow({
+          where: {
+            id: cachedRunId,
+          },
+          select: {
+            id: true,
+            friendlyId: true,
+            traceId: true,
+            spanId: true,
+            parentSpanId: true,
+            createdAt: true,
+            completedAt: true,
+            taskIdentifier: true,
+            projectId: true,
+            runtimeEnvironmentId: true,
+            environmentType: true,
+            isTest: true,
+            organizationId: true,
+          },
+        })
+      );
+
+      if (cachedRunError) {
+        logger.error("[cachedRunCompleted] Failed to find cached run", {
+          error: cachedRunError,
+          cachedRunId,
+        });
+        return;
+      }
+
+      const [blockedRunError, blockedRun] = await tryCatch(
+        $replica.taskRun.findFirst({
+          where: {
+            id: blockedRunId,
+          },
+          select: {
+            id: true,
+            friendlyId: true,
+            traceId: true,
+            spanId: true,
+            parentSpanId: true,
+            createdAt: true,
+            completedAt: true,
+            taskIdentifier: true,
+            projectId: true,
+            runtimeEnvironmentId: true,
+            environmentType: true,
+            isTest: true,
+            organizationId: true,
+          },
+        })
+      );
+
+      if (blockedRunError) {
+        logger.error("[cachedRunCompleted] Failed to find blocked run", {
+          error: blockedRunError,
+          blockedRunId,
+        });
+      }
 
       if (!blockedRun) {
         logger.error("[cachedRunCompleted] Blocked run not found", {
@@ -224,100 +263,125 @@ export function registerRunEngineEventBusHandlers() {
         return;
       }
 
-      const eventStore = getTaskEventStoreTableForRun(blockedRun);
-
-      const completedEvent = await eventRepository.completeEvent(
-        eventStore,
-        span.id,
-        span.createdAt,
-        time,
-        {
+      const [completeCachedRunEventError] = await tryCatch(
+        eventRepository.completeCachedRunEvent({
+          run: cachedRun,
+          blockedRun,
+          spanId,
+          parentSpanId,
+          spanCreatedAt: span.createdAt,
+          isError: hasError,
           endTime: time,
-          attributes: {
-            isError: hasError,
-          },
-        }
+        })
       );
 
-      if (!completedEvent) {
-        logger.error("[cachedRunCompleted] Failed to complete event for unknown reason", {
-          span,
+      if (completeCachedRunEventError) {
+        logger.error("[cachedRunCompleted] Failed to complete cached run event", {
+          error: completeCachedRunEventError,
+          cachedRunId,
         });
-        return;
       }
-    } catch (error) {
-      logger.error("[cachedRunCompleted] Failed to complete event for unknown reason", {
-        error: error instanceof Error ? error.message : error,
-        span,
-      });
     }
-  });
+  );
 
   engine.eventBus.on("runExpired", async ({ time, run }) => {
-    try {
-      const eventStore = getTaskEventStoreTableForRun(run);
+    if (!run.ttl) {
+      return;
+    }
 
-      const completedEvent = await eventRepository.completeEvent(
-        eventStore,
-        run.spanId,
-        run.createdAt,
-        run.completedAt ?? undefined,
-        {
-          endTime: time,
-          attributes: {
-            isError: true,
-          },
-          events: [
-            {
-              name: "exception",
-              time,
-              properties: {
-                exception: {
-                  message: `Run expired because the TTL (${run.ttl}) was reached`,
-                },
-              },
-            },
-          ],
-        }
-      );
+    const [taskRunError, taskRun] = await tryCatch(
+      $replica.taskRun.findFirstOrThrow({
+        where: {
+          id: run.id,
+        },
+        select: {
+          id: true,
+          friendlyId: true,
+          traceId: true,
+          spanId: true,
+          parentSpanId: true,
+          createdAt: true,
+          completedAt: true,
+          taskIdentifier: true,
+          projectId: true,
+          runtimeEnvironmentId: true,
+          environmentType: true,
+          isTest: true,
+          organizationId: true,
+        },
+      })
+    );
 
-      if (!completedEvent) {
-        logger.error("[runFailed] Failed to complete event for unknown reason", {
-          runId: run.id,
-          spanId: run.spanId,
-        });
-        return;
-      }
-    } catch (error) {
-      logger.error("[runExpired] Failed to complete event", {
-        error: error instanceof Error ? error.message : error,
+    if (taskRunError) {
+      logger.error("[runExpired] Failed to find task run", {
+        error: taskRunError,
+        runId: run.id,
+      });
+      return;
+    }
+
+    const [completeExpiredRunEventError] = await tryCatch(
+      eventRepository.completeExpiredRunEvent({
+        run: taskRun,
+        endTime: time,
+        ttl: run.ttl,
+      })
+    );
+
+    if (completeExpiredRunEventError) {
+      logger.error("[runExpired] Failed to complete expired run event", {
+        error: completeExpiredRunEventError,
         runId: run.id,
-        spanId: run.spanId,
       });
     }
   });
 
   engine.eventBus.on("runCancelled", async ({ time, run }) => {
-    try {
-      const eventStore = getTaskEventStoreTableForRun(run);
-
-      const inProgressEvents = await eventRepository.queryIncompleteEvents(
-        eventStore,
-        {
-          runId: run.friendlyId,
+    const [taskRunError, taskRun] = await tryCatch(
+      $replica.taskRun.findFirstOrThrow({
+        where: {
+          id: run.id,
         },
-        run.createdAt,
-        run.completedAt ?? undefined
-      );
+        select: {
+          id: true,
+          friendlyId: true,
+          traceId: true,
+          spanId: true,
+          parentSpanId: true,
+          createdAt: true,
+          completedAt: true,
+          taskIdentifier: true,
+          projectId: true,
+          runtimeEnvironmentId: true,
+          environmentType: true,
+          isTest: true,
+          organizationId: true,
+        },
+      })
+    );
 
-      const error = createJsonErrorObject(run.error);
-
-      await eventRepository.cancelEvents(inProgressEvents, time, error.message);
-    } catch (error) {
-      logger.error("[runCancelled] Failed to cancel event", {
-        error: error instanceof Error ? error.message : error,
+    if (taskRunError) {
+      logger.error("[runCancelled] Task run not found", {
+        error: taskRunError,
+        runId: run.id,
+      });
+      return;
+    }
+
+    const error = createJsonErrorObject(run.error);
+
+    const [cancelRunEventError] = await tryCatch(
+      eventRepository.cancelRunEvent({
+        reason: error.message,
+        run: taskRun,
+        cancelledAt: time,
+      })
+    );
+
+    if (cancelRunEventError) {
+      logger.error("[runCancelled] Failed to cancel run event", {
+        error: cancelRunEventError,
         runId: run.id,
-        spanId: run.spanId,
       });
     }
   });

apps/webapp/app/v3/services/aiRunFilterService.server.ts

@@ -175,6 +175,21 @@ export class AIRunFilterService {
   - "using large machine" → machines: ["large-1x", "large-2x"]
   - "root only" → rootOnly: true
   
+  Time-specific patterns:
+  - "around 8am today" → from/to: 7am-9am today (1-2 hour window around the time)
+  - "at 2pm yesterday" → from/to: 2pm-2:59pm yesterday (exact hour)
+  - "this morning" → from/to: 6am-12pm today
+  - "this afternoon" → from/to: 12pm-6pm today
+  - "this evening" → from/to: 6pm-10pm today
+  - "started around X" / "that started at X" → same as time filtering (treat "started" as temporal filter)
+  - "began around X" / "ran at X" → same as time filtering
+  
+  When handling specific times:
+  - "around" adds ±1 hour buffer (e.g., "around 8am" = 7am-9am)
+  - "at" means exact hour (e.g., "at 2pm" = 2pm-2:59pm)
+  - For relative days: "today" = current date, "yesterday" = current date - 1 day
+  - Convert times to ISO format for from/to filters
+  
   Use the available tools to look up actual tags, versions, queues, and tasks in the environment when the user mentions them. This will help you provide accurate filter values.
   
   Unless they specify they only want root runs, set rootOnly to false.

apps/webapp/app/v3/services/cancelAttempt.server.ts

@@ -72,25 +72,6 @@ export class CancelAttemptService extends BaseService {
           error: isCancellable ? { type: "STRING_ERROR", raw: reason } : undefined,
         });
       });
-
-      const inProgressEvents = await eventRepository.queryIncompleteEvents(
-        getTaskEventStoreTableForRun(taskRunAttempt.taskRun),
-        {
-          runId: taskRunAttempt.taskRun.friendlyId,
-        },
-        taskRunAttempt.taskRun.createdAt,
-        taskRunAttempt.taskRun.completedAt ?? undefined
-      );
-
-      logger.debug("Cancelling in-progress events", {
-        inProgressEvents: inProgressEvents.map((event) => event.id),
-      });
-
-      await Promise.all(
-        inProgressEvents.map((event) => {
-          return eventRepository.cancelEvent(event, cancelledAt, reason);
-        })
-      );
     });
   }
 }

apps/webapp/app/v3/services/cancelTaskRunV1.server.ts

@@ -1,4 +1,4 @@
-import { type Prisma, type TaskRun } from "@trigger.dev/database";
+import { type Prisma } from "@trigger.dev/database";
 import assertNever from "assert-never";
 import { logger } from "~/services/logger.server";
 import { eventRepository } from "../eventRepository.server";
@@ -8,9 +8,9 @@ import { CANCELLABLE_ATTEMPT_STATUSES, isCancellableRunStatus } from "../taskSta
 import { BaseService } from "./baseService.server";
 import { CancelAttemptService } from "./cancelAttempt.server";
 import { CancelTaskAttemptDependenciesService } from "./cancelTaskAttemptDependencies.server";
-import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
-import { getTaskEventStoreTableForRun } from "../taskEventStore.server";
 import { CancelableTaskRun } from "./cancelTaskRun.server";
+import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
+import { tryCatch } from "@trigger.dev/core/utils";
 
 type ExtendedTaskRun = Prisma.TaskRunGetPayload<{
   include: {
@@ -92,6 +92,7 @@ export class CancelTaskRunServiceV1 extends BaseService {
         },
         runtimeEnvironment: true,
         lockedToVersion: true,
+        project: true,
       },
       attemptStatus: "CANCELED",
       error: {
@@ -100,21 +101,20 @@ export class CancelTaskRunServiceV1 extends BaseService {
       },
     });
 
-    const inProgressEvents = await eventRepository.queryIncompleteEvents(
-      getTaskEventStoreTableForRun(taskRun),
-      {
-        runId: taskRun.friendlyId,
-      },
-      taskRun.createdAt,
-      taskRun.completedAt ?? undefined
+    const [cancelRunEventError] = await tryCatch(
+      eventRepository.cancelRunEvent({
+        reason: opts.reason,
+        run: cancelledTaskRun,
+        cancelledAt: opts.cancelledAt,
+      })
     );
 
-    logger.debug("Cancelling in-progress events", {
-      inProgressEvents: inProgressEvents.map((event) => event.id),
-      eventCount: inProgressEvents.length,
-    });
-
-    await eventRepository.cancelEvents(inProgressEvents, opts.cancelledAt, opts.reason);
+    if (cancelRunEventError) {
+      logger.error("[CancelTaskRunServiceV1] Failed to cancel run event", {
+        error: cancelRunEventError,
+        runId: cancelledTaskRun.id,
+      });
+    }
 
     // Cancel any in progress attempts
     if (opts.cancelAttempts) {

apps/webapp/app/v3/services/completeAttempt.server.ts

@@ -33,6 +33,7 @@ import { CancelAttemptService } from "./cancelAttempt.server";
 import { CreateCheckpointService } from "./createCheckpoint.server";
 import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
 import { RetryAttemptService } from "./retryAttempt.server";
+import { tryCatch } from "@trigger.dev/core/utils";
 
 type FoundAttempt = Awaited<ReturnType<typeof findAttempt>>;
 
@@ -164,27 +165,20 @@ export class CompleteAttemptService extends BaseService {
       env,
     });
 
-    // Now we need to "complete" the task run event/span
-    await eventRepository.completeEvent(
-      getTaskEventStoreTableForRun(taskRunAttempt.taskRun),
-      taskRunAttempt.taskRun.spanId,
-      taskRunAttempt.taskRun.createdAt,
-      taskRunAttempt.taskRun.completedAt ?? undefined,
-      {
+    const [completeSuccessfulRunEventError] = await tryCatch(
+      eventRepository.completeSuccessfulRunEvent({
+        run: taskRunAttempt.taskRun,
         endTime: new Date(),
-        attributes: {
-          isError: false,
-          output:
-            completion.outputType === "application/store" || completion.outputType === "text/plain"
-              ? completion.output
-              : completion.output
-              ? (safeJsonParse(completion.output) as Attributes)
-              : undefined,
-          outputType: completion.outputType,
-        },
-      }
+      })
     );
 
+    if (completeSuccessfulRunEventError) {
+      logger.error("[CompleteAttemptService] Failed to complete successful run event", {
+        error: completeSuccessfulRunEventError,
+        runId: taskRunAttempt.taskRunId,
+      });
+    }
+
     return "COMPLETED";
   }
 
@@ -213,7 +207,7 @@ export class CompleteAttemptService extends BaseService {
         taskRunAttempt.friendlyId,
         taskRunAttempt.taskRunId,
         new Date(),
-        "Cancelled by user",
+        "Canceled by user",
         env
       );
 
@@ -322,29 +316,21 @@ export class CompleteAttemptService extends BaseService {
       exitRun(taskRunAttempt.taskRunId);
     }
 
-    // Now we need to "complete" the task run event/span
-    await eventRepository.completeEvent(
-      getTaskEventStoreTableForRun(taskRunAttempt.taskRun),
-      taskRunAttempt.taskRun.spanId,
-      taskRunAttempt.taskRun.createdAt,
-      taskRunAttempt.taskRun.completedAt ?? undefined,
-      {
+    const [completeFailedRunEventError] = await tryCatch(
+      eventRepository.completeFailedRunEvent({
+        run: taskRunAttempt.taskRun,
         endTime: failedAt,
-        attributes: {
-          isError: true,
-        },
-        events: [
-          {
-            name: "exception",
-            time: failedAt,
-            properties: {
-              exception: createExceptionPropertiesFromError(sanitizedError),
-            },
-          },
-        ],
-      }
+        exception: createExceptionPropertiesFromError(sanitizedError),
+      })
     );
 
+    if (completeFailedRunEventError) {
+      logger.error("[CompleteAttemptService] Failed to complete failed run event", {
+        error: completeFailedRunEventError,
+        runId: taskRunAttempt.taskRunId,
+      });
+    }
+
     await this._prisma.taskRun.update({
       where: {
         id: taskRunAttempt.taskRunId,
@@ -385,64 +371,43 @@ export class CompleteAttemptService extends BaseService {
       return "COMPLETED";
     }
 
-    const inProgressEvents = await eventRepository.queryIncompleteEvents(
-      getTaskEventStoreTableForRun(taskRunAttempt.taskRun),
-      {
-        runId: taskRunAttempt.taskRun.friendlyId,
-      },
-      taskRunAttempt.taskRun.createdAt,
-      taskRunAttempt.taskRun.completedAt ?? undefined
-    );
-
     // Handle in-progress events
     switch (status) {
       case "CRASHED": {
-        logger.debug("[CompleteAttemptService] Crashing in-progress events", {
-          inProgressEvents: inProgressEvents.map((event) => event.id),
-        });
-
-        await Promise.all(
-          inProgressEvents.map((event) => {
-            return eventRepository.crashEvent({
-              event,
-              crashedAt: failedAt,
-              exception: createExceptionPropertiesFromError(sanitizedError),
-            });
+        const [createAttemptFailedEventError] = await tryCatch(
+          eventRepository.createAttemptFailedRunEvent({
+            run: taskRunAttempt.taskRun,
+            endTime: failedAt,
+            attemptNumber: taskRunAttempt.number,
+            exception: createExceptionPropertiesFromError(sanitizedError),
           })
         );
 
+        if (createAttemptFailedEventError) {
+          logger.error("[CompleteAttemptService] Failed to create attempt failed run event", {
+            error: createAttemptFailedEventError,
+            runId: taskRunAttempt.taskRunId,
+          });
+        }
+
         break;
       }
       case "SYSTEM_FAILURE": {
-        logger.debug("[CompleteAttemptService] Failing in-progress events", {
-          inProgressEvents: inProgressEvents.map((event) => event.id),
-        });
-
-        await Promise.all(
-          inProgressEvents.map((event) => {
-            return eventRepository.completeEvent(
-              getTaskEventStoreTableForRun(taskRunAttempt.taskRun),
-              event.spanId,
-              taskRunAttempt.taskRun.createdAt,
-              taskRunAttempt.taskRun.completedAt ?? undefined,
-              {
-                endTime: failedAt,
-                attributes: {
-                  isError: true,
-                },
-                events: [
-                  {
-                    name: "exception",
-                    time: failedAt,
-                    properties: {
-                      exception: createExceptionPropertiesFromError(sanitizedError),
-                    },
-                  },
-                ],
-              }
-            );
+        const [createAttemptFailedEventError] = await tryCatch(
+          eventRepository.createAttemptFailedRunEvent({
+            run: taskRunAttempt.taskRun,
+            endTime: failedAt,
+            attemptNumber: taskRunAttempt.number,
+            exception: createExceptionPropertiesFromError(sanitizedError),
           })
         );
+
+        if (createAttemptFailedEventError) {
+          logger.error("[CompleteAttemptService] Failed to create attempt failed run event", {
+            error: createAttemptFailedEventError,
+            runId: taskRunAttempt.taskRunId,
+          });
+        }
       }
     }
 

apps/webapp/app/v3/services/crashTaskRun.server.ts

@@ -8,6 +8,7 @@ import { sanitizeError, TaskRunErrorCodes, TaskRunInternalError } from "@trigger
 import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
 import { FailedTaskRunRetryHelper } from "../failedTaskRun.server";
 import { getTaskEventStoreTableForRun } from "../taskEventStore.server";
+import { tryCatch } from "@trigger.dev/core/utils";
 
 export type CrashTaskRunServiceOptions = {
   reason?: string;
@@ -120,34 +121,25 @@ export class CrashTaskRunService extends BaseService {
       },
     });
 
-    const inProgressEvents = await eventRepository.queryIncompleteEvents(
-      getTaskEventStoreTableForRun(taskRun),
-      {
-        runId: taskRun.friendlyId,
-      },
-      taskRun.createdAt,
-      taskRun.completedAt ?? undefined,
-      options?.overrideCompletion
-    );
-
-    logger.debug("[CrashTaskRunService] Crashing in-progress events", {
-      inProgressEvents: inProgressEvents.map((event) => event.id),
-    });
-
-    await Promise.all(
-      inProgressEvents.map((event) => {
-        return eventRepository.crashEvent({
-          event: event,
-          crashedAt: opts.crashedAt,
-          exception: {
-            type: opts.errorCode ?? TaskRunErrorCodes.TASK_RUN_CRASHED,
-            message: opts.reason,
-            stacktrace: opts.logs,
-          },
-        });
+    const [createAttemptFailedEventError] = await tryCatch(
+      eventRepository.completeFailedRunEvent({
+        run: crashedTaskRun,
+        endTime: opts.crashedAt,
+        exception: {
+          type: opts.errorCode ?? TaskRunErrorCodes.TASK_RUN_CRASHED,
+          message: opts.reason,
+          stacktrace: opts.logs,
+        },
       })
     );
 
+    if (createAttemptFailedEventError) {
+      logger.error("[CrashTaskRunService] Failed to complete failed run event", {
+        error: createAttemptFailedEventError,
+        runId: crashedTaskRun.id,
+      });
+    }
+
     if (!opts.crashAttempts) {
       return;
     }

apps/webapp/app/v3/services/deployment.server.ts

@@ -0,0 +1,234 @@
+import { type AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { BaseService } from "./baseService.server";
+import { errAsync, fromPromise, okAsync } from "neverthrow";
+import { type WorkerDeployment } from "@trigger.dev/database";
+import { logger, type GitMeta } from "@trigger.dev/core/v3";
+import { TimeoutDeploymentService } from "./timeoutDeployment.server";
+import { env } from "~/env.server";
+import { createRemoteImageBuild } from "../remoteImageBuilder.server";
+import { FINAL_DEPLOYMENT_STATUSES } from "./failDeployment.server";
+
+export class DeploymentService extends BaseService {
+  /**
+   * Progresses a deployment from PENDING to INSTALLING and then to BUILDING.
+   * Also extends the deployment timeout.
+   *
+   * When progressing to BUILDING, the remote Depot build is also created.
+   *
+   * Only acts when the current status allows. Not idempotent.
+   *
+   * @param authenticatedEnv The environment which the deployment belongs to.
+   * @param friendlyId The friendly deployment ID.
+   * @param updates Optional deployment details to persist.
+   */
+
+  public progressDeployment(
+    authenticatedEnv: AuthenticatedEnvironment,
+    friendlyId: string,
+    updates: Partial<Pick<WorkerDeployment, "contentHash" | "runtime"> & { git: GitMeta }>
+  ) {
+    const getDeployment = () =>
+      fromPromise(
+        this._prisma.workerDeployment.findFirst({
+          where: {
+            friendlyId,
+            environmentId: authenticatedEnv.id,
+          },
+          select: {
+            status: true,
+            id: true,
+          },
+        }),
+        (error) => ({
+          type: "other" as const,
+          cause: error,
+        })
+      ).andThen((deployment) => {
+        if (!deployment) {
+          return errAsync({ type: "deployment_not_found" as const });
+        }
+        return okAsync(deployment);
+      });
+
+    const validateDeployment = (deployment: Pick<WorkerDeployment, "id" | "status">) => {
+      if (deployment.status !== "PENDING" && deployment.status !== "INSTALLING") {
+        logger.warn(
+          "Attempted progressing deployment that is not in PENDING or INSTALLING status",
+          {
+            deployment,
+          }
+        );
+        return errAsync({ type: "deployment_cannot_be_progressed" as const });
+      }
+
+      return okAsync(deployment);
+    };
+
+    const progressToInstalling = (deployment: Pick<WorkerDeployment, "id">) =>
+      fromPromise(
+        this._prisma.workerDeployment.updateMany({
+          where: { id: deployment.id, status: "PENDING" }, // status could've changed in the meantime, we're not locking the row
+          data: {
+            ...updates,
+            status: "INSTALLING",
+            startedAt: new Date(),
+          },
+        }),
+        (error) => ({
+          type: "other" as const,
+          cause: error,
+        })
+      ).andThen((result) => {
+        if (result.count === 0) {
+          return errAsync({ type: "deployment_cannot_be_progressed" as const });
+        }
+        return okAsync({ id: deployment.id, status: "INSTALLING" as const });
+      });
+
+    const createRemoteBuild = (deployment: Pick<WorkerDeployment, "id">) =>
+      fromPromise(createRemoteImageBuild(authenticatedEnv.project), (error) => ({
+        type: "failed_to_create_remote_build" as const,
+        cause: error,
+      }));
+
+    const progressToBuilding = (deployment: Pick<WorkerDeployment, "id">) =>
+      createRemoteBuild(deployment)
+        .andThen((externalBuildData) =>
+          fromPromise(
+            this._prisma.workerDeployment.updateMany({
+              where: { id: deployment.id, status: "INSTALLING" }, // status could've changed in the meantime, we're not locking the row
+              data: {
+                ...updates,
+                externalBuildData,
+                status: "BUILDING",
+                installedAt: new Date(),
+              },
+            }),
+            (error) => ({
+              type: "other" as const,
+              cause: error,
+            })
+          )
+        )
+        .andThen((result) => {
+          if (result.count === 0) {
+            return errAsync({ type: "deployment_cannot_be_progressed" as const });
+          }
+          return okAsync({ id: deployment.id, status: "BUILDING" as const });
+        });
+
+    const extendTimeout = (deployment: Pick<WorkerDeployment, "id" | "status">) =>
+      fromPromise(
+        TimeoutDeploymentService.enqueue(
+          deployment.id,
+          deployment.status,
+          deployment.status === "INSTALLING"
+            ? "Installing dependencies timed out"
+            : "Building timed out",
+          new Date(Date.now() + env.DEPLOY_TIMEOUT_MS)
+        ),
+        (error) => ({
+          type: "failed_to_extend_deployment_timeout" as const,
+          cause: error,
+        })
+      );
+
+    return getDeployment()
+      .andThen(validateDeployment)
+      .andThen((deployment) => {
+        if (deployment.status === "PENDING") {
+          return progressToInstalling(deployment);
+        }
+        return progressToBuilding(deployment);
+      })
+      .andThen(extendTimeout)
+      .map(() => undefined);
+  }
+
+  /**
+   * Cancels a deployment that is not yet in a final state.
+   *
+   * Only acts when the current status is not final. Not idempotent.
+   *
+   * @param authenticatedEnv The environment which the deployment belongs to.
+   * @param friendlyId The friendly deployment ID.
+   * @param data Cancelation reason.
+   */
+  public cancelDeployment(
+    authenticatedEnv: Pick<AuthenticatedEnvironment, "projectId">,
+    friendlyId: string,
+    data?: Partial<Pick<WorkerDeployment, "canceledReason">>
+  ) {
+    const getDeployment = () =>
+      fromPromise(
+        this._prisma.workerDeployment.findFirst({
+          where: {
+            friendlyId,
+            projectId: authenticatedEnv.projectId,
+          },
+          select: {
+            status: true,
+            id: true,
+          },
+        }),
+        (error) => ({
+          type: "other" as const,
+          cause: error,
+        })
+      ).andThen((deployment) => {
+        if (!deployment) {
+          return errAsync({ type: "deployment_not_found" as const });
+        }
+        return okAsync(deployment);
+      });
+
+    const validateDeployment = (deployment: Pick<WorkerDeployment, "id" | "status">) => {
+      if (FINAL_DEPLOYMENT_STATUSES.includes(deployment.status)) {
+        logger.warn("Attempted cancelling deployment in a final state", {
+          deployment,
+        });
+        return errAsync({ type: "deployment_cannot_be_cancelled" as const });
+      }
+
+      return okAsync(deployment);
+    };
+
+    const cancelDeployment = (deployment: Pick<WorkerDeployment, "id">) =>
+      fromPromise(
+        this._prisma.workerDeployment.updateMany({
+          where: {
+            id: deployment.id,
+            status: {
+              notIn: FINAL_DEPLOYMENT_STATUSES, // status could've changed in the meantime, we're not locking the row
+            },
+          },
+          data: {
+            status: "CANCELED",
+            canceledAt: new Date(),
+            canceledReason: data?.canceledReason,
+          },
+        }),
+        (error) => ({
+          type: "other" as const,
+          cause: error,
+        })
+      ).andThen((result) => {
+        if (result.count === 0) {
+          return errAsync({ type: "deployment_cannot_be_cancelled" as const });
+        }
+        return okAsync({ id: deployment.id });
+      });
+
+    const deleteTimeout = (deployment: Pick<WorkerDeployment, "id">) =>
+      fromPromise(TimeoutDeploymentService.dequeue(deployment.id, this._prisma), (error) => ({
+        type: "failed_to_delete_deployment_timeout" as const,
+        cause: error,
+      }));
+
+    return getDeployment()
+      .andThen(validateDeployment)
+      .andThen(cancelDeployment)
+      .andThen(deleteTimeout)
+      .map(() => undefined);
+  }
+}

apps/webapp/app/v3/services/expireEnqueuedRun.server.ts

@@ -5,6 +5,7 @@ import { eventRepository } from "../eventRepository.server";
 import { BaseService } from "./baseService.server";
 import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
 import { getTaskEventStoreTableForRun } from "../taskEventStore.server";
+import { tryCatch } from "@trigger.dev/core/utils";
 
 export class ExpireEnqueuedRunService extends BaseService {
   public static async ack(runId: string, tx?: PrismaClientOrTransaction) {
@@ -78,28 +79,21 @@ export class ExpireEnqueuedRunService extends BaseService {
       },
     });
 
-    await eventRepository.completeEvent(
-      getTaskEventStoreTableForRun(run),
-      run.spanId,
-      run.createdAt,
-      run.completedAt ?? undefined,
-      {
-        endTime: new Date(),
-        attributes: {
-          isError: true,
-        },
-        events: [
-          {
-            name: "exception",
-            time: new Date(),
-            properties: {
-              exception: {
-                message: `Run expired because the TTL (${run.ttl}) was reached`,
-              },
-            },
-          },
-        ],
+    if (run.ttl) {
+      const [completeExpiredRunEventError] = await tryCatch(
+        eventRepository.completeExpiredRunEvent({
+          run,
+          endTime: new Date(),
+          ttl: run.ttl,
+        })
+      );
+
+      if (completeExpiredRunEventError) {
+        logger.error("[ExpireEnqueuedRunService] Failed to complete expired run event", {
+          error: completeExpiredRunEventError,
+          runId: run.id,
+        });
       }
-    );
+    }
   }
 }

apps/webapp/app/v3/services/failDeployment.server.ts

@@ -1,11 +1,11 @@
 import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server";
 import { BaseService } from "./baseService.server";
 import { logger } from "~/services/logger.server";
-import { WorkerDeploymentStatus } from "@trigger.dev/database";
-import { FailDeploymentRequestBody } from "@trigger.dev/core/v3/schemas";
-import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { type WorkerDeploymentStatus } from "@trigger.dev/database";
+import { type FailDeploymentRequestBody } from "@trigger.dev/core/v3/schemas";
+import { type AuthenticatedEnvironment } from "~/services/apiAuth.server";
 
-const FINAL_DEPLOYMENT_STATUSES: WorkerDeploymentStatus[] = [
+export const FINAL_DEPLOYMENT_STATUSES: WorkerDeploymentStatus[] = [
   "CANCELED",
   "DEPLOYED",
   "FAILED",

apps/webapp/app/v3/services/initializeDeployment.server.ts

@@ -19,7 +19,40 @@ export class InitializeDeploymentService extends BaseService {
     environment: AuthenticatedEnvironment,
     payload: InitializeDeploymentRequestBody
   ) {
-    return this.traceWithEnv("call", environment, async (span) => {
+    return this.traceWithEnv("call", environment, async () => {
+      if (payload.gitMeta?.commitSha?.startsWith("deployment_")) {
+        // When we introduced automatic deployments via the build server, we slightly changed the deployment flow
+        // mainly in the initialization and starting step: now deployments are first initialized in the `PENDING` status
+        // and updated to `BUILDING` once the build server dequeues the build job.
+        // Newer versions of the `deploy` command in the CLI will automatically attach to the existing deployment
+        // and continue with the build process. For older versions, we can't change the command's client-side behavior,
+        // so we need to handle this case here in the initialization endpoint. As we control the env variables which
+        // the git meta is extracted from in the build server, we can use those to pass the existing deployment ID
+        // to this endpoint. This doesn't affect the git meta on the deployment as it is set prior to this step using the
+        // /start endpoint. It's a rather hacky solution, but it will do for now as it enables us to avoid degrading the
+        // build server experience for users with older CLI versions. We'll eventually be able to remove this workaround
+        // once we stop supporting 3.x CLI versions.
+
+        const existingDeploymentId = payload.gitMeta.commitSha;
+        const existingDeployment = await this._prisma.workerDeployment.findFirst({
+          where: {
+            environmentId: environment.id,
+            friendlyId: existingDeploymentId,
+          },
+        });
+
+        if (!existingDeployment) {
+          throw new ServiceValidationError(
+            "Existing deployment not found during deployment initialization"
+          );
+        }
+
+        return {
+          deployment: existingDeployment,
+          imageRef: existingDeployment.imageReference ?? "",
+        };
+      }
+
       if (payload.type === "UNMANAGED") {
         throw new ServiceValidationError("UNMANAGED deployments are not supported");
       }
@@ -54,8 +87,12 @@ export class InitializeDeploymentService extends BaseService {
         );
       }
 
-      // Try and create a depot build and get back the external build data
-      const externalBuildData = await createRemoteImageBuild(environment.project);
+      // For the `PENDING` initial status, defer the creation of the Depot build until the deployment is started.
+      // This helps avoid Depot token expiration issues.
+      const externalBuildData =
+        payload.initialStatus === "PENDING"
+          ? undefined
+          : await createRemoteImageBuild(environment.project);
 
       const triggeredBy = payload.userId
         ? await this._prisma.user.findFirst({
@@ -99,6 +136,10 @@ export class InitializeDeploymentService extends BaseService {
 
       const { imageRef, isEcr, repoCreated } = imageRefResult;
 
+      // we keep using `BUILDING` as the initial status if not explicitly set
+      // to avoid changing the behavior for deployments not created in the build server
+      const initialStatus = payload.initialStatus ?? "BUILDING";
+
       logger.debug("Creating deployment", {
         environmentId: environment.id,
         projectId: environment.projectId,
@@ -108,6 +149,7 @@ export class InitializeDeploymentService extends BaseService {
         imageRef,
         isEcr,
         repoCreated,
+        initialStatus,
       });
 
       const deployment = await this._prisma.workerDeployment.create({
@@ -116,7 +158,7 @@ export class InitializeDeploymentService extends BaseService {
           contentHash: payload.contentHash,
           shortCode: deploymentShortCode,
           version: nextVersion,
-          status: "BUILDING",
+          status: initialStatus,
           environmentId: environment.id,
           projectId: environment.projectId,
           externalBuildData,
@@ -126,14 +168,18 @@ export class InitializeDeploymentService extends BaseService {
           imagePlatform: env.DEPLOY_IMAGE_PLATFORM,
           git: payload.gitMeta ?? undefined,
           runtime: payload.runtime ?? undefined,
+          startedAt: initialStatus === "BUILDING" ? new Date() : undefined,
         },
       });
 
+      const timeoutMs =
+        deployment.status === "PENDING" ? env.DEPLOY_QUEUE_TIMEOUT_MS : env.DEPLOY_TIMEOUT_MS;
+
       await TimeoutDeploymentService.enqueue(
         deployment.id,
-        "BUILDING",
+        deployment.status,
         "Building timed out",
-        new Date(Date.now() + env.DEPLOY_TIMEOUT_MS)
+        new Date(Date.now() + timeoutMs)
       );
 
       return {

apps/webapp/app/v3/taskEventStore.server.ts

@@ -21,6 +21,7 @@ export type TraceEvent = Pick<
   | "events"
   | "environmentType"
   | "kind"
+  | "attemptNumber"
 >;
 
 export type DetailedTraceEvent = Pick<
@@ -47,6 +48,7 @@ export type DetailedTraceEvent = Pick<
   | "machinePreset"
   | "properties"
   | "output"
+  | "attemptNumber"
 >;
 
 export type TaskEventStoreTable = "taskEvent" | "taskEventPartitioned";
@@ -188,7 +190,8 @@ export class TaskEventStore {
           level,
           events,
           "environmentType",
-          "kind"
+          "kind",
+          "attemptNumber"
         FROM "TaskEventPartitioned"
         WHERE
           "traceId" = ${traceId}
@@ -220,7 +223,8 @@ export class TaskEventStore {
           level,
           events,
           "environmentType",
-          "kind"
+          "kind",
+          "attemptNumber"
         FROM "TaskEvent"
         WHERE "traceId" = ${traceId}
           ${
@@ -273,7 +277,8 @@ export class TaskEventStore {
           "queueName",
           "machinePreset",
           properties,
-          output
+          output,
+          "attemptNumber"
         FROM "TaskEventPartitioned"
         WHERE
           "traceId" = ${traceId}
@@ -311,7 +316,8 @@ export class TaskEventStore {
           "queueName",
           "machinePreset",
           properties,
-          output
+          output,
+          "attemptNumber"
         FROM "TaskEvent"
         WHERE "traceId" = ${traceId}
           ${

apps/webapp/evals/aiRunFilter.eval.ts

@@ -216,6 +216,37 @@ evalite("AI Run Filter", {
           },
         }),
       },
+      // Time-specific queries with hours
+      {
+        input: "tasks that started around 8am today",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            from: new Date(new Date().toDateString() + " 07:00:00").getTime(),
+            to: new Date(new Date().toDateString() + " 09:00:00").getTime(),
+          },
+        }),
+      },
+      {
+        input: "runs that started at 2pm yesterday",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            from: new Date(new Date(Date.now() - 24*60*60*1000).toDateString() + " 14:00:00").getTime(),
+            to: new Date(new Date(Date.now() - 24*60*60*1000).toDateString() + " 14:59:59").getTime(),
+          },
+        }),
+      },
+      {
+        input: "any runs started this morning",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            from: new Date(new Date().toDateString() + " 06:00:00").getTime(),
+            to: new Date(new Date().toDateString() + " 12:00:00").getTime(),
+          },
+        }),
+      },
       // Ambiguous cases that should return errors
       {
         input: "Show me something",

apps/webapp/package.json

@@ -159,6 +159,7 @@
     "match-sorter": "^6.3.4",
     "morgan": "^1.10.0",
     "nanoid": "3.3.8",
+    "neverthrow": "^8.2.0",
     "non.geist": "^1.0.2",
     "octokit": "^3.2.1",
     "ohash": "^1.1.3",

apps/webapp/test/engine/triggerTask.test.ts

@@ -16,7 +16,7 @@ vi.mock("~/services/platform.v3.server", async (importOriginal) => {
 
 import { RunEngine } from "@internal/run-engine";
 import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests";
-import { containerTest } from "@internal/testcontainers";
+import { assertNonNullable, containerTest } from "@internal/testcontainers";
 import { trace } from "@opentelemetry/api";
 import { IOPacket } from "@trigger.dev/core/v3";
 import { TaskRun } from "@trigger.dev/database";
@@ -31,11 +31,15 @@ import {
   TagValidationParams,
   TracedEventSpan,
   TraceEventConcern,
+  TriggerRacepoints,
+  TriggerRacepointSystem,
   TriggerTaskRequest,
   TriggerTaskValidator,
   ValidationResult,
 } from "~/runEngine/types";
 import { RunEngineTriggerTaskService } from "../../app/runEngine/services/triggerTask.server";
+import { promiseWithResolvers } from "@trigger.dev/core";
+import { setTimeout } from "node:timers/promises";
 
 vi.setConfig({ testTimeout: 30_000 }); // 30 seconds timeout
 
@@ -108,6 +112,29 @@ class MockTraceEventConcern implements TraceEventConcern {
   }
 }
 
+type TriggerRacepoint = { promise: Promise<void>; resolve: (value: void) => void };
+
+class MockTriggerRacepointSystem implements TriggerRacepointSystem {
+  private racepoints: Record<string, TriggerRacepoint | undefined> = {};
+
+  async waitForRacepoint({ id }: { racepoint: TriggerRacepoints; id: string }): Promise<void> {
+    const racepoint = this.racepoints[id];
+
+    if (racepoint) {
+      return racepoint.promise;
+    }
+
+    return Promise.resolve();
+  }
+
+  registerRacepoint(racepoint: TriggerRacepoints, id: string): TriggerRacepoint {
+    const { promise, resolve } = promiseWithResolvers<void>();
+    this.racepoints[id] = { promise, resolve };
+
+    return { promise, resolve };
+  }
+}
+
 describe("RunEngineTriggerTaskService", () => {
   containerTest("should trigger a task with minimal options", async ({ prisma, redisOptions }) => {
     const engine = new RunEngine({
@@ -312,6 +339,228 @@ describe("RunEngineTriggerTaskService", () => {
     await engine.quit();
   });
 
+  containerTest(
+    "should handle idempotency keys when the engine throws an RunDuplicateIdempotencyKeyError",
+    async ({ prisma, redisOptions }) => {
+      const engine = new RunEngine({
+        prisma,
+        worker: {
+          redis: redisOptions,
+          workers: 1,
+          tasksPerWorker: 10,
+          pollIntervalMs: 100,
+        },
+        queue: {
+          redis: redisOptions,
+        },
+        runLock: {
+          redis: redisOptions,
+        },
+        machines: {
+          defaultMachine: "small-1x",
+          machines: {
+            "small-1x": {
+              name: "small-1x" as const,
+              cpu: 0.5,
+              memory: 0.5,
+              centsPerMs: 0.0001,
+            },
+          },
+          baseCostInCents: 0.0005,
+        },
+        tracer: trace.getTracer("test", "0.0.0"),
+        logLevel: "debug",
+      });
+
+      const parentTask = "parent-task";
+
+      const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
+
+      const taskIdentifier = "test-task";
+
+      //create background worker
+      await setupBackgroundWorker(engine, authenticatedEnvironment, [parentTask, taskIdentifier]);
+
+      const parentRun1 = await engine.trigger(
+        {
+          number: 1,
+          friendlyId: "run_p1",
+          environment: authenticatedEnvironment,
+          taskIdentifier: parentTask,
+          payload: "{}",
+          payloadType: "application/json",
+          context: {},
+          traceContext: {},
+          traceId: "t12345",
+          spanId: "s12345",
+          queue: `task/${parentTask}`,
+          isTest: false,
+          tags: [],
+          workerQueue: "main",
+        },
+        prisma
+      );
+
+      //dequeue parent and create the attempt
+      await setTimeout(500);
+      const dequeued = await engine.dequeueFromWorkerQueue({
+        consumerId: "test_12345",
+        workerQueue: "main",
+      });
+      await engine.startRunAttempt({
+        runId: parentRun1.id,
+        snapshotId: dequeued[0].snapshot.id,
+      });
+
+      const parentRun2 = await engine.trigger(
+        {
+          number: 2,
+          friendlyId: "run_p2",
+          environment: authenticatedEnvironment,
+          taskIdentifier: parentTask,
+          payload: "{}",
+          payloadType: "application/json",
+          context: {},
+          traceContext: {},
+          traceId: "t12346",
+          spanId: "s12346",
+          queue: `task/${parentTask}`,
+          isTest: false,
+          tags: [],
+          workerQueue: "main",
+        },
+        prisma
+      );
+
+      await setTimeout(500);
+      const dequeued2 = await engine.dequeueFromWorkerQueue({
+        consumerId: "test_12345",
+        workerQueue: "main",
+      });
+      await engine.startRunAttempt({
+        runId: parentRun2.id,
+        snapshotId: dequeued2[0].snapshot.id,
+      });
+
+      const queuesManager = new DefaultQueueManager(prisma, engine);
+
+      const idempotencyKeyConcern = new IdempotencyKeyConcern(
+        prisma,
+        engine,
+        new MockTraceEventConcern()
+      );
+
+      const triggerRacepointSystem = new MockTriggerRacepointSystem();
+
+      const triggerTaskService = new RunEngineTriggerTaskService({
+        engine,
+        prisma,
+        runNumberIncrementer: new MockRunNumberIncrementer(),
+        payloadProcessor: new MockPayloadProcessor(),
+        queueConcern: queuesManager,
+        idempotencyKeyConcern,
+        validator: new MockTriggerTaskValidator(),
+        traceEventConcern: new MockTraceEventConcern(),
+        tracer: trace.getTracer("test", "0.0.0"),
+        metadataMaximumSize: 1024 * 1024 * 1, // 1MB
+        triggerRacepointSystem,
+      });
+
+      const idempotencyKey = "test-idempotency-key";
+
+      const racepoint = triggerRacepointSystem.registerRacepoint("idempotencyKey", idempotencyKey);
+
+      const childTriggerPromise1 = triggerTaskService.call({
+        taskId: taskIdentifier,
+        environment: authenticatedEnvironment,
+        body: {
+          payload: { test: "test" },
+          options: {
+            idempotencyKey,
+            parentRunId: parentRun1.friendlyId,
+            resumeParentOnCompletion: true,
+          },
+        },
+      });
+
+      const childTriggerPromise2 = triggerTaskService.call({
+        taskId: taskIdentifier,
+        environment: authenticatedEnvironment,
+        body: {
+          payload: { test: "test" },
+          options: {
+            idempotencyKey,
+            parentRunId: parentRun2.friendlyId,
+            resumeParentOnCompletion: true,
+          },
+        },
+      });
+
+      await setTimeout(500);
+
+      // Now we can resolve the racepoint
+      racepoint.resolve();
+
+      const result = await childTriggerPromise1;
+      const result2 = await childTriggerPromise2;
+
+      expect(result).toBeDefined();
+      expect(result?.run.friendlyId).toBeDefined();
+      expect(result?.run.status).toBe("PENDING");
+
+      const run = await prisma.taskRun.findUnique({
+        where: {
+          id: result?.run.id,
+        },
+      });
+
+      expect(run).toBeDefined();
+      expect(run?.friendlyId).toBe(result?.run.friendlyId);
+      expect(run?.engine).toBe("V2");
+      expect(run?.queuedAt).toBeDefined();
+      expect(run?.queue).toBe(`task/${taskIdentifier}`);
+
+      expect(result2).toBeDefined();
+      expect(result2?.run.friendlyId).toBe(result?.run.friendlyId);
+
+      const parent1ExecutionData = await engine.getRunExecutionData({ runId: parentRun1.id });
+      assertNonNullable(parent1ExecutionData);
+      expect(parent1ExecutionData.snapshot.executionStatus).toBe("EXECUTING_WITH_WAITPOINTS");
+
+      const parent2ExecutionData = await engine.getRunExecutionData({ runId: parentRun2.id });
+      assertNonNullable(parent2ExecutionData);
+      expect(parent2ExecutionData.snapshot.executionStatus).toBe("EXECUTING_WITH_WAITPOINTS");
+
+      const parent1RunWaitpoint = await prisma.taskRunWaitpoint.findFirst({
+        where: {
+          taskRunId: parentRun1.id,
+        },
+        include: {
+          waitpoint: true,
+        },
+      });
+
+      assertNonNullable(parent1RunWaitpoint);
+      expect(parent1RunWaitpoint.waitpoint.type).toBe("RUN");
+      expect(parent1RunWaitpoint.waitpoint.completedByTaskRunId).toBe(result?.run.id);
+
+      const parent2RunWaitpoint = await prisma.taskRunWaitpoint.findFirst({
+        where: {
+          taskRunId: parentRun2.id,
+        },
+        include: {
+          waitpoint: true,
+        },
+      });
+
+      assertNonNullable(parent2RunWaitpoint);
+      expect(parent2RunWaitpoint.waitpoint.type).toBe("RUN");
+      expect(parent2RunWaitpoint.waitpoint.completedByTaskRunId).toBe(result2?.run.id);
+
+      await engine.quit();
+    }
+  );
+
   containerTest(
     "should resolve queue names correctly when locked to version",
     async ({ prisma, redisOptions }) => {

apps/webapp/test/fairDequeuingStrategy.test.ts

@@ -270,8 +270,8 @@ describe("FairDequeuingStrategy", () => {
 
       console.log("Second distribution took", distribute2Duration, "ms");
 
-      // Make sure the second call is more than 2 times faster than the first
-      expect(distribute2Duration).toBeLessThan(withTolerance(distribute1Duration / 2));
+      // Make sure the second call is faster than the first
+      expect(distribute2Duration).toBeLessThan(distribute1Duration);
 
       const startDistribute3 = performance.now();
 
@@ -284,8 +284,8 @@ describe("FairDequeuingStrategy", () => {
 
       console.log("Third distribution took", distribute3Duration, "ms");
 
-      // Make sure the third call is more than 4 times the second
-      expect(withTolerance(distribute3Duration)).toBeGreaterThan(distribute2Duration * 4);
+      // Make sure the third call is faster than the second
+      expect(withTolerance(distribute3Duration)).toBeGreaterThan(distribute2Duration);
     }
   );
 

docs/config/extensions/additionalFiles.mdx

@@ -13,10 +13,11 @@ import { additionalFiles } from "@trigger.dev/build/extensions/core";
 export default defineConfig({
   project: "<project ref>",
   // Your other config settings...
+  // We strongly recommend setting this to false
+  // When set to `false`, the current working directory will be set to the build directory, which more closely matches production behavior.
+  legacyDevProcessCwdBehaviour: false, // Default: true
   build: {
-    extensions: [
-      additionalFiles({ files: ["wrangler/wrangler.toml", "./assets/**", "./fonts/**"] }),
-    ],
+    extensions: [additionalFiles({ files: ["./assets/**", "wrangler/wrangler.toml"] })],
   },
 });
 ```
@@ -25,4 +26,13 @@ This will copy the files specified in the `files` array to the build directory.
 
 This extension effects both the `dev` and the `deploy` commands, and the resulting paths will be the same for both.
 
+If you use `legacyDevProcessCwdBehaviour: false`, you can then do this:
+
+```ts
+import path from "node:path";
+
+// You can use `process.cwd()` if you use `legacyDevProcessCwdBehaviour: false`
+const interRegularFont = path.join(process.cwd(), "assets/Inter-Regular.ttf");
+```
+
 <Note>The root of the project is the directory that contains the trigger.config.ts file</Note>

docs/docs.json

@@ -336,12 +336,14 @@
           {
             "group": "Example projects",
             "pages": [
+              "guides/example-projects/anchor-browser-web-scraper",
               "guides/example-projects/batch-llm-evaluator",
               "guides/example-projects/claude-thinking-chatbot",
               "guides/example-projects/human-in-the-loop-workflow",
               "guides/example-projects/mastra-agents-with-memory",
               "guides/example-projects/meme-generator-human-in-the-loop",
               "guides/example-projects/openai-agents-sdk-typescript-playground",
+              "guides/example-projects/product-image-generator",
               "guides/example-projects/realtime-csv-importer",
               "guides/example-projects/realtime-fal-ai",
               "guides/example-projects/turborepo-monorepo-prisma",
@@ -375,6 +377,7 @@
               "guides/examples/puppeteer",
               "guides/examples/react-pdf",
               "guides/examples/react-email",
+              "guides/examples/replicate-image-generation",
               "guides/examples/resend-email-sequence",
               "guides/examples/satori",
               "guides/examples/scrape-hacker-news",

docs/guides/example-projects/anchor-browser-web-scraper.mdx

@@ -0,0 +1,124 @@
+---
+title: "Automated website monitoring with Anchor Browser"
+sidebarTitle: "Anchor Browser web scraper"
+description: "Automated web monitoring using Trigger.dev's task scheduling and Anchor Browser's AI-powered browser automation."
+---
+
+import WebScrapingWarning from "/snippets/web-scraping-warning.mdx";
+
+<WebScrapingWarning />
+
+## Overview
+
+This example demonstrates automated web monitoring using Trigger.dev's task scheduling and Anchor Browser's AI-powered browser automation tools.
+
+The task runs daily at 5pm ET to find the cheapest Broadway tickets available for same-day shows.
+
+**How it works:**
+
+- Trigger.dev schedules and executes the monitoring task
+- Anchor Browser spins up a remote browser session with an AI agent
+- The AI agent uses computer vision and natural language processing to analyze the TDF website
+- AI agent returns the lowest-priced show with specific details: name, price, and showtime
+
+## Tech stack
+
+- **[Node.js](https://nodejs.org)** runtime environment (version 18.2 or higher)
+- **[Trigger.dev](https://trigger.dev)** for task scheduling and task orchestration
+- **[Anchor Browser](https://anchorbrowser.io/)** for AI-powered browser automation
+- **[Playwright](https://playwright.dev/)** for browser automation libraries (handled via external dependencies)
+
+## GitHub repo
+
+<Card
+  title="View the Anchor Browser web scraper repo"
+  icon="GitHub"
+  href="https://github.com/triggerdotdev/examples/tree/main/anchor-browser-web-scraper"
+>
+  Click here to view the full code for this project in our examples repository on GitHub. You can
+  fork it and use it as a starting point for your own project.
+</Card>
+
+## Relevant code
+
+### Broadway ticket monitor task
+
+This task runs daily at 5pm ET, in [src/trigger/broadway-monitor.ts](https://github.com/triggerdotdev/examples/tree/main/anchor-browser-web-scraper/src/trigger/broadway-monitor.ts):
+
+```ts
+import { schedules } from "@trigger.dev/sdk";
+import Anchorbrowser from "anchorbrowser";
+
+export const broadwayMonitor = schedules.task({
+  id: "broadway-ticket-monitor",
+  cron: "0 21 * * *",
+  run: async (payload, { ctx }) => {
+    const client = new Anchorbrowser({
+      apiKey: process.env.ANCHOR_BROWSER_API_KEY!,
+    });
+
+    let session;
+    try {
+      // Create explicit session to get live view URL
+      session = await client.sessions.create();
+      console.log(`Session ID: ${session.data.id}`);
+      console.log(`Live View URL: https://live.anchorbrowser.io?sessionId=${session.data.id}`);
+
+      const response = await client.tools.performWebTask({
+        sessionId: session.data.id,
+        url: "https://www.tdf.org/discount-ticket-programs/tkts-by-tdf/tkts-live/",
+        prompt: `Look for the "Broadway Shows" section on this page. Find the show with the absolute lowest starting price available right now and return the show name, current lowest price, and show time. Be very specific about the current price you see. Format as: Show: [name], Price: [exact current price], Time: [time]`,
+      });
+
+      console.log("Raw response:", response);
+
+      const result = response.data.result?.result || response.data.result || response.data;
+
+      if (result && typeof result === "string" && result.includes("Show:")) {
+        console.log(`🎭 Best Broadway Deal Found!`);
+        console.log(result);
+
+        return {
+          success: true,
+          bestDeal: result,
+          liveViewUrl: `https://live.anchorbrowser.io?sessionId=${session.data.id}`,
+        };
+      } else {
+        console.log("No Broadway deals found today");
+        return { success: true, message: "No deals found" };
+      }
+    } finally {
+      if (session?.data?.id) {
+        try {
+          await client.sessions.delete(session.data.id);
+        } catch (cleanupError) {
+          console.warn("Failed to cleanup session:", cleanupError);
+        }
+      }
+    }
+  },
+});
+```
+
+### Build configuration
+
+Since Anchor Browser uses browser automation libraries (Playwright) under the hood, we need to configure Trigger.dev to handle these dependencies properly by excluding them from the build bundle in [trigger.config.ts](https://github.com/triggerdotdev/examples/tree/main/anchor-browser-web-scraper/trigger.config.ts):
+
+```ts
+import { defineConfig } from "@trigger.dev/sdk";
+
+export default defineConfig({
+  project: "proj_your_project_id_here", // Get from Trigger.dev dashboard
+  maxDuration: 3600, // 1 hour - plenty of time for web automation
+  dirs: ["./src/trigger"],
+  build: {
+    external: ["playwright-core", "playwright", "chromium-bidi"],
+  },
+});
+```
+
+## Learn more
+
+- View the [Anchor Browser docs](https://anchorbrowser.io/docs) to learn more about Anchor Browser's AI-powered browser automation tools.
+- Check out the source code for the [Anchor Browser web scraper repo](https://github.com/triggerdotdev/examples/tree/main/anchor-browser-web-scraper) on GitHub.
+- Browser our [example projects](/guides/introduction) to see how you can use Trigger.dev with other services.

docs/guides/example-projects/product-image-generator.mdx

@@ -0,0 +1,61 @@
+---
+title: "Product image generator using Replicate and Trigger.dev"
+sidebarTitle: "Product image generator"
+description: "AI-powered product image generator that transforms basic product photos into professional marketing shots using Replicate's image generation models"
+---
+
+## Overview
+
+This project demonstrates how to build an AI-powered product image generator that transforms basic product photos into professional marketing shots. Users upload a product image and receive three professionally styled variations: clean product shots, lifestyle scenes, and hero shots with dramatic lighting.
+
+## Video
+
+<video
+  controls
+  className="w-full aspect-video"
+  src="https://content.trigger.dev/product-image-generator-example.mp4"
+/>
+
+## GitHub repo
+
+Clone this repo and follow the instructions in the `README.md` file to get started.
+
+<Card
+  title="View the product image generator repo"
+  icon="GitHub"
+  href="https://github.com/triggerdotdev/examples/tree/main/product-image-generator"
+>
+  Click here to view the full code in our examples repository on GitHub. You can fork it and use it
+  as a starting point for your project.
+</Card>
+
+## Tech stack
+
+- [**Next.js**](https://nextjs.org/) – frontend React framework
+- [**Replicate**](https://replicate.com/docs) – AI image generation using the `google/nano-banana` image-to-image model
+- [**UploadThing**](https://uploadthing.com/) – file upload management and server callbacks
+- [**Cloudflare R2**](https://developers.cloudflare.com/r2/) – scalable image storage with public URLs
+
+## How it works
+
+The application orchestrates image generation through two main tasks: [`generateImages`](https://github.com/triggerdotdev/examples/blob/main/product-image-generator/app/trigger/generate-images.ts) coordinates batch processing, while [`generateImage`](https://github.com/triggerdotdev/examples/blob/main/product-image-generator/app/trigger/generate-images.ts) handles individual style generation.
+
+Each generation task enhances prompts with style-specific instructions, calls Replicate's `google/nano-banana` image-to-image model, creates waitpoint tokens for async webhook handling, and uploads results to Cloudflare R2. The frontend displays real-time progress updates via React hooks as tasks complete.
+
+Style presets include clean product shots (white background), lifestyle scenes (person holding product), and hero shots (dramatic lighting).
+
+## Relevant code
+
+- **Image generation tasks** – batch processing with waitpoints for Replicate webhook callbacks ([`app/trigger/generate-images.ts`](https://github.com/triggerdotdev/examples/blob/main/product-image-generator/app/trigger/generate-images.ts))
+- **Upload handler** – UploadThing integration that triggers batch generation ([`app/api/uploadthing/core.ts`](https://github.com/triggerdotdev/examples/blob/main/product-image-generator/app/api/uploadthing/core.ts))
+- **Real-time progress UI** – live task updates using React hooks ([`app/components/GeneratedCard.tsx`](https://github.com/triggerdotdev/examples/blob/main/product-image-generator/app/components/GeneratedCard.tsx))
+- **Custom prompt interface** – user-defined style generation ([`app/components/CustomPromptCard.tsx`](https://github.com/triggerdotdev/examples/blob/main/product-image-generator/app/components/CustomPromptCard.tsx))
+- **Main app component** – layout and state management ([`app/ProductImageGenerator.tsx`](https://github.com/triggerdotdev/examples/blob/main/product-image-generator/app/ProductImageGenerator.tsx))
+
+## Learn more
+
+- [**Waitpoints**](/wait-for-token) – pause tasks for async webhook callbacks
+- [**React hooks**](/realtime/react-hooks/overview) – real-time task updates and frontend integration
+- [**Batch operations**](/triggering#tasks-batchtrigger) – parallel task execution patterns
+- [**Replicate API**](https://replicate.com/docs/get-started/nextjs) – AI model integration
+- [**UploadThing**](https://docs.uploadthing.com/) – file upload handling and server callbacks

docs/guides/examples/replicate-image-generation.mdx

@@ -0,0 +1,109 @@
+---
+title: "Image-to-image generation using Replicate and nano-banana"
+sidebarTitle: "Replicate image generation"
+description: "Learn how to generate images from source image URLs using Replicate and Trigger.dev."
+---
+
+## Overview
+
+This example demonstrates how to use Trigger.dev to generate images from source image URLs using [Replicate](https://replicate.com/), the [nano-banana-image-to-image](https://replicate.com/meta/nano-banana-image-to-image) model.
+
+## Task code
+
+```tsx trigger/generateImage.tsx
+import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import { task, wait } from "@trigger.dev/sdk";
+import Replicate, { Prediction } from "replicate";
+
+// Initialize clients
+const replicate = new Replicate({
+  auth: process.env.REPLICATE_API_TOKEN,
+});
+
+const s3Client = new S3Client({
+  region: "auto",
+  endpoint: process.env.R2_ENDPOINT,
+  credentials: {
+    accessKeyId: process.env.R2_ACCESS_KEY_ID ?? "",
+    secretAccessKey: process.env.R2_SECRET_ACCESS_KEY ?? "",
+  },
+});
+
+const model = "google/nano-banana";
+
+export const generateImageAndUploadToR2 = task({
+  id: "generate-image-and-upload-to-r2",
+  run: async (payload: { prompt: string; imageUrl: string }) => {
+    const { prompt, imageUrl } = payload;
+
+    const token = await wait.createToken({
+      timeout: "10m",
+    });
+
+    // Use Flux with structured prompt
+    const output = await replicate.predictions.create({
+      model: model,
+      input: { prompt, image_input: [imageUrl] },
+      // pass the provided URL to Replicate's webhook, so they can "callback"
+      webhook: token.url,
+      webhook_events_filter: ["completed"],
+    });
+
+    const result = await wait.forToken<Prediction>(token).unwrap();
+    // unwrap() throws a timeout error or returns the result 👆
+
+    if (!result.ok) {
+      throw new Error("Failed to create prediction");
+    }
+
+    const generatedImageUrl = result.output.output;
+
+    const image = await fetch(generatedImageUrl);
+    const imageBuffer = Buffer.from(await image.arrayBuffer());
+
+    const base64Image = Buffer.from(imageBuffer).toString("base64");
+
+    const timestamp = Date.now();
+    const filename = `generated-${timestamp}.png`;
+
+    // Generate unique key for R2
+    const sanitizedFileName = filename.replace(/[^a-zA-Z0-9.-]/g, "_");
+    const r2Key = `uploaded-images/${timestamp}-${sanitizedFileName}`;
+
+    const uploadParams = {
+      Bucket: process.env.R2_BUCKET,
+      Key: r2Key,
+      Body: imageBuffer,
+      ContentType: "image/png",
+      // Add cache control for better performance
+      CacheControl: "public, max-age=31536000", // 1 year
+    };
+
+    const uploadResult = await s3Client.send(new PutObjectCommand(uploadParams));
+
+    // Construct the public URL using the R2_PUBLIC_URL env var
+    const publicUrl = `${process.env.R2_PUBLIC_URL}/${r2Key}`;
+
+    return {
+      success: true,
+      publicUrl,
+      originalPrompt: prompt,
+      sourceImageUrl: imageUrl,
+    };
+  },
+});
+```
+
+## Environment variables
+
+You will need to set the following environment variables:
+
+```
+TRIGGER_SECRET_KEY=<your-trigger-secret-key>
+REPLICATE_API_TOKEN=<your-replicate-api-token>
+R2_ENDPOINT=<your-r2-endpoint>
+R2_ACCESS_KEY_ID=<your-r2-access-key-id>
+R2_SECRET_ACCESS_KEY=<your-r2-secret-access-key>
+R2_BUCKET=<your-r2-bucket>
+R2_PUBLIC_URL=<your-r2-public-url>
+```

docs/guides/frameworks/supabase-edge-functions-basic.mdx

@@ -77,7 +77,7 @@ Replace the placeholder code in your `edge-function-trigger/index.ts` file with
 // Setup type definitions for built-in Supabase Runtime APIs
 import "jsr:@supabase/functions-js/edge-runtime.d.ts";
 // Import the Trigger.dev SDK - replace "<your-sdk-version>" with the version of the SDK you are using, e.g. "3.0.0". You can find this in your package.json file.
-import { tasks } from "npm:@trigger.dev/sdk@3.0.0/v3";
+import { tasks } from "npm:@trigger.dev/sdk@3.0.0";
 // Import your task type from your /trigger folder
 import type { helloWorldTask } from "../../../src/trigger/example.ts";
 //     👆 **type-only** import

docs/guides/frameworks/supabase-edge-functions-database-webhooks.mdx

@@ -330,7 +330,7 @@ supabase functions new video-processing-handler
 ```ts functions/video-processing-handler/index.ts
 // Setup type definitions for built-in Supabase Runtime APIs
 import "jsr:@supabase/functions-js/edge-runtime.d.ts";
-import { tasks } from "npm:@trigger.dev/sdk@latest/v3";
+import { tasks } from "npm:@trigger.dev/sdk@latest";
 // Import the videoProcessAndUpdate task from the trigger folder
 import type { videoProcessAndUpdate } from "../../../src/trigger/videoProcessAndUpdate.ts";
 //     👆 type only import

docs/guides/introduction.mdx

@@ -45,12 +45,14 @@ Example projects are full projects with example repos you can fork and use. Thes
 
 | Example project                                                                                                 | Description                                                                                                                            | Framework | GitHub                                                                                                         |
 | :-------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------- | :-------- | :------------------------------------------------------------------------------------------------------------- |
+| [Anchor Browser web scraper](/guides/example-projects/anchor-browser-web-scraper)                               | Monitor a website and find the cheapest tickets for a show.                                                                            | —         | [View the repo](https://github.com/triggerdotdev/examples/tree/main/anchor-browser-web-scraper)                |
 | [Batch LLM Evaluator](/guides/example-projects/batch-llm-evaluator)                                             | Evaluate multiple LLM models and stream the results to the frontend.                                                                   | Next.js   | [View the repo](https://github.com/triggerdotdev/examples/tree/main/batch-llm-evaluator)                       |
 | [Claude thinking chatbot](/guides/example-projects/claude-thinking-chatbot)                                     | Use Vercel's AI SDK and Anthropic's Claude 3.7 model to create a thinking chatbot.                                                     | Next.js   | [View the repo](https://github.com/triggerdotdev/examples/tree/main/claude-thinking-chatbot)                   |
 | [Human-in-the-loop workflow](/guides/example-projects/human-in-the-loop-workflow)                               | Create audio summaries of newspaper articles using a human-in-the-loop workflow built with ReactFlow and Trigger.dev waitpoint tokens. | Next.js   | [View the repo](https://github.com/triggerdotdev/examples/tree/main/article-summary-workflow)                  |
 | [Mastra agents with memory](/guides/example-projects/mastra-agents-with-memory)                                 | Use Mastra to create a weather agent that can collect live weather data and generate clothing recommendations.                         | —         | [View the repo](https://github.com/triggerdotdev/examples/tree/main/mastra-agents)                             |
 | [OpenAI Agents SDK for Python guardrails](/guides/example-projects/openai-agent-sdk-guardrails)                 | Use the OpenAI Agents SDK for Python to create a guardrails system for your AI agents.                                                 | —         | [View the repo](https://github.com/triggerdotdev/examples/tree/main/openai-agent-sdk-guardrails-examples)      |
 | [OpenAI Agents SDK for TypeScript playground](/guides/example-projects/openai-agents-sdk-typescript-playground) | A playground containing 7 AI agents using the OpenAI Agents SDK for TypeScript with Trigger.dev.                                       | —         | [View the repo](https://github.com/triggerdotdev/examples/tree/main/openai-agents-sdk-with-trigger-playground) |
+| [Product image generator](/guides/example-projects/product-image-generator)                                     | Transform basic product photos into professional marketing shots using Replicate's image generation models.                            | Next.js   | [View the repo](https://github.com/triggerdotdev/examples/tree/main/product-image-generator)                   |
 | [Python web crawler](/guides/python/python-crawl4ai)                                                            | Use Python, Crawl4AI and Playwright to create a headless web crawler with Trigger.dev.                                                 | —         | [View the repo](https://github.com/triggerdotdev/examples/tree/main/python-crawl4ai)                           |
 | [Realtime CSV Importer](/guides/example-projects/realtime-csv-importer)                                         | Upload a CSV file and see the progress of the task streamed to the frontend.                                                           | Next.js   | [View the repo](https://github.com/triggerdotdev/examples/tree/main/realtime-csv-importer)                     |
 | [Realtime Fal.ai image generation](/guides/example-projects/realtime-fal-ai)                                    | Generate an image from a prompt using Fal.ai and show the progress of the task on the frontend using Realtime.                         | Next.js   | [View the repo](https://github.com/triggerdotdev/examples/tree/main/realtime-fal-ai-image-generation)          |
@@ -78,6 +80,7 @@ Task code you can copy and paste to use in your project. They can all be extende
 | [React email](/guides/examples/react-email)                                   | Send an email using React Email.                                                                                                                     |
 | [React to PDF](/guides/examples/react-pdf)                                    | Use `react-pdf` to generate a PDF and save it to Cloudflare R2.                                                                                      |
 | [Resend email sequence](/guides/examples/resend-email-sequence)               | Send a sequence of emails over several days using Resend with Trigger.dev.                                                                           |
+| [Replicate image generation](/guides/examples/replicate-image-generation)     | Learn how to generate images from source image URLs using Replicate and Trigger.dev.                                                                 |
 | [Satori](/guides/examples/satori)                                             | Generate OG images using React Satori.                                                                                                               |
 | [Scrape Hacker News](/guides/examples/scrape-hacker-news)                     | Scrape Hacker News using BrowserBase and Puppeteer, summarize the articles with ChatGPT and send an email of the summary every weekday using Resend. |
 | [Sentry error tracking](/guides/examples/sentry-error-tracking)               | Automatically send errors to Sentry from your tasks.                                                                                                 |

docs/guides/python/python-crawl4ai.mdx

@@ -62,7 +62,7 @@ After you've initialized your project with Trigger.dev, add these build settings
 ```ts trigger.config.ts
 import { defineConfig } from "@trigger.dev/sdk";
 import { pythonExtension } from "@trigger.dev/python/extension";
-import type { BuildContext, BuildExtension } from "@trigger.dev/core/v3/build";
+import type { BuildContext, BuildExtension } from "@trigger.dev/core/build";
 
 export default defineConfig({
   project: "<project ref>",

docs/how-to-reduce-your-spend.mdx

@@ -6,6 +6,7 @@ description: "Tips and best practices to reduce your costs on Trigger.dev"
 ## Check out your usage page regularly
 
 Monitor your usage dashboard to understand your spending patterns. You can see:
+
 - Your most expensive tasks
 - Your total duration by task
 - Number of runs by task
@@ -18,6 +19,7 @@ You can view your usage page by clicking the "Organization" menu in the top left
 ## Create billing alerts
 
 Configure billing alerts in your dashboard to get notified when you approach spending thresholds. This helps you:
+
 - Catch unexpected cost increases early
 - Identify runaway tasks before they become expensive
 
@@ -43,7 +45,7 @@ export const lightTask = task({
 
 // Only use larger machines when necessary
 export const heavyTask = task({
-  id: "heavy-task", 
+  id: "heavy-task",
   machine: "medium-1x", // 1 vCPU, 2 GB RAM
   run: async (payload) => {
     // CPU/memory intensive operations
@@ -64,11 +66,14 @@ export const expensiveApiCall = task({
   id: "expensive-api-call",
   run: async (payload: { userId: string }) => {
     // This expensive operation will only run once per user
-    await wait.for({ seconds: 30 }, { 
-      idempotencyKey: `user-processing-${payload.userId}`,
-      idempotencyKeyTTL: "1h" 
-    });
-    
+    await wait.for(
+      { seconds: 30 },
+      {
+        idempotencyKey: `user-processing-${payload.userId}`,
+        idempotencyKeyTTL: "1h",
+      }
+    );
+
     const result = await processUserData(payload.userId);
     return result;
   },
@@ -105,7 +110,7 @@ export const processItems = task({
   id: "process-items",
   run: async (payload: { items: string[] }) => {
     // Process all items in parallel
-    const promises = payload.items.map(item => processItem(item));
+    const promises = payload.items.map((item) => processItem(item));
     // This works very well for API calls
     await Promise.all(promises);
   },
@@ -133,7 +138,7 @@ export const apiTask = task({
 This is very useful for intermittent errors, but if there's a permanent error you don't want to retry because you will just keep failing and waste compute. Use [AbortTaskRunError](/errors-retrying#using-aborttaskrunerror) to prevent a retry:
 
 ```ts
-import { task, AbortTaskRunError } from "@trigger.dev/sdk/v3";
+import { task, AbortTaskRunError } from "@trigger.dev/sdk";
 
 export const someTask = task({
   id: "some-task",
@@ -145,13 +150,11 @@ export const someTask = task({
       throw new AbortTaskRunError(result.error);
     }
 
-    return result
+    return result;
   },
 });
 ```
 
-
-
 ## Use appropriate maxDuration settings
 
 Set realistic maxDurations to prevent runs from executing for too long:

docs/introduction.mdx

@@ -28,7 +28,7 @@ mode: "center"
 
 Trigger.dev is an open source background jobs framework that lets you write reliable workflows in plain async code. Run long-running AI tasks, handle complex background jobs, and build AI agents with built-in queuing, automatic retries, and real-time monitoring. No timeouts, elastic scaling, and zero infrastructure management required.
 
-We provide everything you need to build and manage background tasks: a CLI and SDK for writing tasks in your existing codebase, support for both [regular](/tasks/overview) and [scheduled](/tasks/scheduled) tasks, full observability through our dashboard, and a [Realtime API](/realtime) with [React hooks](/realtime/react-hooks#realtime-hooks) for showing task status in your frontend. You can use [Trigger.dev Cloud](https://cloud.trigger.dev) or [self-host](/open-source-self-hosting) on your own infrastructure.
+We provide everything you need to build and manage background tasks: a CLI and SDK for writing tasks in your existing codebase, support for both [regular](/tasks/overview) and [scheduled](/tasks/scheduled) tasks, full observability through our dashboard, and a [Realtime API](/realtime) with [React hooks](/realtime/react-hooks#realtime-hooks) for showing task status in your frontend. You can use [Trigger.dev Cloud](https://cloud.trigger.dev) or [self-host](/self-hosting/overview) on your own infrastructure.
 
 ## Learn the concepts
 

docs/machines.mdx

@@ -6,7 +6,7 @@ description: "Configure the number of vCPUs and GBs of RAM you want the task to
 The `machine` configuration is optional. Using higher spec machines will increase the cost of running the task but can also improve the performance of the task if it is CPU or memory bound.
 
 ```ts /trigger/heavy-task.ts
-import {  task } from "@trigger.dev/sdk";
+import { task } from "@trigger.dev/sdk";
 
 export const heavyTask = task({
   id: "heavy-task",
@@ -30,15 +30,15 @@ export const config: TriggerConfig = {
 
 ## Machine configurations
 
-| Preset              | vCPU | Memory | Disk space |
-| :------------------ | :--- | :----- | :--------- |
-| micro               | 0.25 | 0.25   | 10GB       |
-| small-1x (default)  | 0.5  | 0.5    | 10GB       |
-| small-2x            | 1    | 1      | 10GB       |
-| medium-1x           | 1    | 2      | 10GB       |
-| medium-2x           | 2    | 4      | 10GB       |
-| large-1x            | 4    | 8      | 10GB       |
-| large-2x            | 8    | 16     | 10GB       |
+| Preset             | vCPU | Memory | Disk space |
+| :----------------- | :--- | :----- | :--------- |
+| micro              | 0.25 | 0.25   | 10GB       |
+| small-1x (default) | 0.5  | 0.5    | 10GB       |
+| small-2x           | 1    | 1      | 10GB       |
+| medium-1x          | 1    | 2      | 10GB       |
+| medium-2x          | 2    | 4      | 10GB       |
+| large-1x           | 4    | 8      | 10GB       |
+| large-2x           | 8    | 16     | 10GB       |
 
 You can view the Trigger.dev cloud pricing for these machines [here](https://trigger.dev/pricing#computePricing).
 
@@ -60,14 +60,855 @@ This is useful when you know that a certain payload will require more memory tha
 
 Sometimes you might see one of your runs fail with an "Out Of Memory" error.
 
-> TASK_PROCESS_OOM_KILLED. Your task ran out of memory. Try increasing the machine specs. If this doesn't fix it there might be a memory leak.
+> TASK_PROCESS_OOM_KILLED. Your run was terminated due to exceeding the machine's memory limit. Try increasing the machine preset in your task options or replay using a larger machine.
 
-We automatically detect common Out Of Memory errors, including when ffmpeg throws an error because it ran out of memory.
+We automatically detect common Out Of Memory errors:
+
+- When using Node.js, if the V8 heap limit is exceeded (this can happen when creating large long-lived objects)
+- When the entire process exceeds the memory limit of the machine the run is executing on.
+- When a child process, such as ffmpeg, causes the memory limit of the machine to be exceeded, and exits with a non-zero code.
+
+### Memory/Resource monitoring
+
+To better understand why an OOM error occurred, we've published a helper class that will log memory debug information at regular intervals.
+
+First, add this `ResourceMonitor` class to your project:
+
+<Accordion title="View ResourceMonitor class">
+
+```ts /src/resourceMonitor.ts
+import { promisify } from "node:util";
+import { exec } from "node:child_process";
+import os from "node:os";
+import { promises as fs } from "node:fs";
+import { type Context, logger } from "@trigger.dev/sdk";
+import { getHeapStatistics } from "node:v8";
+import { PerformanceObserver, constants } from "node:perf_hooks";
+
+const execAsync = promisify(exec);
+
+export type DiskMetrics = {
+  total: number;
+  used: number;
+  free: number;
+  percentUsed: number;
+  warning?: string;
+};
+
+export type MemoryMetrics = {
+  total: number;
+  free: number;
+  used: number;
+  percentUsed: number;
+};
+
+export type NodeProcessMetrics = {
+  memoryUsage: number;
+  memoryUsagePercent: number;
+  heapUsed: number;
+  heapSizeLimit: number;
+  heapUsagePercent: number;
+  availableHeap: number;
+  isNearHeapLimit: boolean;
+};
+
+export type TargetProcessMetrics = {
+  method: string;
+  processName: string;
+  count: number;
+  processes: ProcessInfo[];
+  averages: {
+    cpu: number;
+    memory: number;
+    rss: number;
+    vsz: number;
+  } | null;
+  totals: {
+    cpu: number;
+    memory: number;
+    rss: number;
+    vsz: number;
+  } | null;
+};
+
+export type ProcessMetrics = {
+  node: NodeProcessMetrics;
+  targetProcess: TargetProcessMetrics | null;
+};
+
+type GCSummary = {
+  count: number;
+  totalDuration: number; // ms
+  avgDuration: number; // ms
+  maxDuration: number; // ms
+  kinds: Record<
+    string,
+    {
+      // breakdown by kind
+      count: number;
+      totalDuration: number;
+      avgDuration: number;
+      maxDuration: number;
+    }
+  >;
+};
+
+type ProcessInfo = {
+  user: string;
+  pid: number;
+  cpu: number;
+  mem: number;
+  vsz: number;
+  rss: number;
+  command: string;
+};
+
+export type SystemMetrics = {
+  disk: DiskMetrics;
+  memory: MemoryMetrics;
+};
+
+export type ResourceMonitorConfig = {
+  dirName?: string;
+  processName?: string;
+  ctx: Context;
+  compactLogging?: boolean;
+};
+
+// Constants
+const DISK_LIMIT_GB = 10;
+const DISK_LIMIT_BYTES = DISK_LIMIT_GB * 1024 * 1024 * 1024; // 10Gi in bytes
+
+export class ResourceMonitor {
+  private logInterval: NodeJS.Timeout | null = null;
+  private logger: typeof logger;
+  private dirName: string;
+  private processName: string | undefined;
+  private ctx: Context;
+  private verbose: boolean;
+  private compactLogging: boolean;
+  private gcObserver: PerformanceObserver | null = null;
+  private bufferedGcEntries: PerformanceEntry[] = [];
+
+  constructor(config: ResourceMonitorConfig) {
+    this.logger = logger;
+    this.dirName = config.dirName ?? "/tmp";
+    this.processName = config.processName;
+    this.ctx = config.ctx;
+    this.verbose = true;
+    this.compactLogging = config.compactLogging ?? false;
+  }
+
+  /**
+   * Start periodic resource monitoring
+   * @param intervalMs Monitoring interval in milliseconds
+   */
+  startMonitoring(intervalMs = 10000): void {
+    if (intervalMs < 1000) {
+      intervalMs = 1000;
+      this.logger.warn("ResourceMonitor: intervalMs is less than 1000, setting to 1000");
+    }
+
+    if (this.logInterval) {
+      clearInterval(this.logInterval);
+    }
+
+    this.logInterval = setInterval(this.logResources.bind(this), intervalMs);
+
+    this.gcObserver = new PerformanceObserver((list) => {
+      this.bufferedGcEntries.push(...list.getEntries());
+    });
+
+    this.gcObserver.observe({ entryTypes: ["gc"], buffered: true });
+  }
+
+  /**
+   * Stop resource monitoring
+   */
+  stopMonitoring(): void {
+    if (this.logInterval) {
+      clearInterval(this.logInterval);
+      this.logInterval = null;
+    }
+
+    if (this.gcObserver) {
+      this.gcObserver.disconnect();
+      this.gcObserver = null;
+    }
+  }
+
+  private async logResources() {
+    try {
+      await this.logResourceSnapshot("ResourceMonitor");
+    } catch (error) {
+      this.logger.error(
+        `Resource monitoring error: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+  }
+
+  /**
+   * Get combined system metrics (disk and memory)
+   */
+  private async getSystemMetrics(): Promise<SystemMetrics> {
+    const [disk, memory] = await Promise.all([this.getDiskMetrics(), this.getMemoryMetrics()]);
+    return { disk, memory };
+  }
+
+  /**
+   * Get disk space information
+   */
+  private async getDiskMetrics(): Promise<DiskMetrics> {
+    try {
+      // Even with permission errors, du will output a total
+      const { stdout, stderr } = await execAsync(`du -sb ${this.dirName} || true`);
+
+      // Get the last line of stdout which contains the total
+      const lastLine = stdout.split("\n").filter(Boolean).pop() || "";
+      const usedBytes = parseInt(lastLine.split("\t")[0], 10);
+
+      const effectiveTotal = DISK_LIMIT_BYTES;
+      const effectiveUsed = Math.min(usedBytes, DISK_LIMIT_BYTES);
+      const effectiveFree = effectiveTotal - effectiveUsed;
+      const percentUsed = (effectiveUsed / effectiveTotal) * 100;
+
+      const metrics: DiskMetrics = {
+        total: effectiveTotal,
+        used: effectiveUsed,
+        free: effectiveFree,
+        percentUsed,
+      };
+
+      // If we had permission errors, add a warning
+      if (stderr.includes("Permission denied") || stderr.includes("cannot access")) {
+        metrics.warning = "Some directories were not accessible";
+      } else if (stderr.includes("No such file or directory")) {
+        metrics.warning = "The directory does not exist";
+      }
+
+      return metrics;
+    } catch (error) {
+      this.logger.error(
+        `Error getting disk metrics: ${error instanceof Error ? error.message : String(error)}`
+      );
+      return {
+        free: DISK_LIMIT_BYTES,
+        total: DISK_LIMIT_BYTES,
+        used: 0,
+        percentUsed: 0,
+        warning: "Failed to measure disk usage",
+      };
+    }
+  }
+
+  /**
+   * Get memory metrics
+   */
+  private getMemoryMetrics(): MemoryMetrics {
+    const total = os.totalmem();
+    const free = os.freemem();
+    const used = total - free;
+    const percentUsed = (used / total) * 100;
+
+    return { total, free, used, percentUsed };
+  }
+
+  /**
+   * Get process-specific metrics using /proc filesystem
+   */
+  private async getProcMetrics(pids: number[]): Promise<ProcessInfo[]> {
+    return Promise.all(
+      pids.map(async (pid) => {
+        try {
+          // Read process status
+          const status = await fs.readFile(`/proc/${pid}/status`, "utf8");
+          const cmdline = await fs.readFile(`/proc/${pid}/cmdline`, "utf8");
+          const stat = await fs.readFile(`/proc/${pid}/stat`, "utf8");
+
+          // Parse VmRSS (resident set size) from status
+          const rss = parseInt(status.match(/VmRSS:\s+(\d+)/)?.[1] ?? "0", 10);
+          // Parse VmSize (virtual memory size) from status
+          const vsz = parseInt(status.match(/VmSize:\s+(\d+)/)?.[1] ?? "0", 10);
+          // Get process owner
+          const user = (await fs.stat(`/proc/${pid}`)).uid.toString();
+
+          // Parse CPU stats from /proc/[pid]/stat
+          const stats = stat.split(" ");
+          const utime = parseInt(stats[13], 10);
+          const stime = parseInt(stats[14], 10);
+          const starttime = parseInt(stats[21], 10);
+
+          // Calculate CPU percentage
+          const totalTime = utime + stime;
+          const uptime = os.uptime();
+          const hertz = 100; // Usually 100 on Linux
+          const elapsedTime = uptime - starttime / hertz;
+          const cpuUsage = 100 * (totalTime / hertz / elapsedTime);
+
+          // Calculate memory percentage against total system memory
+          const totalMem = os.totalmem();
+          const memoryPercent = (rss * 1024 * 100) / totalMem;
+
+          return {
+            user,
+            pid,
+            cpu: cpuUsage,
+            mem: memoryPercent,
+            vsz,
+            rss,
+            command: cmdline.replace(/\0/g, " ").trim(),
+          };
+        } catch (error) {
+          return null;
+        }
+      })
+    ).then((results) => results.filter((r): r is ProcessInfo => r !== null));
+  }
+
+  /**
+   * Find PIDs for a process name using /proc filesystem
+   */
+  private async findPidsByName(processName?: string): Promise<number[]> {
+    if (!processName) {
+      return [];
+    }
+
+    try {
+      const pids: number[] = [];
+      const procDirs = await fs.readdir("/proc");
+
+      for (const dir of procDirs) {
+        if (!/^\d+$/.test(dir)) continue;
+
+        const processPid = parseInt(dir, 10);
+
+        // Ignore processes that have a lower PID than our own PID
+        if (processPid <= process.pid) {
+          continue;
+        }
+
+        try {
+          const cmdline = await fs.readFile(`/proc/${dir}/cmdline`, "utf8");
+          if (cmdline.includes(processName)) {
+            pids.push(parseInt(dir, 10));
+          }
+        } catch {
+          // Ignore errors reading individual process info
+          continue;
+        }
+      }
+
+      return pids;
+    } catch {
+      return [];
+    }
+  }
+
+  /**
+   * Get process-specific metrics
+   */
+  private async getProcessMetrics(): Promise<ProcessMetrics> {
+    // Get Node.js process metrics
+    const totalMemory = os.totalmem();
+    // Convert GB to bytes (machine.memory is in GB)
+    const machineMemoryBytes = this.ctx.machine
+      ? this.ctx.machine.memory * 1024 * 1024 * 1024
+      : totalMemory;
+    const nodeMemoryUsage = process.memoryUsage();
+
+    // Node process percentage is based on machine memory if available, otherwise system memory
+    const nodeMemoryPercent = (nodeMemoryUsage.rss / machineMemoryBytes) * 100;
+    const heapStats = getHeapStatistics();
+
+    const nodeMetrics: NodeProcessMetrics = {
+      memoryUsage: nodeMemoryUsage.rss,
+      memoryUsagePercent: nodeMemoryPercent,
+      heapUsed: nodeMemoryUsage.heapUsed,
+      heapSizeLimit: heapStats.heap_size_limit,
+      heapUsagePercent: (heapStats.used_heap_size / heapStats.heap_size_limit) * 100,
+      availableHeap: heapStats.total_available_size,
+      isNearHeapLimit: heapStats.used_heap_size / heapStats.heap_size_limit > 0.8,
+    };
+
+    let method = "ps";
+
+    try {
+      let processes: ProcessInfo[] = [];
+
+      // Try ps first, fall back to /proc if it fails
+      try {
+        const { stdout: psOutput } = await execAsync(
+          `ps aux | grep ${this.processName} | grep -v grep`
+        );
+
+        if (psOutput.trim()) {
+          processes = psOutput
+            .trim()
+            .split("\n")
+            .filter((line) => {
+              const parts = line.trim().split(/\s+/);
+              const pid = parseInt(parts[1], 10);
+
+              // Ignore processes that have a lower PID than our own PID
+              return pid > process.pid;
+            })
+            .map((line) => {
+              const parts = line.trim().split(/\s+/);
+              return {
+                user: parts[0],
+                pid: parseInt(parts[1], 10),
+                cpu: parseFloat(parts[2]),
+                mem: parseFloat(parts[3]),
+                vsz: parseInt(parts[4], 10),
+                rss: parseInt(parts[5], 10),
+                command: parts.slice(10).join(" "),
+              };
+            });
+        }
+      } catch {
+        // ps failed, try /proc instead
+        method = "proc";
+        const pids = await this.findPidsByName(this.processName);
+        processes = await this.getProcMetrics(pids);
+      }
+
+      if (processes.length === 0) {
+        return {
+          node: nodeMetrics,
+          targetProcess: this.processName
+            ? {
+                method,
+                processName: this.processName,
+                count: 0,
+                processes: [],
+                averages: null,
+                totals: null,
+              }
+            : null,
+        };
+      }
+
+      // For CPU:
+      // - ps shows CPU percentage per core (e.g., 100% = 1 core)
+      // - machine.cpu is in cores (e.g., 0.5 = half a core)
+      // - we want to show percentage of allocated CPU (e.g., 100% = using all allocated CPU)
+      const availableCpu = this.ctx.machine?.cpu ?? os.cpus().length;
+      const cpuNormalizer = availableCpu * 100; // Convert to basis points for better precision with fractional CPUs
+
+      // For Memory:
+      // - ps 'mem' is already a percentage of system memory
+      // - we need to convert it to a percentage of machine memory
+      // - if machine memory is 0.5GB and system has 16GB, we multiply the percentage by 32
+      const memoryScaleFactor = this.ctx.machine ? totalMemory / machineMemoryBytes : 1;
+
+      const totals = processes.reduce(
+        (acc, proc) => ({
+          cpu: acc.cpu + proc.cpu,
+          // Scale memory percentage to machine memory
+          // TODO: test this
+          memory: acc.memory + proc.mem * memoryScaleFactor,
+          rss: acc.rss + proc.rss,
+          vsz: acc.vsz + proc.vsz,
+        }),
+        { cpu: 0, memory: 0, rss: 0, vsz: 0 }
+      );
+
+      const count = processes.length;
+
+      const averages = {
+        cpu: totals.cpu / (count * cpuNormalizer),
+        memory: totals.memory / count,
+        rss: totals.rss / count,
+        vsz: totals.vsz / count,
+      };
+
+      return {
+        node: nodeMetrics,
+        targetProcess: this.processName
+          ? {
+              method,
+              processName: this.processName,
+              count,
+              processes,
+              averages,
+              totals: {
+                cpu: totals.cpu / cpuNormalizer,
+                memory: totals.memory,
+                rss: totals.rss,
+                vsz: totals.vsz,
+              },
+            }
+          : null,
+      };
+    } catch (error) {
+      return {
+        node: nodeMetrics,
+        targetProcess: this.processName
+          ? {
+              method,
+              processName: this.processName,
+              count: 0,
+              processes: [],
+              averages: null,
+              totals: null,
+            }
+          : null,
+      };
+    }
+  }
+
+  /**
+   * Log a snapshot of current resource usage
+   */
+  async logResourceSnapshot(label = "Resource Snapshot"): Promise<void> {
+    try {
+      const payload = await this.getResourceSnapshotPayload();
+      const enhancedLabel = this.compactLogging
+        ? this.createCompactLabel(payload, label)
+        : this.createEnhancedLabel(payload, label);
+
+      if (payload.process.node.isNearHeapLimit) {
+        this.logger.warn(`${enhancedLabel}: Node is near heap limit`, payload);
+      } else {
+        this.logger.info(enhancedLabel, payload);
+      }
+    } catch (error) {
+      this.logger.error(
+        `Error logging resource snapshot: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+  }
+
+  /**
+   * Create an enhanced log label with key metrics for quick scanning
+   */
+  private createEnhancedLabel(payload: any, baseLabel: string): string {
+    const parts: string[] = [baseLabel];
+
+    // System resources with text indicators
+    const diskPercent = parseFloat(payload.system.disk.percentUsed);
+    const memoryPercent = parseFloat(payload.system.memory.percentUsed);
+    const diskIndicator = this.getTextIndicator(diskPercent, 80, 90);
+    const memIndicator = this.getTextIndicator(memoryPercent, 80, 90);
+    parts.push(`Disk:${diskPercent.toFixed(1).padStart(5)}%${diskIndicator}`);
+    parts.push(`Mem:${memoryPercent.toFixed(1).padStart(5)}%${memIndicator}`);
+
+    // Node process metrics with text indicators
+    const nodeMemPercent = parseFloat(payload.process.node.memoryUsagePercent);
+    const heapPercent = parseFloat(payload.process.node.heapUsagePercent);
+    const nodeIndicator = this.getTextIndicator(nodeMemPercent, 70, 85);
+    const heapIndicator = this.getTextIndicator(heapPercent, 70, 85);
+    parts.push(`Node:${nodeMemPercent.toFixed(1).padStart(4)}%${nodeIndicator}`);
+    parts.push(`Heap:${heapPercent.toFixed(1).padStart(4)}%${heapIndicator}`);
+
+    // Target process metrics (if available)
+    if (payload.process.targetProcess && payload.process.targetProcess.count > 0) {
+      const targetCpu = payload.process.targetProcess.totals?.cpuPercent || "0";
+      const targetMem = payload.process.targetProcess.totals?.memoryPercent || "0";
+      const targetCpuNum = parseFloat(targetCpu);
+      const targetMemNum = parseFloat(targetMem);
+      const cpuIndicator = this.getTextIndicator(targetCpuNum, 80, 90);
+      const memIndicator = this.getTextIndicator(targetMemNum, 80, 90);
+      parts.push(
+        `${payload.process.targetProcess.processName}:${targetCpu.padStart(
+          4
+        )}%${cpuIndicator}/${targetMem.padStart(4)}%${memIndicator}`
+      );
+    }
+
+    // GC activity with performance indicators
+    if (payload.gc && payload.gc.count > 0) {
+      const avgDuration = payload.gc.avgDuration;
+      const gcIndicator = this.getTextIndicator(avgDuration, 5, 10, true);
+      parts.push(
+        `GC:${payload.gc.count.toString().padStart(2)}(${avgDuration
+          .toFixed(1)
+          .padStart(4)}ms)${gcIndicator}`
+      );
+    }
+
+    // Machine constraints
+    if (payload.constraints) {
+      parts.push(`[${payload.constraints.cpu}CPU/${payload.constraints.memoryGB}GB]`);
+    }
+
+    // Warning indicators (only show critical ones in the main label)
+    const criticalWarnings: string[] = [];
+    if (payload.process.node.isNearHeapLimit) criticalWarnings.push("HEAP_LIMIT");
+    if (diskPercent > 90) criticalWarnings.push("DISK_CRITICAL");
+    if (memoryPercent > 95) criticalWarnings.push("MEM_CRITICAL");
+    if (payload.system.disk.warning) criticalWarnings.push("DISK_WARN");
+
+    if (criticalWarnings.length > 0) {
+      parts.push(`[${criticalWarnings.join(",")}]`);
+    }
+
+    return parts.join(" | ");
+  }
+
+  /**
+   * Get text-based indicator for percentage values
+   */
+  private getTextIndicator(
+    value: number,
+    warningThreshold: number,
+    criticalThreshold: number,
+    isDuration = false
+  ): string {
+    if (isDuration) {
+      // For duration values, higher is worse
+      if (value >= criticalThreshold) return " [CRIT]";
+      if (value >= warningThreshold) return " [WARN]";
+      return " [OK]";
+    } else {
+      // For percentage values, higher is worse
+      if (value >= criticalThreshold) return " [CRIT]";
+      if (value >= warningThreshold) return " [WARN]";
+      return " [OK]";
+    }
+  }
+
+  /**
+   * Create a compact version of the enhanced label for high-frequency logging
+   */
+  private createCompactLabel(payload: any, baseLabel: string): string {
+    const parts: string[] = [baseLabel];
+
+    // Only show critical metrics in compact mode
+    const diskPercent = parseFloat(payload.system.disk.percentUsed);
+    const memoryPercent = parseFloat(payload.system.memory.percentUsed);
+    const heapPercent = parseFloat(payload.process.node.heapUsagePercent);
+
+    // Use single character indicators for compactness
+    const diskIndicator = diskPercent > 90 ? "!" : diskPercent > 80 ? "?" : ".";
+    const memIndicator = memoryPercent > 95 ? "!" : memoryPercent > 80 ? "?" : ".";
+    const heapIndicator = heapPercent > 85 ? "!" : heapPercent > 70 ? "?" : ".";
+
+    parts.push(`D:${diskPercent.toFixed(0).padStart(2)}%${diskIndicator}`);
+    parts.push(`M:${memoryPercent.toFixed(0).padStart(2)}%${memIndicator}`);
+    parts.push(`H:${heapPercent.toFixed(0).padStart(2)}%${heapIndicator}`);
+
+    // GC activity (only if significant)
+    if (payload.gc && payload.gc.count > 0 && payload.gc.avgDuration > 2) {
+      const gcIndicator =
+        payload.gc.avgDuration > 10 ? "!" : payload.gc.avgDuration > 5 ? "?" : ".";
+      parts.push(`GC:${payload.gc.count}${gcIndicator}`);
+    }
+
+    return parts.join(" ");
+  }
+
+  async getResourceSnapshotPayload() {
+    const [systemMetrics, processMetrics] = await Promise.all([
+      this.getSystemMetrics(),
+      this.getProcessMetrics(),
+    ]);
+
+    const gcSummary = summarizeGCEntries(this.bufferedGcEntries);
+    this.bufferedGcEntries = [];
+
+    const formatBytes = (bytes: number) => (bytes / (1024 * 1024)).toFixed(2);
+    const formatPercent = (value: number) => value.toFixed(1);
+
+    return {
+      system: {
+        disk: {
+          limitGiB: DISK_LIMIT_GB,
+          dirName: this.dirName,
+          usedGiB: (systemMetrics.disk.used / (1024 * 1024 * 1024)).toFixed(2),
+          freeGiB: (systemMetrics.disk.free / (1024 * 1024 * 1024)).toFixed(2),
+          percentUsed: formatPercent(systemMetrics.disk.percentUsed),
+          warning: systemMetrics.disk.warning,
+        },
+        memory: {
+          freeGB: (systemMetrics.memory.free / (1024 * 1024 * 1024)).toFixed(2),
+          percentUsed: formatPercent(systemMetrics.memory.percentUsed),
+        },
+      },
+      gc: gcSummary,
+      constraints: this.ctx.machine
+        ? {
+            cpu: this.ctx.machine.cpu,
+            memoryGB: this.ctx.machine.memory,
+            diskGB: DISK_LIMIT_BYTES / (1024 * 1024 * 1024),
+          }
+        : {
+            cpu: os.cpus().length,
+            memoryGB: Math.floor(os.totalmem() / (1024 * 1024 * 1024)),
+            note: "Using system resources (no machine constraints specified)",
+          },
+      process: {
+        node: {
+          memoryUsageMB: formatBytes(processMetrics.node.memoryUsage),
+          memoryUsagePercent: formatPercent(processMetrics.node.memoryUsagePercent),
+          heapUsedMB: formatBytes(processMetrics.node.heapUsed),
+          heapSizeLimitMB: formatBytes(processMetrics.node.heapSizeLimit),
+          heapUsagePercent: formatPercent(processMetrics.node.heapUsagePercent),
+          availableHeapMB: formatBytes(processMetrics.node.availableHeap),
+          isNearHeapLimit: processMetrics.node.isNearHeapLimit,
+          ...(this.verbose
+            ? {
+                heapStats: getHeapStatistics(),
+              }
+            : {}),
+        },
+        targetProcess: processMetrics.targetProcess
+          ? {
+              method: processMetrics.targetProcess.method,
+              processName: processMetrics.targetProcess.processName,
+              count: processMetrics.targetProcess.count,
+              averages: processMetrics.targetProcess.averages
+                ? {
+                    cpuPercent: formatPercent(processMetrics.targetProcess.averages.cpu * 100),
+                    memoryPercent: formatPercent(processMetrics.targetProcess.averages.memory),
+                    rssMB: formatBytes(processMetrics.targetProcess.averages.rss * 1024),
+                    vszMB: formatBytes(processMetrics.targetProcess.averages.vsz * 1024),
+                  }
+                : null,
+              totals: processMetrics.targetProcess.totals
+                ? {
+                    cpuPercent: formatPercent(processMetrics.targetProcess.totals.cpu * 100),
+                    memoryPercent: formatPercent(processMetrics.targetProcess.totals.memory),
+                    rssMB: formatBytes(processMetrics.targetProcess.totals.rss * 1024),
+                    vszMB: formatBytes(processMetrics.targetProcess.totals.vsz * 1024),
+                  }
+                : null,
+            }
+          : null,
+      },
+      timestamp: new Date().toISOString(),
+    };
+  }
+}
+
+function summarizeGCEntries(entries: PerformanceEntry[]): GCSummary {
+  if (entries.length === 0) {
+    return {
+      count: 0,
+      totalDuration: 0,
+      avgDuration: 0,
+      maxDuration: 0,
+      kinds: {},
+    };
+  }
+
+  let totalDuration = 0;
+  let maxDuration = 0;
+  const kinds: Record<string, { count: number; totalDuration: number; maxDuration: number }> = {};
+
+  for (const e of entries) {
+    const duration = e.duration;
+    totalDuration += duration;
+    if (duration > maxDuration) maxDuration = duration;
+
+    const kind = kindName((e as any)?.detail?.kind ?? "unknown");
+    if (!kinds[kind]) {
+      kinds[kind] = { count: 0, totalDuration: 0, maxDuration: 0 };
+    }
+    kinds[kind].count += 1;
+    kinds[kind].totalDuration += duration;
+    if (duration > kinds[kind].maxDuration) kinds[kind].maxDuration = duration;
+  }
+
+  // finalize averages
+  const avgDuration = totalDuration / entries.length;
+  const kindsWithAvg: GCSummary["kinds"] = {};
+  for (const [kind, stats] of Object.entries(kinds)) {
+    kindsWithAvg[kind] = {
+      count: stats.count,
+      totalDuration: stats.totalDuration,
+      avgDuration: stats.totalDuration / stats.count,
+      maxDuration: stats.maxDuration,
+    };
+  }
+
+  return {
+    count: entries.length,
+    totalDuration,
+    avgDuration,
+    maxDuration,
+    kinds: kindsWithAvg,
+  };
+}
+
+const kindName = (k: number | string) => {
+  if (typeof k === "number") {
+    return (
+      {
+        [constants.NODE_PERFORMANCE_GC_MAJOR]: "major",
+        [constants.NODE_PERFORMANCE_GC_MINOR]: "minor",
+        [constants.NODE_PERFORMANCE_GC_INCREMENTAL]: "incremental",
+        [constants.NODE_PERFORMANCE_GC_WEAKCB]: "weak-cb",
+      }[k] ?? `kind:${k}`
+    );
+  }
+  return k;
+};
+```
+
+</Accordion>
+
+Then, in your task, you can create an instance of the `ResourceMonitor` class and start monitoring memory, disk, and CPU usage:
+
+```ts /src/trigger/example.ts
+import { task, logger, wait } from "@trigger.dev/sdk";
+import { ResourceMonitor } from "../resourceMonitor.js";
+
+// Middleware to enable the resource monitor
+tasks.middleware("resource-monitor", async ({ ctx, next }) => {
+  const resourceMonitor = new ResourceMonitor({
+    ctx,
+  });
+
+  // Only enable the resource monitor if the environment variable is set
+  if (process.env.RESOURCE_MONITOR_ENABLED === "1") {
+    resourceMonitor.startMonitoring(1_000);
+  }
+
+  await next();
+
+  resourceMonitor.stopMonitoring();
+});
+
+export const resourceMonitorTest = task({
+  id: "resource-monitor-test",
+  run: async (payload: any, { ctx }) => {
+    const interval = createMemoryPressure();
+
+    await setTimeout(180_000);
+
+    clearInterval(interval);
+
+    return {
+      message: "Hello, resources!",
+    };
+  },
+});
+```
+
+This will produce logs that look like this:
+
+![Resource monitor logs](/images/machines-resource-monitor-logs.png)
+
+If you are spawning a child process and you want to monitor its memory usage, you can pass the `processName` option to the `ResourceMonitor` class:
+
+```ts /src/trigger/example.ts
+const resourceMonitor = new ResourceMonitor({
+  ctx,
+  processName: "ffmpeg",
+});
+```
+
+This will produce logs that includes the memory and CPU usage of the `ffmpeg` process:
+
+![Resource monitor logs](/images/machines-resource-monitor-ffmpeg.png)
+
+### Explicit OOM errors
 
 You can explicitly throw an Out Of Memory error in your task. This can be useful if you use a native package that detects it's going to run out of memory and then stops before it runs out. If you can detect this, you can then throw this error.
 
 ```ts /trigger/heavy-task.ts
-import {  task } from "@trigger.dev/sdk";
+import { task } from "@trigger.dev/sdk";
 import { OutOfMemoryError } from "@trigger.dev/sdk";
 
 export const yourTask = task({
@@ -88,7 +929,7 @@ If OOM errors happen regularly you need to either optimize the memory-efficiency
 If you are seeing rare OOM errors, it might make sense to add a setting to your task to retry with a large machine when an OOM happens:
 
 ```ts /trigger/heavy-task.ts
-import {  task } from "@trigger.dev/sdk";
+import { task } from "@trigger.dev/sdk";
 
 export const yourTask = task({
   id: "your-task",
@@ -105,5 +946,7 @@ export const yourTask = task({
 ```
 
 <Note>
-  This will only retry the task if you get an OOM error. It won't permanently change the machine that a new run starts on, so if you consistently see OOM errors you should change the machine in the `machine` property.
+  This will only retry the task if you get an OOM error. It won't permanently change the machine
+  that a new run starts on, so if you consistently see OOM errors you should change the machine in
+  the `machine` property.
 </Note>

docs/realtime/auth.mdx

@@ -23,7 +23,7 @@ You can create a Public Access Token using the `auth.createPublicToken` function
 
 ```tsx
 // Somewhere in your backend code
-import { auth } from "@trigger.dev/sdk/v3";
+import { auth } from "@trigger.dev/sdk";
 
 const publicToken = await auth.createPublicToken(); // 👈 this public access token has no permissions, so is pretty useless!
 ```
@@ -33,7 +33,7 @@ const publicToken = await auth.createPublicToken(); // 👈 this public access t
 By default a Public Access Token has no permissions. You must specify the scopes you need when creating a Public Access Token:
 
 ```ts
-import { auth } from "@trigger.dev/sdk/v3";
+import { auth } from "@trigger.dev/sdk";
 
 const publicToken = await auth.createPublicToken({
   scopes: {
@@ -47,7 +47,7 @@ const publicToken = await auth.createPublicToken({
 This will allow the token to read all runs, which is probably not what you want. You can specify only certain runs by passing an array of run IDs:
 
 ```ts
-import { auth } from "@trigger.dev/sdk/v3";
+import { auth } from "@trigger.dev/sdk";
 
 const publicToken = await auth.createPublicToken({
   scopes: {
@@ -61,7 +61,7 @@ const publicToken = await auth.createPublicToken({
 You can scope the token to only read certain tasks:
 
 ```ts
-import { auth } from "@trigger.dev/sdk/v3";
+import { auth } from "@trigger.dev/sdk";
 
 const publicToken = await auth.createPublicToken({
   scopes: {
@@ -75,7 +75,7 @@ const publicToken = await auth.createPublicToken({
 Or tags:
 
 ```ts
-import { auth } from "@trigger.dev/sdk/v3";
+import { auth } from "@trigger.dev/sdk";
 
 const publicToken = await auth.createPublicToken({
   scopes: {
@@ -89,7 +89,7 @@ const publicToken = await auth.createPublicToken({
 Or a specific batch of runs:
 
 ```ts
-import { auth } from "@trigger.dev/sdk/v3";
+import { auth } from "@trigger.dev/sdk";
 
 const publicToken = await auth.createPublicToken({
   scopes: {
@@ -103,7 +103,7 @@ const publicToken = await auth.createPublicToken({
 You can also combine scopes. For example, to read runs with specific tags and for specific tasks:
 
 ```ts
-import { auth } from "@trigger.dev/sdk/v3";
+import { auth } from "@trigger.dev/sdk";
 
 const publicToken = await auth.createPublicToken({
   scopes: {
@@ -120,7 +120,7 @@ const publicToken = await auth.createPublicToken({
 By default, Public Access Token's expire after 15 minutes. You can specify a different expiration time when creating a Public Access Token:
 
 ```ts
-import { auth } from "@trigger.dev/sdk/v3";
+import { auth } from "@trigger.dev/sdk";
 
 const publicToken = await auth.createPublicToken({
   expirationTime: "1hr",
@@ -156,7 +156,7 @@ For triggering tasks from your frontend, you need special "trigger" tokens. Thes
 ### Creating Trigger Tokens
 
 ```ts
-import { auth } from "@trigger.dev/sdk/v3";
+import { auth } from "@trigger.dev/sdk";
 
 // Somewhere in your backend code
 const triggerToken = await auth.createTriggerPublicToken("my-task");
@@ -167,7 +167,7 @@ const triggerToken = await auth.createTriggerPublicToken("my-task");
 You can pass multiple tasks to create a token that can trigger multiple tasks:
 
 ```ts
-import { auth } from "@trigger.dev/sdk/v3";
+import { auth } from "@trigger.dev/sdk";
 
 // Somewhere in your backend code
 const triggerToken = await auth.createTriggerPublicToken(["my-task-1", "my-task-2"]);
@@ -178,7 +178,7 @@ const triggerToken = await auth.createTriggerPublicToken(["my-task-1", "my-task-
 You can also create tokens that can be used multiple times:
 
 ```ts
-import { auth } from "@trigger.dev/sdk/v3";
+import { auth } from "@trigger.dev/sdk";
 
 // Somewhere in your backend code
 const triggerToken = await auth.createTriggerPublicToken("my-task", {
@@ -191,7 +191,7 @@ const triggerToken = await auth.createTriggerPublicToken("my-task", {
 These tokens also expire, with the default expiration time being 15 minutes. You can specify a custom expiration time:
 
 ```ts
-import { auth } from "@trigger.dev/sdk/v3";
+import { auth } from "@trigger.dev/sdk";
 
 // Somewhere in your backend code
 const triggerToken = await auth.createTriggerPublicToken("my-task", {

docs/realtime/backend/overview.mdx

@@ -30,7 +30,7 @@ See our [authentication guide](/realtime/auth) for detailed information on creat
 Subscribe to a run:
 
 ```ts
-import { runs, tasks } from "@trigger.dev/sdk/v3";
+import { runs, tasks } from "@trigger.dev/sdk";
 
 // Trigger a task
 const handle = await tasks.trigger("my-task", { some: "data" });

docs/realtime/backend/streams.mdx

@@ -20,7 +20,7 @@ Streams use the metadata system to send data chunks in real-time. You register a
 Here's how to stream data from OpenAI in your task:
 
 ```ts
-import { task, metadata } from "@trigger.dev/sdk/v3";
+import { task, metadata } from "@trigger.dev/sdk";
 import OpenAI from "openai";
 
 const openai = new OpenAI({
@@ -64,7 +64,7 @@ export const myTask = task({
 You can subscribe to the stream using the `runs.subscribeToRun` method with `.withStreams()`:
 
 ```ts
-import { runs } from "@trigger.dev/sdk/v3";
+import { runs } from "@trigger.dev/sdk";
 import type { myTask, STREAMS } from "./trigger/my-task";
 
 // Somewhere in your backend
@@ -91,7 +91,7 @@ async function subscribeToStream(runId: string) {
 You can register and subscribe to multiple streams in the same task:
 
 ```ts
-import { task, metadata } from "@trigger.dev/sdk/v3";
+import { task, metadata } from "@trigger.dev/sdk";
 import OpenAI from "openai";
 
 const openai = new OpenAI({
@@ -138,7 +138,7 @@ export const myTask = task({
 Then subscribe to both streams:
 
 ```ts
-import { runs } from "@trigger.dev/sdk/v3";
+import { runs } from "@trigger.dev/sdk";
 import type { myTask, STREAMS } from "./trigger/my-task";
 
 // Somewhere in your backend
@@ -170,7 +170,7 @@ The [AI SDK](https://sdk.vercel.ai/docs/introduction) provides a higher-level AP
 
 ```ts
 import { openai } from "@ai-sdk/openai";
-import { logger, metadata, runs, schemaTask } from "@trigger.dev/sdk/v3";
+import { logger, metadata, runs, schemaTask } from "@trigger.dev/sdk";
 import { streamText } from "ai";
 import { z } from "zod";
 
@@ -215,7 +215,7 @@ When using tools with the AI SDK, you can access tool calls and results using th
 
 ```ts
 import { openai } from "@ai-sdk/openai";
-import { logger, metadata, runs, schemaTask } from "@trigger.dev/sdk/v3";
+import { logger, metadata, runs, schemaTask } from "@trigger.dev/sdk";
 import { streamText, tool, type TextStreamPart } from "ai";
 import { z } from "zod";
 
@@ -283,7 +283,7 @@ You can define a Trigger.dev task that can be used as a tool, and will automatic
 
 ```ts
 import { openai } from "@ai-sdk/openai";
-import { logger, metadata, runs, schemaTask, toolTask } from "@trigger.dev/sdk/v3";
+import { logger, metadata, runs, schemaTask, toolTask } from "@trigger.dev/sdk";
 import { streamText, tool, type TextStreamPart } from "ai";
 import { z } from "zod";
 

docs/realtime/backend/subscribe.mdx

@@ -11,7 +11,7 @@ These functions allow you to subscribe to run updates from your backend code. Ea
 Subscribes to all changes to a specific run.
 
 ```ts Example
-import { runs } from "@trigger.dev/sdk/v3";
+import { runs } from "@trigger.dev/sdk";
 
 for await (const run of runs.subscribeToRun("run_1234")) {
   console.log(run);
@@ -29,7 +29,7 @@ This function subscribes to all changes to a run. It returns an async iterator t
 Subscribes to all changes to runs with a specific tag.
 
 ```ts Example
-import { runs } from "@trigger.dev/sdk/v3";
+import { runs } from "@trigger.dev/sdk";
 
 for await (const run of runs.subscribeToRunsWithTag("user:1234")) {
   console.log(run);
@@ -47,7 +47,7 @@ This function subscribes to all changes to runs with a specific tag. It returns
 Subscribes to all changes for runs in a batch.
 
 ```ts Example
-import { runs } from "@trigger.dev/sdk/v3";
+import { runs } from "@trigger.dev/sdk";
 
 for await (const run of runs.subscribeToBatch("batch_1234")) {
   console.log(run);
@@ -65,7 +65,7 @@ This function subscribes to all changes for runs in a batch. It returns an async
 You can infer the types of the run's payload and output by passing the type of the task to the subscribe functions:
 
 ```ts
-import { runs, tasks } from "@trigger.dev/sdk/v3";
+import { runs, tasks } from "@trigger.dev/sdk";
 import type { myTask } from "./trigger/my-task";
 
 async function myBackend() {
@@ -85,7 +85,7 @@ async function myBackend() {
 When using `subscribeToRunsWithTag`, you can pass a union of task types:
 
 ```ts
-import { runs } from "@trigger.dev/sdk/v3";
+import { runs } from "@trigger.dev/sdk";
 import type { myTask, myOtherTask } from "./trigger/my-task";
 
 for await (const run of runs.subscribeToRunsWithTag<typeof myTask | typeof myOtherTask>("my-tag")) {
@@ -130,7 +130,7 @@ This example task updates the progress of a task as it processes items.
 
 ```ts
 // Your task code
-import { task, metadata } from "@trigger.dev/sdk/v3";
+import { task, metadata } from "@trigger.dev/sdk";
 
 export const progressTask = task({
   id: "progress-task",
@@ -165,7 +165,7 @@ We can now subscribe to the runs and receive real-time metadata updates.
 
 ```ts
 // Somewhere in your backend code
-import { runs } from "@trigger.dev/sdk/v3";
+import { runs } from "@trigger.dev/sdk";
 import type { progressTask } from "./trigger/progress-task";
 
 async function monitorProgress(runId: string) {
@@ -199,7 +199,7 @@ For more information on how to write tasks that use the metadata API, as well as
 You can get type safety for your metadata by defining types:
 
 ```ts
-import { runs } from "@trigger.dev/sdk/v3";
+import { runs } from "@trigger.dev/sdk";
 import type { progressTask } from "./trigger/progress-task";
 
 interface ProgressMetadata {

docs/realtime/how-it-works.mdx

@@ -25,7 +25,7 @@ The run object returned by Realtime subscriptions is optimized for streaming upd
 After you trigger a task, you can subscribe to the run using the `runs.subscribeToRun` function. This function returns an async iterator that you can use to get updates on the run status.
 
 ```ts
-import { runs, tasks } from "@trigger.dev/sdk/v3";
+import { runs, tasks } from "@trigger.dev/sdk";
 
 // Somewhere in your backend code
 async function myBackend() {
@@ -43,7 +43,7 @@ Every time the run changes, the async iterator will yield the updated run. You c
 Alternatively, you can subscribe to changes to any run that includes a specific tag (or tags) using the `runs.subscribeToRunsWithTag` function.
 
 ```ts
-import { runs } from "@trigger.dev/sdk/v3";
+import { runs } from "@trigger.dev/sdk";
 
 // Somewhere in your backend code
 for await (const run of runs.subscribeToRunsWithTag("user:1234")) {
@@ -55,7 +55,7 @@ for await (const run of runs.subscribeToRunsWithTag("user:1234")) {
 If you've used `batchTrigger` to trigger multiple runs, you can also subscribe to changes to all the runs triggered in the batch using the `runs.subscribeToBatch` function.
 
 ```ts
-import { runs } from "@trigger.dev/sdk/v3";
+import { runs } from "@trigger.dev/sdk";
 
 // Somewhere in your backend code
 for await (const run of runs.subscribeToBatch("batch-id")) {

docs/realtime/run-object.mdx

@@ -126,7 +126,7 @@ Type-safety is supported for the run object, so you can infer the types of the r
 You can infer the types of the run's payload and output by passing the type of the task to the `subscribeToRun` function. This will give you type-safe access to the run's payload and output.
 
 ```ts
-import { runs, tasks } from "@trigger.dev/sdk/v3";
+import { runs, tasks } from "@trigger.dev/sdk";
 import type { myTask } from "./trigger/my-task";
 
 // Somewhere in your backend code
@@ -148,7 +148,7 @@ async function myBackend() {
 When using `subscribeToRunsWithTag`, you can pass a union of task types for all the possible tasks that can have the tag.
 
 ```ts
-import { runs } from "@trigger.dev/sdk/v3";
+import { runs } from "@trigger.dev/sdk";
 import type { myTask, myOtherTask } from "./trigger/my-task";
 
 // Somewhere in your backend code

docs/runs/metadata.mdx

@@ -523,7 +523,7 @@ Using metadata updates in conjunction with our [Realtime React hooks](/realtime/
 Track progress with percentage and current step:
 
 ```ts
-import { task, metadata } from "@trigger.dev/sdk/v3";
+import { task, metadata } from "@trigger.dev/sdk";
 
 export const batchProcessingTask = task({
   id: "batch-processing",
@@ -550,7 +550,7 @@ export const batchProcessingTask = task({
 Append log entries while maintaining status:
 
 ```ts
-import { task, metadata } from "@trigger.dev/sdk/v3";
+import { task, metadata } from "@trigger.dev/sdk";
 
 export const deploymentTask = task({
   id: "deployment",
@@ -584,7 +584,7 @@ export const deploymentTask = task({
 Store user information and notification preferences:
 
 ```ts
-import { task, metadata } from "@trigger.dev/sdk/v3";
+import { task, metadata } from "@trigger.dev/sdk";
 
 export const userTask = task({
   id: "user-task",

docs/snippets/node-versions.mdx

@@ -13,7 +13,7 @@ Trigger.dev runs your tasks on specific Node.js versions:
 You can change the runtime by setting the `runtime` field in your `trigger.config.ts` file.
 
 ```ts
-import { defineConfig } from "@trigger.dev/sdk/v3";
+import { defineConfig } from "@trigger.dev/sdk";
 
 export default defineConfig({
   // "node", "node-22" or "bun"

docs/triggering.mdx

@@ -961,6 +961,16 @@ If you don't specify a region it will use the default for your project. Go to th
 
 The region is where your runs are executed, it does not change where the run payload, output, tags, logs, or are any other data is stored.
 
+### `machine`
+
+You can override the default machine preset when you trigger a run:
+
+```ts
+await yourTask.trigger(payload, { machine: "large-1x" });
+```
+
+If you don't specify a machine it will use the machine preset for your task (or the default for your project). For more information read [the machines guide](/machines).
+
 ## Large Payloads
 
 We recommend keeping your task payloads as small as possible. We currently have a hard limit on task payloads above 10MB.