Merge remote-tracking branch 'origin/main' into fix/resolve-waitpoints

.changeset/pre.json

@@ -25,11 +25,14 @@
     "green-lions-relate",
     "hip-cups-wave",
     "honest-files-decide",
+    "itchy-games-sort",
     "late-chairs-ring",
     "moody-squids-count",
     "nice-colts-boil",
     "polite-impalas-care",
     "polite-lies-fix",
+    "real-rats-drop",
+    "red-chairs-begin",
     "red-wasps-cover",
     "shiny-kiwis-beam",
     "smart-coins-hammer",
@@ -42,6 +45,7 @@
     "tricky-houses-invite",
     "two-tigers-dream",
     "weak-jobs-hide",
-    "wet-deers-think"
+    "wet-deers-think",
+    "wet-steaks-reflect"
   ]
 }

.changeset/real-rats-drop.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+Add onCancel lifecycle hook

.changeset/wet-steaks-reflect.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+If you pass a directory when calling deploy we validate it exists and give helpful hints

.github/workflows/publish-worker-v4.yml

@@ -1,4 +1,4 @@
-name: "⚒️ Publish Worker RE2"
+name: "⚒️ Publish Worker (v4)"
 
 on:
   workflow_call:

.github/workflows/unit-tests.yml

@@ -8,6 +8,32 @@ jobs:
     name: "🧪 Unit Tests"
     runs-on: ubuntu-latest
     steps:
+      - name: 🔧 Disable IPv6
+        run: |
+          sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
+          sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1
+          sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=1
+
+      - name: 🔧 Configure docker address pool
+        run: |
+          CONFIG='{
+            "default-address-pools" : [
+              {
+                "base" : "172.17.0.0/12",
+                "size" : 20
+              },
+              {
+                "base" : "192.168.0.0/16",
+                "size" : 24
+              }
+            ]
+          }'
+          mkdir -p /etc/docker
+          echo "$CONFIG" | sudo tee /etc/docker/daemon.json
+
+      - name: 🔧 Restart docker daemon
+        run: sudo systemctl restart docker
+
       - name: ⬇️ Checkout repo
         uses: actions/checkout@v4
         with:

apps/webapp/app/assets/icons/TextInlineIcon.tsx

@@ -0,0 +1,41 @@
+export function TextInlineIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M3 3V21"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M13 21L13 16"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M13 8L13 3"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M7 12L20 12"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M17.5 15.5L21 12L17.5 8.5"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+    </svg>
+  );
+}

apps/webapp/app/assets/icons/TextWrapIcon.tsx

@@ -0,0 +1,34 @@
+export function TextWrapIcon({ className }: { className?: string }) {
+  return (
+    <svg className={className} viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+      <path
+        d="M3 3V21"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M21 21V3"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M7 7H14C15.6569 7 17 8.34315 17 10V13C17 14.6569 15.6569 16 14 16H9"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M11 13L8 16L11 19"
+        stroke="currentColor"
+        strokeWidth="2"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+    </svg>
+  );
+}

apps/webapp/app/components/code/CodeBlock.tsx

@@ -3,11 +3,13 @@ import { Clipboard, ClipboardCheck } from "lucide-react";
 import type { Language, PrismTheme } from "prism-react-renderer";
 import { Highlight, Prism } from "prism-react-renderer";
 import { forwardRef, ReactNode, useCallback, useEffect, useState } from "react";
+import { TextWrapIcon } from "~/assets/icons/TextWrapIcon";
 import { cn } from "~/utils/cn";
 import { Button } from "../primitives/Buttons";
 import { Dialog, DialogContent, DialogHeader, DialogTitle } from "../primitives/Dialog";
 import { Paragraph } from "../primitives/Paragraph";
 import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "../primitives/Tooltip";
+import { TextInlineIcon } from "~/assets/icons/TextInlineIcon";
 
 //This is a fork of https://github.com/mantinedev/mantine/blob/master/src/mantine-prism/src/Prism/Prism.tsx
 //it didn't support highlighting lines by dimming the rest of the code, or animations on the highlighting
@@ -31,6 +33,9 @@ type CodeBlockProps = {
   /** Show copy to clipboard button */
   showCopyButton?: boolean;
 
+  /** Show text wrapping button */
+  showTextWrapping?: boolean;
+
   /** Display line numbers */
   showLineNumbers?: boolean;
 
@@ -183,6 +188,7 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
   (
     {
       showCopyButton = true,
+      showTextWrapping = false,
       showLineNumbers = true,
       showOpenInModal = true,
       highlightedRanges,
@@ -202,6 +208,7 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
     const [copied, setCopied] = useState(false);
     const [modalCopied, setModalCopied] = useState(false);
     const [isModalOpen, setIsModalOpen] = useState(false);
+    const [isWrapped, setIsWrapped] = useState(false);
 
     const onCopied = useCallback(
       (event: React.MouseEvent<HTMLButtonElement>) => {
@@ -263,6 +270,25 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
               showChrome ? "right-1.5 top-1.5" : "top-2.5"
             )}
           >
+            {showTextWrapping && (
+              <TooltipProvider>
+                <Tooltip disableHoverableContent>
+                  <TooltipTrigger
+                    onClick={() => setIsWrapped(!isWrapped)}
+                    className="transition-colors focus-custom hover:cursor-pointer hover:text-text-bright"
+                  >
+                    {isWrapped ? (
+                      <TextInlineIcon className="size-4" />
+                    ) : (
+                      <TextWrapIcon className="size-4" />
+                    )}
+                  </TooltipTrigger>
+                  <TooltipContent side="left" className="text-xs">
+                    {isWrapped ? "Unwrap" : "Wrap"}
+                  </TooltipContent>
+                </Tooltip>
+              </TooltipProvider>
+            )}
             {showCopyButton && (
               <TooltipProvider>
                 <Tooltip open={copied || mouseOver} disableHoverableContent>
@@ -311,16 +337,27 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
               maxLineWidth={maxLineWidth}
               className="px-2 py-3"
               preClassName="text-xs"
+              isWrapped={isWrapped}
             />
           ) : (
             <div
               dir="ltr"
-              className="overflow-auto px-2 py-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600"
+              className={cn(
+                "px-2 py-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600",
+                !isWrapped && "overflow-x-auto",
+                isWrapped && "overflow-y-auto"
+              )}
               style={{
                 maxHeight,
               }}
             >
-              <pre className="relative mr-2 p-2 font-mono text-xs leading-relaxed" dir="ltr">
+              <pre
+                className={cn(
+                  "relative mr-2 p-2 font-mono text-xs leading-relaxed",
+                  isWrapped && "[&_span]:whitespace-pre-wrap [&_span]:break-words"
+                )}
+                dir="ltr"
+              >
                 {code}
               </pre>
             </div>
@@ -355,6 +392,7 @@ export const CodeBlock = forwardRef<HTMLDivElement, CodeBlockProps>(
                 maxLineWidth={maxLineWidth}
                 className="min-h-full"
                 preClassName="text-sm"
+                isWrapped={isWrapped}
               />
             ) : (
               <div
@@ -410,6 +448,7 @@ type HighlightCodeProps = {
   maxLineWidth?: number;
   className?: string;
   preClassName?: string;
+  isWrapped: boolean;
 };
 
 function HighlightCode({
@@ -421,11 +460,11 @@ function HighlightCode({
   maxLineWidth,
   className,
   preClassName,
+  isWrapped,
 }: HighlightCodeProps) {
   const [isLoaded, setIsLoaded] = useState(false);
 
   useEffect(() => {
-    // This ensures the language definitions are loaded
     Promise.all([
       //@ts-ignore
       import("prismjs/components/prism-json"),
@@ -434,16 +473,23 @@ function HighlightCode({
     ]).then(() => setIsLoaded(true));
   }, []);
 
+  const containerClasses = cn(
+    "px-3 py-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600",
+    !isWrapped && "overflow-x-auto",
+    isWrapped && "overflow-y-auto",
+    className
+  );
+
+  const preClasses = cn(
+    "relative mr-2 font-mono leading-relaxed",
+    preClassName,
+    isWrapped && "[&_span]:whitespace-pre-wrap [&_span]:break-words"
+  );
+
   if (!isLoaded) {
     return (
-      <div
-        dir="ltr"
-        className={cn(
-          "overflow-auto px-3 py-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600",
-          className
-        )}
-      >
-        <pre className={cn("relative mr-2 font-mono leading-relaxed", preClassName)}>{code}</pre>
+      <div dir="ltr" className={containerClasses}>
+        <pre className={preClasses}>{code}</pre>
       </div>
     );
   }
@@ -457,22 +503,8 @@ function HighlightCode({
         getLineProps,
         getTokenProps,
       }) => (
-        <div
-          dir="ltr"
-          className={cn(
-            "overflow-auto px-3 py-3 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600",
-            className
-          )}
-        >
-          <pre
-            className={cn(
-              "relative mr-2 font-mono leading-relaxed",
-              inheritedClassName,
-              preClassName
-            )}
-            style={inheritedStyle}
-            dir="ltr"
-          >
+        <div dir="ltr" className={containerClasses}>
+          <pre className={cn(preClasses, inheritedClassName)} style={inheritedStyle} dir="ltr">
             {tokens
               .map((line, index) => {
                 if (index === tokens.length - 1 && line.length === 1 && line[0].content === "\n") {
@@ -495,7 +527,8 @@ function HighlightCode({
                     {...lineProps}
                     className={cn(
                       "flex w-full justify-start transition-opacity duration-500",
-                      lineProps.className
+                      lineProps.className,
+                      isWrapped && "flex-wrap"
                     )}
                     style={{
                       opacity: shouldDim ? dimAmount : undefined,
@@ -504,9 +537,10 @@ function HighlightCode({
                   >
                     {showLineNumbers && (
                       <div
-                        className={
-                          "mr-2 flex-none select-none text-right text-charcoal-500 transition-opacity duration-500"
-                        }
+                        className={cn(
+                          "mr-2 flex-none select-none text-right text-charcoal-500 transition-opacity duration-500",
+                          isWrapped && "sticky left-0"
+                        )}
                         style={{
                           width: `calc(8 * ${(maxLineWidth as number) / 16}rem)`,
                         }}

apps/webapp/app/components/metrics/BigNumber.tsx

@@ -1,7 +1,10 @@
 import { type ReactNode } from "react";
-import { AnimatedNumber } from "../primitives/AnimatedNumber";
-import { Spinner } from "../primitives/Spinner";
 import { cn } from "~/utils/cn";
+import { formatNumber, formatNumberCompact } from "~/utils/numberFormatter";
+import { Header3 } from "../primitives/Headers";
+import { Spinner } from "../primitives/Spinner";
+import { SimpleTooltip } from "../primitives/Tooltip";
+import { AnimatedNumber } from "../primitives/AnimatedNumber";
 
 interface BigNumberProps {
   title: ReactNode;
@@ -13,6 +16,7 @@ interface BigNumberProps {
   accessory?: ReactNode;
   suffix?: string;
   suffixClassName?: string;
+  compactThreshold?: number;
 }
 
 export function BigNumber({
@@ -25,25 +29,39 @@ export function BigNumber({
   accessory,
   animate = false,
   loading = false,
+  compactThreshold,
 }: BigNumberProps) {
   const v = value ?? defaultValue;
+
+  const shouldCompact =
+    typeof compactThreshold === "number" && v !== undefined && v >= compactThreshold;
+
   return (
-    <div className="grid grid-rows-[1.5rem_auto] gap-4 rounded-sm border border-grid-dimmed bg-background-bright p-4">
-      <div className="flex items-center justify-between">
-        <div className="text-2sm text-text-dimmed">{title}</div>
+    <div className="flex flex-col justify-between gap-4 rounded-sm border border-grid-dimmed bg-background-bright p-4">
+      <div className="flex flex-wrap items-center justify-between gap-2">
+        <Header3 className="leading-6">{title}</Header3>
         {accessory && <div className="flex-shrink-0">{accessory}</div>}
       </div>
       <div
         className={cn(
-          "h-[3.75rem] text-[3.75rem] font-normal tabular-nums leading-none text-text-bright",
+          "text-[3.75rem] font-normal tabular-nums leading-none text-text-bright",
           valueClassName
         )}
       >
         {loading ? (
           <Spinner className="size-6" />
         ) : v !== undefined ? (
-          <div className="flex items-baseline gap-1">
-            {animate ? <AnimatedNumber value={v} /> : v}
+          <div className="flex flex-wrap items-baseline gap-2">
+            {shouldCompact ? (
+              <SimpleTooltip
+                button={animate ? <AnimatedNumber value={v} /> : formatNumberCompact(v)}
+                content={formatNumber(v)}
+              />
+            ) : animate ? (
+              <AnimatedNumber value={v} />
+            ) : (
+              formatNumber(v)
+            )}
             {suffix && <div className={cn("text-xs", suffixClassName)}>{suffix}</div>}
           </div>
         ) : (

apps/webapp/app/components/primitives/AnimatedNumber.tsx

@@ -1,4 +1,4 @@
-import { motion, useSpring, useTransform, useMotionValue, animate } from "framer-motion";
+import { animate, motion, useMotionValue, useTransform } from "framer-motion";
 import { useEffect } from "react";
 
 export function AnimatedNumber({ value }: { value: number }) {

apps/webapp/app/components/primitives/DateTime.tsx

@@ -98,7 +98,45 @@ export function formatDateTime(
 }
 
 export function formatDateTimeISO(date: Date, timeZone: string): string {
-  return new Date(date.toLocaleString("en-US", { timeZone })).toISOString();
+  // Special handling for UTC
+  if (timeZone === "UTC") {
+    return date.toISOString();
+  }
+
+  // Get the date parts in the target timezone
+  const dateFormatter = new Intl.DateTimeFormat("en-US", {
+    timeZone,
+    year: "numeric",
+    month: "2-digit",
+    day: "2-digit",
+    hour: "2-digit",
+    minute: "2-digit",
+    second: "2-digit",
+    hour12: false,
+  });
+
+  // Get the timezone offset for this specific date
+  const timeZoneFormatter = new Intl.DateTimeFormat("en-US", {
+    timeZone,
+    timeZoneName: "longOffset",
+  });
+
+  const dateParts = Object.fromEntries(
+    dateFormatter.formatToParts(date).map(({ type, value }) => [type, value])
+  );
+
+  const timeZoneParts = timeZoneFormatter.formatToParts(date);
+  const offset =
+    timeZoneParts.find((part) => part.type === "timeZoneName")?.value.replace("GMT", "") ||
+    "+00:00";
+
+  // Format: YYYY-MM-DDThh:mm:ss.sss±hh:mm
+  return (
+    `${dateParts.year}-${dateParts.month}-${dateParts.day}T` +
+    `${dateParts.hour}:${dateParts.minute}:${dateParts.second}.${String(
+      date.getMilliseconds()
+    ).padStart(3, "0")}${offset}`
+  );
 }
 
 // New component that only shows date when it changes

apps/webapp/app/components/primitives/LoadingBarDivider.tsx

@@ -7,7 +7,7 @@ type LoadingBarDividerProps = {
 
 export function LoadingBarDivider({ isLoading }: LoadingBarDividerProps) {
   return (
-    <div className="relative h-px w-full bg-grid-bright">
+    <div className="relative h-px w-full overflow-hidden bg-grid-bright">
       <AnimationDivider isLoading={isLoading} />
     </div>
   );
@@ -21,11 +21,9 @@ export function AnimationDivider({ isLoading }: LoadingBarDividerProps) {
     if (isPresent) {
       const enterAnimation = async () => {
         await animate(
-          [
-            [scope.current, { width: "30%", left: "35%" }, { duration: 1, ease: "easeIn" }],
-            [scope.current, { width: 0, left: "100%" }, { duration: 1, ease: "easeOut" }],
-          ],
-          { repeat: Infinity, repeatType: "reverse" }
+          scope.current,
+          { left: ["-100%", "100%"], width: "100%" },
+          { duration: 2, ease: "easeOut", repeat: Infinity }
         );
       };
       enterAnimation();

apps/webapp/app/components/runs/v3/PacketDisplay.tsx

@@ -44,6 +44,7 @@ export function PacketDisplay({
           code={data}
           maxLines={20}
           showLineNumbers={false}
+          showTextWrapping
         />
       );
     }

apps/webapp/app/components/runs/v3/RunIcon.tsx

@@ -97,6 +97,7 @@ export function RunIcon({ name, className, spanName }: TaskIconProps) {
     case "task-hook-onResume":
     case "task-hook-onComplete":
     case "task-hook-cleanup":
+    case "task-hook-onCancel":
       return <FunctionIcon className={cn(className, "text-text-dimmed")} />;
     case "task-hook-onFailure":
     case "task-hook-catchError":

apps/webapp/app/components/runs/v3/TaskTriggerSource.tsx

@@ -12,10 +12,12 @@ export function TaskTriggerSourceIcon({
 }) {
   switch (source) {
     case "STANDARD": {
-      return <TaskIconSmall className="size-[1.125rem] text-tasks" />;
+      return <TaskIconSmall className="size-[1.125rem] min-w-[1.125rem] text-tasks" />;
     }
     case "SCHEDULED": {
-      return <ClockIcon className={cn("size-[1.125rem] text-schedules", className)} />;
+      return (
+        <ClockIcon className={cn("size-[1.125rem] min-w-[1.125rem] text-schedules", className)} />
+      );
     }
   }
 }

apps/webapp/app/env.server.ts

@@ -460,6 +460,7 @@ const EnvironmentSchema = z.object({
   RUN_ENGINE_REUSE_SNAPSHOT_COUNT: z.coerce.number().int().default(0),
   RUN_ENGINE_MAXIMUM_ENV_COUNT: z.coerce.number().int().optional(),
   RUN_ENGINE_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000),
+  RUN_ENGINE_RETRY_WARM_START_THRESHOLD_MS: z.coerce.number().int().default(30_000),
 
   RUN_ENGINE_WORKER_REDIS_HOST: z
     .string()
@@ -717,7 +718,7 @@ const EnvironmentSchema = z.object({
 
   SLACK_BOT_TOKEN: z.string().optional(),
   SLACK_SIGNUP_REASON_CHANNEL_ID: z.string().optional(),
-  
+
   // kapa.ai
   KAPA_AI_WEBSITE_ID: z.string().optional(),
 });

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx

@@ -69,6 +69,7 @@ import { docsPath, EnvironmentParamSchema, v3BillingPath } from "~/utils/pathBui
 import { PauseEnvironmentService } from "~/v3/services/pauseEnvironment.server";
 import { PauseQueueService } from "~/v3/services/pauseQueue.server";
 import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route";
+import { Header3 } from "~/components/primitives/Headers";
 import { Input } from "~/components/primitives/Input";
 import { useThrottle } from "~/hooks/useThrottle";
 
@@ -257,13 +258,30 @@ export default function Page() {
               suffix={env.paused && environment.queued > 0 ? "paused" : undefined}
               animate
               accessory={<EnvironmentPauseResumeButton env={env} />}
-              valueClassName={env.paused ? "text-amber-500" : undefined}
+              valueClassName={env.paused ? "text-warning" : undefined}
+              compactThreshold={1000000}
+            />
+            <BigNumber
+              title="Running"
+              value={environment.running}
+              animate
+              valueClassName={
+                environment.running === environment.concurrencyLimit ? "text-warning" : undefined
+              }
+              suffix={
+                environment.running === environment.concurrencyLimit
+                  ? "At concurrency limit"
+                  : undefined
+              }
+              compactThreshold={1000000}
             />
-            <BigNumber title="Running" value={environment.running} animate />
             <BigNumber
               title="Concurrency limit"
               value={environment.concurrencyLimit}
               animate
+              valueClassName={
+                environment.running === environment.concurrencyLimit ? "text-warning" : undefined
+              }
               accessory={
                 plan ? (
                   plan?.v3Subscription?.plan?.limits.concurrentRuns.canExceed ? (
@@ -307,7 +325,37 @@ export default function Page() {
                   <TableRow>
                     <TableHeaderCell>Name</TableHeaderCell>
                     <TableHeaderCell alignment="right">Queued</TableHeaderCell>
-                    <TableHeaderCell alignment="right">Running</TableHeaderCell>
+                    <TableHeaderCell alignment="right">Running/limit</TableHeaderCell>
+                    <TableHeaderCell
+                      alignment="right"
+                      tooltip={
+                        <div className="max-w-xs space-y-2 p-1 text-left">
+                          <div className="space-y-0.5">
+                            <Header3>Environment</Header3>
+                            <Paragraph
+                              variant="small"
+                              className="!text-wrap text-text-dimmed"
+                              spacing
+                            >
+                              This queue is limited by your environment's concurrency limit of{" "}
+                              {environment.concurrencyLimit}.
+                            </Paragraph>
+                          </div>
+                          <div className="space-y-0.5">
+                            <Header3>User</Header3>
+                            <Paragraph
+                              variant="small"
+                              className="!text-wrap text-text-dimmed"
+                              spacing
+                            >
+                              This queue is limited by a concurrency limit set in your code.
+                            </Paragraph>
+                          </div>
+                        </div>
+                      }
+                    >
+                      Limited by
+                    </TableHeaderCell>
                     <TableHeaderCell
                       alignment="right"
                       tooltip={
@@ -334,7 +382,6 @@ export default function Page() {
                     >
                       Release on waitpoint
                     </TableHeaderCell>
-                    <TableHeaderCell alignment="right">Concurrency limit</TableHeaderCell>
                     <TableHeaderCell className="w-[1%] pl-24">
                       <span className="sr-only">Pause/resume</span>
                     </TableHeaderCell>
@@ -342,80 +389,103 @@ export default function Page() {
                 </TableHeader>
                 <TableBody>
                   {queues.length > 0 ? (
-                    queues.map((queue) => (
-                      <TableRow key={queue.name}>
-                        <TableCell>
-                          <span className="flex items-center gap-2">
-                            {queue.type === "task" ? (
-                              <SimpleTooltip
-                                button={
-                                  <TaskIconSmall
-                                    className={cn(
-                                      "size-[1.125rem] text-blue-500",
-                                      queue.paused && "opacity-50"
-                                    )}
-                                  />
-                                }
-                                content={`This queue was automatically created from your "${queue.name}" task`}
-                              />
-                            ) : (
-                              <SimpleTooltip
-                                button={
-                                  <RectangleStackIcon
-                                    className={cn(
-                                      "size-[1.125rem] text-purple-500",
-                                      queue.paused && "opacity-50"
-                                    )}
-                                  />
-                                }
-                                content={`This is a custom queue you added in your code.`}
-                              />
+                    queues.map((queue) => {
+                      const limit = queue.concurrencyLimit ?? environment.concurrencyLimit;
+                      const isAtLimit = queue.running === limit;
+                      return (
+                        <TableRow key={queue.name}>
+                          <TableCell>
+                            <span className="flex items-center gap-2">
+                              {queue.type === "task" ? (
+                                <SimpleTooltip
+                                  button={
+                                    <TaskIconSmall
+                                      className={cn(
+                                        "size-[1.125rem] text-blue-500",
+                                        queue.paused && "opacity-50"
+                                      )}
+                                    />
+                                  }
+                                  content={`This queue was automatically created from your "${queue.name}" task`}
+                                />
+                              ) : (
+                                <SimpleTooltip
+                                  button={
+                                    <RectangleStackIcon
+                                      className={cn(
+                                        "size-[1.125rem] text-purple-500",
+                                        queue.paused && "opacity-50"
+                                      )}
+                                    />
+                                  }
+                                  content={`This is a custom queue you added in your code.`}
+                                />
+                              )}
+                              <span className={queue.paused ? "opacity-50" : undefined}>
+                                {queue.name}
+                              </span>
+                              {queue.paused ? (
+                                <Badge variant="extra-small" className="text-warning">
+                                  Paused
+                                </Badge>
+                              ) : null}
+                              {isAtLimit ? (
+                                <Badge variant="extra-small" className="text-warning">
+                                  At concurrency limit
+                                </Badge>
+                              ) : null}
+                            </span>
+                          </TableCell>
+                          <TableCell
+                            alignment="right"
+                            className={queue.paused ? "opacity-50" : undefined}
+                          >
+                            {queue.queued}
+                          </TableCell>
+                          <TableCell
+                            alignment="right"
+                            className={cn(
+                              queue.paused ? "tabular-nums opacity-50" : undefined,
+                              isAtLimit && "text-warning"
                             )}
-                            <span className={queue.paused ? "opacity-50" : undefined}>
-                              {queue.name}
+                          >
+                            {queue.running}/
+                            <span
+                              className={cn(
+                                "tabular-nums text-text-dimmed",
+                                isAtLimit && "text-warning"
+                              )}
+                            >
+                              {limit}
                             </span>
-                            {queue.paused ? (
-                              <Badge variant="extra-small" className="text-warning">
-                                Paused
-                              </Badge>
-                            ) : null}
-                          </span>
-                        </TableCell>
-                        <TableCell
-                          alignment="right"
-                          className={queue.paused ? "opacity-50" : undefined}
-                        >
-                          {queue.queued}
-                        </TableCell>
-                        <TableCell
-                          alignment="right"
-                          className={queue.paused ? "opacity-50" : undefined}
-                        >
-                          {queue.running}
-                        </TableCell>
-                        <TableCell
-                          alignment="right"
-                          className={queue.paused ? "opacity-50" : undefined}
-                        >
-                          {queue.releaseConcurrencyOnWaitpoint ? "Yes" : "No"}
-                        </TableCell>
-                        <TableCell
-                          alignment="right"
-                          className={queue.paused ? "opacity-50" : undefined}
-                        >
-                          {queue.concurrencyLimit ?? (
-                            <span className="text-text-dimmed">
-                              Max ({environment.concurrencyLimit})
-                            </span>
-                          )}
-                        </TableCell>
-                        <TableCellMenu
-                          isSticky
-                          visibleButtons={queue.paused && <QueuePauseResumeButton queue={queue} />}
-                          hiddenButtons={!queue.paused && <QueuePauseResumeButton queue={queue} />}
-                        />
-                      </TableRow>
-                    ))
+                          </TableCell>
+                          <TableCell
+                            alignment="right"
+                            className={cn(
+                              queue.paused ? "opacity-50" : undefined,
+                              isAtLimit && "text-warning"
+                            )}
+                          >
+                            {queue.concurrencyLimit ? "User" : "Environment"}
+                          </TableCell>
+                          <TableCell
+                            alignment="right"
+                            className={queue.paused ? "opacity-50" : undefined}
+                          >
+                            {queue.releaseConcurrencyOnWaitpoint ? "Yes" : "No"}
+                          </TableCell>
+                          <TableCellMenu
+                            isSticky
+                            visibleButtons={
+                              queue.paused && <QueuePauseResumeButton queue={queue} />
+                            }
+                            hiddenButtons={
+                              !queue.paused && <QueuePauseResumeButton queue={queue} />
+                            }
+                          />
+                        </TableRow>
+                      );
+                    })
                   ) : (
                     <TableRow>
                       <TableCell colSpan={6}>
@@ -503,7 +573,7 @@ function EnvironmentPauseResumeButton({
                     type="button"
                     variant="secondary/small"
                     LeadingIcon={env.paused ? PlayIcon : PauseIcon}
-                    leadingIconClassName={env.paused ? "text-success" : "text-amber-500"}
+                    leadingIconClassName={env.paused ? "text-success" : "text-warning"}
                   >
                     {env.paused ? "Resume..." : "Pause environment..."}
                   </Button>
@@ -512,8 +582,8 @@ function EnvironmentPauseResumeButton({
             </TooltipTrigger>
             <TooltipContent className={"text-xs"}>
               {env.paused
-                ? `Resume processing runs in ${environmentFullTitle(env)}.`
-                : `Pause processing runs in ${environmentFullTitle(env)}.`}
+                ? `Resume processing runs in ${environmentFullTitle(env)}`
+                : `Pause processing runs in ${environmentFullTitle(env)}`}
             </TooltipContent>
           </Tooltip>
         </TooltipProvider>
@@ -540,7 +610,9 @@ function EnvironmentPauseResumeButton({
                   type="submit"
                   disabled={isLoading}
                   variant={env.paused ? "primary/medium" : "danger/medium"}
-                  LeadingIcon={isLoading ? <Spinner /> : env.paused ? PlayIcon : PauseIcon}
+                  LeadingIcon={
+                    isLoading ? <Spinner color="white" /> : env.paused ? PlayIcon : PauseIcon
+                  }
                   shortcut={{ modifiers: ["mod"], key: "enter" }}
                 >
                   {env.paused ? "Resume environment" : "Pause environment"}
@@ -582,7 +654,7 @@ function QueuePauseResumeButton({
                     type="button"
                     variant="tertiary/small"
                     LeadingIcon={queue.paused ? PlayIcon : PauseIcon}
-                    leadingIconClassName={queue.paused ? "text-success" : "text-amber-500"}
+                    leadingIconClassName={queue.paused ? "text-success" : "text-warning"}
                   >
                     {queue.paused ? "Resume..." : "Pause..."}
                   </Button>
@@ -703,7 +775,7 @@ export function QueueFilters() {
   const search = searchParams.get("query") ?? "";
 
   return (
-    <div className="flex w-full px-3 pb-3">
+    <div className="flex w-full border-t border-grid-dimmed px-1.5 py-1.5">
       <Input
         name="search"
         placeholder="Search queue name"

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx

@@ -647,7 +647,12 @@ function RunBody({
                 <Property.Item>
                   <Property.Label>Max duration</Property.Label>
                   <Property.Value>
-                    {run.maxDurationInSeconds ? `${run.maxDurationInSeconds}s` : "–"}
+                    {run.maxDurationInSeconds
+                      ? `${run.maxDurationInSeconds}s (${formatDurationMilliseconds(
+                          run.maxDurationInSeconds * 1000,
+                          { style: "short" }
+                        )})`
+                      : "–"}
                   </Property.Value>
                 </Property.Item>
                 <Property.Item>

apps/webapp/app/routes/resources.orgs.$organizationSlug.select-plan.tsx

@@ -678,7 +678,7 @@ export function TierPro({
                   <Button
                     variant="primary/medium"
                     disabled={isLoading}
-                    LeadingIcon={isLoading ? () => <Spinner color="dark" /> : undefined}
+                    LeadingIcon={isLoading ? () => <Spinner color="white" /> : undefined}
                     form="subscribe-pro"
                   >
                     {`Upgrade to ${plan.title}`}

apps/webapp/app/v3/runEngine.server.ts

@@ -95,6 +95,7 @@ function createRunEngine() {
         ...(env.RUN_ENGINE_RUN_QUEUE_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
       },
     },
+    retryWarmStartThresholdMs: env.RUN_ENGINE_RETRY_WARM_START_THRESHOLD_MS,
   });
 
   return engine;

apps/webapp/app/v3/services/cancelTaskRun.server.ts

@@ -47,29 +47,6 @@ export class CancelTaskRunService extends BaseService {
       tx: this._prisma,
     });
 
-    const inProgressEvents = await eventRepository.queryIncompleteEvents(
-      getTaskEventStoreTableForRun(taskRun),
-      {
-        runId: taskRun.friendlyId,
-      },
-      taskRun.createdAt,
-      taskRun.completedAt ?? undefined
-    );
-
-    logger.debug("Cancelling in-progress events", {
-      inProgressEvents: inProgressEvents.map((event) => event.id),
-    });
-
-    await Promise.all(
-      inProgressEvents.map((event) => {
-        return eventRepository.cancelEvent(
-          event,
-          options?.cancelledAt ?? new Date(),
-          options?.reason ?? "Run cancelled"
-        );
-      })
-    );
-
     return {
       id: result.run.id,
     };

apps/webapp/test/components/DateTime.test.ts

@@ -0,0 +1,54 @@
+import { describe, it, expect } from "vitest";
+import { formatDateTimeISO } from "~/components/primitives/DateTime";
+
+describe("formatDateTimeISO", () => {
+  it("should format UTC dates with Z suffix", () => {
+    const date = new Date("2025-04-29T14:01:19.000Z");
+    const result = formatDateTimeISO(date, "UTC");
+    expect(result).toBe("2025-04-29T14:01:19.000Z");
+  });
+
+  describe("British Time (Europe/London)", () => {
+    it("should format with +01:00 during BST (summer)", () => {
+      // BST - British Summer Time (last Sunday in March to last Sunday in October)
+      const summerDate = new Date("2025-07-15T14:01:19.000Z");
+      const result = formatDateTimeISO(summerDate, "Europe/London");
+      expect(result).toBe("2025-07-15T15:01:19.000+01:00");
+    });
+
+    it("should format with +00:00 during GMT (winter)", () => {
+      // GMT - Greenwich Mean Time (winter)
+      const winterDate = new Date("2025-01-15T14:01:19.000Z");
+      const result = formatDateTimeISO(winterDate, "Europe/London");
+      expect(result).toBe("2025-01-15T14:01:19.000+00:00");
+    });
+  });
+
+  describe("US Pacific Time (America/Los_Angeles)", () => {
+    it("should format with -07:00 during PDT (summer)", () => {
+      // PDT - Pacific Daylight Time (second Sunday in March to first Sunday in November)
+      const summerDate = new Date("2025-07-15T14:01:19.000Z");
+      const result = formatDateTimeISO(summerDate, "America/Los_Angeles");
+      expect(result).toBe("2025-07-15T07:01:19.000-07:00");
+    });
+
+    it("should format with -08:00 during PST (winter)", () => {
+      // PST - Pacific Standard Time (winter)
+      const winterDate = new Date("2025-01-15T14:01:19.000Z");
+      const result = formatDateTimeISO(winterDate, "America/Los_Angeles");
+      expect(result).toBe("2025-01-15T06:01:19.000-08:00");
+    });
+  });
+
+  it("should preserve milliseconds", () => {
+    const date = new Date("2025-04-29T14:01:19.123Z");
+    const result = formatDateTimeISO(date, "UTC");
+    expect(result).toBe("2025-04-29T14:01:19.123Z");
+  });
+
+  it("should preserve milliseconds, not UTC", () => {
+    const date = new Date("2025-04-29T14:01:19.123Z");
+    const result = formatDateTimeISO(date, "Europe/London");
+    expect(result).toBe("2025-04-29T15:01:19.123+01:00");
+  });
+});

apps/webapp/tsconfig.check.json

@@ -1,6 +1,8 @@
 {
   "extends": "./tsconfig.json",
   "compilerOptions": {
+    "lib": ["DOM", "DOM.Iterable", "DOM.AsyncIterable", "ES2022"],
+    "target": "ES2022",
     "noEmit": true,
     "paths": {
       "~/*": ["./app/*"],

docs/upgrade-to-v4.mdx

@@ -170,6 +170,107 @@ tasks.onComplete(({ ctx, result }) => {
 });
 ```
 
+### onCancel
+
+<Note>Available in v4.0.0-beta.12 and later.</Note>
+
+You can now define an `onCancel` hook that is called when a run is cancelled. This is useful if you want to clean up any resources that were allocated for the run.
+
+```ts
+tasks.onCancel(({ ctx, signal }) => {
+  console.log("Run cancelled", signal);
+});
+```
+
+You can use the `onCancel` hook along with the `signal` passed into the run function to interrupt a call to an external service, for example using the [streamText](https://ai-sdk.dev/docs/reference/ai-sdk-core/stream-text) function from the AI SDK:
+
+```ts
+import { logger, tasks, schemaTask } from "@trigger.dev/sdk";
+import { streamText } from "ai";
+import { z } from "zod";
+
+export const interruptibleChat = schemaTask({
+  id: "interruptible-chat",
+  description: "Chat with the AI",
+  schema: z.object({
+    prompt: z.string().describe("The prompt to chat with the AI"),
+  }),
+  run: async ({ prompt }, { signal }) => {
+    const chunks: TextStreamPart<{}>[] = [];
+
+    // 👇 This is a global onCancel hook, but it's inside of the run function
+    tasks.onCancel(async () => {
+      // We have access to the chunks here, and can save them to the database
+      await saveChunksToDatabase(chunks);
+    });
+
+    try {
+      const result = streamText({
+        model: getModel(),
+        prompt,
+        experimental_telemetry: {
+          isEnabled: true,
+        },
+        tools: {},
+        abortSignal: signal, // 👈 Pass the signal to the streamText function, which aborts with the run is cancelled
+        onChunk: ({ chunk }) => {
+          chunks.push(chunk);
+        },
+      });
+
+      const textParts = [];
+
+      for await (const part of result.textStream) {
+        textParts.push(part);
+      }
+
+      return textParts.join("");
+    } catch (error) {
+      if (error instanceof Error && error.name === "AbortError") {
+        // streamText will throw an AbortError if the signal is aborted, so we can handle it here
+      } else {
+        throw error;
+      }
+    }
+  },
+});
+```
+
+The `onCancel` hook can optionally wait for the `run` function to finish, and access the output of the run:
+
+```ts
+import { logger, task } from "@trigger.dev/sdk";
+import { setTimeout } from "node:timers/promises";
+
+export const cancelExampleTask = task({
+  id: "cancel-example",
+  // Signal will be aborted when the task is cancelled 👇
+  run: async (payload: { message: string }, { signal }) => {
+    try {
+      // We pass the signal to setTimeout to abort the timeout if the task is cancelled
+      await setTimeout(10_000, undefined, { signal });
+    } catch (error) {
+      // Ignore the abort error
+    }
+
+    // Do some more work here
+
+    return {
+      message: "Hello, world!",
+    };
+  },
+  onCancel: async ({ runPromise }) => {
+    // You can await the runPromise to get the output of the task
+    const output = await runPromise;
+  },
+});
+```
+
+<Note>
+  You will have up to 30 seconds to complete the `runPromise` in the `onCancel` hook. After that
+  point the process will be killed.
+</Note>
+
 ### Improved middleware and locals
 
 Our task middleware system is now much more useful. Previously it only ran "around" the `run` function, but now we've hoisted it to the top level and it now runs before/after all the other hooks.
@@ -704,7 +805,7 @@ export const myTask = task({
   id: "my-task",
   onStart: ({ payload, ctx }) => {},
   // The run function still uses separate parameters
-  run: async ( payload, { ctx }) => {},
+  run: async (payload, { ctx }) => {},
 });
 ```
 
@@ -760,4 +861,4 @@ const batchHandle = await tasks.batchTrigger([
 // Now you need to call runs.list()
 const runs = await batchHandle.runs.list();
 console.log(runs);
-```
+```

internal-packages/run-engine/src/engine/index.ts

@@ -304,6 +304,7 @@ export class RunEngine {
       waitpointSystem: this.waitpointSystem,
       delayedRunSystem: this.delayedRunSystem,
       machines: this.options.machines,
+      retryWarmStartThresholdMs: this.options.retryWarmStartThresholdMs,
     });
 
     this.dequeueSystem = new DequeueSystem({

internal-packages/run-engine/src/engine/tests/attemptFailures.test.ts

@@ -155,7 +155,7 @@ describe("RunEngine attempt failures", () => {
       expect(executionData4.run.attemptNumber).toBe(2);
       expect(executionData4.run.status).toBe("COMPLETED_SUCCESSFULLY");
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 
@@ -266,7 +266,7 @@ describe("RunEngine attempt failures", () => {
       expect(executionData3.run.attemptNumber).toBe(1);
       expect(executionData3.run.status).toBe("COMPLETED_WITH_ERRORS");
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 
@@ -375,7 +375,7 @@ describe("RunEngine attempt failures", () => {
       expect(executionData3.run.attemptNumber).toBe(1);
       expect(executionData3.run.status).toBe("CRASHED");
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 
@@ -482,7 +482,7 @@ describe("RunEngine attempt failures", () => {
       expect(executionData.run.attemptNumber).toBe(1);
       expect(executionData.run.status).toBe("CRASHED");
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 
@@ -639,7 +639,7 @@ describe("RunEngine attempt failures", () => {
       expect(executionData4.run.attemptNumber).toBe(2);
       expect(executionData4.run.status).toBe("COMPLETED_SUCCESSFULLY");
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 
@@ -803,7 +803,7 @@ describe("RunEngine attempt failures", () => {
       expect(finalExecutionData.run.attemptNumber).toBe(2);
       expect(finalExecutionData.run.status).toBe("CRASHED");
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 });

internal-packages/run-engine/src/engine/tests/batchTrigger.test.ts

@@ -177,7 +177,7 @@ describe("RunEngine batchTrigger", () => {
       });
       expect(batchAfter2?.status).toBe("COMPLETED");
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 });

internal-packages/run-engine/src/engine/tests/batchTriggerAndWait.test.ts

@@ -352,7 +352,7 @@ describe("RunEngine batchTriggerAndWait", () => {
       });
       expect(batchAfter?.status === "COMPLETED");
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 
@@ -570,7 +570,7 @@ describe("RunEngine batchTriggerAndWait", () => {
         );
         expect(parentAfterTriggerAndWait.batch).toBeUndefined();
       } finally {
-        engine.quit();
+        await engine.quit();
       }
     }
   );

internal-packages/run-engine/src/engine/tests/cancelling.test.ts

@@ -220,7 +220,7 @@ describe("RunEngine cancelling", () => {
         );
         expect(envConcurrencyCompleted).toBe(0);
       } finally {
-        engine.quit();
+        await engine.quit();
       }
     }
   );
@@ -321,7 +321,7 @@ describe("RunEngine cancelling", () => {
       );
       expect(envConcurrencyCompleted).toBe(0);
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 

internal-packages/run-engine/src/engine/tests/checkpoints.test.ts

@@ -1375,7 +1375,7 @@ describe("RunEngine checkpoints", () => {
       });
       expect(batchAfter?.status === "COMPLETED");
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 });

internal-packages/run-engine/src/engine/tests/delays.test.ts

@@ -86,7 +86,7 @@ describe("RunEngine delays", () => {
       assertNonNullable(executionData2);
       expect(executionData2.snapshot.executionStatus).toBe("QUEUED");
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 
@@ -183,7 +183,7 @@ describe("RunEngine delays", () => {
       assertNonNullable(executionData3);
       expect(executionData3.snapshot.executionStatus).toBe("QUEUED");
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 
@@ -287,7 +287,7 @@ describe("RunEngine delays", () => {
 
       expect(run3.status).toBe("EXPIRED");
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 
@@ -398,7 +398,7 @@ describe("RunEngine delays", () => {
       expect(executionData4.snapshot.executionStatus).toBe("FINISHED");
       expect(executionData4.run.status).toBe("CANCELED");
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 });

internal-packages/run-engine/src/engine/tests/dequeuing.test.ts

@@ -77,7 +77,7 @@ describe("RunEngine dequeuing", () => {
 
       expect(dequeued.length).toBe(5);
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 
@@ -169,7 +169,7 @@ describe("RunEngine dequeuing", () => {
         const queueLength3 = await engine.runQueue.lengthOfEnvQueue(authenticatedEnvironment);
         expect(queueLength3).toBe(12);
       } finally {
-        engine.quit();
+        await engine.quit();
       }
     }
   );

internal-packages/run-engine/src/engine/tests/pendingVersion.test.ts

@@ -158,7 +158,7 @@ describe("RunEngine pending version", () => {
         );
         expect(queueLength2).toBe(2);
       } finally {
-        engine.quit();
+        await engine.quit();
       }
     }
   );
@@ -319,7 +319,7 @@ describe("RunEngine pending version", () => {
         );
         expect(queueLength3).toBe(1);
       } finally {
-        engine.quit();
+        await engine.quit();
       }
     }
   );

internal-packages/run-engine/src/engine/tests/priority.test.ts

@@ -103,7 +103,7 @@ describe("RunEngine priority", () => {
         expect(dequeue2.length).toBe(1);
         expect(dequeue2[0].run.friendlyId).toBe(runs[2].friendlyId);
       } finally {
-        engine.quit();
+        await engine.quit();
       }
     }
   );
@@ -197,7 +197,7 @@ describe("RunEngine priority", () => {
         expect(dequeue[3].run.friendlyId).toBe(runs[4].friendlyId);
         expect(dequeue[4].run.friendlyId).toBe(runs[0].friendlyId);
       } finally {
-        engine.quit();
+        await engine.quit();
       }
     }
   );

internal-packages/run-engine/src/engine/tests/trigger.test.ts

@@ -198,7 +198,7 @@ describe("RunEngine trigger()", () => {
       expect(runWaitpointAfter[0].type).toBe("RUN");
       expect(runWaitpointAfter[0].output).toBe(`{"foo":"bar"}`);
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 
@@ -325,7 +325,7 @@ describe("RunEngine trigger()", () => {
       expect(output.type).toBe(error.type);
       expect(runWaitpointAfter[0].outputIsError).toBe(true);
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 });

internal-packages/run-engine/src/engine/tests/triggerAndWait.test.ts

@@ -189,7 +189,7 @@ describe("RunEngine triggerAndWait", () => {
       );
       expect(parentExecutionDataAfter.completedWaitpoints![0].output).toBe('{"foo":"bar"}');
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 
@@ -445,7 +445,7 @@ describe("RunEngine triggerAndWait", () => {
         );
         expect(parent2ExecutionDataAfter.completedWaitpoints![0].output).toBe('{"foo":"bar"}');
       } finally {
-        engine.quit();
+        await engine.quit();
       }
     }
   );

internal-packages/run-engine/src/engine/tests/ttl.test.ts

@@ -102,7 +102,7 @@ describe("RunEngine ttl", () => {
       );
       expect(envConcurrencyCompleted).toBe(0);
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 });

internal-packages/run-engine/src/engine/tests/waitpoints.test.ts

@@ -121,7 +121,7 @@ describe("RunEngine Waitpoints", () => {
       const executionDataAfter = await engine.getRunExecutionData({ runId: run.id });
       expect(executionDataAfter?.snapshot.executionStatus).toBe("EXECUTING");
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 
@@ -261,7 +261,7 @@ describe("RunEngine Waitpoints", () => {
       });
       expect(runWaitpoint).toBeNull();
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 
@@ -400,7 +400,7 @@ describe("RunEngine Waitpoints", () => {
         });
         expect(runWaitpoint).toBeNull();
       } finally {
-        engine.quit();
+        await engine.quit();
       }
     }
   );
@@ -516,7 +516,7 @@ describe("RunEngine Waitpoints", () => {
       });
       expect(runWaitpoint).toBeNull();
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 
@@ -664,7 +664,7 @@ describe("RunEngine Waitpoints", () => {
           expect(runWaitpoints.length).toBe(0);
         }
       } finally {
-        engine.quit();
+        await engine.quit();
       }
     }
   );
@@ -814,7 +814,7 @@ describe("RunEngine Waitpoints", () => {
         const isTimeout = isWaitpointOutputTimeout(waitpoint2.output);
         expect(isTimeout).toBe(true);
       } finally {
-        engine.quit();
+        await engine.quit();
       }
     }
   );
@@ -966,7 +966,7 @@ describe("RunEngine Waitpoints", () => {
       expect(waitpoint2.status).toBe("COMPLETED");
       expect(waitpoint2.outputIsError).toBe(false);
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 
@@ -1126,7 +1126,7 @@ describe("RunEngine Waitpoints", () => {
       expect(waitpoint2.status).toBe("COMPLETED");
       expect(waitpoint2.outputIsError).toBe(false);
     } finally {
-      engine.quit();
+      await engine.quit();
     }
   });
 

internal-packages/run-engine/src/run-queue/index.ts

@@ -541,7 +541,7 @@ export class RunQueue {
           }
         }
 
-        await this.#callNackMessage({ message });
+        await this.#callNackMessage({ message, retryAt });
 
         return true;
       },

internal-packages/run-engine/src/run-queue/tests/nack.test.ts

@@ -214,4 +214,65 @@ describe("RunQueue.nackMessage", () => {
       }
     }
   );
+
+  redisTest(
+    "nacking a message with retryAt sets the correct requeue time",
+    async ({ redisContainer }) => {
+      const queue = new RunQueue({
+        ...testOptions,
+        queueSelectionStrategy: new FairQueueSelectionStrategy({
+          redis: {
+            keyPrefix: "runqueue:test:",
+            host: redisContainer.getHost(),
+            port: redisContainer.getPort(),
+          },
+          keys: testOptions.keys,
+        }),
+        redis: {
+          keyPrefix: "runqueue:test:",
+          host: redisContainer.getHost(),
+          port: redisContainer.getPort(),
+        },
+      });
+
+      try {
+        const envMasterQueue = `env:${authenticatedEnvDev.id}`;
+
+        // Enqueue message
+        await queue.enqueueMessage({
+          env: authenticatedEnvDev,
+          message: messageDev,
+          masterQueues: ["main", envMasterQueue],
+        });
+
+        // Dequeue message
+        const dequeued = await queue.dequeueMessageFromMasterQueue(
+          "test_12345",
+          envMasterQueue,
+          10
+        );
+        expect(dequeued.length).toBe(1);
+
+        // Set retryAt to 5 seconds in the future
+        const retryAt = Date.now() + 5000;
+        await queue.nackMessage({
+          orgId: messageDev.orgId,
+          messageId: messageDev.runId,
+          retryAt,
+        });
+
+        // Check the score of the message in the queue
+        const queueKey = queue.keys.queueKey(authenticatedEnvDev, messageDev.queue);
+        const score = await queue.oldestMessageInQueue(authenticatedEnvDev, messageDev.queue);
+        expect(typeof score).toBe("number");
+        if (typeof score !== "number") {
+          throw new Error("Expected score to be a number, but got undefined");
+        }
+        // Should be within 100ms of retryAt
+        expect(Math.abs(score - retryAt)).toBeLessThanOrEqual(100);
+      } finally {
+        await queue.quit();
+      }
+    }
+  );
 });

internal-packages/testcontainers/package.json

@@ -10,10 +10,11 @@
     "ioredis": "^5.3.2"
   },
   "devDependencies": {
-    "@testcontainers/postgresql": "^10.13.1",
-    "@testcontainers/redis": "^10.13.1",
+    "@testcontainers/postgresql": "^10.25.0",
+    "@testcontainers/redis": "^10.25.0",
     "@trigger.dev/core": "workspace:*",
-    "testcontainers": "^10.13.1",
+    "std-env": "^3.9.0",
+    "testcontainers": "^10.25.0",
     "tinyexec": "^0.3.0",
     "vitest": "^1.4.0"
   },

internal-packages/testcontainers/src/docker.ts

@@ -0,0 +1,160 @@
+import { x } from "tinyexec";
+
+function stringToLines(str: string): string[] {
+  return str.split("\n").filter(Boolean);
+}
+
+function lineToWords(line: string): string[] {
+  return line.trim().split(/\s+/);
+}
+
+async function getDockerNetworks(): Promise<string[]> {
+  try {
+    const result = await x("docker", ["network", "ls" /* , "--no-trunc" */]);
+    return stringToLines(result.stdout);
+  } catch (error) {
+    console.error(error);
+    return ["error: check additional logs for more details"];
+  }
+}
+
+async function getDockerContainers(): Promise<string[]> {
+  try {
+    const result = await x("docker", ["ps", "-a" /* , "--no-trunc" */]);
+    return stringToLines(result.stdout);
+  } catch (error) {
+    console.error(error);
+    return ["error: check additional logs for more details"];
+  }
+}
+
+type DockerResource = { id: string; name: string };
+
+type DockerNetworkAttachment = DockerResource & {
+  containers: string[];
+};
+
+export async function getDockerNetworkAttachments(): Promise<DockerNetworkAttachment[]> {
+  let attachments: DockerNetworkAttachment[] = [];
+  let networks: DockerResource[] = [];
+
+  try {
+    const result = await x("docker", [
+      "network",
+      "ls",
+      "--format",
+      '{{.ID | printf "%.12s"}} {{.Name}}',
+    ]);
+
+    const lines = stringToLines(result.stdout);
+
+    for (const line of lines) {
+      const [id, name] = lineToWords(line);
+
+      if (!id || !name) {
+        continue;
+      }
+
+      networks.push({ id, name });
+    }
+  } catch (err) {
+    console.error("Failed to list docker networks:", err);
+  }
+
+  for (const { id, name } of networks) {
+    try {
+      // Get containers, one per line: id name\n
+      const containersResult = await x("docker", [
+        "network",
+        "inspect",
+        "--format",
+        '{{range $k, $v := .Containers}}{{$k | printf "%.12s"}} {{$v.Name}}\n{{end}}',
+        id,
+      ]);
+
+      const containers = stringToLines(containersResult.stdout);
+
+      attachments.push({ id, name, containers });
+    } catch (err) {
+      console.error(`Failed to inspect network ${id}:`, err);
+      attachments.push({ id, name, containers: [] });
+    }
+  }
+
+  return attachments;
+}
+
+type DockerContainerNetwork = DockerResource & {
+  networks: string[];
+};
+
+export async function getDockerContainerNetworks(): Promise<DockerContainerNetwork[]> {
+  let results: DockerContainerNetwork[] = [];
+  let containers: DockerResource[] = [];
+
+  try {
+    const result = await x("docker", [
+      "ps",
+      "-a",
+      "--format",
+      '{{.ID | printf "%.12s"}} {{.Names}}',
+    ]);
+
+    const lines = stringToLines(result.stdout);
+
+    for (const line of lines) {
+      const [id, name] = lineToWords(line);
+
+      if (!id || !name) {
+        continue;
+      }
+
+      containers.push({ id, name });
+    }
+  } catch (err) {
+    console.error("Failed to list docker containers:", err);
+  }
+
+  for (const { id, name } of containers) {
+    try {
+      const inspectResult = await x("docker", [
+        "inspect",
+        "--format",
+        '{{ range $k, $v := .NetworkSettings.Networks }}{{ $k | printf "%.12s" }} {{ $v.Name }}\n{{ end }}',
+        id,
+      ]);
+
+      const networks = stringToLines(inspectResult.stdout);
+
+      results.push({ id, name, networks });
+    } catch (err) {
+      console.error(`Failed to inspect container ${id}:`, err);
+      results.push({ id, name: String(err), networks: [] });
+    }
+  }
+
+  return results;
+}
+
+export type DockerDiagnostics = {
+  containers?: string[];
+  networks?: string[];
+  containerNetworks?: DockerContainerNetwork[];
+  networkAttachments?: DockerNetworkAttachment[];
+};
+
+export async function getDockerDiagnostics(): Promise<DockerDiagnostics> {
+  const [containers, networks, networkAttachments, containerNetworks] = await Promise.all([
+    getDockerContainers(),
+    getDockerNetworks(),
+    getDockerNetworkAttachments(),
+    getDockerContainerNetworks(),
+  ]);
+
+  return {
+    containers,
+    networks,
+    containerNetworks,
+    networkAttachments,
+  };
+}

internal-packages/testcontainers/src/index.ts

@@ -3,8 +3,15 @@ import { StartedRedisContainer } from "@testcontainers/redis";
 import { PrismaClient } from "@trigger.dev/database";
 import { RedisOptions } from "ioredis";
 import { Network, type StartedNetwork } from "testcontainers";
-import { test } from "vitest";
-import { createElectricContainer, createPostgresContainer, createRedisContainer } from "./utils";
+import { TaskContext, test } from "vitest";
+import {
+  createElectricContainer,
+  createPostgresContainer,
+  createRedisContainer,
+  useContainer,
+  withContainerSetup,
+} from "./utils";
+import { getTaskMetadata, logCleanup, logSetup } from "./logs";
 
 export { assertNonNullable } from "./utils";
 export { StartedRedisContainer };
@@ -31,38 +38,50 @@ type ContainerWithElectricContext = NetworkContext & PostgresContext & ElectricC
 
 type Use<T> = (value: T) => Promise<void>;
 
-const network = async ({}, use: Use<StartedNetwork>) => {
+const network = async ({ task }: TaskContext, use: Use<StartedNetwork>) => {
+  const testName = task.name;
+
+  logSetup("network: starting", { testName });
+
+  const start = Date.now();
   const network = await new Network().start();
+  const startDurationMs = Date.now() - start;
+
+  const metadata = {
+    ...getTaskMetadata(task),
+    networkId: network.getId().slice(0, 12),
+    networkName: network.getName(),
+    startDurationMs,
+  };
+
+  logSetup("network: started", metadata);
+
   try {
     await use(network);
   } finally {
-    try {
-      await network.stop();
-    } catch (error) {
-      console.warn("Network stop error (ignored):", error);
-    }
     // Make sure to stop the network after use
+    await logCleanup("network", network.stop(), metadata);
   }
 };
 
 const postgresContainer = async (
-  { network }: { network: StartedNetwork },
+  { network, task }: { network: StartedNetwork } & TaskContext,
   use: Use<StartedPostgreSqlContainer>
 ) => {
-  const { container } = await createPostgresContainer(network);
-  try {
-    await use(container);
-  } finally {
-    // WARNING: Testcontainers by default will not wait until the container has stopped. It will simply issue the stop command and return immediately.
-    // If you need to wait for the container to be stopped, you can provide a timeout. The unit of timeout option here is second
-    await container.stop({ timeout: 10 });
-  }
+  const { container, metadata } = await withContainerSetup({
+    name: "postgresContainer",
+    task,
+    setup: createPostgresContainer(network),
+  });
+
+  await useContainer("postgresContainer", { container, task, use: () => use(container) });
 };
 
 const prisma = async (
-  { postgresContainer }: { postgresContainer: StartedPostgreSqlContainer },
+  { postgresContainer, task }: { postgresContainer: StartedPostgreSqlContainer } & TaskContext,
   use: Use<PrismaClient>
 ) => {
+  const testName = task.name;
   const url = postgresContainer.getConnectionUri();
 
   console.log("Initializing Prisma with URL:", url);
@@ -77,27 +96,26 @@ const prisma = async (
   try {
     await use(prisma);
   } finally {
-    await prisma.$disconnect();
+    await logCleanup("prisma", prisma.$disconnect(), { testName });
   }
 };
 
 export const postgresTest = test.extend<PostgresContext>({ network, postgresContainer, prisma });
 
 const redisContainer = async (
-  { network }: { network: StartedNetwork },
+  { network, task }: { network: StartedNetwork } & TaskContext,
   use: Use<StartedRedisContainer>
 ) => {
-  const { container } = await createRedisContainer({
-    port: 6379,
-    network,
+  const { container, metadata } = await withContainerSetup({
+    name: "redisContainer",
+    task,
+    setup: createRedisContainer({
+      port: 6379,
+      network,
+    }),
   });
-  try {
-    await use(container);
-  } finally {
-    // WARNING: Testcontainers by default will not wait until the container has stopped. It will simply issue the stop command and return immediately.
-    // If you need to wait for the container to be stopped, you can provide a timeout. The unit of timeout option here is second
-    await container.stop({ timeout: 10 });
-  }
+
+  await useContainer("redisContainer", { container, task, use: () => use(container) });
 };
 
 const redisOptions = async (
@@ -139,17 +157,17 @@ const electricOrigin = async (
   {
     postgresContainer,
     network,
-  }: { postgresContainer: StartedPostgreSqlContainer; network: StartedNetwork },
+    task,
+  }: { postgresContainer: StartedPostgreSqlContainer; network: StartedNetwork } & TaskContext,
   use: Use<string>
 ) => {
-  const { origin, container } = await createElectricContainer(postgresContainer, network);
-  try {
-    await use(origin);
-  } finally {
-    // WARNING: Testcontainers by default will not wait until the container has stopped. It will simply issue the stop command and return immediately.
-    // If you need to wait for the container to be stopped, you can provide a timeout. The unit of timeout option here is second
-    await container.stop({ timeout: 10 });
-  }
+  const { origin, container, metadata } = await withContainerSetup({
+    name: "electricContainer",
+    task,
+    setup: createElectricContainer(postgresContainer, network),
+  });
+
+  await useContainer("electricContainer", { container, task, use: () => use(origin) });
 };
 
 export const containerTest = test.extend<ContainerContext>({

internal-packages/testcontainers/src/logs.ts

@@ -0,0 +1,101 @@
+import { env, isCI } from "std-env";
+import { TaskContext } from "vitest";
+import { DockerDiagnostics, getDockerDiagnostics } from "./docker";
+import { StartedTestContainer } from "testcontainers";
+
+let setupOrder = 0;
+
+export function logSetup(resource: string, metadata: Record<string, unknown>) {
+  const order = setupOrder++;
+
+  if (!isCI) {
+    return;
+  }
+
+  console.log(
+    JSON.stringify({
+      type: "setup",
+      order,
+      resource,
+      timestamp: new Date().toISOString(),
+      ...metadata,
+    })
+  );
+}
+
+export function getContainerMetadata(container: StartedTestContainer) {
+  return {
+    containerName: container.getName(),
+    containerId: container.getId().slice(0, 12),
+    containerNetworkNames: container.getNetworkNames(),
+  };
+}
+
+export function getTaskMetadata(task: TaskContext["task"]) {
+  return {
+    testName: task.name,
+  };
+}
+
+let cleanupOrder = 0;
+let activeCleanups = 0;
+
+/**
+ * Logs the cleanup of a resource.
+ * @param resource - The resource that is being cleaned up.
+ * @param promise - The cleanup promise to await..
+ */
+export async function logCleanup(
+  resource: string,
+  promise: Promise<unknown>,
+  metadata: Record<string, unknown> = {}
+) {
+  const start = new Date();
+  const order = cleanupOrder++;
+  const activeAtStart = ++activeCleanups;
+
+  let error: unknown = null;
+
+  try {
+    await promise;
+  } catch (err) {
+    error = err instanceof Error ? err.message : String(err);
+  }
+
+  const end = new Date();
+  const durationMs = end.getTime() - start.getTime();
+  const activeAtEnd = --activeCleanups;
+  const parallel = activeAtStart > 1 || activeAtEnd > 0;
+
+  if (!isCI) {
+    return;
+  }
+
+  let dockerDiagnostics: DockerDiagnostics = {};
+
+  // Only run docker diagnostics if there was an error or cleanup took longer than 5s
+  if (error || durationMs > 5000 || env.DOCKER_DIAGNOSTICS) {
+    try {
+      dockerDiagnostics = await getDockerDiagnostics();
+    } catch (diagnosticErr) {
+      console.error("Failed to get docker diagnostics:", diagnosticErr);
+    }
+  }
+
+  console.log(
+    JSON.stringify({
+      type: "cleanup",
+      order,
+      resource,
+      durationMs,
+      start: start.toISOString(),
+      end: end.toISOString(),
+      parallel,
+      error,
+      activeAtStart,
+      activeAtEnd,
+      ...metadata,
+      ...dockerDiagnostics,
+    })
+  );
+}

internal-packages/testcontainers/src/utils.ts

@@ -1,10 +1,14 @@
 import { PostgreSqlContainer, StartedPostgreSqlContainer } from "@testcontainers/postgresql";
 import { RedisContainer, StartedRedisContainer } from "@testcontainers/redis";
+import { tryCatch } from "@trigger.dev/core";
 import Redis from "ioredis";
 import path from "path";
-import { GenericContainer, StartedNetwork, Wait } from "testcontainers";
+import { isDebug } from "std-env";
+import { GenericContainer, StartedNetwork, StartedTestContainer, Wait } from "testcontainers";
 import { x } from "tinyexec";
-import { expect } from "vitest";
+import { expect, TaskContext } from "vitest";
+import { getContainerMetadata, getTaskMetadata, logCleanup } from "./logs";
+import { logSetup } from "./logs";
 
 export async function createPostgresContainer(network: StartedNetwork) {
   const container = await new PostgreSqlContainer("docker.io/postgres:14")
@@ -67,7 +71,12 @@ export async function createRedisContainer({
     .start();
 
   // Add a verification step
-  await verifyRedisConnection(startedContainer);
+  const [error] = await tryCatch(verifyRedisConnection(startedContainer));
+
+  if (error) {
+    await startedContainer.stop({ timeout: 30 });
+    throw new Error("verifyRedisConnection error", { cause: error });
+  }
 
   return {
     container: startedContainer,
@@ -87,12 +96,28 @@ async function verifyRedisConnection(container: StartedRedisContainer) {
     },
   });
 
+  const containerMetadata = {
+    containerId: container.getId().slice(0, 12),
+    containerName: container.getName(),
+    containerNetworkNames: container.getNetworkNames(),
+  };
+
   redis.on("error", (error) => {
-    // swallow the error
+    if (isDebug) {
+      console.log("verifyRedisConnection: client error", error, containerMetadata);
+    }
+
+    // Don't throw here, we'll do that below if the ping fails
   });
 
   try {
     await redis.ping();
+  } catch (error) {
+    if (isDebug) {
+      console.log("verifyRedisConnection: ping error", error, containerMetadata);
+    }
+
+    throw new Error("verifyRedisConnection: ping error", { cause: error });
   } finally {
     await redis.quit();
   }
@@ -126,3 +151,56 @@ export function assertNonNullable<T>(value: T): asserts value is NonNullable<T>
   expect(value).toBeDefined();
   expect(value).not.toBeNull();
 }
+
+export async function withContainerSetup<T>({
+  name,
+  task,
+  setup,
+}: {
+  name: string;
+  task: TaskContext["task"];
+  setup: Promise<T extends { container: StartedTestContainer } ? T : never>;
+}): Promise<T & { metadata: Record<string, unknown> }> {
+  const testName = task.name;
+  logSetup(`${name}: starting`, { testName });
+
+  const start = Date.now();
+  const result = await setup;
+  const startDurationMs = Date.now() - start;
+
+  const metadata = {
+    ...getTaskMetadata(task),
+    ...getContainerMetadata(result.container),
+    startDurationMs,
+  };
+
+  logSetup(`${name}: started`, metadata);
+
+  return { ...result, metadata };
+}
+
+export async function useContainer<TContainer extends StartedTestContainer>(
+  name: string,
+  {
+    container,
+    task,
+    use,
+  }: { container: TContainer; task: TaskContext["task"]; use: () => Promise<void> }
+) {
+  const metadata = {
+    ...getTaskMetadata(task),
+    ...getContainerMetadata(container),
+    useDurationMs: 0,
+  };
+
+  try {
+    const start = Date.now();
+    await use();
+    const useDurationMs = Date.now() - start;
+    metadata.useDurationMs = useDurationMs;
+  } finally {
+    // WARNING: Testcontainers by default will not wait until the container has stopped. It will simply issue the stop command and return immediately.
+    // If you need to wait for the container to be stopped, you can provide a timeout. The unit of timeout option here is second
+    await logCleanup(name, container.stop({ timeout: 10 }), metadata);
+  }
+}

internal-packages/testcontainers/tsconfig.json

@@ -13,6 +13,7 @@
     "skipLibCheck": true,
     "noEmit": true,
     "strict": true,
+    "noUncheckedIndexedAccess": true,
     "paths": {
       "@trigger.dev/core": ["../../packages/core/src/index"],
       "@trigger.dev/core/*": ["../../packages/core/src/*"],

packages/build/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @trigger.dev/build
 
+## 4.0.0-v4-beta.12
+
+### Patch Changes
+
+- Updated dependencies:
+  - `@trigger.dev/core@4.0.0-v4-beta.12`
+
 ## 4.0.0-v4-beta.11
 
 ### Patch Changes

packages/build/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/build",
-  "version": "4.0.0-v4-beta.11",
+  "version": "4.0.0-v4-beta.12",
   "description": "trigger.dev build extensions",
   "license": "MIT",
   "publishConfig": {
@@ -69,7 +69,7 @@
     "check-exports": "attw --pack ."
   },
   "dependencies": {
-    "@trigger.dev/core": "workspace:4.0.0-v4-beta.11",
+    "@trigger.dev/core": "workspace:4.0.0-v4-beta.12",
     "pkg-types": "^1.1.3",
     "tinyglobby": "^0.2.2",
     "tsconfck": "3.1.3"

packages/cli-v3/CHANGELOG.md

@@ -1,5 +1,16 @@
 # trigger.dev
 
+## 4.0.0-v4-beta.12
+
+### Patch Changes
+
+- Display clickable links in Cursor terminal ([#1998](https://github.com/triggerdotdev/trigger.dev/pull/1998))
+- Added AI assistance link when you have build errors ([#1925](https://github.com/triggerdotdev/trigger.dev/pull/1925))
+- If you pass a directory when calling deploy we validate it exists and give helpful hints ([#2013](https://github.com/triggerdotdev/trigger.dev/pull/2013))
+- Updated dependencies:
+  - `@trigger.dev/build@4.0.0-v4-beta.12`
+  - `@trigger.dev/core@4.0.0-v4-beta.12`
+
 ## 4.0.0-v4-beta.11
 
 ### Patch Changes

packages/cli-v3/package.json

@@ -1,6 +1,6 @@
 {
   "name": "trigger.dev",
-  "version": "4.0.0-v4-beta.11",
+  "version": "4.0.0-v4-beta.12",
   "description": "A Command-Line Interface for Trigger.dev (v3) projects",
   "type": "module",
   "license": "MIT",
@@ -93,8 +93,8 @@
     "@opentelemetry/sdk-trace-base": "1.25.1",
     "@opentelemetry/sdk-trace-node": "1.25.1",
     "@opentelemetry/semantic-conventions": "1.25.1",
-    "@trigger.dev/build": "workspace:4.0.0-v4-beta.11",
-    "@trigger.dev/core": "workspace:4.0.0-v4-beta.11",
+    "@trigger.dev/build": "workspace:4.0.0-v4-beta.12",
+    "@trigger.dev/core": "workspace:4.0.0-v4-beta.12",
     "ansi-escapes": "^7.0.0",
     "c12": "^1.11.1",
     "chalk": "^5.2.0",

packages/cli-v3/src/commands/deploy.ts

@@ -35,6 +35,7 @@ import { spinner } from "../utilities/windows.js";
 import { login } from "./login.js";
 import { updateTriggerPackages } from "./update.js";
 import { setGithubActionsOutputAndEnvVars } from "../utilities/githubActions.js";
+import { isDirectory } from "../utilities/fileSystem.js";
 
 const DeployCommandOptions = CommonCommandOptions.extend({
   dryRun: z.boolean().default(false),
@@ -169,7 +170,24 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) {
     await updateTriggerPackages(dir, { ...options }, true, true);
   }
 
-  const projectPath = resolve(process.cwd(), dir);
+  const cwd = process.cwd();
+  const projectPath = resolve(cwd, dir);
+
+  if (dir !== "." && !isDirectory(projectPath)) {
+    if (dir === "staging" || dir === "prod") {
+      throw new Error(`To deploy to ${dir}, you need to pass "--env ${dir}", not just "${dir}".`);
+    }
+
+    if (dir === "production") {
+      throw new Error(`To deploy to production, you need to pass "--env prod", not "production".`);
+    }
+
+    if (dir === "stg") {
+      throw new Error(`To deploy to staging, you need to pass "--env staging", not "stg".`);
+    }
+
+    throw new Error(`Directory "${dir}" not found at ${projectPath}`);
+  }
 
   const authorization = await login({
     embedded: true,

packages/cli-v3/src/entryPoints/dev-run-worker.ts

@@ -23,6 +23,7 @@ import {
   TaskRunExecution,
   timeout,
   TriggerConfig,
+  UsageMeasurement,
   waitUntil,
   WorkerManifest,
   WorkerToExecutorMessageCatalog,
@@ -232,7 +233,10 @@ async function bootstrap() {
 
 let _execution: TaskRunExecution | undefined;
 let _isRunning = false;
+let _isCancelled = false;
 let _tracingSDK: TracingSDK | undefined;
+let _executionMeasurement: UsageMeasurement | undefined;
+const cancelController = new AbortController();
 
 const zodIpc = new ZodIpcConnection({
   listenSchema: WorkerToExecutorMessageCatalog,
@@ -403,18 +407,17 @@ const zodIpc = new ZodIpcConnection({
             getNumberEnvVar("TRIGGER_RUN_METADATA_FLUSH_INTERVAL", 1000)
           );
 
-          const measurement = usage.start();
+          _executionMeasurement = usage.start();
 
-          // This lives outside of the executor because this will eventually be moved to the controller level
-          const signal = execution.run.maxDuration
-            ? timeout.abortAfterTimeout(execution.run.maxDuration)
-            : undefined;
+          const timeoutController = timeout.abortAfterTimeout(execution.run.maxDuration);
+
+          const signal = AbortSignal.any([cancelController.signal, timeoutController.signal]);
 
           const { result } = await executor.execute(execution, metadata, traceContext, signal);
 
-          const usageSample = usage.stop(measurement);
+          if (_isRunning && !_isCancelled) {
+            const usageSample = usage.stop(_executionMeasurement);
 
-          if (_isRunning) {
             return sender.send("TASK_RUN_COMPLETED", {
               execution,
               result: {
@@ -452,7 +455,16 @@ const zodIpc = new ZodIpcConnection({
         });
       }
     },
-    FLUSH: async ({ timeoutInMs }, sender) => {
+    CANCEL: async ({ timeoutInMs }) => {
+      _isCancelled = true;
+      cancelController.abort("run cancelled");
+      await callCancelHooks(timeoutInMs);
+      if (_executionMeasurement) {
+        usage.stop(_executionMeasurement);
+      }
+      await flushAll(timeoutInMs);
+    },
+    FLUSH: async ({ timeoutInMs }) => {
       await flushAll(timeoutInMs);
     },
     RESOLVE_WAITPOINT: async ({ waitpoint }) => {
@@ -461,6 +473,18 @@ const zodIpc = new ZodIpcConnection({
   },
 });
 
+async function callCancelHooks(timeoutInMs: number = 10_000) {
+  const now = performance.now();
+
+  try {
+    await Promise.race([lifecycleHooks.callOnCancelHookListeners(), setTimeout(timeoutInMs)]);
+  } finally {
+    const duration = performance.now() - now;
+
+    log(`Called cancel hooks in ${duration}ms`);
+  }
+}
+
 async function flushAll(timeoutInMs: number = 10_000) {
   const now = performance.now();
 

packages/cli-v3/src/entryPoints/managed-run-worker.ts

@@ -22,6 +22,7 @@ import {
   TaskRunExecution,
   timeout,
   TriggerConfig,
+  UsageMeasurement,
   waitUntil,
   WorkerManifest,
   WorkerToExecutorMessageCatalog,
@@ -229,7 +230,10 @@ async function bootstrap() {
 
 let _execution: TaskRunExecution | undefined;
 let _isRunning = false;
+let _isCancelled = false;
 let _tracingSDK: TracingSDK | undefined;
+let _executionMeasurement: UsageMeasurement | undefined;
+const cancelController = new AbortController();
 
 const zodIpc = new ZodIpcConnection({
   listenSchema: WorkerToExecutorMessageCatalog,
@@ -398,18 +402,17 @@ const zodIpc = new ZodIpcConnection({
             getNumberEnvVar("TRIGGER_RUN_METADATA_FLUSH_INTERVAL", 1000)
           );
 
-          const measurement = usage.start();
+          _executionMeasurement = usage.start();
 
-          // This lives outside of the executor because this will eventually be moved to the controller level
-          const signal = execution.run.maxDuration
-            ? timeout.abortAfterTimeout(execution.run.maxDuration)
-            : undefined;
+          const timeoutController = timeout.abortAfterTimeout(execution.run.maxDuration);
+
+          const signal = AbortSignal.any([cancelController.signal, timeoutController.signal]);
 
           const { result } = await executor.execute(execution, metadata, traceContext, signal);
 
-          const usageSample = usage.stop(measurement);
+          if (_isRunning && !_isCancelled) {
+            const usageSample = usage.stop(_executionMeasurement);
 
-          if (_isRunning) {
             return sender.send("TASK_RUN_COMPLETED", {
               execution,
               result: {
@@ -448,12 +451,33 @@ const zodIpc = new ZodIpcConnection({
     FLUSH: async ({ timeoutInMs }, sender) => {
       await flushAll(timeoutInMs);
     },
+    CANCEL: async ({ timeoutInMs }, sender) => {
+      _isCancelled = true;
+      cancelController.abort("run cancelled");
+      await callCancelHooks(timeoutInMs);
+      if (_executionMeasurement) {
+        usage.stop(_executionMeasurement);
+      }
+      await flushAll(timeoutInMs);
+    },
     RESOLVE_WAITPOINT: async ({ waitpoint }) => {
       sharedWorkerRuntime.resolveWaitpoints([waitpoint]);
     },
   },
 });
 
+async function callCancelHooks(timeoutInMs: number = 10_000) {
+  const now = performance.now();
+
+  try {
+    await Promise.race([lifecycleHooks.callOnCancelHookListeners(), setTimeout(timeoutInMs)]);
+  } finally {
+    const duration = performance.now() - now;
+
+    console.log(`Called cancel hooks in ${duration}ms`);
+  }
+}
+
 async function flushAll(timeoutInMs: number = 10_000) {
   const now = performance.now();
 

packages/cli-v3/src/executions/taskRunProcess.ts

@@ -111,9 +111,9 @@ export class TaskRunProcess {
     this._isBeingCancelled = true;
 
     try {
-      await this.#flush();
+      await this.#cancel();
     } catch (err) {
-      console.error("Error flushing task run process", { err });
+      console.error("Error cancelling task run process", { err });
     }
 
     await this.kill();
@@ -122,6 +122,10 @@ export class TaskRunProcess {
   async cleanup(kill = true) {
     this._isPreparedForNextRun = false;
 
+    if (this._isBeingCancelled) {
+      return;
+    }
+
     try {
       await this.#flush();
     } catch (err) {
@@ -221,10 +225,17 @@ export class TaskRunProcess {
     await this._ipc?.sendWithAck("FLUSH", { timeoutInMs }, timeoutInMs + 1_000);
   }
 
+  async #cancel(timeoutInMs: number = 30_000) {
+    logger.debug("sending cancel message to task run process", { pid: this.pid, timeoutInMs });
+
+    await this._ipc?.sendWithAck("CANCEL", { timeoutInMs }, timeoutInMs + 1_000);
+  }
+
   async execute(
     params: TaskRunProcessExecuteParams,
     isWarmStart?: boolean
   ): Promise<TaskRunExecutionResult> {
+    this._isBeingCancelled = false;
     this._isPreparedForNextRun = false;
     this._isPreparedForNextAttempt = false;
 

packages/core/CHANGELOG.md

@@ -1,5 +1,7 @@
 # internal-platform
 
+## 4.0.0-v4-beta.12
+
 ## 4.0.0-v4-beta.11
 
 ## 4.0.0-v4-beta.10

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/core",
-  "version": "4.0.0-v4-beta.11",
+  "version": "4.0.0-v4-beta.12",
   "description": "Core code used across the Trigger.dev SDK and platform",
   "license": "MIT",
   "publishConfig": {

packages/core/src/utils.ts

@@ -16,3 +16,25 @@ export async function tryCatch<T, E = Error>(
     return [error as E, null];
   }
 }
+
+export type Deferred<T> = {
+  promise: Promise<T>;
+  resolve: (value: T) => void;
+  reject: (reason?: any) => void;
+};
+
+export function promiseWithResolvers<T>(): Deferred<T> {
+  let resolve!: (value: T) => void;
+  let reject!: (reason?: any) => void;
+
+  const promise = new Promise<T>((_resolve, _reject) => {
+    resolve = _resolve;
+    reject = _reject;
+  });
+
+  return {
+    promise,
+    resolve,
+    reject,
+  };
+}

packages/core/src/v3/lifecycle-hooks-api.ts

@@ -32,4 +32,7 @@ export type {
   AnyOnCleanupHookFunction,
   TaskCleanupHookParams,
   TaskWait,
+  TaskCancelHookParams,
+  OnCancelHookFunction,
+  AnyOnCancelHookFunction,
 } from "./lifecycleHooks/types.js";

packages/core/src/v3/lifecycleHooks/index.ts

@@ -13,6 +13,7 @@ import {
   AnyOnStartHookFunction,
   AnyOnSuccessHookFunction,
   AnyOnWaitHookFunction,
+  AnyOnCancelHookFunction,
   RegisteredHookFunction,
   RegisterHookFunctionParams,
   TaskWait,
@@ -260,6 +261,33 @@ export class LifecycleHooksAPI {
     this.#getManager().registerOnResumeHookListener(listener);
   }
 
+  public registerGlobalCancelHook(hook: RegisterHookFunctionParams<AnyOnCancelHookFunction>): void {
+    this.#getManager().registerGlobalCancelHook(hook);
+  }
+
+  public registerTaskCancelHook(
+    taskId: string,
+    hook: RegisterHookFunctionParams<AnyOnCancelHookFunction>
+  ): void {
+    this.#getManager().registerTaskCancelHook(taskId, hook);
+  }
+
+  public getTaskCancelHook(taskId: string): AnyOnCancelHookFunction | undefined {
+    return this.#getManager().getTaskCancelHook(taskId);
+  }
+
+  public getGlobalCancelHooks(): RegisteredHookFunction<AnyOnCancelHookFunction>[] {
+    return this.#getManager().getGlobalCancelHooks();
+  }
+
+  public callOnCancelHookListeners(): Promise<void> {
+    return this.#getManager().callOnCancelHookListeners();
+  }
+
+  public registerOnCancelHookListener(listener: () => Promise<void>): void {
+    this.#getManager().registerOnCancelHookListener(listener);
+  }
+
   #getManager(): LifecycleHooksManager {
     return getGlobal(API_NAME) ?? NOOP_LIFECYCLE_HOOKS_MANAGER;
   }

packages/core/src/v3/lifecycleHooks/manager.ts

@@ -13,6 +13,7 @@ import {
   AnyOnMiddlewareHookFunction,
   AnyOnCleanupHookFunction,
   TaskWait,
+  AnyOnCancelHookFunction,
 } from "./types.js";
 
 export class StandardLifecycleHooksManager implements LifecycleHooksManager {
@@ -37,9 +38,6 @@ export class StandardLifecycleHooksManager implements LifecycleHooksManager {
   private taskCompleteHooks: Map<string, RegisteredHookFunction<AnyOnCompleteHookFunction>> =
     new Map();
 
-  private globalWaitHooks: Map<string, RegisteredHookFunction<AnyOnWaitHookFunction>> = new Map();
-  private taskWaitHooks: Map<string, RegisteredHookFunction<AnyOnWaitHookFunction>> = new Map();
-
   private globalResumeHooks: Map<string, RegisteredHookFunction<AnyOnResumeHookFunction>> =
     new Map();
   private taskResumeHooks: Map<string, RegisteredHookFunction<AnyOnResumeHookFunction>> = new Map();
@@ -59,9 +57,25 @@ export class StandardLifecycleHooksManager implements LifecycleHooksManager {
   private taskCleanupHooks: Map<string, RegisteredHookFunction<AnyOnCleanupHookFunction>> =
     new Map();
 
+  private globalWaitHooks: Map<string, RegisteredHookFunction<AnyOnWaitHookFunction>> = new Map();
+  private taskWaitHooks: Map<string, RegisteredHookFunction<AnyOnWaitHookFunction>> = new Map();
   private onWaitHookListeners: ((wait: TaskWait) => Promise<void>)[] = [];
+
   private onResumeHookListeners: ((wait: TaskWait) => Promise<void>)[] = [];
 
+  private globalCancelHooks: Map<string, RegisteredHookFunction<AnyOnCancelHookFunction>> =
+    new Map();
+  private taskCancelHooks: Map<string, RegisteredHookFunction<AnyOnCancelHookFunction>> = new Map();
+  private onCancelHookListeners: (() => Promise<void>)[] = [];
+
+  registerOnCancelHookListener(listener: () => Promise<void>): void {
+    this.onCancelHookListeners.push(listener);
+  }
+
+  async callOnCancelHookListeners(): Promise<void> {
+    await Promise.allSettled(this.onCancelHookListeners.map((listener) => listener()));
+  }
+
   registerOnWaitHookListener(listener: (wait: TaskWait) => Promise<void>): void {
     this.onWaitHookListeners.push(listener);
   }
@@ -394,9 +408,65 @@ export class StandardLifecycleHooksManager implements LifecycleHooksManager {
   getGlobalCleanupHooks(): RegisteredHookFunction<AnyOnCleanupHookFunction>[] {
     return Array.from(this.globalCleanupHooks.values());
   }
+
+  registerGlobalCancelHook(hook: RegisterHookFunctionParams<AnyOnCancelHookFunction>): void {
+    const id = generateHookId(hook);
+
+    this.globalCancelHooks.set(id, {
+      id,
+      name: hook.id,
+      fn: hook.fn,
+    });
+  }
+
+  registerTaskCancelHook(
+    taskId: string,
+    hook: RegisterHookFunctionParams<AnyOnCancelHookFunction>
+  ): void {
+    const id = generateHookId(hook);
+
+    this.taskCancelHooks.set(taskId, {
+      id,
+      name: hook.id,
+      fn: hook.fn,
+    });
+  }
+
+  getGlobalCancelHooks(): RegisteredHookFunction<AnyOnCancelHookFunction>[] {
+    return Array.from(this.globalCancelHooks.values());
+  }
+
+  getTaskCancelHook(taskId: string): AnyOnCancelHookFunction | undefined {
+    return this.taskCancelHooks.get(taskId)?.fn;
+  }
 }
 
 export class NoopLifecycleHooksManager implements LifecycleHooksManager {
+  registerOnCancelHookListener(listener: () => Promise<void>): void {
+    // Noop
+  }
+
+  async callOnCancelHookListeners(): Promise<void> {
+    // Noop
+  }
+
+  registerGlobalCancelHook(hook: RegisterHookFunctionParams<AnyOnCancelHookFunction>): void {}
+
+  registerTaskCancelHook(
+    taskId: string,
+    hook: RegisterHookFunctionParams<AnyOnCancelHookFunction>
+  ): void {
+    // Noop
+  }
+
+  getTaskCancelHook(taskId: string): AnyOnCancelHookFunction | undefined {
+    return undefined;
+  }
+
+  getGlobalCancelHooks(): RegisteredHookFunction<AnyOnCancelHookFunction>[] {
+    return [];
+  }
+
   registerOnWaitHookListener(listener: (wait: TaskWait) => Promise<void>): void {
     // Noop
   }

packages/core/src/v3/lifecycleHooks/types.ts

@@ -7,7 +7,7 @@ export type TaskInitHookParams<TPayload = unknown> = {
   ctx: TaskRunContext;
   payload: TPayload;
   task: string;
-  signal?: AbortSignal;
+  signal: AbortSignal;
 };
 
 export type OnInitHookFunction<TPayload, TInitOutput extends TaskInitOutput> = (
@@ -23,7 +23,7 @@ export type TaskStartHookParams<
   ctx: TaskRunContext;
   payload: TPayload;
   task: string;
-  signal?: AbortSignal;
+  signal: AbortSignal;
   init?: TInitOutput;
 };
 
@@ -60,7 +60,7 @@ export type TaskWaitHookParams<
   ctx: TaskRunContext;
   payload: TPayload;
   task: string;
-  signal?: AbortSignal;
+  signal: AbortSignal;
   init?: TInitOutput;
 };
 
@@ -78,7 +78,7 @@ export type TaskResumeHookParams<
   wait: TaskWait;
   payload: TPayload;
   task: string;
-  signal?: AbortSignal;
+  signal: AbortSignal;
   init?: TInitOutput;
 };
 
@@ -96,7 +96,7 @@ export type TaskFailureHookParams<
   payload: TPayload;
   task: string;
   error: unknown;
-  signal?: AbortSignal;
+  signal: AbortSignal;
   init?: TInitOutput;
 };
 
@@ -115,7 +115,7 @@ export type TaskSuccessHookParams<
   payload: TPayload;
   task: string;
   output: TOutput;
-  signal?: AbortSignal;
+  signal: AbortSignal;
   init?: TInitOutput;
 };
 
@@ -152,7 +152,7 @@ export type TaskCompleteHookParams<
   payload: TPayload;
   task: string;
   result: TaskCompleteResult<TOutput>;
-  signal?: AbortSignal;
+  signal: AbortSignal;
   init?: TInitOutput;
 };
 
@@ -188,7 +188,7 @@ export type TaskCatchErrorHookParams<
   retry?: RetryOptions;
   retryAt?: Date;
   retryDelayInMs?: number;
-  signal?: AbortSignal;
+  signal: AbortSignal;
   init?: TInitOutput;
 };
 
@@ -203,7 +203,7 @@ export type TaskMiddlewareHookParams<TPayload = unknown> = {
   ctx: TaskRunContext;
   payload: TPayload;
   task: string;
-  signal?: AbortSignal;
+  signal: AbortSignal;
   next: () => Promise<void>;
 };
 
@@ -220,7 +220,7 @@ export type TaskCleanupHookParams<
   ctx: TaskRunContext;
   payload: TPayload;
   task: string;
-  signal?: AbortSignal;
+  signal: AbortSignal;
   init?: TInitOutput;
 };
 
@@ -230,6 +230,29 @@ export type OnCleanupHookFunction<TPayload, TInitOutput extends TaskInitOutput =
 
 export type AnyOnCleanupHookFunction = OnCleanupHookFunction<unknown, TaskInitOutput>;
 
+export type TaskCancelHookParams<
+  TPayload = unknown,
+  TRunOutput = any,
+  TInitOutput extends TaskInitOutput = TaskInitOutput,
+> = {
+  ctx: TaskRunContext;
+  payload: TPayload;
+  task: string;
+  runPromise: Promise<TRunOutput>;
+  init?: TInitOutput;
+  signal: AbortSignal;
+};
+
+export type OnCancelHookFunction<
+  TPayload,
+  TRunOutput = any,
+  TInitOutput extends TaskInitOutput = TaskInitOutput,
+> = (
+  params: TaskCancelHookParams<TPayload, TRunOutput, TInitOutput>
+) => undefined | void | Promise<undefined | void>;
+
+export type AnyOnCancelHookFunction = OnCancelHookFunction<unknown, unknown, TaskInitOutput>;
+
 export interface LifecycleHooksManager {
   registerGlobalInitHook(hook: RegisterHookFunctionParams<AnyOnInitHookFunction>): void;
   registerTaskInitHook(
@@ -307,4 +330,15 @@ export interface LifecycleHooksManager {
 
   callOnResumeHookListeners(wait: TaskWait): Promise<void>;
   registerOnResumeHookListener(listener: (wait: TaskWait) => Promise<void>): void;
+
+  registerGlobalCancelHook(hook: RegisterHookFunctionParams<AnyOnCancelHookFunction>): void;
+  registerTaskCancelHook(
+    taskId: string,
+    hook: RegisterHookFunctionParams<AnyOnCancelHookFunction>
+  ): void;
+  getGlobalCancelHooks(): RegisteredHookFunction<AnyOnCancelHookFunction>[];
+  getTaskCancelHook(taskId: string): AnyOnCancelHookFunction | undefined;
+
+  registerOnCancelHookListener(listener: () => Promise<void>): void;
+  callOnCancelHookListeners(): Promise<void>;
 }

packages/core/src/v3/schemas/messages.ts

@@ -208,6 +208,12 @@ export const WorkerToExecutorMessageCatalog = {
     }),
     callback: z.void(),
   },
+  CANCEL: {
+    message: z.object({
+      timeoutInMs: z.number(),
+    }),
+    callback: z.void(),
+  },
   RESOLVE_WAITPOINT: {
     message: z.object({
       version: z.literal("v1").default("v1"),

packages/core/src/v3/timeout/api.ts

@@ -4,8 +4,8 @@ import { TimeoutManager } from "./types.js";
 const API_NAME = "timeout";
 
 class NoopTimeoutManager implements TimeoutManager {
-  abortAfterTimeout(timeoutInSeconds: number): AbortSignal {
-    return new AbortController().signal;
+  abortAfterTimeout(timeoutInSeconds?: number): AbortController {
+    return new AbortController();
   }
 }
 
@@ -25,11 +25,11 @@ export class TimeoutAPI implements TimeoutManager {
   }
 
   public get signal(): AbortSignal | undefined {
-    return this.#getManagerManager().signal;
+    return this.#getManager().signal;
   }
 
-  public abortAfterTimeout(timeoutInSeconds: number): AbortSignal {
-    return this.#getManagerManager().abortAfterTimeout(timeoutInSeconds);
+  public abortAfterTimeout(timeoutInSeconds?: number): AbortController {
+    return this.#getManager().abortAfterTimeout(timeoutInSeconds);
   }
 
   public setGlobalManager(manager: TimeoutManager): boolean {
@@ -40,7 +40,7 @@ export class TimeoutAPI implements TimeoutManager {
     unregisterGlobal(API_NAME);
   }
 
-  #getManagerManager(): TimeoutManager {
+  #getManager(): TimeoutManager {
     return getGlobal(API_NAME) ?? NOOP_TIMEOUT_MANAGER;
   }
 }

packages/core/src/v3/timeout/types.ts

@@ -1,5 +1,5 @@
 export interface TimeoutManager {
-  abortAfterTimeout: (timeoutInSeconds: number) => AbortSignal;
+  abortAfterTimeout: (timeoutInSeconds?: number) => AbortController;
   signal?: AbortSignal;
 }
 

packages/core/src/v3/timeout/usageTimeoutManager.ts

@@ -4,6 +4,7 @@ import { TaskRunExceededMaxDuration, TimeoutManager } from "./types.js";
 export class UsageTimeoutManager implements TimeoutManager {
   private _abortController: AbortController;
   private _abortSignal: AbortSignal | undefined;
+  private _intervalId: NodeJS.Timeout | undefined;
 
   constructor(private readonly usageManager: UsageManager) {
     this._abortController = new AbortController();
@@ -13,15 +14,23 @@ export class UsageTimeoutManager implements TimeoutManager {
     return this._abortSignal;
   }
 
-  abortAfterTimeout(timeoutInSeconds: number): AbortSignal {
+  abortAfterTimeout(timeoutInSeconds?: number): AbortController {
     this._abortSignal = this._abortController.signal;
 
+    if (!timeoutInSeconds) {
+      return this._abortController;
+    }
+
+    if (this._intervalId) {
+      clearInterval(this._intervalId);
+    }
+
     // Now we need to start an interval that will measure usage and abort the signal if the usage is too high
-    const intervalId = setInterval(() => {
+    this._intervalId = setInterval(() => {
       const sample = this.usageManager.sample();
       if (sample) {
         if (sample.cpuTime > timeoutInSeconds * 1000) {
-          clearInterval(intervalId);
+          clearInterval(this._intervalId);
 
           this._abortController.abort(
             new TaskRunExceededMaxDuration(timeoutInSeconds, sample.cpuTime / 1000)
@@ -30,6 +39,6 @@ export class UsageTimeoutManager implements TimeoutManager {
       }
     }, 1000);
 
-    return this._abortSignal;
+    return this._abortController;
   }
 }

packages/core/src/v3/types/tasks.ts

@@ -12,6 +12,7 @@ import {
   OnStartHookFunction,
   OnSuccessHookFunction,
   OnWaitHookFunction,
+  OnCancelHookFunction,
 } from "../lifecycleHooks/types.js";
 import { RunTags } from "../schemas/api.js";
 import {
@@ -88,28 +89,36 @@ export type RunFnParams<TInitOutput extends InitOutput> = Prettify<{
   ctx: Context;
   /** If you use the `init` function, this will be whatever you returned. */
   init?: TInitOutput;
-  /** Abort signal that is aborted when a task run exceeds it's maxDuration. Can be used to automatically cancel downstream requests */
-  signal?: AbortSignal;
+  /** Abort signal that is aborted when a task run exceeds it's maxDuration or if the task run is cancelled. Can be used to automatically cancel downstream requests */
+  signal: AbortSignal;
 }>;
 
 export type MiddlewareFnParams = Prettify<{
   ctx: Context;
   next: () => Promise<void>;
-  /** Abort signal that is aborted when a task run exceeds it's maxDuration. Can be used to automatically cancel downstream requests */
-  signal?: AbortSignal;
+  /** Abort signal that is aborted when a task run exceeds it's maxDuration or if the task run is cancelled. Can be used to automatically cancel downstream requests */
+  signal: AbortSignal;
 }>;
 
 export type InitFnParams = Prettify<{
   ctx: Context;
-  /** Abort signal that is aborted when a task run exceeds it's maxDuration. Can be used to automatically cancel downstream requests */
-  signal?: AbortSignal;
+  /** Abort signal that is aborted when a task run exceeds it's maxDuration or if the task run is cancelled. Can be used to automatically cancel downstream requests */
+  signal: AbortSignal;
 }>;
 
 export type StartFnParams = Prettify<{
   ctx: Context;
   init?: InitOutput;
-  /** Abort signal that is aborted when a task run exceeds it's maxDuration. Can be used to automatically cancel downstream requests */
-  signal?: AbortSignal;
+  /** Abort signal that is aborted when a task run exceeds it's maxDuration or if the task run is cancelled. Can be used to automatically cancel downstream requests */
+  signal: AbortSignal;
+}>;
+
+export type CancelFnParams = Prettify<{
+  ctx: Context;
+  /** Abort signal that is aborted when a task run exceeds it's maxDuration or if the task run is cancelled. Can be used to automatically cancel downstream requests */
+  signal: AbortSignal;
+  runPromise: Promise<unknown>;
+  init?: InitOutput;
 }>;
 
 export type Context = TaskRunContext;
@@ -296,6 +305,7 @@ type CommonTaskOptions<
   onResume?: OnResumeHookFunction<TPayload>;
   onWait?: OnWaitHookFunction<TPayload>;
   onComplete?: OnCompleteHookFunction<TPayload, TOutput>;
+  onCancel?: OnCancelHookFunction<TPayload, TOutput, TInitOutput>;
 
   /**
    * middleware allows you to run code "around" the run function. This can be useful for logging, metrics, or other cross-cutting concerns.

packages/core/src/v3/usage-api.ts

@@ -3,3 +3,5 @@
 import { UsageAPI } from "./usage/api.js";
 /** Entrypoint for usage API */
 export const usage = UsageAPI.getInstance();
+
+export type { UsageMeasurement, UsageSample } from "./usage/types.js";

packages/core/src/v3/usage/devUsageManager.ts

@@ -74,7 +74,9 @@ export class DevUsageManager implements UsageManager {
 
     const sample = measurement.sample();
 
-    this._currentMeasurements.delete(measurement.id);
+    if (this._currentMeasurements.has(measurement.id)) {
+      this._currentMeasurements.delete(measurement.id);
+    }
 
     return sample;
   }

packages/core/src/v3/workers/taskExecutor.ts

@@ -1,4 +1,4 @@
-import { SpanKind } from "@opentelemetry/api";
+import { Context, context, SpanKind, trace } from "@opentelemetry/api";
 import { VERSION } from "../../version.js";
 import { ApiError, RateLimitError } from "../apiClient/errors.js";
 import { ConsoleInterceptor } from "../consoleInterceptor.js";
@@ -51,6 +51,7 @@ import {
   stringifyIO,
 } from "../utils/ioSerialization.js";
 import { calculateNextRetryDelay } from "../utils/retries.js";
+import { promiseWithResolvers } from "../../utils.js";
 
 export type TaskExecutorOptions = {
   tracingSDK: TracingSDK;
@@ -90,7 +91,7 @@ export class TaskExecutor {
     execution: TaskRunExecution,
     worker: ServerBackgroundWorker,
     traceContext: Record<string, unknown>,
-    signal?: AbortSignal,
+    signal: AbortSignal,
     isWarmStart?: boolean
   ): Promise<{ result: TaskRunExecutionResult }> {
     const ctx = TaskRunContext.parse(execution);
@@ -120,6 +121,8 @@ export class TaskExecutor {
     const result = await this._tracer.startActiveSpan(
       attemptMessage,
       async (span) => {
+        const attemptContext = context.active();
+
         return await this._consoleInterceptor.intercept(console, async () => {
           let parsedPayload: any;
           let initOutput: any;
@@ -150,6 +153,26 @@ export class TaskExecutor {
             await this.#callOnResumeFunctions(wait, parsedPayload, ctx, initOutput, signal);
           });
 
+          const {
+            promise: runPromise,
+            resolve: runResolve,
+            reject: runReject,
+          } = promiseWithResolvers<void>();
+
+          // Make sure the run promise does not cause unhandled promise rejections
+          runPromise.catch(() => {});
+
+          lifecycleHooks.registerOnCancelHookListener(async () => {
+            await this.#callOnCancelFunctions(
+              runPromise,
+              parsedPayload,
+              ctx,
+              initOutput,
+              signal,
+              attemptContext
+            );
+          });
+
           const executeTask = async (payload: any) => {
             const [runError, output] = await tryCatch(
               (async () => {
@@ -172,6 +195,8 @@ export class TaskExecutor {
             );
 
             if (runError) {
+              runReject(runError);
+
               const [handleErrorError, handleErrorResult] = await tryCatch(
                 this.#handleError(execution, runError, payload, ctx, initOutput, signal)
               );
@@ -220,6 +245,8 @@ export class TaskExecutor {
               } satisfies TaskRunExecutionResult;
             }
 
+            runResolve(output);
+
             const [outputError, stringifiedOutput] = await tryCatch(stringifyIO(output));
 
             if (outputError) {
@@ -336,7 +363,7 @@ export class TaskExecutor {
     execution: TaskRunExecution,
     hooks: RegisteredHookFunction<AnyOnMiddlewareHookFunction>[],
     executeTask: (payload: unknown) => Promise<TaskRunExecutionResult>,
-    signal?: AbortSignal
+    signal: AbortSignal
   ) {
     let output: any;
     let executeError: unknown;
@@ -384,7 +411,7 @@ export class TaskExecutor {
     return output;
   }
 
-  async #callRun(payload: unknown, ctx: TaskRunContext, init: unknown, signal?: AbortSignal) {
+  async #callRun(payload: unknown, ctx: TaskRunContext, init: unknown, signal: AbortSignal) {
     const runFn = this.task.fns.run;
 
     if (!runFn) {
@@ -392,30 +419,29 @@ export class TaskExecutor {
     }
 
     // Create a promise that rejects when the signal aborts
-    const abortPromise = signal
-      ? new Promise((_, reject) => {
-          signal.addEventListener("abort", () => {
-            const maxDuration = ctx.run.maxDuration;
-            reject(
-              new InternalError({
-                code: TaskRunErrorCodes.MAX_DURATION_EXCEEDED,
-                message: `Run exceeded maximum compute time (maxDuration) of ${maxDuration} seconds`,
-              })
-            );
-          });
-        })
-      : undefined;
+    const abortPromise = new Promise((_, reject) => {
+      signal.addEventListener("abort", () => {
+        if (typeof signal.reason === "string" && signal.reason.includes("cancel")) {
+          console.log("abortPromise: cancel");
+          return;
+        }
+
+        const maxDuration = ctx.run.maxDuration;
+        reject(
+          new InternalError({
+            code: TaskRunErrorCodes.MAX_DURATION_EXCEEDED,
+            message: `Run exceeded maximum compute time (maxDuration) of ${maxDuration} seconds`,
+          })
+        );
+      });
+    });
 
     return runTimelineMetrics.measureMetric("trigger.dev/execution", "run", async () => {
       return await this._tracer.startActiveSpan(
         "run()",
         async (span) => {
-          if (abortPromise) {
-            // Race between the run function and the abort promise
-            return await Promise.race([runFn(payload, { ctx, init, signal }), abortPromise]);
-          }
-
-          return await runFn(payload, { ctx, init, signal });
+          // Race between the run function and the abort promise
+          return await Promise.race([runFn(payload, { ctx, init, signal }), abortPromise]);
         },
         {
           attributes: { [SemanticInternalAttributes.STYLE_ICON]: "task-fn-run" },
@@ -429,7 +455,7 @@ export class TaskExecutor {
     payload: unknown,
     ctx: TaskRunContext,
     initOutput: TaskInitOutput,
-    signal?: AbortSignal
+    signal: AbortSignal
   ) {
     const globalWaitHooks = lifecycleHooks.getGlobalWaitHooks();
     const taskWaitHook = lifecycleHooks.getTaskWaitHook(this.task.id);
@@ -496,12 +522,94 @@ export class TaskExecutor {
     );
   }
 
+  async #callOnCancelFunctions(
+    runPromise: Promise<any>,
+    payload: unknown,
+    ctx: TaskRunContext,
+    initOutput: TaskInitOutput,
+    signal: AbortSignal,
+    attemptContext: Context
+  ) {
+    const globalCancelHooks = lifecycleHooks.getGlobalCancelHooks();
+    const taskCancelHook = lifecycleHooks.getTaskCancelHook(this.task.id);
+
+    if (globalCancelHooks.length === 0 && !taskCancelHook) {
+      return;
+    }
+
+    const result = await runTimelineMetrics.measureMetric(
+      "trigger.dev/execution",
+      "onCancel",
+      async () => {
+        for (const hook of globalCancelHooks) {
+          const [hookError] = await tryCatch(
+            this._tracer.startActiveSpan(
+              "onCancel()",
+              async (span) => {
+                await hook.fn({
+                  payload,
+                  ctx,
+                  signal,
+                  task: this.task.id,
+                  init: initOutput,
+                  runPromise,
+                });
+              },
+              {
+                attributes: {
+                  [SemanticInternalAttributes.STYLE_ICON]: "task-hook-onCancel",
+                  [SemanticInternalAttributes.COLLAPSED]: true,
+                  ...this.#lifecycleHookAccessoryAttributes(hook.name),
+                },
+              },
+              attemptContext
+            )
+          );
+
+          if (hookError) {
+            throw hookError;
+          }
+        }
+
+        if (taskCancelHook) {
+          const [hookError] = await tryCatch(
+            this._tracer.startActiveSpan(
+              "onCancel()",
+              async (span) => {
+                await taskCancelHook({
+                  payload,
+                  ctx,
+                  signal,
+                  task: this.task.id,
+                  init: initOutput,
+                  runPromise,
+                });
+              },
+              {
+                attributes: {
+                  [SemanticInternalAttributes.STYLE_ICON]: "task-hook-onCancel",
+                  [SemanticInternalAttributes.COLLAPSED]: true,
+                  ...this.#lifecycleHookAccessoryAttributes("task"),
+                },
+              },
+              attemptContext
+            )
+          );
+
+          if (hookError) {
+            throw hookError;
+          }
+        }
+      }
+    );
+  }
+
   async #callOnResumeFunctions(
     wait: TaskWait,
     payload: unknown,
     ctx: TaskRunContext,
     initOutput: TaskInitOutput,
-    signal?: AbortSignal
+    signal: AbortSignal
   ) {
     const globalResumeHooks = lifecycleHooks.getGlobalResumeHooks();
     const taskResumeHook = lifecycleHooks.getTaskResumeHook(this.task.id);
@@ -568,7 +676,7 @@ export class TaskExecutor {
     );
   }
 
-  async #callInitFunctions(payload: unknown, ctx: TaskRunContext, signal?: AbortSignal) {
+  async #callInitFunctions(payload: unknown, ctx: TaskRunContext, signal: AbortSignal) {
     const globalInitHooks = lifecycleHooks.getGlobalInitHooks();
     const taskInitHook = lifecycleHooks.getTaskInitHook(this.task.id);
 
@@ -671,7 +779,7 @@ export class TaskExecutor {
     output: any,
     ctx: TaskRunContext,
     initOutput: any,
-    signal?: AbortSignal
+    signal: AbortSignal
   ) {
     const globalSuccessHooks = lifecycleHooks.getGlobalSuccessHooks();
     const taskSuccessHook = lifecycleHooks.getTaskSuccessHook(this.task.id);
@@ -746,7 +854,7 @@ export class TaskExecutor {
     error: unknown,
     ctx: TaskRunContext,
     initOutput: any,
-    signal?: AbortSignal
+    signal: AbortSignal
   ) {
     const globalFailureHooks = lifecycleHooks.getGlobalFailureHooks();
     const taskFailureHook = lifecycleHooks.getTaskFailureHook(this.task.id);
@@ -832,7 +940,7 @@ export class TaskExecutor {
     payload: unknown,
     ctx: TaskRunContext,
     initOutput: any,
-    signal?: AbortSignal
+    signal: AbortSignal
   ) {
     const globalStartHooks = lifecycleHooks.getGlobalStartHooks();
     const taskStartHook = lifecycleHooks.getTaskStartHook(this.task.id);
@@ -898,7 +1006,7 @@ export class TaskExecutor {
     payload: unknown,
     ctx: TaskRunContext,
     initOutput: any,
-    signal?: AbortSignal
+    signal: AbortSignal
   ) {
     await this.#callCleanupFunctions(payload, ctx, initOutput, signal);
     await this.#blockForWaitUntil();
@@ -908,7 +1016,7 @@ export class TaskExecutor {
     payload: unknown,
     ctx: TaskRunContext,
     initOutput: any,
-    signal?: AbortSignal
+    signal: AbortSignal
   ) {
     const globalCleanupHooks = lifecycleHooks.getGlobalCleanupHooks();
     const taskCleanupHook = lifecycleHooks.getTaskCleanupHook(this.task.id);
@@ -1001,7 +1109,7 @@ export class TaskExecutor {
     payload: any,
     ctx: TaskRunContext,
     init: TaskInitOutput,
-    signal?: AbortSignal
+    signal: AbortSignal
   ): Promise<
     | { status: "retry"; retry: TaskRunExecutionRetry; error?: unknown }
     | { status: "skipped"; error?: unknown }
@@ -1191,7 +1299,7 @@ export class TaskExecutor {
     result: TaskCompleteResult<unknown>,
     ctx: TaskRunContext,
     initOutput: any,
-    signal?: AbortSignal
+    signal: AbortSignal
   ) {
     const globalCompleteHooks = lifecycleHooks.getGlobalCompleteHooks();
     const taskCompleteHook = lifecycleHooks.getTaskCompleteHook(this.task.id);

packages/core/test/taskExecutor.test.ts

@@ -1905,5 +1905,7 @@ function executeTask(
     engine: "V2",
   };
 
-  return executor.execute(execution, worker, {}, signal);
+  const $signal = signal ? signal : new AbortController().signal;
+
+  return executor.execute(execution, worker, {}, $signal);
 }

packages/python/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @trigger.dev/python
 
+## 4.0.0-v4-beta.12
+
+### Patch Changes
+
+- Updated dependencies:
+  - `@trigger.dev/sdk@4.0.0-v4-beta.12`
+  - `@trigger.dev/build@4.0.0-v4-beta.12`
+  - `@trigger.dev/core@4.0.0-v4-beta.12`
+
 ## 4.0.0-v4-beta.11
 
 ### Patch Changes

packages/python/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/python",
-  "version": "4.0.0-v4-beta.11",
+  "version": "4.0.0-v4-beta.12",
   "description": "Python runtime and build extension for Trigger.dev",
   "license": "MIT",
   "publishConfig": {
@@ -45,7 +45,7 @@
     "check-exports": "attw --pack ."
   },
   "dependencies": {
-    "@trigger.dev/core": "workspace:4.0.0-v4-beta.11",
+    "@trigger.dev/core": "workspace:4.0.0-v4-beta.12",
     "tinyexec": "^0.3.2"
   },
   "devDependencies": {
@@ -56,12 +56,12 @@
     "tsx": "4.17.0",
     "esbuild": "^0.23.0",
     "@arethetypeswrong/cli": "^0.15.4",
-    "@trigger.dev/build": "workspace:4.0.0-v4-beta.11",
-    "@trigger.dev/sdk": "workspace:4.0.0-v4-beta.11"
+    "@trigger.dev/build": "workspace:4.0.0-v4-beta.12",
+    "@trigger.dev/sdk": "workspace:4.0.0-v4-beta.12"
   },
   "peerDependencies": {
-    "@trigger.dev/sdk": "workspace:^4.0.0-v4-beta.11",
-    "@trigger.dev/build": "workspace:^4.0.0-v4-beta.11"
+    "@trigger.dev/sdk": "workspace:^4.0.0-v4-beta.12",
+    "@trigger.dev/build": "workspace:^4.0.0-v4-beta.12"
   },
   "engines": {
     "node": ">=18.20.0"

packages/react-hooks/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @trigger.dev/react-hooks
 
+## 4.0.0-v4-beta.12
+
+### Patch Changes
+
+- Updated dependencies:
+  - `@trigger.dev/core@4.0.0-v4-beta.12`
+
 ## 4.0.0-v4-beta.11
 
 ### Patch Changes

packages/react-hooks/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/react-hooks",
-  "version": "4.0.0-v4-beta.11",
+  "version": "4.0.0-v4-beta.12",
   "description": "trigger.dev react hooks",
   "license": "MIT",
   "publishConfig": {
@@ -37,7 +37,7 @@
     "check-exports": "attw --pack ."
   },
   "dependencies": {
-    "@trigger.dev/core": "workspace:^4.0.0-v4-beta.11",
+    "@trigger.dev/core": "workspace:^4.0.0-v4-beta.12",
     "swr": "^2.2.5"
   },
   "devDependencies": {

packages/redis-worker/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @trigger.dev/redis-worker
 
+## 4.0.0-v4-beta.12
+
+### Patch Changes
+
+- Updated dependencies:
+  - `@trigger.dev/core@4.0.0-v4-beta.12`
+
 ## 4.0.0-v4-beta.11
 
 ### Patch Changes

packages/redis-worker/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/redis-worker",
-  "version": "4.0.0-v4-beta.11",
+  "version": "4.0.0-v4-beta.12",
   "description": "Redis worker for trigger.dev",
   "license": "MIT",
   "publishConfig": {
@@ -23,7 +23,7 @@
     "test": "vitest --sequence.concurrent=false --no-file-parallelism"
   },
   "dependencies": {
-    "@trigger.dev/core": "workspace:4.0.0-v4-beta.11",
+    "@trigger.dev/core": "workspace:4.0.0-v4-beta.12",
     "lodash.omit": "^4.5.0",
     "nanoid": "^5.0.7",
     "p-limit": "^6.2.0",

packages/rsc/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @trigger.dev/rsc
 
+## 4.0.0-v4-beta.12
+
+### Patch Changes
+
+- Updated dependencies:
+  - `@trigger.dev/core@4.0.0-v4-beta.12`
+
 ## 4.0.0-v4-beta.11
 
 ### Patch Changes

packages/rsc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/rsc",
-  "version": "4.0.0-v4-beta.11",
+  "version": "4.0.0-v4-beta.12",
   "description": "trigger.dev rsc",
   "license": "MIT",
   "publishConfig": {
@@ -37,14 +37,14 @@
     "check-exports": "attw --pack ."
   },
   "dependencies": {
-    "@trigger.dev/core": "workspace:^4.0.0-v4-beta.11",
+    "@trigger.dev/core": "workspace:^4.0.0-v4-beta.12",
     "mlly": "^1.7.1",
     "react": "19.0.0-rc.1",
     "react-dom": "19.0.0-rc.1"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "^0.15.4",
-    "@trigger.dev/build": "workspace:^4.0.0-v4-beta.11",
+    "@trigger.dev/build": "workspace:^4.0.0-v4-beta.12",
     "@types/node": "^20.14.14",
     "@types/react": "*",
     "@types/react-dom": "*",

packages/trigger-sdk/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @trigger.dev/sdk
 
+## 4.0.0-v4-beta.12
+
+### Patch Changes
+
+- Display clickable links in Cursor terminal ([#1998](https://github.com/triggerdotdev/trigger.dev/pull/1998))
+- Add onCancel lifecycle hook ([#2022](https://github.com/triggerdotdev/trigger.dev/pull/2022))
+- Updated dependencies:
+  - `@trigger.dev/core@4.0.0-v4-beta.12`
+
 ## 4.0.0-v4-beta.11
 
 ### Patch Changes

packages/trigger-sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/sdk",
-  "version": "4.0.0-v4-beta.11",
+  "version": "4.0.0-v4-beta.12",
   "description": "trigger.dev Node.JS SDK",
   "license": "MIT",
   "publishConfig": {
@@ -52,7 +52,7 @@
     "@opentelemetry/api": "1.9.0",
     "@opentelemetry/api-logs": "0.52.1",
     "@opentelemetry/semantic-conventions": "1.25.1",
-    "@trigger.dev/core": "workspace:4.0.0-v4-beta.11",
+    "@trigger.dev/core": "workspace:4.0.0-v4-beta.12",
     "chalk": "^5.2.0",
     "cronstrue": "^2.21.0",
     "debug": "^4.3.4",

packages/trigger-sdk/src/v3/hooks.ts

@@ -11,6 +11,7 @@ import {
   type AnyOnResumeHookFunction,
   type AnyOnCatchErrorHookFunction,
   type AnyOnMiddlewareHookFunction,
+  type AnyOnCancelHookFunction,
 } from "@trigger.dev/core/v3";
 
 export type {
@@ -25,6 +26,7 @@ export type {
   AnyOnResumeHookFunction,
   AnyOnCatchErrorHookFunction,
   AnyOnMiddlewareHookFunction,
+  AnyOnCancelHookFunction,
 };
 
 export function onStart(name: string, fn: AnyOnStartHookFunction): void;
@@ -131,3 +133,15 @@ export function middleware(
     fn: typeof fnOrName === "function" ? fnOrName : fn!,
   });
 }
+
+export function onCancel(name: string, fn: AnyOnCancelHookFunction): void;
+export function onCancel(fn: AnyOnCancelHookFunction): void;
+export function onCancel(
+  fnOrName: string | AnyOnCancelHookFunction,
+  fn?: AnyOnCancelHookFunction
+): void {
+  lifecycleHooks.registerGlobalCancelHook({
+    id: typeof fnOrName === "string" ? fnOrName : fnOrName.name ? fnOrName.name : undefined,
+    fn: typeof fnOrName === "function" ? fnOrName : fn!,
+  });
+}

packages/trigger-sdk/src/v3/shared.ts

@@ -42,6 +42,7 @@ import type {
   AnyOnStartHookFunction,
   AnyOnSuccessHookFunction,
   AnyOnWaitHookFunction,
+  AnyOnCancelHookFunction,
   AnyRunHandle,
   AnyRunTypes,
   AnyTask,
@@ -1637,4 +1638,10 @@ function registerTaskLifecycleHooks<
       fn: params.cleanup as AnyOnCleanupHookFunction,
     });
   }
+
+  if (params.onCancel) {
+    lifecycleHooks.registerTaskCancelHook(taskId, {
+      fn: params.onCancel as AnyOnCancelHookFunction,
+    });
+  }
 }

packages/trigger-sdk/src/v3/tasks.ts

@@ -8,6 +8,7 @@ import {
   onHandleError,
   onCatchError,
   middleware,
+  onCancel,
 } from "./hooks.js";
 import {
   batchTrigger,
@@ -95,6 +96,7 @@ export const tasks = {
   onComplete,
   onWait,
   onResume,
+  onCancel,
   /** @deprecated Use catchError instead */
   handleError: onHandleError,
   catchError: onCatchError,

pnpm-lock.yaml

@@ -1006,17 +1006,20 @@ importers:
         version: 5.3.2
     devDependencies:
       '@testcontainers/postgresql':
-        specifier: ^10.13.1
-        version: 10.13.1
+        specifier: ^10.25.0
+        version: 10.25.0
       '@testcontainers/redis':
-        specifier: ^10.13.1
-        version: 10.13.1
+        specifier: ^10.25.0
+        version: 10.25.0
       '@trigger.dev/core':
         specifier: workspace:*
         version: link:../../packages/core
+      std-env:
+        specifier: ^3.9.0
+        version: 3.9.0
       testcontainers:
-        specifier: ^10.13.1
-        version: 10.13.1
+        specifier: ^10.25.0
+        version: 10.25.0
       tinyexec:
         specifier: ^0.3.0
         version: 0.3.0
@@ -1077,7 +1080,7 @@ importers:
   packages/build:
     dependencies:
       '@trigger.dev/core':
-        specifier: workspace:4.0.0-v4-beta.11
+        specifier: workspace:4.0.0-v4-beta.12
         version: link:../core
       pkg-types:
         specifier: ^1.1.3
@@ -1153,10 +1156,10 @@ importers:
         specifier: 1.25.1
         version: 1.25.1
       '@trigger.dev/build':
-        specifier: workspace:4.0.0-v4-beta.11
+        specifier: workspace:4.0.0-v4-beta.12
         version: link:../build
       '@trigger.dev/core':
-        specifier: workspace:4.0.0-v4-beta.11
+        specifier: workspace:4.0.0-v4-beta.12
         version: link:../core
       ansi-escapes:
         specifier: ^7.0.0
@@ -1488,7 +1491,7 @@ importers:
   packages/python:
     dependencies:
       '@trigger.dev/core':
-        specifier: workspace:4.0.0-v4-beta.11
+        specifier: workspace:4.0.0-v4-beta.12
         version: link:../core
       tinyexec:
         specifier: ^0.3.2
@@ -1498,10 +1501,10 @@ importers:
         specifier: ^0.15.4
         version: 0.15.4
       '@trigger.dev/build':
-        specifier: workspace:4.0.0-v4-beta.11
+        specifier: workspace:4.0.0-v4-beta.12
         version: link:../build
       '@trigger.dev/sdk':
-        specifier: workspace:4.0.0-v4-beta.11
+        specifier: workspace:4.0.0-v4-beta.12
         version: link:../trigger-sdk
       '@types/node':
         specifier: 20.14.14
@@ -1525,7 +1528,7 @@ importers:
   packages/react-hooks:
     dependencies:
       '@trigger.dev/core':
-        specifier: workspace:^4.0.0-v4-beta.11
+        specifier: workspace:^4.0.0-v4-beta.12
         version: link:../core
       react:
         specifier: ^18.0 || ^19.0 || ^19.0.0-rc
@@ -1559,7 +1562,7 @@ importers:
   packages/redis-worker:
     dependencies:
       '@trigger.dev/core':
-        specifier: workspace:4.0.0-v4-beta.11
+        specifier: workspace:4.0.0-v4-beta.12
         version: link:../core
       lodash.omit:
         specifier: ^4.5.0
@@ -1605,7 +1608,7 @@ importers:
   packages/rsc:
     dependencies:
       '@trigger.dev/core':
-        specifier: workspace:^4.0.0-v4-beta.11
+        specifier: workspace:^4.0.0-v4-beta.12
         version: link:../core
       mlly:
         specifier: ^1.7.1
@@ -1621,7 +1624,7 @@ importers:
         specifier: ^0.15.4
         version: 0.15.4
       '@trigger.dev/build':
-        specifier: workspace:^4.0.0-v4-beta.11
+        specifier: workspace:^4.0.0-v4-beta.12
         version: link:../build
       '@types/node':
         specifier: ^20.14.14
@@ -1654,7 +1657,7 @@ importers:
         specifier: 1.25.1
         version: 1.25.1
       '@trigger.dev/core':
-        specifier: workspace:4.0.0-v4-beta.11
+        specifier: workspace:4.0.0-v4-beta.12
         version: link:../core
       chalk:
         specifier: ^5.2.0
@@ -8075,7 +8078,6 @@ packages:
     dependencies:
       '@grpc/proto-loader': 0.7.13
       '@js-sdsl/ordered-map': 4.4.2
-    dev: false
 
   /@grpc/grpc-js@1.8.17:
     resolution: {integrity: sha512-DGuSbtMFbaRsyffMf+VEkVu8HkSXEUfO3UyGJNtqxW9ABdtTIA+2UXAJpwbJS+xfQxuwqLUeELmL6FuZkOqPxw==}
@@ -8093,7 +8095,6 @@ packages:
       long: 5.2.3
       protobufjs: 7.3.2
       yargs: 17.7.2
-    dev: false
 
   /@grpc/proto-loader@0.7.7:
     resolution: {integrity: sha512-1TIeXOi8TuSCQprPItwoMymZXxWT0CPxUhkrkeCUH+D8U7QDwQ6b7SUz2MaLuWM2llT+J/TVFLmQI5KtML3BhQ==}
@@ -8697,7 +8698,6 @@ packages:
 
   /@js-sdsl/ordered-map@4.4.2:
     resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==}
-    dev: false
 
   /@jsep-plugin/assignment@1.3.0(jsep@1.4.0):
     resolution: {integrity: sha512-VVgV+CXrhbMI3aSusQyclHkenWSAm95WaiKrMxRFam3JSUiIaQjoMIw2sEs/OX4XifnqeQUN4DYbJjlA8EfktQ==}
@@ -17796,19 +17796,21 @@ packages:
       zod: 3.23.8
     dev: false
 
-  /@testcontainers/postgresql@10.13.1:
-    resolution: {integrity: sha512-HAh/3uLAzAhOmzXsOE6hVxkvetczPnX/Zoyt+SgK7QotW98Npr1MDx8OKiaLGTJ8XkIvVvS4Ch6bl+frt4pnkQ==}
+  /@testcontainers/postgresql@10.25.0:
+    resolution: {integrity: sha512-VkpqpX9YZ8aq4wfk6sJRopGTmlBdE1kErzAFWJ/1pY/XrEZ7nxdfFBG+En2icQnbv3BIFQYysEKxEFMNB+hQVw==}
     dependencies:
-      testcontainers: 10.13.1
+      testcontainers: 10.25.0
     transitivePeerDependencies:
+      - bare-buffer
       - supports-color
     dev: true
 
-  /@testcontainers/redis@10.13.1:
-    resolution: {integrity: sha512-pXg15o4oTRaEyb5xryQZUdePtoRId/+3TeU7vnUgDpqOmRacF8/7zL7jqs13uPh1uea6M7a8MDgHQM8j8kXZUg==}
+  /@testcontainers/redis@10.25.0:
+    resolution: {integrity: sha512-ALNrrnYnB59kV5c/EjiUkzn0roCtcnOu2KfHHF8xBi3vq3dYSqzADL8rL2BExeoFhyaEtlUT9P4ZecRB60O+/Q==}
     dependencies:
-      testcontainers: 10.13.1
+      testcontainers: 10.25.0
     transitivePeerDependencies:
+      - bare-buffer
       - supports-color
     dev: true
 
@@ -20359,6 +20361,12 @@ packages:
   /bare-events@2.4.2:
     resolution: {integrity: sha512-qMKFd2qG/36aA4GwvKq8MxnPgCQAmBWmSyLWsJcbn8v03wvIPQ/hG1Ms8bPzndZxMDoHpxez5VOS+gC9Yi24/Q==}
     requiresBuild: true
+    dev: false
+    optional: true
+
+  /bare-events@2.5.4:
+    resolution: {integrity: sha512-+gFfDkR8pj4/TrWCGUGWmJIkBwuxPS5F+a5yWjOHQt2hHvNZd5YLzadjmDUtFmMM4y429bnKLa8bYBMHcYdnQA==}
+    requiresBuild: true
     optional: true
 
   /bare-fs@2.3.5:
@@ -20368,11 +20376,36 @@ packages:
       bare-events: 2.4.2
       bare-path: 2.1.3
       bare-stream: 2.3.0
+    dev: false
+    optional: true
+
+  /bare-fs@4.1.4:
+    resolution: {integrity: sha512-r8+26Voz8dGX3AYpJdFb1ZPaUSM8XOLCZvy+YGpRTmwPHIxA7Z3Jov/oMPtV7hfRQbOnH8qGlLTzQAbgtdNN0Q==}
+    engines: {bare: '>=1.16.0'}
+    requiresBuild: true
+    peerDependencies:
+      bare-buffer: '*'
+    peerDependenciesMeta:
+      bare-buffer:
+        optional: true
+    dependencies:
+      bare-events: 2.5.4
+      bare-path: 3.0.0
+      bare-stream: 2.6.5(bare-events@2.5.4)
+    dev: true
     optional: true
 
   /bare-os@2.4.4:
     resolution: {integrity: sha512-z3UiI2yi1mK0sXeRdc4O1Kk8aOa/e+FNWZcTiPB/dfTWyLypuE99LibgRaQki914Jq//yAWylcAt+mknKdixRQ==}
     requiresBuild: true
+    dev: false
+    optional: true
+
+  /bare-os@3.6.1:
+    resolution: {integrity: sha512-uaIjxokhFidJP+bmmvKSgiMzj2sV5GPHaZVAIktcxcpCyBFFWO+YlikVAdhmUo2vYFvFhOXIAlldqV29L8126g==}
+    engines: {bare: '>=1.14.0'}
+    requiresBuild: true
+    dev: true
     optional: true
 
   /bare-path@2.1.3:
@@ -20380,6 +20413,15 @@ packages:
     requiresBuild: true
     dependencies:
       bare-os: 2.4.4
+    dev: false
+    optional: true
+
+  /bare-path@3.0.0:
+    resolution: {integrity: sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==}
+    requiresBuild: true
+    dependencies:
+      bare-os: 3.6.1
+    dev: true
     optional: true
 
   /bare-stream@2.3.0:
@@ -20388,6 +20430,24 @@ packages:
     dependencies:
       b4a: 1.6.6
       streamx: 2.20.1
+    dev: false
+    optional: true
+
+  /bare-stream@2.6.5(bare-events@2.5.4):
+    resolution: {integrity: sha512-jSmxKJNJmHySi6hC42zlZnq00rga4jjxcgNZjY9N5WlOe/iOoGRtdwGsHzQv2RlH2KOYMwGUXhf2zXd32BA9RA==}
+    requiresBuild: true
+    peerDependencies:
+      bare-buffer: '*'
+      bare-events: '*'
+    peerDependenciesMeta:
+      bare-buffer:
+        optional: true
+      bare-events:
+        optional: true
+    dependencies:
+      bare-events: 2.5.4
+      streamx: 2.22.0
+    dev: true
     optional: true
 
   /base64-js@1.5.1:
@@ -22192,19 +22252,7 @@ packages:
     resolution: {integrity: sha512-plizRs/Vf15H+GCVxq2EUvyPK7ei9b/cVesHvjnX4xaXjM9spHe2Ytq0BitndFgvTJ3E3NljPNUEl7BAN43iZw==}
     engines: {node: '>= 6.0.0'}
     dependencies:
-      yaml: 2.3.1
-    dev: true
-
-  /docker-modem@3.0.8:
-    resolution: {integrity: sha512-f0ReSURdM3pcKPNS30mxOHSbaFLcknGmQjwSfmbcdOw1XWKXVhukM3NJHhr7NpY9BIyyWQb0EBo3KQvvuU5egQ==}
-    engines: {node: '>= 8.0'}
-    dependencies:
-      debug: 4.4.0(supports-color@10.0.0)
-      readable-stream: 3.6.0
-      split-ca: 1.0.1
-      ssh2: 1.16.0
-    transitivePeerDependencies:
-      - supports-color
+      yaml: 2.7.1
     dev: true
 
   /docker-modem@5.0.6:
@@ -22217,18 +22265,6 @@ packages:
       ssh2: 1.16.0
     transitivePeerDependencies:
       - supports-color
-    dev: false
-
-  /dockerode@3.3.5:
-    resolution: {integrity: sha512-/0YNa3ZDNeLr/tSckmD69+Gq+qVNhvKfAHNeZJBnp7EOP6RGKV8ORrJHkUn20So5wU+xxT7+1n5u8PjHbfjbSA==}
-    engines: {node: '>= 8.0'}
-    dependencies:
-      '@balena/dockerignore': 1.0.2
-      docker-modem: 3.0.8
-      tar-fs: 2.0.1
-    transitivePeerDependencies:
-      - supports-color
-    dev: true
 
   /dockerode@4.0.4:
     resolution: {integrity: sha512-6GYP/EdzEY50HaOxTVTJ2p+mB5xDHTMJhS+UoGrVyS6VC+iQRh7kZ4FRpUYq6nziby7hPqWhOrFFUFTMUZJJ5w==}
@@ -22245,6 +22281,21 @@ packages:
       - supports-color
     dev: false
 
+  /dockerode@4.0.6:
+    resolution: {integrity: sha512-FbVf3Z8fY/kALB9s+P9epCpWhfi/r0N2DgYYcYpsAUlaTxPjdsitsFobnltb+lyCgAIvf9C+4PSWlTnHlJMf1w==}
+    engines: {node: '>= 8.0'}
+    dependencies:
+      '@balena/dockerignore': 1.0.2
+      '@grpc/grpc-js': 1.12.6
+      '@grpc/proto-loader': 0.7.13
+      docker-modem: 5.0.6
+      protobufjs: 7.3.2
+      tar-fs: 2.1.2
+      uuid: 10.0.0
+    transitivePeerDependencies:
+      - supports-color
+    dev: true
+
   /doctrine@2.1.0:
     resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==}
     engines: {node: '>=0.10.0'}
@@ -24603,6 +24654,11 @@ packages:
     resolution: {integrity: sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==}
     engines: {node: '>=8'}
 
+  /get-port@7.1.0:
+    resolution: {integrity: sha512-QB9NKEeDg3xxVwCCwJQ9+xycaz6pBB6iQ76wiWMl1927n0Kir6alPiP+yuiICLLU4jpMe08dXfpebuQppFA2zw==}
+    engines: {node: '>=16'}
+    dev: true
+
   /get-proto@1.0.1:
     resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==}
     engines: {node: '>= 0.4'}
@@ -32570,6 +32626,10 @@ packages:
   /std-env@3.8.1:
     resolution: {integrity: sha512-vj5lIj3Mwf9D79hBkltk5qmkFI+biIKWS2IBxEyEU3AX1tUf7AoL8nSazCOiiqQsGKIq01SClsKEzweu34uwvA==}
 
+  /std-env@3.9.0:
+    resolution: {integrity: sha512-UGvjygr6F6tpH7o2qyqR6QYpwraIjKSdtzyBdyytFOHmPZY917kwdwLG0RbOjWOnKmnm3PeHjaoLLMie7kPLQw==}
+    dev: true
+
   /stoppable@1.1.0:
     resolution: {integrity: sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==}
     engines: {node: '>=4', npm: '>=6'}
@@ -32605,7 +32665,18 @@ packages:
       queue-tick: 1.0.1
       text-decoder: 1.2.0
     optionalDependencies:
-      bare-events: 2.4.2
+      bare-events: 2.5.4
+
+  /streamx@2.22.0:
+    resolution: {integrity: sha512-sLh1evHOzBy/iWRiR6d1zRcLao4gGZr3C1kzNz4fopCOKJb6xD9ub8Mpi9Mr1R6id5o43S+d93fI48UC5uM9aw==}
+    requiresBuild: true
+    dependencies:
+      fast-fifo: 1.3.2
+      text-decoder: 1.2.0
+    optionalDependencies:
+      bare-events: 2.5.4
+    dev: true
+    optional: true
 
   /strict-event-emitter@0.5.1:
     resolution: {integrity: sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==}
@@ -33203,6 +33274,7 @@ packages:
       mkdirp-classic: 0.5.3
       pump: 3.0.0
       tar-stream: 2.2.0
+    dev: false
 
   /tar-fs@2.1.1:
     resolution: {integrity: sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==}
@@ -33213,6 +33285,15 @@ packages:
       tar-stream: 2.2.0
     dev: true
 
+  /tar-fs@2.1.2:
+    resolution: {integrity: sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==}
+    dependencies:
+      chownr: 1.1.4
+      mkdirp-classic: 0.5.3
+      pump: 3.0.0
+      tar-stream: 2.2.0
+    dev: true
+
   /tar-fs@3.0.6:
     resolution: {integrity: sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w==}
     dependencies:
@@ -33221,6 +33302,19 @@ packages:
     optionalDependencies:
       bare-fs: 2.3.5
       bare-path: 2.1.3
+    dev: false
+
+  /tar-fs@3.0.8:
+    resolution: {integrity: sha512-ZoROL70jptorGAlgAYiLoBLItEKw/fUxg9BSYK/dF/GAGYFJOJJJMvjPAKDJraCXFwadD456FCuvLWgfhMsPwg==}
+    dependencies:
+      pump: 3.0.0
+      tar-stream: 3.1.7
+    optionalDependencies:
+      bare-fs: 4.1.4
+      bare-path: 3.0.0
+    transitivePeerDependencies:
+      - bare-buffer
+    dev: true
 
   /tar-stream@2.2.0:
     resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==}
@@ -33380,25 +33474,26 @@ packages:
       minimatch: 9.0.5
     dev: true
 
-  /testcontainers@10.13.1:
-    resolution: {integrity: sha512-JBbOhxmygj/ouH/47GnoVNt+c55Telh/45IjVxEbDoswsLchVmJiuKiw/eF6lE5i7LN+/99xsrSCttI3YRtirg==}
+  /testcontainers@10.25.0:
+    resolution: {integrity: sha512-X3x6cjorEMgei1vVx3M7dnTMzWoWOTi4krpUf3C2iOvOcwsaMUHbca9J4yzpN65ieiWhcK2dA5dxpZyUonwC2Q==}
     dependencies:
       '@balena/dockerignore': 1.0.2
       '@types/dockerode': 3.3.35
       archiver: 7.0.1
       async-lock: 1.4.1
       byline: 5.0.0
-      debug: 4.3.7(supports-color@10.0.0)
+      debug: 4.4.0(supports-color@10.0.0)
       docker-compose: 0.24.8
-      dockerode: 3.3.5
-      get-port: 5.1.1
+      dockerode: 4.0.6
+      get-port: 7.1.0
       proper-lockfile: 4.1.2
       properties-reader: 2.3.0
       ssh-remote-port-forward: 1.0.4
-      tar-fs: 3.0.6
+      tar-fs: 3.0.8
       tmp: 0.2.3
-      undici: 5.28.4
+      undici: 5.29.0
     transitivePeerDependencies:
+      - bare-buffer
       - supports-color
     dev: true
 
@@ -34341,6 +34436,14 @@ packages:
     engines: {node: '>=14.0'}
     dependencies:
       '@fastify/busboy': 2.0.0
+    dev: false
+
+  /undici@5.29.0:
+    resolution: {integrity: sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==}
+    engines: {node: '>=14.0'}
+    dependencies:
+      '@fastify/busboy': 2.0.0
+    dev: true
 
   /unenv-nightly@1.10.0-1717606461.a117952:
     resolution: {integrity: sha512-u3TfBX02WzbHTpaEfWEKwDijDSFAHcgXkayUZ+MVDrjhLFvgAJzFGTSTmwlEhwWi2exyRQey23ah9wELMM6etg==}
@@ -34758,7 +34861,6 @@ packages:
   /uuid@10.0.0:
     resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==}
     hasBin: true
-    dev: false
 
   /uuid@3.4.0:
     resolution: {integrity: sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==}
@@ -35274,7 +35376,7 @@ packages:
       magic-string: 0.30.17
       pathe: 1.1.2
       picocolors: 1.1.1
-      std-env: 3.8.1
+      std-env: 3.9.0
       strip-literal: 2.1.0
       tinybench: 2.9.0
       tinypool: 0.8.3

references/d3-chat/src/trigger/chat.ts

@@ -1,7 +1,7 @@
 import { anthropic } from "@ai-sdk/anthropic";
 import { openai } from "@ai-sdk/openai";
 import { ai } from "@trigger.dev/sdk/ai";
-import { logger, metadata, schemaTask, wait } from "@trigger.dev/sdk/v3";
+import { logger, metadata, schemaTask, tasks, wait } from "@trigger.dev/sdk/v3";
 import { sql } from "@vercel/postgres";
 import { streamText, TextStreamPart, tool } from "ai";
 import { nanoid } from "nanoid";
@@ -110,7 +110,7 @@ export const todoChat = schemaTask({
       ),
     userId: z.string(),
   }),
-  run: async ({ input, userId }) => {
+  run: async ({ input, userId }, { signal }) => {
     metadata.set("user_id", userId);
 
     const system = `
@@ -157,6 +157,8 @@ export const todoChat = schemaTask({
 
     const prompt = input;
 
+    const chunks: TextStreamPart<TOOLS>[] = [];
+
     const result = streamText({
       model: getModel(),
       system,
@@ -174,6 +176,10 @@ export const todoChat = schemaTask({
       experimental_telemetry: {
         isEnabled: true,
       },
+      abortSignal: signal,
+      onChunk: ({ chunk }) => {
+        chunks.push(chunk);
+      },
     });
 
     const stream = await metadata.stream("fullStream", result.fullStream);
@@ -213,3 +219,49 @@ function getModel() {
     return anthropic("claude-3-5-sonnet-latest");
   }
 }
+
+export const interruptibleChat = schemaTask({
+  id: "interruptible-chat",
+  description: "Chat with the AI",
+  schema: z.object({
+    prompt: z.string().describe("The prompt to chat with the AI"),
+  }),
+  run: async ({ prompt }, { signal }) => {
+    const chunks: TextStreamPart<{}>[] = [];
+
+    // 👇 This is a global onCancel hook, but it's inside of the run function
+    tasks.onCancel(async () => {
+      // We have access to the chunks here
+      logger.info("interruptible-chat: task cancelled with chunks", { chunks });
+    });
+
+    try {
+      const result = streamText({
+        model: getModel(),
+        prompt,
+        experimental_telemetry: {
+          isEnabled: true,
+        },
+        tools: {},
+        abortSignal: signal,
+        onChunk: ({ chunk }) => {
+          chunks.push(chunk);
+        },
+      });
+
+      const textParts = [];
+
+      for await (const part of result.textStream) {
+        textParts.push(part);
+      }
+
+      return textParts.join("");
+    } catch (error) {
+      if (error instanceof Error && error.name === "AbortError") {
+        // streamText will throw an AbortError if the signal is aborted, so we can handle it here
+      } else {
+        throw error;
+      }
+    }
+  },
+});

references/hello-world/src/trigger/example.ts

@@ -1,4 +1,4 @@
-import { batch, logger, task, timeout, wait } from "@trigger.dev/sdk";
+import { batch, logger, task, tasks, timeout, wait } from "@trigger.dev/sdk";
 import { setTimeout } from "timers/promises";
 import { ResourceMonitor } from "../resourceMonitor.js";
 
@@ -207,6 +207,54 @@ export const hooksTask = task({
   cleanup: async ({ ctx, payload }) => {
     logger.info("Hello, world from the cleanup hook", { payload });
   },
+  onCancel: async ({ payload }) => {
+    logger.info("Hello, world from the onCancel hook", { payload });
+  },
+});
+
+export const cancelExampleTask = task({
+  id: "cancel-example",
+  // Signal will be aborted when the task is cancelled 👇
+  run: async (payload: { timeoutInSeconds: number }, { signal }) => {
+    logger.info("Hello, world from the cancel task", {
+      timeoutInSeconds: payload.timeoutInSeconds,
+    });
+
+    // This is a global hook that will be called if the task is cancelled
+    tasks.onCancel(async () => {
+      logger.info("global task onCancel hook but inside of the run function baby!");
+    });
+
+    await logger.trace("timeout", async (span) => {
+      try {
+        // We pass the signal to setTimeout to abort the timeout if the task is cancelled
+        await setTimeout(payload.timeoutInSeconds * 1000, undefined, { signal });
+      } catch (error) {
+        // If the timeout is aborted, this error will be thrown, we can handle it here
+        logger.error("Timeout error", { error });
+      }
+    });
+
+    logger.info("Hello, world from the cancel task after the timeout", {
+      timeoutInSeconds: payload.timeoutInSeconds,
+    });
+
+    return {
+      message: "Hello, world!",
+    };
+  },
+  onCancel: async ({ payload, runPromise }) => {
+    logger.info("Hello, world from the onCancel hook", { payload });
+    // You can await the runPromise to get the output of the task
+    const output = await runPromise;
+
+    logger.info("Hello, world from the onCancel hook after the run", { payload, output });
+
+    // You can do work inside the onCancel hook, up to 30 seconds
+    await setTimeout(10_000);
+
+    logger.info("Hello, world from the onCancel hook after the timeout", { payload });
+  },
 });
 
 export const resourceMonitorTest = task({

references/hello-world/src/trigger/init.ts

@@ -6,6 +6,10 @@ tasks.middleware("db", ({ ctx, payload, next }) => {
   return next();
 });
 
+tasks.onCancel(async ({ ctx, payload }) => {
+  logger.info("Hello, world from the global cancel", { ctx, payload });
+});
+
 // tasks.onSuccess(({ ctx, payload, output }) => {
 //   logger.info("Hello, world from the success", { ctx, payload });
 // });