Release 3.0.0-beta.25

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.changeset/little-crabs-cross.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Fix issues with consecutive waits

.changeset/pre.json

@@ -67,6 +67,7 @@
     "lemon-jobs-repair",
     "light-bulldogs-press",
     "light-dragons-complain",
+    "little-crabs-cross",
     "loud-actors-remember",
     "many-ligers-pump",
     "mighty-camels-joke",
@@ -79,6 +80,7 @@
     "polite-rockets-matter",
     "poor-flowers-cross",
     "purple-garlics-shop",
+    "rare-lamps-promise",
     "rare-roses-float",
     "real-planets-stare",
     "rich-kangaroos-unite",
@@ -90,6 +92,7 @@
     "sharp-zebras-serve",
     "shiny-coats-cry",
     "silly-suits-switch",
+    "six-ligers-exist",
     "slow-buses-own",
     "smart-needles-move",
     "smart-olives-eat",
@@ -101,6 +104,7 @@
     "swift-dragons-peel",
     "tall-bees-wave",
     "tame-guests-know",
+    "tender-moose-tell",
     "tender-oranges-rhyme",
     "tidy-balloons-suffer",
     "tidy-dryers-sleep",

.changeset/rare-lamps-promise.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Add a postInstall option to allow running scripts after dependencies have been installed in deployed images

.changeset/six-ligers-exist.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Improve the display of non-object return types in the run trace viewer

.changeset/tender-moose-tell.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Use locked package versions when resolving dependencies in deployed workers

apps/coordinator/src/index.ts

@@ -157,16 +157,18 @@ class Checkpointer {
     return this.#abortControllers.has(runId);
   }
 
-  cancelCheckpoint(runId: string) {
+  cancelCheckpoint(runId: string): boolean {
     const controller = this.#abortControllers.get(runId);
 
     if (!controller) {
       logger.debug("Nothing to cancel", { runId });
-      return;
+      return false;
     }
 
     controller.abort("cancelCheckpointing()");
     this.#abortControllers.delete(runId);
+
+    return true;
   }
 
   async #checkpointAndPush({
@@ -725,10 +727,18 @@ class TaskCoordinator {
           checkpointable.resolve();
         });
 
-        socket.on("CANCEL_CHECKPOINT", async (message) => {
+        socket.on("CANCEL_CHECKPOINT", async (message, callback) => {
           logger.log("[CANCEL_CHECKPOINT]", message);
 
-          this.#cancelCheckpoint(socket.data.runId);
+          if (message.version === "v1") {
+            this.#cancelCheckpoint(socket.data.runId);
+            // v1 has no callback
+            return;
+          }
+
+          const checkpointCanceled = this.#cancelCheckpoint(socket.data.runId);
+
+          callback({ version: "v2", checkpointCanceled });
         });
 
         socket.on("WAIT_FOR_DURATION", async (message, callback) => {
@@ -933,7 +943,9 @@ class TaskCoordinator {
     }
 
     // Cancel checkpointing procedure
-    this.#checkpointer.cancelCheckpoint(runId);
+    const checkpointCanceled = this.#checkpointer.cancelCheckpoint(runId);
+
+    return checkpointCanceled;
   }
 
   #createHttpServer() {

apps/kubernetes-provider/src/index.ts

@@ -212,6 +212,7 @@ class KubernetesTaskOperations implements TaskOperations {
             {
               name: "populate-taskinfo",
               image: "docker.io/library/busybox",
+              imagePullPolicy: "IfNotPresent",
               command: ["/bin/sh", "-c"],
               args: ["printenv COORDINATOR_HOST | tee /etc/taskinfo/coordinator-host"],
               env: [

apps/webapp/.gitignore

@@ -17,4 +17,5 @@ build-storybook.log
 .storybook-out
 storybook-static
 
-/prisma/seed.js
+/prisma/seed.js
+/prisma/populate.js

apps/webapp/app/components/admin/debugTooltip.tsx

@@ -0,0 +1,27 @@
+import { ShieldCheckIcon } from "@heroicons/react/20/solid";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger,
+} from "~/components/primitives/Tooltip";
+import { useHasAdminAccess } from "~/hooks/useUser";
+
+export function AdminDebugTooltip({ children }: { children: React.ReactNode }) {
+  const hasAdminAccess = useHasAdminAccess();
+
+  if (!hasAdminAccess) {
+    return null;
+  }
+
+  return (
+    <TooltipProvider>
+      <Tooltip>
+        <TooltipTrigger>
+          <ShieldCheckIcon className="h-5 w-5" />
+        </TooltipTrigger>
+        <TooltipContent className="flex items-center gap-1">{children}</TooltipContent>
+      </Tooltip>
+    </TooltipProvider>
+  );
+}

apps/webapp/app/components/primitives/TooltipPortal.tsx

@@ -0,0 +1,103 @@
+import type { VirtualElement as IVirtualElement } from "@popperjs/core";
+import { ReactNode, useEffect, useState } from "react";
+import { createPortal } from "react-dom";
+import { usePopper } from "react-popper";
+import { useEvent } from "react-use";
+import useLazyRef from "~/hooks/useLazyRef";
+
+// Recharts 3.x will have portal support, but until then we're using this:
+//https://github.com/recharts/recharts/issues/2458#issuecomment-1063463873
+
+export interface PopperPortalProps {
+  active?: boolean;
+  children: ReactNode;
+}
+
+export default function TooltipPortal({ active = true, children }: PopperPortalProps) {
+  const [portalElement, setPortalElement] = useState<HTMLDivElement>();
+  const [popperElement, setPopperElement] = useState<HTMLDivElement | null>();
+  const virtualElementRef = useLazyRef(() => new VirtualElement());
+
+  const { styles, attributes, update } = usePopper(
+    virtualElementRef.current,
+    popperElement,
+    POPPER_OPTIONS
+  );
+
+  useEffect(() => {
+    const el = document.createElement("div");
+    document.body.appendChild(el);
+    setPortalElement(el);
+    return () => el.remove();
+  }, []);
+
+  useEvent("mousemove", ({ clientX: x, clientY: y }) => {
+    virtualElementRef.current?.update(x, y);
+    if (!active) return;
+    update?.();
+  });
+
+  useEffect(() => {
+    if (!active) return;
+    update?.();
+  }, [active, update]);
+
+  if (!portalElement) return null;
+
+  return createPortal(
+    <div
+      ref={setPopperElement}
+      {...attributes.popper}
+      style={{
+        ...styles.popper,
+        zIndex: 1000,
+        display: active ? "block" : "none",
+      }}
+    >
+      {children}
+    </div>,
+    portalElement
+  );
+}
+
+class VirtualElement implements IVirtualElement {
+  private rect = {
+    width: 0,
+    height: 0,
+    top: 0,
+    right: 0,
+    bottom: 0,
+    left: 0,
+    x: 0,
+    y: 0,
+    toJSON() {
+      return this;
+    },
+  };
+
+  update(x: number, y: number) {
+    this.rect.y = y;
+    this.rect.top = y;
+    this.rect.bottom = y;
+
+    this.rect.x = x;
+    this.rect.left = x;
+    this.rect.right = x;
+  }
+
+  getBoundingClientRect(): DOMRect {
+    return this.rect;
+  }
+}
+
+const POPPER_OPTIONS: Parameters<typeof usePopper>[2] = {
+  placement: "right-start",
+  modifiers: [
+    {
+      name: "offset",
+      options: {
+        offset: [8, 8],
+      },
+    },
+  ],
+};

apps/webapp/app/components/primitives/TreeView/TreeView.tsx

@@ -1,10 +1,9 @@
 import { VirtualItem, Virtualizer, useVirtualizer } from "@tanstack/react-virtual";
 import { motion } from "framer-motion";
 import { MutableRefObject, RefObject, useCallback, useEffect, useReducer, useRef } from "react";
-import { UnmountClosed } from "react-collapse";
 import { cn } from "~/utils/cn";
 import { NodeState, NodesState, reducer } from "./reducer";
-import { applyFilterToState, concreteStateFromInput, selectedIdFromState } from "./utils";
+import { concreteStateFromInput, selectedIdFromState } from "./utils";
 
 export type TreeViewProps<TData> = {
   tree: FlatTree<TData>;
@@ -104,23 +103,22 @@ export function TreeView<TData>({
             if (!node) return null;
             const state = nodes[node.id];
             if (!state) return null;
+            if (!state.visible) return null;
             return (
               <div
                 key={node.id}
                 data-index={virtualItem.index}
                 ref={virtualizer.measureElement}
-                className="overflow-clip [&_.ReactCollapse--collapse]:transition-all"
+                className="overflow-clip"
                 {...getNodeProps(node.id)}
               >
-                <UnmountClosed key={node.id} isOpened={state.visible}>
-                  {renderNode({
-                    node,
-                    state,
-                    index: virtualItem.index,
-                    virtualizer: virtualizer,
-                    virtualItem,
-                  })}
-                </UnmountClosed>
+                {renderNode({
+                  node,
+                  state,
+                  index: virtualItem.index,
+                  virtualizer: virtualizer,
+                  virtualItem,
+                })}
               </div>
             );
           })}
@@ -130,19 +128,23 @@ export function TreeView<TData>({
   );
 }
 
-type TreeStateHookProps<TData> = {
+export type Filter<TData, TFilterValue> = {
+  value?: TFilterValue;
+  fn: (value: TFilterValue, node: FlatTreeItem<TData>) => boolean;
+};
+
+type TreeStateHookProps<TData, TFilterValue> = {
   tree: FlatTree<TData>;
   selectedId?: string;
   collapsedIds?: string[];
   onSelectedIdChanged?: (selectedId: string | undefined) => void;
-  onCollapsedIdsChanged?: (collapsedIds: string[]) => void;
   estimatedRowHeight: (params: {
     node: FlatTreeItem<TData>;
     state: NodeState;
     index: number;
   }) => number;
   parentRef: RefObject<any>;
-  filter?: (node: FlatTreeItem<TData>) => boolean;
+  filter?: Filter<TData, TFilterValue>;
 };
 
 //this is so Framer Motion can be used to render the components
@@ -178,24 +180,24 @@ export type UseTreeStateOutput = {
   scrollToNode: (id: string) => void;
 };
 
-export function useTree<TData>({
+export function useTree<TData, TFilterValue>({
   tree,
   selectedId,
   collapsedIds,
   onSelectedIdChanged,
-  onCollapsedIdsChanged,
   parentRef,
   estimatedRowHeight,
   filter,
-}: TreeStateHookProps<TData>): UseTreeStateOutput {
+}: TreeStateHookProps<TData, TFilterValue>): UseTreeStateOutput {
   const previousNodeCount = useRef(tree.length);
   const previousSelectedId = useRef<string | undefined>(selectedId);
 
   const [state, dispatch] = useReducer(
     reducer,
-    concreteStateFromInput({ tree, selectedId, collapsedIds })
+    concreteStateFromInput({ tree, selectedId, collapsedIds, filter })
   );
 
+  //fire onSelectedIdChanged()
   useEffect(() => {
     const selectedId = selectedIdFromState(state.nodes);
     if (selectedId !== previousSelectedId.current) {
@@ -204,12 +206,7 @@ export function useTree<TData>({
     }
   }, [state.changes.selectedId]);
 
-  useEffect(() => {
-    if (state.changes.collapsedIds) {
-      onCollapsedIdsChanged?.(state.changes.collapsedIds);
-    }
-  }, [state.changes.collapsedIds]);
-
+  //update tree when the number of nodes changes
   useEffect(() => {
     if (tree.length !== previousNodeCount.current) {
       previousNodeCount.current = tree.length;
@@ -217,9 +214,25 @@ export function useTree<TData>({
     }
   }, [previousNodeCount.current, tree.length]);
 
+  //update the filter, if it's changed
+  const previousFilter = useRef(filter);
+  useEffect(() => {
+    //check if the value (not reference) of the filter is the same
+    const previousValue = previousFilter.current
+      ? JSON.stringify(previousFilter.current.value)
+      : undefined;
+    const newValue = filter ? JSON.stringify(filter.value) : undefined;
+
+    previousFilter.current = filter;
+
+    if (previousValue !== newValue) {
+      dispatch({ type: "UPDATE_FILTER", payload: { filter } });
+    }
+  }, [filter?.value]);
+
   const virtualizer = useVirtualizer({
-    count: tree.length,
-    getItemKey: (index) => tree[index].id,
+    count: state.visibleNodeIds.length,
+    getItemKey: (index) => state.visibleNodeIds[index],
     getScrollElement: () => parentRef.current,
     estimateSize: (index: number) => {
       return estimatedRowHeight({
@@ -233,7 +246,7 @@ export function useTree<TData>({
 
   const scrollToNodeFn = useCallback(
     (id: string) => {
-      const itemIndex = tree.findIndex((node) => node.id === id);
+      const itemIndex = state.visibleNodeIds.findIndex((n) => n === id);
 
       if (itemIndex !== -1) {
         virtualizer.scrollToIndex(itemIndex, { align: "auto" });
@@ -269,21 +282,21 @@ export function useTree<TData>({
 
   const expandNode = useCallback(
     (id: string, scrollToNode = true) => {
-      dispatch({ type: "EXPAND_NODE", payload: { id, tree, scrollToNode, scrollToNodeFn } });
+      dispatch({ type: "EXPAND_NODE", payload: { id, scrollToNode, scrollToNodeFn } });
     },
     [state]
   );
 
   const collapseNode = useCallback(
     (id: string) => {
-      dispatch({ type: "COLLAPSE_NODE", payload: { id, tree } });
+      dispatch({ type: "COLLAPSE_NODE", payload: { id } });
     },
     [state]
   );
 
   const toggleExpandNode = useCallback(
     (id: string, scrollToNode = true) => {
-      dispatch({ type: "TOGGLE_EXPAND_NODE", payload: { id, tree, scrollToNode, scrollToNodeFn } });
+      dispatch({ type: "TOGGLE_EXPAND_NODE", payload: { id, scrollToNode, scrollToNodeFn } });
     },
     [state]
   );
@@ -292,7 +305,7 @@ export function useTree<TData>({
     (scrollToNode = true) => {
       dispatch({
         type: "SELECT_FIRST_VISIBLE_NODE",
-        payload: { tree, scrollToNode, scrollToNodeFn },
+        payload: { scrollToNode, scrollToNodeFn },
       });
     },
     [tree, state]
@@ -302,7 +315,7 @@ export function useTree<TData>({
     (scrollToNode = true) => {
       dispatch({
         type: "SELECT_LAST_VISIBLE_NODE",
-        payload: { tree, scrollToNode, scrollToNodeFn },
+        payload: { scrollToNode, scrollToNodeFn },
       });
     },
     [tree, state]
@@ -312,7 +325,7 @@ export function useTree<TData>({
     (scrollToNode = true) => {
       dispatch({
         type: "SELECT_NEXT_VISIBLE_NODE",
-        payload: { tree, scrollToNode, scrollToNodeFn },
+        payload: { scrollToNode, scrollToNodeFn },
       });
     },
     [state]
@@ -322,7 +335,7 @@ export function useTree<TData>({
     (scrollToNode = true) => {
       dispatch({
         type: "SELECT_PREVIOUS_VISIBLE_NODE",
-        payload: { tree, scrollToNode, scrollToNodeFn },
+        payload: { scrollToNode, scrollToNodeFn },
       });
     },
     [state]
@@ -332,7 +345,7 @@ export function useTree<TData>({
     (scrollToNode = true) => {
       dispatch({
         type: "SELECT_PARENT_NODE",
-        payload: { tree, scrollToNode, scrollToNodeFn },
+        payload: { scrollToNode, scrollToNodeFn },
       });
     },
     [state]
@@ -340,35 +353,35 @@ export function useTree<TData>({
 
   const expandAllBelowDepth = useCallback(
     (depth: number) => {
-      dispatch({ type: "EXPAND_ALL_BELOW_DEPTH", payload: { tree, depth } });
+      dispatch({ type: "EXPAND_ALL_BELOW_DEPTH", payload: { depth } });
     },
     [state]
   );
 
   const collapseAllBelowDepth = useCallback(
     (depth: number) => {
-      dispatch({ type: "COLLAPSE_ALL_BELOW_DEPTH", payload: { tree, depth } });
+      dispatch({ type: "COLLAPSE_ALL_BELOW_DEPTH", payload: { depth } });
     },
     [state]
   );
 
   const expandLevel = useCallback(
     (level: number) => {
-      dispatch({ type: "EXPAND_LEVEL", payload: { tree, level } });
+      dispatch({ type: "EXPAND_LEVEL", payload: { level } });
     },
     [state]
   );
 
   const collapseLevel = useCallback(
     (level: number) => {
-      dispatch({ type: "COLLAPSE_LEVEL", payload: { tree, level } });
+      dispatch({ type: "COLLAPSE_LEVEL", payload: { level } });
     },
     [state]
   );
 
   const toggleExpandLevel = useCallback(
     (level: number) => {
-      dispatch({ type: "TOGGLE_EXPAND_LEVEL", payload: { tree, level } });
+      dispatch({ type: "TOGGLE_EXPAND_LEVEL", payload: { level } });
     },
     [state]
   );
@@ -480,7 +493,7 @@ export function useTree<TData>({
 
   return {
     selected: selectedIdFromState(state.nodes),
-    nodes: filter ? applyFilterToState(tree, state.nodes, filter) : state.nodes,
+    nodes: state.nodes,
     getTreeProps,
     getNodeProps,
     selectNode,

apps/webapp/app/components/primitives/TreeView/reducer.ts

@@ -1,5 +1,7 @@
-import { FlatTree } from "./TreeView";
+import assertNever from "assert-never";
+import { Filter, FlatTree } from "./TreeView";
 import {
+  applyFilterToState,
   applyVisibility,
   collapsedIdsFromState,
   concreteStateFromInput,
@@ -18,12 +20,15 @@ export type NodeState = {
 
 export type Changes = {
   selectedId: string | undefined;
-  collapsedIds: string[] | undefined;
 };
 
 export type TreeState = {
+  tree: FlatTree<any>;
   nodes: NodesState;
+  filteredNodes: NodesState;
   changes: Changes;
+  filter: Filter<any, any> | undefined;
+  visibleNodeIds: string[];
 };
 
 export type NodesState = Record<string, NodeState>;
@@ -71,7 +76,6 @@ type ExpandNodeAction = {
   type: "EXPAND_NODE";
   payload: {
     id: string;
-    tree: FlatTree<any>;
   } & WithScrollToNode;
 };
 
@@ -79,7 +83,6 @@ type CollapseNodeAction = {
   type: "COLLAPSE_NODE";
   payload: {
     id: string;
-    tree: FlatTree<any>;
   };
 };
 
@@ -87,7 +90,6 @@ type ToggleExpandNodeAction = {
   type: "TOGGLE_EXPAND_NODE";
   payload: {
     id: string;
-    tree: FlatTree<any>;
   } & WithScrollToNode;
 };
 
@@ -95,7 +97,6 @@ type ExpandAllBelowDepthAction = {
   type: "EXPAND_ALL_BELOW_DEPTH";
   payload: {
     depth: number;
-    tree: FlatTree<any>;
   };
 };
 
@@ -103,7 +104,6 @@ type CollapseAllBelowDepthAction = {
   type: "COLLAPSE_ALL_BELOW_DEPTH";
   payload: {
     depth: number;
-    tree: FlatTree<any>;
   };
 };
 
@@ -111,7 +111,6 @@ type ExpandLevelAction = {
   type: "EXPAND_LEVEL";
   payload: {
     level: number;
-    tree: FlatTree<any>;
   };
 };
 
@@ -119,7 +118,6 @@ type CollapseLevelAction = {
   type: "COLLAPSE_LEVEL";
   payload: {
     level: number;
-    tree: FlatTree<any>;
   };
 };
 
@@ -127,43 +125,39 @@ type ToggleExpandLevelAction = {
   type: "TOGGLE_EXPAND_LEVEL";
   payload: {
     level: number;
-    tree: FlatTree<any>;
   };
 };
 
 type SelectFirstVisibleNodeAction = {
   type: "SELECT_FIRST_VISIBLE_NODE";
-  payload: {
-    tree: FlatTree<any>;
-  } & WithScrollToNode;
+  payload: {} & WithScrollToNode;
 };
 
 type SelectLastVisibleNodeAction = {
   type: "SELECT_LAST_VISIBLE_NODE";
-  payload: {
-    tree: FlatTree<any>;
-  } & WithScrollToNode;
+  payload: {} & WithScrollToNode;
 };
 
 type SelectNextVisibleNodeAction = {
   type: "SELECT_NEXT_VISIBLE_NODE";
-  payload: {
-    tree: FlatTree<any>;
-  } & WithScrollToNode;
+  payload: {} & WithScrollToNode;
 };
 
 type SelectPreviousVisibleNodeAction = {
   type: "SELECT_PREVIOUS_VISIBLE_NODE";
-  payload: {
-    tree: FlatTree<any>;
-  } & WithScrollToNode;
+  payload: {} & WithScrollToNode;
 };
 
 type SelectParentNodeAction = {
   type: "SELECT_PARENT_NODE";
+  payload: {} & WithScrollToNode;
+};
+
+type UpdateFilterAction = {
+  type: "UPDATE_FILTER";
   payload: {
-    tree: FlatTree<any>;
-  } & WithScrollToNode;
+    filter: Filter<any, any> | undefined;
+  };
 };
 
 export type Action =
@@ -184,7 +178,8 @@ export type Action =
   | SelectLastVisibleNodeAction
   | SelectNextVisibleNodeAction
   | SelectPreviousVisibleNodeAction
-  | SelectParentNodeAction;
+  | SelectParentNodeAction
+  | UpdateFilterAction;
 
 export function reducer(state: TreeState, action: Action): TreeState {
   switch (action.type) {
@@ -204,7 +199,12 @@ export function reducer(state: TreeState, action: Action): TreeState {
         action.payload.scrollToNodeFn(action.payload.id);
       }
 
-      return { nodes: newNodes, changes: generateChanges(state.nodes, newNodes) };
+      return applyFilterToState({
+        ...state,
+        tree: state.tree,
+        nodes: newNodes,
+        changes: generateChanges(state.nodes, newNodes),
+      });
     }
     case "DESELECT_NODE": {
       const nodes = {
@@ -212,28 +212,36 @@ export function reducer(state: TreeState, action: Action): TreeState {
         [action.payload.id]: { ...state.nodes[action.payload.id], selected: false },
       };
 
-      return { nodes, changes: generateChanges(state.nodes, nodes) };
+      return applyFilterToState({
+        ...state,
+        nodes,
+        changes: generateChanges(state.nodes, nodes),
+      });
     }
     case "DESELECT_ALL_NODES": {
       const nodes = Object.fromEntries(
         Object.entries(state.nodes).map(([key, value]) => [key, { ...value, selected: false }])
       );
-      return { nodes, changes: generateChanges(state.nodes, nodes) };
+      return applyFilterToState({
+        ...state,
+        nodes,
+        changes: generateChanges(state.nodes, nodes),
+      });
     }
     case "TOGGLE_NODE_SELECTION": {
       const currentlySelected = state.nodes[action.payload.id]?.selected ?? false;
       if (currentlySelected) {
         return reducer(state, { type: "DESELECT_NODE", payload: { id: action.payload.id } });
-      } else {
-        return reducer(state, {
-          type: "SELECT_NODE",
-          payload: {
-            id: action.payload.id,
-            scrollToNode: action.payload.scrollToNode,
-            scrollToNodeFn: action.payload.scrollToNodeFn,
-          },
-        });
       }
+
+      return reducer(state, {
+        type: "SELECT_NODE",
+        payload: {
+          id: action.payload.id,
+          scrollToNode: action.payload.scrollToNode,
+          scrollToNodeFn: action.payload.scrollToNodeFn,
+        },
+      });
     }
     case "EXPAND_NODE": {
       const newNodes = {
@@ -245,37 +253,44 @@ export function reducer(state: TreeState, action: Action): TreeState {
         action.payload.scrollToNodeFn(action.payload.id);
       }
 
-      const visibleNodes = applyVisibility(action.payload.tree, newNodes);
-      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+      const visibleNodes = applyVisibility(state.tree, newNodes);
+      return applyFilterToState({
+        ...state,
+        nodes: visibleNodes,
+        changes: generateChanges(state.nodes, visibleNodes),
+      });
     }
     case "COLLAPSE_NODE": {
-      const visibleNodes = applyVisibility(action.payload.tree, {
+      const visibleNodes = applyVisibility(state.tree, {
         ...state.nodes,
         [action.payload.id]: { ...state.nodes[action.payload.id], expanded: false },
       });
-      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+      return applyFilterToState({
+        ...state,
+        nodes: visibleNodes,
+        changes: generateChanges(state.nodes, visibleNodes),
+      });
     }
     case "TOGGLE_EXPAND_NODE": {
       const currentlyExpanded = state.nodes[action.payload.id]?.expanded ?? true;
       if (currentlyExpanded) {
         return reducer(state, {
           type: "COLLAPSE_NODE",
-          payload: { id: action.payload.id, tree: action.payload.tree },
-        });
-      } else {
-        return reducer(state, {
-          type: "EXPAND_NODE",
-          payload: {
-            id: action.payload.id,
-            tree: action.payload.tree,
-            scrollToNode: action.payload.scrollToNode,
-            scrollToNodeFn: action.payload.scrollToNodeFn,
-          },
+          payload: { id: action.payload.id },
         });
       }
+
+      return reducer(state, {
+        type: "EXPAND_NODE",
+        payload: {
+          id: action.payload.id,
+          scrollToNode: action.payload.scrollToNode,
+          scrollToNodeFn: action.payload.scrollToNodeFn,
+        },
+      });
     }
     case "EXPAND_ALL_BELOW_DEPTH": {
-      const nodesToExpand = action.payload.tree.filter(
+      const nodesToExpand = state.tree.filter(
         (n) => n.level >= action.payload.depth && n.hasChildren
       );
 
@@ -289,11 +304,15 @@ export function reducer(state: TreeState, action: Action): TreeState {
         ])
       );
 
-      const visibleNodes = applyVisibility(action.payload.tree, newNodes);
-      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+      const visibleNodes = applyVisibility(state.tree, newNodes);
+      return applyFilterToState({
+        ...state,
+        nodes: visibleNodes,
+        changes: generateChanges(state.nodes, visibleNodes),
+      });
     }
     case "COLLAPSE_ALL_BELOW_DEPTH": {
-      const nodesToCollapse = action.payload.tree.filter(
+      const nodesToCollapse = state.tree.filter(
         (n) => n.level >= action.payload.depth && n.hasChildren
       );
 
@@ -307,11 +326,15 @@ export function reducer(state: TreeState, action: Action): TreeState {
         ])
       );
 
-      const visibleNodes = applyVisibility(action.payload.tree, newNodes);
-      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+      const visibleNodes = applyVisibility(state.tree, newNodes);
+      return applyFilterToState({
+        ...state,
+        nodes: visibleNodes,
+        changes: generateChanges(state.nodes, visibleNodes),
+      });
     }
     case "EXPAND_LEVEL": {
-      const nodesToExpand = action.payload.tree.filter(
+      const nodesToExpand = state.tree.filter(
         (n) => n.level <= action.payload.level && n.hasChildren
       );
 
@@ -325,11 +348,15 @@ export function reducer(state: TreeState, action: Action): TreeState {
         ])
       );
 
-      const visibleNodes = applyVisibility(action.payload.tree, newNodes);
-      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+      const visibleNodes = applyVisibility(state.tree, newNodes);
+      return applyFilterToState({
+        ...state,
+        nodes: visibleNodes,
+        changes: generateChanges(state.nodes, visibleNodes),
+      });
     }
     case "COLLAPSE_LEVEL": {
-      const nodesToCollapse = action.payload.tree.filter(
+      const nodesToCollapse = state.tree.filter(
         (n) => n.level === action.payload.level && n.hasChildren
       );
 
@@ -343,13 +370,17 @@ export function reducer(state: TreeState, action: Action): TreeState {
         ])
       );
 
-      const visibleNodes = applyVisibility(action.payload.tree, newNodes);
-      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+      const visibleNodes = applyVisibility(state.tree, newNodes);
+      return applyFilterToState({
+        ...state,
+        nodes: visibleNodes,
+        changes: generateChanges(state.nodes, visibleNodes),
+      });
     }
     case "TOGGLE_EXPAND_LEVEL": {
       //first get the first item at that level in the tree. If it is expanded, collapse all nodes at that level
       //if it is collapsed, expand all nodes at that level
-      const nodesAtLevel = action.payload.tree.filter(
+      const nodesAtLevel = state.tree.filter(
         (n) => n.level === action.payload.level && n.hasChildren
       );
       const firstNode = nodesAtLevel[0];
@@ -364,21 +395,19 @@ export function reducer(state: TreeState, action: Action): TreeState {
           type: "COLLAPSE_LEVEL",
           payload: {
             level: action.payload.level,
-            tree: action.payload.tree,
-          },
-        });
-      } else {
-        return reducer(state, {
-          type: "EXPAND_LEVEL",
-          payload: {
-            level: action.payload.level,
-            tree: action.payload.tree,
           },
         });
       }
+
+      return reducer(state, {
+        type: "EXPAND_LEVEL",
+        payload: {
+          level: action.payload.level,
+        },
+      });
     }
     case "SELECT_FIRST_VISIBLE_NODE": {
-      const node = firstVisibleNode(action.payload.tree, state.nodes);
+      const node = firstVisibleNode(state.tree, state.filteredNodes);
       if (node) {
         return reducer(state, {
           type: "SELECT_NODE",
@@ -389,9 +418,11 @@ export function reducer(state: TreeState, action: Action): TreeState {
           },
         });
       }
+
+      return state;
     }
     case "SELECT_LAST_VISIBLE_NODE": {
-      const node = lastVisibleNode(action.payload.tree, state.nodes);
+      const node = lastVisibleNode(state.tree, state.filteredNodes);
       if (node) {
         return reducer(state, {
           type: "SELECT_NODE",
@@ -402,6 +433,8 @@ export function reducer(state: TreeState, action: Action): TreeState {
           },
         });
       }
+
+      return state;
     }
     case "SELECT_NEXT_VISIBLE_NODE": {
       const selected = selectedIdFromState(state.nodes);
@@ -409,14 +442,13 @@ export function reducer(state: TreeState, action: Action): TreeState {
         return reducer(state, {
           type: "SELECT_FIRST_VISIBLE_NODE",
           payload: {
-            tree: action.payload.tree,
             scrollToNode: action.payload.scrollToNode,
             scrollToNodeFn: action.payload.scrollToNodeFn,
           },
         });
       }
 
-      const visible = visibleNodes(action.payload.tree, state.nodes);
+      const visible = visibleNodes(state.tree, state.filteredNodes);
       const selectedIndex = visible.findIndex((node) => node.id === selected);
       const nextNode = visible[selectedIndex + 1];
       if (nextNode) {
@@ -429,6 +461,8 @@ export function reducer(state: TreeState, action: Action): TreeState {
           },
         });
       }
+
+      return state;
     }
     case "SELECT_PREVIOUS_VISIBLE_NODE": {
       const selected = selectedIdFromState(state.nodes);
@@ -437,16 +471,15 @@ export function reducer(state: TreeState, action: Action): TreeState {
         return reducer(state, {
           type: "SELECT_FIRST_VISIBLE_NODE",
           payload: {
-            tree: action.payload.tree,
             scrollToNode: action.payload.scrollToNode,
             scrollToNodeFn: action.payload.scrollToNodeFn,
           },
         });
       }
 
-      const visible = visibleNodes(action.payload.tree, state.nodes);
+      const visible = visibleNodes(state.tree, state.filteredNodes);
       const selectedIndex = visible.findIndex((node) => node.id === selected);
-      const previousNode = visible[selectedIndex - 1];
+      const previousNode = visible[Math.max(0, selectedIndex - 1)];
       if (previousNode) {
         return reducer(state, {
           type: "SELECT_NODE",
@@ -467,19 +500,18 @@ export function reducer(state: TreeState, action: Action): TreeState {
         return reducer(state, {
           type: "SELECT_FIRST_VISIBLE_NODE",
           payload: {
-            tree: action.payload.tree,
             scrollToNode: action.payload.scrollToNode,
             scrollToNodeFn: action.payload.scrollToNodeFn,
           },
         });
       }
 
-      const selectedNode = action.payload.tree.find((node) => node.id === selected);
+      const selectedNode = state.tree.find((node) => node.id === selected);
       if (!selectedNode) {
         return state;
       }
 
-      const parentNode = action.payload.tree.find((node) => node.id === selectedNode.parentId);
+      const parentNode = state.tree.find((node) => node.id === selectedNode.parentId);
       if (parentNode) {
         return reducer(state, {
           type: "SELECT_NODE",
@@ -498,12 +530,23 @@ export function reducer(state: TreeState, action: Action): TreeState {
       const selectedId = selectedIdFromState(state.nodes);
       const collapsedIds = collapsedIdsFromState(state.nodes);
       const newState = concreteStateFromInput({
+        ...state,
         tree: action.payload.tree,
         selectedId,
         collapsedIds,
       });
       return newState;
     }
+    case "UPDATE_FILTER": {
+      const newState = applyFilterToState({
+        ...state,
+        filter: action.payload.filter,
+      });
+      return newState;
+    }
+    default: {
+      assertNever(action);
+    }
   }
 
   throw new Error(`Unhandled action type: ${(action as any).type}`);

apps/webapp/app/components/primitives/TreeView/utils.ts

@@ -1,4 +1,4 @@
-import { FlatTree, FlatTreeItem } from "./TreeView";
+import { Filter, FlatTree, FlatTreeItem } from "./TreeView";
 import { Changes, NodeState, NodesState, TreeState } from "./reducer";
 
 type PartialNodeState = Record<string, Partial<NodeState>>;
@@ -8,10 +8,12 @@ const defaultExpanded = true;
 
 export function concreteStateFromInput({
   tree,
+  filter,
   selectedId,
   collapsedIds,
 }: {
   tree: FlatTree<any>;
+  filter: Filter<any, any> | undefined;
   selectedId: string | undefined;
   collapsedIds: string[] | undefined;
 }): TreeState {
@@ -35,10 +37,15 @@ export function concreteStateFromInput({
       }
     }
   }
+  const nodes = concreteStateFromPartialState(tree, state);
 
   return {
-    nodes: concreteStateFromPartialState(tree, state),
-    changes: { selectedId, collapsedIds: [] },
+    tree,
+    nodes,
+    changes: { selectedId },
+    filter,
+    filteredNodes: nodes,
+    visibleNodeIds: visibleNodes(tree, nodes).map((node) => node.id),
   };
 }
 
@@ -82,28 +89,48 @@ export function selectedIdFromState(state: NodesState): string | undefined {
   return selected?.[0];
 }
 
-export function applyFilterToState<TData>(
-  tree: FlatTree<TData>,
-  inputNodes: NodesState,
-  filter: (node: FlatTreeItem<TData>) => boolean
-): NodesState {
+export function applyFilterToState<TData>({
+  tree,
+  nodes,
+  filter,
+  visibleNodeIds,
+  changes,
+}: TreeState): TreeState {
+  if (!filter || !filter.value) {
+    return {
+      tree,
+      nodes,
+      filteredNodes: nodes,
+      changes,
+      filter,
+      visibleNodeIds: visibleNodes(tree, nodes).map((node) => node.id),
+    };
+  }
+
   //we need to do two passes, first collect all the nodes that are results
   const newFilteredOut = new Set<string>();
   for (const node of tree) {
-    if (!filter(node)) {
+    if (!filter.fn(filter.value, node)) {
       newFilteredOut.add(node.id);
     }
   }
 
   //nothing is filtered out
   if (newFilteredOut.size === 0) {
-    return inputNodes;
+    return {
+      tree,
+      nodes,
+      filteredNodes: nodes,
+      changes,
+      filter,
+      visibleNodeIds: visibleNodes(tree, nodes).map((node) => node.id),
+    };
   }
 
   //copy of nodes
-  const nodes = { ...inputNodes };
+  const filteredNodes = { ...nodes };
 
-  const selected = selectedIdFromState(nodes);
+  const selected = selectedIdFromState(filteredNodes);
 
   const visible = new Set<string>();
   const expanded = new Set<string>();
@@ -148,28 +175,35 @@ export function applyFilterToState<TData>(
 
   //now set the visibility and expanded state
   for (const id of hidden) {
-    nodes[id] = { ...nodes[id], visible: false };
+    filteredNodes[id] = { ...filteredNodes[id], visible: false };
   }
   for (const id of visible) {
-    nodes[id] = { ...nodes[id], visible: true };
+    filteredNodes[id] = { ...filteredNodes[id], visible: true };
   }
 
   for (const id of collapsed) {
-    nodes[id] = { ...nodes[id], expanded: false };
+    filteredNodes[id] = { ...filteredNodes[id], expanded: false };
   }
   for (const id of expanded) {
-    nodes[id] = { ...nodes[id], expanded: true };
+    filteredNodes[id] = { ...filteredNodes[id], expanded: true };
   }
 
   if (selected) {
     if (visible.has(selected)) {
-      nodes[selected] = { ...nodes[selected], selected: true };
+      filteredNodes[selected] = { ...filteredNodes[selected], selected: true };
     } else {
-      nodes[selected] = { ...nodes[selected], selected: false };
+      filteredNodes[selected] = { ...filteredNodes[selected], selected: false };
     }
   }
 
-  return nodes;
+  return {
+    tree,
+    nodes,
+    filteredNodes,
+    changes,
+    filter,
+    visibleNodeIds: visibleNodes(tree, filteredNodes).map((node) => node.id),
+  };
 }
 
 export function visibleNodes(tree: FlatTree<any>, nodes: NodesState) {
@@ -215,6 +249,5 @@ export function generateChanges(a: NodesState, b: NodesState): Changes {
 
   return {
     selectedId: selectedIdA !== selectedIdB ? selectedIdB : undefined,
-    collapsedIds: collapsedChanges.length > 0 ? collapsedChanges : undefined,
   };
 }

apps/webapp/app/components/runs/v3/LiveTimer.tsx

@@ -1,18 +1,14 @@
 import { formatDuration } from "@trigger.dev/core/v3";
-import { useState, useEffect } from "react";
-import { Paragraph } from "~/components/primitives/Paragraph";
-import { cn } from "~/utils/cn";
+import { useEffect, useState } from "react";
 
 export function LiveTimer({
   startTime,
   endTime,
   updateInterval = 250,
-  className,
 }: {
   startTime: Date;
   endTime?: Date;
   updateInterval?: number;
-  className?: string;
 }) {
   const [now, setNow] = useState<Date>();
 
@@ -30,13 +26,13 @@ export function LiveTimer({
   }, [startTime]);
 
   return (
-    <Paragraph variant="extra-small" className={cn("whitespace-nowrap tabular-nums", className)}>
+    <>
       {formatDuration(startTime, now, {
         style: "short",
         maxDecimalPoints: 0,
         units: ["d", "h", "m", "s"],
       })}
-    </Paragraph>
+    </>
   );
 }
 

apps/webapp/app/components/runs/v3/TaskRunStatus.tsx

@@ -38,6 +38,15 @@ export const RUNNING_STATUSES: TaskRunStatus[] = [
   "WAITING_TO_RESUME",
 ];
 
+export const FINISHED_STATUSES: TaskRunStatus[] = [
+  "COMPLETED_SUCCESSFULLY",
+  "CANCELED",
+  "COMPLETED_WITH_ERRORS",
+  "INTERRUPTED",
+  "SYSTEM_FAILURE",
+  "CRASHED",
+];
+
 export function descriptionForTaskRunStatus(status: TaskRunStatus): string {
   return taskRunStatusDescriptions[status];
 }

apps/webapp/app/components/runs/v3/TaskRunsTable.tsx

@@ -3,7 +3,6 @@ import { StopIcon } from "@heroicons/react/24/outline";
 import { BeakerIcon, BookOpenIcon, CheckIcon } from "@heroicons/react/24/solid";
 import { useLocation } from "@remix-run/react";
 import { formatDuration } from "@trigger.dev/core/v3";
-import { User } from "@trigger.dev/database";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
 import { Dialog, DialogTrigger } from "~/components/primitives/Dialog";
 import { useEnvironments } from "~/hooks/useEnvironments";
@@ -28,6 +27,7 @@ import {
 import { CancelRunDialog } from "./CancelRunDialog";
 import { ReplayRunDialog } from "./ReplayRunDialog";
 import { TaskRunStatusCombo } from "./TaskRunStatus";
+import { LiveTimer } from "./LiveTimer";
 
 type RunsTableProps = {
   total: number;
@@ -94,9 +94,15 @@ export function TaskRunsTable({
                   {run.startedAt ? <DateTime date={run.startedAt} /> : "–"}
                 </TableCell>
                 <TableCell to={path}>
-                  {formatDuration(run.startedAt, run.completedAt, {
-                    style: "short",
-                  })}
+                  {run.startedAt && run.finishedAt ? (
+                    formatDuration(new Date(run.startedAt), new Date(run.finishedAt), {
+                      style: "short",
+                    })
+                  ) : run.startedAt ? (
+                    <LiveTimer startTime={new Date(run.startedAt)} />
+                  ) : (
+                    "–"
+                  )}
                 </TableCell>
                 <TableCell to={path}>
                   {run.isTest ? (

apps/webapp/app/hooks/useDebounce.ts

@@ -1,6 +1,11 @@
 import { useRef } from "react";
 
-//a function that you call with a debounce delay, the function will only be called after the delay has passed
+/**
+ * A function that you call with a debounce delay, the function will only be called after the delay has passed
+ *
+ * @param fn The function to debounce
+ * @param delay In ms
+ */
 export function useDebounce<T extends (...args: any[]) => any>(fn: T, delay: number) {
   const timeout = useRef<ReturnType<typeof setTimeout>>();
 

apps/webapp/app/hooks/useLazyRef.ts

@@ -0,0 +1,11 @@
+import { useRef, MutableRefObject } from "react";
+
+const useLazyRef = <T>(initialValFunc: () => T) => {
+  const ref: MutableRefObject<T | null> = useRef(null);
+  if (ref.current === null) {
+    ref.current = initialValFunc();
+  }
+  return ref;
+};
+
+export default useLazyRef;

apps/webapp/app/hooks/useUser.ts

@@ -26,3 +26,9 @@ export function useUser(matches?: UIMatch[]): User {
 export function useUserChanged(callback: (user: User | undefined) => void) {
   useChanged(useOptionalUser, callback);
 }
+
+export function useHasAdminAccess(matches?: UIMatch[]): boolean {
+  const user = useOptionalUser(matches);
+
+  return Boolean(user?.admin);
+}

apps/webapp/app/models/admin.server.ts

@@ -168,6 +168,12 @@ export async function adminGetOrganizations(userId: string, { page, search }: Se
                 mode: "insensitive",
               },
             },
+            {
+              id: {
+                contains: search,
+                mode: "insensitive",
+              },
+            },
           ],
         }
       : undefined,

apps/webapp/app/models/message.server.ts

@@ -1,6 +1,6 @@
 import { json, Session } from "@remix-run/node";
-import { redirect } from "remix-typedjson";
 import { createCookieSessionStorage } from "@remix-run/node";
+import { redirect } from "remix-typedjson";
 import { env } from "~/env.server";
 
 export type ToastMessage = {

apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts

@@ -103,6 +103,7 @@ export class DeploymentPresenter {
                 exportName: "asc",
               },
             },
+            sdkVersion: true,
           },
         },
         triggeredBy: {
@@ -135,6 +136,7 @@ export class DeploymentPresenter {
         },
         deployedBy: deployment.triggeredBy,
         errorData: this.#prepareErrorData(deployment.errorData),
+        sdkVersion: deployment.worker?.sdkVersion,
       },
     };
   }

apps/webapp/app/presenters/v3/RunListPresenter.server.ts

@@ -1,9 +1,10 @@
 import { Prisma, TaskRunStatus } from "@trigger.dev/database";
 import { Direction } from "~/components/runs/RunStatuses";
-import { sqlDatabaseSchema, PrismaClient, prisma } from "~/db.server";
+import { FINISHED_STATUSES } from "~/components/runs/v3/TaskRunStatus";
+import { sqlDatabaseSchema } from "~/db.server";
 import { displayableEnvironments } from "~/models/runtimeEnvironment.server";
-import { getUsername } from "~/utils/username";
 import { CANCELLABLE_STATUSES } from "~/v3/services/cancelTaskRun.server";
+import { BasePresenter } from "./basePresenter.server";
 
 type RunListOptions = {
   userId?: string;
@@ -28,13 +29,7 @@ export type RunList = Awaited<ReturnType<RunListPresenter["call"]>>;
 export type RunListItem = RunList["runs"][0];
 export type RunListAppliedFilters = RunList["filters"];
 
-export class RunListPresenter {
-  #prismaClient: PrismaClient;
-
-  constructor(prismaClient: PrismaClient = prisma) {
-    this.#prismaClient = prismaClient;
-  }
-
+export class RunListPresenter extends BasePresenter {
   public async call({
     userId,
     projectSlug,
@@ -60,7 +55,7 @@ export class RunListPresenter {
       to !== undefined;
 
     // Find the project scoped to the organization
-    const project = await this.#prismaClient.project.findFirstOrThrow({
+    const project = await this._replica.project.findFirstOrThrow({
       select: {
         id: true,
         environments: {
@@ -88,7 +83,7 @@ export class RunListPresenter {
     });
 
     //get all possible tasks
-    const possibleTasks = await this.#prismaClient.backgroundWorkerTask.findMany({
+    const possibleTasks = await this._replica.backgroundWorkerTask.findMany({
       distinct: ["slug"],
       where: {
         projectId: project.id,
@@ -96,7 +91,7 @@ export class RunListPresenter {
     });
 
     //get the runs
-    let runs = await this.#prismaClient.$queryRaw<
+    let runs = await this._replica.$queryRaw<
       {
         id: string;
         number: BigInt;
@@ -107,10 +102,9 @@ export class RunListPresenter {
         status: TaskRunStatus;
         createdAt: Date;
         lockedAt: Date | null;
-        completedAt: Date | null;
+        updatedAt: Date;
         isTest: boolean;
         spanId: string;
-        attempts: BigInt;
       }[]
     >`
     SELECT
@@ -123,20 +117,13 @@ export class RunListPresenter {
     tr.status AS status,
     tr."createdAt" AS "createdAt",
     tr."lockedAt" AS "lockedAt",
-    tra."completedAt" AS "completedAt",
+    tr."updatedAt" AS "updatedAt",
     tr."isTest" AS "isTest",
-    tr."spanId" AS "spanId",
-    COUNT(tra.id) AS attempts
+    tr."spanId" AS "spanId"
   FROM
     ${sqlDatabaseSchema}."TaskRun" tr
   LEFT JOIN
-    (
-      SELECT *,
-        ROW_NUMBER() OVER (PARTITION BY "taskRunId" ORDER BY "createdAt" DESC) rn
-      FROM ${sqlDatabaseSchema}."TaskRunAttempt"
-    ) tra ON tr.id = tra."taskRunId" AND tra.rn = 1
-  LEFT JOIN
-    ${sqlDatabaseSchema}."BackgroundWorker" bw ON tra."backgroundWorkerId" = bw.id
+    ${sqlDatabaseSchema}."BackgroundWorker" bw ON tr."lockedToVersionId" = bw.id
   WHERE
       -- project
       tr."projectId" = ${project.id}
@@ -154,15 +141,11 @@ export class RunListPresenter {
           ? Prisma.sql`AND tr."taskIdentifier" IN (${Prisma.join(tasks)})`
           : Prisma.empty
       }
-      ${hasStatusFilters ? Prisma.sql`AND (` : Prisma.empty}
       ${
         statuses && statuses.length > 0
-          ? Prisma.sql`tr.status = ANY(ARRAY[${Prisma.join(statuses)}]::"TaskRunStatus"[])`
+          ? Prisma.sql`AND tr.status = ANY(ARRAY[${Prisma.join(statuses)}]::"TaskRunStatus"[])`
           : Prisma.empty
       }
-      ${statuses && statuses.length > 0 && hasStatusFilters ? Prisma.sql` OR ` : Prisma.empty}
-      ${hasStatusFilters ? Prisma.sql`tr.status IS NULL` : Prisma.empty}
-      ${hasStatusFilters ? Prisma.sql`) ` : Prisma.empty}
       ${
         environments && environments.length > 0
           ? Prisma.sql`AND tr."runtimeEnvironmentId" IN (${Prisma.join(environments)})`
@@ -179,8 +162,6 @@ export class RunListPresenter {
           ? Prisma.sql`AND tr."createdAt" <= ${new Date(to).toISOString()}::timestamp`
           : Prisma.empty
       } 
-  GROUP BY
-    tr."friendlyId", tr."taskIdentifier", tr."runtimeEnvironmentId", tr.id, bw.version, tra.status, tr."createdAt", tra."startedAt", tra."completedAt"
   ORDER BY
     ${direction === "forward" ? Prisma.sql`tr.id DESC` : Prisma.sql`tr.id ASC`}
   LIMIT ${pageSize + 1}`;
@@ -219,19 +200,21 @@ export class RunListPresenter {
           throw new Error(`Environment not found for TaskRun ${run.id}`);
         }
 
+        const hasFinished = FINISHED_STATUSES.includes(run.status);
+
         return {
           id: run.id,
           friendlyId: run.runFriendlyId,
           number: Number(run.number),
-          createdAt: run.createdAt,
-          startedAt: run.lockedAt,
-          completedAt: run.completedAt,
+          createdAt: run.createdAt.toISOString(),
+          startedAt: run.lockedAt ? run.lockedAt.toISOString() : undefined,
+          hasFinished,
+          finishedAt: hasFinished ? run.updatedAt.toISOString() : undefined,
           isTest: run.isTest,
           status: run.status,
           version: run.version,
           taskIdentifier: run.taskIdentifier,
           spanId: run.spanId,
-          attempts: Number(run.attempts),
           isReplayable: true,
           isCancellable: CANCELLABLE_STATUSES.includes(run.status),
           environment: displayableEnvironments(environment, userId),

apps/webapp/app/presenters/v3/RunPresenter.server.ts

@@ -38,6 +38,7 @@ export class RunPresenter {
             id: true,
             type: true,
             slug: true,
+            organizationId: true,
             orgMember: {
               select: {
                 user: {
@@ -66,9 +67,13 @@ export class RunPresenter {
     if (!traceSummary) {
       return {
         run: {
+          id: run.id,
           number: run.number,
           friendlyId: run.friendlyId,
+          traceId: run.traceId,
           environment: {
+            id: run.runtimeEnvironment.id,
+            organizationId: run.runtimeEnvironment.organizationId,
             type: run.runtimeEnvironment.type,
             slug: run.runtimeEnvironment.slug,
             userId: run.runtimeEnvironment.orgMember?.user.id,
@@ -118,9 +123,13 @@ export class RunPresenter {
 
     return {
       run: {
+        id: run.id,
         number: run.number,
         friendlyId: run.friendlyId,
+        traceId: run.traceId,
         environment: {
+          id: run.runtimeEnvironment.id,
+          organizationId: run.runtimeEnvironment.organizationId,
           type: run.runtimeEnvironment.type,
           slug: run.runtimeEnvironment.slug,
           userId: run.runtimeEnvironment.orgMember?.user.id,

apps/webapp/app/presenters/v3/SpanPresenter.server.ts

@@ -42,7 +42,7 @@ export class SpanPresenter {
     const output =
       span.outputType === "application/store"
         ? `/resources/packets/${span.environmentId}/${span.output}`
-        : typeof span.output !== "undefined" && span.output !== null
+        : typeof span.output !== "undefined"
         ? await prettyPrintPacket(span.output, span.outputType ?? undefined)
         : undefined;
 

apps/webapp/app/presenters/v3/TaskListPresenter.server.ts

@@ -4,16 +4,15 @@ import {
   TaskRunStatus,
   TaskTriggerSource,
 } from "@trigger.dev/database";
-import { PrismaClient, prisma, sqlDatabaseSchema } from "~/db.server";
+import { QUEUED_STATUSES, RUNNING_STATUSES } from "~/components/runs/v3/TaskRunStatus";
+import { sqlDatabaseSchema } from "~/db.server";
 import { Organization } from "~/models/organization.server";
 import { Project } from "~/models/project.server";
+import { displayableEnvironments } from "~/models/runtimeEnvironment.server";
 import { User } from "~/models/user.server";
 import { sortEnvironments } from "~/services/environmentSort.server";
 import { logger } from "~/services/logger.server";
-import { getUsername } from "~/utils/username";
 import { BasePresenter } from "./basePresenter.server";
-import { QUEUED_STATUSES, RUNNING_STATUSES } from "~/components/runs/v3/TaskRunStatus";
-import { displayableEnvironments } from "~/models/runtimeEnvironment.server";
 
 export type Task = {
   slug: string;
@@ -26,10 +25,6 @@ export type Task = {
     type: RuntimeEnvironmentType;
     userName?: string;
   }[];
-  latestRun?: {
-    createdAt: Date;
-    status: TaskRunStatus;
-  };
 };
 
 type Return = Awaited<ReturnType<TaskListPresenter["call"]>>;
@@ -98,39 +93,8 @@ export class TaskListPresenter extends BasePresenter {
     JOIN ${sqlDatabaseSchema}."BackgroundWorkerTask" tasks ON tasks."workerId" = workers.id
     ORDER BY slug ASC;`;
 
-    let latestRuns = [] as {
-      createdAt: Date;
-      status: TaskRunStatus;
-      taskIdentifier: string;
-    }[];
-
-    if (tasks.length > 0) {
-      const uniqueTaskSlugs = new Set(tasks.map((t) => t.slug));
-      latestRuns = await this._replica.$queryRaw<
-        {
-          createdAt: Date;
-          status: TaskRunStatus;
-          taskIdentifier: string;
-        }[]
-      >`
-      SELECT * FROM (
-        SELECT
-          "createdAt",
-          "status",
-          "taskIdentifier",
-          ROW_NUMBER() OVER (PARTITION BY "taskIdentifier" ORDER BY "updatedAt" DESC) AS rn
-        FROM
-          ${sqlDatabaseSchema}."TaskRun"
-        WHERE
-          "taskIdentifier" IN(${Prisma.join(Array.from(uniqueTaskSlugs))})
-          AND "projectId" = ${project.id}
-      ) t
-      WHERE rn = 1;`;
-    }
-
     //group by the task identifier (task.slug). Add the latestRun and add all the environments.
     const outputTasks = tasks.reduce((acc, task) => {
-      const latestRun = latestRuns.find((r) => r.taskIdentifier === task.slug);
       const environment = project.environments.find((env) => env.id === task.runtimeEnvironmentId);
       if (!environment) {
         throw new Error(`Environment not found for TaskRun ${task.id}`);
@@ -151,13 +115,6 @@ export class TaskListPresenter extends BasePresenter {
       //order the environments
       existingTask.environments = sortEnvironments(existingTask.environments);
 
-      existingTask.latestRun = latestRun
-        ? {
-            createdAt: latestRun.createdAt,
-            status: latestRun.status,
-          }
-        : undefined;
-
       return acc;
     }, [] as Task[]);
 
@@ -186,6 +143,10 @@ export class TaskListPresenter extends BasePresenter {
   }
 
   async #getActivity(tasks: string[], projectId: string) {
+    if (tasks.length === 0) {
+      return {};
+    }
+
     const activity = await this._replica.$queryRaw<
       {
         taskIdentifier: string;
@@ -257,6 +218,10 @@ export class TaskListPresenter extends BasePresenter {
   }
 
   async #getRunningStats(tasks: string[], projectId: string) {
+    if (tasks.length === 0) {
+      return {};
+    }
+
     const statuses = await this._replica.$queryRaw<
       {
         taskIdentifier: string;
@@ -305,6 +270,10 @@ export class TaskListPresenter extends BasePresenter {
   }
 
   async #getAverageDurations(tasks: string[], projectId: string) {
+    if (tasks.length === 0) {
+      return {};
+    }
+
     const durations = await this._replica.$queryRaw<
       {
         taskIdentifier: string;

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.settings/route.tsx

@@ -232,7 +232,7 @@ export default function Page() {
                   <FormError id={projectSlug.errorId}>{projectSlug.error}</FormError>
                   <FormError>{deleteForm.error}</FormError>
                   <Hint>
-                    This change is irreversible, so please be certain. Type in the Project slug
+                    This change is irreversible, so please be certain. Type in the Project slug{" "}
                     <InlineCode variant="extra-small">{project.slug}</InlineCode> and then press
                     Delete.
                   </Hint>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam._index/route.tsx

@@ -1,23 +1,26 @@
 import { ChatBubbleLeftRightIcon, ChevronDownIcon, ChevronUpIcon } from "@heroicons/react/20/solid";
 import { useRevalidator } from "@remix-run/react";
 import { LoaderFunctionArgs } from "@remix-run/server-runtime";
-import { formatDuration, formatDurationMilliseconds } from "@trigger.dev/core/v3";
+import { formatDurationMilliseconds } from "@trigger.dev/core/v3";
 import { TaskRunStatus } from "@trigger.dev/database";
 import { Fragment, Suspense, useEffect, useState } from "react";
-import { Bar, BarChart, ResponsiveContainer, Tooltip, TooltipProps, XAxis, YAxis } from "recharts";
+import { Bar, BarChart, ResponsiveContainer, Tooltip, TooltipProps } from "recharts";
 import { TypedAwait, typeddefer, useTypedLoaderData } from "remix-typedjson";
 import { Feedback } from "~/components/Feedback";
 import { InitCommandV3, TriggerDevStepV3, TriggerLoginStepV3 } from "~/components/SetupCommands";
 import { StepContentContainer } from "~/components/StepContentContainer";
+import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
 import { InlineCode } from "~/components/code/InlineCode";
 import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
 import { MainCenteredContainer, PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { Button } from "~/components/primitives/Buttons";
 import { Callout } from "~/components/primitives/Callout";
-import { DateTime, formatDateTime } from "~/components/primitives/DateTime";
+import { formatDateTime } from "~/components/primitives/DateTime";
 import { Header1, Header2, Header3 } from "~/components/primitives/Headers";
-import { NavBar, PageTitle } from "~/components/primitives/PageHeader";
+import { Input } from "~/components/primitives/Input";
+import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
+import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
 import { Spinner } from "~/components/primitives/Spinner";
 import { StepNumber } from "~/components/primitives/StepNumber";
 import {
@@ -31,13 +34,9 @@ import {
   TableRow,
 } from "~/components/primitives/Table";
 import { SimpleTooltip } from "~/components/primitives/Tooltip";
+import TooltipPortal from "~/components/primitives/TooltipPortal";
 import { TaskFunctionName } from "~/components/runs/v3/TaskPath";
-import {
-  TaskRunStatusCombo,
-  TaskRunStatusIcon,
-  runStatusClassNameColor,
-  runStatusTitle,
-} from "~/components/runs/v3/TaskRunStatus";
+import { TaskRunStatusCombo } from "~/components/runs/v3/TaskRunStatus";
 import {
   TaskTriggerSourceIcon,
   taskTriggerSourceDescription,
@@ -45,8 +44,8 @@ import {
 import { useEventSource } from "~/hooks/useEventSource";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
-import { useUser } from "~/hooks/useUser";
-import { TaskActivity, TaskListPresenter } from "~/presenters/v3/TaskListPresenter.server";
+import { useTextFilter } from "~/hooks/useTextFilter";
+import { Task, TaskActivity, TaskListPresenter } from "~/presenters/v3/TaskListPresenter.server";
 import { requireUserId } from "~/services/session.server";
 import { cn } from "~/utils/cn";
 import { ProjectParamSchema, v3RunsPath, v3TasksStreamingPath } from "~/utils/pathBuilder";
@@ -84,6 +83,31 @@ export default function Page() {
   const project = useProject();
   const { tasks, userHasTasks, activity, runningStats, durations } =
     useTypedLoaderData<typeof loader>();
+  const { filterText, setFilterText, filteredItems } = useTextFilter<Task>({
+    items: tasks,
+    filter: (task, text) => {
+      if (task.slug.toLowerCase().includes(text.toLowerCase())) {
+        return true;
+      }
+
+      if (
+        task.exportName.toLowerCase().includes(text.toLowerCase().replace("(", "").replace(")", ""))
+      ) {
+        return true;
+      }
+
+      if (task.filePath.toLowerCase().includes(text.toLowerCase())) {
+        return true;
+      }
+
+      if (task.triggerSource === "SCHEDULED" && "scheduled".includes(text.toLowerCase())) {
+        return true;
+      }
+
+      return false;
+    },
+  });
+
   const hasTasks = tasks.length > 0;
 
   //live reload the page when the tasks change
@@ -103,13 +127,43 @@ export default function Page() {
     <PageContainer>
       <NavBar>
         <PageTitle title="Tasks" />
+        <PageAccessories>
+          <AdminDebugTooltip>
+            <PropertyTable>
+              {tasks.map((task) => (
+                <Property label={task.exportName} key={task.slug}>
+                  <div className="flex items-center gap-2">
+                    <Paragraph variant="extra-small/bright/mono">
+                      {task.environments
+                        .map((e) =>
+                          e.userName ? `${e.userName}/${e.id}` : `${e.type.slice(0, 3)}/${e.id}`
+                        )
+                        .join(", ")}
+                    </Paragraph>
+                  </div>
+                </Property>
+              ))}
+            </PropertyTable>
+          </AdminDebugTooltip>
+        </PageAccessories>
       </NavBar>
       <PageBody>
-        <div className={cn("grid h-full grid-cols-1 gap-4")}>
-          <div className="h-full">
-            {hasTasks ? (
-              <div className="flex flex-col gap-4 pb-4">
-                {!userHasTasks && <UserHasNoTasks />}
+        <div className={cn("grid h-full grid-rows-1")}>
+          {hasTasks ? (
+            <div className="flex flex-col gap-4 pb-4">
+              {!userHasTasks && <UserHasNoTasks />}
+              <div className="pb-4">
+                <div className="h-8">
+                  <Input
+                    placeholder="Search tasks"
+                    variant="tertiary"
+                    icon="search"
+                    fullWidth={true}
+                    value={filterText}
+                    onChange={(e) => setFilterText(e.target.value)}
+                    autoFocus
+                  />
+                </div>
                 <Table>
                   <TableHeader>
                     <TableRow>
@@ -120,13 +174,12 @@ export default function Page() {
                       <TableHeaderCell>Activity (7d)</TableHeaderCell>
                       <TableHeaderCell>Avg. duration</TableHeaderCell>
                       <TableHeaderCell>Environments</TableHeaderCell>
-                      <TableHeaderCell>Last run</TableHeaderCell>
                       <TableHeaderCell hiddenLabel>Go to page</TableHeaderCell>
                     </TableRow>
                   </TableHeader>
                   <TableBody>
-                    {tasks.length > 0 ? (
-                      tasks.map((task) => {
+                    {filteredItems.length > 0 ? (
+                      filteredItems.map((task) => {
                         const path = v3RunsPath(organization, project, {
                           tasks: [task.slug],
                         });
@@ -218,30 +271,12 @@ export default function Page() {
                                 ))}
                               </div>
                             </TableCell>
-                            <TableCell to={path}>
-                              {task.latestRun ? (
-                                <div
-                                  className={cn(
-                                    "flex items-center gap-1",
-                                    runStatusClassNameColor(task.latestRun.status)
-                                  )}
-                                >
-                                  <TaskRunStatusIcon
-                                    status={task.latestRun.status}
-                                    className="h-4 w-4"
-                                  />
-                                  <DateTime date={task.latestRun.createdAt} />
-                                </div>
-                              ) : (
-                                "Never run"
-                              )}
-                            </TableCell>
                             <TableCellChevron to={path} />
                           </TableRow>
                         );
                       })
                     ) : (
-                      <TableBlankRow colSpan={6}>
+                      <TableBlankRow colSpan={8}>
                         <Paragraph variant="small" className="flex items-center justify-center">
                           No tasks match your filters
                         </Paragraph>
@@ -250,12 +285,12 @@ export default function Page() {
                   </TableBody>
                 </Table>
               </div>
-            ) : (
-              <MainCenteredContainer className="max-w-prose">
-                <CreateTaskInstructions />
-              </MainCenteredContainer>
-            )}
-          </div>
+            </div>
+          ) : (
+            <MainCenteredContainer className="max-w-prose">
+              <CreateTaskInstructions />
+            </MainCenteredContainer>
+          )}
         </div>
       </PageBody>
     </PageContainer>
@@ -362,7 +397,9 @@ function TaskActivityGraph({ activity }: { activity: TaskActivity }) {
           content={<CustomTooltip />}
           allowEscapeViewBox={{ x: true, y: true }}
           wrapperStyle={{ zIndex: 1000 }}
+          animationDuration={0}
         />
+
         {/* The background */}
         <Bar
           dataKey="bg"
@@ -425,18 +462,21 @@ const CustomTooltip = ({ active, payload, label }: TooltipProps<number, string>)
     }));
     const title = payload[0].payload.day as string;
     const formattedDate = formatDateTime(new Date(title), "UTC", [], false, false);
+
     return (
-      <div className="rounded-sm border border-grid-bright bg-background-dimmed px-3 py-2">
-        <Header3 className="border-b-charcoal-650 border-b pb-2">{formattedDate}</Header3>
-        <div className="mt-2 grid grid-cols-[1fr_auto] gap-2 text-xs text-text-bright">
-          {items.map((item) => (
-            <Fragment key={item.status}>
-              <TaskRunStatusCombo status={item.status} />
-              <p>{item.value}</p>
-            </Fragment>
-          ))}
+      <TooltipPortal active={active}>
+        <div className="rounded-sm border border-grid-bright bg-background-dimmed px-3 py-2">
+          <Header3 className="border-b-charcoal-650 border-b pb-2">{formattedDate}</Header3>
+          <div className="mt-2 grid grid-cols-[1fr_auto] gap-2 text-xs text-text-bright">
+            {items.map((item) => (
+              <Fragment key={item.status}>
+                <TaskRunStatusCombo status={item.status} />
+                <p>{item.value}</p>
+              </Fragment>
+            ))}
+          </div>
         </div>
-      </div>
+      </TooltipPortal>
     );
   }
 

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.apikeys/route.tsx

@@ -1,7 +1,8 @@
-import { BookOpenIcon } from "@heroicons/react/20/solid";
+import { BookOpenIcon, LightBulbIcon, ShieldCheckIcon } from "@heroicons/react/20/solid";
 import { Form } from "@remix-run/react";
 import { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
+import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
 import { EnvironmentLabel, environmentTitle } from "~/components/environments/EnvironmentLabel";
 import { RegenerateApiKeyModal } from "~/components/environments/RegenerateApiKeyModal";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
@@ -10,8 +11,9 @@ import { Callout } from "~/components/primitives/Callout";
 import { ClipboardField } from "~/components/primitives/ClipboardField";
 import { DateTime } from "~/components/primitives/DateTime";
 import { Header3 } from "~/components/primitives/Headers";
-import { PageAccessories, NavBar, PageTitle } from "~/components/primitives/PageHeader";
+import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
+import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
 import {
   Table,
   TableBody,
@@ -24,7 +26,6 @@ import {
 import { TextLink } from "~/components/primitives/TextLink";
 import { prisma } from "~/db.server";
 import { useFeatures } from "~/hooks/useFeatures";
-import { useProject } from "~/hooks/useProject";
 import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
 import { createEnvironment } from "~/models/organization.server";
 import { ApiKeysPresenter } from "~/presenters/v3/ApiKeysPresenter.server";
@@ -132,6 +133,18 @@ export default function Page() {
       <NavBar>
         <PageTitle title="API keys" />
         <PageAccessories>
+          <AdminDebugTooltip>
+            <PropertyTable>
+              {environments.map((environment) => (
+                <Property label={environment.slug} key={environment.id}>
+                  <div className="flex items-center gap-2">
+                    <Paragraph variant="extra-small/bright/mono">{environment.id}</Paragraph>
+                  </div>
+                </Property>
+              ))}
+            </PropertyTable>
+          </AdminDebugTooltip>
+
           <LinkButton
             variant={"minimal/small"}
             LeadingIcon={BookOpenIcon}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.deployments.$deploymentParam/route.tsx

@@ -92,6 +92,9 @@ export default function Page() {
               <DeploymentStatus status={deployment.status} className="text-sm" />
             </Property>
             <Property label="Tasks">{deployment.tasks ? deployment.tasks.length : "–"}</Property>
+            <Property label="SDK Version">
+              {deployment.sdkVersion ? deployment.sdkVersion : "–"}
+            </Property>
             <Property label="Started at">
               <Paragraph variant="small/bright">
                 <DateTimeAccurate date={deployment.createdAt} /> UTC

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.environment-variables.new/route.tsx

@@ -38,6 +38,7 @@ import { cn } from "~/utils/cn";
 import { ProjectParamSchema, v3EnvironmentVariablesPath } from "~/utils/pathBuilder";
 import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server";
 import { EnvironmentVariableKey } from "~/v3/environmentVariables/repository";
+import dotenv from "dotenv";
 
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const userId = await requireUserId(request);
@@ -183,7 +184,7 @@ export default function Page() {
         }
       }}
     >
-      <DialogContent>
+      <DialogContent className="md:max-w-2xl lg:max-w-3xl">
         <DialogHeader>New environment variables</DialogHeader>
         <Form
           method="post"
@@ -308,20 +309,12 @@ function VariableFields({
     if (!clipboardData) return;
 
     let text = clipboardData.getData("text");
-    //replace carriage returns
-    text = text.replace(/\r/g, "");
-    const lines = text.split("\n");
+    if (!text) return;
 
-    const keyValuePairs = lines.flatMap((line) => {
-      if (line.trim().startsWith("#")) return [];
-
-      const split = line.split("=");
-      if (split.length === 2) {
-        return [{ key: split[0], value: split[1] }];
-      }
-      return [];
-    });
+    const variables = dotenv.parse(text);
+    const keyValuePairs = Object.entries(variables).map(([key, value]) => ({ key, value }));
 
+    //do the default paste
     if (keyValuePairs.length === 0) return;
 
     //prevent default pasting

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx

@@ -1,6 +1,4 @@
 import {
-  ArrowsPointingInIcon,
-  ArrowsPointingOutIcon,
   ChevronDownIcon,
   ChevronRightIcon,
   MagnifyingGlassMinusIcon,
@@ -17,7 +15,8 @@ import {
 } from "@trigger.dev/core/v3";
 import { RuntimeEnvironmentType } from "@trigger.dev/database";
 import { motion } from "framer-motion";
-import { useEffect, useRef, useState } from "react";
+import { useCallback, useEffect, useRef, useState } from "react";
+import { useHotkeys } from "react-hotkeys-hook";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { ShowParentIcon, ShowParentIconSelected } from "~/assets/icons/ShowParentIcon";
 import tileBgPath from "~/assets/images/error-banner-tile@2x.png";
@@ -26,11 +25,13 @@ import { InlineCode } from "~/components/code/InlineCode";
 import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
 import { MainCenteredContainer, PageBody } from "~/components/layout/AppLayout";
 import { Badge } from "~/components/primitives/Badge";
-import { Button, LinkButton } from "~/components/primitives/Buttons";
+import { LinkButton } from "~/components/primitives/Buttons";
 import { Callout } from "~/components/primitives/Callout";
+import { Header3 } from "~/components/primitives/Headers";
 import { Input } from "~/components/primitives/Input";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
+import { Popover, PopoverArrowTrigger, PopoverContent } from "~/components/primitives/Popover";
 import {
   ResizableHandle,
   ResizablePanel,
@@ -66,10 +67,8 @@ import {
   v3RunsPath,
 } from "~/utils/pathBuilder";
 import { SpanView } from "../resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route";
-import { number } from "zod";
-import { useHotkeys } from "react-hotkeys-hook";
-import { Popover, PopoverArrowTrigger, PopoverContent } from "~/components/primitives/Popover";
-import { Header3 } from "~/components/primitives/Headers";
+import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
+import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
 
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const userId = await requireUserId(request);
@@ -119,6 +118,33 @@ export default function Page() {
             title={`Run #${run.number}`}
           />
           <PageAccessories>
+            <AdminDebugTooltip>
+              <PropertyTable>
+                <Property label="ID">
+                  <div className="flex items-center gap-2">
+                    <Paragraph variant="extra-small/bright/mono">{run.id}</Paragraph>
+                  </div>
+                </Property>
+                <Property label="Trace ID">
+                  <div className="flex items-center gap-2">
+                    <Paragraph variant="extra-small/bright/mono">{run.traceId}</Paragraph>
+                  </div>
+                </Property>
+                <Property label="Env ID">
+                  <div className="flex items-center gap-2">
+                    <Paragraph variant="extra-small/bright/mono">{run.environment.id}</Paragraph>
+                  </div>
+                </Property>
+                <Property label="Org ID">
+                  <div className="flex items-center gap-2">
+                    <Paragraph variant="extra-small/bright/mono">
+                      {run.environment.organizationId}
+                    </Paragraph>
+                  </div>
+                </Property>
+              </PropertyTable>
+            </AdminDebugTooltip>
+
             <EnvironmentLabel
               size="large"
               environment={run.environment}
@@ -168,6 +194,33 @@ export default function Page() {
           title={`Run #${run.number}`}
         />
         <PageAccessories>
+          <AdminDebugTooltip>
+            <PropertyTable>
+              <Property label="ID">
+                <div className="flex items-center gap-2">
+                  <Paragraph variant="extra-small/bright/mono">{run.id}</Paragraph>
+                </div>
+              </Property>
+              <Property label="Trace ID">
+                <div className="flex items-center gap-2">
+                  <Paragraph variant="extra-small/bright/mono">{run.traceId}</Paragraph>
+                </div>
+              </Property>
+              <Property label="Env ID">
+                <div className="flex items-center gap-2">
+                  <Paragraph variant="extra-small/bright/mono">{run.environment.id}</Paragraph>
+                </div>
+              </Property>
+              <Property label="Org ID">
+                <div className="flex items-center gap-2">
+                  <Paragraph variant="extra-small/bright/mono">
+                    {run.environment.organizationId}
+                  </Paragraph>
+                </div>
+              </Property>
+            </PropertyTable>
+          </AdminDebugTooltip>
+
           <EnvironmentLabel size="large" environment={run.environment} userName={usernameForEnv} />
         </PageAccessories>
       </NavBar>
@@ -268,29 +321,25 @@ function TasksTreeView({
     onSelectedIdChanged,
     estimatedRowHeight: () => 32,
     parentRef,
-    filter: (node) => {
-      const nodePassesErrorTest = (errorsOnly && node.data.isError) || !errorsOnly;
-      if (!nodePassesErrorTest) return false;
+    filter: {
+      value: { text: filterText, errorsOnly },
+      fn: (value, node) => {
+        const nodePassesErrorTest = (value.errorsOnly && node.data.isError) || !value.errorsOnly;
+        if (!nodePassesErrorTest) return false;
 
-      if (filterText === "") return true;
-      if (node.data.message.toLowerCase().includes(filterText.toLowerCase())) {
-        return true;
-      }
-      return false;
+        if (value.text === "") return true;
+        if (node.data.message.toLowerCase().includes(value.text.toLowerCase())) {
+          return true;
+        }
+        return false;
+      },
     },
   });
 
   return (
     <div className="grid h-full grid-rows-[2.5rem_1fr_3.25rem] overflow-hidden">
       <div className="mx-3 flex items-center justify-between gap-2 border-b border-grid-dimmed">
-        <Input
-          placeholder="Search log"
-          variant="tertiary"
-          icon="search"
-          fullWidth={true}
-          value={filterText}
-          onChange={(e) => setFilterText(e.target.value)}
-        />
+        <SearchField onChange={setFilterText} />
         <div className="flex items-center gap-2">
           <Switch
             variant="small"
@@ -1004,3 +1053,27 @@ function NumberShortcuts({ toggleLevel }: { toggleLevel: (depth: number) => void
     </div>
   );
 }
+
+function SearchField({ onChange }: { onChange: (value: string) => void }) {
+  const [value, setValue] = useState("");
+
+  const updateFilterText = useDebounce((text: string) => {
+    onChange(text);
+  }, 250);
+
+  const updateValue = useCallback((value: string) => {
+    setValue(value);
+    updateFilterText(value);
+  }, []);
+
+  return (
+    <Input
+      placeholder="Search log"
+      variant="tertiary"
+      icon="search"
+      fullWidth={true}
+      value={value}
+      onChange={(e) => updateValue(e.target.value)}
+    />
+  );
+}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs._index/route.tsx

@@ -1,7 +1,7 @@
 import { BeakerIcon, BookOpenIcon } from "@heroicons/react/24/solid";
 import { useNavigation } from "@remix-run/react";
 import { LoaderFunctionArgs } from "@remix-run/server-runtime";
-import { typedjson, useTypedLoaderData } from "remix-typedjson";
+import { TypedAwait, typeddefer, typedjson, useTypedLoaderData } from "remix-typedjson";
 import { TaskIcon } from "~/assets/icons/TaskIcon";
 import { BlankstateInstructions } from "~/components/BlankstateInstructions";
 import { StepContentContainer } from "~/components/StepContentContainer";
@@ -22,6 +22,8 @@ import { cn } from "~/utils/cn";
 import { ProjectParamSchema, v3ProjectPath, v3TestPath } from "~/utils/pathBuilder";
 import { ListPagination } from "../../components/ListPagination";
 import { TextLink } from "~/components/primitives/TextLink";
+import { Spinner } from "~/components/primitives/Spinner";
+import { Suspense } from "react";
 
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const userId = await requireUserId(request);
@@ -33,7 +35,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
     TaskRunListSearchFilters.parse(s);
 
   const presenter = new RunListPresenter();
-  const list = await presenter.call({
+  const list = presenter.call({
     userId,
     projectSlug: projectParam,
     tasks,
@@ -46,13 +48,13 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
     cursor: cursor,
   });
 
-  return typedjson({
-    list,
+  return typeddefer({
+    data: list,
   });
 };
 
 export default function Page() {
-  const { list } = useTypedLoaderData<typeof loader>();
+  const { data } = useTypedLoaderData<typeof loader>();
   const navigation = useNavigation();
   const isLoading = navigation.state !== "idle";
   const project = useProject();
@@ -64,36 +66,53 @@ export default function Page() {
         <PageTitle title="Runs" />
       </NavBar>
       <PageBody>
-        {list.runs.length === 0 && !list.hasFilters ? (
-          list.possibleTasks.length === 0 ? (
-            <CreateFirstTaskInstructions />
-          ) : (
-            <RunTaskInstructions />
-          )
-        ) : (
-          <div className={cn("grid h-fit grid-cols-1 gap-4")}>
-            <div>
-              <div className="mb-2 flex items-center justify-between gap-x-2">
-                <RunsFilters
-                  possibleEnvironments={project.environments}
-                  possibleTasks={list.possibleTasks}
-                />
-                <div className="flex items-center justify-end gap-x-2">
-                  <ListPagination list={list} />
-                </div>
+        <Suspense
+          fallback={
+            <div className="flex items-center justify-center py-2">
+              <div className="mx-auto flex items-center gap-2">
+                <Spinner />
+                <Paragraph variant="small">Loading runs</Paragraph>
               </div>
-
-              <TaskRunsTable
-                total={list.runs.length}
-                hasFilters={list.hasFilters}
-                filters={list.filters}
-                runs={list.runs}
-                isLoading={isLoading}
-              />
-              <ListPagination list={list} className="mt-2 justify-end" />
             </div>
-          </div>
-        )}
+          }
+        >
+          <TypedAwait resolve={data}>
+            {(list) => (
+              <>
+                {list.runs.length === 0 && !list.hasFilters ? (
+                  list.possibleTasks.length === 0 ? (
+                    <CreateFirstTaskInstructions />
+                  ) : (
+                    <RunTaskInstructions />
+                  )
+                ) : (
+                  <div className={cn("grid h-fit grid-cols-1 gap-4")}>
+                    <div>
+                      <div className="mb-2 flex items-center justify-between gap-x-2">
+                        <RunsFilters
+                          possibleEnvironments={project.environments}
+                          possibleTasks={list.possibleTasks}
+                        />
+                        <div className="flex items-center justify-end gap-x-2">
+                          <ListPagination list={list} />
+                        </div>
+                      </div>
+
+                      <TaskRunsTable
+                        total={list.runs.length}
+                        hasFilters={list.hasFilters}
+                        filters={list.filters}
+                        runs={list.runs}
+                        isLoading={isLoading}
+                      />
+                      <ListPagination list={list} className="mt-2 justify-end" />
+                    </div>
+                  </div>
+                )}
+              </>
+            )}
+          </TypedAwait>
+        </Suspense>
       </PageBody>
     </>
   );

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.schedules/route.tsx

@@ -4,6 +4,7 @@ import { Outlet, useLocation, useParams } from "@remix-run/react";
 import { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { BlankstateInstructions } from "~/components/BlankstateInstructions";
+import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
 import { InlineCode } from "~/components/code/InlineCode";
 import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
 import { MainCenteredContainer, PageBody, PageContainer } from "~/components/layout/AppLayout";
@@ -12,6 +13,7 @@ import { DateTime } from "~/components/primitives/DateTime";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { PaginationControls } from "~/components/primitives/Pagination";
 import { Paragraph } from "~/components/primitives/Paragraph";
+import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
 import {
   ResizableHandle,
   ResizablePanel,
@@ -93,6 +95,18 @@ export default function Page() {
       <NavBar>
         <PageTitle title="Schedules" />
         <PageAccessories>
+          <AdminDebugTooltip>
+            <PropertyTable>
+              {schedules.map((schedule) => (
+                <Property label={schedule.friendlyId} key={schedule.id}>
+                  <div className="flex items-center gap-2">
+                    <Paragraph variant="extra-small/bright/mono">{schedule.id}</Paragraph>
+                  </div>
+                </Property>
+              ))}
+            </PropertyTable>
+          </AdminDebugTooltip>
+
           <LinkButton
             LeadingIcon={PlusIcon}
             to={`${v3NewSchedulePath(organization, project)}${location.search}`}
@@ -261,10 +275,10 @@ function SchedulesTable({
                   {schedule.userProvidedDeduplicationKey ? schedule.deduplicationKey : "–"}
                 </TableCell>
                 <TableCell to={path} className={cellClass}>
-                  <DateTime date={schedule.nextRun} />
+                  <DateTime date={schedule.nextRun} timeZone="utc" />
                 </TableCell>
                 <TableCell to={path} className={cellClass}>
-                  {schedule.lastRun ? <DateTime date={schedule.lastRun} /> : "–"}
+                  {schedule.lastRun ? <DateTime date={schedule.lastRun} timeZone="utc" /> : "–"}
                 </TableCell>
                 <TableCell to={path} className={cellClass}>
                   <div className="flex gap-1">

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.settings/route.tsx

@@ -3,6 +3,7 @@ import { parse } from "@conform-to/zod";
 import { Form, useActionData, useNavigation } from "@remix-run/react";
 import { ActionFunction, json } from "@remix-run/server-runtime";
 import { z } from "zod";
+import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
 import { InlineCode } from "~/components/code/InlineCode";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { Button } from "~/components/primitives/Buttons";
@@ -14,7 +15,9 @@ import { Hint } from "~/components/primitives/Hint";
 import { Input } from "~/components/primitives/Input";
 import { InputGroup } from "~/components/primitives/InputGroup";
 import { Label } from "~/components/primitives/Label";
-import { NavBar, PageTitle } from "~/components/primitives/PageHeader";
+import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
+import { Paragraph } from "~/components/primitives/Paragraph";
+import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
 import { prisma } from "~/db.server";
 import { useProject } from "~/hooks/useProject";
 import { redirectWithSuccessMessage } from "~/models/message.server";
@@ -110,6 +113,23 @@ export default function Page() {
     <PageContainer>
       <NavBar>
         <PageTitle title={`${project.name} project settings`} />
+
+        <PageAccessories>
+          <AdminDebugTooltip>
+            <PropertyTable>
+              <Property label="ID">
+                <div className="flex items-center gap-2">
+                  <Paragraph variant="extra-small/bright/mono">{project.id}</Paragraph>
+                </div>
+              </Property>
+              <Property label="Org ID">
+                <div className="flex items-center gap-2">
+                  <Paragraph variant="extra-small/bright/mono">{project.organizationId}</Paragraph>
+                </div>
+              </Property>
+            </PropertyTable>
+          </AdminDebugTooltip>
+        </PageAccessories>
       </NavBar>
 
       <PageBody>

apps/webapp/app/routes/_app.orgs.$organizationSlug.settings/route.tsx

@@ -239,7 +239,7 @@ export default function Page() {
                   <FormError id={organizationSlug.errorId}>{organizationSlug.error}</FormError>
                   <FormError>{deleteForm.error}</FormError>
                   <Hint>
-                    This change is irreversible, so please be certain. Type in the Organization slug
+                    This change is irreversible, so please be certain. Type in the Organization slug{" "}
                     <InlineCode variant="extra-small">{organization.slug}</InlineCode> and then
                     press Delete.
                   </Hint>

apps/webapp/app/routes/_app.orgs.$organizationSlug.team/route.tsx

@@ -8,6 +8,7 @@ import { UseDataFunctionReturn, typedjson, useTypedLoaderData } from "remix-type
 import invariant from "tiny-invariant";
 import { z } from "zod";
 import { UserAvatar } from "~/components/UserProfilePhoto";
+import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
 import {
   Alert,
@@ -23,8 +24,9 @@ import { Button, ButtonContent, LinkButton } from "~/components/primitives/Butto
 import { DateTime } from "~/components/primitives/DateTime";
 import { Header2, Header3 } from "~/components/primitives/Headers";
 import { NamedIcon } from "~/components/primitives/NamedIcon";
-import { NavBar, PageTitle } from "~/components/primitives/PageHeader";
+import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
+import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
 import { SimpleTooltip } from "~/components/primitives/Tooltip";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useUser } from "~/hooks/useUser";
@@ -107,6 +109,28 @@ export default function Page() {
     <PageContainer>
       <NavBar>
         <PageTitle title="Team" />
+
+        <PageAccessories>
+          <AdminDebugTooltip>
+            <PropertyTable>
+              <Property label="Org ID">
+                <div className="flex items-center gap-2">
+                  <Paragraph variant="extra-small/bright/mono">{organization.id}</Paragraph>
+                </div>
+              </Property>
+
+              {members.map((member) => (
+                <Property label={member.user.name} key={member.id}>
+                  <div className="flex items-center gap-2">
+                    <Paragraph variant="extra-small/bright/mono">
+                      {member.user.email} - {member.user.id}
+                    </Paragraph>
+                  </div>
+                </Property>
+              ))}
+            </PropertyTable>
+          </AdminDebugTooltip>
+        </PageAccessories>
       </NavBar>
       <PageBody>
         <Header2>Members</Header2>

apps/webapp/app/routes/_app/route.tsx

@@ -15,9 +15,12 @@ export const loader = async ({ request }: LoaderFunctionArgs) => {
     return redirect(confirmBasicDetailsPath());
   }
 
-  return typedjson({
-    headers: [["Set-Cookie", await commitSession(await clearRedirectTo(request))]],
-  });
+  return typedjson(
+    {},
+    {
+      headers: { "Set-Cookie": await commitSession(await clearRedirectTo(request)) },
+    }
+  );
 };
 
 export default function App() {

apps/webapp/app/routes/admin.orgs.tsx

@@ -1,11 +1,9 @@
 import { MagnifyingGlassIcon } from "@heroicons/react/20/solid";
 import { Form } from "@remix-run/react";
 import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime";
-import { redirect } from "@remix-run/server-runtime";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { z } from "zod";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
-import { Header1 } from "~/components/primitives/Headers";
 import { Input } from "~/components/primitives/Input";
 import { PaginationControls } from "~/components/primitives/Pagination";
 import { Paragraph } from "~/components/primitives/Paragraph";
@@ -18,10 +16,8 @@ import {
   TableHeaderCell,
   TableRow,
 } from "~/components/primitives/Table";
-import { useUser } from "~/hooks/useUser";
-import { adminGetOrganizations, adminGetUsers, setV3Enabled } from "~/models/admin.server";
+import { adminGetOrganizations, setV3Enabled } from "~/models/admin.server";
 import { redirectWithSuccessMessage } from "~/models/message.server";
-import { commitImpersonationSession, setImpersonationId } from "~/services/impersonation.server";
 import { requireUserId } from "~/services/session.server";
 import { createSearchParams } from "~/utils/searchParams";
 

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx

@@ -342,7 +342,9 @@ function Timeline({ startTime, duration, inProgress, isError }: TimelineProps) {
           <DateTimeAccurate date={startTime} />
         </Paragraph>
         {state === "pending" ? (
-          <LiveTimer startTime={startTime} className="" />
+          <Paragraph variant="extra-small" className={cn("whitespace-nowrap tabular-nums")}>
+            <LiveTimer startTime={startTime} />
+          </Paragraph>
         ) : (
           <Paragraph variant="small">
             <DateTimeAccurate

apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts

@@ -63,6 +63,14 @@ export const action: ActionFunction = async ({ request, params }) => {
       { spanId: newRun.spanId }
     );
 
+    logger.debug("Replayed run", {
+      taskRunId: taskRun.id,
+      taskRunFriendlyId: taskRun.friendlyId,
+      newRunId: newRun.id,
+      newRunFriendlyId: newRun.friendlyId,
+      runPath,
+    });
+
     return redirectWithSuccessMessage(runPath, request, `Replaying run`);
   } catch (error) {
     if (error instanceof Error) {

apps/webapp/app/routes/storybook.tree-view/route.tsx

@@ -157,17 +157,17 @@ function TreeViewParent({
     onSelectedIdChanged: (id) => {
       console.log("onSelectedIdChanged", id);
     },
-    onCollapsedIdsChanged: (ids) => {
-      console.log("onCollapsedIdsChanged", ids);
-    },
     estimatedRowHeight: () => 32,
     parentRef,
-    filter: (node) => {
-      if (filterText === "") return true;
-      if (node.data.title.toLowerCase().includes(filterText.toLowerCase())) {
-        return true;
-      }
-      return false;
+    filter: {
+      value: filterText,
+      fn: (text, node) => {
+        if (text === "") return true;
+        if (node.data.title.toLowerCase().includes(text.toLowerCase())) {
+          return true;
+        }
+        return false;
+      },
     },
   });
 

apps/webapp/app/v3/eventRepository.server.ts

@@ -13,6 +13,7 @@ import {
   correctErrorStackTrace,
   createPacketAttributesAsJson,
   flattenAttributes,
+  NULL_SENTINEL,
   isExceptionSpanEvent,
   omit,
   unflattenAttributes,
@@ -438,21 +439,10 @@ export class EventRepository {
       return;
     }
 
-    const output = isEmptyJson(fullEvent.output)
-      ? null
-      : unflattenAttributes(fullEvent.output as Attributes);
+    const output = rehydrateJson(fullEvent.output);
+    const payload = rehydrateJson(fullEvent.payload);
 
-    const payload = isEmptyJson(fullEvent.payload)
-      ? null
-      : unflattenAttributes(fullEvent.payload as Attributes);
-
-    const show = unflattenAttributes(
-      filteredAttributes(fullEvent.properties as Attributes, SemanticInternalAttributes.SHOW)
-    )[SemanticInternalAttributes.SHOW] as
-      | {
-          actions?: boolean;
-        }
-      | undefined;
+    const show = rehydrateShow(fullEvent.properties);
 
     const properties = sanitizedAttributes(fullEvent.properties);
 
@@ -1046,7 +1036,7 @@ function isEmptyJson(json: Prisma.JsonValue) {
   return false;
 }
 
-function sanitizedAttributes(json: Prisma.JsonValue): Record<string, unknown> | undefined {
+function sanitizedAttributes(json: Prisma.JsonValue) {
   if (json === null || json === undefined) {
     return;
   }
@@ -1143,3 +1133,57 @@ function getNowInNanoseconds(): bigint {
 function getDateFromNanoseconds(nanoseconds: bigint) {
   return new Date(Number(nanoseconds) / 1_000_000);
 }
+
+function rehydrateJson(json: Prisma.JsonValue): any {
+  if (json === null) {
+    return undefined;
+  }
+
+  if (json === NULL_SENTINEL) {
+    return null;
+  }
+
+  if (typeof json === "string") {
+    return json;
+  }
+
+  if (typeof json === "number") {
+    return json;
+  }
+
+  if (typeof json === "boolean") {
+    return json;
+  }
+
+  if (Array.isArray(json)) {
+    return json.map((item) => rehydrateJson(item));
+  }
+
+  if (typeof json === "object") {
+    return unflattenAttributes(json as Attributes);
+  }
+
+  return null;
+}
+
+function rehydrateShow(properties: Prisma.JsonValue): { actions?: boolean } | undefined {
+  if (properties === null || properties === undefined) {
+    return;
+  }
+
+  if (typeof properties !== "object") {
+    return;
+  }
+
+  if (Array.isArray(properties)) {
+    return;
+  }
+
+  const actions = properties[SemanticInternalAttributes.SHOW_ACTIONS];
+
+  if (typeof actions === "boolean") {
+    return { actions };
+  }
+
+  return;
+}

apps/webapp/app/v3/marqs/index.server.ts

@@ -19,6 +19,7 @@ import {
   MarQSQueuePriorityStrategy,
   MessagePayload,
   QueueCapacities,
+  QueueRange,
 } from "./types";
 
 const tracer = trace.getTracer("marqs");
@@ -231,7 +232,7 @@ export class MarQS {
           return;
         }
 
-        const message = await this.#readMessage(messageData.messageId);
+        const message = await this.readMessage(messageData.messageId);
 
         if (message) {
           span.setAttributes({
@@ -308,7 +309,7 @@ export class MarQS {
           return;
         }
 
-        const message = await this.#readMessage(messageData.messageId);
+        const message = await this.readMessage(messageData.messageId);
 
         if (message) {
           span.setAttributes({
@@ -336,7 +337,7 @@ export class MarQS {
     return this.#trace(
       "acknowledgeMessage",
       async (span) => {
-        const message = await this.#readMessage(messageId);
+        const message = await this.readMessage(messageId);
 
         if (!message) {
           return;
@@ -374,12 +375,13 @@ export class MarQS {
   public async replaceMessage(
     messageId: string,
     messageData: Record<string, unknown>,
-    timestamp?: number
+    timestamp?: number,
+    inplace?: boolean
   ) {
     return this.#trace(
       "replaceMessage",
       async (span) => {
-        const oldMessage = await this.#readMessage(messageId);
+        const oldMessage = await this.readMessage(messageId);
 
         if (!oldMessage) {
           return;
@@ -392,6 +394,27 @@ export class MarQS {
           [SemanticAttributes.PARENT_QUEUE]: oldMessage.parentQueue,
         });
 
+        const traceContext = {
+          traceparent: oldMessage.data.traceparent,
+          tracestate: oldMessage.data.tracestate,
+        };
+
+        const newMessage: MessagePayload = {
+          version: "1",
+          // preserve original trace context
+          data: { ...messageData, ...traceContext },
+          queue: oldMessage.queue,
+          concurrencyKey: oldMessage.concurrencyKey,
+          timestamp: timestamp ?? Date.now(),
+          messageId,
+          parentQueue: oldMessage.parentQueue,
+        };
+
+        if (inplace) {
+          await this.#callReplaceMessage(newMessage);
+          return;
+        }
+
         await this.#callAcknowledgeMessage({
           parentQueue: oldMessage.parentQueue,
           messageKey: this.keys.messageKey(messageId),
@@ -403,16 +426,6 @@ export class MarQS {
           messageId,
         });
 
-        const newMessage: MessagePayload = {
-          version: "1",
-          data: messageData,
-          queue: oldMessage.queue,
-          concurrencyKey: oldMessage.concurrencyKey,
-          timestamp: timestamp ?? Date.now(),
-          messageId,
-          parentQueue: oldMessage.parentQueue,
-        };
-
         await this.#callEnqueueMessage(newMessage);
       },
       {
@@ -455,7 +468,7 @@ export class MarQS {
     return this.#trace(
       "nackMessage",
       async (span) => {
-        const message = await this.#readMessage(messageId);
+        const message = await this.readMessage(messageId);
 
         if (!message) {
           return;
@@ -505,7 +518,7 @@ export class MarQS {
     return this.options.visibilityTimeoutInMs ?? 300000;
   }
 
-  async #readMessage(messageId: string) {
+  async readMessage(messageId: string) {
     return this.#trace(
       "readMessage",
       async (span) => {
@@ -551,7 +564,7 @@ export class MarQS {
           parentQueue
         );
 
-        const queues = await this.#zrangeWithScores(parentQueue, range[0], range[1]);
+        const queues = await this.#getChildQueuesWithScores(parentQueue, range);
 
         const queuesWithScores = await this.#calculateQueueScores(queues, calculateCapacities);
 
@@ -562,21 +575,25 @@ export class MarQS {
           selectionId
         );
 
-        if (typeof choice !== "string") {
-          return;
-        }
-
         span.setAttributes({
           ...flattenAttributes(queues, "marqs.queues"),
         });
         span.setAttributes({
           ...flattenAttributes(queuesWithScores, "marqs.queuesWithScores"),
         });
-        span.setAttribute("marqs.nextRange", range);
-        span.setAttribute("marqs.queueCount", queues.length);
-        span.setAttribute("marqs.queueChoice", choice);
+        span.setAttribute("nextRange.offset", range.offset);
+        span.setAttribute("nextRange.count", range.count);
+        span.setAttribute("queueCount", queues.length);
 
-        return choice;
+        if (typeof choice !== "string") {
+          span.setAttribute("noQueueChoice", true);
+
+          return;
+        } else {
+          span.setAttribute("queueChoice", choice);
+
+          return choice;
+        }
       },
       {
         kind: SpanKind.CONSUMER,
@@ -620,12 +637,19 @@ export class MarQS {
     });
   }
 
-  async #zrangeWithScores(
+  async #getChildQueuesWithScores(
     key: string,
-    min: number,
-    max: number
+    range: QueueRange
   ): Promise<Array<{ value: string; score: number }>> {
-    const valuesWithScores = await this.redis.zrange(key, min, max, "WITHSCORES");
+    const valuesWithScores = await this.redis.zrangebyscore(
+      key,
+      "-inf",
+      Date.now(),
+      "WITHSCORES",
+      "LIMIT",
+      range.offset,
+      range.count
+    );
     const result: Array<{ value: string; score: number }> = [];
 
     for (let i = 0; i < valuesWithScores.length; i += 2) {
@@ -881,6 +905,17 @@ export class MarQS {
     };
   }
 
+  async #callReplaceMessage(message: MessagePayload) {
+    logger.debug("Calling replaceMessage", {
+      messagePayload: message,
+    });
+
+    return this.redis.replaceMessage(
+      this.keys.messageKey(message.messageId),
+      JSON.stringify(message)
+    );
+  }
+
   async #callAcknowledgeMessage({
     parentQueue,
     messageKey,
@@ -1185,6 +1220,25 @@ return {messageId, messageScore} -- Return message details
       `,
     });
 
+    this.redis.defineCommand("replaceMessage", {
+      numberOfKeys: 1,
+      lua: `
+local messageKey = KEYS[1]
+local messageData = ARGV[1]
+
+-- Check if message exists
+local existingMessage = redis.call('GET', messageKey)
+
+-- Do nothing if it doesn't
+if #existingMessage == nil then
+    return nil
+end
+
+-- Replace the message
+redis.call('SET', messageKey, messageData, 'GET')
+      `,
+    });
+
     this.redis.defineCommand("acknowledgeMessage", {
       numberOfKeys: 7,
       lua: `
@@ -1406,6 +1460,12 @@ declare module "ioredis" {
       callback?: Callback<[string, string]>
     ): Result<[string, string] | null, Context>;
 
+    replaceMessage(
+      messageKey: string,
+      messageData: string,
+      callback?: Callback<void>
+    ): Result<void, Context>;
+
     acknowledgeMessage(
       parentQueue: string,
       messageKey: string,
@@ -1489,7 +1549,7 @@ function getMarQSClient() {
 
       return new MarQS({
         keysProducer: new MarQSShortKeyProducer(KEY_PREFIX),
-        queuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 12 }),
+        queuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 36 }),
         envQueuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 12 }),
         workers: 1,
         redis: redisOptions,

apps/webapp/app/v3/marqs/priorityStrategy.server.ts

@@ -1,5 +1,10 @@
 import { RedisOptions } from "ioredis";
-import { MarQSQueuePriorityStrategy, PriorityStrategyChoice, QueueWithScores } from "./types";
+import {
+  MarQSQueuePriorityStrategy,
+  PriorityStrategyChoice,
+  QueueRange,
+  QueueWithScores,
+} from "./types";
 import { nanoid } from "nanoid";
 import seedrandom from "seedrandom";
 
@@ -26,9 +31,7 @@ export class DynamicWeightedChoiceStrategy implements MarQSQueuePriorityStrategy
     throw new Error("Method not implemented.");
   }
 
-  nextCandidateSelection(
-    parentQueue: string
-  ): Promise<{ range: [number, number]; selectionId: string }> {
+  nextCandidateSelection(parentQueue: string): Promise<{ range: QueueRange; selectionId: string }> {
     throw new Error("Method not implemented.");
   }
 }
@@ -39,13 +42,18 @@ export type SimpleWeightedChoiceStrategyOptions = {
 };
 
 export class SimpleWeightedChoiceStrategy implements MarQSQueuePriorityStrategy {
-  private _nextRangesByParentQueue: Map<string, [number, number]> = new Map();
+  private _nextRangesByParentQueue: Map<string, QueueRange> = new Map();
   private _randomGenerator = seedrandom(this.options.randomSeed);
 
   constructor(private options: SimpleWeightedChoiceStrategyOptions) {}
 
-  private nextRangeForParentQueue(parentQueue: string) {
-    return this._nextRangesByParentQueue.get(parentQueue) ?? [0, this.options.queueSelectionCount];
+  private nextRangeForParentQueue(parentQueue: string): QueueRange {
+    return (
+      this._nextRangesByParentQueue.get(parentQueue) ?? {
+        offset: 0,
+        count: this.options.queueSelectionCount,
+      }
+    );
   }
 
   chooseQueue(
@@ -55,25 +63,23 @@ export class SimpleWeightedChoiceStrategy implements MarQSQueuePriorityStrategy
   ): PriorityStrategyChoice {
     const filteredQueues = filterQueuesAtCapacity(queues);
 
-    if (filteredQueues.length === 0) {
-      if (queues.length === this.options.queueSelectionCount) {
-        const nextRangeForParentQueue = this.nextRangeForParentQueue(parentQueue);
-        const nextRange: [number, number] = nextRangeForParentQueue
-          ? [
-              nextRangeForParentQueue[1],
-              nextRangeForParentQueue[1] + this.options.queueSelectionCount,
-            ]
-          : [this.options.queueSelectionCount, this.options.queueSelectionCount * 2];
-        // If all queues are at capacity, and we were passed the max number of queues, then we will slide the window "to the right"
-        this._nextRangesByParentQueue.set(parentQueue, nextRange);
-      } else {
-        this._nextRangesByParentQueue.delete(parentQueue);
-      }
-
-      return { abort: true };
+    if (queues.length === this.options.queueSelectionCount) {
+      const nextRangeForParentQueue = this.nextRangeForParentQueue(parentQueue);
+      const nextRange: QueueRange = nextRangeForParentQueue
+        ? {
+            offset: nextRangeForParentQueue.offset + this.options.queueSelectionCount,
+            count: this.options.queueSelectionCount,
+          }
+        : { offset: this.options.queueSelectionCount, count: this.options.queueSelectionCount };
+      // If all queues are at capacity, and we were passed the max number of queues, then we will slide the window "to the right"
+      this._nextRangesByParentQueue.set(parentQueue, nextRange);
+    } else {
+      this._nextRangesByParentQueue.delete(parentQueue);
     }
 
-    this._nextRangesByParentQueue.delete(parentQueue);
+    if (filteredQueues.length === 0) {
+      return { abort: true };
+    }
 
     const queueWeights = this.#calculateQueueWeights(filteredQueues);
 
@@ -82,7 +88,7 @@ export class SimpleWeightedChoiceStrategy implements MarQSQueuePriorityStrategy
 
   async nextCandidateSelection(
     parentQueue: string
-  ): Promise<{ range: [number, number]; selectionId: string }> {
+  ): Promise<{ range: QueueRange; selectionId: string }> {
     return { range: this.nextRangeForParentQueue(parentQueue), selectionId: nanoid(24) };
   }
 

apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts

@@ -27,14 +27,15 @@ import { generateFriendlyId } from "../friendlyIdentifiers";
 import { socketIo } from "../handleSocketIo.server";
 import { findCurrentWorkerDeployment } from "../models/workerDeployment.server";
 import { RestoreCheckpointService } from "../services/restoreCheckpoint.server";
-import { tracer } from "../tracer.server";
+import { SEMINTATTRS_FORCE_RECORDING, tracer } from "../tracer.server";
+import { CrashTaskRunService } from "../services/crashTaskRun.server";
 
 const WithTraceContext = z.object({
   traceparent: z.string().optional(),
   tracestate: z.string().optional(),
 });
 
-const MessageBody = z.discriminatedUnion("type", [
+export const SharedQueueMessageBody = z.discriminatedUnion("type", [
   WithTraceContext.extend({
     type: z.literal("EXECUTE"),
     taskIdentifier: z.string(),
@@ -51,8 +52,14 @@ const MessageBody = z.discriminatedUnion("type", [
     resumableAttemptId: z.string(),
     checkpointEventId: z.string(),
   }),
+  WithTraceContext.extend({
+    type: z.literal("FAIL"),
+    reason: z.string(),
+  }),
 ]);
 
+export type SharedQueueMessageBody = z.infer<typeof SharedQueueMessageBody>;
+
 type BackgroundWorkerWithTasks = BackgroundWorker & { tasks: BackgroundWorkerTask[] };
 
 export type SharedQueueConsumerOptions = {
@@ -60,7 +67,6 @@ export type SharedQueueConsumerOptions = {
   traceTimeoutSeconds?: number;
   nextTickInterval?: number;
   interval?: number;
-  parentContext?: Context;
 };
 
 export class SharedQueueConsumer {
@@ -86,7 +92,6 @@ export class SharedQueueConsumer {
       traceTimeoutSeconds: options.traceTimeoutSeconds ?? 60, // 60 seconds
       nextTickInterval: options.nextTickInterval ?? 1000, // 1 second
       interval: options.interval ?? 100, // 100ms
-      parentContext: options.parentContext ?? ROOT_CONTEXT,
     };
   }
 
@@ -186,19 +191,17 @@ export class SharedQueueConsumer {
     ) {
       this.#endCurrentSpan();
 
-      const parentContext = this._options.parentContext ?? ROOT_CONTEXT;
-
       // Create a new trace
       this._currentSpan = tracer.startSpan(
         "SharedQueueConsumer.doWork()",
         {
           kind: SpanKind.CONSUMER,
         },
-        parentContext
+        ROOT_CONTEXT
       );
 
       // Get the span trace context
-      this._currentSpanContext = trace.setSpan(parentContext, this._currentSpan);
+      this._currentSpanContext = trace.setSpan(ROOT_CONTEXT, this._currentSpan);
 
       this._perTraceCountdown = this._options.maximumItemsPerTrace;
       this._lastNewTrace = new Date();
@@ -233,7 +236,7 @@ export class SharedQueueConsumer {
 
     logger.log("dequeueMessageInSharedQueue()", { queueMessage: message });
 
-    const messageBody = MessageBody.safeParse(message.data);
+    const messageBody = SharedQueueMessageBody.safeParse(message.data);
 
     if (!messageBody.success) {
       logger.error("Failed to parse message", {
@@ -411,11 +414,21 @@ export class SharedQueueConsumer {
         });
 
         if (!queue) {
+          logger.debug("SharedQueueConsumer queue not found, so nacking message", {
+            queueMessage: message,
+            taskRunQueue: lockedTaskRun.queue,
+            runtimeEnvironmentId: lockedTaskRun.runtimeEnvironmentId,
+          });
+
           await this.#nackAndDoMoreWork(message.messageId, this._options.nextTickInterval);
           return;
         }
 
         if (!this._enabled) {
+          logger.debug("SharedQueueConsumer not enabled, so nacking message", {
+            queueMessage: message,
+          });
+
           await marqs?.nackMessage(message.messageId);
           return;
         }
@@ -521,6 +534,11 @@ export class SharedQueueConsumer {
             }),
           ]);
 
+          logger.error("SharedQueueConsumer errored, so nacking message", {
+            queueMessage: message,
+            error: e instanceof Error ? { name: e.name, message: e.message, stack: e.stack } : e,
+          });
+
           await this.#nackAndDoMoreWork(message.messageId);
           return;
         }
@@ -739,6 +757,34 @@ export class SharedQueueConsumer {
 
         break;
       }
+      // Fail for whatever reason, usually runs that have been resumed but stopped heartbeating
+      case "FAIL": {
+        const existingTaskRun = await prisma.taskRun.findUnique({
+          where: {
+            id: message.messageId,
+          },
+        });
+
+        if (!existingTaskRun) {
+          logger.error("No existing task run to fail", {
+            queueMessage: messageBody,
+            messageId: message.messageId,
+          });
+
+          await this.#ackAndDoMoreWork(message.messageId);
+          return;
+        }
+
+        // TODO: Consider failing the attempt and retrying instead. This may not be a good idea, as dequeued FAIL messages tend to point towards critical, persistent errors.
+        const service = new CrashTaskRunService();
+        await service.call(existingTaskRun.id, {
+          crashAttempts: true,
+          reason: messageBody.data.reason,
+        });
+
+        await this.#ackAndDoMoreWork(message.messageId);
+        return;
+      }
     }
 
     this.#doMoreWork();

apps/webapp/app/v3/marqs/types.ts

@@ -18,6 +18,8 @@ export type QueueWithScores = {
   age: number;
 };
 
+export type QueueRange = { offset: number; count: number };
+
 export interface MarQSKeyProducer {
   queueConcurrencyLimitKey(env: AuthenticatedEnvironment, queue: string): string;
   envConcurrencyLimitKey(env: AuthenticatedEnvironment): string;
@@ -69,9 +71,7 @@ export interface MarQSQueuePriorityStrategy {
    *
    * @returns The scores and the selectionId for the next candidate selection
    */
-  nextCandidateSelection(
-    parentQueue: string
-  ): Promise<{ range: [number, number]; selectionId: string }>;
+  nextCandidateSelection(parentQueue: string): Promise<{ range: QueueRange; selectionId: string }>;
 }
 
 export const MessagePayload = z.object({

apps/webapp/app/v3/services/createCheckpoint.server.ts

@@ -10,6 +10,7 @@ import { generateFriendlyId } from "../friendlyIdentifiers";
 import { marqs } from "~/v3/marqs/index.server";
 import { CreateCheckpointRestoreEventService } from "./createCheckpointRestoreEvent.server";
 import { BaseService } from "./baseService.server";
+import { CrashTaskRunService } from "./crashTaskRun.server";
 
 const FREEZABLE_RUN_STATUSES: TaskRunStatus[] = ["EXECUTING", "RETRYING_AFTER_FAILURE"];
 const FREEZABLE_ATTEMPT_STATUSES: TaskRunAttemptStatus[] = ["EXECUTING", "FAILED"];
@@ -61,6 +62,14 @@ export class CreateCheckpointService extends BaseService {
           status: attempt.taskRun.status,
         },
       });
+
+      // This should only affect CLIs < beta.24, in very limited scenarios
+      const service = new CrashTaskRunService(this._prisma);
+      await service.call(attempt.taskRunId, {
+        crashAttempts: true,
+        reason: "Unfreezable state: Please upgrade your CLI",
+      });
+
       return;
     }
 

apps/webapp/app/v3/services/resumeAttempt.server.ts

@@ -2,13 +2,14 @@ import {
   CoordinatorToPlatformMessages,
   TaskRunExecution,
   TaskRunExecutionResult,
+  WaitReason,
 } from "@trigger.dev/core/v3";
 import type { InferSocketMessageSchema } from "@trigger.dev/core/v3/zodSocket";
 import { $transaction, PrismaClientOrTransaction } from "~/db.server";
 import { logger } from "~/services/logger.server";
 import { marqs } from "~/v3/marqs/index.server";
 import { socketIo } from "../handleSocketIo.server";
-import { sharedQueueTasks } from "../marqs/sharedQueueConsumer.server";
+import { SharedQueueMessageBody, sharedQueueTasks } from "../marqs/sharedQueueConsumer.server";
 import { BaseService } from "./baseService.server";
 import { TaskRunAttempt } from "@trigger.dev/database";
 
@@ -91,12 +92,13 @@ export class ResumeAttemptService extends BaseService {
 
       switch (params.type) {
         case "WAIT_FOR_DURATION": {
-          logger.error(
-            "Attempt requested resume after duration wait, this is unexpected and likely a bug",
-            { attemptId: attempt.id }
-          );
+          logger.debug("Sending duration wait resume message", {
+            attemptId: attempt.id,
+            attemptFriendlyId: params.attemptFriendlyId,
+          });
+
+          await this.#setPostResumeStatuses(attempt, tx);
 
-          // Attempts should not request resume for duration waits, this is just here as a backup
           socketIo.coordinatorNamespace.emit("RESUME_AFTER_DURATION", {
             version: "v1",
             attemptId: attempt.id,
@@ -119,6 +121,9 @@ export class ResumeAttemptService extends BaseService {
             logger.error("No task dependency", { attemptId: attempt.id });
             return;
           }
+
+          await this.#handleDependencyResume(attempt, completedAttemptIds, tx);
+
           break;
         }
         case "WAIT_FOR_BATCH": {
@@ -136,6 +141,9 @@ export class ResumeAttemptService extends BaseService {
             logger.error("No batch dependency", { attemptId: attempt.id });
             return;
           }
+
+          await this.#handleDependencyResume(attempt, completedAttemptIds, tx);
+
           break;
         }
         default: {
@@ -143,7 +151,8 @@ export class ResumeAttemptService extends BaseService {
         }
       }
 
-      await this.#handleDependencyResume(attempt, completedAttemptIds, tx);
+      // Prevent infinite restores by failing runs that don't heartbeat after post-restore resume requests
+      await this.#replaceResumeWithFailMessage(attempt.taskRunId, params.type);
     });
   }
 
@@ -215,7 +224,20 @@ export class ResumeAttemptService extends BaseService {
       executions.push(executionPayload.execution);
     }
 
-    const updated = await tx.taskRunAttempt.update({
+    await this.#setPostResumeStatuses(attempt, tx);
+
+    socketIo.coordinatorNamespace.emit("RESUME_AFTER_DEPENDENCY", {
+      version: "v1",
+      runId: attempt.taskRunId,
+      attemptId: attempt.id,
+      attemptFriendlyId: attempt.friendlyId,
+      completions,
+      executions,
+    });
+  }
+
+  async #setPostResumeStatuses(attempt: TaskRunAttempt, tx: PrismaClientOrTransaction) {
+    return await tx.taskRunAttempt.update({
       where: {
         id: attempt.id,
       },
@@ -230,14 +252,51 @@ export class ResumeAttemptService extends BaseService {
         },
       },
     });
+  }
 
-    socketIo.coordinatorNamespace.emit("RESUME_AFTER_DEPENDENCY", {
-      version: "v1",
-      runId: attempt.taskRunId,
-      attemptId: attempt.id,
-      attemptFriendlyId: attempt.friendlyId,
-      completions,
-      executions,
-    });
+  async #replaceResumeWithFailMessage(messageId: string, waitReason: WaitReason) {
+    const currentMessage = await marqs?.readMessage(messageId);
+
+    if (!currentMessage) {
+      logger.debug("No message to replace", { messageId, waitReason });
+      return;
+    }
+
+    const currentBody = SharedQueueMessageBody.safeParse(currentMessage.data);
+
+    if (!currentBody.success) {
+      logger.debug("Invalid message body", { messageId, waitReason, currentBody });
+      return;
+    }
+
+    const currentType = currentBody.data.type;
+
+    if (currentType !== "RESUME" && currentType !== "RESUME_AFTER_DURATION") {
+      logger.debug("Not a resume message", { messageId, waitReason, currentBody });
+      return;
+    }
+
+    let reason = "Worker unresponsive after restore";
+
+    switch (waitReason) {
+      case "WAIT_FOR_DURATION":
+        reason = "Worker unresponsive after waiting for duration";
+        break;
+      case "WAIT_FOR_TASK":
+        reason = "Worker unresponsive after waiting for task";
+        break;
+      case "WAIT_FOR_BATCH":
+        reason = "Worker unresponsive after waiting for batch task";
+        break;
+      default:
+        break;
+    }
+
+    const failMessage: SharedQueueMessageBody = {
+      type: "FAIL",
+      reason,
+    };
+
+    return await marqs?.replaceMessage(messageId, failMessage, undefined, true);
   }
 }

apps/webapp/app/v3/sharedSocketConnection.ts

@@ -1,19 +1,17 @@
+import { trace } from "@opentelemetry/api";
 import { clientWebsocketMessages, serverWebsocketMessages } from "@trigger.dev/core/v3";
+import type { StructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
 import {
+  MessageCatalogToSocketIoEvents,
   ZodMessageHandler,
   ZodMessageSender,
-  MessageCatalogToSocketIoEvents,
 } from "@trigger.dev/core/v3/zodMessageHandler";
-import type { StructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
 import { Evt } from "evt";
 import { randomUUID } from "node:crypto";
+import type { DisconnectReason, Namespace, Socket } from "socket.io";
+import { env } from "~/env.server";
 import { logger } from "~/services/logger.server";
 import { SharedQueueConsumer } from "./marqs/sharedQueueConsumer.server";
-import type { DisconnectReason, Namespace, Socket } from "socket.io";
-import { ROOT_CONTEXT, Span, SpanKind, trace } from "@opentelemetry/api";
-import { env } from "~/env.server";
-
-const tracer = trace.getTracer("sharedQueueConsumerPool");
 
 interface SharedQueueConsumerPoolOptions {
   sender: ZodMessageSender<typeof serverWebsocketMessages>;
@@ -22,22 +20,8 @@ interface SharedQueueConsumerPoolOptions {
 
 class SharedQueueConsumerPool {
   #consumers: SharedQueueConsumer[];
-  #span: Span;
 
   constructor(opts: SharedQueueConsumerPoolOptions) {
-    this.#span = tracer.startSpan(
-      "SharedQueueConsumerPool()",
-      {
-        kind: SpanKind.CONSUMER,
-        attributes: {
-          "pool.size": opts.poolSize,
-        },
-      },
-      ROOT_CONTEXT
-    );
-
-    const spanContext = trace.setSpan(ROOT_CONTEXT, this.#span);
-
     this.#consumers = Array(opts.poolSize)
       .fill(null)
       .map(
@@ -45,7 +29,6 @@ class SharedQueueConsumerPool {
           new SharedQueueConsumer(opts.sender, {
             interval: env.SHARED_QUEUE_CONSUMER_INTERVAL_MS,
             nextTickInterval: env.SHARED_QUEUE_CONSUMER_NEXT_TICK_INTERVAL_MS,
-            parentContext: spanContext,
           })
       );
   }
@@ -56,7 +39,6 @@ class SharedQueueConsumerPool {
 
   async stop() {
     await Promise.allSettled(this.#consumers.map((consumer) => consumer.stop()));
-    this.#span.end();
   }
 }
 
@@ -100,7 +82,7 @@ export class SharedSocketConnection {
       },
     });
 
-    logger.log("Starting SharedQueueConsumer pool", {
+    logger.debug("Starting SharedQueueConsumer pool", {
       poolSize: opts.poolSize ?? this._defaultPoolSize,
     });
 

apps/webapp/app/v3/tracer.server.ts

@@ -125,7 +125,7 @@ function getTracer() {
     );
   } else {
     if (env.INTERNAL_OTEL_TRACE_LOGGING_ENABLED === "1") {
-      console.log(`🔦 Tracer: Logger exporter enabled`);
+      console.log(`🔦 Tracer: Logger exporter enabled (sampling = ${samplingRate})`);
 
       const loggerExporter = new LoggerSpanExporter();
 

apps/webapp/package.json

@@ -16,6 +16,8 @@
     "typecheck": "tsc -p ./tsconfig.check.json",
     "db:seed": "node prisma/seed.js",
     "db:seed:local": "ts-node prisma/seed.ts",
+    "build:db:populate": "esbuild --platform=node --bundle --minify --format=cjs ./prisma/populate.ts --outdir=prisma",
+    "db:populate": "node prisma/populate.js --",
     "generate:sourcemaps": "remix build --sourcemap",
     "clean:sourcemaps": "run-s clean:sourcemaps:*",
     "clean:sourcemaps:public": "rimraf ./build/**/*.map",
@@ -58,6 +60,7 @@
     "@opentelemetry/sdk-trace-base": "^1.22.0",
     "@opentelemetry/sdk-trace-node": "^1.22.0",
     "@opentelemetry/semantic-conventions": "^1.22.0",
+    "@popperjs/core": "^2.11.8",
     "@prisma/instrumentation": "^5.11.0",
     "@radix-ui/react-alert-dialog": "^1.0.4",
     "@radix-ui/react-dialog": "^1.0.3",
@@ -105,6 +108,7 @@
     "cronstrue": "^2.21.0",
     "cross-env": "^7.0.3",
     "cuid": "^2.1.8",
+    "dotenv": "^16.4.5",
     "emails": "workspace:*",
     "evt": "^2.4.13",
     "express": "^4.18.1",
@@ -136,6 +140,7 @@
     "react-collapse": "^5.1.1",
     "react-dom": "^18.2.0",
     "react-hotkeys-hook": "^4.4.1",
+    "react-popper": "^2.3.0",
     "react-resizable-panels": "^2.0.9",
     "react-stately": "^3.29.1",
     "react-use": "^17.4.0",
@@ -227,4 +232,4 @@
   "engines": {
     "node": ">=16.0.0"
   }
-}
+}

apps/webapp/prisma/populate.ts

@@ -0,0 +1,101 @@
+// Bulk adds data to the database for testing
+// Call it like this
+// 1. pnpm run build:db:populate
+// 2. pnpm run db:populate -- --projectRef=proj_liazlkfgmfcusswwgohl --taskIdentifier=child-task --runCount=100000
+import { generateFriendlyId } from "~/v3/friendlyIdentifiers";
+import { prisma } from "../app/db.server";
+
+async function populate() {
+  if (process.env.NODE_ENV !== "development") {
+    return;
+  }
+
+  const projectRef = getArg("projectRef");
+  if (!projectRef) {
+    throw new Error("projectRef is required");
+  }
+
+  const project = await prisma.project.findUnique({
+    include: {
+      environments: true,
+    },
+    where: {
+      externalRef: projectRef,
+    },
+  });
+  if (!project) {
+    throw new Error("Project not found");
+  }
+
+  const taskIdentifier = getArg("taskIdentifier");
+  if (!taskIdentifier) {
+    throw new Error("taskIdentifier is required");
+  }
+
+  const runCount = parseInt(getArg("runCount") || "100");
+
+  const task = await prisma.backgroundWorkerTask.findFirst({
+    where: {
+      projectId: project.id,
+      slug: taskIdentifier,
+    },
+    orderBy: {
+      createdAt: "desc",
+    },
+  });
+
+  if (!task) {
+    throw new Error("Task not found");
+  }
+
+  const runs = await prisma.taskRun.createMany({
+    data: Array(runCount)
+      .fill(0)
+      .map((_, index) => {
+        const friendlyId = generateFriendlyId("run");
+
+        return {
+          status: "CANCELED",
+          number: index + 1,
+          friendlyId,
+          runtimeEnvironmentId: project.environments[randomIndex(project.environments)].id,
+          projectId: project.id,
+          taskIdentifier,
+          payload: JSON.stringify({ foo: "bar" }),
+          traceId: "traceId",
+          spanId: "spanId",
+          queue: "task/${taskIdentifier}",
+        };
+      }),
+    skipDuplicates: true,
+  });
+
+  console.log(`Added ${runs.count} runs`);
+}
+
+function getArg(name: string) {
+  const args = process.argv.slice(2);
+
+  let value = "";
+
+  args.forEach((val) => {
+    if (val.startsWith(`--${name}=`)) {
+      value = val.split("=")[1];
+    }
+  });
+
+  return !value ? undefined : value;
+}
+
+function randomIndex<T>(array: T[]) {
+  return Math.floor(Math.random() * array.length);
+}
+
+populate()
+  .catch((e) => {
+    console.error(e);
+    process.exit(1);
+  })
+  .finally(async () => {
+    await prisma.$disconnect();
+  });

apps/webapp/remix.config.js

@@ -21,6 +21,7 @@ module.exports = {
     "random-words",
     "superjson",
   ],
+  browserNodeBuiltinsPolyfill: { modules: { path: true, os: true, crypto: true } },
   watchPaths: async () => {
     return [
       "../../packages/core/src/**/*",

apps/webapp/test/marqs.test.ts

@@ -88,7 +88,10 @@ describe("SimpleWeightedChoiceStrategy", () => {
 
     const nextSelection = await stategy.nextCandidateSelection("parentQueue");
 
-    expect(nextSelection).toEqual({ range: [3, 6], selectionId: expect.any(String) });
+    expect(nextSelection).toEqual({
+      range: { offset: 3, count: 3 },
+      selectionId: expect.any(String),
+    });
 
     // Now pass some queues that have some capacity
     const chosenQueue2 = stategy.chooseQueue(
@@ -129,7 +132,10 @@ describe("SimpleWeightedChoiceStrategy", () => {
 
     const nextSelection2 = await stategy.nextCandidateSelection("parentQueue");
 
-    expect(nextSelection2).toEqual({ range: [0, 3], selectionId: expect.any(String) });
+    expect(nextSelection2).toEqual({
+      range: { offset: 6, count: 3 },
+      selectionId: expect.any(String),
+    });
   });
 
   it("should adjust the next filter range only if passed the maximum number of queues", async () => {
@@ -167,7 +173,10 @@ describe("SimpleWeightedChoiceStrategy", () => {
 
     const nextSelection = await stategy.nextCandidateSelection("parentQueue");
 
-    expect(nextSelection).toEqual({ range: [0, 3], selectionId: expect.any(String) });
+    expect(nextSelection).toEqual({
+      range: { offset: 0, count: 3 },
+      selectionId: expect.any(String),
+    });
   });
 
   it("should adjust the next candidate range ONLY for the matching parent queue", async () => {
@@ -182,7 +191,7 @@ describe("SimpleWeightedChoiceStrategy", () => {
           queue: "queue1",
           age: 4497,
           capacities: {
-            queue: { current: 10, limit: 10 },
+            queue: { current: 0, limit: 10 },
             env: { current: 0, limit: 10 },
             org: { current: 0, limit: 10 },
           },
@@ -210,15 +219,21 @@ describe("SimpleWeightedChoiceStrategy", () => {
       "selectionId"
     );
 
-    expect(chosenQueue).toEqual({ abort: true });
+    expect(chosenQueue).toEqual("queue1");
 
     const nextSelection = await stategy.nextCandidateSelection("parentQueue2");
 
-    expect(nextSelection).toEqual({ range: [0, 3], selectionId: expect.any(String) });
+    expect(nextSelection).toEqual({
+      range: { offset: 0, count: 3 },
+      selectionId: expect.any(String),
+    });
 
     const nextSelection2 = await stategy.nextCandidateSelection("parentQueue");
 
-    expect(nextSelection2).toEqual({ range: [3, 6], selectionId: expect.any(String) });
+    expect(nextSelection2).toEqual({
+      range: { offset: 3, count: 3 },
+      selectionId: expect.any(String),
+    });
 
     const chosenQueue2 = stategy.chooseQueue(
       [
@@ -250,14 +265,52 @@ describe("SimpleWeightedChoiceStrategy", () => {
           },
         },
       ],
-      "parentQueue2",
+      "parentQueue",
       "selectionId"
     );
 
-    expect(chosenQueue2).toEqual("queue3");
+    expect(chosenQueue2).toEqual("queue2");
 
-    const nextSelection3 = await stategy.nextCandidateSelection("parentQueue2");
+    const nextSelection3 = await stategy.nextCandidateSelection("parentQueue");
 
-    expect(nextSelection3).toEqual({ range: [0, 3], selectionId: expect.any(String) });
+    expect(nextSelection3).toEqual({
+      range: { offset: 6, count: 3 },
+      selectionId: expect.any(String),
+    });
+
+    // Not passed 3 queues, so the range should be reset (we've reached the end)
+    const chosenQueue3 = stategy.chooseQueue(
+      [
+        {
+          queue: "queue1",
+          age: 4497,
+          capacities: {
+            queue: { current: 0, limit: 10 },
+            env: { current: 0, limit: 10 },
+            org: { current: 0, limit: 10 },
+          },
+        },
+        {
+          queue: "queue2",
+          age: 19670,
+          capacities: {
+            queue: { current: 0, limit: 10 },
+            env: { current: 0, limit: 10 },
+            org: { current: 0, limit: 10 },
+          },
+        },
+      ],
+      "parentQueue",
+      "selectionId"
+    );
+
+    expect(chosenQueue3).toEqual("queue2");
+
+    const nextSelection4 = await stategy.nextCandidateSelection("parentQueue");
+
+    expect(nextSelection4).toEqual({
+      range: { offset: 0, count: 3 },
+      selectionId: expect.any(String),
+    });
   });
 });

docs/v3/errors-retrying.mdx

@@ -29,6 +29,8 @@ This task will retry 10 times with exponential backoff.
 One way to gain reliability is to break your work into smaller tasks and [trigger](/v3/triggering) them from each other. Each task can have its own retrying behavior:
 
 ```ts /trigger/multiple-tasks.ts
+import { task } from "@trigger.dev/sdk/v3";
+
 export const myTask = task({
   id: "my-task",
   retry: {
@@ -66,6 +68,8 @@ We provide some useful functions that you can use to retry smaller parts of a ta
 You can retry a block of code that can throw an error, with the same retry settings as a task.
 
 ```ts /trigger/retry-on-throw.ts
+import { task, logger, retry } from "@trigger.dev/sdk/v3";
+
 export const retryOnThrow = task({
   id: "retry-on-throw",
   run: async (payload: any) => {
@@ -100,6 +104,8 @@ You can use `fetch`, `axios`, or any other library in your code.
 But we do provide a convenient function to perform HTTP requests with conditional retrying based on the response:
 
 ```ts /trigger/retry-fetch.ts
+import { task, logger, retry } from "@trigger.dev/sdk/v3";
+
 export const taskWithFetchRetries = task({
   id: "task-with-fetch-retries",
   run: async ({ payload, ctx }) => {
@@ -193,6 +199,8 @@ In this complicated example:
 - If we've run out of requests or tokens we retry at the time specified in the headers.
 
 ```ts
+import { task } from "@trigger.dev/sdk/v3";
+
 export const openaiTask = task({
   id: "openai-task",
   retry: {

integrations/airtable/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @trigger.dev/airtable
 
+## 3.0.0-beta.25
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.25
+- @trigger.dev/sdk@3.0.0-beta.25
+
+## 3.0.0-beta.24
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.24
+- @trigger.dev/sdk@3.0.0-beta.24
+
 ## 3.0.0-beta.23
 
 ### Patch Changes

integrations/airtable/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/airtable",
-  "version": "3.0.0-beta.23",
+  "version": "3.0.0-beta.25",
   "description": "Trigger.dev integration for airtable",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -25,8 +25,8 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
     "airtable": "^0.12.1",
     "zod": "3.22.3"
   },

integrations/github/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @trigger.dev/github
 
+## 3.0.0-beta.25
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.25
+- @trigger.dev/sdk@3.0.0-beta.25
+
+## 3.0.0-beta.24
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.24
+- @trigger.dev/sdk@3.0.0-beta.24
+
 ## 3.0.0-beta.23
 
 ### Patch Changes

integrations/github/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/github",
-  "version": "3.0.0-beta.23",
+  "version": "3.0.0-beta.25",
   "description": "The official GitHub integration for Trigger.dev",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -30,8 +30,8 @@
     "@octokit/request-error": "^5.0.1",
     "@octokit/webhooks": "^12.0.10",
     "octokit": "^3.1.2",
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
     "zod": "3.22.3"
   },
   "engines": {

integrations/linear/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @trigger.dev/linear
 
+## 3.0.0-beta.25
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.25
+- @trigger.dev/sdk@3.0.0-beta.25
+
+## 3.0.0-beta.24
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.24
+- @trigger.dev/sdk@3.0.0-beta.24
+
 ## 3.0.0-beta.23
 
 ### Patch Changes

integrations/linear/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/linear",
-  "version": "3.0.0-beta.23",
+  "version": "3.0.0-beta.25",
   "description": "Trigger.dev integration for @linear/sdk",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -26,8 +26,8 @@
   },
   "dependencies": {
     "@linear/sdk": "^8.0.0",
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
     "zod": "3.22.3"
   },
   "engines": {

integrations/openai/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @trigger.dev/slack
 
+## 3.0.0-beta.25
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.25
+- @trigger.dev/sdk@3.0.0-beta.25
+
+## 3.0.0-beta.24
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.24
+- @trigger.dev/sdk@3.0.0-beta.24
+
 ## 3.0.0-beta.23
 
 ### Patch Changes

integrations/openai/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/openai",
-  "version": "3.0.0-beta.23",
+  "version": "3.0.0-beta.25",
   "description": "The official OpenAI integration for Trigger.dev",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -42,8 +42,8 @@
   },
   "dependencies": {
     "openai": "^4.16.1",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23"
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25"
   },
   "engines": {
     "node": ">=18.0.0"

integrations/plain/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @trigger.dev/plain
 
+## 3.0.0-beta.25
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.25
+- @trigger.dev/sdk@3.0.0-beta.25
+
+## 3.0.0-beta.24
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.24
+- @trigger.dev/sdk@3.0.0-beta.24
+
 ## 3.0.0-beta.23
 
 ### Patch Changes

integrations/plain/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/plain",
-  "version": "3.0.0-beta.23",
+  "version": "3.0.0-beta.25",
   "description": "The official Plain.com integration for Trigger.dev",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -24,8 +24,8 @@
     "build:tsup": "tsup"
   },
   "dependencies": {
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
     "@team-plain/typescript-sdk": "^2.7.0"
   },
   "engines": {

integrations/replicate/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @trigger.dev/replicate
 
+## 3.0.0-beta.25
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.25
+- @trigger.dev/sdk@3.0.0-beta.25
+
+## 3.0.0-beta.24
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.24
+- @trigger.dev/sdk@3.0.0-beta.24
+
 ## 3.0.0-beta.23
 
 ### Patch Changes

integrations/replicate/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/replicate",
-  "version": "3.0.0-beta.23",
+  "version": "3.0.0-beta.25",
   "description": "Trigger.dev integration for replicate",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -25,8 +25,8 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
     "replicate": "^0.18.1",
     "zod": "3.22.3"
   },

integrations/resend/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @trigger.dev/resend
 
+## 3.0.0-beta.25
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.25
+- @trigger.dev/sdk@3.0.0-beta.25
+
+## 3.0.0-beta.24
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.24
+- @trigger.dev/sdk@3.0.0-beta.24
+
 ## 3.0.0-beta.23
 
 ### Patch Changes

integrations/resend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/resend",
-  "version": "3.0.0-beta.23",
+  "version": "3.0.0-beta.25",
   "description": "The official Resend.com integration for Trigger.dev",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -24,8 +24,8 @@
     "build:tsup": "tsup"
   },
   "dependencies": {
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
     "resend": "^2.1.0"
   },
   "engines": {

integrations/sendgrid/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @trigger.dev/sendgrid
 
+## 3.0.0-beta.25
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.25
+- @trigger.dev/sdk@3.0.0-beta.25
+
+## 3.0.0-beta.24
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.24
+- @trigger.dev/sdk@3.0.0-beta.24
+
 ## 3.0.0-beta.23
 
 ### Patch Changes

integrations/sendgrid/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/sendgrid",
-  "version": "3.0.0-beta.23",
+  "version": "3.0.0-beta.25",
   "description": "Trigger.dev integration for @sendgrid/mail",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -26,8 +26,8 @@
   },
   "dependencies": {
     "@sendgrid/mail": "^7.7.0",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23"
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25"
   },
   "engines": {
     "node": ">=16.8.0"

integrations/shopify/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @trigger.dev/shopify
 
+## 3.0.0-beta.25
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.25
+- @trigger.dev/sdk@3.0.0-beta.25
+
+## 3.0.0-beta.24
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.24
+- @trigger.dev/sdk@3.0.0-beta.24
+
 ## 3.0.0-beta.23
 
 ### Patch Changes

integrations/shopify/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/shopify",
-  "version": "3.0.0-beta.23",
+  "version": "3.0.0-beta.25",
   "description": "Trigger.dev integration for @shopify/shopify-api",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -26,8 +26,8 @@
   },
   "dependencies": {
     "@shopify/shopify-api": "^8.0.2",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
     "zod": "3.22.3"
   },
   "engines": {

integrations/slack/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @trigger.dev/slack
 
+## 3.0.0-beta.25
+
+### Patch Changes
+
+- @trigger.dev/sdk@3.0.0-beta.25
+
+## 3.0.0-beta.24
+
+### Patch Changes
+
+- @trigger.dev/sdk@3.0.0-beta.24
+
 ## 3.0.0-beta.23
 
 ### Patch Changes

integrations/slack/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/slack",
-  "version": "3.0.0-beta.23",
+  "version": "3.0.0-beta.25",
   "description": "The official Slack integration for Trigger.dev",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -25,7 +25,7 @@
   },
   "dependencies": {
     "@slack/web-api": "^6.8.1",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
     "zod": "3.22.3"
   },
   "engines": {

integrations/stripe/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @trigger.dev/stripe
 
+## 3.0.0-beta.25
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.25
+- @trigger.dev/sdk@3.0.0-beta.25
+
+## 3.0.0-beta.24
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.24
+- @trigger.dev/sdk@3.0.0-beta.24
+
 ## 3.0.0-beta.23
 
 ### Patch Changes

integrations/stripe/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/stripe",
-  "version": "3.0.0-beta.23",
+  "version": "3.0.0-beta.25",
   "description": "Trigger.dev integration for stripe",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -25,8 +25,8 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
     "stripe": "^12.14.0",
     "zod": "3.22.3"
   },

integrations/supabase/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @trigger.dev/supabase
 
+## 3.0.0-beta.25
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.25
+- @trigger.dev/sdk@3.0.0-beta.25
+
+## 3.0.0-beta.24
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.24
+- @trigger.dev/sdk@3.0.0-beta.24
+
 ## 3.0.0-beta.23
 
 ### Patch Changes

integrations/supabase/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/supabase",
-  "version": "3.0.0-beta.23",
+  "version": "3.0.0-beta.25",
   "description": "Trigger.dev integration for @supabase/supabase-js",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -26,8 +26,8 @@
   },
   "dependencies": {
     "@supabase/supabase-js": "^2.26.0",
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
     "supabase-management-js": "^1.0.0",
     "zod": "3.22.3"
   },

integrations/typeform/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @trigger.dev/typeform
 
+## 3.0.0-beta.25
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.25
+- @trigger.dev/sdk@3.0.0-beta.25
+
+## 3.0.0-beta.24
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.24
+- @trigger.dev/sdk@3.0.0-beta.24
+
 ## 3.0.0-beta.23
 
 ### Patch Changes

integrations/typeform/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/typeform",
-  "version": "3.0.0-beta.23",
+  "version": "3.0.0-beta.25",
   "description": "The official Typeform integration for Trigger.dev",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -24,8 +24,8 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.23",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.23",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
     "@typeform/api-client": "^1.8.0",
     "zod": "3.22.3"
   },

package.json

@@ -14,6 +14,7 @@
     "db:migrate": "turbo run db:migrate:deploy generate",
     "db:seed": "turbo run db:seed",
     "db:studio": "turbo run db:studio",
+    "db:populate": "turbo run db:populate",
     "dev": "turbo run dev --parallel",
     "i:dev": "infisical run -- turbo run dev --parallel",
     "format": "prettier . --write --config prettier.config.js",
@@ -72,4 +73,4 @@
       "engine.io-parser@5.2.2": "patches/engine.io-parser@5.2.2.patch"
     }
   }
-}
+}

packages/astro/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @trigger.dev/astro
 
+## 3.0.0-beta.25
+
+### Patch Changes
+
+- @trigger.dev/sdk@3.0.0-beta.25
+
+## 3.0.0-beta.24
+
+### Patch Changes
+
+- @trigger.dev/sdk@3.0.0-beta.24
+
 ## 3.0.0-beta.23
 
 ### Patch Changes

packages/astro/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@trigger.dev/astro",
   "description": "An Astro-native integration for Trigger.dev background jobs platform",
-  "version": "3.0.0-beta.23",
+  "version": "3.0.0-beta.25",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "files": [
@@ -20,7 +20,7 @@
     "build:tsup": "tsup"
   },
   "peerDependencies": {
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.23"
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.25"
   },
   "devDependencies": {
     "astro": "^3.0.12",

packages/cli-v3/CHANGELOG.md

@@ -1,5 +1,23 @@
 # trigger.dev
 
+## 3.0.0-beta.25
+
+### Patch Changes
+
+- e337b2165: Add a postInstall option to allow running scripts after dependencies have been installed in deployed images
+- c37c82231: Use locked package versions when resolving dependencies in deployed workers
+- Updated dependencies [e337b2165]
+- Updated dependencies [9e5382951]
+  - @trigger.dev/core@3.0.0-beta.25
+
+## 3.0.0-beta.24
+
+### Patch Changes
+
+- 83dc87155: Fix issues with consecutive waits
+- Updated dependencies [83dc87155]
+  - @trigger.dev/core@3.0.0-beta.24
+
 ## 3.0.0-beta.23
 
 ### Patch Changes

packages/cli-v3/package.json

@@ -1,6 +1,6 @@
 {
   "name": "trigger.dev",
-  "version": "3.0.0-beta.23",
+  "version": "3.0.0-beta.25",
   "description": "A Command-Line Interface for Trigger.dev (v3) projects",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -86,7 +86,7 @@
     "@opentelemetry/sdk-trace-base": "^1.22.0",
     "@opentelemetry/sdk-trace-node": "^1.22.0",
     "@opentelemetry/semantic-conventions": "^1.22.0",
-    "@trigger.dev/core": "workspace:3.0.0-beta.23",
+    "@trigger.dev/core": "workspace:3.0.0-beta.25",
     "@types/degit": "^2.8.3",
     "chalk": "^5.2.0",
     "chokidar": "^3.5.3",

packages/cli-v3/src/Containerfile.prod

@@ -19,6 +19,8 @@ COPY --chown=node:node . .
 USER node
 RUN npm ci --no-fund --no-audit && npm cache clean --force
 
+__POST_INSTALL__
+
 # Development or production stage builds upon the base stage
 FROM base AS final
 

packages/cli-v3/src/commands/deploy.ts

@@ -1283,6 +1283,8 @@ async function compileProject(
 
       const dependencies = await gatherRequiredDependencies(allImports, config, javascriptProject);
 
+      logger.debug("gatherRequiredDependencies()", { dependencies });
+
       const packageJsonContents = {
         name: "trigger-worker",
         version: "0.0.0",
@@ -1325,8 +1327,19 @@ async function compileProject(
 
       // Write the Containerfile to /tmp/dir/Containerfile
       const containerFilePath = join(cliRootPath(), "Containerfile.prod");
-      // Copy the Containerfile to /tmp/dir/Containerfile
-      await copyFile(containerFilePath, join(tempDir, "Containerfile"));
+
+      let containerFileContents = readFileSync(containerFilePath, "utf-8");
+
+      if (config.postInstall) {
+        containerFileContents = containerFileContents.replace(
+          "__POST_INSTALL__",
+          `RUN ${config.postInstall}`
+        );
+      } else {
+        containerFileContents = containerFileContents.replace("__POST_INSTALL__", "");
+      }
+
+      await writeFile(join(tempDir, "Containerfile"), containerFileContents);
 
       const contentHasher = createHash("sha256");
       contentHasher.update(Buffer.from(entryPointOutputFile.text));
@@ -1536,6 +1549,7 @@ async function gatherRequiredDependencies(
   project: JavascriptProject
 ) {
   const dependencies: Record<string, string> = {};
+  const resolvablePackageNames = new Set<string>();
 
   for (const file of imports) {
     if ((file.kind !== "require-call" && file.kind !== "dynamic-import") || !file.external) {
@@ -1544,26 +1558,32 @@ async function gatherRequiredDependencies(
 
     const packageName = detectPackageNameFromImportPath(file.path);
 
-    if (dependencies[packageName]) {
+    if (!packageName) {
       continue;
     }
 
-    const externalDependencyVersion = await project.resolve(packageName);
+    resolvablePackageNames.add(packageName);
+  }
 
-    if (externalDependencyVersion) {
-      dependencies[packageName] = stripWorkspaceFromVersion(externalDependencyVersion);
-      continue;
-    }
+  const resolvedPackageVersions = await project.resolveAll(Array.from(resolvablePackageNames));
+  const missingPackages = Array.from(resolvablePackageNames).filter(
+    (packageName) => !resolvedPackageVersions[packageName]
+  );
 
+  for (const missingPackage of missingPackages) {
     const internalDependencyVersion =
-      (packageJson.dependencies as Record<string, string>)[packageName] ??
-      detectDependencyVersion(packageName);
+      (packageJson.dependencies as Record<string, string>)[missingPackage] ??
+      detectDependencyVersion(missingPackage);
 
     if (internalDependencyVersion) {
-      dependencies[packageName] = stripWorkspaceFromVersion(internalDependencyVersion);
+      dependencies[missingPackage] = stripWorkspaceFromVersion(internalDependencyVersion);
     }
   }
 
+  for (const [packageName, version] of Object.entries(resolvedPackageVersions)) {
+    dependencies[packageName] = version;
+  }
+
   if (config.additionalPackages) {
     for (const packageName of config.additionalPackages) {
       if (dependencies[packageName]) {

packages/cli-v3/src/utilities/javascriptProject.ts

@@ -5,51 +5,10 @@ import { logger } from "./logger";
 import { PackageManager, getUserPackageManager } from "./getUserPackageManager";
 import { PackageJson } from "type-fest";
 import { assertExhaustive } from "./assertExhaustive";
+import { builtinModules } from "node:module";
 
 export type ResolveOptions = { allowDev: boolean };
 
-const BuiltInModules = new Set([
-  "assert",
-  "async_hooks",
-  "buffer",
-  "child_process",
-  "cluster",
-  "console",
-  "constants",
-  "crypto",
-  "dgram",
-  "dns",
-  "domain",
-  "events",
-  "fs",
-  "http",
-  "http2",
-  "https",
-  "inspector",
-  "module",
-  "net",
-  "os",
-  "path",
-  "perf_hooks",
-  "process",
-  "punycode",
-  "querystring",
-  "readline",
-  "repl",
-  "stream",
-  "string_decoder",
-  "timers",
-  "tls",
-  "trace_events",
-  "tty",
-  "url",
-  "util",
-  "v8",
-  "vm",
-  "worker_threads",
-  "zlib",
-]);
-
 export class JavascriptProject {
   private _packageJson?: PackageJson;
   private _packageManager?: PackageManager;
@@ -84,27 +43,73 @@ export class JavascriptProject {
     }
   }
 
+  async resolveAll(
+    packageNames: string[],
+    options?: ResolveOptions
+  ): Promise<Record<string, string>> {
+    const externalPackages = packageNames.filter((packageName) => !isBuiltInModule(packageName));
+
+    const opts = { allowDev: false, ...options };
+
+    const command = await this.#getCommand();
+
+    try {
+      const versions = await command.resolveDependencyVersions(externalPackages, {
+        cwd: this.projectPath,
+      });
+
+      if (versions) {
+        logger.debug(`Resolved [${externalPackages.join(", ")}] version using ${command.name}`, {
+          versions,
+        });
+      }
+
+      // Merge the resolved versions with the package.json dependencies
+      const missingPackages = externalPackages.filter((packageName) => !versions[packageName]);
+      const missingPackageVersions: Record<string, string> = {};
+
+      for (const packageName of missingPackages) {
+        const packageJsonVersion = this.packageJson.dependencies?.[packageName];
+
+        if (typeof packageJsonVersion === "string") {
+          logger.debug(`Resolved ${packageName} version using package.json`, {
+            packageJsonVersion,
+          });
+
+          missingPackageVersions[packageName] = packageJsonVersion;
+        }
+
+        if (opts.allowDev) {
+          const devPackageJsonVersion = this.packageJson.devDependencies?.[packageName];
+
+          if (typeof devPackageJsonVersion === "string") {
+            logger.debug(`Resolved ${packageName} version using devDependencies`, {
+              devPackageJsonVersion,
+            });
+
+            missingPackageVersions[packageName] = devPackageJsonVersion;
+          }
+        }
+      }
+
+      return { ...versions, ...missingPackageVersions };
+    } catch (error) {
+      logger.debug(`Failed to resolve dependency versions using ${command.name}`, {
+        packageNames,
+        error,
+      });
+
+      return {};
+    }
+  }
+
   async resolve(packageName: string, options?: ResolveOptions): Promise<string | undefined> {
-    if (BuiltInModules.has(packageName)) {
+    if (isBuiltInModule(packageName)) {
       return undefined;
     }
 
     const opts = { allowDev: false, ...options };
 
-    const packageJsonVersion = this.packageJson.dependencies?.[packageName];
-
-    if (typeof packageJsonVersion === "string") {
-      return packageJsonVersion;
-    }
-
-    if (opts.allowDev) {
-      const devPackageJsonVersion = this.packageJson.devDependencies?.[packageName];
-
-      if (typeof devPackageJsonVersion === "string") {
-        return devPackageJsonVersion;
-      }
-    }
-
     const command = await this.#getCommand();
 
     try {
@@ -113,8 +118,30 @@ export class JavascriptProject {
       });
 
       if (version) {
+        logger.debug(`Resolved ${packageName} version using ${command.name}`, { version });
+
         return version;
       }
+
+      const packageJsonVersion = this.packageJson.dependencies?.[packageName];
+
+      if (typeof packageJsonVersion === "string") {
+        logger.debug(`Resolved ${packageName} version using package.json`, { packageJsonVersion });
+
+        return packageJsonVersion;
+      }
+
+      if (opts.allowDev) {
+        const devPackageJsonVersion = this.packageJson.devDependencies?.[packageName];
+
+        if (typeof devPackageJsonVersion === "string") {
+          logger.debug(`Resolved ${packageName} version using devDependencies`, {
+            devPackageJsonVersion,
+          });
+
+          return devPackageJsonVersion;
+        }
+      }
     } catch (error) {
       logger.debug(`Failed to resolve dependency version using ${command.name}`, {
         packageName,
@@ -176,6 +203,11 @@ interface PackageManagerCommands {
     packageName: string,
     options: PackageManagerOptions
   ): Promise<string | undefined>;
+
+  resolveDependencyVersions(
+    packageNames: string[],
+    options: PackageManagerOptions
+  ): Promise<Record<string, string>>;
 }
 
 class PNPMCommands implements PackageManagerCommands {
@@ -197,7 +229,7 @@ class PNPMCommands implements PackageManagerCommands {
     const { stdout } = await $({ cwd: options.cwd })`${this.cmd} list ${packageName} -r --json`;
     const result = JSON.parse(stdout) as PnpmList;
 
-    logger.debug(`Resolving ${packageName} version using ${this.name}`, { result });
+    logger.debug(`Resolving ${packageName} version using ${this.name}`);
 
     // Return the first dependency version that matches the package name
     for (const dep of result) {
@@ -208,6 +240,31 @@ class PNPMCommands implements PackageManagerCommands {
       }
     }
   }
+
+  async resolveDependencyVersions(
+    packageNames: string[],
+    options: PackageManagerOptions
+  ): Promise<Record<string, string>> {
+    const { stdout } = await $({ cwd: options.cwd })`${this.cmd} list ${packageNames} -r --json`;
+    const result = JSON.parse(stdout) as PnpmList;
+
+    logger.debug(`Resolving ${packageNames.join(" ")} version using ${this.name}`);
+
+    const results: Record<string, string> = {};
+
+    // Return the first dependency version that matches the package name
+    for (const dep of result) {
+      for (const packageName of packageNames) {
+        const dependency = dep.dependencies?.[packageName];
+
+        if (dependency) {
+          results[packageName] = dependency.version;
+        }
+      }
+    }
+
+    return results;
+  }
 }
 
 type NpmDependency = {
@@ -246,6 +303,28 @@ class NPMCommands implements PackageManagerCommands {
     return this.#recursivelySearchDependencies(output.dependencies, packageName);
   }
 
+  async resolveDependencyVersions(
+    packageNames: string[],
+    options: PackageManagerOptions
+  ): Promise<Record<string, string>> {
+    const { stdout } = await $({ cwd: options.cwd })`${this.cmd} list ${packageNames} --json`;
+    const output = JSON.parse(stdout) as NpmListOutput;
+
+    logger.debug(`Resolving ${packageNames.join(" ")} version using ${this.name}`, { output });
+
+    const results: Record<string, string> = {};
+
+    for (const packageName of packageNames) {
+      const version = this.#recursivelySearchDependencies(output.dependencies, packageName);
+
+      if (version) {
+        results[packageName] = version;
+      }
+    }
+
+    return results;
+  }
+
   #recursivelySearchDependencies(
     dependencies: Record<string, NpmDependency>,
     packageName: string
@@ -286,7 +365,7 @@ class YarnCommands implements PackageManagerCommands {
 
     const lines = stdout.split("\n");
 
-    logger.debug(`Resolving ${packageName} version using ${this.name}`, { lines });
+    logger.debug(`Resolving ${packageName} version using ${this.name}`);
 
     for (const line of lines) {
       const json = JSON.parse(line);
@@ -296,4 +375,54 @@ class YarnCommands implements PackageManagerCommands {
       }
     }
   }
+
+  async resolveDependencyVersions(
+    packageNames: string[],
+    options: PackageManagerOptions
+  ): Promise<Record<string, string>> {
+    const { stdout } = await $({ cwd: options.cwd })`${this.cmd} info ${packageNames} --json`;
+
+    const lines = stdout.split("\n");
+
+    logger.debug(`Resolving ${packageNames.join(" ")} version using ${this.name}`);
+
+    const results: Record<string, string> = {};
+
+    for (const line of lines) {
+      const json = JSON.parse(line);
+
+      const packageName = this.#parseYarnValueIntoPackageName(json.value);
+
+      if (packageNames.includes(packageName)) {
+        results[packageName] = json.children.Version;
+      }
+    }
+
+    return results;
+  }
+
+  // The "value" when doing yarn info is formatted like this:
+  // "package-name@npm:version" or "package-name@workspace:version"
+  // This function will parse the value into just the package name.
+  // This correctly handles scoped packages as well e.g. @scope/package-name@npm:version
+  #parseYarnValueIntoPackageName(value: string): string {
+    const parts = value.split("@");
+
+    // If the value does not contain an "@" symbol, then it's just the package name
+    if (parts.length === 3) {
+      return parts[1] as string;
+    }
+
+    // If the value contains an "@" symbol, then the package name is the first part
+    return parts[0] as string;
+  }
+}
+
+function isBuiltInModule(module: string): boolean {
+  // if the module has node: prefix, it's a built-in module
+  if (module.startsWith("node:")) {
+    return true;
+  }
+
+  return builtinModules.includes(module);
 }

packages/cli-v3/src/workers/prod/backgroundWorker.ts

@@ -13,6 +13,7 @@ import {
   TaskRunExecution,
   TaskRunExecutionPayload,
   TaskRunExecutionResult,
+  WaitReason,
   correctErrorStackTrace,
 } from "@trigger.dev/core/v3";
 import { ZodIpcConnection } from "@trigger.dev/core/v3/zodIpc";
@@ -68,8 +69,10 @@ export class ProdBackgroundWorker {
   > = new Evt();
 
   public preCheckpointNotification = Evt.create<{ willCheckpointAndRestore: boolean }>();
+  public checkpointCanceledNotification = Evt.create<{ checkpointCanceled: boolean }>();
+
   public onReadyForCheckpoint = Evt.create<{ version?: "v1" }>();
-  public onCancelCheckpoint = Evt.create<{ version?: "v1" }>();
+  public onCancelCheckpoint = Evt.create<{ version?: "v1" | "v2"; reason?: WaitReason }>();
 
   private _onClose: Evt<void> = new Evt();
 
@@ -251,6 +254,9 @@ export class ProdBackgroundWorker {
       this.preCheckpointNotification.attach((message) => {
         taskRunProcess.preCheckpointNotification.post(message);
       });
+      this.checkpointCanceledNotification.attach((message) => {
+        taskRunProcess.checkpointCanceledNotification.post(message);
+      });
 
       await taskRunProcess.initialize();
 
@@ -377,8 +383,10 @@ class TaskRunProcess {
   > = new Evt();
 
   public preCheckpointNotification = Evt.create<{ willCheckpointAndRestore: boolean }>();
+  public checkpointCanceledNotification = Evt.create<{ checkpointCanceled: boolean }>();
+
   public onReadyForCheckpoint = Evt.create<{ version?: "v1" }>();
-  public onCancelCheckpoint = Evt.create<{ version?: "v1" }>();
+  public onCancelCheckpoint = Evt.create<{ version?: "v1" | "v2"; reason?: WaitReason }>();
 
   constructor(
     private execution: ProdTaskRunExecution,
@@ -434,26 +442,62 @@ class TaskRunProcess {
           this.onTaskHeartbeat.post(message.id);
         },
         TASKS_READY: async (message) => {},
+        WAIT_FOR_TASK: async (message) => {
+          this.onWaitForTask.post(message);
+        },
         WAIT_FOR_BATCH: async (message) => {
           this.onWaitForBatch.post(message);
         },
         WAIT_FOR_DURATION: async (message) => {
+          // Post to coordinator
           this.onWaitForDuration.post(message);
 
-          // The coordinator will let us know if a checkpoint is about to happen
-          // We then pass this back down to the runtime in the child process
-          const { willCheckpointAndRestore } = await this.preCheckpointNotification.waitFor();
+          try {
+            // ..and wait for response
+            const { willCheckpointAndRestore } = await this.preCheckpointNotification.waitFor(
+              30_000
+            );
 
-          return { willCheckpointAndRestore };
-        },
-        WAIT_FOR_TASK: async (message) => {
-          this.onWaitForTask.post(message);
+            return {
+              willCheckpointAndRestore,
+            };
+          } catch (error) {
+            console.error("Error while waiting for pre-checkpoint notification", error);
+
+            // Assume we won't get checkpointed
+            return {
+              willCheckpointAndRestore: false,
+            };
+          }
         },
         READY_FOR_CHECKPOINT: async (message) => {
           this.onReadyForCheckpoint.post(message);
         },
         CANCEL_CHECKPOINT: async (message) => {
+          const version = "v2";
+
+          // Post to coordinator
           this.onCancelCheckpoint.post(message);
+
+          try {
+            // ..and wait for response
+            const { checkpointCanceled } = await this.checkpointCanceledNotification.waitFor(
+              30_000
+            );
+
+            return {
+              version,
+              checkpointCanceled,
+            };
+          } catch (error) {
+            console.error("Error while waiting for checkpoint cancellation", error);
+
+            // Assume it's been canceled
+            return {
+              version,
+              checkpointCanceled: true,
+            };
+          }
         },
       },
     });

packages/cli-v3/src/workers/prod/entry-point.ts

@@ -79,21 +79,33 @@ class ProdWorker {
     });
 
     this.#backgroundWorker.onReadyForCheckpoint.attach(async (message) => {
+      // Flush before checkpointing so we don't flush the same spans again after restore
+      await this.#backgroundWorker.flushTelemetry();
       this.#coordinatorSocket.socket.emit("READY_FOR_CHECKPOINT", { version: "v1" });
     });
 
+    // Currently, this is only used for duration waits. Might need adjusting for other use cases.
     this.#backgroundWorker.onCancelCheckpoint.attach(async (message) => {
-      logger.log("onCancelCheckpoint() clearing paused state, don't wait for post start hook", {
-        paused: this.paused,
-        nextResumeAfter: this.nextResumeAfter,
-        waitForPostStart: this.waitForPostStart,
-      });
+      logger.log("onCancelCheckpoint", { message });
 
-      this.paused = false;
-      this.nextResumeAfter = undefined;
-      this.waitForPostStart = false;
+      const { checkpointCanceled } = await this.#coordinatorSocket.socket.emitWithAck(
+        "CANCEL_CHECKPOINT",
+        {
+          version: "v2",
+          reason: message.reason,
+        }
+      );
 
-      this.#coordinatorSocket.socket.emit("CANCEL_CHECKPOINT", { version: "v1" });
+      if (checkpointCanceled) {
+        if (message.reason === "WAIT_FOR_DURATION") {
+          // Worker will resume immediately
+          this.paused = false;
+          this.nextResumeAfter = undefined;
+          this.waitForPostStart = false;
+        }
+      }
+
+      this.#backgroundWorker.checkpointCanceledNotification.post({ checkpointCanceled });
     });
 
     this.#backgroundWorker.onWaitForDuration.attach(async (message) => {
@@ -216,7 +228,7 @@ class ProdWorker {
     }
   }
 
-  #prepareForWait(reason: WaitReason, willCheckpointAndRestore: boolean) {
+  async #prepareForWait(reason: WaitReason, willCheckpointAndRestore: boolean) {
     logger.log(`prepare for ${reason}`, { willCheckpointAndRestore });
 
     this.#backgroundWorker.preCheckpointNotification.post({ willCheckpointAndRestore });
@@ -225,6 +237,12 @@ class ProdWorker {
       this.paused = true;
       this.nextResumeAfter = reason;
       this.waitForPostStart = true;
+
+      if (reason === "WAIT_FOR_TASK" || reason === "WAIT_FOR_BATCH") {
+        // Flush before checkpointing so we don't flush the same spans again after restore
+        // Duration waits do this via the "ready for checkpoint" event instead
+        await this.#backgroundWorker.flushTelemetry();
+      }
     }
   }
 
@@ -253,6 +271,7 @@ class ProdWorker {
   #resumeAfterDuration() {
     this.paused = false;
     this.nextResumeAfter = undefined;
+    this.waitForPostStart = false;
 
     this.#backgroundWorker.waitCompletedNotification();
   }
@@ -267,6 +286,7 @@ class ProdWorker {
     return headers;
   }
 
+  // FIXME: If the the worker can't connect for a while, this runs MANY times - it should only run once
   #createCoordinatorSocket(host: string) {
     const extraHeaders = this.#returnValidatedExtraHeaders({
       "x-machine-name": MACHINE_NAME,
@@ -342,6 +362,7 @@ class ProdWorker {
 
           this.paused = false;
           this.nextResumeAfter = undefined;
+          this.waitForPostStart = false;
 
           for (let i = 0; i < message.completions.length; i++) {
             const completion = message.completions[i];
@@ -428,6 +449,25 @@ class ProdWorker {
           return;
         }
 
+        if (this.paused) {
+          if (!this.nextResumeAfter) {
+            return;
+          }
+
+          if (!this.attemptFriendlyId) {
+            logger.error("Missing friendly ID");
+            return;
+          }
+
+          socket.emit("READY_FOR_RESUME", {
+            version: "v1",
+            attemptFriendlyId: this.attemptFriendlyId,
+            type: this.nextResumeAfter,
+          });
+
+          return;
+        }
+
         if (process.env.INDEX_TASKS === "true") {
           try {
             const taskResources = await this.#initializeWorker();
@@ -519,30 +559,6 @@ class ProdWorker {
           }
         }
 
-        if (this.paused) {
-          if (!this.nextResumeAfter) {
-            return;
-          }
-
-          if (!this.attemptFriendlyId) {
-            logger.error("Missing friendly ID");
-            return;
-          }
-
-          if (this.nextResumeAfter === "WAIT_FOR_DURATION") {
-            this.#resumeAfterDuration();
-            return;
-          }
-
-          socket.emit("READY_FOR_RESUME", {
-            version: "v1",
-            attemptFriendlyId: this.attemptFriendlyId,
-            type: this.nextResumeAfter,
-          });
-
-          return;
-        }
-
         if (this.executing) {
           return;
         }
@@ -587,7 +603,8 @@ class ProdWorker {
           case "/status": {
             return reply.json({
               executing: this.executing,
-              pause: this.paused,
+              paused: this.paused,
+              completed: this.completed.size,
               nextResumeAfter: this.nextResumeAfter,
             });
           }

packages/cli-v3/src/workers/prod/worker-facade.ts

@@ -174,7 +174,7 @@ const zodIpc = new ZodIpcConnection({
       prodRuntimeManager.resumeTask(completion, execution);
     },
     WAIT_COMPLETED_NOTIFICATION: async () => {
-      prodRuntimeManager.resumeAfterRestore();
+      prodRuntimeManager.resumeAfterDuration();
     },
     CLEANUP: async ({ flush, kill }, sender) => {
       if (kill) {

packages/cli/CHANGELOG.md

@@ -1,5 +1,22 @@
 # create-trigger
 
+## 3.0.0-beta.25
+
+### Patch Changes
+
+- Updated dependencies [e337b2165]
+- Updated dependencies [9e5382951]
+  - @trigger.dev/core@3.0.0-beta.25
+  - @trigger.dev/yalt@3.0.0-beta.25
+
+## 3.0.0-beta.24
+
+### Patch Changes
+
+- Updated dependencies [83dc87155]
+  - @trigger.dev/core@3.0.0-beta.24
+  - @trigger.dev/yalt@3.0.0-beta.24
+
 ## 3.0.0-beta.23
 
 ### Patch Changes

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/cli",
-  "version": "3.0.0-beta.23",
+  "version": "3.0.0-beta.25",
   "description": "The Trigger.dev CLI",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",

packages/core-apps/CHANGELOG.md

@@ -1,5 +1,9 @@
 # @trigger.dev/core-apps
 
+## 3.0.0-beta.25
+
+## 3.0.0-beta.24
+
 ## 3.0.0-beta.23
 
 ## 3.0.0-beta.22

packages/core-apps/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@trigger.dev/core-apps",
   "description": "Backend core code used across apps",
-  "version": "3.0.0-beta.23",
+  "version": "3.0.0-beta.25",
   "private": true,
   "license": "MIT",
   "main": "./dist/index.js",

packages/core-backend/CHANGELOG.md

@@ -1,5 +1,9 @@
 # @trigger.dev/core-backend
 
+## 3.0.0-beta.25
+
+## 3.0.0-beta.24
+
 ## 3.0.0-beta.23
 
 ## 3.0.0-beta.22

packages/core-backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trigger.dev/core-backend",
-  "version": "3.0.0-beta.23",
+  "version": "3.0.0-beta.25",
   "description": "Core code used across `@trigger.dev/sdk` and Trigger.dev server",
   "license": "MIT",
   "main": "./dist/index.js",

packages/core/CHANGELOG.md

@@ -1,5 +1,18 @@
 # internal-platform
 
+## 3.0.0-beta.25
+
+### Patch Changes
+
+- e337b2165: Add a postInstall option to allow running scripts after dependencies have been installed in deployed images
+- 9e5382951: Improve the display of non-object return types in the run trace viewer
+
+## 3.0.0-beta.24
+
+### Patch Changes
+
+- 83dc87155: Fix issues with consecutive waits
+
 ## 3.0.0-beta.23
 
 ## 3.0.0-beta.22