Release 3.0.0-beta.25

chore: Update version for release (beta) (#1080 )
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
2024-05-02 16:00:43 +01:00 · 2024-05-02 15:58:42 +01:00 · 2024-05-02 14:12:16 +01:00 · 2024-05-02 12:31:55 +01:00 · 2024-05-02 11:56:35 +01:00 · 2024-05-01 22:41:50 +01:00
211 changed files with 7235 additions and 2084 deletions
@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Hoist uncaughtException handler to the top of workers to better report error messages
@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Fix error stack traces
@@ -0,0 +1,7 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Implement task.onSuccess/onFailure and config.onSuccess/onFailure
@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Fix issues with consecutive waits
@@ -0,0 +1,83 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Adds support for `emitDecoratorMetadata: true` and `experimentalDecorators: true` in your tsconfig using the [`@anatine/esbuild-decorators`](https://github.com/anatine/esbuildnx/tree/main/packages/esbuild-decorators) package. This allows you to use libraries like TypeORM: 
+
+```ts orm/index.ts
+import "reflect-metadata";
+import { DataSource } from "typeorm";
+import { Entity, Column, PrimaryColumn } from "typeorm";
+
+@Entity()
+export class Photo {
+  @PrimaryColumn()
+  id!: number;
+
+  @Column()
+  name!: string;
+
+  @Column()
+  description!: string;
+
+  @Column()
+  filename!: string;
+
+  @Column()
+  views!: number;
+
+  @Column()
+  isPublished!: boolean;
+}
+
+export const AppDataSource = new DataSource({
+  type: "postgres",
+  host: "localhost",
+  port: 5432,
+  username: "postgres",
+  password: "postgres",
+  database: "v3-catalog",
+  entities: [Photo],
+  synchronize: true,
+  logging: false,
+});
+```
+
+And then in your trigger.config.ts file you can initialize the datasource using the new `init` option:
+
+```ts trigger.config.ts
+import type { TriggerConfig } from "@trigger.dev/sdk/v3";
+import { AppDataSource } from "@/trigger/orm";
+
+export const config: TriggerConfig = {
+  // ... other options here
+  init: async (payload, { ctx }) => {
+    await AppDataSource.initialize();
+  },
+};
+```
+
+Now you are ready to use this in your tasks:
+
+```ts
+import { task } from "@trigger.dev/sdk/v3";
+import { AppDataSource, Photo } from "./orm";
+
+export const taskThatUsesDecorators = task({
+  id: "taskThatUsesDecorators",
+  run: async (payload: { message: string }) => {
+    console.log("Creating a photo...");
+
+    const photo = new Photo();
+    photo.id = 2;
+    photo.name = "Me and Bears";
+    photo.description = "I am near polar bears";
+    photo.filename = "photo-with-bears.jpg";
+    photo.views = 1;
+    photo.isPublished = true;
+
+    await AppDataSource.manager.save(photo);
+  },
+});
+```
@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Fix dev CLI output when not printing update messages
@@ -46,9 +46,12 @@
  "changesets": [
    "angry-eagles-trade",
    "beige-pens-dance",
+    "big-tomatoes-deliver",
    "breezy-gorillas-mate",
    "chilled-hornets-move",
    "clean-pianos-listen",
+    "clever-apes-collect",
+    "clever-carrots-travel",
    "cool-glasses-bake",
    "cuddly-feet-approve",
    "dry-walls-check",
@@ -64,15 +67,20 @@
    "lemon-jobs-repair",
    "light-bulldogs-press",
    "light-dragons-complain",
+    "little-crabs-cross",
    "loud-actors-remember",
    "many-ligers-pump",
    "mighty-camels-joke",
+    "mighty-flowers-train",
    "nasty-jars-pump",
    "new-rivers-tell",
    "ninety-pets-travel",
    "odd-poets-own",
    "polite-ducks-switch",
+    "polite-rockets-matter",
    "poor-flowers-cross",
+    "purple-garlics-shop",
+    "rare-lamps-promise",
    "rare-roses-float",
    "real-planets-stare",
    "rich-kangaroos-unite",
@@ -84,6 +92,7 @@
    "sharp-zebras-serve",
    "shiny-coats-cry",
    "silly-suits-switch",
+    "six-ligers-exist",
    "slow-buses-own",
    "smart-needles-move",
    "smart-olives-eat",
@@ -95,6 +104,7 @@
    "swift-dragons-peel",
    "tall-bees-wave",
    "tame-guests-know",
+    "tender-moose-tell",
    "tender-oranges-rhyme",
    "tidy-balloons-suffer",
    "tidy-dryers-sleep",
@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Add typescript as a dependency so the esbuild-decorator will work even when running in npx
@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Add a postInstall option to allow running scripts after dependencies have been installed in deployed images
@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Improve the display of non-object return types in the run trace viewer
@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Use locked package versions when resolving dependencies in deployed workers
@@ -4,13 +4,15 @@ on:
 jobs:
  publish:
    runs-on: ubuntu-latest
+    env:
+      PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING: 1
    outputs:
      version: ${{ steps.get_version.outputs.version }}
      short_sha: ${{ steps.get_commit.outputs.sha_short }}
    steps:
      - name: Setup Depot CLI
        uses: depot/setup-action@v1
-        
+
      - name: ⬇️ Checkout repo
        uses: actions/checkout@v3
        with:
@@ -157,16 +157,18 @@ class Checkpointer {
    return this.#abortControllers.has(runId);
  }

-  cancelCheckpoint(runId: string) {
+  cancelCheckpoint(runId: string): boolean {
    const controller = this.#abortControllers.get(runId);

    if (!controller) {
      logger.debug("Nothing to cancel", { runId });
-      return;
+      return false;
    }

    controller.abort("cancelCheckpointing()");
    this.#abortControllers.delete(runId);
+
+    return true;
  }

  async #checkpointAndPush({
@@ -725,10 +727,18 @@ class TaskCoordinator {
          checkpointable.resolve();
        });

-        socket.on("CANCEL_CHECKPOINT", async (message) => {
+        socket.on("CANCEL_CHECKPOINT", async (message, callback) => {
          logger.log("[CANCEL_CHECKPOINT]", message);

-          this.#cancelCheckpoint(socket.data.runId);
+          if (message.version === "v1") {
+            this.#cancelCheckpoint(socket.data.runId);
+            // v1 has no callback
+            return;
+          }
+
+          const checkpointCanceled = this.#cancelCheckpoint(socket.data.runId);
+
+          callback({ version: "v2", checkpointCanceled });
        });

        socket.on("WAIT_FOR_DURATION", async (message, callback) => {
@@ -933,7 +943,9 @@ class TaskCoordinator {
    }

    // Cancel checkpointing procedure
-    this.#checkpointer.cancelCheckpoint(runId);
+    const checkpointCanceled = this.#checkpointer.cancelCheckpoint(runId);
+
+    return checkpointCanceled;
  }

  #createHttpServer() {
@@ -212,6 +212,7 @@ class KubernetesTaskOperations implements TaskOperations {
            {
              name: "populate-taskinfo",
              image: "docker.io/library/busybox",
+              imagePullPolicy: "IfNotPresent",
              command: ["/bin/sh", "-c"],
              args: ["printenv COORDINATOR_HOST | tee /etc/taskinfo/coordinator-host"],
              env: [
@@ -17,4 +17,5 @@ build-storybook.log
 .storybook-out
 storybook-static

-/prisma/seed.js
+/prisma/seed.js
+/prisma/populate.js
@@ -206,3 +206,39 @@ export function TriggerDevStepV3() {
    </ClientTabs>
  );
 }
+
+export function TriggerLoginStepV3() {
+  return (
+    <ClientTabs defaultValue="npm">
+      <ClientTabsList>
+        <ClientTabsTrigger value={"npm"}>npm</ClientTabsTrigger>
+        <ClientTabsTrigger value={"pnpm"}>pnpm</ClientTabsTrigger>
+        <ClientTabsTrigger value={"yarn"}>yarn</ClientTabsTrigger>
+      </ClientTabsList>
+      <ClientTabsContent value={"npm"}>
+        <ClipboardField
+          variant="primary/medium"
+          iconButton
+          className="mb-4"
+          value={`npx trigger.dev@${v3PackageTag} login`}
+        />
+      </ClientTabsContent>
+      <ClientTabsContent value={"pnpm"}>
+        <ClipboardField
+          variant="primary/medium"
+          iconButton
+          className="mb-4"
+          value={`pnpm dlx trigger.dev@${v3PackageTag} login`}
+        />
+      </ClientTabsContent>
+      <ClientTabsContent value={"yarn"}>
+        <ClipboardField
+          variant="primary/medium"
+          iconButton
+          className="mb-4"
+          value={`yarn dlx trigger.dev@${v3PackageTag} login`}
+        />
+      </ClientTabsContent>
+    </ClientTabs>
+  );
+}
@@ -0,0 +1,27 @@
+import { ShieldCheckIcon } from "@heroicons/react/20/solid";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger,
+} from "~/components/primitives/Tooltip";
+import { useHasAdminAccess } from "~/hooks/useUser";
+
+export function AdminDebugTooltip({ children }: { children: React.ReactNode }) {
+  const hasAdminAccess = useHasAdminAccess();
+
+  if (!hasAdminAccess) {
+    return null;
+  }
+
+  return (
+    <TooltipProvider>
+      <Tooltip>
+        <TooltipTrigger>
+          <ShieldCheckIcon className="h-5 w-5" />
+        </TooltipTrigger>
+        <TooltipContent className="flex items-center gap-1">{children}</TooltipContent>
+      </Tooltip>
+    </TooltipProvider>
+  );
+}
@@ -1,5 +1,9 @@
 import { Paragraph } from "./Paragraph";

-export function Hint({ children }: { children: React.ReactNode }) {
-  return <Paragraph variant="extra-small">{children}</Paragraph>;
+export function Hint({ children, className }: { children: React.ReactNode; className?: string }) {
+  return (
+    <Paragraph variant="extra-small" className={className}>
+      {children}
+    </Paragraph>
+  );
 }
@@ -0,0 +1,103 @@
+import type { VirtualElement as IVirtualElement } from "@popperjs/core";
+import { ReactNode, useEffect, useState } from "react";
+import { createPortal } from "react-dom";
+import { usePopper } from "react-popper";
+import { useEvent } from "react-use";
+import useLazyRef from "~/hooks/useLazyRef";
+
+// Recharts 3.x will have portal support, but until then we're using this:
+//https://github.com/recharts/recharts/issues/2458#issuecomment-1063463873
+
+export interface PopperPortalProps {
+  active?: boolean;
+  children: ReactNode;
+}
+
+export default function TooltipPortal({ active = true, children }: PopperPortalProps) {
+  const [portalElement, setPortalElement] = useState<HTMLDivElement>();
+  const [popperElement, setPopperElement] = useState<HTMLDivElement | null>();
+  const virtualElementRef = useLazyRef(() => new VirtualElement());
+
+  const { styles, attributes, update } = usePopper(
+    virtualElementRef.current,
+    popperElement,
+    POPPER_OPTIONS
+  );
+
+  useEffect(() => {
+    const el = document.createElement("div");
+    document.body.appendChild(el);
+    setPortalElement(el);
+    return () => el.remove();
+  }, []);
+
+  useEvent("mousemove", ({ clientX: x, clientY: y }) => {
+    virtualElementRef.current?.update(x, y);
+    if (!active) return;
+    update?.();
+  });
+
+  useEffect(() => {
+    if (!active) return;
+    update?.();
+  }, [active, update]);
+
+  if (!portalElement) return null;
+
+  return createPortal(
+    <div
+      ref={setPopperElement}
+      {...attributes.popper}
+      style={{
+        ...styles.popper,
+        zIndex: 1000,
+        display: active ? "block" : "none",
+      }}
+    >
+      {children}
+    </div>,
+    portalElement
+  );
+}
+
+class VirtualElement implements IVirtualElement {
+  private rect = {
+    width: 0,
+    height: 0,
+    top: 0,
+    right: 0,
+    bottom: 0,
+    left: 0,
+    x: 0,
+    y: 0,
+    toJSON() {
+      return this;
+    },
+  };
+
+  update(x: number, y: number) {
+    this.rect.y = y;
+    this.rect.top = y;
+    this.rect.bottom = y;
+
+    this.rect.x = x;
+    this.rect.left = x;
+    this.rect.right = x;
+  }
+
+  getBoundingClientRect(): DOMRect {
+    return this.rect;
+  }
+}
+
+const POPPER_OPTIONS: Parameters<typeof usePopper>[2] = {
+  placement: "right-start",
+  modifiers: [
+    {
+      name: "offset",
+      options: {
+        offset: [8, 8],
+      },
+    },
+  ],
+};
@@ -1,10 +1,9 @@
 import { VirtualItem, Virtualizer, useVirtualizer } from "@tanstack/react-virtual";
 import { motion } from "framer-motion";
 import { MutableRefObject, RefObject, useCallback, useEffect, useReducer, useRef } from "react";
-import { UnmountClosed } from "react-collapse";
 import { cn } from "~/utils/cn";
 import { NodeState, NodesState, reducer } from "./reducer";
-import { applyFilterToState, concreteStateFromInput, selectedIdFromState } from "./utils";
+import { concreteStateFromInput, selectedIdFromState } from "./utils";

 export type TreeViewProps<TData> = {
  tree: FlatTree<TData>;
@@ -104,23 +103,22 @@ export function TreeView<TData>({
            if (!node) return null;
            const state = nodes[node.id];
            if (!state) return null;
+            if (!state.visible) return null;
            return (
              <div
                key={node.id}
                data-index={virtualItem.index}
                ref={virtualizer.measureElement}
-                className="overflow-clip [&_.ReactCollapse--collapse]:transition-all"
+                className="overflow-clip"
                {...getNodeProps(node.id)}
              >
-                <UnmountClosed key={node.id} isOpened={state.visible}>
-                  {renderNode({
-                    node,
-                    state,
-                    index: virtualItem.index,
-                    virtualizer: virtualizer,
-                    virtualItem,
-                  })}
-                </UnmountClosed>
+                {renderNode({
+                  node,
+                  state,
+                  index: virtualItem.index,
+                  virtualizer: virtualizer,
+                  virtualItem,
+                })}
              </div>
            );
          })}
@@ -130,19 +128,23 @@ export function TreeView<TData>({
  );
 }

-type TreeStateHookProps<TData> = {
+export type Filter<TData, TFilterValue> = {
+  value?: TFilterValue;
+  fn: (value: TFilterValue, node: FlatTreeItem<TData>) => boolean;
+};
+
+type TreeStateHookProps<TData, TFilterValue> = {
  tree: FlatTree<TData>;
  selectedId?: string;
  collapsedIds?: string[];
  onSelectedIdChanged?: (selectedId: string | undefined) => void;
-  onCollapsedIdsChanged?: (collapsedIds: string[]) => void;
  estimatedRowHeight: (params: {
    node: FlatTreeItem<TData>;
    state: NodeState;
    index: number;
  }) => number;
  parentRef: RefObject<any>;
-  filter?: (node: FlatTreeItem<TData>) => boolean;
+  filter?: Filter<TData, TFilterValue>;
 };

 //this is so Framer Motion can be used to render the components
@@ -178,24 +180,24 @@ export type UseTreeStateOutput = {
  scrollToNode: (id: string) => void;
 };

-export function useTree<TData>({
+export function useTree<TData, TFilterValue>({
  tree,
  selectedId,
  collapsedIds,
  onSelectedIdChanged,
-  onCollapsedIdsChanged,
  parentRef,
  estimatedRowHeight,
  filter,
-}: TreeStateHookProps<TData>): UseTreeStateOutput {
+}: TreeStateHookProps<TData, TFilterValue>): UseTreeStateOutput {
  const previousNodeCount = useRef(tree.length);
  const previousSelectedId = useRef<string | undefined>(selectedId);

  const [state, dispatch] = useReducer(
    reducer,
-    concreteStateFromInput({ tree, selectedId, collapsedIds })
+    concreteStateFromInput({ tree, selectedId, collapsedIds, filter })
  );

+  //fire onSelectedIdChanged()
  useEffect(() => {
    const selectedId = selectedIdFromState(state.nodes);
    if (selectedId !== previousSelectedId.current) {
@@ -204,12 +206,7 @@ export function useTree<TData>({
    }
  }, [state.changes.selectedId]);

-  useEffect(() => {
-    if (state.changes.collapsedIds) {
-      onCollapsedIdsChanged?.(state.changes.collapsedIds);
-    }
-  }, [state.changes.collapsedIds]);
-
+  //update tree when the number of nodes changes
  useEffect(() => {
    if (tree.length !== previousNodeCount.current) {
      previousNodeCount.current = tree.length;
@@ -217,9 +214,25 @@ export function useTree<TData>({
    }
  }, [previousNodeCount.current, tree.length]);

+  //update the filter, if it's changed
+  const previousFilter = useRef(filter);
+  useEffect(() => {
+    //check if the value (not reference) of the filter is the same
+    const previousValue = previousFilter.current
+      ? JSON.stringify(previousFilter.current.value)
+      : undefined;
+    const newValue = filter ? JSON.stringify(filter.value) : undefined;
+
+    previousFilter.current = filter;
+
+    if (previousValue !== newValue) {
+      dispatch({ type: "UPDATE_FILTER", payload: { filter } });
+    }
+  }, [filter?.value]);
+
  const virtualizer = useVirtualizer({
-    count: tree.length,
-    getItemKey: (index) => tree[index].id,
+    count: state.visibleNodeIds.length,
+    getItemKey: (index) => state.visibleNodeIds[index],
    getScrollElement: () => parentRef.current,
    estimateSize: (index: number) => {
      return estimatedRowHeight({
@@ -233,7 +246,7 @@ export function useTree<TData>({

  const scrollToNodeFn = useCallback(
    (id: string) => {
-      const itemIndex = tree.findIndex((node) => node.id === id);
+      const itemIndex = state.visibleNodeIds.findIndex((n) => n === id);

      if (itemIndex !== -1) {
        virtualizer.scrollToIndex(itemIndex, { align: "auto" });
@@ -269,21 +282,21 @@ export function useTree<TData>({

  const expandNode = useCallback(
    (id: string, scrollToNode = true) => {
-      dispatch({ type: "EXPAND_NODE", payload: { id, tree, scrollToNode, scrollToNodeFn } });
+      dispatch({ type: "EXPAND_NODE", payload: { id, scrollToNode, scrollToNodeFn } });
    },
    [state]
  );

  const collapseNode = useCallback(
    (id: string) => {
-      dispatch({ type: "COLLAPSE_NODE", payload: { id, tree } });
+      dispatch({ type: "COLLAPSE_NODE", payload: { id } });
    },
    [state]
  );

  const toggleExpandNode = useCallback(
    (id: string, scrollToNode = true) => {
-      dispatch({ type: "TOGGLE_EXPAND_NODE", payload: { id, tree, scrollToNode, scrollToNodeFn } });
+      dispatch({ type: "TOGGLE_EXPAND_NODE", payload: { id, scrollToNode, scrollToNodeFn } });
    },
    [state]
  );
@@ -292,7 +305,7 @@ export function useTree<TData>({
    (scrollToNode = true) => {
      dispatch({
        type: "SELECT_FIRST_VISIBLE_NODE",
-        payload: { tree, scrollToNode, scrollToNodeFn },
+        payload: { scrollToNode, scrollToNodeFn },
      });
    },
    [tree, state]
@@ -302,7 +315,7 @@ export function useTree<TData>({
    (scrollToNode = true) => {
      dispatch({
        type: "SELECT_LAST_VISIBLE_NODE",
-        payload: { tree, scrollToNode, scrollToNodeFn },
+        payload: { scrollToNode, scrollToNodeFn },
      });
    },
    [tree, state]
@@ -312,7 +325,7 @@ export function useTree<TData>({
    (scrollToNode = true) => {
      dispatch({
        type: "SELECT_NEXT_VISIBLE_NODE",
-        payload: { tree, scrollToNode, scrollToNodeFn },
+        payload: { scrollToNode, scrollToNodeFn },
      });
    },
    [state]
@@ -322,7 +335,7 @@ export function useTree<TData>({
    (scrollToNode = true) => {
      dispatch({
        type: "SELECT_PREVIOUS_VISIBLE_NODE",
-        payload: { tree, scrollToNode, scrollToNodeFn },
+        payload: { scrollToNode, scrollToNodeFn },
      });
    },
    [state]
@@ -332,7 +345,7 @@ export function useTree<TData>({
    (scrollToNode = true) => {
      dispatch({
        type: "SELECT_PARENT_NODE",
-        payload: { tree, scrollToNode, scrollToNodeFn },
+        payload: { scrollToNode, scrollToNodeFn },
      });
    },
    [state]
@@ -340,35 +353,35 @@ export function useTree<TData>({

  const expandAllBelowDepth = useCallback(
    (depth: number) => {
-      dispatch({ type: "EXPAND_ALL_BELOW_DEPTH", payload: { tree, depth } });
+      dispatch({ type: "EXPAND_ALL_BELOW_DEPTH", payload: { depth } });
    },
    [state]
  );

  const collapseAllBelowDepth = useCallback(
    (depth: number) => {
-      dispatch({ type: "COLLAPSE_ALL_BELOW_DEPTH", payload: { tree, depth } });
+      dispatch({ type: "COLLAPSE_ALL_BELOW_DEPTH", payload: { depth } });
    },
    [state]
  );

  const expandLevel = useCallback(
    (level: number) => {
-      dispatch({ type: "EXPAND_LEVEL", payload: { tree, level } });
+      dispatch({ type: "EXPAND_LEVEL", payload: { level } });
    },
    [state]
  );

  const collapseLevel = useCallback(
    (level: number) => {
-      dispatch({ type: "COLLAPSE_LEVEL", payload: { tree, level } });
+      dispatch({ type: "COLLAPSE_LEVEL", payload: { level } });
    },
    [state]
  );

  const toggleExpandLevel = useCallback(
    (level: number) => {
-      dispatch({ type: "TOGGLE_EXPAND_LEVEL", payload: { tree, level } });
+      dispatch({ type: "TOGGLE_EXPAND_LEVEL", payload: { level } });
    },
    [state]
  );
@@ -480,7 +493,7 @@ export function useTree<TData>({

  return {
    selected: selectedIdFromState(state.nodes),
-    nodes: filter ? applyFilterToState(tree, state.nodes, filter) : state.nodes,
+    nodes: state.nodes,
    getTreeProps,
    getNodeProps,
    selectNode,
@@ -1,5 +1,7 @@
-import { FlatTree } from "./TreeView";
+import assertNever from "assert-never";
+import { Filter, FlatTree } from "./TreeView";
 import {
+  applyFilterToState,
  applyVisibility,
  collapsedIdsFromState,
  concreteStateFromInput,
@@ -18,12 +20,15 @@ export type NodeState = {

 export type Changes = {
  selectedId: string | undefined;
-  collapsedIds: string[] | undefined;
 };

 export type TreeState = {
+  tree: FlatTree<any>;
  nodes: NodesState;
+  filteredNodes: NodesState;
  changes: Changes;
+  filter: Filter<any, any> | undefined;
+  visibleNodeIds: string[];
 };

 export type NodesState = Record<string, NodeState>;
@@ -71,7 +76,6 @@ type ExpandNodeAction = {
  type: "EXPAND_NODE";
  payload: {
    id: string;
-    tree: FlatTree<any>;
  } & WithScrollToNode;
 };

@@ -79,7 +83,6 @@ type CollapseNodeAction = {
  type: "COLLAPSE_NODE";
  payload: {
    id: string;
-    tree: FlatTree<any>;
  };
 };

@@ -87,7 +90,6 @@ type ToggleExpandNodeAction = {
  type: "TOGGLE_EXPAND_NODE";
  payload: {
    id: string;
-    tree: FlatTree<any>;
  } & WithScrollToNode;
 };

@@ -95,7 +97,6 @@ type ExpandAllBelowDepthAction = {
  type: "EXPAND_ALL_BELOW_DEPTH";
  payload: {
    depth: number;
-    tree: FlatTree<any>;
  };
 };

@@ -103,7 +104,6 @@ type CollapseAllBelowDepthAction = {
  type: "COLLAPSE_ALL_BELOW_DEPTH";
  payload: {
    depth: number;
-    tree: FlatTree<any>;
  };
 };

@@ -111,7 +111,6 @@ type ExpandLevelAction = {
  type: "EXPAND_LEVEL";
  payload: {
    level: number;
-    tree: FlatTree<any>;
  };
 };

@@ -119,7 +118,6 @@ type CollapseLevelAction = {
  type: "COLLAPSE_LEVEL";
  payload: {
    level: number;
-    tree: FlatTree<any>;
  };
 };

@@ -127,43 +125,39 @@ type ToggleExpandLevelAction = {
  type: "TOGGLE_EXPAND_LEVEL";
  payload: {
    level: number;
-    tree: FlatTree<any>;
  };
 };

 type SelectFirstVisibleNodeAction = {
  type: "SELECT_FIRST_VISIBLE_NODE";
-  payload: {
-    tree: FlatTree<any>;
-  } & WithScrollToNode;
+  payload: {} & WithScrollToNode;
 };

 type SelectLastVisibleNodeAction = {
  type: "SELECT_LAST_VISIBLE_NODE";
-  payload: {
-    tree: FlatTree<any>;
-  } & WithScrollToNode;
+  payload: {} & WithScrollToNode;
 };

 type SelectNextVisibleNodeAction = {
  type: "SELECT_NEXT_VISIBLE_NODE";
-  payload: {
-    tree: FlatTree<any>;
-  } & WithScrollToNode;
+  payload: {} & WithScrollToNode;
 };

 type SelectPreviousVisibleNodeAction = {
  type: "SELECT_PREVIOUS_VISIBLE_NODE";
-  payload: {
-    tree: FlatTree<any>;
-  } & WithScrollToNode;
+  payload: {} & WithScrollToNode;
 };

 type SelectParentNodeAction = {
  type: "SELECT_PARENT_NODE";
+  payload: {} & WithScrollToNode;
+};
+
+type UpdateFilterAction = {
+  type: "UPDATE_FILTER";
  payload: {
-    tree: FlatTree<any>;
-  } & WithScrollToNode;
+    filter: Filter<any, any> | undefined;
+  };
 };

 export type Action =
@@ -184,7 +178,8 @@ export type Action =
  | SelectLastVisibleNodeAction
  | SelectNextVisibleNodeAction
  | SelectPreviousVisibleNodeAction
-  | SelectParentNodeAction;
+  | SelectParentNodeAction
+  | UpdateFilterAction;

 export function reducer(state: TreeState, action: Action): TreeState {
  switch (action.type) {
@@ -204,7 +199,12 @@ export function reducer(state: TreeState, action: Action): TreeState {
        action.payload.scrollToNodeFn(action.payload.id);
      }

-      return { nodes: newNodes, changes: generateChanges(state.nodes, newNodes) };
+      return applyFilterToState({
+        ...state,
+        tree: state.tree,
+        nodes: newNodes,
+        changes: generateChanges(state.nodes, newNodes),
+      });
    }
    case "DESELECT_NODE": {
      const nodes = {
@@ -212,28 +212,36 @@ export function reducer(state: TreeState, action: Action): TreeState {
        [action.payload.id]: { ...state.nodes[action.payload.id], selected: false },
      };

-      return { nodes, changes: generateChanges(state.nodes, nodes) };
+      return applyFilterToState({
+        ...state,
+        nodes,
+        changes: generateChanges(state.nodes, nodes),
+      });
    }
    case "DESELECT_ALL_NODES": {
      const nodes = Object.fromEntries(
        Object.entries(state.nodes).map(([key, value]) => [key, { ...value, selected: false }])
      );
-      return { nodes, changes: generateChanges(state.nodes, nodes) };
+      return applyFilterToState({
+        ...state,
+        nodes,
+        changes: generateChanges(state.nodes, nodes),
+      });
    }
    case "TOGGLE_NODE_SELECTION": {
      const currentlySelected = state.nodes[action.payload.id]?.selected ?? false;
      if (currentlySelected) {
        return reducer(state, { type: "DESELECT_NODE", payload: { id: action.payload.id } });
-      } else {
-        return reducer(state, {
-          type: "SELECT_NODE",
-          payload: {
-            id: action.payload.id,
-            scrollToNode: action.payload.scrollToNode,
-            scrollToNodeFn: action.payload.scrollToNodeFn,
-          },
-        });
      }
+
+      return reducer(state, {
+        type: "SELECT_NODE",
+        payload: {
+          id: action.payload.id,
+          scrollToNode: action.payload.scrollToNode,
+          scrollToNodeFn: action.payload.scrollToNodeFn,
+        },
+      });
    }
    case "EXPAND_NODE": {
      const newNodes = {
@@ -245,37 +253,44 @@ export function reducer(state: TreeState, action: Action): TreeState {
        action.payload.scrollToNodeFn(action.payload.id);
      }

-      const visibleNodes = applyVisibility(action.payload.tree, newNodes);
-      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+      const visibleNodes = applyVisibility(state.tree, newNodes);
+      return applyFilterToState({
+        ...state,
+        nodes: visibleNodes,
+        changes: generateChanges(state.nodes, visibleNodes),
+      });
    }
    case "COLLAPSE_NODE": {
-      const visibleNodes = applyVisibility(action.payload.tree, {
+      const visibleNodes = applyVisibility(state.tree, {
        ...state.nodes,
        [action.payload.id]: { ...state.nodes[action.payload.id], expanded: false },
      });
-      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+      return applyFilterToState({
+        ...state,
+        nodes: visibleNodes,
+        changes: generateChanges(state.nodes, visibleNodes),
+      });
    }
    case "TOGGLE_EXPAND_NODE": {
      const currentlyExpanded = state.nodes[action.payload.id]?.expanded ?? true;
      if (currentlyExpanded) {
        return reducer(state, {
          type: "COLLAPSE_NODE",
-          payload: { id: action.payload.id, tree: action.payload.tree },
-        });
-      } else {
-        return reducer(state, {
-          type: "EXPAND_NODE",
-          payload: {
-            id: action.payload.id,
-            tree: action.payload.tree,
-            scrollToNode: action.payload.scrollToNode,
-            scrollToNodeFn: action.payload.scrollToNodeFn,
-          },
+          payload: { id: action.payload.id },
        });
      }
+
+      return reducer(state, {
+        type: "EXPAND_NODE",
+        payload: {
+          id: action.payload.id,
+          scrollToNode: action.payload.scrollToNode,
+          scrollToNodeFn: action.payload.scrollToNodeFn,
+        },
+      });
    }
    case "EXPAND_ALL_BELOW_DEPTH": {
-      const nodesToExpand = action.payload.tree.filter(
+      const nodesToExpand = state.tree.filter(
        (n) => n.level >= action.payload.depth && n.hasChildren
      );

@@ -289,11 +304,15 @@ export function reducer(state: TreeState, action: Action): TreeState {
        ])
      );

-      const visibleNodes = applyVisibility(action.payload.tree, newNodes);
-      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+      const visibleNodes = applyVisibility(state.tree, newNodes);
+      return applyFilterToState({
+        ...state,
+        nodes: visibleNodes,
+        changes: generateChanges(state.nodes, visibleNodes),
+      });
    }
    case "COLLAPSE_ALL_BELOW_DEPTH": {
-      const nodesToCollapse = action.payload.tree.filter(
+      const nodesToCollapse = state.tree.filter(
        (n) => n.level >= action.payload.depth && n.hasChildren
      );

@@ -307,11 +326,15 @@ export function reducer(state: TreeState, action: Action): TreeState {
        ])
      );

-      const visibleNodes = applyVisibility(action.payload.tree, newNodes);
-      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+      const visibleNodes = applyVisibility(state.tree, newNodes);
+      return applyFilterToState({
+        ...state,
+        nodes: visibleNodes,
+        changes: generateChanges(state.nodes, visibleNodes),
+      });
    }
    case "EXPAND_LEVEL": {
-      const nodesToExpand = action.payload.tree.filter(
+      const nodesToExpand = state.tree.filter(
        (n) => n.level <= action.payload.level && n.hasChildren
      );

@@ -325,11 +348,15 @@ export function reducer(state: TreeState, action: Action): TreeState {
        ])
      );

-      const visibleNodes = applyVisibility(action.payload.tree, newNodes);
-      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+      const visibleNodes = applyVisibility(state.tree, newNodes);
+      return applyFilterToState({
+        ...state,
+        nodes: visibleNodes,
+        changes: generateChanges(state.nodes, visibleNodes),
+      });
    }
    case "COLLAPSE_LEVEL": {
-      const nodesToCollapse = action.payload.tree.filter(
+      const nodesToCollapse = state.tree.filter(
        (n) => n.level === action.payload.level && n.hasChildren
      );

@@ -343,13 +370,17 @@ export function reducer(state: TreeState, action: Action): TreeState {
        ])
      );

-      const visibleNodes = applyVisibility(action.payload.tree, newNodes);
-      return { nodes: visibleNodes, changes: generateChanges(state.nodes, visibleNodes) };
+      const visibleNodes = applyVisibility(state.tree, newNodes);
+      return applyFilterToState({
+        ...state,
+        nodes: visibleNodes,
+        changes: generateChanges(state.nodes, visibleNodes),
+      });
    }
    case "TOGGLE_EXPAND_LEVEL": {
      //first get the first item at that level in the tree. If it is expanded, collapse all nodes at that level
      //if it is collapsed, expand all nodes at that level
-      const nodesAtLevel = action.payload.tree.filter(
+      const nodesAtLevel = state.tree.filter(
        (n) => n.level === action.payload.level && n.hasChildren
      );
      const firstNode = nodesAtLevel[0];
@@ -364,21 +395,19 @@ export function reducer(state: TreeState, action: Action): TreeState {
          type: "COLLAPSE_LEVEL",
          payload: {
            level: action.payload.level,
-            tree: action.payload.tree,
-          },
-        });
-      } else {
-        return reducer(state, {
-          type: "EXPAND_LEVEL",
-          payload: {
-            level: action.payload.level,
-            tree: action.payload.tree,
          },
        });
      }
+
+      return reducer(state, {
+        type: "EXPAND_LEVEL",
+        payload: {
+          level: action.payload.level,
+        },
+      });
    }
    case "SELECT_FIRST_VISIBLE_NODE": {
-      const node = firstVisibleNode(action.payload.tree, state.nodes);
+      const node = firstVisibleNode(state.tree, state.filteredNodes);
      if (node) {
        return reducer(state, {
          type: "SELECT_NODE",
@@ -389,9 +418,11 @@ export function reducer(state: TreeState, action: Action): TreeState {
          },
        });
      }
+
+      return state;
    }
    case "SELECT_LAST_VISIBLE_NODE": {
-      const node = lastVisibleNode(action.payload.tree, state.nodes);
+      const node = lastVisibleNode(state.tree, state.filteredNodes);
      if (node) {
        return reducer(state, {
          type: "SELECT_NODE",
@@ -402,6 +433,8 @@ export function reducer(state: TreeState, action: Action): TreeState {
          },
        });
      }
+
+      return state;
    }
    case "SELECT_NEXT_VISIBLE_NODE": {
      const selected = selectedIdFromState(state.nodes);
@@ -409,14 +442,13 @@ export function reducer(state: TreeState, action: Action): TreeState {
        return reducer(state, {
          type: "SELECT_FIRST_VISIBLE_NODE",
          payload: {
-            tree: action.payload.tree,
            scrollToNode: action.payload.scrollToNode,
            scrollToNodeFn: action.payload.scrollToNodeFn,
          },
        });
      }

-      const visible = visibleNodes(action.payload.tree, state.nodes);
+      const visible = visibleNodes(state.tree, state.filteredNodes);
      const selectedIndex = visible.findIndex((node) => node.id === selected);
      const nextNode = visible[selectedIndex + 1];
      if (nextNode) {
@@ -429,6 +461,8 @@ export function reducer(state: TreeState, action: Action): TreeState {
          },
        });
      }
+
+      return state;
    }
    case "SELECT_PREVIOUS_VISIBLE_NODE": {
      const selected = selectedIdFromState(state.nodes);
@@ -437,16 +471,15 @@ export function reducer(state: TreeState, action: Action): TreeState {
        return reducer(state, {
          type: "SELECT_FIRST_VISIBLE_NODE",
          payload: {
-            tree: action.payload.tree,
            scrollToNode: action.payload.scrollToNode,
            scrollToNodeFn: action.payload.scrollToNodeFn,
          },
        });
      }

-      const visible = visibleNodes(action.payload.tree, state.nodes);
+      const visible = visibleNodes(state.tree, state.filteredNodes);
      const selectedIndex = visible.findIndex((node) => node.id === selected);
-      const previousNode = visible[selectedIndex - 1];
+      const previousNode = visible[Math.max(0, selectedIndex - 1)];
      if (previousNode) {
        return reducer(state, {
          type: "SELECT_NODE",
@@ -467,19 +500,18 @@ export function reducer(state: TreeState, action: Action): TreeState {
        return reducer(state, {
          type: "SELECT_FIRST_VISIBLE_NODE",
          payload: {
-            tree: action.payload.tree,
            scrollToNode: action.payload.scrollToNode,
            scrollToNodeFn: action.payload.scrollToNodeFn,
          },
        });
      }

-      const selectedNode = action.payload.tree.find((node) => node.id === selected);
+      const selectedNode = state.tree.find((node) => node.id === selected);
      if (!selectedNode) {
        return state;
      }

-      const parentNode = action.payload.tree.find((node) => node.id === selectedNode.parentId);
+      const parentNode = state.tree.find((node) => node.id === selectedNode.parentId);
      if (parentNode) {
        return reducer(state, {
          type: "SELECT_NODE",
@@ -498,12 +530,23 @@ export function reducer(state: TreeState, action: Action): TreeState {
      const selectedId = selectedIdFromState(state.nodes);
      const collapsedIds = collapsedIdsFromState(state.nodes);
      const newState = concreteStateFromInput({
+        ...state,
        tree: action.payload.tree,
        selectedId,
        collapsedIds,
      });
      return newState;
    }
+    case "UPDATE_FILTER": {
+      const newState = applyFilterToState({
+        ...state,
+        filter: action.payload.filter,
+      });
+      return newState;
+    }
+    default: {
+      assertNever(action);
+    }
  }

  throw new Error(`Unhandled action type: ${(action as any).type}`);
@@ -1,4 +1,4 @@
-import { FlatTree, FlatTreeItem } from "./TreeView";
+import { Filter, FlatTree, FlatTreeItem } from "./TreeView";
 import { Changes, NodeState, NodesState, TreeState } from "./reducer";

 type PartialNodeState = Record<string, Partial<NodeState>>;
@@ -8,10 +8,12 @@ const defaultExpanded = true;

 export function concreteStateFromInput({
  tree,
+  filter,
  selectedId,
  collapsedIds,
 }: {
  tree: FlatTree<any>;
+  filter: Filter<any, any> | undefined;
  selectedId: string | undefined;
  collapsedIds: string[] | undefined;
 }): TreeState {
@@ -35,10 +37,15 @@ export function concreteStateFromInput({
      }
    }
  }
+  const nodes = concreteStateFromPartialState(tree, state);

  return {
-    nodes: concreteStateFromPartialState(tree, state),
-    changes: { selectedId, collapsedIds: [] },
+    tree,
+    nodes,
+    changes: { selectedId },
+    filter,
+    filteredNodes: nodes,
+    visibleNodeIds: visibleNodes(tree, nodes).map((node) => node.id),
  };
 }

@@ -82,28 +89,48 @@ export function selectedIdFromState(state: NodesState): string | undefined {
  return selected?.[0];
 }

-export function applyFilterToState<TData>(
-  tree: FlatTree<TData>,
-  inputNodes: NodesState,
-  filter: (node: FlatTreeItem<TData>) => boolean
-): NodesState {
+export function applyFilterToState<TData>({
+  tree,
+  nodes,
+  filter,
+  visibleNodeIds,
+  changes,
+}: TreeState): TreeState {
+  if (!filter || !filter.value) {
+    return {
+      tree,
+      nodes,
+      filteredNodes: nodes,
+      changes,
+      filter,
+      visibleNodeIds: visibleNodes(tree, nodes).map((node) => node.id),
+    };
+  }
+
  //we need to do two passes, first collect all the nodes that are results
  const newFilteredOut = new Set<string>();
  for (const node of tree) {
-    if (!filter(node)) {
+    if (!filter.fn(filter.value, node)) {
      newFilteredOut.add(node.id);
    }
  }

  //nothing is filtered out
  if (newFilteredOut.size === 0) {
-    return inputNodes;
+    return {
+      tree,
+      nodes,
+      filteredNodes: nodes,
+      changes,
+      filter,
+      visibleNodeIds: visibleNodes(tree, nodes).map((node) => node.id),
+    };
  }

  //copy of nodes
-  const nodes = { ...inputNodes };
+  const filteredNodes = { ...nodes };

-  const selected = selectedIdFromState(nodes);
+  const selected = selectedIdFromState(filteredNodes);

  const visible = new Set<string>();
  const expanded = new Set<string>();
@@ -148,28 +175,35 @@ export function applyFilterToState<TData>(

  //now set the visibility and expanded state
  for (const id of hidden) {
-    nodes[id] = { ...nodes[id], visible: false };
+    filteredNodes[id] = { ...filteredNodes[id], visible: false };
  }
  for (const id of visible) {
-    nodes[id] = { ...nodes[id], visible: true };
+    filteredNodes[id] = { ...filteredNodes[id], visible: true };
  }

  for (const id of collapsed) {
-    nodes[id] = { ...nodes[id], expanded: false };
+    filteredNodes[id] = { ...filteredNodes[id], expanded: false };
  }
  for (const id of expanded) {
-    nodes[id] = { ...nodes[id], expanded: true };
+    filteredNodes[id] = { ...filteredNodes[id], expanded: true };
  }

  if (selected) {
    if (visible.has(selected)) {
-      nodes[selected] = { ...nodes[selected], selected: true };
+      filteredNodes[selected] = { ...filteredNodes[selected], selected: true };
    } else {
-      nodes[selected] = { ...nodes[selected], selected: false };
+      filteredNodes[selected] = { ...filteredNodes[selected], selected: false };
    }
  }

-  return nodes;
+  return {
+    tree,
+    nodes,
+    filteredNodes,
+    changes,
+    filter,
+    visibleNodeIds: visibleNodes(tree, filteredNodes).map((node) => node.id),
+  };
 }

 export function visibleNodes(tree: FlatTree<any>, nodes: NodesState) {
@@ -215,6 +249,5 @@ export function generateChanges(a: NodesState, b: NodesState): Changes {

  return {
    selectedId: selectedIdA !== selectedIdB ? selectedIdB : undefined,
-    collapsedIds: collapsedChanges.length > 0 ? collapsedChanges : undefined,
  };
 }
@@ -1,18 +1,14 @@
 import { formatDuration } from "@trigger.dev/core/v3";
-import { useState, useEffect } from "react";
-import { Paragraph } from "~/components/primitives/Paragraph";
-import { cn } from "~/utils/cn";
+import { useEffect, useState } from "react";

 export function LiveTimer({
  startTime,
  endTime,
  updateInterval = 250,
-  className,
 }: {
  startTime: Date;
  endTime?: Date;
  updateInterval?: number;
-  className?: string;
 }) {
  const [now, setNow] = useState<Date>();

@@ -30,13 +26,13 @@ export function LiveTimer({
  }, [startTime]);

  return (
-    <Paragraph variant="extra-small" className={cn("whitespace-nowrap tabular-nums", className)}>
+    <>
      {formatDuration(startTime, now, {
        style: "short",
        maxDecimalPoints: 0,
        units: ["d", "h", "m", "s"],
      })}
-    </Paragraph>
+    </>
  );
 }

@@ -73,14 +73,7 @@ export function SpanCodePathAccessory({
    >
      {accessory.items.map((item, index) => (
        <Fragment key={index}>
-          <span
-            className={cn(
-              "truncate",
-              index === accessory.items.length - 1 ? "text-sun-100" : "text-text-dimmed"
-            )}
-          >
-            {item.text}
-          </span>
+          <span className={cn("truncate", "text-text-dimmed")}>{item.text}</span>
          {index < accessory.items.length - 1 && (
            <span className="text-text-dimmed">
              <ChevronRightIcon className="h-4 w-4" />
@@ -27,7 +27,7 @@ type TaskFunctionNameProps = {

 export function TaskFunctionName({ variant, functionName, className }: TaskFunctionNameProps) {
  return (
-    <InlineCode variant={variant} className={cn("text-sun-100", className)}>
+    <InlineCode variant={variant} className={cn("text-text-dimmed", className)}>
      {`${functionName}()`}
    </InlineCode>
  );
@@ -30,6 +30,23 @@ const taskRunStatusDescriptions: Record<TaskRunStatus, string> = {
  CRASHED: "Task has crashed and won't be retried",
 };

+export const QUEUED_STATUSES: TaskRunStatus[] = ["PENDING", "WAITING_FOR_DEPLOY"];
+
+export const RUNNING_STATUSES: TaskRunStatus[] = [
+  "EXECUTING",
+  "RETRYING_AFTER_FAILURE",
+  "WAITING_TO_RESUME",
+];
+
+export const FINISHED_STATUSES: TaskRunStatus[] = [
+  "COMPLETED_SUCCESSFULLY",
+  "CANCELED",
+  "COMPLETED_WITH_ERRORS",
+  "INTERRUPTED",
+  "SYSTEM_FAILURE",
+  "CRASHED",
+];
+
 export function descriptionForTaskRunStatus(status: TaskRunStatus): string {
  return taskRunStatusDescriptions[status];
 }
@@ -3,7 +3,6 @@ import { StopIcon } from "@heroicons/react/24/outline";
 import { BeakerIcon, BookOpenIcon, CheckIcon } from "@heroicons/react/24/solid";
 import { useLocation } from "@remix-run/react";
 import { formatDuration } from "@trigger.dev/core/v3";
-import { User } from "@trigger.dev/database";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
 import { Dialog, DialogTrigger } from "~/components/primitives/Dialog";
 import { useEnvironments } from "~/hooks/useEnvironments";
@@ -28,6 +27,7 @@ import {
 import { CancelRunDialog } from "./CancelRunDialog";
 import { ReplayRunDialog } from "./ReplayRunDialog";
 import { TaskRunStatusCombo } from "./TaskRunStatus";
+import { LiveTimer } from "./LiveTimer";

 type RunsTableProps = {
  total: number;
@@ -36,7 +36,6 @@ type RunsTableProps = {
  showJob?: boolean;
  runs: RunListItem[];
  isLoading?: boolean;
-  currentUser: User;
 };

 export function TaskRunsTable({
@@ -45,7 +44,6 @@ export function TaskRunsTable({
  filters,
  runs,
  isLoading = false,
-  currentUser,
 }: RunsTableProps) {
  const organization = useOrganization();
  const project = useProject();
@@ -78,15 +76,16 @@ export function TaskRunsTable({
        ) : (
          runs.map((run) => {
            const path = v3RunSpanPath(organization, project, run, { spanId: run.spanId });
-            const usernameForEnv =
-              currentUser.id !== run.environment.userId ? run.environment.userName : undefined;
            return (
              <TableRow key={run.id}>
                <TableCell to={path}>#{run.number}</TableCell>
                <TableCell to={path}>{run.taskIdentifier}</TableCell>
                <TableCell to={path}>{run.version ?? "–"}</TableCell>
                <TableCell to={path}>
-                  <EnvironmentLabel environment={run.environment} userName={usernameForEnv} />
+                  <EnvironmentLabel
+                    environment={run.environment}
+                    userName={run.environment.userName}
+                  />
                </TableCell>
                <TableCell to={path}>
                  <TaskRunStatusCombo status={run.status} />
@@ -95,9 +94,15 @@ export function TaskRunsTable({
                  {run.startedAt ? <DateTime date={run.startedAt} /> : "–"}
                </TableCell>
                <TableCell to={path}>
-                  {formatDuration(run.startedAt, run.completedAt, {
-                    style: "short",
-                  })}
+                  {run.startedAt && run.finishedAt ? (
+                    formatDuration(new Date(run.startedAt), new Date(run.finishedAt), {
+                      style: "short",
+                    })
+                  ) : run.startedAt ? (
+                    <LiveTimer startTime={new Date(run.startedAt)} />
+                  ) : (
+                    "–"
+                  )}
                </TableCell>
                <TableCell to={path}>
                  {run.isTest ? (
@@ -196,7 +201,12 @@ function BlankState({ isLoading, filters }: Pick<RunsTableProps, "isLoading" | "
            {environment ? (
              <>
                {" "}
-                in <EnvironmentLabel environment={environment} size="large" />
+                in{" "}
+                <EnvironmentLabel
+                  environment={environment}
+                  userName={environment.userName}
+                  size="large"
+                />
              </>
            ) : null}
          </Paragraph>
@@ -70,23 +70,21 @@ export { Prisma };

 export const prisma = singleton("prisma", getClient);

+export const $replica: Omit<PrismaClient, "$transaction"> = singleton(
+  "replica",
+  () => getReplicaClient() ?? prisma
+);
+
 function getClient() {
  const { DATABASE_URL } = process.env;
  invariant(typeof DATABASE_URL === "string", "DATABASE_URL env var not set");

-  const databaseUrl = new URL(DATABASE_URL);
+  const databaseUrl = extendQueryParams(DATABASE_URL, {
+    connection_limit: env.DATABASE_CONNECTION_LIMIT.toString(),
+    pool_timeout: env.DATABASE_POOL_TIMEOUT.toString(),
+  });

-  // We need to add the connection_limit and pool_timeout query params to the url, in a way that works if the DATABASE_URL already has query params
-  const query = databaseUrl.searchParams;
-  query.set("connection_limit", env.DATABASE_CONNECTION_LIMIT.toString());
-  query.set("pool_timeout", env.DATABASE_POOL_TIMEOUT.toString());
-  databaseUrl.search = query.toString();
-
-  // Remove the username:password in the url and print that to the console
-  const urlWithoutCredentials = new URL(databaseUrl.href);
-  urlWithoutCredentials.password = "";
-
-  console.log(`🔌 setting up prisma client to ${urlWithoutCredentials.toString()}`);
+  console.log(`🔌 setting up prisma client to ${redactUrlSecrets(databaseUrl)}`);

  const client = new PrismaClient({
    datasources: {
@@ -134,6 +132,68 @@ function getClient() {
  return client;
 }

+function getReplicaClient() {
+  if (!env.DATABASE_READ_REPLICA_URL) {
+    console.log(`🔌 No database replica, using the regular client`);
+    return;
+  }
+
+  const replicaUrl = extendQueryParams(env.DATABASE_READ_REPLICA_URL, {
+    connection_limit: env.DATABASE_CONNECTION_LIMIT.toString(),
+    pool_timeout: env.DATABASE_POOL_TIMEOUT.toString(),
+  });
+
+  console.log(`🔌 setting up read replica connection to ${redactUrlSecrets(replicaUrl)}`);
+
+  const replicaClient = new PrismaClient({
+    datasources: {
+      db: {
+        url: replicaUrl.href,
+      },
+    },
+    log: [
+      {
+        emit: "stdout",
+        level: "error",
+      },
+      {
+        emit: "stdout",
+        level: "info",
+      },
+      {
+        emit: "stdout",
+        level: "warn",
+      },
+    ],
+  });
+
+  // connect eagerly
+  replicaClient.$connect();
+
+  console.log(`🔌 read replica connected`);
+
+  return replicaClient;
+}
+
+function extendQueryParams(hrefOrUrl: string | URL, queryParams: Record<string, string>) {
+  const url = new URL(hrefOrUrl);
+  const query = url.searchParams;
+
+  for (const [key, val] of Object.entries(queryParams)) {
+    query.set(key, val);
+  }
+
+  url.search = query.toString();
+
+  return url;
+}
+
+function redactUrlSecrets(hrefOrUrl: string | URL) {
+  const url = new URL(hrefOrUrl);
+  url.password = "";
+  return url.href;
+}
+
 export type { PrismaClient } from "@trigger.dev/database";

 export const PrismaErrorSchema = z.object({
@@ -19,6 +19,7 @@ const EnvironmentSchema = z.object({
      isValidDatabaseUrl,
      "DIRECT_URL is invalid, for details please check the additional output above this message."
    ),
+  DATABASE_READ_REPLICA_URL: z.string().optional(),
  SESSION_SECRET: z.string(),
  MAGIC_LINK_SECRET: z.string(),
  ENCRYPTION_KEY: z.string(),
@@ -1,6 +1,11 @@
 import { useRef } from "react";

-//a function that you call with a debounce delay, the function will only be called after the delay has passed
+/**
+ * A function that you call with a debounce delay, the function will only be called after the delay has passed
+ *
+ * @param fn The function to debounce
+ * @param delay In ms
+ */
 export function useDebounce<T extends (...args: any[]) => any>(fn: T, delay: number) {
  const timeout = useRef<ReturnType<typeof setTimeout>>();

@@ -10,20 +10,3 @@ export function useEnvironments(matches?: UIMatch[]) {

  return project.environments;
 }
-
-export function useDevEnvironment(matches?: UIMatch[]) {
-  const user = useUser();
-  const environments = useEnvironments(matches);
-  if (!environments) return;
-
-  return environments.find(
-    (environment) => environment.type === "DEVELOPMENT" && environment.userId === user.id
-  );
-}
-
-export function useProdEnvironment(matches?: UIMatch[]) {
-  const environments = useEnvironments(matches);
-  if (!environments) return;
-
-  return environments.find((environment) => environment.type === "PRODUCTION");
-}
@@ -0,0 +1,11 @@
+import { useRef, MutableRefObject } from "react";
+
+const useLazyRef = <T>(initialValFunc: () => T) => {
+  const ref: MutableRefObject<T | null> = useRef(null);
+  if (ref.current === null) {
+    ref.current = initialValFunc();
+  }
+  return ref;
+};
+
+export default useLazyRef;
@@ -0,0 +1,72 @@
+import { Reducer, useReducer } from "react";
+
+export type ListState<T> = {
+  items: T[];
+};
+
+type AppendAction<T> = {
+  type: "append";
+  items: T[];
+};
+
+type UpdateAction<T> = {
+  type: "update";
+  index: number;
+  item: T;
+};
+
+type DeleteAction<T> = {
+  type: "delete";
+  index: number;
+};
+
+type InsertAfter<T> = {
+  type: "insertAfter";
+  index: number;
+  items: T[];
+};
+
+type Action<T> = AppendAction<T> | UpdateAction<T> | DeleteAction<T> | InsertAfter<T>;
+
+function reducer<T>(state: ListState<T>, action: Action<T>): ListState<T> {
+  switch (action.type) {
+    case "append":
+      return { items: [...state.items, ...action.items] };
+    case "update":
+      return {
+        items: state.items.map((v, i) => (i === action.index ? action.item : v)),
+      };
+    case "delete":
+      return { items: state.items.filter((_, i) => i !== action.index) };
+    case "insertAfter":
+      return {
+        items: [
+          ...state.items.slice(0, action.index + 1),
+          ...action.items,
+          ...state.items.slice(action.index + 1),
+        ],
+      };
+  }
+}
+
+type HookReturn<T> = {
+  items: T[];
+  append: (items: T[]) => void;
+  update: (index: number, item: T) => void;
+  delete: (index: number) => void;
+  insertAfter: (index: number, items: T[]) => void;
+};
+
+export function useList<T>(initialItems: T[]): HookReturn<T> {
+  const [state, dispatch] = useReducer<Reducer<ListState<T>, Action<T>>>(reducer, {
+    items: initialItems,
+  });
+
+  return {
+    items: state.items,
+    append: (items: T[]) => dispatch({ type: "append", items }),
+    update: (index: number, item: T) => dispatch({ type: "update", index, item }),
+    delete: (index: number) => dispatch({ type: "delete", index }),
+    insertAfter: (index: number, items: T[]) => dispatch({ type: "insertAfter", index, items }),
+  };
+}
@@ -26,3 +26,9 @@ export function useUser(matches?: UIMatch[]): User {
 export function useUserChanged(callback: (user: User | undefined) => void) {
  useChanged(useOptionalUser, callback);
 }
+
+export function useHasAdminAccess(matches?: UIMatch[]): boolean {
+  const user = useOptionalUser(matches);
+
+  return Boolean(user?.admin);
+}
@@ -168,6 +168,12 @@ export async function adminGetOrganizations(userId: string, { page, search }: Se
                mode: "insensitive",
              },
            },
+            {
+              id: {
+                contains: search,
+                mode: "insensitive",
+              },
+            },
          ],
        }
      : undefined,
@@ -1,6 +1,6 @@
 import { json, Session } from "@remix-run/node";
-import { redirect } from "remix-typedjson";
 import { createCookieSessionStorage } from "@remix-run/node";
+import { redirect } from "remix-typedjson";
 import { env } from "~/env.server";

 export type ToastMessage = {
@@ -87,8 +87,8 @@ export async function createOrganization(
 }

 export async function createEnvironment(
-  organization: Organization,
-  project: Project,
+  organization: Pick<Organization, "id">,
+  project: Pick<Project, "id">,
  type: RuntimeEnvironment["type"],
  member?: OrgMember,
  prismaClient: PrismaClientOrTransaction = prisma
@@ -87,7 +87,7 @@ export async function createProject(
  // Create the dev and prod environments
  await createEnvironment(organization, project, "PRODUCTION");

-  if (project.version === "V2") {
+  if (version === "v2") {
    await createEnvironment(organization, project, "STAGING");
  }

@@ -1,5 +1,6 @@
-import type { RuntimeEnvironment } from "@trigger.dev/database";
+import type { Prisma, RuntimeEnvironment } from "@trigger.dev/database";
 import { prisma } from "~/db.server";
+import { getUsername } from "~/utils/username";

 export type { RuntimeEnvironment };

@@ -118,3 +119,36 @@ export async function disconnectSession(environmentId: string) {
    return session;
  });
 }
+
+type DisplayableInputEnvironment = Prisma.RuntimeEnvironmentGetPayload<{
+  select: {
+    id: true;
+    type: true;
+    orgMember: {
+      select: {
+        user: {
+          select: {
+            id: true;
+            name: true;
+            displayName: true;
+          };
+        };
+      };
+    };
+  };
+}>;
+
+export function displayableEnvironments(
+  environment: DisplayableInputEnvironment,
+  userId: string | undefined
+) {
+  return {
+    id: environment.id,
+    type: environment.type,
+    userName: environment.orgMember
+      ? environment.orgMember.user.id === userId
+        ? undefined
+        : getUsername(environment.orgMember.user)
+      : undefined,
+  };
+}
@@ -1,6 +1,8 @@
 import { PrismaClient, prisma } from "~/db.server";
 import { Project } from "~/models/project.server";
+import { displayableEnvironments } from "~/models/runtimeEnvironment.server";
 import { User } from "~/models/user.server";
+import { sortEnvironments } from "~/services/environmentSort.server";

 export class ProjectPresenter {
  #prismaClient: PrismaClient;
@@ -49,7 +51,13 @@ export class ProjectPresenter {
            type: true,
            orgMember: {
              select: {
-                userId: true,
+                user: {
+                  select: {
+                    id: true,
+                    name: true,
+                    displayName: true,
+                  },
+                },
              },
            },
            apiKey: true,
@@ -76,13 +84,12 @@ export class ProjectPresenter {
      hasInactiveExternalTriggers: project._count.sources > 0,
      jobCount: project._count.jobs,
      httpEndpointCount: project._count.httpEndpoints,
-      environments: project.environments.map((environment) => ({
-        id: environment.id,
-        slug: environment.slug,
-        type: environment.type,
-        apiKey: environment.apiKey,
-        userId: environment.orgMember?.userId,
-      })),
+      environments: sortEnvironments(
+        project.environments.map((environment) => ({
+          ...displayableEnvironments(environment, userId),
+          userId: environment.orgMember?.user.id,
+        }))
+      ),
    };
  }
 }
@@ -74,6 +74,7 @@ export class ApiKeysPresenter {
          environmentVariableCount: environment._count.environmentVariableValues,
        }))
      ),
+      hasStaging: environments.some((environment) => environment.type === "STAGING"),
    };
  }
 }
@@ -103,6 +103,7 @@ export class DeploymentPresenter {
                exportName: "asc",
              },
            },
+            sdkVersion: true,
          },
        },
        triggeredBy: {
@@ -135,6 +136,7 @@ export class DeploymentPresenter {
        },
        deployedBy: deployment.triggeredBy,
        errorData: this.#prepareErrorData(deployment.errorData),
+        sdkVersion: deployment.worker?.sdkVersion,
      },
    };
  }
@@ -1,10 +1,13 @@
 import { Prisma, TaskRunStatus } from "@trigger.dev/database";
 import { Direction } from "~/components/runs/RunStatuses";
-import { sqlDatabaseSchema, PrismaClient, prisma } from "~/db.server";
-import { getUsername } from "~/utils/username";
+import { FINISHED_STATUSES } from "~/components/runs/v3/TaskRunStatus";
+import { sqlDatabaseSchema } from "~/db.server";
+import { displayableEnvironments } from "~/models/runtimeEnvironment.server";
 import { CANCELLABLE_STATUSES } from "~/v3/services/cancelTaskRun.server";
+import { BasePresenter } from "./basePresenter.server";

 type RunListOptions = {
+  userId?: string;
  projectSlug: string;
  //filters
  tasks?: string[];
@@ -26,14 +29,9 @@ export type RunList = Awaited<ReturnType<RunListPresenter["call"]>>;
 export type RunListItem = RunList["runs"][0];
 export type RunListAppliedFilters = RunList["filters"];

-export class RunListPresenter {
-  #prismaClient: PrismaClient;
-
-  constructor(prismaClient: PrismaClient = prisma) {
-    this.#prismaClient = prismaClient;
-  }
-
+export class RunListPresenter extends BasePresenter {
  public async call({
+    userId,
    projectSlug,
    tasks,
    versions,
@@ -57,7 +55,7 @@ export class RunListPresenter {
      to !== undefined;

    // Find the project scoped to the organization
-    const project = await this.#prismaClient.project.findFirstOrThrow({
+    const project = await this._replica.project.findFirstOrThrow({
      select: {
        id: true,
        environments: {
@@ -85,7 +83,7 @@ export class RunListPresenter {
    });

    //get all possible tasks
-    const possibleTasks = await this.#prismaClient.backgroundWorkerTask.findMany({
+    const possibleTasks = await this._replica.backgroundWorkerTask.findMany({
      distinct: ["slug"],
      where: {
        projectId: project.id,
@@ -93,7 +91,7 @@ export class RunListPresenter {
    });

    //get the runs
-    let runs = await this.#prismaClient.$queryRaw<
+    let runs = await this._replica.$queryRaw<
      {
        id: string;
        number: BigInt;
@@ -104,10 +102,9 @@ export class RunListPresenter {
        status: TaskRunStatus;
        createdAt: Date;
        lockedAt: Date | null;
-        completedAt: Date | null;
+        updatedAt: Date;
        isTest: boolean;
        spanId: string;
-        attempts: BigInt;
      }[]
    >`
    SELECT
@@ -120,20 +117,13 @@ export class RunListPresenter {
    tr.status AS status,
    tr."createdAt" AS "createdAt",
    tr."lockedAt" AS "lockedAt",
-    tra."completedAt" AS "completedAt",
+    tr."updatedAt" AS "updatedAt",
    tr."isTest" AS "isTest",
-    tr."spanId" AS "spanId",
-    COUNT(tra.id) AS attempts
+    tr."spanId" AS "spanId"
  FROM
    ${sqlDatabaseSchema}."TaskRun" tr
  LEFT JOIN
-    (
-      SELECT *,
-        ROW_NUMBER() OVER (PARTITION BY "taskRunId" ORDER BY "createdAt" DESC) rn
-      FROM ${sqlDatabaseSchema}."TaskRunAttempt"
-    ) tra ON tr.id = tra."taskRunId" AND tra.rn = 1
-  LEFT JOIN
-    ${sqlDatabaseSchema}."BackgroundWorker" bw ON tra."backgroundWorkerId" = bw.id
+    ${sqlDatabaseSchema}."BackgroundWorker" bw ON tr."lockedToVersionId" = bw.id
  WHERE
      -- project
      tr."projectId" = ${project.id}
@@ -151,15 +141,11 @@ export class RunListPresenter {
          ? Prisma.sql`AND tr."taskIdentifier" IN (${Prisma.join(tasks)})`
          : Prisma.empty
      }
-      ${hasStatusFilters ? Prisma.sql`AND (` : Prisma.empty}
      ${
        statuses && statuses.length > 0
-          ? Prisma.sql`tr.status = ANY(ARRAY[${Prisma.join(statuses)}]::"TaskRunStatus"[])`
+          ? Prisma.sql`AND tr.status = ANY(ARRAY[${Prisma.join(statuses)}]::"TaskRunStatus"[])`
          : Prisma.empty
      }
-      ${statuses && statuses.length > 0 && hasStatusFilters ? Prisma.sql` OR ` : Prisma.empty}
-      ${hasStatusFilters ? Prisma.sql`tr.status IS NULL` : Prisma.empty}
-      ${hasStatusFilters ? Prisma.sql`) ` : Prisma.empty}
      ${
        environments && environments.length > 0
          ? Prisma.sql`AND tr."runtimeEnvironmentId" IN (${Prisma.join(environments)})`
@@ -176,8 +162,6 @@ export class RunListPresenter {
          ? Prisma.sql`AND tr."createdAt" <= ${new Date(to).toISOString()}::timestamp`
          : Prisma.empty
      } 
-  GROUP BY
-    tr."friendlyId", tr."taskIdentifier", tr."runtimeEnvironmentId", tr.id, bw.version, tra.status, tr."createdAt", tra."startedAt", tra."completedAt"
  ORDER BY
    ${direction === "forward" ? Prisma.sql`tr.id DESC` : Prisma.sql`tr.id ASC`}
  LIMIT ${pageSize + 1}`;
@@ -216,27 +200,24 @@ export class RunListPresenter {
          throw new Error(`Environment not found for TaskRun ${run.id}`);
        }

+        const hasFinished = FINISHED_STATUSES.includes(run.status);
+
        return {
          id: run.id,
          friendlyId: run.runFriendlyId,
          number: Number(run.number),
-          createdAt: run.createdAt,
-          startedAt: run.lockedAt,
-          completedAt: run.completedAt,
+          createdAt: run.createdAt.toISOString(),
+          startedAt: run.lockedAt ? run.lockedAt.toISOString() : undefined,
+          hasFinished,
+          finishedAt: hasFinished ? run.updatedAt.toISOString() : undefined,
          isTest: run.isTest,
          status: run.status,
          version: run.version,
          taskIdentifier: run.taskIdentifier,
          spanId: run.spanId,
-          attempts: Number(run.attempts),
          isReplayable: true,
          isCancellable: CANCELLABLE_STATUSES.includes(run.status),
-          environment: {
-            type: environment.type,
-            slug: environment.slug,
-            userId: environment.orgMember?.user.id,
-            userName: getUsername(environment.orgMember?.user),
-          },
+          environment: displayableEnvironments(environment, userId),
        };
      }),
      pagination: {
@@ -38,6 +38,7 @@ export class RunPresenter {
            id: true,
            type: true,
            slug: true,
+            organizationId: true,
            orgMember: {
              select: {
                user: {
@@ -66,9 +67,13 @@ export class RunPresenter {
    if (!traceSummary) {
      return {
        run: {
+          id: run.id,
          number: run.number,
          friendlyId: run.friendlyId,
+          traceId: run.traceId,
          environment: {
+            id: run.runtimeEnvironment.id,
+            organizationId: run.runtimeEnvironment.organizationId,
            type: run.runtimeEnvironment.type,
            slug: run.runtimeEnvironment.slug,
            userId: run.runtimeEnvironment.orgMember?.user.id,
@@ -118,9 +123,13 @@ export class RunPresenter {

    return {
      run: {
+        id: run.id,
        number: run.number,
        friendlyId: run.friendlyId,
+        traceId: run.traceId,
        environment: {
+          id: run.runtimeEnvironment.id,
+          organizationId: run.runtimeEnvironment.organizationId,
          type: run.runtimeEnvironment.type,
          slug: run.runtimeEnvironment.slug,
          userId: run.runtimeEnvironment.orgMember?.user.id,
@@ -42,7 +42,7 @@ export class SpanPresenter {
    const output =
      span.outputType === "application/store"
        ? `/resources/packets/${span.environmentId}/${span.output}`
-        : typeof span.output !== "undefined" && span.output !== null
+        : typeof span.output !== "undefined"
        ? await prettyPrintPacket(span.output, span.outputType ?? undefined)
        : undefined;

@@ -1,19 +1,37 @@
-import { Prisma, TaskRunStatus, TaskTriggerSource } from "@trigger.dev/database";
-import { PrismaClient, prisma, sqlDatabaseSchema } from "~/db.server";
+import {
+  Prisma,
+  RuntimeEnvironmentType,
+  TaskRunStatus,
+  TaskTriggerSource,
+} from "@trigger.dev/database";
+import { QUEUED_STATUSES, RUNNING_STATUSES } from "~/components/runs/v3/TaskRunStatus";
+import { sqlDatabaseSchema } from "~/db.server";
 import { Organization } from "~/models/organization.server";
 import { Project } from "~/models/project.server";
+import { displayableEnvironments } from "~/models/runtimeEnvironment.server";
 import { User } from "~/models/user.server";
-import { getUsername } from "~/utils/username";
+import { sortEnvironments } from "~/services/environmentSort.server";
+import { logger } from "~/services/logger.server";
+import { BasePresenter } from "./basePresenter.server";

-export type Task = Awaited<ReturnType<TaskListPresenter["call"]>>[0];
+export type Task = {
+  slug: string;
+  exportName: string;
+  filePath: string;
+  createdAt: Date;
+  triggerSource: TaskTriggerSource;
+  environments: {
+    id: string;
+    type: RuntimeEnvironmentType;
+    userName?: string;
+  }[];
+};

-export class TaskListPresenter {
-  #prismaClient: PrismaClient;
+type Return = Awaited<ReturnType<TaskListPresenter["call"]>>;

-  constructor(prismaClient: PrismaClient = prisma) {
-    this.#prismaClient = prismaClient;
-  }
+export type TaskActivity = Awaited<Return["activity"]>[string];

+export class TaskListPresenter extends BasePresenter {
  public async call({
    userId,
    projectSlug,
@@ -23,7 +41,7 @@ export class TaskListPresenter {
    projectSlug: Project["slug"];
    organizationSlug: Organization["slug"];
  }) {
-    const project = await this.#prismaClient.project.findFirstOrThrow({
+    const project = await this._replica.project.findFirstOrThrow({
      select: {
        id: true,
        environments: {
@@ -53,7 +71,7 @@ export class TaskListPresenter {
      },
    });

-    const tasks = await this.#prismaClient.$queryRaw<
+    const tasks = await this._replica.$queryRaw<
      {
        id: string;
        slug: string;
@@ -64,73 +82,217 @@ export class TaskListPresenter {
        triggerSource: TaskTriggerSource;
      }[]
    >`
-    SELECT DISTINCT ON(bwt.slug, bwt."runtimeEnvironmentId")
-      bwt.slug,
-      bwt.id,
-      bwt."exportName",
-      bwt."filePath",
-      bwt."runtimeEnvironmentId",
-      bwt."createdAt",
-      bwt."triggerSource"
-    FROM
-      ${sqlDatabaseSchema}."BackgroundWorkerTask" as bwt
-    WHERE bwt."projectId" = ${project.id}
-    ORDER BY
-      bwt.slug,
-      bwt."runtimeEnvironmentId",
-      bwt."createdAt" DESC;`;
+    WITH workers AS (
+      SELECT DISTINCT ON ("runtimeEnvironmentId") id, "runtimeEnvironmentId", version
+      FROM ${sqlDatabaseSchema}."BackgroundWorker"
+      WHERE "runtimeEnvironmentId" IN (${Prisma.join(project.environments.map((e) => e.id))})
+      ORDER BY "runtimeEnvironmentId", "createdAt" DESC
+    )
+    SELECT tasks.id, slug, "filePath", "exportName", "triggerSource", tasks."runtimeEnvironmentId", tasks."createdAt"
+    FROM workers
+    JOIN ${sqlDatabaseSchema}."BackgroundWorkerTask" tasks ON tasks."workerId" = workers.id
+    ORDER BY slug ASC;`;

-    let latestRuns = [] as {
-      createdAt: Date;
-      status: TaskRunStatus;
-      lockedById: string;
-    }[];
-
-    if (tasks.length > 0) {
-      latestRuns = await this.#prismaClient.$queryRaw<
-        {
-          createdAt: Date;
-          status: TaskRunStatus;
-          lockedById: string;
-        }[]
-      >`
-      SELECT * FROM (
-        SELECT
-          "createdAt",
-          "status",
-          "lockedById",
-          ROW_NUMBER() OVER (PARTITION BY "lockedById" ORDER BY "updatedAt" DESC) AS rn
-        FROM
-          ${sqlDatabaseSchema}."TaskRun"
-        WHERE
-          "lockedById" IN(${Prisma.join(tasks.map((t) => t.id))})
-            ) t
-            WHERE rn = 1;`;
-    }
-
-    return tasks.map((task) => {
-      const latestRun = latestRuns.find((r) => r.lockedById === task.id);
+    //group by the task identifier (task.slug). Add the latestRun and add all the environments.
+    const outputTasks = tasks.reduce((acc, task) => {
      const environment = project.environments.find((env) => env.id === task.runtimeEnvironmentId);
      if (!environment) {
        throw new Error(`Environment not found for TaskRun ${task.id}`);
      }

-      return {
-        ...task,
-        environment: {
-          id: environment.id,
-          type: environment.type,
-          slug: environment.slug,
-          userId: environment.orgMember?.user.id,
-          userName: getUsername(environment.orgMember?.user),
-        },
-        latestRun: latestRun
-          ? {
-              createdAt: latestRun.createdAt,
-              status: latestRun.status,
-            }
-          : undefined,
-      };
-    });
+      let existingTask = acc.find((t) => t.slug === task.slug);
+
+      if (!existingTask) {
+        existingTask = {
+          ...task,
+          environments: [],
+        };
+        acc.push(existingTask);
+      }
+
+      existingTask.environments.push(displayableEnvironments(environment, userId));
+
+      //order the environments
+      existingTask.environments = sortEnvironments(existingTask.environments);
+
+      return acc;
+    }, [] as Task[]);
+
+    //then get the activity for each task
+    const activity = this.#getActivity(
+      outputTasks.map((t) => t.slug),
+      project.id
+    );
+
+    const runningStats = this.#getRunningStats(
+      outputTasks.map((t) => t.slug),
+      project.id
+    );
+
+    const durations = this.#getAverageDurations(
+      outputTasks.map((t) => t.slug),
+      project.id
+    );
+
+    const userEnvironment = project.environments.find((e) => e.orgMember?.user.id === userId);
+    const userHasTasks = userEnvironment
+      ? outputTasks.some((t) => t.environments.some((e) => e.id === userEnvironment.id))
+      : false;
+
+    return { tasks: outputTasks, userHasTasks, activity, runningStats, durations };
+  }
+
+  async #getActivity(tasks: string[], projectId: string) {
+    if (tasks.length === 0) {
+      return {};
+    }
+
+    const activity = await this._replica.$queryRaw<
+      {
+        taskIdentifier: string;
+        status: TaskRunStatus;
+        day: Date;
+        count: BigInt;
+      }[]
+    >`
+    SELECT 
+    tr."taskIdentifier", 
+    tr."status",
+    DATE(tr."createdAt") as day, 
+    COUNT(*) 
+  FROM 
+    ${sqlDatabaseSchema}."TaskRun" as tr
+  WHERE 
+    tr."taskIdentifier" IN (${Prisma.join(tasks)})
+    AND tr."projectId" = ${projectId}
+    AND tr."createdAt" >= (current_date - interval '6 days')
+  GROUP BY 
+    tr."taskIdentifier", 
+    tr."status", 
+    day
+  ORDER BY 
+    tr."taskIdentifier" ASC,
+    day ASC,
+    tr."status" ASC;`;
+
+    //today with no time
+    const today = new Date();
+    today.setUTCHours(0, 0, 0, 0);
+
+    return activity.reduce((acc, a) => {
+      let existingTask = acc[a.taskIdentifier];
+
+      if (!existingTask) {
+        existingTask = [];
+        //populate the array with the past 7 days
+        for (let i = 6; i >= 0; i--) {
+          const day = new Date(today);
+          day.setUTCDate(today.getDate() - i);
+          day.setUTCHours(0, 0, 0, 0);
+
+          existingTask.push({
+            day: day.toISOString(),
+            [TaskRunStatus.COMPLETED_SUCCESSFULLY]: 0,
+          } as { day: string } & Record<TaskRunStatus, number>);
+        }
+
+        acc[a.taskIdentifier] = existingTask;
+      }
+
+      const dayString = a.day.toISOString();
+      const day = existingTask.find((d) => d.day === dayString);
+
+      if (!day) {
+        logger.warn(`Day not found for TaskRun`, {
+          day: dayString,
+          taskIdentifier: a.taskIdentifier,
+          existingTask,
+        });
+        return acc;
+      }
+
+      day[a.status] = Number(a.count);
+
+      return acc;
+    }, {} as Record<string, ({ day: string } & Record<TaskRunStatus, number>)[]>);
+  }
+
+  async #getRunningStats(tasks: string[], projectId: string) {
+    if (tasks.length === 0) {
+      return {};
+    }
+
+    const statuses = await this._replica.$queryRaw<
+      {
+        taskIdentifier: string;
+        status: TaskRunStatus;
+        count: BigInt;
+      }[]
+    >`
+    SELECT 
+    tr."taskIdentifier", 
+    tr."status",
+    COUNT(*) 
+  FROM 
+    ${sqlDatabaseSchema}."TaskRun" as tr
+  WHERE 
+    tr."taskIdentifier" IN (${Prisma.join(tasks)})
+    AND tr."projectId" = ${projectId}
+    AND tr."status" IN ('PENDING', 'WAITING_FOR_DEPLOY', 'EXECUTING', 'RETRYING_AFTER_FAILURE', 'WAITING_TO_RESUME')
+  GROUP BY 
+    tr."taskIdentifier", 
+    tr."status"
+  ORDER BY 
+    tr."taskIdentifier" ASC,
+    tr."status" ASC;`;
+
+    return statuses.reduce((acc, a) => {
+      let existingTask = acc[a.taskIdentifier];
+
+      if (!existingTask) {
+        existingTask = {
+          queued: 0,
+          running: 0,
+        };
+
+        acc[a.taskIdentifier] = existingTask;
+      }
+
+      if (QUEUED_STATUSES.includes(a.status)) {
+        existingTask.queued += Number(a.count);
+      }
+      if (RUNNING_STATUSES.includes(a.status)) {
+        existingTask.running += Number(a.count);
+      }
+
+      return acc;
+    }, {} as Record<string, { queued: number; running: number }>);
+  }
+
+  async #getAverageDurations(tasks: string[], projectId: string) {
+    if (tasks.length === 0) {
+      return {};
+    }
+
+    const durations = await this._replica.$queryRaw<
+      {
+        taskIdentifier: string;
+        duration: Number;
+      }[]
+    >`    
+    SELECT 
+      tr."taskIdentifier", 
+      AVG(EXTRACT(EPOCH FROM (tr."updatedAt" - tr."lockedAt"))) as duration
+      FROM 
+      ${sqlDatabaseSchema}."TaskRun" as tr
+    WHERE 
+      tr."taskIdentifier" IN (${Prisma.join(tasks)})
+      AND tr."projectId" = ${projectId}
+      AND tr."createdAt" >= (current_date - interval '6 days')
+      AND tr."status" IN ('COMPLETED_SUCCESSFULLY', 'COMPLETED_WITH_ERRORS')
+    GROUP BY 
+      tr."taskIdentifier";`;
+
+    return Object.fromEntries(durations.map((s) => [s.taskIdentifier, Number(s.duration)]));
  }
 }
@@ -63,17 +63,18 @@ export class TestPresenter {
    const searchParams = createSearchParams(url, TestSearchParams);

    //no environmentId
-    if (!searchParams.success || !searchParams.params.get("environment")) {
+    if (!searchParams.success) {
      return {
        hasSelectedEnvironment: false as const,
        environments,
      };
    }

+    //default to dev environment
+    const environment = searchParams.params.get("environment") ?? "dev";
+
    //is the environmentId valid?
-    const matchingEnvironment = project.environments.find(
-      (env) => env.slug === searchParams.params.get("environment")
-    );
+    const matchingEnvironment = project.environments.find((env) => env.slug === environment);
    if (!matchingEnvironment) {
      return {
        hasSelectedEnvironment: false as const,
@@ -101,7 +102,7 @@ export class TestPresenter {
      WHERE "runtimeEnvironmentId" = ${matchingEnvironment.id}
    ),
    latest_workers AS (SELECT * FROM workers WHERE rn = 1)
-    SELECT bwt.id, version, slug as "taskIdentifier", "filePath", "exportName", bwt."friendlyId" 
+    SELECT bwt.id, version, slug as "taskIdentifier", "filePath", "exportName", bwt."friendlyId", bwt."triggerSource"
    FROM latest_workers
    JOIN ${sqlDatabaseSchema}."BackgroundWorkerTask" bwt ON bwt."workerId" = latest_workers.id
    ORDER BY bwt."exportName" ASC;
@@ -171,17 +171,23 @@ export class TestTaskPresenter {
        return {
          triggerSource: "SCHEDULED",
          task: taskWithEnvironment,
-          runs: await Promise.all(
-            latestRuns.map(async (r) => {
-              const number = Number(r.number);
+          runs: (
+            await Promise.all(
+              latestRuns.map(async (r) => {
+                const number = Number(r.number);

-              return {
-                ...r,
-                number,
-                payload: await getScheduleTaskRunPayload(r),
-              };
-            })
-          ),
+                const payload = await getScheduleTaskRunPayload(r);
+
+                if (payload.success) {
+                  return {
+                    ...r,
+                    number,
+                    payload: payload.data,
+                  };
+                }
+              })
+            )
+          ).filter(Boolean),
        };
    }
  }
@@ -189,6 +195,6 @@ export class TestTaskPresenter {

 async function getScheduleTaskRunPayload(run: RawRun) {
  const payload = await parsePacket({ data: run.payload, dataType: run.payloadType });
-  const parsed = ScheduledTaskPayload.parse(payload);
+  const parsed = ScheduledTaskPayload.safeParse(payload);
  return parsed;
 }
@@ -1,10 +1,13 @@
 import { Span, SpanKind } from "@opentelemetry/api";
-import { PrismaClientOrTransaction, prisma } from "~/db.server";
+import { $replica, PrismaClientOrTransaction, prisma } from "~/db.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { attributesFromAuthenticatedEnv, tracer } from "../../v3/tracer.server";

 export abstract class BasePresenter {
-  constructor(protected readonly _prisma: PrismaClientOrTransaction = prisma) {}
+  constructor(
+    protected readonly _prisma: PrismaClientOrTransaction = prisma,
+    protected readonly _replica: PrismaClientOrTransaction = $replica
+  ) {}

  protected async traceWithEnv<T>(
    trace: string,
@@ -232,7 +232,7 @@ export default function Page() {
                  <FormError id={projectSlug.errorId}>{projectSlug.error}</FormError>
                  <FormError>{deleteForm.error}</FormError>
                  <Hint>
-                    This change is irreversible, so please be certain. Type in the Project slug
+                    This change is irreversible, so please be certain. Type in the Project slug{" "}
                    <InlineCode variant="extra-small">{project.slug}</InlineCode> and then press
                    Delete.
                  </Hint>
@@ -1,5 +1,4 @@
 import { ChatBubbleLeftRightIcon, Squares2X2Icon } from "@heroicons/react/20/solid";
-import invariant from "tiny-invariant";
 import { AstroLogo } from "~/assets/logos/AstroLogo";
 import { Feedback } from "~/components/Feedback";
 import { InitCommand, RunDevCommand, TriggerDevStep } from "~/components/SetupCommands";
@@ -11,18 +10,17 @@ import { Header1 } from "~/components/primitives/Headers";
 import { Paragraph } from "~/components/primitives/Paragraph";
 import { StepNumber } from "~/components/primitives/StepNumber";
 import { useAppOrigin } from "~/hooks/useAppOrigin";
-import { useDevEnvironment } from "~/hooks/useEnvironments";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
 import { useProjectSetupComplete } from "~/hooks/useProjectSetupComplete";
 import { projectSetupPath } from "~/utils/pathBuilder";
+import { useV2OnboardingApiKey } from "../_app.orgs.$organizationSlug.projects.$projectParam.setup/route";

 export default function SetUpAstro() {
  const organization = useOrganization();
  const project = useProject();
  useProjectSetupComplete();
-  const devEnvironment = useDevEnvironment();
-  invariant(devEnvironment, "Dev environment must be defined");
+  const { apiKey } = useV2OnboardingApiKey();
  const appOrigin = useAppOrigin();

  return (
@@ -67,7 +65,7 @@ export default function SetUpAstro() {
            title="Run the CLI 'init' command in an existing Astro project"
          />
          <StepContentContainer>
-            <InitCommand appOrigin={appOrigin} apiKey={devEnvironment.apiKey} />
+            <InitCommand appOrigin={appOrigin} apiKey={apiKey} />

            <Paragraph spacing variant="small">
              You’ll notice a new folder in your project called 'jobs'. We’ve added a very simple
@@ -1,5 +1,4 @@
 import { ChatBubbleLeftRightIcon, Squares2X2Icon } from "@heroicons/react/20/solid";
-import invariant from "tiny-invariant";
 import { ExpressLogo } from "~/assets/logos/ExpressLogo";
 import { Feedback } from "~/components/Feedback";
 import { RunDevCommand, TriggerDevStep } from "~/components/SetupCommands";
@@ -11,20 +10,17 @@ import { ClipboardField } from "~/components/primitives/ClipboardField";
 import { Header1 } from "~/components/primitives/Headers";
 import { Paragraph } from "~/components/primitives/Paragraph";
 import { StepNumber } from "~/components/primitives/StepNumber";
-import { useAppOrigin } from "~/hooks/useAppOrigin";
-import { useDevEnvironment } from "~/hooks/useEnvironments";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
 import { useProjectSetupComplete } from "~/hooks/useProjectSetupComplete";
 import { projectSetupPath } from "~/utils/pathBuilder";
+import { useV2OnboardingApiKey } from "../_app.orgs.$organizationSlug.projects.$projectParam.setup/route";

 export default function Page() {
  const organization = useOrganization();
  const project = useProject();
  useProjectSetupComplete();
-  const devEnvironment = useDevEnvironment();
-  invariant(devEnvironment, "Dev environment must be defined");
-  const appOrigin = useAppOrigin();
+  const { apiKey } = useV2OnboardingApiKey();

  return (
    <div className="mx-auto max-w-3xl pt-16">
@@ -73,7 +69,7 @@ export default function Page() {
              <ClipboardField
                secure
                className="w-fit"
-                value={devEnvironment.apiKey}
+                value={apiKey}
                variant={"secondary/medium"}
                icon={<Badge variant="outline">Server</Badge>}
              />
@@ -1,5 +1,4 @@
 import { ChatBubbleLeftRightIcon, Squares2X2Icon } from "@heroicons/react/20/solid";
-import invariant from "tiny-invariant";
 import { NestjsLogo } from "~/assets/logos/NestjsLogo";
 import { Feedback } from "~/components/Feedback";
 import { TriggerDevStep } from "~/components/SetupCommands";
@@ -11,12 +10,12 @@ import { Header1 } from "~/components/primitives/Headers";
 import { Paragraph } from "~/components/primitives/Paragraph";
 import { StepNumber } from "~/components/primitives/StepNumber";
 import { useAppOrigin } from "~/hooks/useAppOrigin";
-import { useDevEnvironment } from "~/hooks/useEnvironments";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
 import { useProjectSetupComplete } from "~/hooks/useProjectSetupComplete";
 import { projectSetupPath } from "~/utils/pathBuilder";
 import { CodeBlock } from "../../components/code/CodeBlock";
+import { useV2OnboardingApiKey } from "../_app.orgs.$organizationSlug.projects.$projectParam.setup/route";

 const AppModuleCode = `
 import { Module } from '@nestjs/common';
@@ -114,11 +113,9 @@ export default function SetupNestJS() {
  const organization = useOrganization();
  const project = useProject();
  useProjectSetupComplete();
-  const devEnvironment = useDevEnvironment();
+  const { apiKey } = useV2OnboardingApiKey();
  const appOrigin = useAppOrigin();

-  invariant(devEnvironment, "devEnvironment is required");
-
  return (
    <div className="mx-auto max-w-3xl pt-16">
      <div className="mb-12 grid place-items-center">
@@ -161,7 +158,7 @@ export default function SetupNestJS() {
          <CodeBlock
            fileName=".env"
            showChrome
-            code={`TRIGGER_API_KEY=${devEnvironment.apiKey}\nTRIGGER_API_URL=${appOrigin}`}
+            code={`TRIGGER_API_KEY=${apiKey}\nTRIGGER_API_URL=${appOrigin}`}
          />
        </StepContentContainer>
        <StepNumber stepNumber="3" title="Add the TriggerDevModule" />
@@ -1,6 +1,5 @@
 import { ChatBubbleLeftRightIcon, Squares2X2Icon } from "@heroicons/react/20/solid";
 import { useState } from "react";
-import invariant from "tiny-invariant";
 import { NextjsLogo } from "~/assets/logos/NextjsLogo";
 import { Feedback } from "~/components/Feedback";
 import { InitCommand, RunDevCommand, TriggerDevStep } from "~/components/SetupCommands";
@@ -21,11 +20,11 @@ import { Paragraph } from "~/components/primitives/Paragraph";
 import { RadioGroup, RadioGroupItem } from "~/components/primitives/RadioButton";
 import { StepNumber } from "~/components/primitives/StepNumber";
 import { useAppOrigin } from "~/hooks/useAppOrigin";
-import { useDevEnvironment } from "~/hooks/useEnvironments";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
 import { useProjectSetupComplete } from "~/hooks/useProjectSetupComplete";
 import { projectSetupPath } from "~/utils/pathBuilder";
+import { useV2OnboardingApiKey } from "../_app.orgs.$organizationSlug.projects.$projectParam.setup/route";

 type SelectionChoices = "use-existing-project" | "create-new-next-app";

@@ -33,12 +32,10 @@ export default function SetupNextjs() {
  const organization = useOrganization();
  const project = useProject();
  useProjectSetupComplete();
-  const devEnvironment = useDevEnvironment();
+  const { apiKey } = useV2OnboardingApiKey();
  const appOrigin = useAppOrigin();
  const [selectedValue, setSelectedValue] = useState<SelectionChoices | null>(null);

-  invariant(devEnvironment, "devEnvironment is required");
-
  return (
    <div className="mx-auto max-w-3xl pt-16">
      <div className="mb-12 grid place-items-center">
@@ -152,7 +149,7 @@ export default function SetupNextjs() {
                title="Run the CLI 'init' command in your new Next.js project"
              />
              <StepContentContainer>
-                <InitCommand appOrigin={appOrigin} apiKey={devEnvironment.apiKey} />
+                <InitCommand appOrigin={appOrigin} apiKey={apiKey} />
                <Paragraph spacing variant="small">
                  You’ll notice a new folder in your project called 'jobs'. We’ve added a very
                  simple example Job in <InlineCode variant="extra-small">examples.ts</InlineCode>{" "}
@@ -179,7 +176,7 @@ export default function SetupNextjs() {
                title="Run the CLI 'init' command in an existing Next.js project"
              />
              <StepContentContainer>
-                <InitCommand appOrigin={appOrigin} apiKey={devEnvironment.apiKey} />
+                <InitCommand appOrigin={appOrigin} apiKey={apiKey} />

                <Paragraph spacing variant="small">
                  You’ll notice a new folder in your project called 'jobs'. We’ve added a very
@@ -1,5 +1,4 @@
 import { ChatBubbleLeftRightIcon, Squares2X2Icon } from "@heroicons/react/20/solid";
-import invariant from "tiny-invariant";
 import { RemixLogo } from "~/assets/logos/RemixLogo";
 import { Feedback } from "~/components/Feedback";
 import { InitCommand, RunDevCommand, TriggerDevStep } from "~/components/SetupCommands";
@@ -11,18 +10,17 @@ import { Header1 } from "~/components/primitives/Headers";
 import { Paragraph } from "~/components/primitives/Paragraph";
 import { StepNumber } from "~/components/primitives/StepNumber";
 import { useAppOrigin } from "~/hooks/useAppOrigin";
-import { useDevEnvironment } from "~/hooks/useEnvironments";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
 import { useProjectSetupComplete } from "~/hooks/useProjectSetupComplete";
 import { projectSetupPath } from "~/utils/pathBuilder";
+import { useV2OnboardingApiKey } from "../_app.orgs.$organizationSlug.projects.$projectParam.setup/route";

 export default function SetUpRemix() {
  const organization = useOrganization();
  const project = useProject();
  useProjectSetupComplete();
-  const devEnvironment = useDevEnvironment();
-  invariant(devEnvironment, "Dev environment must be defined");
+  const { apiKey } = useV2OnboardingApiKey();
  const appOrigin = useAppOrigin();

  return (
@@ -67,7 +65,7 @@ export default function SetUpRemix() {
            title="Run the CLI 'init' command in an existing Remix project"
          />
          <StepContentContainer>
-            <InitCommand appOrigin={appOrigin} apiKey={devEnvironment.apiKey} />
+            <InitCommand appOrigin={appOrigin} apiKey={apiKey} />

            <Paragraph spacing variant="small">
              You’ll notice a new folder in your project called 'jobs'. We’ve added a very simple
@@ -1,5 +1,4 @@
 import { ChatBubbleLeftRightIcon, Squares2X2Icon } from "@heroicons/react/20/solid";
-import invariant from "tiny-invariant";
 import { SvelteKitLogo } from "~/assets/logos/SveltekitLogo";
 import { Feedback } from "~/components/Feedback";
 import { RunDevCommand, TriggerDevStep } from "~/components/SetupCommands";
@@ -11,18 +10,18 @@ import { ClipboardField } from "~/components/primitives/ClipboardField";
 import { Header1 } from "~/components/primitives/Headers";
 import { Paragraph } from "~/components/primitives/Paragraph";
 import { StepNumber } from "~/components/primitives/StepNumber";
-import { useDevEnvironment } from "~/hooks/useEnvironments";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
 import { useProjectSetupComplete } from "~/hooks/useProjectSetupComplete";
 import { projectSetupPath } from "~/utils/pathBuilder";
+import { useV2OnboardingApiKey } from "../_app.orgs.$organizationSlug.projects.$projectParam.setup/route";

 export default function SetUpSveltekit() {
  const organization = useOrganization();
  const project = useProject();
  useProjectSetupComplete();
-  const devEnvironment = useDevEnvironment();
-  invariant(devEnvironment, "Dev environment must be defined");
+  const { apiKey } = useV2OnboardingApiKey();
+
  return (
    <div className="mx-auto max-w-3xl pt-16">
      <div className="mb-12 grid place-items-center">
@@ -70,7 +69,7 @@ export default function SetUpSveltekit() {
              <ClipboardField
                secure
                className="w-fit"
-                value={devEnvironment.apiKey}
+                value={apiKey}
                variant={"secondary/medium"}
                icon={<Badge variant="outline">Server</Badge>}
              />
@@ -1,4 +1,48 @@
 import { Outlet } from "@remix-run/react";
+import { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { typedjson } from "remix-typedjson";
+import { prisma } from "~/db.server";
+import { useTypedMatchData, useTypedMatchesData } from "~/hooks/useTypedMatchData";
+import { requireUserId } from "~/services/session.server";
+import { ProjectParamSchema } from "~/utils/pathBuilder";
+
+export const loader = async ({ request, params }: LoaderFunctionArgs) => {
+  const userId = await requireUserId(request);
+  const { organizationSlug, projectParam } = ProjectParamSchema.parse(params);
+
+  const environment = await prisma.runtimeEnvironment.findFirst({
+    where: {
+      organization: {
+        slug: organizationSlug,
+      },
+      project: {
+        slug: projectParam,
+      },
+      orgMember: {
+        userId,
+      },
+    },
+  });
+
+  if (!environment) {
+    throw new Response("Not Found", { status: 404 });
+  }
+
+  return typedjson({
+    apiKey: environment.apiKey,
+  });
+};
+
+export function useV2OnboardingApiKey() {
+  const routeMatch = useTypedMatchesData<typeof loader>({
+    id: "routes/_app.orgs.$organizationSlug.projects.$projectParam.setup",
+  });
+  if (!routeMatch) {
+    throw new Error("Route match not found");
+  }
+
+  return routeMatch;
+}

 export default function Page() {
  return (
@@ -1,21 +1,27 @@
-import { ChatBubbleLeftRightIcon } from "@heroicons/react/20/solid";
+import { ChatBubbleLeftRightIcon, ChevronDownIcon, ChevronUpIcon } from "@heroicons/react/20/solid";
 import { useRevalidator } from "@remix-run/react";
 import { LoaderFunctionArgs } from "@remix-run/server-runtime";
-import { TaskRunAttemptStatus, TaskRunStatus } from "@trigger.dev/database";
-import { useEffect } from "react";
-import { typedjson, useTypedLoaderData } from "remix-typedjson";
-import invariant from "tiny-invariant";
+import { formatDurationMilliseconds } from "@trigger.dev/core/v3";
+import { TaskRunStatus } from "@trigger.dev/database";
+import { Fragment, Suspense, useEffect, useState } from "react";
+import { Bar, BarChart, ResponsiveContainer, Tooltip, TooltipProps } from "recharts";
+import { TypedAwait, typeddefer, useTypedLoaderData } from "remix-typedjson";
 import { Feedback } from "~/components/Feedback";
-import { InitCommandV3, TriggerDevStepV3 } from "~/components/SetupCommands";
+import { InitCommandV3, TriggerDevStepV3, TriggerLoginStepV3 } from "~/components/SetupCommands";
 import { StepContentContainer } from "~/components/StepContentContainer";
+import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
 import { InlineCode } from "~/components/code/InlineCode";
 import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
 import { MainCenteredContainer, PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { Button } from "~/components/primitives/Buttons";
-import { DateTime } from "~/components/primitives/DateTime";
-import { Header1 } from "~/components/primitives/Headers";
-import { NavBar, PageTitle } from "~/components/primitives/PageHeader";
+import { Callout } from "~/components/primitives/Callout";
+import { formatDateTime } from "~/components/primitives/DateTime";
+import { Header1, Header2, Header3 } from "~/components/primitives/Headers";
+import { Input } from "~/components/primitives/Input";
+import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
+import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
+import { Spinner } from "~/components/primitives/Spinner";
 import { StepNumber } from "~/components/primitives/StepNumber";
 import {
  Table,
@@ -28,18 +34,18 @@ import {
  TableRow,
 } from "~/components/primitives/Table";
 import { SimpleTooltip } from "~/components/primitives/Tooltip";
-import { TaskFunctionName, TaskPath } from "~/components/runs/v3/TaskPath";
+import TooltipPortal from "~/components/primitives/TooltipPortal";
+import { TaskFunctionName } from "~/components/runs/v3/TaskPath";
 import { TaskRunStatusCombo } from "~/components/runs/v3/TaskRunStatus";
 import {
  TaskTriggerSourceIcon,
  taskTriggerSourceDescription,
 } from "~/components/runs/v3/TaskTriggerSource";
-import { useDevEnvironment } from "~/hooks/useEnvironments";
 import { useEventSource } from "~/hooks/useEventSource";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
-import { useUser } from "~/hooks/useUser";
-import { TaskListPresenter } from "~/presenters/v3/TaskListPresenter.server";
+import { useTextFilter } from "~/hooks/useTextFilter";
+import { Task, TaskActivity, TaskListPresenter } from "~/presenters/v3/TaskListPresenter.server";
 import { requireUserId } from "~/services/session.server";
 import { cn } from "~/utils/cn";
 import { ProjectParamSchema, v3RunsPath, v3TasksStreamingPath } from "~/utils/pathBuilder";
@@ -50,14 +56,18 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {

  try {
    const presenter = new TaskListPresenter();
-    const tasks = await presenter.call({
+    const { tasks, userHasTasks, activity, runningStats, durations } = await presenter.call({
      userId,
      organizationSlug,
      projectSlug: projectParam,
    });

-    return typedjson({
+    return typeddefer({
      tasks,
+      userHasTasks,
+      activity,
+      runningStats,
+      durations,
    });
  } catch (error) {
    console.error(error);
@@ -71,8 +81,33 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
 export default function Page() {
  const organization = useOrganization();
  const project = useProject();
-  const user = useUser();
-  const { tasks } = useTypedLoaderData<typeof loader>();
+  const { tasks, userHasTasks, activity, runningStats, durations } =
+    useTypedLoaderData<typeof loader>();
+  const { filterText, setFilterText, filteredItems } = useTextFilter<Task>({
+    items: tasks,
+    filter: (task, text) => {
+      if (task.slug.toLowerCase().includes(text.toLowerCase())) {
+        return true;
+      }
+
+      if (
+        task.exportName.toLowerCase().includes(text.toLowerCase().replace("(", "").replace(")", ""))
+      ) {
+        return true;
+      }
+
+      if (task.filePath.toLowerCase().includes(text.toLowerCase())) {
+        return true;
+      }
+
+      if (task.triggerSource === "SCHEDULED" && "scheduled".includes(text.toLowerCase())) {
+        return true;
+      }
+
+      return false;
+    },
+  });
+
  const hasTasks = tasks.length > 0;

  //live reload the page when the tasks change
@@ -92,40 +127,64 @@ export default function Page() {
    <PageContainer>
      <NavBar>
        <PageTitle title="Tasks" />
+        <PageAccessories>
+          <AdminDebugTooltip>
+            <PropertyTable>
+              {tasks.map((task) => (
+                <Property label={task.exportName} key={task.slug}>
+                  <div className="flex items-center gap-2">
+                    <Paragraph variant="extra-small/bright/mono">
+                      {task.environments
+                        .map((e) =>
+                          e.userName ? `${e.userName}/${e.id}` : `${e.type.slice(0, 3)}/${e.id}`
+                        )
+                        .join(", ")}
+                    </Paragraph>
+                  </div>
+                </Property>
+              ))}
+            </PropertyTable>
+          </AdminDebugTooltip>
+        </PageAccessories>
      </NavBar>
      <PageBody>
-        <div className={cn("grid h-full grid-cols-1 gap-4")}>
-          <div className="h-full">
-            {hasTasks ? (
-              <div className="flex flex-col gap-4">
+        <div className={cn("grid h-full grid-rows-1")}>
+          {hasTasks ? (
+            <div className="flex flex-col gap-4 pb-4">
+              {!userHasTasks && <UserHasNoTasks />}
+              <div className="pb-4">
+                <div className="h-8">
+                  <Input
+                    placeholder="Search tasks"
+                    variant="tertiary"
+                    icon="search"
+                    fullWidth={true}
+                    value={filterText}
+                    onChange={(e) => setFilterText(e.target.value)}
+                    autoFocus
+                  />
+                </div>
                <Table>
                  <TableHeader>
                    <TableRow>
                      <TableHeaderCell>Task ID</TableHeaderCell>
                      <TableHeaderCell>Task</TableHeaderCell>
-                      <TableHeaderCell>Path</TableHeaderCell>
-                      <TableHeaderCell>Environment</TableHeaderCell>
-                      <TableHeaderCell>Last run</TableHeaderCell>
-                      <TableHeaderCell>
-                        <div className="sr-only">Last run status</div>
-                      </TableHeaderCell>
-                      <TableHeaderCell>Created at</TableHeaderCell>
+                      <TableHeaderCell>Running</TableHeaderCell>
+                      <TableHeaderCell>Queued</TableHeaderCell>
+                      <TableHeaderCell>Activity (7d)</TableHeaderCell>
+                      <TableHeaderCell>Avg. duration</TableHeaderCell>
+                      <TableHeaderCell>Environments</TableHeaderCell>
                      <TableHeaderCell hiddenLabel>Go to page</TableHeaderCell>
                    </TableRow>
                  </TableHeader>
                  <TableBody>
-                    {tasks.length > 0 ? (
-                      tasks.map((task) => {
-                        const usernameForEnv =
-                          user.id !== task.environment.userId
-                            ? task.environment.userName
-                            : undefined;
+                    {filteredItems.length > 0 ? (
+                      filteredItems.map((task) => {
                        const path = v3RunsPath(organization, project, {
                          tasks: [task.slug],
-                          environments: [task.environment.id],
                        });
                        return (
-                          <TableRow key={task.id} className="group">
+                          <TableRow key={task.slug} className="group">
                            <TableCell to={path}>
                              <div className="flex items-center gap-2">
                                <SimpleTooltip
@@ -135,50 +194,89 @@ export default function Page() {
                                <span>{task.slug}</span>
                              </div>
                            </TableCell>
-                            <TableCell to={path}>
+                            <TableCell to={path} className="py-0" actionClassName="py-0">
                              <TaskFunctionName
                                functionName={task.exportName}
-                                variant="extra-small"
+                                variant="extra-extra-small"
                              />
                            </TableCell>
-                            <TableCell to={path}>{task.filePath}</TableCell>
-                            <TableCell to={path}>
-                              <EnvironmentLabel
-                                environment={task.environment}
-                                userName={usernameForEnv}
-                              />
+                            <TableCell to={path} className="p-0">
+                              <Suspense
+                                fallback={
+                                  <>
+                                    <Spinner color="muted" />
+                                  </>
+                                }
+                              >
+                                <TypedAwait resolve={runningStats}>
+                                  {(data) => {
+                                    const taskData = data[task.slug];
+                                    return taskData?.running ?? "0";
+                                  }}
+                                </TypedAwait>
+                              </Suspense>
                            </TableCell>
-
-                            <TableCell to={path}>
-                              {task.latestRun ? (
-                                <div
-                                  className={cn(
-                                    "flex items-center gap-2",
-                                    classForTaskRunStatus(task.latestRun.status)
-                                  )}
-                                >
-                                  <DateTime date={task.latestRun.createdAt} />
-                                </div>
-                              ) : (
-                                "Never run"
-                              )}
+                            <TableCell to={path} className="p-0">
+                              <Suspense fallback={<></>}>
+                                <TypedAwait resolve={runningStats}>
+                                  {(data) => {
+                                    const taskData = data[task.slug];
+                                    return taskData?.queued ?? "0";
+                                  }}
+                                </TypedAwait>
+                              </Suspense>
+                            </TableCell>
+                            <TableCell to={path} className="p-0" actionClassName="py-0">
+                              <Suspense fallback={<TaskActivityBlankState />}>
+                                <TypedAwait resolve={activity}>
+                                  {(data) => {
+                                    const taskData = data[task.slug];
+                                    return (
+                                      <>
+                                        {taskData !== undefined ? (
+                                          <div className="h-6 w-[5.125rem] rounded-sm">
+                                            <TaskActivityGraph activity={taskData} />
+                                          </div>
+                                        ) : (
+                                          <TaskActivityBlankState />
+                                        )}
+                                      </>
+                                    );
+                                  }}
+                                </TypedAwait>
+                              </Suspense>
+                            </TableCell>
+                            <TableCell to={path} className="p-0">
+                              <Suspense fallback={<></>}>
+                                <TypedAwait resolve={durations}>
+                                  {(data) => {
+                                    const taskData = data[task.slug];
+                                    return taskData
+                                      ? formatDurationMilliseconds(taskData * 1000, {
+                                          style: "short",
+                                        })
+                                      : "–";
+                                  }}
+                                </TypedAwait>
+                              </Suspense>
                            </TableCell>
                            <TableCell to={path}>
-                              {task.latestRun ? (
-                                <TaskRunStatusCombo status={task.latestRun.status} />
-                              ) : (
-                                "–"
-                              )}
-                            </TableCell>
-                            <TableCell to={path}>
-                              <DateTime date={task.createdAt} />
+                              <div className="space-x-2">
+                                {task.environments.map((environment) => (
+                                  <EnvironmentLabel
+                                    key={environment.id}
+                                    environment={environment}
+                                    userName={environment.userName}
+                                  />
+                                ))}
+                              </div>
                            </TableCell>
                            <TableCellChevron to={path} />
                          </TableRow>
                        );
                      })
                    ) : (
-                      <TableBlankRow colSpan={6}>
+                      <TableBlankRow colSpan={8}>
                        <Paragraph variant="small" className="flex items-center justify-center">
                          No tasks match your filters
                        </Paragraph>
@@ -187,29 +285,21 @@ export default function Page() {
                  </TableBody>
                </Table>
              </div>
-            ) : (
+            </div>
+          ) : (
+            <MainCenteredContainer className="max-w-prose">
              <CreateTaskInstructions />
-            )}
-          </div>
+            </MainCenteredContainer>
+          )}
        </div>
      </PageBody>
    </PageContainer>
  );
 }

-function classForTaskRunStatus(status: TaskRunStatus) {
-  switch (status) {
-    case "SYSTEM_FAILURE":
-    case "COMPLETED_WITH_ERRORS":
-      return "text-error";
-    default:
-      return "";
-  }
-}
-
 function CreateTaskInstructions() {
  return (
-    <MainCenteredContainer className="max-w-prose">
+    <div>
      <div className="mb-6 flex items-center justify-between border-b">
        <Header1 spacing>Get setup in 3 minutes</Header1>
        <div className="flex items-center gap-2">
@@ -240,6 +330,155 @@ function CreateTaskInstructions() {
      <StepContentContainer>
        <Paragraph>This page will automatically refresh.</Paragraph>
      </StepContentContainer>
-    </MainCenteredContainer>
+    </div>
  );
 }
+
+function UserHasNoTasks() {
+  const [open, setOpen] = useState(false);
+
+  return (
+    <Callout
+      variant="info"
+      cta={
+        <Button
+          variant="tertiary/small"
+          TrailingIcon={open ? ChevronUpIcon : ChevronDownIcon}
+          onClick={() => setOpen((o) => !o)}
+        >
+          {open ? "Close" : "Setup your dev environment"}
+        </Button>
+      }
+    >
+      {open ? (
+        <div>
+          <Header2 spacing>Get setup in 3 minutes</Header2>
+
+          <StepNumber stepNumber="1" title="Open up your project" className="mt-6" />
+          <StepContentContainer>
+            <Paragraph>You'll need to open a terminal at the root of your project.</Paragraph>
+          </StepContentContainer>
+          <StepNumber stepNumber="2" title="Run the CLI 'login' command" />
+          <StepContentContainer>
+            <TriggerLoginStepV3 />
+          </StepContentContainer>
+          <StepNumber stepNumber="3" title="Run the CLI 'dev' command" />
+          <StepContentContainer>
+            <TriggerDevStepV3 />
+          </StepContentContainer>
+          <StepNumber stepNumber="4" title="Waiting for tasks" displaySpinner />
+          <StepContentContainer>
+            <Paragraph>This page will automatically refresh.</Paragraph>
+          </StepContentContainer>
+        </div>
+      ) : (
+        "Your DEV environment isn't setup yet."
+      )}
+    </Callout>
+  );
+}
+
+function TaskActivityGraph({ activity }: { activity: TaskActivity }) {
+  return (
+    <ResponsiveContainer width="100%" height="100%">
+      <BarChart
+        data={activity}
+        margin={{
+          top: 0,
+          right: 0,
+          left: 0,
+          bottom: 0,
+        }}
+        width={82}
+        height={24}
+      >
+        <Tooltip
+          cursor={{ fill: "transparent" }}
+          content={<CustomTooltip />}
+          allowEscapeViewBox={{ x: true, y: true }}
+          wrapperStyle={{ zIndex: 1000 }}
+          animationDuration={0}
+        />
+
+        {/* The background */}
+        <Bar
+          dataKey="bg"
+          background={{ fill: "#212327" }}
+          strokeWidth={0}
+          stackId="a"
+          barSize={10}
+          isAnimationActive={false}
+        />
+        <Bar dataKey="PENDING" fill="#5F6570" stackId="a" strokeWidth={0} barSize={10} />
+        <Bar dataKey="WAITING_FOR_DEPLOY" fill="#F59E0B" stackId="a" strokeWidth={0} barSize={10} />
+        <Bar dataKey="EXECUTING" fill="#3B82F6" stackId="a" strokeWidth={0} barSize={10} />
+        <Bar
+          dataKey="RETRYING_AFTER_FAILURE"
+          fill="#3B82F6"
+          stackId="a"
+          strokeWidth={0}
+          barSize={10}
+        />
+        <Bar dataKey="WAITING_TO_RESUME" fill="#3B82F6" stackId="a" strokeWidth={0} barSize={10} />
+        <Bar
+          dataKey="COMPLETED_SUCCESSFULLY"
+          fill="#28BF5C"
+          stackId="a"
+          strokeWidth={0}
+          barSize={10}
+        />
+        <Bar dataKey="CANCELED" fill="#5F6570" stackId="a" strokeWidth={0} barSize={10} />
+        <Bar
+          dataKey="COMPLETED_WITH_ERRORS"
+          fill="#F43F5E"
+          stackId="a"
+          strokeWidth={0}
+          barSize={10}
+        />
+        <Bar dataKey="INTERRUPTED" fill="#F43F5E" stackId="a" strokeWidth={0} barSize={10} />
+        <Bar dataKey="SYSTEM_FAILURE" fill="#F43F5E" stackId="a" strokeWidth={0} barSize={10} />
+        <Bar dataKey="PAUSED" fill="#FCD34D" stackId="a" strokeWidth={0} barSize={10} />
+        <Bar dataKey="CRASHED" fill="#F43F5E" stackId="a" strokeWidth={0} barSize={10} />
+      </BarChart>
+    </ResponsiveContainer>
+  );
+}
+
+function TaskActivityBlankState() {
+  return (
+    <div className="flex h-6 w-[5.125rem] items-center gap-0.5 rounded-sm">
+      {[...Array(7)].map((_, i) => (
+        <div key={i} className="h-full w-2.5 bg-[#212327]" />
+      ))}
+    </div>
+  );
+}
+
+const CustomTooltip = ({ active, payload, label }: TooltipProps<number, string>) => {
+  if (active && payload) {
+    const items = payload.map((p) => ({
+      status: p.dataKey as TaskRunStatus,
+      value: p.value,
+    }));
+    const title = payload[0].payload.day as string;
+    const formattedDate = formatDateTime(new Date(title), "UTC", [], false, false);
+
+    return (
+      <TooltipPortal active={active}>
+        <div className="rounded-sm border border-grid-bright bg-background-dimmed px-3 py-2">
+          <Header3 className="border-b-charcoal-650 border-b pb-2">{formattedDate}</Header3>
+          <div className="mt-2 grid grid-cols-[1fr_auto] gap-2 text-xs text-text-bright">
+            {items.map((item) => (
+              <Fragment key={item.status}>
+                <TaskRunStatusCombo status={item.status} />
+                <p>{item.value}</p>
+              </Fragment>
+            ))}
+          </div>
+        </div>
+      </TooltipPortal>
+    );
+  }
+
+  return null;
+};
@@ -1,15 +1,19 @@
-import { BookOpenIcon } from "@heroicons/react/20/solid";
-import { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { BookOpenIcon, LightBulbIcon, ShieldCheckIcon } from "@heroicons/react/20/solid";
+import { Form } from "@remix-run/react";
+import { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
+import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
 import { EnvironmentLabel, environmentTitle } from "~/components/environments/EnvironmentLabel";
 import { RegenerateApiKeyModal } from "~/components/environments/RegenerateApiKeyModal";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
-import { LinkButton } from "~/components/primitives/Buttons";
+import { Button, LinkButton } from "~/components/primitives/Buttons";
+import { Callout } from "~/components/primitives/Callout";
 import { ClipboardField } from "~/components/primitives/ClipboardField";
 import { DateTime } from "~/components/primitives/DateTime";
 import { Header3 } from "~/components/primitives/Headers";
-import { PageAccessories, NavBar, PageTitle } from "~/components/primitives/PageHeader";
+import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
+import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
 import {
  Table,
  TableBody,
@@ -20,11 +24,14 @@ import {
  TableRow,
 } from "~/components/primitives/Table";
 import { TextLink } from "~/components/primitives/TextLink";
-import { useProject } from "~/hooks/useProject";
+import { prisma } from "~/db.server";
+import { useFeatures } from "~/hooks/useFeatures";
+import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
+import { createEnvironment } from "~/models/organization.server";
 import { ApiKeysPresenter } from "~/presenters/v3/ApiKeysPresenter.server";
 import { requireUserId } from "~/services/session.server";
 import { cn } from "~/utils/cn";
-import { ProjectParamSchema, docsPath } from "~/utils/pathBuilder";
+import { ProjectParamSchema, docsPath, v3ApiKeysPath } from "~/utils/pathBuilder";

 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
  const userId = await requireUserId(request);
@@ -32,13 +39,14 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {

  try {
    const presenter = new ApiKeysPresenter();
-    const { environments } = await presenter.call({
+    const { environments, hasStaging } = await presenter.call({
      userId,
      projectSlug: projectParam,
    });

    return typedjson({
      environments,
+      hasStaging,
    });
  } catch (error) {
    console.error(error);
@@ -49,15 +57,94 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
  }
 };

+export const action = async ({ request, params }: ActionFunctionArgs) => {
+  const userId = await requireUserId(request);
+  const { organizationSlug, projectParam } = ProjectParamSchema.parse(params);
+
+  if (request.method.toUpperCase() !== "POST") {
+    return { status: 405, body: "Method Not Allowed" };
+  }
+
+  const project = await prisma.project.findUnique({
+    where: {
+      slug: params.projectParam,
+      organization: {
+        members: {
+          some: {
+            userId,
+          },
+        },
+      },
+    },
+    select: {
+      id: true,
+      organizationId: true,
+      environments: {
+        select: {
+          type: true,
+        },
+      },
+    },
+  });
+
+  if (!project) {
+    return redirectWithErrorMessage(
+      v3ApiKeysPath({ slug: organizationSlug }, { slug: projectParam }),
+      request,
+      "Project not found"
+    );
+  }
+
+  if (project.environments.some((env) => env.type === "STAGING")) {
+    return redirectWithErrorMessage(
+      v3ApiKeysPath({ slug: organizationSlug }, { slug: projectParam }),
+      request,
+      "You already have a staging environment"
+    );
+  }
+
+  const environment = await createEnvironment(
+    { id: project.organizationId },
+    { id: project.id },
+    "STAGING"
+  );
+
+  if (!environment) {
+    return redirectWithErrorMessage(
+      v3ApiKeysPath({ slug: organizationSlug }, { slug: projectParam }),
+      request,
+      "Failed to create staging environment"
+    );
+  }
+
+  return redirectWithSuccessMessage(
+    v3ApiKeysPath({ slug: organizationSlug }, { slug: projectParam }),
+    request,
+    "Staging environment created"
+  );
+};
+
 export default function Page() {
-  const { environments } = useTypedLoaderData<typeof loader>();
-  const project = useProject();
+  const { environments, hasStaging } = useTypedLoaderData<typeof loader>();
+  const { isManagedCloud } = useFeatures();

  return (
    <PageContainer>
      <NavBar>
        <PageTitle title="API keys" />
        <PageAccessories>
+          <AdminDebugTooltip>
+            <PropertyTable>
+              {environments.map((environment) => (
+                <Property label={environment.slug} key={environment.id}>
+                  <div className="flex items-center gap-2">
+                    <Paragraph variant="extra-small/bright/mono">{environment.id}</Paragraph>
+                  </div>
+                </Property>
+              ))}
+            </PropertyTable>
+          </AdminDebugTooltip>
+
          <LinkButton
            variant={"minimal/small"}
            LeadingIcon={BookOpenIcon}
@@ -127,6 +214,21 @@ export default function Page() {
                ))}
              </TableBody>
            </Table>
+
+            {!hasStaging && (
+              <Callout
+                variant="info"
+                cta={
+                  <Form method="post">
+                    <Button variant="tertiary/small">Enable Staging</Button>
+                  </Form>
+                }
+              >
+                {isManagedCloud
+                  ? "The Staging environment will be a paid feature when we add billing. In the interim you can enable it for free."
+                  : "You can add a Staging environment to your project."}
+              </Callout>
+            )}
          </div>
        </div>
      </PageBody>
@@ -92,6 +92,9 @@ export default function Page() {
              <DeploymentStatus status={deployment.status} className="text-sm" />
            </Property>
            <Property label="Tasks">{deployment.tasks ? deployment.tasks.length : "–"}</Property>
+            <Property label="SDK Version">
+              {deployment.sdkVersion ? deployment.sdkVersion : "–"}
+            </Property>
            <Property label="Started at">
              <Paragraph variant="small/bright">
                <DateTimeAccurate date={deployment.createdAt} /> UTC
@@ -1,35 +1,44 @@
-import { Submission, conform, useForm } from "@conform-to/react";
+import {
+  FieldConfig,
+  list,
+  requestIntent,
+  useFieldList,
+  useFieldset,
+  useForm,
+} from "@conform-to/react";
 import { parse } from "@conform-to/zod";
-import { Form, useActionData, useLocation, useNavigate, useNavigation } from "@remix-run/react";
+import { PlusIcon, XMarkIcon } from "@heroicons/react/20/solid";
+import { Form, useActionData, useNavigate, useNavigation } from "@remix-run/react";
 import { ActionFunctionArgs, LoaderFunctionArgs, json } from "@remix-run/server-runtime";
-import { Fragment, useEffect, useRef, useState } from "react";
+import { RefObject, useCallback, useEffect, useRef, useState } from "react";
 import { redirect, typedjson, useTypedLoaderData } from "remix-typedjson";
 import { z } from "zod";
-import { InlineCode } from "~/components/code/InlineCode";
-import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
+import {
+  environmentTextClassName,
+  environmentTitle,
+} from "~/components/environments/EnvironmentLabel";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
-import { Callout } from "~/components/primitives/Callout";
+import { Checkbox } from "~/components/primitives/Checkbox";
 import { Dialog, DialogContent, DialogHeader } from "~/components/primitives/Dialog";
 import { Fieldset } from "~/components/primitives/Fieldset";
 import { FormButtons } from "~/components/primitives/FormButtons";
 import { FormError } from "~/components/primitives/FormError";
+import { Hint } from "~/components/primitives/Hint";
 import { Input } from "~/components/primitives/Input";
 import { InputGroup } from "~/components/primitives/InputGroup";
 import { Label } from "~/components/primitives/Label";
 import { Switch } from "~/components/primitives/Switch";
 import { prisma } from "~/db.server";
+import { useList } from "~/hooks/useList";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
-import { redirectWithSuccessMessage } from "~/models/message.server";
 import { EnvironmentVariablesPresenter } from "~/presenters/v3/EnvironmentVariablesPresenter.server";
 import { requireUserId } from "~/services/session.server";
-import {
-  ProjectParamSchema,
-  v3EnvironmentVariablesPath,
-  v3NewEnvironmentVariablesPath,
-} from "~/utils/pathBuilder";
+import { cn } from "~/utils/cn";
+import { ProjectParamSchema, v3EnvironmentVariablesPath } from "~/utils/pathBuilder";
 import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server";
-import { CreateEnvironmentVariable } from "~/v3/environmentVariables/repository";
+import { EnvironmentVariableKey } from "~/v3/environmentVariables/repository";
+import dotenv from "dotenv";

 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
  const userId = await requireUserId(request);
@@ -47,7 +56,6 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
      environments,
    });
  } catch (error) {
-    console.error(error);
    throw new Response(undefined, {
      status: 400,
      statusText: "Something went wrong, if this problem persists please contact support.",
@@ -55,9 +63,39 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
  }
 };

+const Variable = z.object({
+  key: EnvironmentVariableKey,
+  value: z.string().nonempty("Value is required"),
+});
+
+type Variable = z.infer<typeof Variable>;
+
 const schema = z.object({
-  action: z.enum(["create", "create-more"]),
-  ...CreateEnvironmentVariable.shape,
+  overwrite: z.preprocess((i) => {
+    if (i === "true") return true;
+    if (i === "false") return false;
+    return;
+  }, z.boolean()),
+  environmentIds: z.preprocess((i) => {
+    if (typeof i === "string") return [i];
+
+    if (Array.isArray(i)) {
+      const ids = i.filter((v) => typeof v === "string" && v !== "");
+      if (ids.length === 0) {
+        return;
+      }
+      return ids;
+    }
+
+    return;
+  }, z.array(z.string(), { required_error: "At least one environment is required" })),
+  variables: z.preprocess((i) => {
+    if (!Array.isArray(i)) {
+      return [];
+    }
+
+    return i;
+  }, Variable.array().nonempty("At least one variable is required")),
 });

 export const action = async ({ request, params }: ActionFunctionArgs) => {
@@ -92,22 +130,22 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
  const result = await repository.create(project.id, userId, submission.value);

  if (!result.success) {
-    submission.error.key = result.error;
+    if (result.variableErrors) {
+      for (const { key, error } of result.variableErrors) {
+        const index = submission.value.variables.findIndex((v) => v.key === key);
+
+        if (index !== -1) {
+          submission.error[`variables[${index}].key`] = error;
+        }
+      }
+    } else {
+      submission.error.variables = result.error;
+    }
+
    return json(submission);
  }

-  switch (submission.value.action) {
-    case "create":
-      return redirect(
-        v3EnvironmentVariablesPath({ slug: organizationSlug }, { slug: projectParam })
-      );
-    case "create-more":
-      return redirectWithSuccessMessage(
-        v3NewEnvironmentVariablesPath({ slug: organizationSlug }, { slug: projectParam }),
-        request,
-        `Created ${submission.value.key} environment variable`
-      );
-  }
+  return redirect(v3EnvironmentVariablesPath({ slug: organizationSlug }, { slug: projectParam }));
 };

 export default function Page() {
@@ -118,15 +156,11 @@ export default function Page() {
  const navigate = useNavigate();
  const organization = useOrganization();
  const project = useProject();
-  const keyFieldRef = useRef<HTMLInputElement>(null);

-  const isLoading =
-    navigation.state !== "idle" &&
-    navigation.formMethod === "post" &&
-    navigation.formData?.get("action") === "create";
+  const isLoading = navigation.state !== "idle" && navigation.formMethod === "post";

-  const [form, { key }] = useForm({
-    id: "create-environment-variable",
+  const [form, { environmentIds, variables }] = useForm({
+    id: "create-environment-variables",
    // TODO: type this
    lastSubmission: lastSubmission as any,
    onValidate({ formData }) {
@@ -141,14 +175,6 @@ export default function Page() {
    setIsOpen(true);
  }, []);

-  useEffect(() => {
-    if (navigation.state !== "idle") return;
-    if (lastSubmission !== undefined) return;
-
-    form.ref.current?.reset();
-    keyFieldRef.current?.focus();
-  }, [navigation.state, lastSubmission]);
-
  return (
    <Dialog
      open={isOpen}
@@ -158,61 +184,64 @@ export default function Page() {
        }
      }}
    >
-      <DialogContent>
-        <DialogHeader>New environment variable</DialogHeader>
-        <Form method="post" {...form.props}>
+      <DialogContent className="md:max-w-2xl lg:max-w-3xl">
+        <DialogHeader>New environment variables</DialogHeader>
+        <Form
+          method="post"
+          {...form.props}
+          className="max-h-[70vh] overflow-y-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600"
+        >
          <Fieldset className="mt-2">
            <InputGroup fullWidth>
-              <Label>Key</Label>
-              <Input
-                {...conform.input(key)}
-                placeholder="e.g. CLIENT_KEY"
-                autoFocus
-                ref={keyFieldRef}
-              />
-            </InputGroup>
-            <InputGroup fullWidth>
-              <div className="flex items-center justify-between">
-                <Label>Values</Label>
-                <Switch
-                  variant="small"
-                  label="Reveal values"
-                  checked={revealAll}
-                  onCheckedChange={(e) => setRevealAll(e.valueOf())}
-                />
-              </div>
-              <div className="grid grid-cols-[auto_1fr] gap-x-2 gap-y-2">
-                {environments.map((environment, index) => {
-                  return (
-                    <Fragment key={environment.id}>
-                      <input
-                        type="hidden"
-                        name={`values[${index}].environmentId`}
-                        value={environment.id}
-                      />
-                      <label
-                        className="flex items-center justify-end"
-                        htmlFor={`values[${index}].value`}
+              <Label>Environments</Label>
+              <div className="flex flex-wrap items-center gap-2">
+                {environments.map((environment) => (
+                  <Checkbox
+                    key={environment.id}
+                    id={environment.id}
+                    value={environment.id}
+                    name="environmentIds"
+                    type="radio"
+                    label={
+                      <span
+                        className={cn("text-xs uppercase", environmentTextClassName(environment))}
                      >
-                        <EnvironmentLabel environment={environment} className="h-5 px-2" />
-                      </label>
-                      <Input
-                        type={revealAll ? "text" : "password"}
-                        name={`values[${index}].value`}
-                        placeholder="Not set"
-                      />
-                    </Fragment>
-                  );
-                })}
+                        {environmentTitle(environment)}
+                      </span>
+                    }
+                    variant="button"
+                  />
+                ))}
              </div>
+              <FormError id={environmentIds.errorId}>{environmentIds.error}</FormError>
+              <Hint>
+                Dev environment variables specified here will be overridden by ones in your .env
+                file when running locally.
+              </Hint>
+            </InputGroup>
+            <Hint>Tip: Paste your .env into this form to populate it:</Hint>
+            <InputGroup fullWidth>
+              <FieldLayout>
+                <Label>Keys</Label>
+                <div className="flex justify-between gap-1">
+                  <Label>Values</Label>
+                  <Switch
+                    variant="small"
+                    label="Reveal"
+                    checked={revealAll}
+                    onCheckedChange={(e) => setRevealAll(e.valueOf())}
+                  />
+                </div>
+              </FieldLayout>
+              <VariableFields
+                revealValues={revealAll}
+                formId={form.id}
+                formRef={form.ref}
+                variablesFields={variables}
+              />
+              <FormError id={variables.errorId}>{variables.error}</FormError>
            </InputGroup>

-            <Callout variant="info" className="inline-flex">
-              Dev environment variables specified here will be overriden by ones in your{" "}
-              <InlineCode variant="extra-small">.env</InlineCode> file when running locally.
-            </Callout>
-
-            <FormError id={key.errorId}>{key.error}</FormError>
            <FormError>{form.error}</FormError>
            <FormButtons
              confirmButton={
@@ -221,18 +250,18 @@ export default function Page() {
                    type="submit"
                    variant="primary/small"
                    disabled={isLoading}
-                    name="action"
-                    value="create-more"
+                    name="overwrite"
+                    value="false"
                  >
-                    {isLoading ? "Saving" : "Save and add another"}
+                    {isLoading ? "Saving" : "Save"}
                  </Button>
                  <Button
                    variant="secondary/small"
                    disabled={isLoading}
-                    name="action"
-                    value="create"
+                    name="overwrite"
+                    value="true"
                  >
-                    {isLoading ? "Saving" : "Save"}
+                    {isLoading ? "Overwriting" : "Overwrite"}
                  </Button>
                </div>
              }
@@ -251,3 +280,156 @@ export default function Page() {
    </Dialog>
  );
 }
+
+function FieldLayout({ children }: { children: React.ReactNode }) {
+  return <div className="grid w-full grid-cols-[1fr_1fr_2rem] gap-2">{children}</div>;
+}
+
+function VariableFields({
+  revealValues,
+  formId,
+  variablesFields,
+  formRef,
+}: {
+  revealValues: boolean;
+  formId?: string;
+  variablesFields: FieldConfig<any>;
+  formRef: RefObject<HTMLFormElement>;
+}) {
+  const {
+    items,
+    append,
+    update,
+    delete: remove,
+    insertAfter,
+  } = useList<Variable>([{ key: "", value: "" }]);
+
+  const handlePaste = useCallback((index: number, e: React.ClipboardEvent<HTMLInputElement>) => {
+    const clipboardData = e.clipboardData;
+    if (!clipboardData) return;
+
+    let text = clipboardData.getData("text");
+    if (!text) return;
+
+    const variables = dotenv.parse(text);
+    const keyValuePairs = Object.entries(variables).map(([key, value]) => ({ key, value }));
+
+    //do the default paste
+    if (keyValuePairs.length === 0) return;
+
+    //prevent default pasting
+    e.preventDefault();
+
+    const [firstPair, ...rest] = keyValuePairs;
+    update(index, firstPair);
+
+    for (const pair of rest) {
+      requestIntent(formRef.current ?? undefined, list.append(variablesFields.name));
+    }
+    insertAfter(index, rest);
+  }, []);
+
+  const fields = useFieldList(formRef, variablesFields);
+
+  return (
+    <>
+      {fields.map((field, index) => {
+        const item = items[index];
+
+        return (
+          <VariableField
+            formId={formId}
+            key={index}
+            index={index}
+            value={item}
+            onChange={(value) => update(index, value)}
+            onPaste={(e) => handlePaste(index, e)}
+            onDelete={() => {
+              requestIntent(
+                formRef.current ?? undefined,
+                list.remove(variablesFields.name, { index })
+              );
+              remove(index);
+            }}
+            showDeleteButton={items.length > 1}
+            showValue={revealValues}
+            config={field}
+          />
+        );
+      })}
+      <Button
+        variant="tertiary/medium"
+        type="button"
+        onClick={() => {
+          requestIntent(formRef.current ?? undefined, list.append(variablesFields.name));
+          append([{ key: "", value: "" }]);
+        }}
+        LeadingIcon={PlusIcon}
+      >
+        Add another
+      </Button>
+    </>
+  );
+}
+
+function VariableField({
+  formId,
+  index,
+  value,
+  onChange,
+  onPaste,
+  onDelete,
+  showDeleteButton,
+  showValue,
+  config,
+}: {
+  formId?: string;
+  index: number;
+  value: Variable;
+  onChange: (value: Variable) => void;
+  onPaste: (e: React.ClipboardEvent<HTMLInputElement>) => void;
+  onDelete: () => void;
+  showDeleteButton: boolean;
+  showValue: boolean;
+  config: FieldConfig<Variable>;
+}) {
+  const ref = useRef<HTMLFieldSetElement>(null);
+  const fields = useFieldset(ref, config);
+  const baseFieldName = `variables[${index}]`;
+
+  return (
+    <fieldset ref={ref}>
+      <FieldLayout>
+        <Input
+          id={`${formId}-${baseFieldName}.key`}
+          name={`${baseFieldName}.key`}
+          placeholder="e.g. CLIENT_KEY"
+          value={value.key}
+          onChange={(e) => onChange({ ...value, key: e.currentTarget.value })}
+          autoFocus={index === 0}
+          onPaste={onPaste}
+        />
+        <Input
+          id={`${formId}-${baseFieldName}.value`}
+          name={`${baseFieldName}.value`}
+          type={showValue ? "text" : "password"}
+          placeholder="Not set"
+          value={value.value}
+          onChange={(e) => onChange({ ...value, value: e.currentTarget.value })}
+        />
+        {showDeleteButton && (
+          <Button
+            variant="minimal/medium"
+            type="button"
+            onClick={() => onDelete()}
+            LeadingIcon={XMarkIcon}
+          />
+        )}
+      </FieldLayout>
+      <div className="space-y-2">
+        <FormError id={fields.key.errorId}>{fields.key.error}</FormError>
+        <FormError id={fields.value.errorId}>{fields.value.error}</FormError>
+      </div>
+    </fieldset>
+  );
+}
@@ -187,8 +187,9 @@ export default function Page() {
              to={v3NewEnvironmentVariablesPath(organization, project)}
              variant="primary/small"
              LeadingIcon={PlusIcon}
+              shortcut={{ key: "n" }}
            >
-              New environment variable
+              Add new
            </LinkButton>
          </div>
          <Table>
@@ -247,7 +248,7 @@ export default function Page() {
          </Table>

          <Callout variant="info" className="mb-4">
-            Dev environment variables specified here will be overriden by ones in your .env file
+            Dev environment variables specified here will be overridden by ones in your .env file
            when running locally.
          </Callout>
        </div>
@@ -1,6 +1,4 @@
 import {
-  ArrowsPointingInIcon,
-  ArrowsPointingOutIcon,
  ChevronDownIcon,
  ChevronRightIcon,
  MagnifyingGlassMinusIcon,
@@ -17,7 +15,8 @@ import {
 } from "@trigger.dev/core/v3";
 import { RuntimeEnvironmentType } from "@trigger.dev/database";
 import { motion } from "framer-motion";
-import { useEffect, useRef, useState } from "react";
+import { useCallback, useEffect, useRef, useState } from "react";
+import { useHotkeys } from "react-hotkeys-hook";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { ShowParentIcon, ShowParentIconSelected } from "~/assets/icons/ShowParentIcon";
 import tileBgPath from "~/assets/images/error-banner-tile@2x.png";
@@ -26,11 +25,13 @@ import { InlineCode } from "~/components/code/InlineCode";
 import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
 import { MainCenteredContainer, PageBody } from "~/components/layout/AppLayout";
 import { Badge } from "~/components/primitives/Badge";
-import { Button, LinkButton } from "~/components/primitives/Buttons";
+import { LinkButton } from "~/components/primitives/Buttons";
 import { Callout } from "~/components/primitives/Callout";
+import { Header3 } from "~/components/primitives/Headers";
 import { Input } from "~/components/primitives/Input";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
+import { Popover, PopoverArrowTrigger, PopoverContent } from "~/components/primitives/Popover";
 import {
  ResizableHandle,
  ResizablePanel,
@@ -66,10 +67,8 @@ import {
  v3RunsPath,
 } from "~/utils/pathBuilder";
 import { SpanView } from "../resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route";
-import { number } from "zod";
-import { useHotkeys } from "react-hotkeys-hook";
-import { Popover, PopoverArrowTrigger, PopoverContent } from "~/components/primitives/Popover";
-import { Header3 } from "~/components/primitives/Headers";
+import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
+import { Property, PropertyTable } from "~/components/primitives/PropertyTable";

 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
  const userId = await requireUserId(request);
@@ -119,6 +118,33 @@ export default function Page() {
            title={`Run #${run.number}`}
          />
          <PageAccessories>
+            <AdminDebugTooltip>
+              <PropertyTable>
+                <Property label="ID">
+                  <div className="flex items-center gap-2">
+                    <Paragraph variant="extra-small/bright/mono">{run.id}</Paragraph>
+                  </div>
+                </Property>
+                <Property label="Trace ID">
+                  <div className="flex items-center gap-2">
+                    <Paragraph variant="extra-small/bright/mono">{run.traceId}</Paragraph>
+                  </div>
+                </Property>
+                <Property label="Env ID">
+                  <div className="flex items-center gap-2">
+                    <Paragraph variant="extra-small/bright/mono">{run.environment.id}</Paragraph>
+                  </div>
+                </Property>
+                <Property label="Org ID">
+                  <div className="flex items-center gap-2">
+                    <Paragraph variant="extra-small/bright/mono">
+                      {run.environment.organizationId}
+                    </Paragraph>
+                  </div>
+                </Property>
+              </PropertyTable>
+            </AdminDebugTooltip>
+
            <EnvironmentLabel
              size="large"
              environment={run.environment}
@@ -168,6 +194,33 @@ export default function Page() {
          title={`Run #${run.number}`}
        />
        <PageAccessories>
+          <AdminDebugTooltip>
+            <PropertyTable>
+              <Property label="ID">
+                <div className="flex items-center gap-2">
+                  <Paragraph variant="extra-small/bright/mono">{run.id}</Paragraph>
+                </div>
+              </Property>
+              <Property label="Trace ID">
+                <div className="flex items-center gap-2">
+                  <Paragraph variant="extra-small/bright/mono">{run.traceId}</Paragraph>
+                </div>
+              </Property>
+              <Property label="Env ID">
+                <div className="flex items-center gap-2">
+                  <Paragraph variant="extra-small/bright/mono">{run.environment.id}</Paragraph>
+                </div>
+              </Property>
+              <Property label="Org ID">
+                <div className="flex items-center gap-2">
+                  <Paragraph variant="extra-small/bright/mono">
+                    {run.environment.organizationId}
+                  </Paragraph>
+                </div>
+              </Property>
+            </PropertyTable>
+          </AdminDebugTooltip>
+
          <EnvironmentLabel size="large" environment={run.environment} userName={usernameForEnv} />
        </PageAccessories>
      </NavBar>
@@ -268,29 +321,25 @@ function TasksTreeView({
    onSelectedIdChanged,
    estimatedRowHeight: () => 32,
    parentRef,
-    filter: (node) => {
-      const nodePassesErrorTest = (errorsOnly && node.data.isError) || !errorsOnly;
-      if (!nodePassesErrorTest) return false;
+    filter: {
+      value: { text: filterText, errorsOnly },
+      fn: (value, node) => {
+        const nodePassesErrorTest = (value.errorsOnly && node.data.isError) || !value.errorsOnly;
+        if (!nodePassesErrorTest) return false;

-      if (filterText === "") return true;
-      if (node.data.message.toLowerCase().includes(filterText.toLowerCase())) {
-        return true;
-      }
-      return false;
+        if (value.text === "") return true;
+        if (node.data.message.toLowerCase().includes(value.text.toLowerCase())) {
+          return true;
+        }
+        return false;
+      },
    },
  });

  return (
    <div className="grid h-full grid-rows-[2.5rem_1fr_3.25rem] overflow-hidden">
      <div className="mx-3 flex items-center justify-between gap-2 border-b border-grid-dimmed">
-        <Input
-          placeholder="Search log"
-          variant="tertiary"
-          icon="search"
-          fullWidth={true}
-          value={filterText}
-          onChange={(e) => setFilterText(e.target.value)}
-        />
+        <SearchField onChange={setFilterText} />
        <div className="flex items-center gap-2">
          <Switch
            variant="small"
@@ -1004,3 +1053,27 @@ function NumberShortcuts({ toggleLevel }: { toggleLevel: (depth: number) => void
    </div>
  );
 }
+
+function SearchField({ onChange }: { onChange: (value: string) => void }) {
+  const [value, setValue] = useState("");
+
+  const updateFilterText = useDebounce((text: string) => {
+    onChange(text);
+  }, 250);
+
+  const updateValue = useCallback((value: string) => {
+    setValue(value);
+    updateFilterText(value);
+  }, []);
+
+  return (
+    <Input
+      placeholder="Search log"
+      variant="tertiary"
+      icon="search"
+      fullWidth={true}
+      value={value}
+      onChange={(e) => updateValue(e.target.value)}
+    />
+  );
+}
@@ -1,7 +1,7 @@
 import { BeakerIcon, BookOpenIcon } from "@heroicons/react/24/solid";
 import { useNavigation } from "@remix-run/react";
 import { LoaderFunctionArgs } from "@remix-run/server-runtime";
-import { typedjson, useTypedLoaderData } from "remix-typedjson";
+import { TypedAwait, typeddefer, typedjson, useTypedLoaderData } from "remix-typedjson";
 import { TaskIcon } from "~/assets/icons/TaskIcon";
 import { BlankstateInstructions } from "~/components/BlankstateInstructions";
 import { StepContentContainer } from "~/components/StepContentContainer";
@@ -22,6 +22,8 @@ import { cn } from "~/utils/cn";
 import { ProjectParamSchema, v3ProjectPath, v3TestPath } from "~/utils/pathBuilder";
 import { ListPagination } from "../../components/ListPagination";
 import { TextLink } from "~/components/primitives/TextLink";
+import { Spinner } from "~/components/primitives/Spinner";
+import { Suspense } from "react";

 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
  const userId = await requireUserId(request);
@@ -33,7 +35,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
    TaskRunListSearchFilters.parse(s);

  const presenter = new RunListPresenter();
-  const list = await presenter.call({
+  const list = presenter.call({
+    userId,
    projectSlug: projectParam,
    tasks,
    versions,
@@ -45,13 +48,13 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
    cursor: cursor,
  });

-  return typedjson({
-    list,
+  return typeddefer({
+    data: list,
  });
 };

 export default function Page() {
-  const { list } = useTypedLoaderData<typeof loader>();
+  const { data } = useTypedLoaderData<typeof loader>();
  const navigation = useNavigation();
  const isLoading = navigation.state !== "idle";
  const project = useProject();
@@ -63,37 +66,53 @@ export default function Page() {
        <PageTitle title="Runs" />
      </NavBar>
      <PageBody>
-        {list.runs.length === 0 && !list.hasFilters ? (
-          list.possibleTasks.length === 0 ? (
-            <CreateFirstTaskInstructions />
-          ) : (
-            <RunTaskInstructions />
-          )
-        ) : (
-          <div className={cn("grid h-fit grid-cols-1 gap-4")}>
-            <div>
-              <div className="mb-2 flex items-center justify-between gap-x-2">
-                <RunsFilters
-                  possibleEnvironments={project.environments}
-                  possibleTasks={list.possibleTasks}
-                />
-                <div className="flex items-center justify-end gap-x-2">
-                  <ListPagination list={list} />
-                </div>
+        <Suspense
+          fallback={
+            <div className="flex items-center justify-center py-2">
+              <div className="mx-auto flex items-center gap-2">
+                <Spinner />
+                <Paragraph variant="small">Loading runs</Paragraph>
              </div>
-
-              <TaskRunsTable
-                total={list.runs.length}
-                hasFilters={list.hasFilters}
-                filters={list.filters}
-                runs={list.runs}
-                isLoading={isLoading}
-                currentUser={user}
-              />
-              <ListPagination list={list} className="mt-2 justify-end" />
            </div>
-          </div>
-        )}
+          }
+        >
+          <TypedAwait resolve={data}>
+            {(list) => (
+              <>
+                {list.runs.length === 0 && !list.hasFilters ? (
+                  list.possibleTasks.length === 0 ? (
+                    <CreateFirstTaskInstructions />
+                  ) : (
+                    <RunTaskInstructions />
+                  )
+                ) : (
+                  <div className={cn("grid h-fit grid-cols-1 gap-4")}>
+                    <div>
+                      <div className="mb-2 flex items-center justify-between gap-x-2">
+                        <RunsFilters
+                          possibleEnvironments={project.environments}
+                          possibleTasks={list.possibleTasks}
+                        />
+                        <div className="flex items-center justify-end gap-x-2">
+                          <ListPagination list={list} />
+                        </div>
+                      </div>
+
+                      <TaskRunsTable
+                        total={list.runs.length}
+                        hasFilters={list.hasFilters}
+                        filters={list.filters}
+                        runs={list.runs}
+                        isLoading={isLoading}
+                      />
+                      <ListPagination list={list} className="mt-2 justify-end" />
+                    </div>
+                  </div>
+                )}
+              </>
+            )}
+          </TypedAwait>
+        </Suspense>
      </PageBody>
    </>
  );
@@ -247,7 +247,6 @@ export default function Page() {
                }}
                runs={schedule.runs}
                isLoading={false}
-                currentUser={user}
              />
            </div>
            <div className="flex flex-col gap-1">
@@ -4,6 +4,7 @@ import { Outlet, useLocation, useParams } from "@remix-run/react";
 import { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { BlankstateInstructions } from "~/components/BlankstateInstructions";
+import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
 import { InlineCode } from "~/components/code/InlineCode";
 import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
 import { MainCenteredContainer, PageBody, PageContainer } from "~/components/layout/AppLayout";
@@ -12,6 +13,7 @@ import { DateTime } from "~/components/primitives/DateTime";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { PaginationControls } from "~/components/primitives/Pagination";
 import { Paragraph } from "~/components/primitives/Paragraph";
+import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
 import {
  ResizableHandle,
  ResizablePanel,
@@ -93,6 +95,18 @@ export default function Page() {
      <NavBar>
        <PageTitle title="Schedules" />
        <PageAccessories>
+          <AdminDebugTooltip>
+            <PropertyTable>
+              {schedules.map((schedule) => (
+                <Property label={schedule.friendlyId} key={schedule.id}>
+                  <div className="flex items-center gap-2">
+                    <Paragraph variant="extra-small/bright/mono">{schedule.id}</Paragraph>
+                  </div>
+                </Property>
+              ))}
+            </PropertyTable>
+          </AdminDebugTooltip>
+
          <LinkButton
            LeadingIcon={PlusIcon}
            to={`${v3NewSchedulePath(organization, project)}${location.search}`}
@@ -261,10 +275,10 @@ function SchedulesTable({
                  {schedule.userProvidedDeduplicationKey ? schedule.deduplicationKey : "–"}
                </TableCell>
                <TableCell to={path} className={cellClass}>
-                  <DateTime date={schedule.nextRun} />
+                  <DateTime date={schedule.nextRun} timeZone="utc" />
                </TableCell>
                <TableCell to={path} className={cellClass}>
-                  {schedule.lastRun ? <DateTime date={schedule.lastRun} /> : "–"}
+                  {schedule.lastRun ? <DateTime date={schedule.lastRun} timeZone="utc" /> : "–"}
                </TableCell>
                <TableCell to={path} className={cellClass}>
                  <div className="flex gap-1">
@@ -3,6 +3,7 @@ import { parse } from "@conform-to/zod";
 import { Form, useActionData, useNavigation } from "@remix-run/react";
 import { ActionFunction, json } from "@remix-run/server-runtime";
 import { z } from "zod";
+import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
 import { InlineCode } from "~/components/code/InlineCode";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { Button } from "~/components/primitives/Buttons";
@@ -14,7 +15,9 @@ import { Hint } from "~/components/primitives/Hint";
 import { Input } from "~/components/primitives/Input";
 import { InputGroup } from "~/components/primitives/InputGroup";
 import { Label } from "~/components/primitives/Label";
-import { NavBar, PageTitle } from "~/components/primitives/PageHeader";
+import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
+import { Paragraph } from "~/components/primitives/Paragraph";
+import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
 import { prisma } from "~/db.server";
 import { useProject } from "~/hooks/useProject";
 import { redirectWithSuccessMessage } from "~/models/message.server";
@@ -110,6 +113,23 @@ export default function Page() {
    <PageContainer>
      <NavBar>
        <PageTitle title={`${project.name} project settings`} />
+
+        <PageAccessories>
+          <AdminDebugTooltip>
+            <PropertyTable>
+              <Property label="ID">
+                <div className="flex items-center gap-2">
+                  <Paragraph variant="extra-small/bright/mono">{project.id}</Paragraph>
+                </div>
+              </Property>
+              <Property label="Org ID">
+                <div className="flex items-center gap-2">
+                  <Paragraph variant="extra-small/bright/mono">{project.organizationId}</Paragraph>
+                </div>
+              </Property>
+            </PropertyTable>
+          </AdminDebugTooltip>
+        </PageAccessories>
      </NavBar>

      <PageBody>
@@ -9,6 +9,7 @@ import {
 } from "~/components/environments/EnvironmentLabel";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { Header2 } from "~/components/primitives/Headers";
+import { Input } from "~/components/primitives/Input";
 import { NavBar, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
 import { RadioButtonCircle } from "~/components/primitives/RadioButton";
@@ -20,6 +21,7 @@ import {
 import { Spinner } from "~/components/primitives/Spinner";
 import {
  Table,
+  TableBlankRow,
  TableBody,
  TableCell,
  TableHeader,
@@ -32,6 +34,7 @@ import { useLinkStatus } from "~/hooks/useLinkStatus";
 import { useOptimisticLocation } from "~/hooks/useOptimisticLocation";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
+import { useTextFilter } from "~/hooks/useTextFilter";
 import {
  SelectedEnvironment,
  TaskListItem,
@@ -67,7 +70,7 @@ export default function Page() {

  //get optimistic location for the segment control
  const optimisticLocation = useOptimisticLocation();
-  const environment = new URLSearchParams(optimisticLocation.search).get("environment");
+  const environment = new URLSearchParams(optimisticLocation.search).get("environment") ?? "dev";

  const navigation = useNavigation();

@@ -150,11 +153,50 @@ function TaskSelector({
  tasks: TaskListItem[];
  environmentSlug: string;
 }) {
-  const organization = useOrganization();
-  const project = useProject();
+  const { filterText, setFilterText, filteredItems } = useTextFilter<TaskListItem>({
+    items: tasks,
+    filter: (task, text) => {
+      if (task.taskIdentifier.toLowerCase().includes(text.toLowerCase())) {
+        return true;
+      }
+
+      if (task.exportName.toLowerCase().includes(text.toLowerCase())) {
+        return true;
+      }
+
+      if (task.filePath.toLowerCase().includes(text.toLowerCase())) {
+        return true;
+      }
+
+      if (task.id.toLowerCase().includes(text.toLowerCase())) {
+        return true;
+      }
+
+      if (task.friendlyId.toLowerCase().includes(text.toLowerCase())) {
+        return true;
+      }
+
+      if (task.triggerSource === "SCHEDULED" && "scheduled".includes(text.toLowerCase())) {
+        return true;
+      }
+
+      return false;
+    },
+  });

  return (
    <div className="divide-y divide-charcoal-800 overflow-y-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600">
+      <div className="px-2 pb-2">
+        <Input
+          placeholder="Search tasks"
+          variant="medium"
+          icon="search"
+          fullWidth={true}
+          value={filterText}
+          autoFocus
+          onChange={(e) => setFilterText(e.target.value)}
+        />
+      </div>
      <Table>
        <TableHeader>
          <TableRow>
@@ -166,42 +208,17 @@ function TaskSelector({
          </TableRow>
        </TableHeader>
        <TableBody>
-          {tasks.map((t) => {
-            const path = v3TestTaskPath(organization, project, t, environmentSlug);
-            const { isActive, isPending } = useLinkStatus(path);
-            return (
-              <TableRow
-                key={t.taskIdentifier}
-                className={cn(
-                  (isActive || isPending) &&
-                    "z-20 rounded-sm outline outline-1 outline-offset-[-1px] outline-secondary"
-                )}
-              >
-                <TableCell to={path} actionClassName="pl-2.5 pr-1 py-1">
-                  <RadioButtonCircle checked={isActive || isPending} />
-                </TableCell>
-                <TableCell to={path} actionClassName="pl-1 pr-2 py-1">
-                  <div className="flex flex-col gap-0.5">
-                    <TaskFunctionName
-                      variant="extra-small"
-                      functionName={t.exportName}
-                      className="-ml-1 inline-flex"
-                    />
-                    <div className="flex items-start gap-1">
-                      <TaskTriggerSourceIcon source={t.triggerSource} className="size-3.5" />
-                      <Paragraph variant="extra-small" className="text-text-dimmed">
-                        {t.taskIdentifier}
-                      </Paragraph>
-                    </div>
-                  </div>
-                </TableCell>
-
-                <TableCell to={path} actionClassName="px-2 py-1">
-                  {t.filePath}
-                </TableCell>
-              </TableRow>
-            );
-          })}
+          {filteredItems.length > 0 ? (
+            filteredItems.map((t) => (
+              <TaskRow key={t.friendlyId} task={t} environmentSlug={environmentSlug} />
+            ))
+          ) : (
+            <TableBlankRow colSpan={3}>
+              <Paragraph spacing variant="small">
+                No tasks match "{filterText}"
+              </Paragraph>
+            </TableBlankRow>
+          )}
        </TableBody>
      </Table>
    </div>
@@ -217,3 +234,43 @@ function NoTaskInstructions({ environment }: { environment?: SelectedEnvironment
    </div>
  );
 }
+
+function TaskRow({ task, environmentSlug }: { task: TaskListItem; environmentSlug: string }) {
+  const organization = useOrganization();
+  const project = useProject();
+
+  const path = v3TestTaskPath(organization, project, task, environmentSlug);
+  const { isActive, isPending } = useLinkStatus(path);
+  return (
+    <TableRow
+      key={task.taskIdentifier}
+      className={cn(
+        (isActive || isPending) &&
+          "z-20 rounded-sm outline outline-1 outline-offset-[-1px] outline-secondary"
+      )}
+    >
+      <TableCell to={path} actionClassName="pl-2.5 pr-1 py-1">
+        <RadioButtonCircle checked={isActive || isPending} />
+      </TableCell>
+      <TableCell to={path} actionClassName="pl-1 pr-2 py-1">
+        <div className="flex flex-col gap-0.5">
+          <TaskFunctionName
+            variant="extra-small"
+            functionName={task.exportName}
+            className="-ml-1 inline-flex"
+          />
+          <div className="flex items-start gap-1">
+            <TaskTriggerSourceIcon source={task.triggerSource} className="size-3.5" />
+            <Paragraph variant="extra-small" className="text-text-dimmed">
+              {task.taskIdentifier}
+            </Paragraph>
+          </div>
+        </div>
+      </TableCell>
+
+      <TableCell to={path} actionClassName="px-2 py-1">
+        {task.filePath}
+      </TableCell>
+    </TableRow>
+  );
+}
@@ -239,7 +239,7 @@ export default function Page() {
                  <FormError id={organizationSlug.errorId}>{organizationSlug.error}</FormError>
                  <FormError>{deleteForm.error}</FormError>
                  <Hint>
-                    This change is irreversible, so please be certain. Type in the Organization slug
+                    This change is irreversible, so please be certain. Type in the Organization slug{" "}
                    <InlineCode variant="extra-small">{organization.slug}</InlineCode> and then
                    press Delete.
                  </Hint>
@@ -8,6 +8,7 @@ import { UseDataFunctionReturn, typedjson, useTypedLoaderData } from "remix-type
 import invariant from "tiny-invariant";
 import { z } from "zod";
 import { UserAvatar } from "~/components/UserProfilePhoto";
+import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
 import {
  Alert,
@@ -23,8 +24,9 @@ import { Button, ButtonContent, LinkButton } from "~/components/primitives/Butto
 import { DateTime } from "~/components/primitives/DateTime";
 import { Header2, Header3 } from "~/components/primitives/Headers";
 import { NamedIcon } from "~/components/primitives/NamedIcon";
-import { NavBar, PageTitle } from "~/components/primitives/PageHeader";
+import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
+import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
 import { SimpleTooltip } from "~/components/primitives/Tooltip";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useUser } from "~/hooks/useUser";
@@ -107,6 +109,28 @@ export default function Page() {
    <PageContainer>
      <NavBar>
        <PageTitle title="Team" />
+
+        <PageAccessories>
+          <AdminDebugTooltip>
+            <PropertyTable>
+              <Property label="Org ID">
+                <div className="flex items-center gap-2">
+                  <Paragraph variant="extra-small/bright/mono">{organization.id}</Paragraph>
+                </div>
+              </Property>
+
+              {members.map((member) => (
+                <Property label={member.user.name} key={member.id}>
+                  <div className="flex items-center gap-2">
+                    <Paragraph variant="extra-small/bright/mono">
+                      {member.user.email} - {member.user.id}
+                    </Paragraph>
+                  </div>
+                </Property>
+              ))}
+            </PropertyTable>
+          </AdminDebugTooltip>
+        </PageAccessories>
      </NavBar>
      <PageBody>
        <Header2>Members</Header2>
@@ -15,9 +15,12 @@ export const loader = async ({ request }: LoaderFunctionArgs) => {
    return redirect(confirmBasicDetailsPath());
  }

-  return typedjson({
-    headers: [["Set-Cookie", await commitSession(await clearRedirectTo(request))]],
-  });
+  return typedjson(
+    {},
+    {
+      headers: { "Set-Cookie": await commitSession(await clearRedirectTo(request)) },
+    }
+  );
 };

 export default function App() {
@@ -1,11 +1,9 @@
 import { MagnifyingGlassIcon } from "@heroicons/react/20/solid";
 import { Form } from "@remix-run/react";
 import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime";
-import { redirect } from "@remix-run/server-runtime";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { z } from "zod";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
-import { Header1 } from "~/components/primitives/Headers";
 import { Input } from "~/components/primitives/Input";
 import { PaginationControls } from "~/components/primitives/Pagination";
 import { Paragraph } from "~/components/primitives/Paragraph";
@@ -18,10 +16,8 @@ import {
  TableHeaderCell,
  TableRow,
 } from "~/components/primitives/Table";
-import { useUser } from "~/hooks/useUser";
-import { adminGetOrganizations, adminGetUsers, setV3Enabled } from "~/models/admin.server";
+import { adminGetOrganizations, setV3Enabled } from "~/models/admin.server";
 import { redirectWithSuccessMessage } from "~/models/message.server";
-import { commitImpersonationSession, setImpersonationId } from "~/services/impersonation.server";
 import { requireUserId } from "~/services/session.server";
 import { createSearchParams } from "~/utils/searchParams";

@@ -342,7 +342,9 @@ function Timeline({ startTime, duration, inProgress, isError }: TimelineProps) {
          <DateTimeAccurate date={startTime} />
        </Paragraph>
        {state === "pending" ? (
-          <LiveTimer startTime={startTime} className="" />
+          <Paragraph variant="extra-small" className={cn("whitespace-nowrap tabular-nums")}>
+            <LiveTimer startTime={startTime} />
+          </Paragraph>
        ) : (
          <Paragraph variant="small">
            <DateTimeAccurate
@@ -63,6 +63,14 @@ export const action: ActionFunction = async ({ request, params }) => {
      { spanId: newRun.spanId }
    );

+    logger.debug("Replayed run", {
+      taskRunId: taskRun.id,
+      taskRunFriendlyId: taskRun.friendlyId,
+      newRunId: newRun.id,
+      newRunFriendlyId: newRun.friendlyId,
+      runPath,
+    });
+
    return redirectWithSuccessMessage(runPath, request, `Replaying run`);
  } catch (error) {
    if (error instanceof Error) {
@@ -157,17 +157,17 @@ function TreeViewParent({
    onSelectedIdChanged: (id) => {
      console.log("onSelectedIdChanged", id);
    },
-    onCollapsedIdsChanged: (ids) => {
-      console.log("onCollapsedIdsChanged", ids);
-    },
    estimatedRowHeight: () => 32,
    parentRef,
-    filter: (node) => {
-      if (filterText === "") return true;
-      if (node.data.title.toLowerCase().includes(filterText.toLowerCase())) {
-        return true;
-      }
-      return false;
+    filter: {
+      value: filterText,
+      fn: (text, node) => {
+        if (text === "") return true;
+        if (node.data.title.toLowerCase().includes(text.toLowerCase())) {
+          return true;
+        }
+        return false;
+      },
    },
  });

@@ -1,4 +1,5 @@
 import { RuntimeEnvironmentType } from "@trigger.dev/database";
+import { logger } from "./logger.server";

 const environmentSortOrder: RuntimeEnvironmentType[] = [
  "DEVELOPMENT",
@@ -9,12 +10,23 @@ const environmentSortOrder: RuntimeEnvironmentType[] = [

 type SortType = {
  type: RuntimeEnvironmentType;
+  userName?: string | null;
 };

 export function sortEnvironments<T extends SortType>(environments: T[]): T[] {
  return environments.sort((a, b) => {
    const aIndex = environmentSortOrder.indexOf(a.type);
    const bIndex = environmentSortOrder.indexOf(b.type);
-    return aIndex - bIndex;
+
+    const difference = aIndex - bIndex;
+
+    if (difference === 0) {
+      //same environment so sort by name
+      const usernameA = a.userName || "";
+      const usernameB = b.userName || "";
+      return usernameA.localeCompare(usernameB);
+    }
+
+    return difference;
  });
 }
@@ -32,6 +32,14 @@ export class ContinueRunService {
          throw new Error("Run is not resumable");
        }

+        // Delete any tasks that are errored
+        const erroredTasks = await tx.task.findMany({
+          where: {
+            runId: runId,
+            status: "ERRORED",
+          },
+        });
+
        await tx.jobRun.update({
          where: { id: runId },
          data: {
@@ -45,9 +53,15 @@ export class ContinueRunService {
          },
        });

+        for (const task of erroredTasks) {
+          await tx.task.delete({
+            where: { id: task.id },
+          });
+        }
+
        await ResumeRunService.enqueue(run, tx);
      },
-      { timeout: 10000 }
+      { timeout: 30_000 }
    );
  }
 }
@@ -1,10 +1,17 @@
-import { Prisma, PrismaClient } from "@trigger.dev/database";
+import { Prisma, PrismaClient, RuntimeEnvironmentType } from "@trigger.dev/database";
 import { z } from "zod";
+import { environmentTitle } from "~/components/environments/EnvironmentLabel";
 import { $transaction, prisma } from "~/db.server";
+import { env } from "~/env.server";
 import { getSecretStore } from "~/services/secrets/secretStore.server";
 import { generateFriendlyId } from "../friendlyIdentifiers";
-import { EnvironmentVariable, ProjectEnvironmentVariable, Repository, Result } from "./repository";
-import { env } from "~/env.server";
+import {
+  CreateResult,
+  EnvironmentVariable,
+  ProjectEnvironmentVariable,
+  Repository,
+  Result,
+} from "./repository";

 function secretKeyProjectPrefix(projectId: string) {
  return `environmentvariable:${projectId}:`;
@@ -35,8 +42,15 @@ export class EnvironmentVariablesRepository implements Repository {
  async create(
    projectId: string,
    userId: string,
-    options: { key: string; values: { value: string; environmentId: string }[] }
-  ): Promise<Result> {
+    options: {
+      overwrite: boolean;
+      environmentIds: string[];
+      variables: {
+        key: string;
+        value: string;
+      }[];
+    }
+  ): Promise<CreateResult> {
    const project = await this.prismaClient.project.findUnique({
      where: {
        id: projectId,
@@ -55,6 +69,18 @@ export class EnvironmentVariablesRepository implements Repository {
            id: true,
          },
        },
+        environmentVariables: {
+          select: {
+            key: true,
+            values: {
+              select: {
+                environment: {
+                  select: { id: true, type: true },
+                },
+              },
+            },
+          },
+        },
      },
    });

@@ -62,58 +88,109 @@ export class EnvironmentVariablesRepository implements Repository {
      return { success: false as const, error: "Project not found" };
    }

-    if (options.values.every((v) => !project.environments.some((e) => e.id === v.environmentId))) {
+    if (options.environmentIds.every((v) => !project.environments.some((e) => e.id === v))) {
      return { success: false as const, error: `Environment not found` };
    }

-    //get rid of empty strings
-    const values = options.values.filter((v) => v.value.trim() !== "");
-
+    //get rid of empty variables
+    const values = options.variables.filter((v) => v.key.trim() !== "" && v.value.trim() !== "");
    if (values.length === 0) {
      return { success: false as const, error: `You must set at least one value` };
    }

+    //check if any of them exist in an environment we're setting
+    if (!options.overwrite) {
+      const existingVariableKeys: { key: string; environments: RuntimeEnvironmentType[] }[] = [];
+      for (const variable of values) {
+        const existingVariable = project.environmentVariables.find((v) => v.key === variable.key);
+        if (
+          existingVariable &&
+          existingVariable.values.some((v) => options.environmentIds.includes(v.environment.id))
+        ) {
+          existingVariableKeys.push({
+            key: variable.key,
+            environments: existingVariable.values
+              .filter((v) => options.environmentIds.includes(v.environment.id))
+              .map((v) => v.environment.type),
+          });
+        }
+      }
+
+      if (existingVariableKeys.length > 0) {
+        return {
+          success: false as const,
+          error: `Some of the variables are already set for these environments`,
+          variableErrors: existingVariableKeys.map((val) => ({
+            key: val.key,
+            error: `Variable already set in ${val.environments
+              .map((e) => environmentTitle({ type: e }))
+              .join(", ")}.`,
+          })),
+        };
+      }
+    }
+
    try {
      const result = await $transaction(this.prismaClient, async (tx) => {
-        const environmentVariable = await tx.environmentVariable.create({
-          data: {
-            key: options.key,
-            friendlyId: generateFriendlyId("envvar"),
-            project: {
-              connect: {
-                id: projectId,
+        for (const variable of values) {
+          const environmentVariable = await tx.environmentVariable.upsert({
+            where: {
+              projectId_key: {
+                key: variable.key,
+                projectId,
              },
            },
-          },
-        });
-
-        const secretStore = getSecretStore("DATABASE", {
-          prismaClient: tx,
-        });
-
-        //create the secret values and references
-        for (const value of values) {
-          const key = secretKey(projectId, value.environmentId, options.key);
-
-          //create the secret reference
-          const secretReference = await tx.secretReference.create({
-            data: {
-              key,
-              provider: "DATABASE",
+            create: {
+              key: variable.key,
+              friendlyId: generateFriendlyId("envvar"),
+              project: {
+                connect: {
+                  id: projectId,
+                },
+              },
            },
+            update: {},
          });

-          const variableValue = await tx.environmentVariableValue.create({
-            data: {
-              variableId: environmentVariable.id,
-              environmentId: value.environmentId,
-              valueReferenceId: secretReference.id,
-            },
+          const secretStore = getSecretStore("DATABASE", {
+            prismaClient: tx,
          });

-          await secretStore.setSecret<{ secret: string }>(key, {
-            secret: value.value,
-          });
+          //set the secret values and references
+          for (const environmentId of options.environmentIds) {
+            const key = secretKey(projectId, environmentId, variable.key);
+
+            //create the secret reference
+            const secretReference = await tx.secretReference.upsert({
+              where: {
+                key,
+              },
+              create: {
+                key,
+                provider: "DATABASE",
+              },
+              update: {},
+            });
+
+            const variableValue = await tx.environmentVariableValue.upsert({
+              where: {
+                variableId_environmentId: {
+                  variableId: environmentVariable.id,
+                  environmentId,
+                },
+              },
+              create: {
+                variableId: environmentVariable.id,
+                environmentId: environmentId,
+                valueReferenceId: secretReference.id,
+              },
+              update: {},
+            });
+
+            await secretStore.setSecret<{ secret: string }>(key, {
+              secret: variable.value,
+            });
+          }
        }
      });

@@ -126,7 +203,7 @@ export class EnvironmentVariablesRepository implements Repository {
        if (error.code === "P2002") {
          return {
            success: false as const,
-            error: `There's already an environment variable called ${options.key}.`,
+            error: `There was already an existing field`,
          };
        }
      }
@@ -1,22 +1,27 @@
 import { RuntimeEnvironmentType } from "@trigger.dev/database";
 import { z } from "zod";

-const EnvironmentVariable = z
+export const EnvironmentVariableKey = z
  .string()
-  .nonempty("Environment variable key is required")
-  .regex(/^\w+$/, "Environment variables can only contain alphanumeric characters and underscores");
+  .nonempty("Key is required")
+  .regex(/^\w+$/, "Keys can only use alphanumeric characters and underscores");

-export const CreateEnvironmentVariable = z.object({
-  key: EnvironmentVariable,
-  values: z.array(
-    z.object({
-      environmentId: z.string(),
-      value: z.string(),
-    })
-  ),
+export const CreateEnvironmentVariables = z.object({
+  environmentIds: z.array(z.string()),
+  variables: z.array(z.object({ key: EnvironmentVariableKey, value: z.string() })),
 });

-export type CreateEnvironmentVariable = z.infer<typeof CreateEnvironmentVariable>;
+export type CreateEnvironmentVariables = z.infer<typeof CreateEnvironmentVariables>;
+
+export type CreateResult =
+  | {
+      success: true;
+    }
+  | {
+      success: false;
+      error: string;
+      variableErrors?: { key: string; error: string }[];
+    };

 export const EditEnvironmentVariable = z.object({
  id: z.string(),
@@ -60,7 +65,11 @@ export type EnvironmentVariable = {
 };

 export interface Repository {
-  create(projectId: string, userId: string, options: CreateEnvironmentVariable): Promise<Result>;
+  create(
+    projectId: string,
+    userId: string,
+    options: CreateEnvironmentVariables
+  ): Promise<CreateResult>;
  edit(projectId: string, userId: string, options: EditEnvironmentVariable): Promise<Result>;
  getProject(projectId: string, userId: string): Promise<ProjectEnvironmentVariable[]>;
  getEnvironment(
@@ -13,6 +13,7 @@ import {
  correctErrorStackTrace,
  createPacketAttributesAsJson,
  flattenAttributes,
+  NULL_SENTINEL,
  isExceptionSpanEvent,
  omit,
  unflattenAttributes,
@@ -438,21 +439,10 @@ export class EventRepository {
      return;
    }

-    const output = isEmptyJson(fullEvent.output)
-      ? null
-      : unflattenAttributes(fullEvent.output as Attributes);
+    const output = rehydrateJson(fullEvent.output);
+    const payload = rehydrateJson(fullEvent.payload);

-    const payload = isEmptyJson(fullEvent.payload)
-      ? null
-      : unflattenAttributes(fullEvent.payload as Attributes);
-
-    const show = unflattenAttributes(
-      filteredAttributes(fullEvent.properties as Attributes, SemanticInternalAttributes.SHOW)
-    )[SemanticInternalAttributes.SHOW] as
-      | {
-          actions?: boolean;
-        }
-      | undefined;
+    const show = rehydrateShow(fullEvent.properties);

    const properties = sanitizedAttributes(fullEvent.properties);

@@ -1046,7 +1036,7 @@ function isEmptyJson(json: Prisma.JsonValue) {
  return false;
 }

-function sanitizedAttributes(json: Prisma.JsonValue): Record<string, unknown> | undefined {
+function sanitizedAttributes(json: Prisma.JsonValue) {
  if (json === null || json === undefined) {
    return;
  }
@@ -1143,3 +1133,57 @@ function getNowInNanoseconds(): bigint {
 function getDateFromNanoseconds(nanoseconds: bigint) {
  return new Date(Number(nanoseconds) / 1_000_000);
 }
+
+function rehydrateJson(json: Prisma.JsonValue): any {
+  if (json === null) {
+    return undefined;
+  }
+
+  if (json === NULL_SENTINEL) {
+    return null;
+  }
+
+  if (typeof json === "string") {
+    return json;
+  }
+
+  if (typeof json === "number") {
+    return json;
+  }
+
+  if (typeof json === "boolean") {
+    return json;
+  }
+
+  if (Array.isArray(json)) {
+    return json.map((item) => rehydrateJson(item));
+  }
+
+  if (typeof json === "object") {
+    return unflattenAttributes(json as Attributes);
+  }
+
+  return null;
+}
+
+function rehydrateShow(properties: Prisma.JsonValue): { actions?: boolean } | undefined {
+  if (properties === null || properties === undefined) {
+    return;
+  }
+
+  if (typeof properties !== "object") {
+    return;
+  }
+
+  if (Array.isArray(properties)) {
+    return;
+  }
+
+  const actions = properties[SemanticInternalAttributes.SHOW_ACTIONS];
+
+  if (typeof actions === "boolean") {
+    return { actions };
+  }
+
+  return;
+}
@@ -19,6 +19,7 @@ import {
  MarQSQueuePriorityStrategy,
  MessagePayload,
  QueueCapacities,
+  QueueRange,
 } from "./types";

 const tracer = trace.getTracer("marqs");
@@ -231,7 +232,7 @@ export class MarQS {
          return;
        }

-        const message = await this.#readMessage(messageData.messageId);
+        const message = await this.readMessage(messageData.messageId);

        if (message) {
          span.setAttributes({
@@ -308,7 +309,7 @@ export class MarQS {
          return;
        }

-        const message = await this.#readMessage(messageData.messageId);
+        const message = await this.readMessage(messageData.messageId);

        if (message) {
          span.setAttributes({
@@ -336,7 +337,7 @@ export class MarQS {
    return this.#trace(
      "acknowledgeMessage",
      async (span) => {
-        const message = await this.#readMessage(messageId);
+        const message = await this.readMessage(messageId);

        if (!message) {
          return;
@@ -374,12 +375,13 @@ export class MarQS {
  public async replaceMessage(
    messageId: string,
    messageData: Record<string, unknown>,
-    timestamp?: number
+    timestamp?: number,
+    inplace?: boolean
  ) {
    return this.#trace(
      "replaceMessage",
      async (span) => {
-        const oldMessage = await this.#readMessage(messageId);
+        const oldMessage = await this.readMessage(messageId);

        if (!oldMessage) {
          return;
@@ -392,6 +394,27 @@ export class MarQS {
          [SemanticAttributes.PARENT_QUEUE]: oldMessage.parentQueue,
        });

+        const traceContext = {
+          traceparent: oldMessage.data.traceparent,
+          tracestate: oldMessage.data.tracestate,
+        };
+
+        const newMessage: MessagePayload = {
+          version: "1",
+          // preserve original trace context
+          data: { ...messageData, ...traceContext },
+          queue: oldMessage.queue,
+          concurrencyKey: oldMessage.concurrencyKey,
+          timestamp: timestamp ?? Date.now(),
+          messageId,
+          parentQueue: oldMessage.parentQueue,
+        };
+
+        if (inplace) {
+          await this.#callReplaceMessage(newMessage);
+          return;
+        }
+
        await this.#callAcknowledgeMessage({
          parentQueue: oldMessage.parentQueue,
          messageKey: this.keys.messageKey(messageId),
@@ -403,16 +426,6 @@ export class MarQS {
          messageId,
        });

-        const newMessage: MessagePayload = {
-          version: "1",
-          data: messageData,
-          queue: oldMessage.queue,
-          concurrencyKey: oldMessage.concurrencyKey,
-          timestamp: timestamp ?? Date.now(),
-          messageId,
-          parentQueue: oldMessage.parentQueue,
-        };
-
        await this.#callEnqueueMessage(newMessage);
      },
      {
@@ -455,7 +468,7 @@ export class MarQS {
    return this.#trace(
      "nackMessage",
      async (span) => {
-        const message = await this.#readMessage(messageId);
+        const message = await this.readMessage(messageId);

        if (!message) {
          return;
@@ -505,7 +518,7 @@ export class MarQS {
    return this.options.visibilityTimeoutInMs ?? 300000;
  }

-  async #readMessage(messageId: string) {
+  async readMessage(messageId: string) {
    return this.#trace(
      "readMessage",
      async (span) => {
@@ -551,7 +564,7 @@ export class MarQS {
          parentQueue
        );

-        const queues = await this.#zrangeWithScores(parentQueue, range[0], range[1]);
+        const queues = await this.#getChildQueuesWithScores(parentQueue, range);

        const queuesWithScores = await this.#calculateQueueScores(queues, calculateCapacities);

@@ -562,21 +575,25 @@ export class MarQS {
          selectionId
        );

-        if (typeof choice !== "string") {
-          return;
-        }
-
        span.setAttributes({
          ...flattenAttributes(queues, "marqs.queues"),
        });
        span.setAttributes({
          ...flattenAttributes(queuesWithScores, "marqs.queuesWithScores"),
        });
-        span.setAttribute("marqs.nextRange", range);
-        span.setAttribute("marqs.queueCount", queues.length);
-        span.setAttribute("marqs.queueChoice", choice);
+        span.setAttribute("nextRange.offset", range.offset);
+        span.setAttribute("nextRange.count", range.count);
+        span.setAttribute("queueCount", queues.length);

-        return choice;
+        if (typeof choice !== "string") {
+          span.setAttribute("noQueueChoice", true);
+
+          return;
+        } else {
+          span.setAttribute("queueChoice", choice);
+
+          return choice;
+        }
      },
      {
        kind: SpanKind.CONSUMER,
@@ -620,12 +637,19 @@ export class MarQS {
    });
  }

-  async #zrangeWithScores(
+  async #getChildQueuesWithScores(
    key: string,
-    min: number,
-    max: number
+    range: QueueRange
  ): Promise<Array<{ value: string; score: number }>> {
-    const valuesWithScores = await this.redis.zrange(key, min, max, "WITHSCORES");
+    const valuesWithScores = await this.redis.zrangebyscore(
+      key,
+      "-inf",
+      Date.now(),
+      "WITHSCORES",
+      "LIMIT",
+      range.offset,
+      range.count
+    );
    const result: Array<{ value: string; score: number }> = [];

    for (let i = 0; i < valuesWithScores.length; i += 2) {
@@ -881,6 +905,17 @@ export class MarQS {
    };
  }

+  async #callReplaceMessage(message: MessagePayload) {
+    logger.debug("Calling replaceMessage", {
+      messagePayload: message,
+    });
+
+    return this.redis.replaceMessage(
+      this.keys.messageKey(message.messageId),
+      JSON.stringify(message)
+    );
+  }
+
  async #callAcknowledgeMessage({
    parentQueue,
    messageKey,
@@ -1185,6 +1220,25 @@ return {messageId, messageScore} -- Return message details
      `,
    });

+    this.redis.defineCommand("replaceMessage", {
+      numberOfKeys: 1,
+      lua: `
+local messageKey = KEYS[1]
+local messageData = ARGV[1]
+
+-- Check if message exists
+local existingMessage = redis.call('GET', messageKey)
+
+-- Do nothing if it doesn't
+if #existingMessage == nil then
+    return nil
+end
+
+-- Replace the message
+redis.call('SET', messageKey, messageData, 'GET')
+      `,
+    });
+
    this.redis.defineCommand("acknowledgeMessage", {
      numberOfKeys: 7,
      lua: `
@@ -1406,6 +1460,12 @@ declare module "ioredis" {
      callback?: Callback<[string, string]>
    ): Result<[string, string] | null, Context>;

+    replaceMessage(
+      messageKey: string,
+      messageData: string,
+      callback?: Callback<void>
+    ): Result<void, Context>;
+
    acknowledgeMessage(
      parentQueue: string,
      messageKey: string,
@@ -1489,7 +1549,7 @@ function getMarQSClient() {

      return new MarQS({
        keysProducer: new MarQSShortKeyProducer(KEY_PREFIX),
-        queuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 12 }),
+        queuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 36 }),
        envQueuePriorityStrategy: new SimpleWeightedChoiceStrategy({ queueSelectionCount: 12 }),
        workers: 1,
        redis: redisOptions,
@@ -1,5 +1,10 @@
 import { RedisOptions } from "ioredis";
-import { MarQSQueuePriorityStrategy, PriorityStrategyChoice, QueueWithScores } from "./types";
+import {
+  MarQSQueuePriorityStrategy,
+  PriorityStrategyChoice,
+  QueueRange,
+  QueueWithScores,
+} from "./types";
 import { nanoid } from "nanoid";
 import seedrandom from "seedrandom";

@@ -26,9 +31,7 @@ export class DynamicWeightedChoiceStrategy implements MarQSQueuePriorityStrategy
    throw new Error("Method not implemented.");
  }

-  nextCandidateSelection(
-    parentQueue: string
-  ): Promise<{ range: [number, number]; selectionId: string }> {
+  nextCandidateSelection(parentQueue: string): Promise<{ range: QueueRange; selectionId: string }> {
    throw new Error("Method not implemented.");
  }
 }
@@ -39,13 +42,18 @@ export type SimpleWeightedChoiceStrategyOptions = {
 };

 export class SimpleWeightedChoiceStrategy implements MarQSQueuePriorityStrategy {
-  private _nextRangesByParentQueue: Map<string, [number, number]> = new Map();
+  private _nextRangesByParentQueue: Map<string, QueueRange> = new Map();
  private _randomGenerator = seedrandom(this.options.randomSeed);

  constructor(private options: SimpleWeightedChoiceStrategyOptions) {}

-  private nextRangeForParentQueue(parentQueue: string) {
-    return this._nextRangesByParentQueue.get(parentQueue) ?? [0, this.options.queueSelectionCount];
+  private nextRangeForParentQueue(parentQueue: string): QueueRange {
+    return (
+      this._nextRangesByParentQueue.get(parentQueue) ?? {
+        offset: 0,
+        count: this.options.queueSelectionCount,
+      }
+    );
  }

  chooseQueue(
@@ -55,25 +63,23 @@ export class SimpleWeightedChoiceStrategy implements MarQSQueuePriorityStrategy
  ): PriorityStrategyChoice {
    const filteredQueues = filterQueuesAtCapacity(queues);

-    if (filteredQueues.length === 0) {
-      if (queues.length === this.options.queueSelectionCount) {
-        const nextRangeForParentQueue = this.nextRangeForParentQueue(parentQueue);
-        const nextRange: [number, number] = nextRangeForParentQueue
-          ? [
-              nextRangeForParentQueue[1],
-              nextRangeForParentQueue[1] + this.options.queueSelectionCount,
-            ]
-          : [this.options.queueSelectionCount, this.options.queueSelectionCount * 2];
-        // If all queues are at capacity, and we were passed the max number of queues, then we will slide the window "to the right"
-        this._nextRangesByParentQueue.set(parentQueue, nextRange);
-      } else {
-        this._nextRangesByParentQueue.delete(parentQueue);
-      }
-
-      return { abort: true };
+    if (queues.length === this.options.queueSelectionCount) {
+      const nextRangeForParentQueue = this.nextRangeForParentQueue(parentQueue);
+      const nextRange: QueueRange = nextRangeForParentQueue
+        ? {
+            offset: nextRangeForParentQueue.offset + this.options.queueSelectionCount,
+            count: this.options.queueSelectionCount,
+          }
+        : { offset: this.options.queueSelectionCount, count: this.options.queueSelectionCount };
+      // If all queues are at capacity, and we were passed the max number of queues, then we will slide the window "to the right"
+      this._nextRangesByParentQueue.set(parentQueue, nextRange);
+    } else {
+      this._nextRangesByParentQueue.delete(parentQueue);
    }

-    this._nextRangesByParentQueue.delete(parentQueue);
+    if (filteredQueues.length === 0) {
+      return { abort: true };
+    }

    const queueWeights = this.#calculateQueueWeights(filteredQueues);

@@ -82,7 +88,7 @@ export class SimpleWeightedChoiceStrategy implements MarQSQueuePriorityStrategy

  async nextCandidateSelection(
    parentQueue: string
-  ): Promise<{ range: [number, number]; selectionId: string }> {
+  ): Promise<{ range: QueueRange; selectionId: string }> {
    return { range: this.nextRangeForParentQueue(parentQueue), selectionId: nanoid(24) };
  }

@@ -27,14 +27,15 @@ import { generateFriendlyId } from "../friendlyIdentifiers";
 import { socketIo } from "../handleSocketIo.server";
 import { findCurrentWorkerDeployment } from "../models/workerDeployment.server";
 import { RestoreCheckpointService } from "../services/restoreCheckpoint.server";
-import { tracer } from "../tracer.server";
+import { SEMINTATTRS_FORCE_RECORDING, tracer } from "../tracer.server";
+import { CrashTaskRunService } from "../services/crashTaskRun.server";

 const WithTraceContext = z.object({
  traceparent: z.string().optional(),
  tracestate: z.string().optional(),
 });

-const MessageBody = z.discriminatedUnion("type", [
+export const SharedQueueMessageBody = z.discriminatedUnion("type", [
  WithTraceContext.extend({
    type: z.literal("EXECUTE"),
    taskIdentifier: z.string(),
@@ -51,8 +52,14 @@ const MessageBody = z.discriminatedUnion("type", [
    resumableAttemptId: z.string(),
    checkpointEventId: z.string(),
  }),
+  WithTraceContext.extend({
+    type: z.literal("FAIL"),
+    reason: z.string(),
+  }),
 ]);

+export type SharedQueueMessageBody = z.infer<typeof SharedQueueMessageBody>;
+
 type BackgroundWorkerWithTasks = BackgroundWorker & { tasks: BackgroundWorkerTask[] };

 export type SharedQueueConsumerOptions = {
@@ -60,7 +67,6 @@ export type SharedQueueConsumerOptions = {
  traceTimeoutSeconds?: number;
  nextTickInterval?: number;
  interval?: number;
-  parentContext?: Context;
 };

 export class SharedQueueConsumer {
@@ -86,7 +92,6 @@ export class SharedQueueConsumer {
      traceTimeoutSeconds: options.traceTimeoutSeconds ?? 60, // 60 seconds
      nextTickInterval: options.nextTickInterval ?? 1000, // 1 second
      interval: options.interval ?? 100, // 100ms
-      parentContext: options.parentContext ?? ROOT_CONTEXT,
    };
  }

@@ -186,19 +191,17 @@ export class SharedQueueConsumer {
    ) {
      this.#endCurrentSpan();

-      const parentContext = this._options.parentContext ?? ROOT_CONTEXT;
-
      // Create a new trace
      this._currentSpan = tracer.startSpan(
        "SharedQueueConsumer.doWork()",
        {
          kind: SpanKind.CONSUMER,
        },
-        parentContext
+        ROOT_CONTEXT
      );

      // Get the span trace context
-      this._currentSpanContext = trace.setSpan(parentContext, this._currentSpan);
+      this._currentSpanContext = trace.setSpan(ROOT_CONTEXT, this._currentSpan);

      this._perTraceCountdown = this._options.maximumItemsPerTrace;
      this._lastNewTrace = new Date();
@@ -233,7 +236,7 @@ export class SharedQueueConsumer {

    logger.log("dequeueMessageInSharedQueue()", { queueMessage: message });

-    const messageBody = MessageBody.safeParse(message.data);
+    const messageBody = SharedQueueMessageBody.safeParse(message.data);

    if (!messageBody.success) {
      logger.error("Failed to parse message", {
@@ -411,11 +414,21 @@ export class SharedQueueConsumer {
        });

        if (!queue) {
+          logger.debug("SharedQueueConsumer queue not found, so nacking message", {
+            queueMessage: message,
+            taskRunQueue: lockedTaskRun.queue,
+            runtimeEnvironmentId: lockedTaskRun.runtimeEnvironmentId,
+          });
+
          await this.#nackAndDoMoreWork(message.messageId, this._options.nextTickInterval);
          return;
        }

        if (!this._enabled) {
+          logger.debug("SharedQueueConsumer not enabled, so nacking message", {
+            queueMessage: message,
+          });
+
          await marqs?.nackMessage(message.messageId);
          return;
        }
@@ -521,6 +534,11 @@ export class SharedQueueConsumer {
            }),
          ]);

+          logger.error("SharedQueueConsumer errored, so nacking message", {
+            queueMessage: message,
+            error: e instanceof Error ? { name: e.name, message: e.message, stack: e.stack } : e,
+          });
+
          await this.#nackAndDoMoreWork(message.messageId);
          return;
        }
@@ -739,6 +757,34 @@ export class SharedQueueConsumer {

        break;
      }
+      // Fail for whatever reason, usually runs that have been resumed but stopped heartbeating
+      case "FAIL": {
+        const existingTaskRun = await prisma.taskRun.findUnique({
+          where: {
+            id: message.messageId,
+          },
+        });
+
+        if (!existingTaskRun) {
+          logger.error("No existing task run to fail", {
+            queueMessage: messageBody,
+            messageId: message.messageId,
+          });
+
+          await this.#ackAndDoMoreWork(message.messageId);
+          return;
+        }
+
+        // TODO: Consider failing the attempt and retrying instead. This may not be a good idea, as dequeued FAIL messages tend to point towards critical, persistent errors.
+        const service = new CrashTaskRunService();
+        await service.call(existingTaskRun.id, {
+          crashAttempts: true,
+          reason: messageBody.data.reason,
+        });
+
+        await this.#ackAndDoMoreWork(message.messageId);
+        return;
+      }
    }

    this.#doMoreWork();
@@ -18,6 +18,8 @@ export type QueueWithScores = {
  age: number;
 };

+export type QueueRange = { offset: number; count: number };
+
 export interface MarQSKeyProducer {
  queueConcurrencyLimitKey(env: AuthenticatedEnvironment, queue: string): string;
  envConcurrencyLimitKey(env: AuthenticatedEnvironment): string;
@@ -69,9 +71,7 @@ export interface MarQSQueuePriorityStrategy {
   *
   * @returns The scores and the selectionId for the next candidate selection
   */
-  nextCandidateSelection(
-    parentQueue: string
-  ): Promise<{ range: [number, number]; selectionId: string }>;
+  nextCandidateSelection(parentQueue: string): Promise<{ range: QueueRange; selectionId: string }>;
 }

 export const MessagePayload = z.object({
@@ -10,6 +10,7 @@ import { generateFriendlyId } from "../friendlyIdentifiers";
 import { marqs } from "~/v3/marqs/index.server";
 import { CreateCheckpointRestoreEventService } from "./createCheckpointRestoreEvent.server";
 import { BaseService } from "./baseService.server";
+import { CrashTaskRunService } from "./crashTaskRun.server";

 const FREEZABLE_RUN_STATUSES: TaskRunStatus[] = ["EXECUTING", "RETRYING_AFTER_FAILURE"];
 const FREEZABLE_ATTEMPT_STATUSES: TaskRunAttemptStatus[] = ["EXECUTING", "FAILED"];
@@ -61,6 +62,14 @@ export class CreateCheckpointService extends BaseService {
          status: attempt.taskRun.status,
        },
      });
+
+      // This should only affect CLIs < beta.24, in very limited scenarios
+      const service = new CrashTaskRunService(this._prisma);
+      await service.call(attempt.taskRunId, {
+        crashAttempts: true,
+        reason: "Unfreezable state: Please upgrade your CLI",
+      });
+
      return;
    }

@@ -2,13 +2,14 @@ import {
  CoordinatorToPlatformMessages,
  TaskRunExecution,
  TaskRunExecutionResult,
+  WaitReason,
 } from "@trigger.dev/core/v3";
 import type { InferSocketMessageSchema } from "@trigger.dev/core/v3/zodSocket";
 import { $transaction, PrismaClientOrTransaction } from "~/db.server";
 import { logger } from "~/services/logger.server";
 import { marqs } from "~/v3/marqs/index.server";
 import { socketIo } from "../handleSocketIo.server";
-import { sharedQueueTasks } from "../marqs/sharedQueueConsumer.server";
+import { SharedQueueMessageBody, sharedQueueTasks } from "../marqs/sharedQueueConsumer.server";
 import { BaseService } from "./baseService.server";
 import { TaskRunAttempt } from "@trigger.dev/database";

@@ -91,12 +92,13 @@ export class ResumeAttemptService extends BaseService {

      switch (params.type) {
        case "WAIT_FOR_DURATION": {
-          logger.error(
-            "Attempt requested resume after duration wait, this is unexpected and likely a bug",
-            { attemptId: attempt.id }
-          );
+          logger.debug("Sending duration wait resume message", {
+            attemptId: attempt.id,
+            attemptFriendlyId: params.attemptFriendlyId,
+          });
+
+          await this.#setPostResumeStatuses(attempt, tx);

-          // Attempts should not request resume for duration waits, this is just here as a backup
          socketIo.coordinatorNamespace.emit("RESUME_AFTER_DURATION", {
            version: "v1",
            attemptId: attempt.id,
@@ -119,6 +121,9 @@ export class ResumeAttemptService extends BaseService {
            logger.error("No task dependency", { attemptId: attempt.id });
            return;
          }
+
+          await this.#handleDependencyResume(attempt, completedAttemptIds, tx);
+
          break;
        }
        case "WAIT_FOR_BATCH": {
@@ -136,6 +141,9 @@ export class ResumeAttemptService extends BaseService {
            logger.error("No batch dependency", { attemptId: attempt.id });
            return;
          }
+
+          await this.#handleDependencyResume(attempt, completedAttemptIds, tx);
+
          break;
        }
        default: {
@@ -143,7 +151,8 @@ export class ResumeAttemptService extends BaseService {
        }
      }

-      await this.#handleDependencyResume(attempt, completedAttemptIds, tx);
+      // Prevent infinite restores by failing runs that don't heartbeat after post-restore resume requests
+      await this.#replaceResumeWithFailMessage(attempt.taskRunId, params.type);
    });
  }

@@ -215,7 +224,20 @@ export class ResumeAttemptService extends BaseService {
      executions.push(executionPayload.execution);
    }

-    const updated = await tx.taskRunAttempt.update({
+    await this.#setPostResumeStatuses(attempt, tx);
+
+    socketIo.coordinatorNamespace.emit("RESUME_AFTER_DEPENDENCY", {
+      version: "v1",
+      runId: attempt.taskRunId,
+      attemptId: attempt.id,
+      attemptFriendlyId: attempt.friendlyId,
+      completions,
+      executions,
+    });
+  }
+
+  async #setPostResumeStatuses(attempt: TaskRunAttempt, tx: PrismaClientOrTransaction) {
+    return await tx.taskRunAttempt.update({
      where: {
        id: attempt.id,
      },
@@ -230,14 +252,51 @@ export class ResumeAttemptService extends BaseService {
        },
      },
    });
+  }

-    socketIo.coordinatorNamespace.emit("RESUME_AFTER_DEPENDENCY", {
-      version: "v1",
-      runId: attempt.taskRunId,
-      attemptId: attempt.id,
-      attemptFriendlyId: attempt.friendlyId,
-      completions,
-      executions,
-    });
+  async #replaceResumeWithFailMessage(messageId: string, waitReason: WaitReason) {
+    const currentMessage = await marqs?.readMessage(messageId);
+
+    if (!currentMessage) {
+      logger.debug("No message to replace", { messageId, waitReason });
+      return;
+    }
+
+    const currentBody = SharedQueueMessageBody.safeParse(currentMessage.data);
+
+    if (!currentBody.success) {
+      logger.debug("Invalid message body", { messageId, waitReason, currentBody });
+      return;
+    }
+
+    const currentType = currentBody.data.type;
+
+    if (currentType !== "RESUME" && currentType !== "RESUME_AFTER_DURATION") {
+      logger.debug("Not a resume message", { messageId, waitReason, currentBody });
+      return;
+    }
+
+    let reason = "Worker unresponsive after restore";
+
+    switch (waitReason) {
+      case "WAIT_FOR_DURATION":
+        reason = "Worker unresponsive after waiting for duration";
+        break;
+      case "WAIT_FOR_TASK":
+        reason = "Worker unresponsive after waiting for task";
+        break;
+      case "WAIT_FOR_BATCH":
+        reason = "Worker unresponsive after waiting for batch task";
+        break;
+      default:
+        break;
+    }
+
+    const failMessage: SharedQueueMessageBody = {
+      type: "FAIL",
+      reason,
+    };
+
+    return await marqs?.replaceMessage(messageId, failMessage, undefined, true);
  }
 }
@@ -76,6 +76,15 @@ export class TriggerTaskService extends BaseService {
          immediate: true,
        },
        async (event, traceContext) => {
+          const runFriendlyId = generateFriendlyId("run");
+
+          const payloadPacket = await this.#handlePayloadPacket(
+            body.payload,
+            body.options?.payloadType ?? "application/json",
+            runFriendlyId,
+            environment
+          );
+
          const lockId = taskIdentifierToLockId(taskId);

          const run = await $transaction(this._prisma, async (tx) => {
@@ -105,15 +114,6 @@ export class TriggerTaskService extends BaseService {
            event.setAttribute("queueName", queueName);
            span.setAttribute("queueName", queueName);

-            const runFriendlyId = generateFriendlyId("run");
-
-            const payloadPacket = await this.#handlePayloadPacket(
-              body.payload,
-              body.options?.payloadType ?? "application/json",
-              runFriendlyId,
-              environment
-            );
-
            const taskRun = await tx.taskRun.create({
              data: {
                status: "PENDING",
@@ -1,19 +1,17 @@
+import { trace } from "@opentelemetry/api";
 import { clientWebsocketMessages, serverWebsocketMessages } from "@trigger.dev/core/v3";
+import type { StructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
 import {
+  MessageCatalogToSocketIoEvents,
  ZodMessageHandler,
  ZodMessageSender,
-  MessageCatalogToSocketIoEvents,
 } from "@trigger.dev/core/v3/zodMessageHandler";
-import type { StructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
 import { Evt } from "evt";
 import { randomUUID } from "node:crypto";
+import type { DisconnectReason, Namespace, Socket } from "socket.io";
+import { env } from "~/env.server";
 import { logger } from "~/services/logger.server";
 import { SharedQueueConsumer } from "./marqs/sharedQueueConsumer.server";
-import type { DisconnectReason, Namespace, Socket } from "socket.io";
-import { ROOT_CONTEXT, Span, SpanKind, trace } from "@opentelemetry/api";
-import { env } from "~/env.server";
-
-const tracer = trace.getTracer("sharedQueueConsumerPool");

 interface SharedQueueConsumerPoolOptions {
  sender: ZodMessageSender<typeof serverWebsocketMessages>;
@@ -22,22 +20,8 @@ interface SharedQueueConsumerPoolOptions {

 class SharedQueueConsumerPool {
  #consumers: SharedQueueConsumer[];
-  #span: Span;

  constructor(opts: SharedQueueConsumerPoolOptions) {
-    this.#span = tracer.startSpan(
-      "SharedQueueConsumerPool()",
-      {
-        kind: SpanKind.CONSUMER,
-        attributes: {
-          "pool.size": opts.poolSize,
-        },
-      },
-      ROOT_CONTEXT
-    );
-
-    const spanContext = trace.setSpan(ROOT_CONTEXT, this.#span);
-
    this.#consumers = Array(opts.poolSize)
      .fill(null)
      .map(
@@ -45,7 +29,6 @@ class SharedQueueConsumerPool {
          new SharedQueueConsumer(opts.sender, {
            interval: env.SHARED_QUEUE_CONSUMER_INTERVAL_MS,
            nextTickInterval: env.SHARED_QUEUE_CONSUMER_NEXT_TICK_INTERVAL_MS,
-            parentContext: spanContext,
          })
      );
  }
@@ -56,7 +39,6 @@ class SharedQueueConsumerPool {

  async stop() {
    await Promise.allSettled(this.#consumers.map((consumer) => consumer.stop()));
-    this.#span.end();
  }
 }

@@ -100,7 +82,7 @@ export class SharedSocketConnection {
      },
    });

-    logger.log("Starting SharedQueueConsumer pool", {
+    logger.debug("Starting SharedQueueConsumer pool", {
      poolSize: opts.poolSize ?? this._defaultPoolSize,
    });

@@ -125,7 +125,7 @@ function getTracer() {
    );
  } else {
    if (env.INTERNAL_OTEL_TRACE_LOGGING_ENABLED === "1") {
-      console.log(`🔦 Tracer: Logger exporter enabled`);
+      console.log(`🔦 Tracer: Logger exporter enabled (sampling = ${samplingRate})`);

      const loggerExporter = new LoggerSpanExporter();

@@ -16,6 +16,8 @@
    "typecheck": "tsc -p ./tsconfig.check.json",
    "db:seed": "node prisma/seed.js",
    "db:seed:local": "ts-node prisma/seed.ts",
+    "build:db:populate": "esbuild --platform=node --bundle --minify --format=cjs ./prisma/populate.ts --outdir=prisma",
+    "db:populate": "node prisma/populate.js --",
    "generate:sourcemaps": "remix build --sourcemap",
    "clean:sourcemaps": "run-s clean:sourcemaps:*",
    "clean:sourcemaps:public": "rimraf ./build/**/*.map",
@@ -58,6 +60,7 @@
    "@opentelemetry/sdk-trace-base": "^1.22.0",
    "@opentelemetry/sdk-trace-node": "^1.22.0",
    "@opentelemetry/semantic-conventions": "^1.22.0",
+    "@popperjs/core": "^2.11.8",
    "@prisma/instrumentation": "^5.11.0",
    "@radix-ui/react-alert-dialog": "^1.0.4",
    "@radix-ui/react-dialog": "^1.0.3",
@@ -105,6 +108,7 @@
    "cronstrue": "^2.21.0",
    "cross-env": "^7.0.3",
    "cuid": "^2.1.8",
+    "dotenv": "^16.4.5",
    "emails": "workspace:*",
    "evt": "^2.4.13",
    "express": "^4.18.1",
@@ -136,10 +140,11 @@
    "react-collapse": "^5.1.1",
    "react-dom": "^18.2.0",
    "react-hotkeys-hook": "^4.4.1",
+    "react-popper": "^2.3.0",
    "react-resizable-panels": "^2.0.9",
    "react-stately": "^3.29.1",
    "react-use": "^17.4.0",
-    "recharts": "^2.8.0",
+    "recharts": "^2.12.6",
    "remix-auth": "^3.6.0",
    "remix-auth-email-link": "2.0.2",
    "remix-auth-github": "^1.6.0",
@@ -227,4 +232,4 @@
  "engines": {
    "node": ">=16.0.0"
  }
-}
+}
@@ -0,0 +1,101 @@
+// Bulk adds data to the database for testing
+// Call it like this
+// 1. pnpm run build:db:populate
+// 2. pnpm run db:populate -- --projectRef=proj_liazlkfgmfcusswwgohl --taskIdentifier=child-task --runCount=100000
+import { generateFriendlyId } from "~/v3/friendlyIdentifiers";
+import { prisma } from "../app/db.server";
+
+async function populate() {
+  if (process.env.NODE_ENV !== "development") {
+    return;
+  }
+
+  const projectRef = getArg("projectRef");
+  if (!projectRef) {
+    throw new Error("projectRef is required");
+  }
+
+  const project = await prisma.project.findUnique({
+    include: {
+      environments: true,
+    },
+    where: {
+      externalRef: projectRef,
+    },
+  });
+  if (!project) {
+    throw new Error("Project not found");
+  }
+
+  const taskIdentifier = getArg("taskIdentifier");
+  if (!taskIdentifier) {
+    throw new Error("taskIdentifier is required");
+  }
+
+  const runCount = parseInt(getArg("runCount") || "100");
+
+  const task = await prisma.backgroundWorkerTask.findFirst({
+    where: {
+      projectId: project.id,
+      slug: taskIdentifier,
+    },
+    orderBy: {
+      createdAt: "desc",
+    },
+  });
+
+  if (!task) {
+    throw new Error("Task not found");
+  }
+
+  const runs = await prisma.taskRun.createMany({
+    data: Array(runCount)
+      .fill(0)
+      .map((_, index) => {
+        const friendlyId = generateFriendlyId("run");
+
+        return {
+          status: "CANCELED",
+          number: index + 1,
+          friendlyId,
+          runtimeEnvironmentId: project.environments[randomIndex(project.environments)].id,
+          projectId: project.id,
+          taskIdentifier,
+          payload: JSON.stringify({ foo: "bar" }),
+          traceId: "traceId",
+          spanId: "spanId",
+          queue: "task/${taskIdentifier}",
+        };
+      }),
+    skipDuplicates: true,
+  });
+
+  console.log(`Added ${runs.count} runs`);
+}
+
+function getArg(name: string) {
+  const args = process.argv.slice(2);
+
+  let value = "";
+
+  args.forEach((val) => {
+    if (val.startsWith(`--${name}=`)) {
+      value = val.split("=")[1];
+    }
+  });
+
+  return !value ? undefined : value;
+}
+
+function randomIndex<T>(array: T[]) {
+  return Math.floor(Math.random() * array.length);
+}
+
+populate()
+  .catch((e) => {
+    console.error(e);
+    process.exit(1);
+  })
+  .finally(async () => {
+    await prisma.$disconnect();
+  });
@@ -21,6 +21,7 @@ module.exports = {
    "random-words",
    "superjson",
  ],
+  browserNodeBuiltinsPolyfill: { modules: { path: true, os: true, crypto: true } },
  watchPaths: async () => {
    return [
      "../../packages/core/src/**/*",
@@ -88,7 +88,10 @@ describe("SimpleWeightedChoiceStrategy", () => {

    const nextSelection = await stategy.nextCandidateSelection("parentQueue");

-    expect(nextSelection).toEqual({ range: [3, 6], selectionId: expect.any(String) });
+    expect(nextSelection).toEqual({
+      range: { offset: 3, count: 3 },
+      selectionId: expect.any(String),
+    });

    // Now pass some queues that have some capacity
    const chosenQueue2 = stategy.chooseQueue(
@@ -129,7 +132,10 @@ describe("SimpleWeightedChoiceStrategy", () => {

    const nextSelection2 = await stategy.nextCandidateSelection("parentQueue");

-    expect(nextSelection2).toEqual({ range: [0, 3], selectionId: expect.any(String) });
+    expect(nextSelection2).toEqual({
+      range: { offset: 6, count: 3 },
+      selectionId: expect.any(String),
+    });
  });

  it("should adjust the next filter range only if passed the maximum number of queues", async () => {
@@ -167,7 +173,10 @@ describe("SimpleWeightedChoiceStrategy", () => {

    const nextSelection = await stategy.nextCandidateSelection("parentQueue");

-    expect(nextSelection).toEqual({ range: [0, 3], selectionId: expect.any(String) });
+    expect(nextSelection).toEqual({
+      range: { offset: 0, count: 3 },
+      selectionId: expect.any(String),
+    });
  });

  it("should adjust the next candidate range ONLY for the matching parent queue", async () => {
@@ -182,7 +191,7 @@ describe("SimpleWeightedChoiceStrategy", () => {
          queue: "queue1",
          age: 4497,
          capacities: {
-            queue: { current: 10, limit: 10 },
+            queue: { current: 0, limit: 10 },
            env: { current: 0, limit: 10 },
            org: { current: 0, limit: 10 },
          },
@@ -210,15 +219,21 @@ describe("SimpleWeightedChoiceStrategy", () => {
      "selectionId"
    );

-    expect(chosenQueue).toEqual({ abort: true });
+    expect(chosenQueue).toEqual("queue1");

    const nextSelection = await stategy.nextCandidateSelection("parentQueue2");

-    expect(nextSelection).toEqual({ range: [0, 3], selectionId: expect.any(String) });
+    expect(nextSelection).toEqual({
+      range: { offset: 0, count: 3 },
+      selectionId: expect.any(String),
+    });

    const nextSelection2 = await stategy.nextCandidateSelection("parentQueue");

-    expect(nextSelection2).toEqual({ range: [3, 6], selectionId: expect.any(String) });
+    expect(nextSelection2).toEqual({
+      range: { offset: 3, count: 3 },
+      selectionId: expect.any(String),
+    });

    const chosenQueue2 = stategy.chooseQueue(
      [
@@ -250,14 +265,52 @@ describe("SimpleWeightedChoiceStrategy", () => {
          },
        },
      ],
-      "parentQueue2",
+      "parentQueue",
      "selectionId"
    );

-    expect(chosenQueue2).toEqual("queue3");
+    expect(chosenQueue2).toEqual("queue2");

-    const nextSelection3 = await stategy.nextCandidateSelection("parentQueue2");
+    const nextSelection3 = await stategy.nextCandidateSelection("parentQueue");

-    expect(nextSelection3).toEqual({ range: [0, 3], selectionId: expect.any(String) });
+    expect(nextSelection3).toEqual({
+      range: { offset: 6, count: 3 },
+      selectionId: expect.any(String),
+    });
+
+    // Not passed 3 queues, so the range should be reset (we've reached the end)
+    const chosenQueue3 = stategy.chooseQueue(
+      [
+        {
+          queue: "queue1",
+          age: 4497,
+          capacities: {
+            queue: { current: 0, limit: 10 },
+            env: { current: 0, limit: 10 },
+            org: { current: 0, limit: 10 },
+          },
+        },
+        {
+          queue: "queue2",
+          age: 19670,
+          capacities: {
+            queue: { current: 0, limit: 10 },
+            env: { current: 0, limit: 10 },
+            org: { current: 0, limit: 10 },
+          },
+        },
+      ],
+      "parentQueue",
+      "selectionId"
+    );
+
+    expect(chosenQueue3).toEqual("queue2");
+
+    const nextSelection4 = await stategy.nextCandidateSelection("parentQueue");
+
+    expect(nextSelection4).toEqual({
+      range: { offset: 0, count: 3 },
+      selectionId: expect.any(String),
+    });
  });
 });
@@ -29,6 +29,8 @@ This task will retry 10 times with exponential backoff.
 One way to gain reliability is to break your work into smaller tasks and [trigger](/v3/triggering) them from each other. Each task can have its own retrying behavior:

 ```ts /trigger/multiple-tasks.ts
+import { task } from "@trigger.dev/sdk/v3";
+
 export const myTask = task({
  id: "my-task",
  retry: {
@@ -66,6 +68,8 @@ We provide some useful functions that you can use to retry smaller parts of a ta
 You can retry a block of code that can throw an error, with the same retry settings as a task.

 ```ts /trigger/retry-on-throw.ts
+import { task, logger, retry } from "@trigger.dev/sdk/v3";
+
 export const retryOnThrow = task({
  id: "retry-on-throw",
  run: async (payload: any) => {
@@ -100,6 +104,8 @@ You can use `fetch`, `axios`, or any other library in your code.
 But we do provide a convenient function to perform HTTP requests with conditional retrying based on the response:

 ```ts /trigger/retry-fetch.ts
+import { task, logger, retry } from "@trigger.dev/sdk/v3";
+
 export const taskWithFetchRetries = task({
  id: "task-with-fetch-retries",
  run: async ({ payload, ctx }) => {
@@ -193,6 +199,8 @@ In this complicated example:
 - If we've run out of requests or tokens we retry at the time specified in the headers.

 ```ts
+import { task } from "@trigger.dev/sdk/v3";
+
 export const openaiTask = task({
  id: "openai-task",
  retry: {
@@ -7,8 +7,10 @@ This simple GitHub action file will deploy you Trigger.dev tasks when new code i

 <Warning>The deploy step will fail if any version mismatches are detected. Please see the [version pinning](/v3/github-actions#version-pinning) section for more details.</Warning>

-```yaml .github/workflows/release-trigger.yml
-name: Deploy to Trigger.dev
+<CodeGroup>
+
+```yaml .github/workflows/release-trigger-prod.yml
+name: Deploy to Trigger.dev (prod)

 on:
  push:
@@ -39,6 +41,37 @@ jobs:
          npx trigger.dev@beta deploy
 ```

+
+```yaml .github/workflows/release-trigger-staging.yml
+name: Deploy to Trigger.dev (staging)
+
+# Requires manually calling the workflow from a branch / commit to deploy to staging
+on:
+  workflow_dispatch:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Use Node.js 20.x
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20.x"
+      
+      - name: Install dependencies
+        run: npm install
+
+      - name: 🚀 Deploy Trigger.dev
+        env:
+          TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }}
+        run: |
+          npx trigger.dev@beta deploy --env staging
+```
+</CodeGroup>
+
 If you already have a GitHub action file, you can just add the final step "🚀 Deploy Trigger.dev" to your existing file.

 You need to add the `TRIGGER_ACCESS_TOKEN` secret to your repository. You can create a new access token by going to your profile page and then clicking on the "Personal Access Tokens" tab.
@@ -79,7 +79,7 @@ export const taskWithRetries = task({
    maxTimeoutInMs: 30_000,
    randomize: false,
  },
-  run: async ({ payload, ctx }) => {
+  run: async (payload: any, { ctx }) => {
    //...
  },
 });
@@ -99,7 +99,7 @@ export const oneAtATime = task({
  queue: {
    concurrencyLimit: 1,
  },
-  run: async ({ payload, ctx }) => {
+  run: async (payload: any, { ctx }) => {
    //...
  },
 });
@@ -116,7 +116,7 @@ export const heavyTask = task({
    cpu: 2,
    memory: 4,
  },
-  run: async ({ payload, ctx }) => {
+  run: async (payload: any, { ctx }) => {
    //...
  },
 });
@@ -124,27 +124,143 @@ export const heavyTask = task({

 ### `init` function

-This function is called before a run attempt.
+This function is called before a run attempt:
+
+```ts /trigger/init.ts
+export const taskWithInit = task({
+  id: "task-with-init",
+  init: async (payload, { ctx }) => {
+    //...
+  },
+  run: async (payload: any, { ctx }) => {
+    //...
+  },
+});
+```
+
+You can also return data from the `init` function that will be available in the params of the `run`, `cleanup`, `onSuccess`, and `onFailure` functions.
+
+```ts /trigger/init-return.ts
+export const taskWithInitReturn = task({
+  id: "task-with-init-return",
+  init: async (payload, { ctx }) => {
+    return { someData: "someValue" };
+  },
+  run: async (payload: any, { ctx, init }) => {
+    console.log(init.someData); // "someValue"
+  },
+});
+```

 ### `cleanup` function

-This function is called after a run attempt has succeeded or failed.
+This function is called after the `run` function is executed, regardless of whether the run was successful or not. It's useful for cleaning up resources, logging, or other side effects.
+
+```ts /trigger/cleanup.ts
+export const taskWithCleanup = task({
+  id: "task-with-cleanup",
+  cleanup: async (payload, { ctx }) => {
+    //...
+  },
+  run: async (payload: any, { ctx }) => {
+    //...
+  },
+});
+```

 ### `middleware` function

 This function is called before the `run` function, it allows you to wrap the run function with custom code. For more information [read the guide](/v3/middleware).

+### `onStart` function
+
+When a task run starts, the `onStart` function is called. It's useful for sending notifications, logging, and other side effects. This function will only be called one per run (not per retry). If you want to run code before each retry, use the `init` function.
+
+```ts /trigger/on-start.ts
+export const taskWithOnStart = task({
+  id: "task-with-on-start",
+  onStart: async (payload, { ctx }) => {
+    //...
+  },
+  run: async (payload: any, { ctx }) => {
+    //...
+  },
+});
+```
+
+You can also define an `onStart` function in your `trigger.config.ts` file to get notified when any task starts.
+
+```ts trigger.config.ts
+import type { TriggerConfig } from "@trigger.dev/sdk/v3";
+
+export const config: TriggerConfig = {
+  onStart: async (payload, { ctx }) => {
+    console.log("Task started", ctx.task.id);
+  },
+};
+```
+
 ### `onSuccess` function

-When a task attempt succeeds, the `onSuccess` function is called. It's useful for sending notifications, logging, or other side effects.
+When a task run succeeds, the `onSuccess` function is called. It's useful for sending notifications, logging, syncing state to your database, or other side effects.

-<Snippet file="coming-soon-slim.mdx" />
+```ts /trigger/on-success.ts
+export const taskWithOnSuccess = task({
+  id: "task-with-on-success",
+  onSuccess: async (payload, output, { ctx }) => {
+    //...
+  },
+  run: async (payload: any, { ctx }) => {
+    //...
+  },
+});
+```

-### `onError` function
+You can also define an `onSuccess` function in your `trigger.config.ts` file to get notified when any task succeeds.

-When a task attempt fails, the `onError` function is called. It's useful for sending notifications, logging, or other side effects.
+```ts trigger.config.ts
+import type { TriggerConfig } from "@trigger.dev/sdk/v3";

-<Snippet file="coming-soon-slim.mdx" />
+export const config: TriggerConfig = {
+  onSuccess: async (payload, output, { ctx }) => {
+    console.log("Task succeeded", ctx.task.id);
+  },
+};
+```
+
+### `onFailure` function
+
+When a task run fails, the `onFailure` function is called. It's useful for sending notifications, logging, or other side effects. It will only be executed once the task run has exhausted all its retries.
+
+```ts /trigger/on-failure.ts
+export const taskWithOnFailure = task({
+  id: "task-with-on-failure",
+  onFailure: async (payload, error, { ctx }) => {
+    //...
+  },
+  run: async (payload: any, { ctx }) => {
+    //...
+  },
+});
+```
+
+You can also define an `onFailure` function in your `trigger.config.ts` file to get notified when any task fails.
+
+```ts trigger.config.ts
+import type { TriggerConfig } from "@trigger.dev/sdk/v3";
+
+export const config: TriggerConfig = {
+  onFailure: async (payload, error, { ctx }) => {
+    console.log("Task failed", ctx.task.id);
+  },
+};
+```
+
+### `handleError` functions
+
+You can define a function that will be called when an error is thrown in the `run` function, that allows you to control how the error is handled and whether the task should be retried.
+
+Read more about `handleError` in our [Errors and Retrying guide](/v3/errors-retrying).

 ## Next steps

@@ -31,6 +31,46 @@ export const config: TriggerConfig = {

 Most of the time you don't need to change anything in this file, or if you do then we will tell you when you the run the CLI command.

+## Global initialization
+
+You can run code before any task is run by adding a `init` function to your `trigger.config.ts` file.
+
+```ts trigger.config.ts
+import type { TriggerConfig } from "@trigger.dev/sdk/v3";
+
+export const config: TriggerConfig = {
+  //..other stuff
+  init: async (payload, { ctx }) => {
+    console.log("I run before any task is run");
+  },
+};
+```
+
+You'll have access to the run payload and the context object. Currently you cannot return anything from this function.
+
+## Lifecycle functions
+
+You can add lifecycle functions to get notified when any task starts, succeeds, or fails using `onStart`, `onSuccess` and `onFailure`:
+
+```ts trigger.config.ts
+import type { TriggerConfig } from "@trigger.dev/sdk/v3";
+
+export const config: TriggerConfig = {
+  //..other stuff
+  onSuccess: async (payload, output, { ctx }) => {
+    console.log("Task succeeded", ctx.task.id);
+  },
+  onFailure: async (payload, error, { ctx }) => {
+    console.log("Task failed", ctx.task.id);
+  },
+  onStart: async (payload, { ctx }) => {
+    console.log("Task started", ctx.task.id);
+  },
+};
+```
+
+Read more about task lifecycle functions in the [tasks overview](/v3/tasks-overview).
+
 ## Instrumentations

 We use OpenTelemetry (OTEL) for our run logs. This means you get a lot of information about your tasks with no effort. But you probably want to add more information to your logs. For example, here's all the Prisma calls automatically logged:
@@ -92,7 +132,6 @@ Prisma works by generating a client from your `prisma.schema` file. This means y

  <Step title="package.json postinstall `prisma generate`">

-
    <CodeGroup>

    ```json default path
@@ -140,6 +179,87 @@ Prisma works by generating a client from your `prisma.schema` file. This means y

 </Steps>

+## TypeORM support
+
+We support using TypeORM with Trigger. You can use decorators in your entities and then use them in your tasks. Here's an example:
+
+```ts orm/index.ts
+import "reflect-metadata";
+import { DataSource } from "typeorm";
+import { Entity, Column, PrimaryColumn } from "typeorm";
+
+@Entity()
+export class Photo {
+  @PrimaryColumn()
+  id!: number;
+
+  @Column()
+  name!: string;
+
+  @Column()
+  description!: string;
+
+  @Column()
+  filename!: string;
+
+  @Column()
+  views!: number;
+
+  @Column()
+  isPublished!: boolean;
+}
+
+export const AppDataSource = new DataSource({
+  type: "postgres",
+  host: "localhost",
+  port: 5432,
+  username: "postgres",
+  password: "postgres",
+  database: "my-database",
+  entities: [Photo],
+  synchronize: true,
+  logging: false,
+});
+```
+
+And then in your trigger.config.ts file you can initialize the datasource using the `onStart` lifecycle function option:
+
+```ts trigger.config.ts
+import type { TriggerConfig } from "@trigger.dev/sdk/v3";
+import { AppDataSource } from "@/trigger/orm";
+
+export const config: TriggerConfig = {
+  // ... other options here
+  onStart: async (payload, { ctx }) => {
+    await AppDataSource.initialize();
+  },
+};
+```
+
+Now you are ready to use this in your tasks:
+
+```ts
+import { task } from "@trigger.dev/sdk/v3";
+import { AppDataSource, Photo } from "./orm";
+
+export const taskThatUsesDecorators = task({
+  id: "task-that-uses-decorators",
+  run: async (payload: { message: string }) => {
+    console.log("Creating a photo...");
+
+    const photo = new Photo();
+    photo.id = 2;
+    photo.name = "Me and Bears";
+    photo.description = "I am near polar bears";
+    photo.filename = "photo-with-bears.jpg";
+    photo.views = 1;
+    photo.isPublished = true;
+
+    await AppDataSource.manager.save(photo);
+  },
+});
+```
+
 ## Troubleshooting

 If you have an issue with bundling let us know on [Discord](https://trigger.dev/discord) or [via email](https://trigger.dev/contact).
@@ -1,5 +1,48 @@
 # @trigger.dev/airtable

+## 3.0.0-beta.25
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.25
+- @trigger.dev/sdk@3.0.0-beta.25
+
+## 3.0.0-beta.24
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.24
+- @trigger.dev/sdk@3.0.0-beta.24
+
+## 3.0.0-beta.23
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.23
+- @trigger.dev/sdk@3.0.0-beta.23
+
+## 3.0.0-beta.22
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.22
+- @trigger.dev/sdk@3.0.0-beta.22
+
+## 3.0.0-beta.21
+
+### Patch Changes
+
+- Updated dependencies [9491a1649]
+  - @trigger.dev/sdk@3.0.0-beta.21
+  - @trigger.dev/integration-kit@3.0.0-beta.21
+
+## 3.0.0-beta.20
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.20
+- @trigger.dev/sdk@3.0.0-beta.20
+
 ## 3.0.0-beta.19

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@trigger.dev/airtable",
-  "version": "3.0.0-beta.19",
+  "version": "3.0.0-beta.25",
  "description": "Trigger.dev integration for airtable",
  "main": "./dist/index.js",
  "types": "./dist/index.d.ts",
@@ -25,8 +25,8 @@
    "typecheck": "tsc --noEmit"
  },
  "dependencies": {
-    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.19",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.19",
+    "@trigger.dev/integration-kit": "workspace:^3.0.0-beta.25",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.25",
    "airtable": "^0.12.1",
    "zod": "3.22.3"
  },
@@ -1,5 +1,48 @@
 # @trigger.dev/github

+## 3.0.0-beta.25
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.25
+- @trigger.dev/sdk@3.0.0-beta.25
+
+## 3.0.0-beta.24
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.24
+- @trigger.dev/sdk@3.0.0-beta.24
+
+## 3.0.0-beta.23
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.23
+- @trigger.dev/sdk@3.0.0-beta.23
+
+## 3.0.0-beta.22
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.22
+- @trigger.dev/sdk@3.0.0-beta.22
+
+## 3.0.0-beta.21
+
+### Patch Changes
+
+- Updated dependencies [9491a1649]
+  - @trigger.dev/sdk@3.0.0-beta.21
+  - @trigger.dev/integration-kit@3.0.0-beta.21
+
+## 3.0.0-beta.20
+
+### Patch Changes
+
+- @trigger.dev/integration-kit@3.0.0-beta.20
+- @trigger.dev/sdk@3.0.0-beta.20
+
 ## 3.0.0-beta.19

 ### Patch Changes
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Eric Allam	bf4e505c70	Release 3.0.0-beta.25	2024-05-02 16:00:43 +01:00
github-actions[bot]	c8538c236c	chore: Update version for release (beta) (#1080 ) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>	2024-05-02 15:58:42 +01:00
Eric Allam	ff6e2461d5	Add admin tooltips to make support/debugging easier for admin users	2024-05-02 14:12:16 +01:00
Eric Allam	3ed016bc2b	Fixed SharedQueueConsumer internal otel logging	2024-05-02 12:31:55 +01:00
Eric Allam	68dd3804a7	Improve the fair queue selection strategy by paginating through all the queues in the parent queue	2024-05-02 11:56:35 +01:00
Eric Allam	95f0c7b780	increase queue selection count to 36	2024-05-01 22:41:50 +01:00
Eric Allam	339c39d739	Fix type issues and unit tests	2024-05-01 22:23:38 +01:00
Eric Allam	bc0ca01319	Fix fair queue priority strategy off-by one error	2024-05-01 22:13:52 +01:00
Eric Allam	c37c822314	Use locked package versions when resolving dependencies in deployed workers	2024-05-01 15:51:39 +01:00
Eric Allam	2dfe191c55	Fixed some docs that were missing imports	2024-05-01 14:47:50 +01:00
Eric Allam	e337b21650	Add a postInstall option to allow running scripts after dependencies have been installed in deployed images	2024-05-01 14:16:52 +01:00
Eric Allam	9e5382951b	Improve the display of non-object return types in the run trace viewer	2024-05-01 13:52:05 +01:00
nicktrn	ed03f4bc15	fix lockfile	2024-05-01 11:33:10 +01:00
github-actions[bot]	9feb0f70b0	chore: Update version for release (beta) (#1079 ) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>	2024-05-01 11:32:15 +01:00
nicktrn	25de9e59af	run workflows	2024-05-01 10:57:09 +01:00
nicktrn	83dc871550	add changeset	2024-05-01 10:48:27 +01:00
nicktrn	62700245a3	v3: fix consecutive waits (#1073 ) * fix spacing for delete hints * don't try to resume deps on wait resume * sending duration wait resumes is not an error anymore * set correct status with new wait resume flow * cancel checkpoint schema v2 * don't mix messages and schemas * prevent unintended case fallthrough in tree view * completely switch to platform-led duration wait resumes * prevent infinite restores * some entries for the catalog * add pg to additional packages * add checkpoint safe timeout * prevent duplicate spans after restore * wait for post start * add sdk version to deploy tab * fail on impossible checkpoint scenarios * remove debug logs	2024-05-01 10:33:30 +01:00
Matt Aitken	6ce820cb45	Test tasks that return different types	2024-04-30 19:07:44 +01:00
Matt Aitken	0f0a6884e8	Environment variables pasting uses dotenv (#1075 ) * Add dotenv package to the webapp frontend, required some polyfills * Use dotenv to parse the pasted env vars. Make the panel wider on larger screens	2024-04-30 14:23:49 +01:00
Matt Aitken	7ff8f0ebab	Task and run page improvements (#1076 ) * TaskListPresenter: if there are no tasks then don’t do stats queries * RunListPresenter, use BasePresenter and the read replica * Added populate script * Simplified the Runs list query, added live timer * Added TaskRun indexes for the RunList * Status can’t be null now we’re using the TaskRun status * Use defer so the page loads and shows a spinner * Improved the loading style * Get rid of latest run info from the tasks table super slow * Fix for the activity graph tooltip getting clipped * Added a code comment crediting the GitHub issue with the portal fix * Add search to the tasks list * Padding * Fix for the schedules columns not being UTC * Remove unused function	2024-04-30 14:19:50 +01:00
Matt Aitken	68455c796a	Fixes/run filtered keyboard nav (#1074 ) * Ensure each switch condition returns a state * Fix for scrolling when filtered * Fix for up/down navigation when filtered	2024-04-29 18:52:46 +01:00
Eric Allam	b0a2c42e0e	Add some additional logging around nacking messages	2024-04-29 17:53:38 +01:00
Andreas Thomas	cac3c32f6a	docs: add 'retry' import in code snippet (#1071 ) * docs: add 'retry' import in code snippet * docs: import all primitives	2024-04-29 17:42:36 +01:00
Matt Aitken	ed8d24fd3d	Run page performance improvements (#1072 ) * lotsOfLogs task now outputs much larger logs * Removed tree view collapse/expand animation * JSDocs for useDebounce * WIP moving filtering into the state * Reworked the reducer to do the filtering	2024-04-29 17:05:26 +01:00
nicktrn	d0ef36260a	stop trying to pull init image when already present	2024-04-29 15:33:46 +01:00
Eric Allam	8eb68dd852	Fix pnpm lock file	2024-04-29 14:50:28 +01:00
github-actions[bot]	a42037da03	chore: Update version for release (beta) (#1070 ) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>	2024-04-29 14:48:01 +01:00
Eric Allam	43bc7ed94e	Hoist uncaughtException handler to the top of workers to better report error messages	2024-04-29 14:22:55 +01:00
Matt Aitken	4fdb7f8288	Bulk insert environment variables (#1069 ) * Fix for #1066. Correct environment username if dev * WIP changing the form * WIP on the form * WIP on repository * Adding environment variables en masse is working * WIP on pasting * useList hook with reducer * Got bulk insert working with pasting… it was a pain * Allow overwriting of values * Set a max height on the new env var form	2024-04-28 18:31:31 +01:00
Matt Aitken	37b9b056c4	Upload payload packets outside of the db transaction	2024-04-28 18:18:33 +01:00
Matt Aitken	801c86bf73	Added search to the test tasks list	2024-04-28 16:10:46 +01:00
Matt Aitken	a1de11a001	Default the test page to the “DEV” tab	2024-04-28 15:59:45 +01:00
Praveen Pendyala	29e9e372ee	Added example of github workflow for deploy to staging (#1068 )	2024-04-26 18:39:34 +01:00
Eric Allam	affc128161	Try to fix prisma generate errors	2024-04-26 18:28:17 +01:00
Eric Allam	40ba8ad0ee	Fix the test page when a scheduled task has recent run payloads that aren’t scheduled payloads	2024-04-26 18:20:04 +01:00
Matt Aitken	96168eb383	Fixes for test page problems	2024-04-26 17:45:06 +01:00
Eric Allam	2c68473cc8	Fix pnpm lock file	2024-04-26 17:14:11 +01:00
github-actions[bot]	469c8a2532	chore: Update version for release (beta) (#1067 ) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>	2024-04-26 17:13:20 +01:00
Eric Allam	ebeb790522	Add typescript as a dependency so the esbuild-decorator will work even when running in npx	2024-04-26 17:00:14 +01:00
Matt Aitken	e765240370	Latest lockfile	2024-04-26 14:37:46 +01:00
github-actions[bot]	38770a91c7	chore: Update version for release (beta) (#1061 ) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>	2024-04-26 14:35:30 +01:00
Matt Aitken	de188a22c5	Option for users to opt-in to a staging environment (#1064 )	2024-04-26 11:29:20 +01:00
Matt Aitken	fae93ac747	New tasks page (with activity) and multi-user fixes (#1063 ) * Fix for “overridden” spelling mistake * Make the task function names grey instead of yellow * The task page now shows each task once * Blankline * WIP on activity bars * Correct size of the graphs * Add data for all days, even if there are no runs * Activity graph is working and has a tooltip * No activity label * Got the bar bg working * Activity chart now has a bg * Added a compound id to make the activity graphs appear faster * Improvements to the activity graph * Tried to disable the bg from animating * Added read replica support to the app, using Nick’s code from his batching PR * Added the read replica to BasePresenter * Added the queued and running columns to the Tasks list * Added avg duration column * Included completed but failed runs in the average duration * Get rid of the old useDevEnvironment hook and remove API keys from the projects query * Unified getting environment info to using a function and doing it in presenters * Removed the path from the tasks table * Onboarding for dev for 2nd+ user. Improved environment sorting. * Removed log * Improved the spacing * Enable staging for new v3 projects	2024-04-26 10:44:39 +01:00
Eric Allam	c3f6557eb6	Create publish preprelease script	2024-04-25 11:26:14 +01:00
Eric Allam	9491a1649c	v3: Support decorators (#1060 ) * Adds support for `emitDecoratorMetadata: true` and `experimentalDecorators: true` in your tsconfig * Implement task.onSuccess/onFailure and config.onSuccess/onFailure * Added onStart and more docs for lifecycle functions * Use onStart instead of init for TypeORM	2024-04-25 11:17:13 +01:00
Eric Allam	0a5aa2dc15	Fix slow job retries by optimizing how errored tasks are deleted	2024-04-24 22:07:12 +01:00
Eric Allam	4a1a5b2e4a	fix pnpm lock file	2024-04-24 15:07:22 +01:00
github-actions[bot]	503882762c	chore: Update version for release (beta) (#1059 ) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>	2024-04-24 15:06:25 +01:00
Eric Allam	e3db257397	Fix error stack traces	2024-04-24 14:18:53 +01:00
Eric Allam	de1cc868e3	Fix dev CLI output when not printing update messages	2024-04-24 14:06:49 +01:00
Eric Allam	ff7fa9e19a	Clear out errored tasks when re-attempting a job run (instead of using the cached error)	2024-04-24 13:00:02 +01:00