Release v4.0.5 (#2531)

* chore: Update version for release

* Release v4.0.5

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: nicktrn <55853254+nicktrn@users.noreply.github.com>

.env.example

@@ -13,6 +13,11 @@ APP_ORIGIN=http://localhost:3030
 ELECTRIC_ORIGIN=http://localhost:3060
 NODE_ENV=development
 
+# Clickhouse
+CLICKHOUSE_URL=http://default:password@localhost:8123
+RUN_REPLICATION_CLICKHOUSE_URL=http://default:password@localhost:8123
+RUN_REPLICATION_ENABLED=1
+
 # Set this to UTC because Node.js uses the system timezone
 TZ="UTC"
 

.vscode/settings.json

@@ -6,5 +6,6 @@
     "**/node_modules/**": true,
     "packages/cli-v3/e2e": true
   },
-  "vitest.disableWorkspaceWarning": true
+  "vitest.disableWorkspaceWarning": true,
+  "typescript.experimental.useTsgo": false
 }

README.md

@@ -13,6 +13,7 @@
 
 [![Twitter Follow](https://img.shields.io/twitter/follow/triggerdotdev?style=social)](https://twitter.com/triggerdotdev)
 [![Discord](https://img.shields.io/discord/1066956501299777596?logo=discord&logoColor=white&color=7289da)](https://discord.gg/nkqV9xBYWy)
+[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/triggerdotdev/trigger.dev)
 [![GitHub stars](https://img.shields.io/github/stars/triggerdotdev/trigger.dev?style=social)](https://github.com/triggerdotdev/trigger.dev)
 
 </div>

apps/supervisor/src/env.ts

@@ -90,6 +90,7 @@ const Env = z.object({
   KUBERNETES_MEMORY_REQUEST_MIN_GB: z.coerce.number().min(0).default(0),
   KUBERNETES_MEMORY_REQUEST_RATIO: z.coerce.number().min(0).max(1).default(1), // Ratio of memory limit, so 1 = 100% of memory limit
   KUBERNETES_MEMORY_OVERHEAD_GB: z.coerce.number().min(0).optional(), // Optional memory overhead to add to the limit in GB
+  KUBERNETES_SCHEDULER_NAME: z.string().optional(), // Custom scheduler name for pods
 
   // Placement tags settings
   PLACEMENT_TAGS_ENABLED: BoolEnv.default(false),

apps/supervisor/src/services/failedPodHandler.ts

@@ -25,6 +25,7 @@ export class FailedPodHandler {
 
   private readonly informer: Informer<V1Pod>;
   private readonly reconnectIntervalMs: number;
+  private reconnecting = false;
 
   // Metrics
   private readonly register: Registry;
@@ -250,21 +251,48 @@ export class FailedPodHandler {
   }
 
   private makeOnError(informerName: string) {
-    return () => this.onError(informerName);
+    return (err?: unknown) => this.onError(informerName, err);
   }
 
-  private async onError(informerName: string) {
+  private async onError(informerName: string, err?: unknown) {
     if (!this.isRunning) {
       this.logger.warn("onError: informer not running");
       return;
     }
 
-    this.logger.error("error event fired", { informerName });
-    this.informerEventsTotal.inc({ namespace: this.namespace, verb: "error" });
+    // Guard against multiple simultaneous reconnections
+    if (this.reconnecting) {
+      this.logger.debug("onError: reconnection already in progress, skipping", {
+        informerName,
+      });
+      return;
+    }
 
-    // Reconnect on errors
-    await setTimeout(this.reconnectIntervalMs);
-    await this.informer.start();
+    this.reconnecting = true;
+
+    try {
+      const error = err instanceof Error ? err : undefined;
+      this.logger.error("error event fired", {
+        informerName,
+        error: error?.message,
+        errorType: error?.name,
+      });
+      this.informerEventsTotal.inc({ namespace: this.namespace, verb: "error" });
+
+      // Reconnect on errors
+      await setTimeout(this.reconnectIntervalMs);
+      await this.informer.start();
+    } catch (handlerError) {
+      const error = handlerError instanceof Error ? handlerError : undefined;
+      this.logger.error("onError: reconnection attempt failed", {
+        informerName,
+        error: error?.message,
+        errorType: error?.name,
+        errorStack: error?.stack,
+      });
+    } finally {
+      this.reconnecting = false;
+    }
   }
 
   private makeOnConnect(informerName: string) {

apps/supervisor/src/workloadManager/kubernetes.ts

@@ -274,6 +274,11 @@ export class KubernetesWorkloadManager implements WorkloadManager {
       restartPolicy: "Never",
       automountServiceAccountToken: false,
       imagePullSecrets: this.getImagePullSecrets(),
+      ...(env.KUBERNETES_SCHEDULER_NAME
+        ? {
+            schedulerName: env.KUBERNETES_SCHEDULER_NAME,
+          }
+        : {}),
       ...(env.KUBERNETES_WORKER_NODETYPE_LABEL
         ? {
             nodeSelector: {
@@ -302,6 +307,7 @@ export class KubernetesWorkloadManager implements WorkloadManager {
       envtype: this.#envTypeToLabelValue(opts.envType),
       org: opts.orgId,
       project: opts.projectId,
+      machine: opts.machine.name,
     };
   }
 

apps/webapp/.env

@@ -0,0 +1 @@
+../../.env

apps/webapp/.gitignore

@@ -9,7 +9,8 @@ node_modules
 
 /app/styles/tailwind.css
 
-
+# Ensure the .env symlink is not removed by accident
+!.env
 
 # Storybook build outputs
 build-storybook.log

apps/webapp/app/components/navigation/SideMenu.tsx

@@ -13,13 +13,14 @@ import {
   GlobeAmericasIcon,
   IdentificationIcon,
   KeyIcon,
+  PencilSquareIcon,
   PlusIcon,
   RectangleStackIcon,
   ServerStackIcon,
   Squares2X2Icon,
   UsersIcon,
 } from "@heroicons/react/20/solid";
-import { useNavigation } from "@remix-run/react";
+import { Link, useNavigation } from "@remix-run/react";
 import { useEffect, useRef, useState, type ReactNode } from "react";
 import simplur from "simplur";
 import { BranchEnvironmentIconSmall } from "~/assets/icons/EnvironmentIcons";
@@ -397,9 +398,15 @@ function ProjectSelector({
       >
         <div className="flex flex-col gap-2 bg-charcoal-750 p-2">
           <div className="flex items-center gap-2.5">
-            <div className="box-content size-10 overflow-clip rounded-sm bg-charcoal-800">
+            <Link
+              to={organizationSettingsPath(organization)}
+              className="group relative box-content size-10 overflow-clip rounded-sm bg-charcoal-800"
+            >
               <Avatar avatar={organization.avatar} size={2.5} orgName={organization.title} />
-            </div>
+              <div className="absolute inset-0 z-10 grid h-full w-full place-items-center bg-black/50 opacity-0 transition group-hover:opacity-100">
+                <PencilSquareIcon className="size-5 text-text-bright" />
+              </div>
+            </Link>
             <div className="space-y-0.5">
               <Paragraph variant="small/bright">{organization.title}</Paragraph>
               <div className="flex items-baseline gap-2">

apps/webapp/app/components/primitives/Buttons.tsx

@@ -276,7 +276,7 @@ export function ButtonContent(props: ButtonContentPropsType) {
       <TooltipProvider>
         <Tooltip>
           <TooltipTrigger asChild>{buttonContent}</TooltipTrigger>
-          <TooltipContent className="text-dimmed flex items-center gap-3 py-1.5 pl-2.5 pr-3 text-xs">
+          <TooltipContent className="flex items-center gap-3 py-1.5 pl-2.5 pr-3 text-xs text-text-bright">
             {tooltip} {shortcut && renderShortcutKey()}
           </TooltipContent>
         </Tooltip>
@@ -298,19 +298,17 @@ export const Button = forwardRef<HTMLButtonElement, ButtonPropsType>(
     const innerRef = useRef<HTMLButtonElement>(null);
     useImperativeHandle(ref, () => innerRef.current as HTMLButtonElement);
 
-    if (props.shortcut) {
-      useShortcutKeys({
-        shortcut: props.shortcut,
-        action: (e) => {
-          if (innerRef.current) {
-            innerRef.current.click();
-            e.preventDefault();
-            e.stopPropagation();
-          }
-        },
-        disabled,
-      });
-    }
+    useShortcutKeys({
+      shortcut: props.shortcut,
+      action: (e) => {
+        if (innerRef.current) {
+          innerRef.current.click();
+          e.preventDefault();
+          e.stopPropagation();
+        }
+      },
+      disabled: disabled || !props.shortcut,
+    });
 
     return (
       <button
@@ -345,16 +343,16 @@ export const LinkButton = ({
   ...props
 }: LinkPropsType) => {
   const innerRef = useRef<HTMLAnchorElement>(null);
-  if (props.shortcut) {
-    useShortcutKeys({
-      shortcut: props.shortcut,
-      action: () => {
-        if (innerRef.current) {
-          innerRef.current.click();
-        }
-      },
-    });
-  }
+  
+  useShortcutKeys({
+    shortcut: props.shortcut,
+    action: () => {
+      if (innerRef.current) {
+        innerRef.current.click();
+      }
+    },
+    disabled: disabled || !props.shortcut,
+  });
 
   if (disabled) {
     return (

apps/webapp/app/components/primitives/CopyableText.tsx

@@ -8,10 +8,12 @@ export function CopyableText({
   value,
   copyValue,
   className,
+  asChild,
 }: {
   value: string;
   copyValue?: string;
   className?: string;
+  asChild?: boolean;
 }) {
   const [isHovered, setIsHovered] = useState(false);
   const { copy, copied } = useCopy(copyValue ?? value);
@@ -35,6 +37,7 @@ export function CopyableText({
             <span
               className={cn(
                 "ml-1 flex size-6 items-center justify-center rounded border border-charcoal-650 bg-charcoal-750",
+                asChild && "p-1",
                 copied
                   ? "text-green-500"
                   : "text-text-dimmed hover:border-charcoal-600 hover:bg-charcoal-700 hover:text-text-bright"
@@ -50,6 +53,7 @@ export function CopyableText({
           content={copied ? "Copied!" : "Copy"}
           className="font-sans"
           disableHoverableContent
+          asChild={asChild}
         />
       </span>
     </span>

apps/webapp/app/components/primitives/Popover.tsx

@@ -5,9 +5,10 @@ import { EllipsisVerticalIcon } from "@heroicons/react/24/solid";
 import * as PopoverPrimitive from "@radix-ui/react-popover";
 import * as React from "react";
 import { DropdownIcon } from "~/assets/icons/DropdownIcon";
+import { Link } from "@remix-run/react";
 import * as useShortcutKeys from "~/hooks/useShortcutKeys";
 import { cn } from "~/utils/cn";
-import { type ButtonContentPropsType, LinkButton } from "./Buttons";
+import { type ButtonContentPropsType, Button, ButtonContent } from "./Buttons";
 import { Paragraph, type ParagraphVariant } from "./Paragraph";
 import { ShortcutKey } from "./ShortcutKey";
 import { type RenderIcon } from "./Icon";
@@ -52,42 +53,78 @@ function PopoverSectionHeader({
   );
 }
 
-function PopoverMenuItem({
-  to,
-  icon,
-  title,
-  isSelected,
-  variant = { variant: "small-menu-item" },
-  leadingIconClassName,
-  className,
-}: {
-  to: string;
-  icon?: RenderIcon;
-  title: React.ReactNode;
-  isSelected?: boolean;
-  variant?: ButtonContentPropsType;
-  leadingIconClassName?: string;
-  className?: string;
-}) {
-  return (
-    <LinkButton
-      to={to}
-      variant={variant.variant}
-      LeadingIcon={icon}
-      leadingIconClassName={leadingIconClassName}
-      fullWidth
-      textAlignLeft
-      TrailingIcon={isSelected ? CheckIcon : undefined}
-      className={cn(
+const PopoverMenuItem = React.forwardRef<
+  HTMLButtonElement | HTMLAnchorElement,
+  {
+    to?: string;
+    icon?: RenderIcon;
+    title: React.ReactNode;
+    isSelected?: boolean;
+    variant?: ButtonContentPropsType;
+    leadingIconClassName?: string;
+    className?: string;
+    onClick?: React.MouseEventHandler;
+    disabled?: boolean;
+  }
+>(
+  (
+    {
+      to,
+      icon,
+      title,
+      isSelected,
+      variant = { variant: "small-menu-item" },
+      leadingIconClassName,
+      className,
+      onClick,
+      disabled,
+    },
+    ref
+  ) => {
+    const contentProps = {
+      variant: variant.variant,
+      LeadingIcon: icon,
+      leadingIconClassName,
+      fullWidth: true,
+      textAlignLeft: true,
+      TrailingIcon: isSelected ? CheckIcon : undefined,
+      className: cn(
         "group-hover:bg-charcoal-700",
         isSelected ? "bg-charcoal-750 group-hover:bg-charcoal-600/50" : undefined,
         className
-      )}
-    >
-      {title}
-    </LinkButton>
-  );
-}
+      ),
+    } as const;
+
+    if (to) {
+      return (
+        <Link
+          to={to}
+          ref={ref as React.Ref<HTMLAnchorElement>}
+          className={cn("group/button focus-custom", contentProps.fullWidth ? "w-full" : "")}
+          onClick={onClick as any}
+        >
+          <ButtonContent {...contentProps}>{title}</ButtonContent>
+        </Link>
+      );
+    }
+
+    return (
+      <button
+        type="button"
+        ref={ref as React.Ref<HTMLButtonElement>}
+        onClick={onClick}
+        disabled={disabled}
+        className={cn(
+          "group/button outline-none focus-custom",
+          contentProps.fullWidth ? "w-full" : ""
+        )}
+      >
+        <ButtonContent {...contentProps}>{title}</ButtonContent>
+      </button>
+    );
+  }
+);
+PopoverMenuItem.displayName = "PopoverMenuItem";
 
 function PopoverCustomTrigger({
   isOpen,

apps/webapp/app/components/runs/v3/DeploymentStatus.tsx

@@ -53,6 +53,7 @@ export function DeploymentStatusIcon({
       return (
         <RectangleStackIcon className={cn(deploymentStatusClassNameColor(status), className)} />
       );
+    case "INSTALLING":
     case "BUILDING":
     case "DEPLOYING":
       return <Spinner className={cn(deploymentStatusClassNameColor(status), className)} />;
@@ -78,6 +79,7 @@ export function deploymentStatusClassNameColor(status: WorkerDeploymentStatus):
   switch (status) {
     case "PENDING":
       return "text-charcoal-500";
+    case "INSTALLING":
     case "BUILDING":
     case "DEPLOYING":
       return "text-pending";
@@ -98,6 +100,8 @@ export function deploymentStatusTitle(status: WorkerDeploymentStatus, isBuilt: b
   switch (status) {
     case "PENDING":
       return "Queued…";
+    case "INSTALLING":
+      return "Installing…";
     case "BUILDING":
       return "Building…";
     case "DEPLOYING":
@@ -127,17 +131,21 @@ export function deploymentStatusTitle(status: WorkerDeploymentStatus, isBuilt: b
 // PENDING and CANCELED are not used so are ommited from the UI
 export const deploymentStatuses: WorkerDeploymentStatus[] = [
   "PENDING",
+  "INSTALLING",
   "BUILDING",
   "DEPLOYING",
   "DEPLOYED",
   "FAILED",
   "TIMED_OUT",
+  "CANCELED",
 ];
 
 export function deploymentStatusDescription(status: WorkerDeploymentStatus): string {
   switch (status) {
     case "PENDING":
       return "The deployment is queued and waiting to be processed.";
+    case "INSTALLING":
+      return "The project dependencies are being installed.";
     case "BUILDING":
       return "The code is being built and prepared for deployment.";
     case "DEPLOYING":

apps/webapp/app/components/runs/v3/RollbackDeploymentDialog.tsx

@@ -1,102 +0,0 @@
-import { ArrowPathIcon } from "@heroicons/react/20/solid";
-import { DialogClose } from "@radix-ui/react-dialog";
-import { Form, useNavigation } from "@remix-run/react";
-import { Button } from "~/components/primitives/Buttons";
-import {
-  DialogContent,
-  DialogDescription,
-  DialogFooter,
-  DialogHeader,
-} from "~/components/primitives/Dialog";
-import { SpinnerWhite } from "~/components/primitives/Spinner";
-
-type RollbackDeploymentDialogProps = {
-  projectId: string;
-  deploymentShortCode: string;
-  redirectPath: string;
-};
-
-export function RollbackDeploymentDialog({
-  projectId,
-  deploymentShortCode,
-  redirectPath,
-}: RollbackDeploymentDialogProps) {
-  const navigation = useNavigation();
-
-  const formAction = `/resources/${projectId}/deployments/${deploymentShortCode}/rollback`;
-  const isLoading = navigation.formAction === formAction;
-
-  return (
-    <DialogContent key="rollback">
-      <DialogHeader>Rollback to this deployment?</DialogHeader>
-      <DialogDescription>
-        This deployment will become the default for all future runs. Tasks triggered but not
-        included in this deploy will remain queued until you roll back to or create a new deployment
-        with these tasks included.
-      </DialogDescription>
-      <DialogFooter>
-        <DialogClose asChild>
-          <Button variant="tertiary/medium">Cancel</Button>
-        </DialogClose>
-        <Form
-          action={`/resources/${projectId}/deployments/${deploymentShortCode}/rollback`}
-          method="post"
-        >
-          <Button
-            type="submit"
-            name="redirectUrl"
-            value={redirectPath}
-            variant="primary/medium"
-            LeadingIcon={isLoading ? SpinnerWhite : ArrowPathIcon}
-            disabled={isLoading}
-            shortcut={{ modifiers: ["mod"], key: "enter" }}
-          >
-            {isLoading ? "Rolling back..." : "Rollback deployment"}
-          </Button>
-        </Form>
-      </DialogFooter>
-    </DialogContent>
-  );
-}
-
-export function PromoteDeploymentDialog({
-  projectId,
-  deploymentShortCode,
-  redirectPath,
-}: RollbackDeploymentDialogProps) {
-  const navigation = useNavigation();
-
-  const formAction = `/resources/${projectId}/deployments/${deploymentShortCode}/promote`;
-  const isLoading = navigation.formAction === formAction;
-
-  return (
-    <DialogContent key="promote">
-      <DialogHeader>Promote this deployment?</DialogHeader>
-      <DialogDescription>
-        This deployment will become the default for all future runs not explicitly tied to a
-        specific deployment.
-      </DialogDescription>
-      <DialogFooter>
-        <DialogClose asChild>
-          <Button variant="tertiary/medium">Cancel</Button>
-        </DialogClose>
-        <Form
-          action={`/resources/${projectId}/deployments/${deploymentShortCode}/promote`}
-          method="post"
-        >
-          <Button
-            type="submit"
-            name="redirectUrl"
-            value={redirectPath}
-            variant="primary/medium"
-            LeadingIcon={isLoading ? SpinnerWhite : ArrowPathIcon}
-            disabled={isLoading}
-            shortcut={{ modifiers: ["mod"], key: "enter" }}
-          >
-            {isLoading ? "Promoting..." : "Promote deployment"}
-          </Button>
-        </Form>
-      </DialogFooter>
-    </DialogContent>
-  );
-}

apps/webapp/app/components/runs/v3/RunFilters.tsx

@@ -58,10 +58,10 @@ import { useProject } from "~/hooks/useProject";
 import { useSearchParams } from "~/hooks/useSearchParam";
 import { type loader as queuesLoader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues";
 import { type loader as versionsLoader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.versions";
-import { type loader as tagsLoader } from "~/routes/resources.projects.$projectParam.runs.tags";
+import { type loader as tagsLoader } from "~/routes/resources.environments.$envId.runs.tags";
 import { Button } from "../../primitives/Buttons";
 import { BulkActionTypeCombo } from "./BulkAction";
-import { appliedSummary, FilterMenuProvider, TimeFilter } from "./SharedFilters";
+import { appliedSummary, FilterMenuProvider, TimeFilter, timeFilters } from "./SharedFilters";
 import { AIFilterInput } from "./AIFilterInput";
 import {
   allTaskRunStatuses,
@@ -280,7 +280,7 @@ export function getRunFiltersFromSearchParams(
     bulkId: searchParams.get("bulkId") ?? undefined,
     tags:
       searchParams.getAll("tags").filter((v) => v.length > 0).length > 0
-        ? searchParams.getAll("tags").map((t) => decodeURIComponent(t))
+        ? searchParams.getAll("tags")
         : undefined,
     from: searchParams.get("from") ?? undefined,
     to: searchParams.get("to") ?? undefined,
@@ -810,8 +810,8 @@ function TagsDropdown({
   searchValue: string;
   onClose?: () => void;
 }) {
-  const project = useProject();
-  const { values, replace } = useSearchParams();
+  const environment = useEnvironment();
+  const { values, value, replace } = useSearchParams();
 
   const handleChange = (values: string[]) => {
     clearSearchValue();
@@ -822,6 +822,12 @@ function TagsDropdown({
     });
   };
 
+  const { period, from, to } = timeFilters({
+    period: value("period"),
+    from: value("from"),
+    to: value("to"),
+  });
+
   const tagValues = values("tags").filter((v) => v !== "");
   const selected = tagValues.length > 0 ? tagValues : undefined;
 
@@ -830,25 +836,34 @@ function TagsDropdown({
   useEffect(() => {
     const searchParams = new URLSearchParams();
     if (searchValue) {
-      searchParams.set("name", encodeURIComponent(searchValue));
+      searchParams.set("name", searchValue);
     }
-    fetcher.load(`/resources/projects/${project.slug}/runs/tags?${searchParams}`);
-  }, [searchValue]);
+    if (period) {
+      searchParams.set("period", period);
+    }
+    if (from) {
+      searchParams.set("from", from.getTime().toString());
+    }
+    if (to) {
+      searchParams.set("to", to.getTime().toString());
+    }
+    fetcher.load(`/resources/environments/${environment.id}/runs/tags?${searchParams}`);
+  }, [environment.id, searchValue, period, from?.getTime(), to?.getTime()]);
 
   const filtered = useMemo(() => {
     let items: string[] = [];
     if (searchValue === "") {
-      items = selected ?? [];
+      items = [...(selected ?? [])];
     }
 
     if (fetcher.data === undefined) {
       return matchSorter(items, searchValue);
     }
 
-    items.push(...fetcher.data.tags.map((t) => t.name));
+    items.push(...fetcher.data.tags);
 
     return matchSorter(Array.from(new Set(items)), searchValue);
-  }, [searchValue, fetcher.data]);
+  }, [searchValue, fetcher.data, selected]);
 
   return (
     <SelectProvider value={selected ?? []} setValue={handleChange} virtualFocus={true}>
@@ -958,7 +973,7 @@ function QueuesDropdown({
       const searchParams = new URLSearchParams();
       searchParams.set("per_page", "25");
       if (searchValue) {
-        searchParams.set("query", encodeURIComponent(s));
+        searchParams.set("query", s);
       }
       fetcher.load(
         `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${
@@ -1220,7 +1235,7 @@ function VersionsDropdown({
     (s) => {
       const searchParams = new URLSearchParams();
       if (searchValue) {
-        searchParams.set("query", encodeURIComponent(s));
+        searchParams.set("query", s);
       }
       fetcher.load(
         `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${

apps/webapp/app/components/runs/v3/WaitpointTokenFilters.tsx

@@ -330,7 +330,7 @@ function TagsDropdown({
   useEffect(() => {
     const searchParams = new URLSearchParams();
     if (searchValue) {
-      searchParams.set("name", encodeURIComponent(searchValue));
+      searchParams.set("name", searchValue);
     }
     fetcher.load(
       `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${environment.slug}/waitpoints/tags?${searchParams}`

apps/webapp/app/entry.server.tsx

@@ -240,6 +240,7 @@ import { logger } from "./services/logger.server";
 import { Prisma } from "./db.server";
 import { registerRunEngineEventBusHandlers } from "./v3/runEngineHandlers.server";
 import { remoteBuildsEnabled } from "./v3/remoteImageBuilder.server";
+import { resourceMonitor } from "./services/resourceMonitor.server";
 
 if (env.EVENT_LOOP_MONITOR_ENABLED === "1") {
   eventLoopMonitor.enable();
@@ -250,3 +251,7 @@ if (remoteBuildsEnabled()) {
 } else {
   console.log("🏗️  Local builds enabled");
 }
+
+if (env.RESOURCE_MONITOR_ENABLED === "1") {
+  resourceMonitor.startMonitoring(1000);
+}

apps/webapp/app/env.server.ts

@@ -59,6 +59,7 @@ const EnvironmentSchema = z
     ADMIN_EMAILS: z.string().refine(isValidRegex, "ADMIN_EMAILS must be a valid regex.").optional(),
     REMIX_APP_PORT: z.string().optional(),
     LOGIN_ORIGIN: z.string().default("http://localhost:3030"),
+    LOGIN_RATE_LIMITS_ENABLED: BoolEnv.default(true),
     APP_ORIGIN: z.string().default("http://localhost:3030"),
     API_ORIGIN: z.string().optional(),
     STREAM_ORIGIN: z.string().optional(),
@@ -492,6 +493,7 @@ const EnvironmentSchema = z
     CENTS_PER_RUN: z.coerce.number().default(0),
 
     EVENT_LOOP_MONITOR_ENABLED: z.string().default("1"),
+    RESOURCE_MONITOR_ENABLED: z.string().default("0"),
     MAXIMUM_LIVE_RELOADING_EVENTS: z.coerce.number().int().default(1000),
     MAXIMUM_TRACE_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(25_000),
     MAXIMUM_TRACE_DETAILED_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(10_000),
@@ -752,8 +754,8 @@ const EnvironmentSchema = z
     /** The max number of runs per API call that we'll dequeue in DEV */
     DEV_DEQUEUE_MAX_RUNS_PER_PULL: z.coerce.number().int().default(10),
 
-    /** The maximum concurrent local run processes executing at once in dev */
-    DEV_MAX_CONCURRENT_RUNS: z.coerce.number().int().default(25),
+    /** The maximum concurrent local run processes executing at once in dev. This is a hard limit */
+    DEV_MAX_CONCURRENT_RUNS: z.coerce.number().int().optional(),
 
     /** The CLI should connect to this for dev runs */
     DEV_ENGINE_URL: z.string().default(process.env.APP_ORIGIN ?? "http://localhost:3030"),
@@ -1028,8 +1030,9 @@ const EnvironmentSchema = z
     TASK_EVENT_PARTITIONING_ENABLED: z.string().default("0"),
     TASK_EVENT_PARTITIONED_WINDOW_IN_SECONDS: z.coerce.number().int().default(60), // 1 minute
 
-    QUEUE_SSE_AUTORELOAD_INTERVAL_MS: z.coerce.number().int().default(5_000),
-    QUEUE_SSE_AUTORELOAD_TIMEOUT_MS: z.coerce.number().int().default(60_000),
+    DEPLOYMENTS_AUTORELOAD_POLL_INTERVAL_MS: z.coerce.number().int().default(5_000),
+    BULK_ACTION_AUTORELOAD_POLL_INTERVAL_MS: z.coerce.number().int().default(1_000),
+    QUEUES_AUTORELOAD_POLL_INTERVAL_MS: z.coerce.number().int().default(5_000),
 
     SLACK_BOT_TOKEN: z.string().optional(),
     SLACK_SIGNUP_REASON_CHANNEL_ID: z.string().optional(),
@@ -1099,6 +1102,7 @@ const EnvironmentSchema = z
     RUN_REPLICATION_INSERT_BASE_DELAY_MS: z.coerce.number().int().default(100),
     RUN_REPLICATION_INSERT_MAX_DELAY_MS: z.coerce.number().int().default(2000),
     RUN_REPLICATION_INSERT_STRATEGY: z.enum(["insert", "insert_async"]).default("insert"),
+    RUN_REPLICATION_DISABLE_PAYLOAD_INSERT: z.string().default("0"),
 
     // Clickhouse
     CLICKHOUSE_URL: z.string(),
@@ -1108,6 +1112,27 @@ const EnvironmentSchema = z
     CLICKHOUSE_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"),
     CLICKHOUSE_COMPRESSION_REQUEST: z.string().default("1"),
 
+    EVENTS_CLICKHOUSE_URL: z
+      .string()
+      .optional()
+      .transform((v) => v ?? process.env.CLICKHOUSE_URL),
+    EVENTS_CLICKHOUSE_KEEP_ALIVE_ENABLED: z.string().default("1"),
+    EVENTS_CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS: z.coerce.number().int().optional(),
+    EVENTS_CLICKHOUSE_MAX_OPEN_CONNECTIONS: z.coerce.number().int().default(10),
+    EVENTS_CLICKHOUSE_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"),
+    EVENTS_CLICKHOUSE_COMPRESSION_REQUEST: z.string().default("1"),
+    EVENTS_CLICKHOUSE_BATCH_SIZE: z.coerce.number().int().default(1000),
+    EVENTS_CLICKHOUSE_FLUSH_INTERVAL_MS: z.coerce.number().int().default(1000),
+    EVENTS_CLICKHOUSE_INSERT_STRATEGY: z.enum(["insert", "insert_async"]).default("insert"),
+    EVENTS_CLICKHOUSE_WAIT_FOR_ASYNC_INSERT: z.string().default("1"),
+    EVENTS_CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE: z.coerce.number().int().default(10485760),
+    EVENTS_CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS: z.coerce.number().int().default(5000),
+    EVENT_REPOSITORY_CLICKHOUSE_ROLLOUT_PERCENT: z.coerce.number().optional(),
+    EVENT_REPOSITORY_DEFAULT_STORE: z.enum(["postgres", "clickhouse"]).default("postgres"),
+    EVENTS_CLICKHOUSE_MAX_TRACE_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(25_000),
+    EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(5_000),
+    EVENTS_CLICKHOUSE_MAX_LIVE_RELOADING_SETTING: z.coerce.number().int().default(2000),
+
     // Bootstrap
     TRIGGER_BOOTSTRAP_ENABLED: z.string().default("0"),
     TRIGGER_BOOTSTRAP_WORKER_GROUP_NAME: z.string().optional(),

apps/webapp/app/hooks/useAutoRevalidate.ts

@@ -0,0 +1,48 @@
+import { useRevalidator } from "@remix-run/react";
+import { useEffect } from "react";
+
+type UseAutoRevalidateOptions = {
+  interval?: number; // in milliseconds
+  onFocus?: boolean;
+  disabled?: boolean;
+};
+
+export function useAutoRevalidate(options: UseAutoRevalidateOptions = {}) {
+  const { interval = 5000, onFocus = true, disabled = false } = options;
+  const revalidator = useRevalidator();
+
+  useEffect(() => {
+    if (!interval || interval <= 0 || disabled) return;
+
+    const intervalId = setInterval(() => {
+      if (revalidator.state === "loading") {
+        return;
+      }
+      revalidator.revalidate();
+    }, interval);
+
+    return () => clearInterval(intervalId);
+  }, [interval, disabled]);
+
+  useEffect(() => {
+    if (!onFocus || disabled) return;
+
+    const handleFocus = () => {
+      if (document.visibilityState === "visible" && revalidator.state !== "loading") {
+        revalidator.revalidate();
+      }
+    };
+
+    // Revalidate when the page becomes visible
+    document.addEventListener("visibilitychange", handleFocus);
+    // Revalidate when the window gains focus
+    window.addEventListener("focus", handleFocus);
+
+    return () => {
+      document.removeEventListener("visibilitychange", handleFocus);
+      window.removeEventListener("focus", handleFocus);
+    };
+  }, [onFocus, disabled]);
+
+  return revalidator;
+}

apps/webapp/app/hooks/useSearchParam.ts

@@ -33,12 +33,7 @@ export function useSearchParams() {
   const value = useCallback(
     (param: string) => {
       const search = new URLSearchParams(location.search);
-      const val = search.get(param) ?? undefined;
-      if (val === undefined) {
-        return val;
-      }
-
-      return decodeURIComponent(val);
+      return search.get(param) ?? undefined;
     },
     [location]
   );
@@ -46,8 +41,7 @@ export function useSearchParams() {
   const values = useCallback(
     (param: string) => {
       const search = new URLSearchParams(location.search);
-      const all = search.getAll(param);
-      return all.map((v) => decodeURIComponent(v));
+      return search.getAll(param);
     },
     [location]
   );

apps/webapp/app/models/member.server.ts

@@ -1,5 +1,9 @@
-import { prisma } from "~/db.server";
+import { type Prisma, prisma } from "~/db.server";
 import { createEnvironment } from "./organization.server";
+import { customAlphabet } from "nanoid";
+
+const tokenValueLength = 40;
+const tokenGenerator = customAlphabet("123456789abcdefghijkmnopqrstuvwxyz", tokenValueLength);
 
 export async function getTeamMembersAndInvites({
   userId,
@@ -95,14 +99,19 @@ export async function inviteMembers({
     throw new Error("User does not have access to this organization");
   }
 
-  const created = await prisma.orgMemberInvite.createMany({
-    data: emails.map((email) => ({
-      email,
-      organizationId: org.id,
-      inviterId: userId,
-      role: "MEMBER",
-    })),
-    skipDuplicates: true,
+  const invites = [...new Set(emails)].map(
+    (email) =>
+      ({
+        email,
+        token: tokenGenerator(),
+        organizationId: org.id,
+        inviterId: userId,
+        role: "MEMBER",
+      } satisfies Prisma.OrgMemberInviteCreateManyInput)
+  );
+
+  await prisma.orgMemberInvite.createMany({
+    data: invites,
   });
 
   return await prisma.orgMemberInvite.findMany({
@@ -147,12 +156,19 @@ export async function getUsersInvites({ email }: { email: string }) {
   });
 }
 
-export async function acceptInvite({ userId, inviteId }: { userId: string; inviteId: string }) {
+export async function acceptInvite({
+  user,
+  inviteId,
+}: {
+  user: { id: string; email: string };
+  inviteId: string;
+}) {
   return await prisma.$transaction(async (tx) => {
     // 1. Delete the invite and get the invite details
     const invite = await tx.orgMemberInvite.delete({
       where: {
         id: inviteId,
+        email: user.email,
       },
       include: {
         organization: {
@@ -167,7 +183,7 @@ export async function acceptInvite({ userId, inviteId }: { userId: string; invit
     const member = await tx.orgMember.create({
       data: {
         organizationId: invite.organizationId,
-        userId,
+        userId: user.id,
         role: invite.role,
       },
     });
@@ -187,7 +203,7 @@ export async function acceptInvite({ userId, inviteId }: { userId: string; invit
     // 4. Check for other invites
     const remainingInvites = await tx.orgMemberInvite.findMany({
       where: {
-        email: invite.email,
+        email: user.email,
       },
     });
 
@@ -195,28 +211,29 @@ export async function acceptInvite({ userId, inviteId }: { userId: string; invit
   });
 }
 
-export async function declineInvite({ userId, inviteId }: { userId: string; inviteId: string }) {
+export async function declineInvite({
+  user,
+  inviteId,
+}: {
+  user: { id: string; email: string };
+  inviteId: string;
+}) {
   return await prisma.$transaction(async (tx) => {
     //1. delete invite
     const declinedInvite = await prisma.orgMemberInvite.delete({
       where: {
         id: inviteId,
+        email: user.email,
       },
       include: {
         organization: true,
       },
     });
 
-    //2. get email
-    const user = await prisma.user.findUnique({
-      where: { id: userId },
-      select: { email: true },
-    });
-
-    //3. check for other invites
+    //2. check for other invites
     const remainingInvites = await prisma.orgMemberInvite.findMany({
       where: {
-        email: user!.email,
+        email: user.email,
       },
     });
 
@@ -224,10 +241,11 @@ export async function declineInvite({ userId, inviteId }: { userId: string; invi
   });
 }
 
-export async function resendInvite({ inviteId }: { inviteId: string }) {
+export async function resendInvite({ inviteId, userId }: { inviteId: string; userId: string }) {
   return await prisma.orgMemberInvite.update({
     where: {
       id: inviteId,
+      inviterId: userId,
     },
     data: {
       updatedAt: new Date(),
@@ -241,26 +259,27 @@ export async function resendInvite({ inviteId }: { inviteId: string }) {
 
 export async function revokeInvite({
   userId,
-  slug,
+  orgSlug,
   inviteId,
 }: {
   userId: string;
-  slug: string;
+  orgSlug: string;
   inviteId: string;
 }) {
-  const org = await prisma.organization.findFirst({
-    where: { slug, members: { some: { userId } } },
-  });
-
-  if (!org) {
-    throw new Error("User does not have access to this organization");
-  }
-  const invite = await prisma.orgMemberInvite.delete({
+  const invite = await prisma.orgMemberInvite.findFirst({
     where: {
       id: inviteId,
-      organizationId: org.id,
+      organization: {
+        slug: orgSlug,
+        members: {
+          some: {
+            userId,
+          },
+        },
+      },
     },
     select: {
+      id: true,
       email: true,
       organization: true,
     },
@@ -270,5 +289,11 @@ export async function revokeInvite({
     throw new Error("Invite not found");
   }
 
+  await prisma.orgMemberInvite.delete({
+    where: {
+      id: invite.id,
+    },
+  });
+
   return { email: invite.email, organization: invite.organization };
 }

apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts

@@ -75,6 +75,7 @@ export class ApiRetrieveRunPresenter {
       },
       select: {
         ...commonRunSelect,
+        traceId: true,
         payload: true,
         payloadType: true,
         output: true,

apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts

@@ -103,6 +103,9 @@ export class DeploymentPresenter {
         deployedAt: true,
         createdAt: true,
         startedAt: true,
+        installedAt: true,
+        canceledAt: true,
+        canceledReason: true,
         git: true,
         promotions: {
           select: {
@@ -147,8 +150,11 @@ export class DeploymentPresenter {
         status: deployment.status,
         createdAt: deployment.createdAt,
         startedAt: deployment.startedAt,
+        installedAt: deployment.installedAt,
         builtAt: deployment.builtAt,
         deployedAt: deployment.deployedAt,
+        canceledAt: deployment.canceledAt,
+        canceledReason: deployment.canceledReason,
         tasks: deployment.worker?.tasks,
         label: deployment.promotions?.[0]?.label,
         environment: {

apps/webapp/app/presenters/v3/QueueListPresenter.server.ts

@@ -114,6 +114,9 @@ export class QueueListPresenter extends BasePresenter {
         name: true,
         orderableName: true,
         concurrencyLimit: true,
+        concurrencyLimitBase: true,
+        concurrencyLimitOverriddenAt: true,
+        concurrencyLimitOverriddenBy: true,
         type: true,
         paused: true,
       },
@@ -135,6 +138,17 @@ export class QueueListPresenter extends BasePresenter {
       ),
     ]);
 
+    // Manually "join" the overridden users because there is no way to implement the relationship
+    // in prisma without adding a foreign key constraint
+    const overriddenByIds = queues.map((q) => q.concurrencyLimitOverriddenBy).filter(Boolean);
+    const overriddenByUsers = await this._replica.user.findMany({
+      where: {
+        id: { in: overriddenByIds },
+      },
+    });
+
+    const overriddenByMap = new Map(overriddenByUsers.map((u) => [u.id, u]));
+
     // Transform queues to include running and queued counts
     return queues.map((queue) =>
       toQueueItem({
@@ -144,6 +158,11 @@ export class QueueListPresenter extends BasePresenter {
         running: results[1][queue.name] ?? 0,
         queued: results[0][queue.name] ?? 0,
         concurrencyLimit: queue.concurrencyLimit ?? null,
+        concurrencyLimitBase: queue.concurrencyLimitBase ?? null,
+        concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt ?? null,
+        concurrencyLimitOverriddenBy: queue.concurrencyLimitOverriddenBy
+          ? overriddenByMap.get(queue.concurrencyLimitOverriddenBy) ?? null
+          : null,
         paused: queue.paused,
       })
     );

apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts

@@ -1,12 +1,18 @@
 import { type AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { engine } from "~/v3/runEngine.server";
 import { BasePresenter } from "./basePresenter.server";
-import { type TaskQueueType } from "@trigger.dev/database";
+import { TaskQueue, User, type TaskQueueType } from "@trigger.dev/database";
 import { assertExhaustive } from "@trigger.dev/core";
 import { determineEngineVersion } from "~/v3/engineVersion.server";
-import { type QueueItem, type RetrieveQueueParam } from "@trigger.dev/core/v3";
+import { type Prettify, type QueueItem, type RetrieveQueueParam } from "@trigger.dev/core/v3";
 import { PrismaClientOrTransaction } from "@trigger.dev/database";
 
+export type FoundQueue = Prettify<
+  Omit<TaskQueue, "concurrencyLimitOverriddenBy"> & {
+    concurrencyLimitOverriddenBy?: User | null;
+  }
+>;
+
 /**
  * Shared queue lookup logic used by both QueueRetrievePresenter and PauseQueueService
  */
@@ -16,22 +22,50 @@ export async function getQueue(
   queue: RetrieveQueueParam
 ) {
   if (typeof queue === "string") {
-    return prismaClient.taskQueue.findFirst({
-      where: {
-        friendlyId: queue,
-        runtimeEnvironmentId: environment.id,
-      },
-    });
+    return joinQueueWithUser(
+      prismaClient,
+      await prismaClient.taskQueue.findFirst({
+        where: {
+          friendlyId: queue,
+          runtimeEnvironmentId: environment.id,
+        },
+      })
+    );
   }
 
   const queueName =
     queue.type === "task" ? `task/${queue.name.replace(/^task\//, "")}` : queue.name;
-  return prismaClient.taskQueue.findFirst({
-    where: {
-      name: queueName,
-      runtimeEnvironmentId: environment.id,
-    },
+  return joinQueueWithUser(
+    prismaClient,
+    await prismaClient.taskQueue.findFirst({
+      where: {
+        name: queueName,
+        runtimeEnvironmentId: environment.id,
+      },
+    })
+  );
+}
+
+async function joinQueueWithUser(
+  prismaClient: PrismaClientOrTransaction,
+  queue?: TaskQueue | null
+): Promise<FoundQueue | undefined> {
+  if (!queue) return undefined;
+  if (!queue.concurrencyLimitOverriddenBy) {
+    return {
+      ...queue,
+      concurrencyLimitOverriddenBy: undefined,
+    };
+  }
+
+  const user = await prismaClient.user.findFirst({
+    where: { id: queue.concurrencyLimitOverriddenBy },
   });
+
+  return {
+    ...queue,
+    concurrencyLimitOverriddenBy: user,
+  };
 }
 
 export class QueueRetrievePresenter extends BasePresenter {
@@ -75,6 +109,9 @@ export class QueueRetrievePresenter extends BasePresenter {
         running: results[1]?.[queue.name] ?? 0,
         queued: results[0]?.[queue.name] ?? 0,
         concurrencyLimit: queue.concurrencyLimit ?? null,
+        concurrencyLimitBase: queue.concurrencyLimitBase ?? null,
+        concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt ?? null,
+        concurrencyLimitOverriddenBy: queue.concurrencyLimitOverriddenBy ?? null,
         paused: queue.paused,
       }),
     };
@@ -104,6 +141,9 @@ export function toQueueItem(data: {
   running: number;
   queued: number;
   concurrencyLimit: number | null;
+  concurrencyLimitBase: number | null;
+  concurrencyLimitOverriddenAt: Date | null;
+  concurrencyLimitOverriddenBy: User | null;
   paused: boolean;
 }): QueueItem & { releaseConcurrencyOnWaitpoint: boolean } {
   return {
@@ -113,9 +153,22 @@ export function toQueueItem(data: {
     type: queueTypeFromType(data.type),
     running: data.running,
     queued: data.queued,
-    concurrencyLimit: data.concurrencyLimit,
     paused: data.paused,
+    concurrencyLimit: data.concurrencyLimit,
+    concurrency: {
+      current: data.concurrencyLimit,
+      base: data.concurrencyLimitBase,
+      override: data.concurrencyLimitOverriddenAt ? data.concurrencyLimit : null,
+      overriddenBy: toQueueConcurrencyOverriddenBy(data.concurrencyLimitOverriddenBy),
+      overriddenAt: data.concurrencyLimitOverriddenAt,
+    },
     // TODO: This needs to be removed but keeping this here for now to avoid breaking existing clients
     releaseConcurrencyOnWaitpoint: true,
   };
 }
+
+function toQueueConcurrencyOverriddenBy(user: User | null) {
+  if (!user) return null;
+
+  return user.displayName ?? user.name ?? null;
+}

apps/webapp/app/presenters/v3/RunPresenter.server.ts

@@ -3,9 +3,11 @@ import { createTreeFromFlatItems, flattenTree } from "~/components/primitives/Tr
 import { prisma, type PrismaClient } from "~/db.server";
 import { createTimelineSpanEventsFromSpanEvents } from "~/utils/timelineSpanEvents";
 import { getUsername } from "~/utils/username";
-import { eventRepository } from "~/v3/eventRepository.server";
+import { resolveEventRepositoryForStore } from "~/v3/eventRepository/index.server";
+import { SpanSummary } from "~/v3/eventRepository/eventRepository.types";
 import { getTaskEventStoreTableForRun } from "~/v3/taskEventStore.server";
 import { isFinalRunStatus } from "~/v3/taskStatus";
+import { env } from "~/env.server";
 
 type Result = Awaited<ReturnType<RunPresenter["call"]>>;
 export type Run = Result["run"];
@@ -28,7 +30,6 @@ export class RunPresenter {
   public async call({
     userId,
     projectSlug,
-    organizationSlug,
     environmentSlug,
     runFriendlyId,
     showDeletedLogs,
@@ -36,7 +37,6 @@ export class RunPresenter {
   }: {
     userId: string;
     projectSlug: string;
-    organizationSlug: string;
     environmentSlug: string;
     runFriendlyId: string;
     showDeletedLogs: boolean;
@@ -47,9 +47,11 @@ export class RunPresenter {
         id: true,
         createdAt: true,
         taskEventStore: true,
+        taskIdentifier: true,
         number: true,
         traceId: true,
         spanId: true,
+        parentSpanId: true,
         friendlyId: true,
         status: true,
         startedAt: true,
@@ -93,6 +95,13 @@ export class RunPresenter {
         friendlyId: runFriendlyId,
         project: {
           slug: projectSlug,
+          organization: {
+            members: {
+              some: {
+                userId,
+              },
+            },
+          },
         },
       },
     });
@@ -132,21 +141,56 @@ export class RunPresenter {
       return {
         run: runData,
         trace: undefined,
+        maximumLiveReloadingSetting: env.MAXIMUM_LIVE_RELOADING_EVENTS,
       };
     }
 
+    const eventRepository = resolveEventRepositoryForStore(run.taskEventStore);
+
     // get the events
-    const traceSummary = await eventRepository.getTraceSummary(
+    let traceSummary = await eventRepository.getTraceSummary(
       getTaskEventStoreTableForRun(run),
+      run.runtimeEnvironment.id,
       run.traceId,
       run.rootTaskRun?.createdAt ?? run.createdAt,
       run.completedAt ?? undefined,
       { includeDebugLogs: showDebug }
     );
+
     if (!traceSummary) {
-      return {
-        run: runData,
-        trace: undefined,
+      const spanSummary: SpanSummary = {
+        id: run.spanId,
+        parentId: run.parentSpanId ?? undefined,
+        runId: run.friendlyId,
+        data: {
+          message: run.taskIdentifier,
+          style: { icon: "task", variant: "primary" },
+          events: [],
+          startTime: run.createdAt,
+          duration: 0,
+          isError:
+            run.status === "COMPLETED_WITH_ERRORS" ||
+            run.status === "CRASHED" ||
+            run.status === "EXPIRED" ||
+            run.status === "SYSTEM_FAILURE" ||
+            run.status === "TIMED_OUT",
+          isPartial:
+            run.status === "DELAYED" ||
+            run.status === "PENDING" ||
+            run.status === "PAUSED" ||
+            run.status === "RETRYING_AFTER_FAILURE" ||
+            run.status === "DEQUEUED" ||
+            run.status === "EXECUTING" ||
+            run.status === "WAITING_TO_RESUME",
+          isCancelled: run.status === "CANCELED",
+          isDebug: false,
+          level: "TRACE",
+        },
+      };
+
+      traceSummary = {
+        rootSpan: spanSummary,
+        spans: [spanSummary],
       };
     }
 
@@ -215,7 +259,9 @@ export class RunPresenter {
         queuedDuration: run.startedAt
           ? millisecondsToNanoseconds(run.startedAt.getTime() - run.createdAt.getTime())
           : undefined,
+        overridesBySpanId: traceSummary.overridesBySpanId,
       },
+      maximumLiveReloadingSetting: eventRepository.maximumLiveReloadingSetting,
     };
   }
 }

apps/webapp/app/presenters/v3/RunStreamPresenter.server.ts

@@ -3,7 +3,7 @@ import { eventStream } from "remix-utils/sse/server";
 import { PrismaClient, prisma } from "~/db.server";
 import { logger } from "~/services/logger.server";
 import { throttle } from "~/utils/throttle";
-import { eventRepository } from "~/v3/eventRepository.server";
+import { tracePubSub } from "~/v3/services/tracePubSub.server";
 
 const pingInterval = 1000;
 
@@ -41,7 +41,7 @@ export class RunStreamPresenter {
 
     let pinger: NodeJS.Timeout | undefined = undefined;
 
-    const { unsubscribe, eventEmitter } = await eventRepository.subscribeToTrace(run.traceId);
+    const { unsubscribe, eventEmitter } = await tracePubSub.subscribeToTrace(run.traceId);
 
     return eventStream(request.signal, (send, close) => {
       const safeSend = (args: { event?: string; data: string }) => {

apps/webapp/app/presenters/v3/RunTagListPresenter.server.ts

@@ -1,8 +1,16 @@
+import { RunsRepository } from "~/services/runsRepository/runsRepository.server";
 import { BasePresenter } from "./basePresenter.server";
+import { clickhouseClient } from "~/services/clickhouseInstance.server";
+import { type PrismaClient } from "@trigger.dev/database";
+import { timeFilters } from "~/components/runs/v3/SharedFilters";
 
 export type TagListOptions = {
-  userId?: string;
+  organizationId: string;
+  environmentId: string;
   projectId: string;
+  period?: string;
+  from?: Date;
+  to?: Date;
   //filters
   name?: string;
   //pagination
@@ -17,40 +25,39 @@ export type TagListItem = TagList["tags"][number];
 
 export class RunTagListPresenter extends BasePresenter {
   public async call({
-    userId,
+    organizationId,
+    environmentId,
     projectId,
     name,
+    period,
+    from,
+    to,
     page = 1,
     pageSize = DEFAULT_PAGE_SIZE,
   }: TagListOptions) {
     const hasFilters = Boolean(name?.trim());
 
-    const tags = await this._replica.taskRunTag.findMany({
-      where: {
-        projectId,
-        name: name
-          ? {
-              startsWith: name,
-              mode: "insensitive",
-            }
-          : undefined,
-      },
-      orderBy: {
-        id: "desc",
-      },
-      take: pageSize + 1,
-      skip: (page - 1) * pageSize,
+    const runsRepository = new RunsRepository({
+      clickhouse: clickhouseClient,
+      prisma: this._replica as PrismaClient,
+    });
+
+    const tags = await runsRepository.listTags({
+      organizationId,
+      projectId,
+      environmentId,
+      query: name,
+      period,
+      from: from ? from.getTime() : undefined,
+      to: to ? to.getTime() : undefined,
+      offset: (page - 1) * pageSize,
+      limit: pageSize + 1,
     });
 
     return {
-      tags: tags
-        .map((tag) => ({
-          id: tag.friendlyId,
-          name: tag.name,
-        }))
-        .slice(0, pageSize),
+      tags: tags.tags,
       currentPage: page,
-      hasMore: tags.length > pageSize,
+      hasMore: tags.tags.length > pageSize,
       hasFilters,
     };
   }

apps/webapp/app/presenters/v3/SpanPresenter.server.ts

@@ -10,13 +10,15 @@ import {
 import { AttemptId, getMaxDuration, parseTraceparent } from "@trigger.dev/core/v3/isomorphic";
 import { RUNNING_STATUSES } from "~/components/runs/v3/TaskRunStatus";
 import { logger } from "~/services/logger.server";
-import { eventRepository, rehydrateAttribute } from "~/v3/eventRepository.server";
+import { rehydrateAttribute } from "~/v3/eventRepository/eventRepository.server";
 import { machinePresetFromRun } from "~/v3/machinePresets.server";
 import { getTaskEventStoreTableForRun, type TaskEventStoreTable } from "~/v3/taskEventStore.server";
 import { isFailedRunStatus, isFinalRunStatus } from "~/v3/taskStatus";
 import { BasePresenter } from "./basePresenter.server";
 import { WaitpointPresenter } from "./WaitpointPresenter.server";
 import { engine } from "~/v3/runEngine.server";
+import { resolveEventRepositoryForStore } from "~/v3/eventRepository/index.server";
+import { IEventRepository, SpanDetail } from "~/v3/eventRepository/eventRepository.types";
 
 type Result = Awaited<ReturnType<SpanPresenter["call"]>>;
 export type Span = NonNullable<NonNullable<Result>["span"]>;
@@ -24,14 +26,16 @@ export type SpanRun = NonNullable<NonNullable<Result>["run"]>;
 type FindRunResult = NonNullable<
   Awaited<ReturnType<InstanceType<typeof SpanPresenter>["findRun"]>>
 >;
-type GetSpanResult = NonNullable<Awaited<ReturnType<(typeof eventRepository)["getSpan"]>>>;
+type GetSpanResult = SpanDetail;
 
 export class SpanPresenter extends BasePresenter {
   public async call({
+    userId,
     projectSlug,
     spanId,
     runFriendlyId,
   }: {
+    userId: string;
     projectSlug: string;
     spanId: string;
     runFriendlyId: string;
@@ -39,6 +43,13 @@ export class SpanPresenter extends BasePresenter {
     const project = await this._replica.project.findFirst({
       where: {
         slug: projectSlug,
+        organization: {
+          members: {
+            some: {
+              userId,
+            },
+          },
+        },
       },
     });
 
@@ -57,6 +68,7 @@ export class SpanPresenter extends BasePresenter {
       },
       where: {
         friendlyId: runFriendlyId,
+        projectId: project.id,
       },
     });
 
@@ -66,14 +78,18 @@ export class SpanPresenter extends BasePresenter {
 
     const { traceId } = parentRun;
 
+    const eventRepository = resolveEventRepositoryForStore(parentRun.taskEventStore);
+
     const eventStore = getTaskEventStoreTableForRun(parentRun);
 
     const run = await this.getRun({
       eventStore,
       traceId,
+      eventRepository,
       spanId,
       createdAt: parentRun.createdAt,
       completedAt: parentRun.completedAt,
+      environmentId: parentRun.runtimeEnvironmentId,
     });
     if (run) {
       return {
@@ -82,15 +98,15 @@ export class SpanPresenter extends BasePresenter {
       };
     }
 
-    //get the run
     const span = await this.#getSpan({
       eventStore,
-      traceId,
       spanId,
+      traceId,
       environmentId: parentRun.runtimeEnvironmentId,
       projectId: parentRun.projectId,
       createdAt: parentRun.createdAt,
       completedAt: parentRun.completedAt,
+      eventRepository,
     });
 
     if (!span) {
@@ -105,30 +121,31 @@ export class SpanPresenter extends BasePresenter {
 
   async getRun({
     eventStore,
+    environmentId,
     traceId,
+    eventRepository,
     spanId,
     createdAt,
     completedAt,
   }: {
     eventStore: TaskEventStoreTable;
+    environmentId: string;
     traceId: string;
+    eventRepository: IEventRepository;
     spanId: string;
     createdAt: Date;
     completedAt: Date | null;
   }) {
-    const span = await eventRepository.getSpan(
+    const originalRunId = await eventRepository.getSpanOriginalRunId(
       eventStore,
+      environmentId,
       spanId,
       traceId,
       createdAt,
       completedAt ?? undefined
     );
 
-    if (!span) {
-      return;
-    }
-
-    const run = await this.findRun({ span, spanId });
+    const run = await this.findRun({ originalRunId, spanId, environmentId });
 
     if (!run) {
       return;
@@ -251,8 +268,9 @@ export class SpanPresenter extends BasePresenter {
       engine: run.engine,
       region,
       workerQueue: run.workerQueue,
+      traceId: run.traceId,
       spanId: run.spanId,
-      isCached: !!span.originalRun,
+      isCached: !!originalRunId,
       machinePreset: machine?.name,
       externalTraceId,
     };
@@ -287,7 +305,15 @@ export class SpanPresenter extends BasePresenter {
     };
   }
 
-  async findRun({ span, spanId }: { span: GetSpanResult; spanId: string }) {
+  async findRun({
+    originalRunId,
+    spanId,
+    environmentId,
+  }: {
+    originalRunId?: string;
+    spanId: string;
+    environmentId: string;
+  }) {
     const run = await this._replica.taskRun.findFirst({
       select: {
         id: true,
@@ -397,12 +423,14 @@ export class SpanPresenter extends BasePresenter {
           },
         },
       },
-      where: span.originalRun
+      where: originalRunId
         ? {
-            friendlyId: span.originalRun,
+            friendlyId: originalRunId,
+            runtimeEnvironmentId: environmentId,
           }
         : {
             spanId,
+            runtimeEnvironmentId: environmentId,
           },
     });
 
@@ -411,6 +439,7 @@ export class SpanPresenter extends BasePresenter {
 
   async #getSpan({
     eventStore,
+    eventRepository,
     traceId,
     spanId,
     environmentId,
@@ -418,6 +447,7 @@ export class SpanPresenter extends BasePresenter {
     createdAt,
     completedAt,
   }: {
+    eventRepository: IEventRepository;
     traceId: string;
     spanId: string;
     environmentId: string;
@@ -428,12 +458,14 @@ export class SpanPresenter extends BasePresenter {
   }) {
     const span = await eventRepository.getSpan(
       eventStore,
+      environmentId,
       spanId,
       traceId,
       createdAt,
       completedAt ?? undefined,
       { includeDebugLogs: true }
     );
+
     if (!span) {
       return;
     }
@@ -445,11 +477,7 @@ export class SpanPresenter extends BasePresenter {
         spanId: true,
         createdAt: true,
         number: true,
-        lockedToVersion: {
-          select: {
-            version: true,
-          },
-        },
+        taskVersion: true,
       },
       where: {
         parentSpanId: spanId,
@@ -457,11 +485,21 @@ export class SpanPresenter extends BasePresenter {
     });
 
     const data = {
-      ...span,
+      spanId: span.spanId,
+      parentId: span.parentId,
+      message: span.message,
+      isError: span.isError,
+      isPartial: span.isPartial,
+      isCancelled: span.isCancelled,
+      level: span.level,
+      startTime: span.startTime,
+      duration: span.duration,
       events: span.events,
+      style: span.style,
       properties: span.properties ? JSON.stringify(span.properties, null, 2) : undefined,
+      entity: span.entity,
+      metadata: span.metadata,
       triggeredRuns,
-      showActionBar: span.show?.actions === true,
     };
 
     switch (span.entity.type) {

apps/webapp/app/presenters/v3/TestTaskPresenter.server.ts

@@ -1,14 +1,17 @@
+import { ClickHouse } from "@internal/clickhouse";
 import { ScheduledTaskPayload, parsePacket, prettyPrintPacket } from "@trigger.dev/core/v3";
 import {
-  type TaskRunTemplate,
   type RuntimeEnvironmentType,
   type TaskRunStatus,
+  type TaskRunTemplate,
+  PrismaClientOrTransaction,
 } from "@trigger.dev/database";
-import { type PrismaClient, prisma, sqlDatabaseSchema } from "~/db.server";
+import parse from "parse-duration";
+import { type PrismaClient } from "~/db.server";
+import { RunsRepository } from "~/services/runsRepository/runsRepository.server";
 import { getTimezones } from "~/utils/timezones.server";
 import { findCurrentWorkerDeployment } from "~/v3/models/workerDeployment.server";
 import { queueTypeFromType } from "./QueueRetrievePresenter.server";
-import parse from "parse-duration";
 
 export type RunTemplate = TaskRunTemplate & {
   scheduledTaskPayload?: ScheduledRun["payload"];
@@ -20,6 +23,8 @@ type TestTaskOptions = {
   environment: {
     id: string;
     type: RuntimeEnvironmentType;
+    projectId: string;
+    organizationId: string;
   };
   taskIdentifier: string;
 };
@@ -111,11 +116,10 @@ export type ScheduledRun = Omit<RawRun, "payload" | "ttl"> & {
 };
 
 export class TestTaskPresenter {
-  #prismaClient: PrismaClient;
-
-  constructor(prismaClient: PrismaClient = prisma) {
-    this.#prismaClient = prismaClient;
-  }
+  constructor(
+    private readonly replica: PrismaClientOrTransaction,
+    private readonly clickhouse: ClickHouse
+  ) {}
 
   public async call({
     userId,
@@ -128,7 +132,7 @@ export class TestTaskPresenter {
         ? (
             await findCurrentWorkerDeployment({ environmentId: environment.id })
           )?.worker?.tasks.find((t) => t.slug === taskIdentifier)
-        : await this.#prismaClient.backgroundWorkerTask.findFirst({
+        : await this.replica.backgroundWorkerTask.findFirst({
             where: {
               slug: taskIdentifier,
               runtimeEnvironmentId: environment.id,
@@ -145,7 +149,7 @@ export class TestTaskPresenter {
     }
 
     const taskQueue = task.queueId
-      ? await this.#prismaClient.taskQueue.findFirst({
+      ? await this.replica.taskQueue.findFirst({
           where: {
             runtimeEnvironmentId: environment.id,
             id: task.queueId,
@@ -159,7 +163,7 @@ export class TestTaskPresenter {
         })
       : undefined;
 
-    const backgroundWorkers = await this.#prismaClient.backgroundWorker.findMany({
+    const backgroundWorkers = await this.replica.backgroundWorker.findMany({
       where: {
         runtimeEnvironmentId: environment.id,
       },
@@ -173,7 +177,7 @@ export class TestTaskPresenter {
       take: 20, // last 20 versions should suffice
     });
 
-    const taskRunTemplates = await this.#prismaClient.taskRunTemplate.findMany({
+    const taskRunTemplates = await this.replica.taskRunTemplate.findMany({
       where: {
         projectId,
         taskSlug: task.slug,
@@ -190,47 +194,55 @@ export class TestTaskPresenter {
     const disableVersionSelection = environment.type === "DEVELOPMENT";
     const allowArbitraryQueues = backgroundWorkers[0]?.engine === "V1";
 
-    const latestRuns = await this.#prismaClient.$queryRaw<RawRun[]>`
-    WITH taskruns AS (
-      SELECT
-          tr.*
-      FROM
-          ${sqlDatabaseSchema}."TaskRun" as tr
-      JOIN
-          ${sqlDatabaseSchema}."BackgroundWorkerTask" as bwt
-      ON
-          tr."taskIdentifier" = bwt.slug
-      WHERE
-          bwt."friendlyId" = ${task.friendlyId} AND
-          tr."runtimeEnvironmentId" = ${environment.id}
-      ORDER BY
-          tr."createdAt" DESC
-      LIMIT 10
-    )
-    SELECT
-        taskr.id,
-        taskr."queue",
-        taskr."friendlyId",
-        taskr."taskIdentifier",
-        taskr."createdAt",
-        taskr.status,
-        taskr.payload,
-        taskr."payloadType",
-        taskr."seedMetadata",
-        taskr."seedMetadataType",
-        taskr."runtimeEnvironmentId",
-        taskr."concurrencyKey",
-        taskr."maxAttempts",
-        taskr."maxDurationInSeconds",
-        taskr."machinePreset",
-        taskr."ttl",
-        taskr."runTags"
-    FROM
-        taskruns AS taskr
-    WHERE
-        taskr."payloadType" = 'application/json' OR taskr."payloadType" = 'application/super+json'
-    ORDER BY
-        taskr."createdAt" DESC;`;
+    // Get the latest runs, for the payloads
+    const runsRepository = new RunsRepository({
+      clickhouse: this.clickhouse,
+      prisma: this.replica as PrismaClient,
+    });
+
+    const runIds = await runsRepository.listRunIds({
+      organizationId: environment.organizationId,
+      environmentId: environment.id,
+      projectId: environment.projectId,
+      tasks: [task.slug],
+      period: "30d",
+      page: {
+        size: 10,
+      },
+    });
+
+    const latestRuns = await this.replica.taskRun.findMany({
+      select: {
+        id: true,
+        queue: true,
+        friendlyId: true,
+        taskIdentifier: true,
+        createdAt: true,
+        status: true,
+        payload: true,
+        payloadType: true,
+        seedMetadata: true,
+        seedMetadataType: true,
+        runtimeEnvironmentId: true,
+        concurrencyKey: true,
+        maxAttempts: true,
+        maxDurationInSeconds: true,
+        machinePreset: true,
+        ttl: true,
+        runTags: true,
+      },
+      where: {
+        id: {
+          in: runIds,
+        },
+        payloadType: {
+          in: ["application/json", "application/super+json"],
+        },
+      },
+      orderBy: {
+        createdAt: "desc",
+      },
+    });
 
     const taskWithEnvironment = {
       id: task.id,
@@ -258,6 +270,12 @@ export class TestTaskPresenter {
               async (r) =>
                 ({
                   ...r,
+                  seedMetadata: r.seedMetadata ?? undefined,
+                  seedMetadataType: r.seedMetadataType ?? undefined,
+                  concurrencyKey: r.concurrencyKey ?? undefined,
+                  maxAttempts: r.maxAttempts ?? undefined,
+                  maxDurationInSeconds: r.maxDurationInSeconds ?? undefined,
+                  machinePreset: r.machinePreset ?? undefined,
                   payload: await prettyPrintPacket(r.payload, r.payloadType),
                   metadata: r.seedMetadata
                     ? await prettyPrintPacket(r.seedMetadata, r.seedMetadataType)
@@ -300,6 +318,12 @@ export class TestTaskPresenter {
                 if (payload.success) {
                   return {
                     ...r,
+                    seedMetadata: r.seedMetadata ?? undefined,
+                    seedMetadataType: r.seedMetadataType ?? undefined,
+                    concurrencyKey: r.concurrencyKey ?? undefined,
+                    maxAttempts: r.maxAttempts ?? undefined,
+                    maxDurationInSeconds: r.maxDurationInSeconds ?? undefined,
+                    machinePreset: r.machinePreset ?? undefined,
                     payload: payload.data,
                     ttlSeconds: r.ttl ? parse(r.ttl, "s") ?? undefined : undefined,
                   } satisfies ScheduledRun;

apps/webapp/app/root.tsx

@@ -20,6 +20,13 @@ export const links: LinksFunction = () => {
   return [{ rel: "stylesheet", href: tailwindStylesheetUrl }];
 };
 
+export const headers = () => ({
+  "Referrer-Policy": "strict-origin-when-cross-origin",
+  "X-Content-Type-Options": "nosniff",
+  "Permissions-Policy":
+    "geolocation=(), microphone=(), camera=(), accelerometer=(), gyroscope=(), magnetometer=(), payment=(), usb=()",
+});
+
 export const meta: MetaFunction = ({ data }) => {
   const typedData = data as UseDataFunctionReturn<typeof loader>;
   return [

apps/webapp/app/routes/_app.github.callback/route.tsx

@@ -1,13 +1,9 @@
-import { type LoaderFunctionArgs, redirect } from "@remix-run/node";
+import { type LoaderFunctionArgs } from "@remix-run/node";
 import { z } from "zod";
 import { validateGitHubAppInstallSession } from "~/services/gitHubSession.server";
 import { linkGitHubAppInstallation, updateGitHubAppInstallation } from "~/services/gitHub.server";
 import { logger } from "~/services/logger.server";
-import {
-  redirectWithErrorMessage,
-  setRequestSuccessMessage,
-  commitSession,
-} from "~/models/message.server";
+import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
 import { tryCatch } from "@trigger.dev/core";
 import { $replica } from "~/db.server";
 import { requireUser } from "~/services/session.server";
@@ -41,7 +37,7 @@ export async function loader({ request }: LoaderFunctionArgs) {
     logger.warn("GitHub App callback with invalid params", {
       queryParams,
     });
-    return redirectWithErrorMessage("/", request, "Failed to install GitHub App");
+    return redirectWithErrorMessage("/", request, "Failed to install GitHub app");
   }
 
   const callbackData = result.data;
@@ -54,7 +50,7 @@ export async function loader({ request }: LoaderFunctionArgs) {
       error: sessionResult.error,
     });
 
-    return redirectWithErrorMessage("/", request, "Failed to install GitHub App");
+    return redirectWithErrorMessage("/", request, "Failed to install GitHub app");
   }
 
   const { organizationId, redirectTo: unsafeRedirectTo } = sessionResult;
@@ -76,7 +72,7 @@ export async function loader({ request }: LoaderFunctionArgs) {
       userId: user.id,
       organizationId,
     });
-    return redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub App");
+    return redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub app");
   }
 
   switch (callbackData.setup_action) {
@@ -89,17 +85,10 @@ export async function loader({ request }: LoaderFunctionArgs) {
         logger.error("Failed to link GitHub App installation", {
           error,
         });
-        return redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub App");
+        return redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub app");
       }
 
-      const session = await setRequestSuccessMessage(request, "GitHub App installed successfully");
-      session.flash("gitHubAppInstalled", true);
-
-      return redirect(redirectTo, {
-        headers: {
-          "Set-Cookie": await commitSession(session),
-        },
-      });
+      return redirectWithSuccessMessage(redirectTo, request, "GitHub App installed successfully");
     }
 
     case "update": {
@@ -112,14 +101,7 @@ export async function loader({ request }: LoaderFunctionArgs) {
         return redirectWithErrorMessage(redirectTo, request, "Failed to update GitHub App");
       }
 
-      const session = await setRequestSuccessMessage(request, "GitHub App updated successfully");
-      session.flash("gitHubAppInstalled", true);
-
-      return redirect(redirectTo, {
-        headers: {
-          "Set-Cookie": await commitSession(session),
-        },
-      });
+      return redirectWithSuccessMessage(redirectTo, request, "GitHub App updated successfully");
     }
 
     case "request": {
@@ -129,17 +111,11 @@ export async function loader({ request }: LoaderFunctionArgs) {
         callbackData,
       });
 
-      const session = await setRequestSuccessMessage(request, "GitHub App installation requested");
-
-      return redirect(redirectTo, {
-        headers: {
-          "Set-Cookie": await commitSession(session),
-        },
-      });
+      return redirectWithSuccessMessage(redirectTo, request, "GitHub App installation requested");
     }
 
     default:
       callbackData satisfies never;
-      return redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub App");
+      return redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub app");
   }
 }

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.bulk-actions.$bulkActionParam/route.tsx

@@ -1,10 +1,9 @@
 import { ArrowPathIcon } from "@heroicons/react/20/solid";
-import { Form, useRevalidator } from "@remix-run/react";
+import { Form } from "@remix-run/react";
 import { type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { tryCatch } from "@trigger.dev/core";
 import type { BulkActionType } from "@trigger.dev/database";
 import { motion } from "framer-motion";
-import { useEffect } from "react";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { z } from "zod";
 import { ExitIcon } from "~/assets/icons/ExitIcon";
@@ -18,8 +17,9 @@ import { Paragraph } from "~/components/primitives/Paragraph";
 import * as Property from "~/components/primitives/PropertyTable";
 import { BulkActionStatusCombo, BulkActionTypeCombo } from "~/components/runs/v3/BulkAction";
 import { UserAvatar } from "~/components/UserProfilePhoto";
+import { env } from "~/env.server";
+import { useAutoRevalidate } from "~/hooks/useAutoRevalidate";
 import { useEnvironment } from "~/hooks/useEnvironment";
-import { useEventSource } from "~/hooks/useEventSource";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
 import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
@@ -72,7 +72,9 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
       throw new Error(error.message);
     }
 
-    return typedjson({ bulkAction: data });
+    const autoReloadPollIntervalMs = env.BULK_ACTION_AUTORELOAD_POLL_INTERVAL_MS;
+
+    return typedjson({ bulkAction: data, autoReloadPollIntervalMs });
   } catch (error) {
     console.error(error);
     throw new Response(undefined, {
@@ -130,30 +132,16 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
 };
 
 export default function Page() {
-  const { bulkAction } = useTypedLoaderData<typeof loader>();
+  const { bulkAction, autoReloadPollIntervalMs } = useTypedLoaderData<typeof loader>();
   const organization = useOrganization();
   const project = useProject();
   const environment = useEnvironment();
 
-  const disabled = bulkAction.status !== "PENDING";
-
-  const streamedEvents = useEventSource(
-    `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${environment.id}/runs/bulkaction/${bulkAction.friendlyId}/stream`,
-    {
-      event: "progress",
-      disabled,
-    }
-  );
-
-  const revalidation = useRevalidator();
-
-  useEffect(() => {
-    if (disabled || streamedEvents === null) {
-      return;
-    }
-
-    revalidation.revalidate();
-  }, [streamedEvents, disabled]);
+  useAutoRevalidate({
+    interval: autoReloadPollIntervalMs,
+    onFocus: true,
+    disabled: bulkAction.status !== "PENDING",
+  });
 
   return (
     <div className="grid h-full max-h-full grid-rows-[2.5rem_2.5rem_1fr_3.25rem] overflow-hidden bg-background-bright">

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.deployments.$deploymentParam/route.tsx

@@ -4,7 +4,6 @@ import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { ExitIcon } from "~/assets/icons/ExitIcon";
 import { GitMetadata } from "~/components/GitMetadata";
 import { RuntimeIcon } from "~/components/RuntimeIcon";
-import { UserAvatar } from "~/components/UserProfilePhoto";
 import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
 import { EnvironmentCombo } from "~/components/environments/EnvironmentLabel";
 import { Badge } from "~/components/primitives/Badge";
@@ -132,7 +131,11 @@ export default function Page() {
                 <Property.Label>Deploy</Property.Label>
                 <Property.Value className="flex items-center gap-2">
                   <span>{deployment.shortCode}</span>
-                  {deployment.label && <Badge variant="outline-rounded">{deployment.label}</Badge>}
+                  {deployment.label && (
+                    <Badge variant="extra-small" className="capitalize">
+                      {deployment.label}
+                    </Badge>
+                  )}
                 </Property.Value>
               </Property.Item>
               <Property.Item>
@@ -155,6 +158,22 @@ export default function Page() {
                   />
                 </Property.Value>
               </Property.Item>
+              {deployment.canceledAt && (
+                <Property.Item>
+                  <Property.Label>Canceled at</Property.Label>
+                  <Property.Value>
+                    <>
+                      <DateTimeAccurate date={deployment.canceledAt} /> UTC
+                    </>
+                  </Property.Value>
+                </Property.Item>
+              )}
+              {deployment.canceledReason && (
+                <Property.Item>
+                  <Property.Label>Cancelation reason</Property.Label>
+                  <Property.Value>{deployment.canceledReason}</Property.Value>
+                </Property.Item>
+              )}
               <Property.Item>
                 <Property.Label>Tasks</Property.Label>
                 <Property.Value>{deployment.tasks ? deployment.tasks.length : "–"}</Property.Value>
@@ -197,6 +216,18 @@ export default function Page() {
                   )}
                 </Property.Value>
               </Property.Item>
+              <Property.Item>
+                <Property.Label>Installed at</Property.Label>
+                <Property.Value>
+                  {deployment.installedAt ? (
+                    <>
+                      <DateTimeAccurate date={deployment.installedAt} /> UTC
+                    </>
+                  ) : (
+                    "–"
+                  )}
+                </Property.Value>
+              </Property.Item>
               <Property.Item>
                 <Property.Label>Built at</Property.Label>
                 <Property.Value>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.deployments/route.tsx

@@ -1,5 +1,18 @@
-import { ArrowUturnLeftIcon, BookOpenIcon } from "@heroicons/react/20/solid";
-import { type MetaFunction, Outlet, useLocation, useNavigate, useParams } from "@remix-run/react";
+import {
+  ArrowPathIcon,
+  ArrowUturnLeftIcon,
+  BookOpenIcon,
+  NoSymbolIcon,
+} from "@heroicons/react/20/solid";
+import {
+  Form,
+  type MetaFunction,
+  Outlet,
+  useLocation,
+  useNavigate,
+  useNavigation,
+  useParams,
+} from "@remix-run/react";
 import { type LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { CogIcon, GitBranchIcon } from "lucide-react";
 import { useEffect } from "react";
@@ -15,7 +28,15 @@ import { MainCenteredContainer, PageBody, PageContainer } from "~/components/lay
 import { Badge } from "~/components/primitives/Badge";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
 import { DateTime } from "~/components/primitives/DateTime";
-import { Dialog, DialogTrigger } from "~/components/primitives/Dialog";
+import { SpinnerWhite } from "~/components/primitives/Spinner";
+import {
+  Dialog,
+  DialogDescription,
+  DialogContent,
+  DialogHeader,
+  DialogTrigger,
+  DialogFooter,
+} from "~/components/primitives/Dialog";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { PaginationControls } from "~/components/primitives/Pagination";
 import { Paragraph } from "~/components/primitives/Paragraph";
@@ -39,10 +60,6 @@ import {
   deploymentStatusDescription,
   deploymentStatuses,
 } from "~/components/runs/v3/DeploymentStatus";
-import {
-  PromoteDeploymentDialog,
-  RollbackDeploymentDialog,
-} from "~/components/runs/v3/RollbackDeploymentDialog";
 import { useEnvironment } from "~/hooks/useEnvironment";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
@@ -61,6 +78,9 @@ import {
 } from "~/utils/pathBuilder";
 import { createSearchParams } from "~/utils/searchParams";
 import { compareDeploymentVersions } from "~/v3/utils/deploymentVersions";
+import { useAutoRevalidate } from "~/hooks/useAutoRevalidate";
+import { env } from "~/env.server";
+import { DialogClose } from "@radix-ui/react-dialog";
 
 export const meta: MetaFunction = () => {
   return [
@@ -116,7 +136,9 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
       ? result.deployments.find((d) => d.version === version)
       : undefined;
 
-    return typedjson({ ...result, selectedDeployment });
+    const autoReloadPollIntervalMs = env.DEPLOYMENTS_AUTORELOAD_POLL_INTERVAL_MS;
+
+    return typedjson({ ...result, selectedDeployment, autoReloadPollIntervalMs });
   } catch (error) {
     console.error(error);
     throw new Response(undefined, {
@@ -137,6 +159,7 @@ export default function Page() {
     selectedDeployment,
     connectedGithubRepository,
     environmentGitHubBranch,
+    autoReloadPollIntervalMs,
   } = useTypedLoaderData<typeof loader>();
   const hasDeployments = totalPages > 0;
 
@@ -144,6 +167,8 @@ export default function Page() {
   const location = useLocation();
   const navigate = useNavigate();
 
+  useAutoRevalidate({ interval: autoReloadPollIntervalMs, onFocus: true });
+
   // If we have a selected deployment from the version param, show it
   useEffect(() => {
     if (selectedDeployment && !deploymentParam) {
@@ -388,7 +413,10 @@ function DeploymentActionsCell({
     compareDeploymentVersions(deployment.version, currentDeployment.version) === -1;
   const canBePromoted = canBeMadeCurrent && !canBeRolledBack;
 
-  if (!canBeRolledBack && !canBePromoted) {
+  const finalStatuses = ["CANCELED", "DEPLOYED", "FAILED", "TIMED_OUT"];
+  const canBeCanceled = !finalStatuses.includes(deployment.status);
+
+  if (!canBeRolledBack && !canBePromoted && !canBeCanceled) {
     return (
       <TableCell to={path} isSelected={isSelected}>
         {""}
@@ -412,7 +440,7 @@ function DeploymentActionsCell({
                   fullWidth
                   textAlignLeft
                 >
-                  Rollback…
+                  Rollback
                 </Button>
               </DialogTrigger>
               <RollbackDeploymentDialog
@@ -432,7 +460,7 @@ function DeploymentActionsCell({
                   fullWidth
                   textAlignLeft
                 >
-                  Promote…
+                  Promote
                 </Button>
               </DialogTrigger>
               <PromoteDeploymentDialog
@@ -442,8 +470,155 @@ function DeploymentActionsCell({
               />
             </Dialog>
           )}
+          {canBeCanceled && (
+            <Dialog>
+              <DialogTrigger asChild>
+                <Button
+                  variant="small-menu-item"
+                  LeadingIcon={NoSymbolIcon}
+                  leadingIconClassName="text-error"
+                  fullWidth
+                  textAlignLeft
+                >
+                  Cancel
+                </Button>
+              </DialogTrigger>
+              <CancelDeploymentDialog
+                projectId={project.id}
+                deploymentShortCode={deployment.shortCode}
+                redirectPath={`${location.pathname}${location.search}`}
+              />
+            </Dialog>
+          )}
         </>
       }
     />
   );
 }
+
+type RollbackDeploymentDialogProps = {
+  projectId: string;
+  deploymentShortCode: string;
+  redirectPath: string;
+};
+
+function RollbackDeploymentDialog({
+  projectId,
+  deploymentShortCode,
+  redirectPath,
+}: RollbackDeploymentDialogProps) {
+  const navigation = useNavigation();
+
+  const formAction = `/resources/${projectId}/deployments/${deploymentShortCode}/rollback`;
+  const isLoading = navigation.formAction === formAction;
+
+  return (
+    <DialogContent key="rollback">
+      <DialogHeader>Rollback to this deployment?</DialogHeader>
+      <DialogDescription>
+        This deployment will become the default for all future runs. Tasks triggered but not
+        included in this deploy will remain queued until you roll back to or create a new deployment
+        with these tasks included.
+      </DialogDescription>
+      <DialogFooter>
+        <DialogClose asChild>
+          <Button variant="tertiary/medium">Cancel</Button>
+        </DialogClose>
+        <Form
+          action={`/resources/${projectId}/deployments/${deploymentShortCode}/rollback`}
+          method="post"
+        >
+          <Button
+            type="submit"
+            name="redirectUrl"
+            value={redirectPath}
+            variant="primary/medium"
+            LeadingIcon={isLoading ? SpinnerWhite : ArrowPathIcon}
+            disabled={isLoading}
+            shortcut={{ modifiers: ["mod"], key: "enter" }}
+          >
+            {isLoading ? "Rolling back..." : "Rollback deployment"}
+          </Button>
+        </Form>
+      </DialogFooter>
+    </DialogContent>
+  );
+}
+
+function PromoteDeploymentDialog({
+  projectId,
+  deploymentShortCode,
+  redirectPath,
+}: RollbackDeploymentDialogProps) {
+  const navigation = useNavigation();
+
+  const formAction = `/resources/${projectId}/deployments/${deploymentShortCode}/promote`;
+  const isLoading = navigation.formAction === formAction;
+
+  return (
+    <DialogContent key="promote">
+      <DialogHeader>Promote this deployment?</DialogHeader>
+      <DialogDescription>
+        This deployment will become the default for all future runs not explicitly tied to a
+        specific deployment.
+      </DialogDescription>
+      <DialogFooter>
+        <DialogClose asChild>
+          <Button variant="tertiary/medium">Cancel</Button>
+        </DialogClose>
+        <Form
+          action={`/resources/${projectId}/deployments/${deploymentShortCode}/promote`}
+          method="post"
+        >
+          <Button
+            type="submit"
+            name="redirectUrl"
+            value={redirectPath}
+            variant="primary/medium"
+            LeadingIcon={isLoading ? SpinnerWhite : ArrowPathIcon}
+            disabled={isLoading}
+            shortcut={{ modifiers: ["mod"], key: "enter" }}
+          >
+            {isLoading ? "Promoting..." : "Promote deployment"}
+          </Button>
+        </Form>
+      </DialogFooter>
+    </DialogContent>
+  );
+}
+
+function CancelDeploymentDialog({
+  projectId,
+  deploymentShortCode,
+  redirectPath,
+}: RollbackDeploymentDialogProps) {
+  const navigation = useNavigation();
+
+  const formAction = `/resources/${projectId}/deployments/${deploymentShortCode}/cancel`;
+  const isLoading = navigation.formAction === formAction;
+
+  return (
+    <DialogContent key="cancel">
+      <DialogHeader>Cancel this deployment?</DialogHeader>
+      <DialogDescription>Canceling a deployment cannot be undone. Are you sure?</DialogDescription>
+      <DialogFooter>
+        <DialogClose asChild>
+          <Button variant="tertiary/medium">Back</Button>
+        </DialogClose>
+        <Form action={formAction} method="post">
+          <Button
+            type="submit"
+            name="redirectUrl"
+            value={redirectPath}
+            variant="danger/medium"
+            LeadingIcon={isLoading ? SpinnerWhite : NoSymbolIcon}
+            disabled={isLoading}
+            shortcut={{ modifiers: ["mod"], key: "enter" }}
+          >
+            {isLoading ? "Canceling..." : "Cancel deployment"}
+          </Button>
+        </Form>
+      </DialogFooter>
+    </DialogContent>
+  );
+}

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx

@@ -1,4 +1,5 @@
 import {
+  AdjustmentsHorizontalIcon,
   ArrowUpCircleIcon,
   BookOpenIcon,
   ChatBubbleLeftEllipsisIcon,
@@ -8,19 +9,14 @@ import {
   RectangleStackIcon,
 } from "@heroicons/react/20/solid";
 import { DialogClose } from "@radix-ui/react-dialog";
-import {
-  Form,
-  useNavigate,
-  useNavigation,
-  useRevalidator,
-  useSearchParams,
-  type MetaFunction,
-} from "@remix-run/react";
+import { Form, useNavigation, useSearchParams, type MetaFunction } from "@remix-run/react";
 import { type ActionFunctionArgs, type LoaderFunctionArgs } from "@remix-run/server-runtime";
 import type { RuntimeEnvironmentType } from "@trigger.dev/database";
+import type { QueueItem } from "@trigger.dev/core/v3/schemas";
 import { useEffect, useState } from "react";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { z } from "zod";
+import { RunsIcon } from "~/assets/icons/RunsIcon";
 import { TaskIconSmall } from "~/assets/icons/TaskIcon";
 import upgradeForQueuesPath from "~/assets/images/queues-dashboard.png";
 import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
@@ -30,13 +26,16 @@ import { Feedback } from "~/components/Feedback";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { BigNumber } from "~/components/metrics/BigNumber";
 import { Badge } from "~/components/primitives/Badge";
-import { Button, ButtonVariant, LinkButton } from "~/components/primitives/Buttons";
+import { Button, LinkButton, type ButtonVariant } from "~/components/primitives/Buttons";
 import { Callout } from "~/components/primitives/Callout";
 import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components/primitives/Dialog";
 import { FormButtons } from "~/components/primitives/FormButtons";
+import { Header3 } from "~/components/primitives/Headers";
+import { Input } from "~/components/primitives/Input";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { PaginationControls } from "~/components/primitives/Pagination";
 import { Paragraph } from "~/components/primitives/Paragraph";
+import { PopoverMenuItem } from "~/components/primitives/Popover";
 import { Spinner } from "~/components/primitives/Spinner";
 import {
   Table,
@@ -55,25 +54,25 @@ import {
   TooltipProvider,
   TooltipTrigger,
 } from "~/components/primitives/Tooltip";
+import { env } from "~/env.server";
+import { useAutoRevalidate } from "~/hooks/useAutoRevalidate";
 import { useEnvironment } from "~/hooks/useEnvironment";
-import { useEventSource } from "~/hooks/useEventSource";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
+import { useThrottle } from "~/hooks/useThrottle";
 import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
 import { findProjectBySlug } from "~/models/project.server";
 import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
+import { getUserById } from "~/models/user.server";
 import { EnvironmentQueuePresenter } from "~/presenters/v3/EnvironmentQueuePresenter.server";
 import { QueueListPresenter } from "~/presenters/v3/QueueListPresenter.server";
 import { requireUserId } from "~/services/session.server";
 import { cn } from "~/utils/cn";
 import { docsPath, EnvironmentParamSchema, v3BillingPath, v3RunsPath } from "~/utils/pathBuilder";
+import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server";
 import { PauseEnvironmentService } from "~/v3/services/pauseEnvironment.server";
 import { PauseQueueService } from "~/v3/services/pauseQueue.server";
 import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route";
-import { Header3 } from "~/components/primitives/Headers";
-import { Input } from "~/components/primitives/Input";
-import { useThrottle } from "~/hooks/useThrottle";
-import { RunsIcon } from "~/assets/icons/RunsIcon";
 
 const SearchParamsSchema = z.object({
   query: z.string().optional(),
@@ -121,9 +120,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
 
     const environmentQueuePresenter = new EnvironmentQueuePresenter();
 
+    const autoReloadPollIntervalMs = env.QUEUES_AUTORELOAD_POLL_INTERVAL_MS;
+
     return typedjson({
       ...queues,
       environment: await environmentQueuePresenter.call(environment),
+      autoReloadPollIntervalMs,
     });
   } catch (error) {
     console.error(error);
@@ -211,34 +213,98 @@ export const action = async ({ request, params }: ActionFunctionArgs) => {
         `Queue ${action === "queue-pause" ? "paused" : "resumed"}`
       );
     }
+    case "queue-override": {
+      const friendlyId = formData.get("friendlyId");
+      const concurrencyLimit = formData.get("concurrencyLimit");
+
+      if (!friendlyId) {
+        return redirectWithErrorMessage(redirectPath, request, "Queue ID is required");
+      }
+
+      if (!concurrencyLimit) {
+        return redirectWithErrorMessage(redirectPath, request, "Concurrency limit is required");
+      }
+
+      const limitNumber = parseInt(concurrencyLimit.toString(), 10);
+      if (isNaN(limitNumber) || limitNumber < 0) {
+        return redirectWithErrorMessage(
+          redirectPath,
+          request,
+          "Concurrency limit must be a valid number"
+        );
+      }
+
+      const user = await getUserById(userId);
+      if (!user) {
+        return redirectWithErrorMessage(redirectPath, request, "User not found");
+      }
+
+      const result = await concurrencySystem.queues.overrideQueueConcurrencyLimit(
+        environment,
+        friendlyId.toString(),
+        limitNumber,
+        user
+      );
+
+      if (!result.isOk()) {
+        return redirectWithErrorMessage(
+          redirectPath,
+          request,
+          "Failed to override queue concurrency limit"
+        );
+      }
+
+      return redirectWithSuccessMessage(
+        redirectPath,
+        request,
+        "Queue concurrency limit overridden"
+      );
+    }
+    case "queue-remove-override": {
+      const friendlyId = formData.get("friendlyId");
+
+      if (!friendlyId) {
+        return redirectWithErrorMessage(redirectPath, request, "Queue ID is required");
+      }
+
+      const result = await concurrencySystem.queues.resetConcurrencyLimit(
+        environment,
+        friendlyId.toString()
+      );
+
+      if (!result.isOk()) {
+        return redirectWithErrorMessage(
+          redirectPath,
+          request,
+          "Failed to reset queue concurrency limit"
+        );
+      }
+
+      return redirectWithSuccessMessage(redirectPath, request, "Queue concurrency limit reset");
+    }
     default:
       return redirectWithErrorMessage(redirectPath, request, "Something went wrong");
   }
 };
 
 export default function Page() {
-  const { environment, queues, success, pagination, code, totalQueues, hasFilters } =
-    useTypedLoaderData<typeof loader>();
+  const {
+    environment,
+    queues,
+    success,
+    pagination,
+    code,
+    totalQueues,
+    hasFilters,
+    autoReloadPollIntervalMs,
+  } = useTypedLoaderData<typeof loader>();
 
   const organization = useOrganization();
   const project = useProject();
   const env = useEnvironment();
   const plan = useCurrentPlan();
 
-  // Reload the page periodically
-  const streamedEvents = useEventSource(
-    `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${env.slug}/queues/stream`,
-    {
-      event: "update",
-    }
-  );
-
-  const revalidation = useRevalidator();
-  useEffect(() => {
-    if (streamedEvents) {
-      revalidation.revalidate();
-    }
-  }, [streamedEvents]);
+  useAutoRevalidate({ interval: autoReloadPollIntervalMs, onFocus: true });
 
   const limitStatus =
     environment.running === environment.concurrencyLimit * environment.burstFactor
@@ -275,17 +341,19 @@ export default function Page() {
               animate
               accessory={
                 <div className="flex items-start gap-1">
+                  {environment.runsEnabled ? <EnvironmentPauseResumeButton env={env} /> : null}
                   <LinkButton
-                    variant="tertiary/small"
+                    variant="secondary/small"
+                    LeadingIcon={RunsIcon}
+                    leadingIconClassName="text-runs"
+                    className="px-2"
                     to={v3RunsPath(organization, project, env, {
                       statuses: ["PENDING"],
                       period: "30d",
                       rootOnly: false,
                     })}
-                  >
-                    View runs
-                  </LinkButton>
-                  {environment.runsEnabled ? <EnvironmentPauseResumeButton env={env} /> : null}
+                    tooltip="View queued runs"
+                  />
                 </div>
               }
               valueClassName={env.paused ? "text-warning" : undefined}
@@ -308,15 +376,17 @@ export default function Page() {
               }
               accessory={
                 <LinkButton
-                  variant="tertiary/small"
+                  variant="secondary/small"
+                  LeadingIcon={RunsIcon}
+                  leadingIconClassName="text-runs"
+                  className="px-2"
                   to={v3RunsPath(organization, project, env, {
                     statuses: ["DEQUEUED", "EXECUTING"],
                     period: "30d",
                     rootOnly: false,
                   })}
-                >
-                  View runs
-                </LinkButton>
+                  tooltip="View runs"
+                />
               }
               compactThreshold={1000000}
             />
@@ -383,7 +453,8 @@ export default function Page() {
                   <TableRow>
                     <TableHeaderCell>Name</TableHeaderCell>
                     <TableHeaderCell alignment="right">Queued</TableHeaderCell>
-                    <TableHeaderCell alignment="right">Running/limit</TableHeaderCell>
+                    <TableHeaderCell alignment="right">Running</TableHeaderCell>
+                    <TableHeaderCell alignment="right">Limit</TableHeaderCell>
                     <TableHeaderCell
                       alignment="right"
                       tooltip={
@@ -409,6 +480,17 @@ export default function Page() {
                               This queue is limited by a concurrency limit set in your code.
                             </Paragraph>
                           </div>
+                          <div className="space-y-0.5">
+                            <Header3>Override</Header3>
+                            <Paragraph
+                              variant="small"
+                              className="!text-wrap text-text-dimmed"
+                              spacing
+                            >
+                              This queue's concurrency limit has been manually overridden from the
+                              dashboard or API.
+                            </Paragraph>
+                          </div>
                         </div>
                       }
                     >
@@ -423,7 +505,7 @@ export default function Page() {
                   {queues.length > 0 ? (
                     queues.map((queue) => {
                       const limit = queue.concurrencyLimit ?? environment.concurrencyLimit;
-                      const isAtLimit = queue.running === limit;
+                      const isAtLimit = queue.running >= limit;
                       const queueFilterableName = `${queue.type === "task" ? "task/" : ""}${
                         queue.name
                       }`;
@@ -459,6 +541,18 @@ export default function Page() {
                               <span className={queue.paused ? "opacity-50" : undefined}>
                                 {queue.name}
                               </span>
+                              {queue.concurrency?.overriddenAt ? (
+                                <SimpleTooltip
+                                  button={
+                                    <Badge variant="extra-small" className="text-text-bright">
+                                      Concurrency limit overridden
+                                    </Badge>
+                                  }
+                                  content="This queue's concurrency limit has been manually overridden from the dashboard or API."
+                                  className="max-w-xs"
+                                  disableHoverableContent
+                                />
+                              ) : null}
                               {queue.paused ? (
                                 <Badge variant="extra-small" className="text-warning">
                                   Paused
@@ -473,31 +567,50 @@ export default function Page() {
                           </TableCell>
                           <TableCell
                             alignment="right"
-                            className={queue.paused ? "opacity-50" : undefined}
+                            className={cn(
+                              "w-[1%] tabular-nums",
+                              queue.paused ? "opacity-50" : undefined
+                            )}
                           >
                             {queue.queued}
                           </TableCell>
                           <TableCell
                             alignment="right"
                             className={cn(
-                              queue.paused ? "tabular-nums opacity-50" : undefined,
+                              "w-[1%] tabular-nums",
+                              queue.paused ? "opacity-50" : undefined,
                               queue.running > 0 && "text-text-bright",
                               isAtLimit && "text-warning"
                             )}
                           >
-                            {queue.running}/
-                            <span className={cn("tabular-nums", isAtLimit && "text-warning")}>
-                              {limit}
-                            </span>
+                            {queue.running}
                           </TableCell>
                           <TableCell
                             alignment="right"
                             className={cn(
+                              "w-[1%] tabular-nums",
                               queue.paused ? "opacity-50" : undefined,
-                              isAtLimit && "text-warning"
+                              queue.concurrency?.overriddenAt && "font-medium text-text-bright"
                             )}
                           >
-                            {queue.concurrencyLimit ? "User" : "Environment"}
+                            {limit}
+                          </TableCell>
+                          <TableCell
+                            alignment="right"
+                            className={cn(
+                              "w-[1%]",
+                              queue.paused ? "opacity-50" : undefined,
+                              isAtLimit && "text-warning",
+                              queue.concurrency?.overriddenAt && "font-medium text-text-bright"
+                            )}
+                          >
+                            {queue.concurrency?.overriddenAt ? (
+                              <span className="text-text-bright">Override</span>
+                            ) : queue.concurrencyLimit ? (
+                              "User"
+                            ) : (
+                              "Environment"
+                            )}
                           </TableCell>
                           <TableCellMenu
                             isSticky
@@ -524,50 +637,43 @@ export default function Page() {
                                     showTooltip={false}
                                   />
                                 )}
-                                <LinkButton
-                                  variant="minimal/small"
+
+                                <PopoverMenuItem
+                                  icon={RunsIcon}
+                                  leadingIconClassName="text-runs"
+                                  title="View all runs"
                                   to={v3RunsPath(organization, project, env, {
                                     queues: [queueFilterableName],
                                     period: "30d",
                                     rootOnly: false,
                                   })}
-                                  fullWidth
-                                  textAlignLeft
-                                  LeadingIcon={RunsIcon}
-                                  leadingIconClassName="text-indigo-500"
-                                >
-                                  View all runs
-                                </LinkButton>
-                                <LinkButton
-                                  variant="minimal/small"
+                                />
+                                <PopoverMenuItem
+                                  icon={RectangleStackIcon}
+                                  leadingIconClassName="text-queues"
+                                  title="View queued runs"
                                   to={v3RunsPath(organization, project, env, {
                                     queues: [queueFilterableName],
                                     statuses: ["PENDING"],
                                     period: "30d",
                                     rootOnly: false,
                                   })}
-                                  fullWidth
-                                  textAlignLeft
-                                  LeadingIcon={RectangleStackIcon}
-                                  leadingIconClassName="text-queues"
-                                >
-                                  View queued runs
-                                </LinkButton>
-                                <LinkButton
-                                  variant="minimal/small"
+                                />
+                                <PopoverMenuItem
+                                  icon={Spinner}
+                                  leadingIconClassName="text-queues animate-none"
+                                  title="View running runs"
                                   to={v3RunsPath(organization, project, env, {
                                     queues: [queueFilterableName],
                                     statuses: ["DEQUEUED", "EXECUTING"],
                                     period: "30d",
                                     rootOnly: false,
                                   })}
-                                  fullWidth
-                                  textAlignLeft
-                                  LeadingIcon={Spinner}
-                                  leadingIconClassName="size-4 animate-none"
-                                >
-                                  View running runs
-                                </LinkButton>
+                                />
+                                <QueueOverrideConcurrencyButton
+                                  queue={queue}
+                                  environmentConcurrencyLimit={environment.concurrencyLimit}
+                                />
                               </>
                             }
                           />
@@ -576,7 +682,7 @@ export default function Page() {
                     })
                   ) : (
                     <TableRow>
-                      <TableCell colSpan={6}>
+                      <TableCell colSpan={7}>
                         <div className="grid place-items-center py-6 text-text-dimmed">
                           <Paragraph>
                             {hasFilters
@@ -733,29 +839,26 @@ function QueuePauseResumeButton({
   fullWidth?: boolean;
   showTooltip?: boolean;
 }) {
-  const navigation = useNavigation();
   const [isOpen, setIsOpen] = useState(false);
 
-  const button = (
-    <Button
-      type="button"
-      variant={variant}
-      LeadingIcon={queue.paused ? PlayIcon : PauseIcon}
-      leadingIconClassName={queue.paused ? "text-success" : "text-warning"}
-      fullWidth={fullWidth}
-      textAlignLeft={fullWidth}
-    >
-      {queue.paused ? "Resume..." : "Pause..."}
-    </Button>
-  );
-
   const trigger = showTooltip ? (
     <div>
       <TooltipProvider disableHoverableContent={true}>
         <Tooltip>
           <TooltipTrigger asChild>
             <div>
-              <DialogTrigger asChild>{button}</DialogTrigger>
+              <DialogTrigger asChild>
+                <Button
+                  type="button"
+                  variant={variant}
+                  LeadingIcon={queue.paused ? PlayIcon : PauseIcon}
+                  leadingIconClassName={queue.paused ? "text-success" : "text-warning"}
+                  fullWidth={fullWidth}
+                  textAlignLeft={fullWidth}
+                >
+                  {queue.paused ? "Resume..." : "Pause..."}
+                </Button>
+              </DialogTrigger>
             </div>
           </TooltipTrigger>
           <TooltipContent side="right" className={"text-xs"}>
@@ -767,7 +870,13 @@ function QueuePauseResumeButton({
       </TooltipProvider>
     </div>
   ) : (
-    <DialogTrigger asChild>{button}</DialogTrigger>
+    <DialogTrigger asChild>
+      <PopoverMenuItem
+        icon={queue.paused ? PlayIcon : PauseIcon}
+        leadingIconClassName={queue.paused ? "text-success" : "text-warning"}
+        title={queue.paused ? "Resume..." : "Pause..."}
+      />
+    </DialogTrigger>
   );
 
   return (
@@ -814,6 +923,124 @@ function QueuePauseResumeButton({
   );
 }
 
+function QueueOverrideConcurrencyButton({
+  queue,
+  environmentConcurrencyLimit,
+}: {
+  queue: QueueItem;
+  environmentConcurrencyLimit: number;
+}) {
+  const navigation = useNavigation();
+  const [isOpen, setIsOpen] = useState(false);
+  const [concurrencyLimit, setConcurrencyLimit] = useState<string>(
+    queue.concurrencyLimit?.toString() ?? environmentConcurrencyLimit.toString()
+  );
+
+  const isOverridden = !!queue.concurrency?.overriddenAt;
+  const currentLimit = queue.concurrencyLimit ?? environmentConcurrencyLimit;
+
+  useEffect(() => {
+    if (navigation.state === "loading" || navigation.state === "idle") {
+      setIsOpen(false);
+    }
+  }, [navigation.state]);
+
+  const isLoading = Boolean(
+    navigation.formData?.get("action") === "queue-override" ||
+      navigation.formData?.get("action") === "queue-remove-override"
+  );
+
+  return (
+    <Dialog open={isOpen} onOpenChange={setIsOpen}>
+      <DialogTrigger asChild>
+        <PopoverMenuItem
+          icon={AdjustmentsHorizontalIcon}
+          title={isOverridden ? "Edit override…" : "Override limit…"}
+        />
+      </DialogTrigger>
+      <DialogContent>
+        <DialogHeader>
+          {isOverridden ? "Edit concurrency override" : "Override concurrency limit"}
+        </DialogHeader>
+        <div className="flex flex-col gap-3 pt-3">
+          {isOverridden ? (
+            <Paragraph>
+              This queue's concurrency limit is currently overridden to {currentLimit}.
+              {typeof queue.concurrency?.base === "number" &&
+                ` The original limit set in code was ${queue.concurrency.base}.`}{" "}
+              You can update the override or remove it to restore the{" "}
+              {typeof queue.concurrency?.base === "number"
+                ? "limit set in code"
+                : "environment concurrency limit"}
+              .
+            </Paragraph>
+          ) : (
+            <Paragraph>
+              Override this queue's concurrency limit. The current limit is {currentLimit}, which is
+              set {queue.concurrencyLimit !== null ? "in code" : "by the environment"}.
+            </Paragraph>
+          )}
+          <Form method="post" onSubmit={() => setIsOpen(false)} className="space-y-3">
+            <input type="hidden" name="friendlyId" value={queue.id} />
+            <div className="space-y-2">
+              <label htmlFor="concurrencyLimit" className="text-sm text-text-bright">
+                Concurrency limit
+              </label>
+              <Input
+                type="number"
+                name="concurrencyLimit"
+                id="concurrencyLimit"
+                min="0"
+                max={environmentConcurrencyLimit}
+                value={concurrencyLimit}
+                onChange={(e) => setConcurrencyLimit(e.target.value)}
+                placeholder={currentLimit.toString()}
+                autoFocus
+              />
+            </div>
+
+            <FormButtons
+              confirmButton={
+                <Button
+                  type="submit"
+                  name="action"
+                  value="queue-override"
+                  disabled={isLoading || !concurrencyLimit}
+                  variant="primary/medium"
+                  LeadingIcon={isLoading && <Spinner color="white" />}
+                  shortcut={{ modifiers: ["mod"], key: "enter" }}
+                >
+                  {isOverridden ? "Update override" : "Override limit"}
+                </Button>
+              }
+              cancelButton={
+                <div className="flex items-center justify-between gap-2">
+                  {isOverridden && (
+                    <Button
+                      type="submit"
+                      name="action"
+                      value="queue-remove-override"
+                      disabled={isLoading}
+                      variant="danger/medium"
+                    >
+                      Remove override
+                    </Button>
+                  )}
+                  <DialogClose asChild>
+                    <Button type="button" variant="tertiary/medium">
+                      Cancel
+                    </Button>
+                  </DialogClose>
+                </div>
+              }
+            />
+          </Form>
+        </div>
+      </DialogContent>
+    </Dialog>
+  );
+}
+
 function EngineVersionUpgradeCallout() {
   return (
     <div className="mt-4 flex max-w-lg flex-col gap-4 rounded-sm border border-grid-bright bg-background-bright px-4">

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx

@@ -97,6 +97,7 @@ import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route";
 import { SpanView } from "../resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route";
 import { useSearchParams } from "~/hooks/useSearchParam";
 import { CopyableText } from "~/components/primitives/CopyableText";
+import type { SpanOverride } from "~/v3/eventRepository/eventRepository.types";
 
 const resizableSettings = {
   parent: {
@@ -142,7 +143,6 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const [error, result] = await tryCatch(
     presenter.call({
       userId,
-      organizationSlug,
       showDeletedLogs: !!impersonationId,
       projectSlug: projectParam,
       runFriendlyId: runParam,
@@ -172,7 +172,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   return json({
     run: result.run,
     trace: result.trace,
-    maximumLiveReloadingSetting: env.MAXIMUM_LIVE_RELOADING_EVENTS,
+    maximumLiveReloadingSetting: result.maximumLiveReloadingSetting,
     resizable: {
       parent,
       tree,
@@ -302,7 +302,8 @@ function TraceView({ run, trace, maximumLiveReloadingSetting, resizable }: Loade
     return <></>;
   }
 
-  const { events, duration, rootSpanStatus, rootStartedAt, queuedDuration } = trace;
+  const { events, duration, rootSpanStatus, rootStartedAt, queuedDuration, overridesBySpanId } =
+    trace;
   const shouldLiveReload = events.length <= maximumLiveReloadingSetting;
 
   const changeToSpan = useDebounce((selectedSpan: string) => {
@@ -324,6 +325,8 @@ function TraceView({ run, trace, maximumLiveReloadingSetting, resizable }: Loade
     // WARNING Don't put the revalidator in the useEffect deps array or bad things will happen
   }, [streamedEvents]); // eslint-disable-line react-hooks/exhaustive-deps
 
+  const spanOverrides = selectedSpanId ? overridesBySpanId?.[selectedSpanId] : undefined;
+
   return (
     <div className={cn("grid h-full max-h-full grid-cols-1 overflow-hidden")}>
       <ResizablePanelGroup
@@ -372,6 +375,7 @@ function TraceView({ run, trace, maximumLiveReloadingSetting, resizable }: Loade
             <SpanView
               runParam={run.friendlyId}
               spanId={selectedSpanId}
+              spanOverrides={spanOverrides as SpanOverride | undefined}
               closePanel={() => replaceSearchParam("span")}
             />
           </ResizablePanel>

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.settings/route.tsx

@@ -14,6 +14,7 @@ import {
   useActionData,
   useNavigation,
   useNavigate,
+  useSearchParams,
 } from "@remix-run/react";
 import { type ActionFunction, type LoaderFunctionArgs, json } from "@remix-run/server-runtime";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
@@ -49,8 +50,6 @@ import {
   redirectBackWithSuccessMessage,
   redirectWithErrorMessage,
   redirectWithSuccessMessage,
-  getSession,
-  commitSession,
 } from "~/models/message.server";
 import { ProjectSettingsService } from "~/services/projectSettings.server";
 import { logger } from "~/services/logger.server";
@@ -61,6 +60,8 @@ import {
   githubAppInstallPath,
   EnvironmentParamSchema,
   v3ProjectSettingsPath,
+  docsPath,
+  v3BillingPath,
 } from "~/utils/pathBuilder";
 import React, { useEffect, useState } from "react";
 import { Select, SelectItem } from "~/components/primitives/Select";
@@ -78,6 +79,7 @@ import { TextLink } from "~/components/primitives/TextLink";
 import { cn } from "~/utils/cn";
 import { ProjectSettingsPresenter } from "~/services/projectSettingsPresenter.server";
 import { type BuildSettings } from "~/v3/buildSettings";
+import { InfoIconTooltip } from "~/components/primitives/Tooltip";
 
 export const meta: MetaFunction = () => {
   return [
@@ -123,22 +125,13 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
 
   const { gitHubApp, buildSettings } = resultOrFail.value;
 
-  const session = await getSession(request.headers.get("Cookie"));
-  const openGitHubRepoConnectionModal = session.get("gitHubAppInstalled") === true;
-  const headers = new Headers({
-    "Set-Cookie": await commitSession(session),
+  return typedjson({
+    githubAppEnabled: gitHubApp.enabled,
+    githubAppInstallations: gitHubApp.installations,
+    connectedGithubRepository: gitHubApp.connectedRepository,
+    isPreviewEnvironmentEnabled: gitHubApp.isPreviewEnvironmentEnabled,
+    buildSettings,
   });
-
-  return typedjson(
-    {
-      githubAppEnabled: gitHubApp.enabled,
-      githubAppInstallations: gitHubApp.installations,
-      connectedGithubRepository: gitHubApp.connectedRepository,
-      openGitHubRepoConnectionModal,
-      buildSettings,
-    },
-    { headers }
-  );
 };
 
 const ConnectGitHubRepoFormSchema = z.object({
@@ -167,14 +160,6 @@ const UpdateBuildSettingsFormSchema = z.object({
     .refine((val) => !val || val.length <= 255, {
       message: "Config file path must not exceed 255 characters",
     }),
-  installDirectory: z
-    .string()
-    .trim()
-    .optional()
-    .transform((val) => (val ? val.replace(/^\/+/, "") : val))
-    .refine((val) => !val || val.length <= 255, {
-      message: "Install directory must not exceed 255 characters",
-    }),
   installCommand: z
     .string()
     .trim()
@@ -185,6 +170,16 @@ const UpdateBuildSettingsFormSchema = z.object({
     .refine((val) => !val || val.length <= 500, {
       message: "Install command must not exceed 500 characters",
     }),
+  preBuildCommand: z
+    .string()
+    .trim()
+    .optional()
+    .refine((val) => !val || !val.includes("\n"), {
+      message: "Pre-build command must be a single line",
+    })
+    .refine((val) => !val || val.length <= 500, {
+      message: "Pre-build command must not exceed 500 characters",
+    }),
 });
 
 type UpdateBuildSettingsFormSchema = z.infer<typeof UpdateBuildSettingsFormSchema>;
@@ -412,11 +407,11 @@ export const action: ActionFunction = async ({ request, params }) => {
       });
     }
     case "update-build-settings": {
-      const { installDirectory, installCommand, triggerConfigFilePath } = submission.value;
+      const { installCommand, preBuildCommand, triggerConfigFilePath } = submission.value;
 
       const resultOrFail = await projectSettingsService.updateBuildSettings(projectId, {
-        installDirectory: installDirectory || undefined,
         installCommand: installCommand || undefined,
+        preBuildCommand: preBuildCommand || undefined,
         triggerConfigFilePath: triggerConfigFilePath || undefined,
       });
 
@@ -448,8 +443,8 @@ export default function Page() {
     githubAppInstallations,
     connectedGithubRepository,
     githubAppEnabled,
-    openGitHubRepoConnectionModal,
     buildSettings,
+    isPreviewEnvironmentEnabled,
   } = useTypedLoaderData<typeof loader>();
   const project = useProject();
   const organization = useOrganization();
@@ -577,14 +572,16 @@ export default function Page() {
                   <Header2 spacing>Git settings</Header2>
                   <div className="w-full rounded-sm border border-grid-dimmed p-4">
                     {connectedGithubRepository ? (
-                      <ConnectedGitHubRepoForm connectedGitHubRepo={connectedGithubRepository} />
+                      <ConnectedGitHubRepoForm
+                        connectedGitHubRepo={connectedGithubRepository}
+                        previewEnvironmentEnabled={isPreviewEnvironmentEnabled}
+                      />
                     ) : (
                       <GitHubConnectionPrompt
                         gitHubAppInstallations={githubAppInstallations ?? []}
                         organizationSlug={organization.slug}
                         projectSlug={project.slug}
                         environmentSlug={environment.slug}
-                        openGitHubRepoConnectionModal={openGitHubRepoConnectionModal}
                       />
                     )}
                   </div>
@@ -667,7 +664,6 @@ function ConnectGitHubRepoModal({
   organizationSlug,
   projectSlug,
   environmentSlug,
-  open = false,
 }: {
   gitHubAppInstallations: GitHubAppInstallation[];
   organizationSlug: string;
@@ -675,7 +671,7 @@ function ConnectGitHubRepoModal({
   environmentSlug: string;
   open?: boolean;
 }) {
-  const [isModalOpen, setIsModalOpen] = useState(open);
+  const [isModalOpen, setIsModalOpen] = useState(false);
   const lastSubmission = useActionData() as any;
   const navigate = useNavigate();
 
@@ -703,6 +699,17 @@ function ConnectGitHubRepoModal({
     },
   });
 
+  const [searchParams, setSearchParams] = useSearchParams();
+  useEffect(() => {
+    const params = new URLSearchParams(searchParams);
+
+    if (params.get("openGithubRepoModal") === "1") {
+      setIsModalOpen(true);
+      params.delete("openGithubRepoModal");
+      setSearchParams(params);
+    }
+  }, [searchParams, setSearchParams]);
+
   useEffect(() => {
     if (lastSubmission && "success" in lastSubmission && lastSubmission.success === true) {
       setIsModalOpen(false);
@@ -759,11 +766,11 @@ function ConnectGitHubRepoModal({
                         navigate(
                           githubAppInstallPath(
                             organizationSlug,
-                            v3ProjectSettingsPath(
+                            `${v3ProjectSettingsPath(
                               { slug: organizationSlug },
                               { slug: projectSlug },
                               { slug: environmentSlug }
-                            )
+                            )}?openGithubRepoModal=1`
                           )
                         );
                       }}
@@ -856,13 +863,11 @@ function GitHubConnectionPrompt({
   organizationSlug,
   projectSlug,
   environmentSlug,
-  openGitHubRepoConnectionModal = false,
 }: {
   gitHubAppInstallations: GitHubAppInstallation[];
   organizationSlug: string;
   projectSlug: string;
   environmentSlug: string;
-  openGitHubRepoConnectionModal?: boolean;
 }) {
   return (
     <Fieldset>
@@ -871,11 +876,11 @@ function GitHubConnectionPrompt({
           <LinkButton
             to={githubAppInstallPath(
               organizationSlug,
-              v3ProjectSettingsPath(
+              `${v3ProjectSettingsPath(
                 { slug: organizationSlug },
                 { slug: projectSlug },
                 { slug: environmentSlug }
-              )
+              )}?openGithubRepoModal=1`
             )}
             variant={"secondary/medium"}
             LeadingIcon={OctoKitty}
@@ -890,7 +895,6 @@ function GitHubConnectionPrompt({
               organizationSlug={organizationSlug}
               projectSlug={projectSlug}
               environmentSlug={environmentSlug}
-              open={openGitHubRepoConnectionModal}
             />
             <span className="flex items-center gap-1 text-xs text-text-dimmed">
               <CheckCircleIcon className="size-4 text-success" /> GitHub app is installed
@@ -913,11 +917,14 @@ type ConnectedGitHubRepo = {
 
 function ConnectedGitHubRepoForm({
   connectedGitHubRepo,
+  previewEnvironmentEnabled,
 }: {
   connectedGitHubRepo: ConnectedGitHubRepo;
+  previewEnvironmentEnabled?: boolean;
 }) {
   const lastSubmission = useActionData() as any;
   const navigation = useNavigation();
+  const organization = useOrganization();
 
   const [hasGitSettingsChanges, setHasGitSettingsChanges] = useState(false);
   const [gitSettingsValues, setGitSettingsValues] = useState({
@@ -1013,10 +1020,10 @@ function ConnectedGitHubRepoForm({
         <Fieldset>
           <InputGroup fullWidth>
             <Hint>
-              Every commit on the selected tracking branch creates a deployment in the corresponding
+              Every push to the selected tracking branch creates a deployment in the corresponding
               environment.
             </Hint>
-            <div className="grid grid-cols-[120px_1fr] gap-3">
+            <div className="mt-1 grid grid-cols-[120px_1fr] gap-3">
               <div className="flex items-center gap-1.5">
                 <EnvironmentIcon environment={{ type: "PRODUCTION" }} className="size-4" />
                 <span className={`text-sm ${environmentTextClassName({ type: "PRODUCTION" })}`}>
@@ -1064,19 +1071,34 @@ function ConnectedGitHubRepoForm({
                   {environmentFullTitle({ type: "PREVIEW" })}
                 </span>
               </div>
-              <Switch
-                name="previewDeploymentsEnabled"
-                defaultChecked={connectedGitHubRepo.previewDeploymentsEnabled}
-                variant="small"
-                label="create preview deployments for pull requests"
-                labelPosition="right"
-                onCheckedChange={(checked) => {
-                  setGitSettingsValues((prev) => ({
-                    ...prev,
-                    previewDeploymentsEnabled: checked,
-                  }));
-                }}
-              />
+              <div className="flex items-center gap-1.5">
+                <Switch
+                  name="previewDeploymentsEnabled"
+                  disabled={!previewEnvironmentEnabled}
+                  defaultChecked={
+                    connectedGitHubRepo.previewDeploymentsEnabled && previewEnvironmentEnabled
+                  }
+                  variant="small"
+                  label="Create preview deployments for pull requests"
+                  labelPosition="right"
+                  onCheckedChange={(checked) => {
+                    setGitSettingsValues((prev) => ({
+                      ...prev,
+                      previewDeploymentsEnabled: checked,
+                    }));
+                  }}
+                />
+                {!previewEnvironmentEnabled && (
+                  <InfoIconTooltip
+                    content={
+                      <span className="text-xs">
+                        <TextLink to={v3BillingPath(organization)}>Upgrade</TextLink> your plan to
+                        enable preview branches
+                      </span>
+                    }
+                  />
+                )}
+              </div>
             </div>
             <FormError>{fields.productionBranch?.error}</FormError>
             <FormError>{fields.stagingBranch?.error}</FormError>
@@ -1110,14 +1132,14 @@ function BuildSettingsForm({ buildSettings }: { buildSettings: BuildSettings })
 
   const [hasBuildSettingsChanges, setHasBuildSettingsChanges] = useState(false);
   const [buildSettingsValues, setBuildSettingsValues] = useState({
-    installDirectory: buildSettings?.installDirectory || "",
+    preBuildCommand: buildSettings?.preBuildCommand || "",
     installCommand: buildSettings?.installCommand || "",
     triggerConfigFilePath: buildSettings?.triggerConfigFilePath || "",
   });
 
   useEffect(() => {
     const hasChanges =
-      buildSettingsValues.installDirectory !== (buildSettings?.installDirectory || "") ||
+      buildSettingsValues.preBuildCommand !== (buildSettings?.preBuildCommand || "") ||
       buildSettingsValues.installCommand !== (buildSettings?.installCommand || "") ||
       buildSettingsValues.triggerConfigFilePath !== (buildSettings?.triggerConfigFilePath || "");
     setHasBuildSettingsChanges(hasChanges);
@@ -1167,7 +1189,7 @@ function BuildSettingsForm({ buildSettings }: { buildSettings: BuildSettings })
           <Input
             {...conform.input(fields.installCommand, { type: "text" })}
             defaultValue={buildSettings?.installCommand || ""}
-            placeholder="e.g., `npm install`, or `bun install`"
+            placeholder="e.g., `npm install`, `pnpm install`, or `bun install`"
             onChange={(e) => {
               setBuildSettingsValues((prev) => ({
                 ...prev,
@@ -1175,26 +1197,30 @@ function BuildSettingsForm({ buildSettings }: { buildSettings: BuildSettings })
               }));
             }}
           />
-          <Hint>Command to install your project dependencies. Auto-detected by default.</Hint>
+          <Hint>
+            Command to install your project dependencies. This will be run from the root directory
+            of your repo. Auto-detected by default.
+          </Hint>
           <FormError id={fields.installCommand.errorId}>{fields.installCommand.error}</FormError>
         </InputGroup>
         <InputGroup fullWidth>
-          <Label htmlFor={fields.installDirectory.id}>Install directory</Label>
+          <Label htmlFor={fields.preBuildCommand.id}>Pre-build command</Label>
           <Input
-            {...conform.input(fields.installDirectory, { type: "text" })}
-            defaultValue={buildSettings?.installDirectory || ""}
-            placeholder=""
+            {...conform.input(fields.preBuildCommand, { type: "text" })}
+            defaultValue={buildSettings?.preBuildCommand || ""}
+            placeholder="e.g., `npm run prisma:generate`"
             onChange={(e) => {
               setBuildSettingsValues((prev) => ({
                 ...prev,
-                installDirectory: e.target.value,
+                preBuildCommand: e.target.value,
               }));
             }}
           />
-          <Hint>The directory where the install command is run in. Auto-detected by default.</Hint>
-          <FormError id={fields.installDirectory.errorId}>
-            {fields.installDirectory.error}
-          </FormError>
+          <Hint>
+            Any command that needs to run before we build and deploy your project. This will be run
+            from the root directory of your repo.
+          </Hint>
+          <FormError id={fields.preBuildCommand.errorId}>{fields.preBuildCommand.error}</FormError>
         </InputGroup>
         <FormError>{buildSettingsForm.error}</FormError>
         <FormButtons

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx

@@ -73,6 +73,8 @@ import { DeleteTaskRunTemplateData, RunTemplateData } from "~/v3/taskRunTemplate
 import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components/primitives/Dialog";
 import { DialogClose, DialogDescription } from "@radix-ui/react-dialog";
 import { FormButtons } from "~/components/primitives/FormButtons";
+import { $replica } from "~/db.server";
+import { clickhouseClient } from "~/services/clickhouseInstance.server";
 
 type FormAction = "create-template" | "delete-template" | "run-scheduled" | "run-standard";
 
@@ -96,7 +98,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
     });
   }
 
-  const presenter = new TestTaskPresenter();
+  const presenter = new TestTaskPresenter($replica, clickhouseClient);
   try {
     const result = await presenter.call({
       userId,

apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.billing-alerts/route.tsx

@@ -47,7 +47,7 @@ export const meta: MetaFunction = () => {
 };
 
 export async function loader({ params, request }: LoaderFunctionArgs) {
-  await requireUserId(request);
+  const userId = await requireUserId(request);
   const { organizationSlug } = OrganizationParamsSchema.parse(params);
 
   const { isManagedCloud } = featuresForRequest(request);
@@ -55,8 +55,8 @@ export async function loader({ params, request }: LoaderFunctionArgs) {
     return redirect(organizationPath({ slug: organizationSlug }));
   }
 
-  const organization = await prisma.organization.findUnique({
-    where: { slug: organizationSlug },
+  const organization = await prisma.organization.findFirst({
+    where: { slug: organizationSlug, members: { some: { userId } } },
   });
 
   if (!organization) {
@@ -181,7 +181,7 @@ export default function Page() {
   const fieldValues = useRef<string[]>(alerts.emails);
   const emailFields = useFieldList(form.ref, { ...emails, defaultValue: alerts.emails });
 
-  const checkboxLevels = [0.75, 0.9, 1.0];
+  const checkboxLevels = [0.75, 0.9, 1.0, 2.0, 5.0];
 
   useEffect(() => {
     if (alerts.emails.length > 0) {

apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.billing/route.tsx

@@ -28,7 +28,7 @@ export const meta: MetaFunction = () => {
 };
 
 export async function loader({ params, request }: LoaderFunctionArgs) {
-  await requireUserId(request);
+  const userId = await requireUserId(request);
   const { organizationSlug } = OrganizationParamsSchema.parse(params);
 
   const { isManagedCloud } = featuresForRequest(request);
@@ -41,8 +41,8 @@ export async function loader({ params, request }: LoaderFunctionArgs) {
     throw new Response(null, { status: 404, statusText: "Plans not found" });
   }
 
-  const organization = await prisma.organization.findUnique({
-    where: { slug: organizationSlug },
+  const organization = await prisma.organization.findFirst({
+    where: { slug: organizationSlug, members: { some: { userId } } },
   });
 
   if (!organization) {

apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.usage/route.tsx

@@ -46,7 +46,7 @@ export const meta: MetaFunction = () => {
 };
 
 export async function loader({ params, request }: LoaderFunctionArgs) {
-  await requireUserId(request);
+  const userId = await requireUserId(request);
   const { organizationSlug } = OrganizationParamsSchema.parse(params);
 
   const { isManagedCloud } = featuresForRequest(request);
@@ -54,8 +54,8 @@ export async function loader({ params, request }: LoaderFunctionArgs) {
     return redirect(organizationPath({ slug: organizationSlug }));
   }
 
-  const organization = await prisma.organization.findUnique({
-    where: { slug: organizationSlug },
+  const organization = await prisma.organization.findFirst({
+    where: { slug: organizationSlug, members: { some: { userId } } },
   });
 
   if (!organization) {

apps/webapp/app/routes/account.tokens/route.tsx

@@ -3,8 +3,8 @@ import { parse } from "@conform-to/zod";
 import { BookOpenIcon, ShieldCheckIcon, TrashIcon } from "@heroicons/react/20/solid";
 import { ShieldExclamationIcon } from "@heroicons/react/24/solid";
 import { DialogClose } from "@radix-ui/react-dialog";
-import { Form, MetaFunction, useActionData, useFetcher } from "@remix-run/react";
-import { ActionFunction, LoaderFunctionArgs, json } from "@remix-run/server-runtime";
+import { Form, type MetaFunction, useActionData, useFetcher } from "@remix-run/react";
+import { type ActionFunction, type LoaderFunctionArgs, json } from "@remix-run/server-runtime";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { z } from "zod";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
@@ -16,7 +16,6 @@ import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components
 import { Fieldset } from "~/components/primitives/Fieldset";
 import { FormButtons } from "~/components/primitives/FormButtons";
 import { FormError } from "~/components/primitives/FormError";
-import { Header2 } from "~/components/primitives/Headers";
 import { Hint } from "~/components/primitives/Hint";
 import { Input } from "~/components/primitives/Input";
 import { InputGroup } from "~/components/primitives/InputGroup";
@@ -36,8 +35,8 @@ import {
 import { SimpleTooltip } from "~/components/primitives/Tooltip";
 import { redirectWithSuccessMessage } from "~/models/message.server";
 import {
-  CreatedPersonalAccessToken,
-  ObfuscatedPersonalAccessToken,
+  type CreatedPersonalAccessToken,
+  type ObfuscatedPersonalAccessToken,
   createPersonalAccessToken,
   getValidPersonalAccessTokens,
   revokePersonalAccessToken,
@@ -53,7 +52,7 @@ export const meta: MetaFunction = () => {
   ];
 };
 
-export const loader = async ({ request, params }: LoaderFunctionArgs) => {
+export const loader = async ({ request }: LoaderFunctionArgs) => {
   const userId = await requireUserId(request);
 
   try {
@@ -113,7 +112,7 @@ export const action: ActionFunction = async ({ request }) => {
     }
     case "revoke": {
       try {
-        await revokePersonalAccessToken(submission.value.tokenId);
+        await revokePersonalAccessToken(submission.value.tokenId, userId);
 
         return redirectWithSuccessMessage(
           personalAccessTokensPath(),
@@ -125,6 +124,7 @@ export const action: ActionFunction = async ({ request }) => {
       }
     }
     default: {
+      submission.value satisfies never;
       return json({ errors: { body: "Invalid action" } }, { status: 400 });
     }
   }

apps/webapp/app/routes/admin._index.tsx

@@ -138,13 +138,17 @@ export default function AdminDashboardRoute() {
                     <TableCell isSticky={true}>
                       <Form method="post" reloadDocument>
                         <input type="hidden" name="id" value={user.id} />
-
                         <Button
                           type="submit"
                           name="action"
                           value="impersonate"
                           className="mr-2"
                           variant="tertiary/small"
+                          shortcut={
+                            users.length === 1
+                              ? { modifiers: ["mod"], key: "enter", enabledOnInputElements: true }
+                              : undefined
+                          }
                         >
                           Impersonate
                         </Button>

apps/webapp/app/routes/admin.api.v1.environments.$environmentId.engine.repair-queues.ts

@@ -0,0 +1,105 @@
+import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
+import pMap from "p-map";
+import { z } from "zod";
+import { $replica, prisma } from "~/db.server";
+import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+import { determineEngineVersion } from "~/v3/engineVersion.server";
+import { engine } from "~/v3/runEngine.server";
+
+const ParamsSchema = z.object({
+  environmentId: z.string(),
+});
+
+const BodySchema = z.object({
+  dryRun: z.boolean().default(true),
+  queues: z.array(z.string()).default([]),
+});
+
+export async function action({ request, params }: ActionFunctionArgs) {
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const user = await prisma.user.findUnique({
+    where: {
+      id: authenticationResult.userId,
+    },
+  });
+
+  if (!user) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  if (!user.admin) {
+    return json({ error: "You must be an admin to perform this action" }, { status: 403 });
+  }
+
+  const parsedParams = ParamsSchema.parse(params);
+
+  const environment = await prisma.runtimeEnvironment.findFirst({
+    where: {
+      id: parsedParams.environmentId,
+    },
+    include: {
+      organization: true,
+      project: true,
+      orgMember: true,
+    },
+  });
+
+  if (!environment) {
+    return json({ error: "Environment not found" }, { status: 404 });
+  }
+
+  const engineVersion = await determineEngineVersion({ environment });
+
+  if (engineVersion === "V1") {
+    return json({ error: "Engine version is V1" }, { status: 400 });
+  }
+
+  const body = await request.json();
+  const parsedBody = BodySchema.parse(body);
+
+  const queues = await $replica.taskQueue.findMany({
+    where: {
+      runtimeEnvironmentId: environment.id,
+      version: "V2",
+      name: parsedBody.queues.length > 0 ? { in: parsedBody.queues } : undefined,
+    },
+    select: {
+      friendlyId: true,
+      name: true,
+      concurrencyLimit: true,
+      type: true,
+      paused: true,
+    },
+    orderBy: {
+      orderableName: "asc",
+    },
+  });
+
+  const repairEnvironmentResults = await engine.repairEnvironment(environment, parsedBody.dryRun);
+
+  const repairResults = await pMap(
+    queues,
+    async (queue) => {
+      const repair = await engine.repairQueue(
+        environment,
+        queue.name,
+        parsedBody.dryRun,
+        repairEnvironmentResults.runIds
+      );
+
+      return {
+        queue: queue.name,
+        ...repair,
+      };
+    },
+    { concurrency: 5 }
+  );
+
+  return json({ environment: repairEnvironmentResults, queues: repairResults });
+}

apps/webapp/app/routes/admin.api.v1.environments.$environmentId.engine.report.ts

@@ -0,0 +1,95 @@
+import { json, LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { $replica, prisma } from "~/db.server";
+import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+import { determineEngineVersion } from "~/v3/engineVersion.server";
+import { engine } from "~/v3/runEngine.server";
+
+const ParamsSchema = z.object({
+  environmentId: z.string(),
+});
+
+const SearchParamsSchema = z.object({
+  verbose: z.string().default("0"),
+  page: z.coerce.number().optional(),
+  per_page: z.coerce.number().optional(),
+});
+
+export async function loader({ request, params }: LoaderFunctionArgs) {
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const user = await prisma.user.findUnique({
+    where: {
+      id: authenticationResult.userId,
+    },
+  });
+
+  if (!user) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  if (!user.admin) {
+    return json({ error: "You must be an admin to perform this action" }, { status: 403 });
+  }
+
+  const parsedParams = ParamsSchema.parse(params);
+
+  const environment = await prisma.runtimeEnvironment.findFirst({
+    where: {
+      id: parsedParams.environmentId,
+    },
+    include: {
+      organization: true,
+      project: true,
+      orgMember: true,
+    },
+  });
+
+  if (!environment) {
+    return json({ error: "Environment not found" }, { status: 404 });
+  }
+
+  const engineVersion = await determineEngineVersion({ environment });
+
+  if (engineVersion === "V1") {
+    return json({ error: "Engine version is V1" }, { status: 400 });
+  }
+
+  const url = new URL(request.url);
+  const searchParams = SearchParamsSchema.parse(Object.fromEntries(url.searchParams));
+
+  const page = searchParams.page ?? 1;
+  const perPage = searchParams.per_page ?? 50;
+
+  const queues = await $replica.taskQueue.findMany({
+    where: {
+      runtimeEnvironmentId: environment.id,
+      version: "V2",
+    },
+    select: {
+      friendlyId: true,
+      name: true,
+      concurrencyLimit: true,
+      type: true,
+      paused: true,
+    },
+    orderBy: {
+      orderableName: "asc",
+    },
+    skip: (page - 1) * perPage,
+    take: perPage,
+  });
+
+  const report = await engine.generateEnvironmentReport(
+    environment,
+    queues,
+    searchParams.verbose === "1"
+  );
+
+  return json(report);
+}

apps/webapp/app/routes/admin.orgs.tsx

@@ -117,6 +117,11 @@ export default function AdminDashboardRoute() {
                         to={`/@/orgs/${org.slug}`}
                         className="mr-2"
                         variant="tertiary/small"
+                        shortcut={
+                          organizations.length === 1
+                            ? { modifiers: ["mod"], key: "enter", enabledOnInputElements: true }
+                            : undefined
+                        }
                       >
                         Impersonate
                       </LinkButton>

apps/webapp/app/routes/api.v1.deployments.$deploymentId.cancel.ts

@@ -1,5 +1,5 @@
 import { type ActionFunctionArgs, json } from "@remix-run/server-runtime";
-import { StartDeploymentRequestBody } from "@trigger.dev/core/v3";
+import { CancelDeploymentRequestBody, tryCatch } from "@trigger.dev/core/v3";
 import { z } from "zod";
 import { authenticateRequest } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
@@ -34,8 +34,8 @@ export async function action({ request, params }: ActionFunctionArgs) {
   const { environment: authenticatedEnv } = authenticationResult.result;
   const { deploymentId } = parsedParams.data;
 
-  const rawBody = await request.json();
-  const body = StartDeploymentRequestBody.safeParse(rawBody);
+  const [, rawBody] = await tryCatch(request.json());
+  const body = CancelDeploymentRequestBody.safeParse(rawBody ?? {});
 
   if (!body.success) {
     return json({ error: "Invalid request body", issues: body.error.issues }, { status: 400 });
@@ -44,23 +44,24 @@ export async function action({ request, params }: ActionFunctionArgs) {
   const deploymentService = new DeploymentService();
 
   return await deploymentService
-    .startDeployment(authenticatedEnv, deploymentId, {
-      contentHash: body.data.contentHash,
-      git: body.data.gitMeta,
-      runtime: body.data.runtime,
+    .cancelDeployment(authenticatedEnv, deploymentId, {
+      canceledReason: body.data.reason,
     })
     .match(
       () => {
-        return json(null, { status: 204 });
+        return new Response(null, { status: 204 });
       },
       (error) => {
         switch (error.type) {
-          case "failed_to_extend_deployment_timeout":
-            return json(null, { status: 204 }); // ignore these errors for now
           case "deployment_not_found":
             return json({ error: "Deployment not found" }, { status: 404 });
-          case "deployment_not_pending":
-            return json({ error: "Deployment is not pending" }, { status: 409 });
+          case "failed_to_delete_deployment_timeout":
+            return new Response(null, { status: 204 }); // not a critical error, ignore
+          case "deployment_cannot_be_cancelled":
+            return json(
+              { error: "Deployment is already in a final state and cannot be canceled" },
+              { status: 409 }
+            );
           case "other":
           default:
             error.type satisfies "other";

apps/webapp/app/routes/api.v1.deployments.$deploymentId.generate-registry-credentials.ts

@@ -0,0 +1,100 @@
+import { type ActionFunctionArgs, json } from "@remix-run/server-runtime";
+import {
+  type GenerateRegistryCredentialsResponseBody,
+  ProgressDeploymentRequestBody,
+  tryCatch,
+} from "@trigger.dev/core/v3";
+import { z } from "zod";
+import { authenticateRequest } from "~/services/apiAuth.server";
+import { logger } from "~/services/logger.server";
+import { DeploymentService } from "~/v3/services/deployment.server";
+
+const ParamsSchema = z.object({
+  deploymentId: z.string(),
+});
+
+export async function action({ request, params }: ActionFunctionArgs) {
+  if (request.method.toUpperCase() !== "POST") {
+    return json({ error: "Method Not Allowed" }, { status: 405 });
+  }
+
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return json({ error: "Invalid params" }, { status: 400 });
+  }
+
+  const authenticationResult = await authenticateRequest(request, {
+    apiKey: true,
+    organizationAccessToken: false,
+    personalAccessToken: false,
+  });
+
+  if (!authenticationResult || !authenticationResult.result.ok) {
+    logger.info("Invalid or missing api key", { url: request.url });
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const { environment: authenticatedEnv } = authenticationResult.result;
+  const { deploymentId } = parsedParams.data;
+
+  const [, rawBody] = await tryCatch(request.json());
+  const body = ProgressDeploymentRequestBody.safeParse(rawBody ?? {});
+
+  if (!body.success) {
+    return json({ error: "Invalid request body", issues: body.error.issues }, { status: 400 });
+  }
+
+  const deploymentService = new DeploymentService();
+
+  return await deploymentService.generateRegistryCredentials(authenticatedEnv, deploymentId).match(
+    (result) => {
+      return json(
+        {
+          username: result.username,
+          password: result.password,
+          expiresAt: result.expiresAt.toISOString(),
+          repositoryUri: result.repositoryUri,
+        } satisfies GenerateRegistryCredentialsResponseBody,
+        { status: 200 }
+      );
+    },
+    (error) => {
+      switch (error.type) {
+        case "deployment_not_found":
+          return json({ error: "Deployment not found" }, { status: 404 });
+        case "deployment_has_no_image_reference":
+          logger.error(
+            "Failed to generate registry credentials: deployment_has_no_image_reference",
+            { deploymentId }
+          );
+          return json({ error: "Deployment has no image reference" }, { status: 409 });
+        case "deployment_is_already_final":
+          return json(
+            { error: "Failed to generate registry credentials: deployment_is_already_final" },
+            { status: 409 }
+          );
+        case "missing_registry_credentials":
+          logger.error("Failed to generate registry credentials: missing_registry_credentials", {
+            deploymentId,
+          });
+          return json({ error: "Missing registry credentials" }, { status: 409 });
+        case "registry_not_supported":
+          logger.error("Failed to generate registry credentials: registry_not_supported", {
+            deploymentId,
+          });
+          return json({ error: "Registry not supported" }, { status: 409 });
+        case "registry_region_not_supported":
+          logger.error("Failed to generate registry credentials: registry_region_not_supported", {
+            deploymentId,
+          });
+          return json({ error: "Registry region not supported" }, { status: 409 });
+        case "other":
+        default:
+          error.type satisfies "other";
+          logger.error("Failed to generate registry credentials", { error: error.cause });
+          return json({ error: "Internal server error" }, { status: 500 });
+      }
+    }
+  );
+}

apps/webapp/app/routes/api.v1.deployments.$deploymentId.progress.ts

@@ -0,0 +1,80 @@
+import { type ActionFunctionArgs, json } from "@remix-run/server-runtime";
+import { ProgressDeploymentRequestBody, tryCatch } from "@trigger.dev/core/v3";
+import { z } from "zod";
+import { authenticateRequest } from "~/services/apiAuth.server";
+import { logger } from "~/services/logger.server";
+import { DeploymentService } from "~/v3/services/deployment.server";
+
+const ParamsSchema = z.object({
+  deploymentId: z.string(),
+});
+
+export async function action({ request, params }: ActionFunctionArgs) {
+  if (request.method.toUpperCase() !== "POST") {
+    return json({ error: "Method Not Allowed" }, { status: 405 });
+  }
+
+  const parsedParams = ParamsSchema.safeParse(params);
+
+  if (!parsedParams.success) {
+    return json({ error: "Invalid params" }, { status: 400 });
+  }
+
+  const authenticationResult = await authenticateRequest(request, {
+    apiKey: true,
+    organizationAccessToken: false,
+    personalAccessToken: false,
+  });
+
+  if (!authenticationResult || !authenticationResult.result.ok) {
+    logger.info("Invalid or missing api key", { url: request.url });
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const { environment: authenticatedEnv } = authenticationResult.result;
+  const { deploymentId } = parsedParams.data;
+
+  const [, rawBody] = await tryCatch(request.json());
+  const body = ProgressDeploymentRequestBody.safeParse(rawBody ?? {});
+
+  if (!body.success) {
+    return json({ error: "Invalid request body", issues: body.error.issues }, { status: 400 });
+  }
+
+  const deploymentService = new DeploymentService();
+
+  return await deploymentService
+    .progressDeployment(authenticatedEnv, deploymentId, {
+      contentHash: body.data.contentHash,
+      git: body.data.gitMeta,
+      runtime: body.data.runtime,
+    })
+    .match(
+      () => {
+        return new Response(null, { status: 204 });
+      },
+      (error) => {
+        switch (error.type) {
+          case "failed_to_extend_deployment_timeout": {
+            logger.warn("Failed to extend deployment timeout", { error: error.cause });
+            return new Response(null, { status: 204 }); // ignore these errors for now
+          }
+          case "deployment_not_found":
+            return json({ error: "Deployment not found" }, { status: 404 });
+          case "deployment_cannot_be_progressed":
+            return json(
+              { error: "Deployment is not in a progressable state (PENDING or INSTALLING)" },
+              { status: 409 }
+            );
+          case "failed_to_create_remote_build": {
+            logger.error("Failed to create remote Depot build", { error: error.cause });
+            return json({ error: "Failed to create remote build" }, { status: 500 });
+          }
+          case "other":
+          default:
+            error.type satisfies "other";
+            return json({ error: "Internal server error" }, { status: 500 });
+        }
+      }
+    );
+}

apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts

@@ -0,0 +1,75 @@
+import { json } from "@remix-run/server-runtime";
+import { type RetrieveQueueParam, RetrieveQueueType } from "@trigger.dev/core/v3";
+import { z } from "zod";
+import { toQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server";
+import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
+import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server";
+
+const BodySchema = z.object({
+  type: RetrieveQueueType.default("id"),
+  concurrencyLimit: z.number().int().min(0).max(100000),
+});
+
+export const { action } = createActionApiRoute(
+  {
+    body: BodySchema,
+    params: z.object({
+      queueParam: z.string().transform((val) => val.replace(/%2F/g, "/")),
+    }),
+  },
+  async ({ params, body, authentication }) => {
+    const input: RetrieveQueueParam =
+      body.type === "id"
+        ? params.queueParam
+        : {
+            type: body.type,
+            name: decodeURIComponent(params.queueParam).replace(/%2F/g, "/"),
+          };
+
+    return concurrencySystem.queues
+      .overrideQueueConcurrencyLimit(authentication.environment, input, body.concurrencyLimit)
+      .match(
+        (queue) => {
+          return json(
+            toQueueItem({
+              friendlyId: queue.friendlyId,
+              name: queue.name,
+              type: queue.type,
+              running: queue.running,
+              queued: queue.queued,
+              concurrencyLimit: queue.concurrencyLimit,
+              concurrencyLimitBase: queue.concurrencyLimitBase,
+              concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt,
+              concurrencyLimitOverriddenBy: null,
+              paused: queue.paused,
+            }),
+            { status: 200 }
+          );
+        },
+        (error) => {
+          switch (error.type) {
+            case "queue_not_found": {
+              return json({ error: "Queue not found" }, { status: 404 });
+            }
+            case "queue_update_failed": {
+              return json({ error: "Failed to update queue concurrency limit" }, { status: 500 });
+            }
+            case "sync_queue_concurrency_to_engine_failed": {
+              return json(
+                { error: "Failed to sync queue concurrency limit to engine" },
+                { status: 500 }
+              );
+            }
+            case "get_queue_stats_failed": {
+              return json({ error: "Failed to get queue stats" }, { status: 500 });
+            }
+            case "other":
+            default: {
+              error.type satisfies "other";
+              return json({ error: "Internal server error" }, { status: 500 });
+            }
+          }
+        }
+      );
+  }
+);

apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts

@@ -0,0 +1,75 @@
+import { json } from "@remix-run/server-runtime";
+import { type RetrieveQueueParam, RetrieveQueueType } from "@trigger.dev/core/v3";
+import { z } from "zod";
+import { toQueueItem } from "~/presenters/v3/QueueRetrievePresenter.server";
+import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
+import { concurrencySystem } from "~/v3/services/concurrencySystemInstance.server";
+
+const BodySchema = z.object({
+  type: RetrieveQueueType.default("id"),
+});
+
+export const { action } = createActionApiRoute(
+  {
+    body: BodySchema,
+    params: z.object({
+      queueParam: z.string().transform((val) => val.replace(/%2F/g, "/")),
+    }),
+  },
+  async ({ params, body, authentication }) => {
+    const input: RetrieveQueueParam =
+      body.type === "id"
+        ? params.queueParam
+        : {
+            type: body.type,
+            name: decodeURIComponent(params.queueParam).replace(/%2F/g, "/"),
+          };
+
+    return concurrencySystem.queues.resetConcurrencyLimit(authentication.environment, input).match(
+      (queue) => {
+        return json(
+          toQueueItem({
+            friendlyId: queue.friendlyId,
+            name: queue.name,
+            type: queue.type,
+            running: queue.running,
+            queued: queue.queued,
+            concurrencyLimit: queue.concurrencyLimit,
+            concurrencyLimitBase: queue.concurrencyLimitBase,
+            concurrencyLimitOverriddenAt: queue.concurrencyLimitOverriddenAt,
+            concurrencyLimitOverriddenBy: null,
+            paused: queue.paused,
+          }),
+          { status: 200 }
+        );
+      },
+      (error) => {
+        switch (error.type) {
+          case "queue_not_found": {
+            return json({ error: "Queue not found" }, { status: 404 });
+          }
+          case "queue_not_overridden": {
+            return json({ error: "Queue is not overridden" }, { status: 400 });
+          }
+          case "queue_update_failed": {
+            return json({ error: "Failed to update queue concurrency limit" }, { status: 500 });
+          }
+          case "sync_queue_concurrency_to_engine_failed": {
+            return json(
+              { error: "Failed to sync queue concurrency limit to engine" },
+              { status: 500 }
+            );
+          }
+          case "get_queue_stats_failed": {
+            return json({ error: "Failed to get queue stats" }, { status: 500 });
+          }
+          case "other":
+          default: {
+            error.type satisfies "other";
+            return json({ error: "Internal server error" }, { status: 500 });
+          }
+        }
+      }
+    );
+  }
+);

apps/webapp/app/routes/api.v1.runs.$runId.events.ts

@@ -2,8 +2,8 @@ import { json } from "@remix-run/server-runtime";
 import { z } from "zod";
 import { getTaskEventStoreTableForRun } from "~/v3/taskEventStore.server";
 import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server";
-import { eventRepository } from "~/v3/eventRepository.server";
 import { ApiRetrieveRunPresenter } from "~/presenters/v3/ApiRetrieveRunPresenter.server";
+import { resolveEventRepositoryForStore } from "~/v3/eventRepository/index.server";
 
 const ParamsSchema = z.object({
   runId: z.string(), // This is the run friendly ID
@@ -30,9 +30,13 @@ export const loader = createLoaderApiRoute(
       superScopes: ["read:runs", "read:all", "admin"],
     },
   },
-  async ({ resource: run }) => {
+  async ({ resource: run, authentication }) => {
+    const eventRepository = resolveEventRepositoryForStore(run.taskEventStore);
+
     const runEvents = await eventRepository.getRunEvents(
       getTaskEventStoreTableForRun(run),
+      authentication.environment.id,
+      run.traceId,
       run.friendlyId,
       run.createdAt,
       run.completedAt ?? undefined

apps/webapp/app/routes/api.v1.runs.$runId.trace.ts

@@ -3,7 +3,7 @@ import { BatchId } from "@trigger.dev/core/v3/isomorphic";
 import { z } from "zod";
 import { $replica } from "~/db.server";
 import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server";
-import { eventRepository } from "~/v3/eventRepository.server";
+import { resolveEventRepositoryForStore } from "~/v3/eventRepository/index.server";
 import { getTaskEventStoreTableForRun } from "~/v3/taskEventStore.server";
 
 const ParamsSchema = z.object({
@@ -35,9 +35,12 @@ export const loader = createLoaderApiRoute(
       superScopes: ["read:runs", "read:all", "admin"],
     },
   },
-  async ({ resource: run }) => {
+  async ({ resource: run, authentication }) => {
+    const eventRepository = resolveEventRepositoryForStore(run.taskEventStore);
+
     const traceSummary = await eventRepository.getTraceDetailedSummary(
       getTaskEventStoreTableForRun(run),
+      authentication.environment.id,
       run.traceId,
       run.createdAt,
       run.completedAt ?? undefined

apps/webapp/app/routes/engine.v1.dev.config.ts

@@ -20,7 +20,11 @@ export const loader = createLoaderApiRoute(
         environmentId: authentication.environment.id,
         dequeueIntervalWithRun: env.DEV_DEQUEUE_INTERVAL_WITH_RUN,
         dequeueIntervalWithoutRun: env.DEV_DEQUEUE_INTERVAL_WITHOUT_RUN,
-        maxConcurrentRuns: env.DEV_MAX_CONCURRENT_RUNS,
+        // Limit max runs to smaller of an optional global limit and the environment limit
+        maxConcurrentRuns: Math.min(
+          env.DEV_MAX_CONCURRENT_RUNS ?? authentication.environment.maximumConcurrencyLimit,
+          authentication.environment.maximumConcurrencyLimit
+        ),
         engineUrl: env.DEV_ENGINE_URL,
       });
     } catch (error) {

apps/webapp/app/routes/engine.v1.dev.runs.$runFriendlyId.logs.debug.ts

@@ -1,79 +1,3 @@
-import { TypedResponse } from "@remix-run/server-runtime";
-import { assertExhaustive } from "@trigger.dev/core/utils";
-import { RunId } from "@trigger.dev/core/v3/isomorphic";
-import {
-  WorkerApiDebugLogBody,
-  WorkerApiRunAttemptStartResponseBody,
-} from "@trigger.dev/core/v3/workers";
-import { z } from "zod";
-import { prisma } from "~/db.server";
-import { logger } from "~/services/logger.server";
-import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
-import { recordRunDebugLog } from "~/v3/eventRepository.server";
-
-// const { action } = createActionApiRoute(
-//   {
-//     params: z.object({
-//       runFriendlyId: z.string(),
-//     }),
-//     body: WorkerApiDebugLogBody,
-//     method: "POST",
-//   },
-//   async ({
-//     authentication,
-//     body,
-//     params,
-//   }): Promise<TypedResponse<WorkerApiRunAttemptStartResponseBody>> => {
-//     const { runFriendlyId } = params;
-
-//     try {
-//       const run = await prisma.taskRun.findFirst({
-//         where: {
-//           friendlyId: params.runFriendlyId,
-//           runtimeEnvironmentId: authentication.environment.id,
-//         },
-//       });
-
-//       if (!run) {
-//         throw new Response("You don't have permissions for this run", { status: 401 });
-//       }
-
-//       const eventResult = await recordRunDebugLog(
-//         RunId.fromFriendlyId(runFriendlyId),
-//         body.message,
-//         {
-//           attributes: {
-//             properties: body.properties,
-//           },
-//           startTime: body.time,
-//         }
-//       );
-
-//       if (eventResult.success) {
-//         return new Response(null, { status: 204 });
-//       }
-
-//       switch (eventResult.code) {
-//         case "FAILED_TO_RECORD_EVENT":
-//           return new Response(null, { status: 400 }); // send a 400 to prevent retries
-//         case "RUN_NOT_FOUND":
-//           return new Response(null, { status: 404 });
-//         default:
-//           return assertExhaustive(eventResult.code);
-//       }
-//     } catch (error) {
-//       logger.error("Failed to record dev log", {
-//         environmentId: authentication.environment.id,
-//         error,
-//       });
-//       throw error;
-//     }
-//   }
-// );
-
-// export { action };
-
-// Create a generic JSON action in remix
 export function action() {
   return new Response(null, { status: 204 });
 }

apps/webapp/app/routes/engine.v1.dev.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts

@@ -1,5 +1,5 @@
 import { json, TypedResponse } from "@remix-run/server-runtime";
-import { MachinePreset } from "@trigger.dev/core/v3";
+import { MachinePreset, SemanticInternalAttributes } from "@trigger.dev/core/v3";
 import { RunId, SnapshotId } from "@trigger.dev/core/v3/isomorphic";
 import {
   WorkerApiRunAttemptStartRequestBody,
@@ -57,7 +57,8 @@ const { action } = createActionApiRoute(
       const envVars = await getEnvVars(
         authentication.environment,
         engineResult.run.id,
-        engineResult.execution.machine ?? defaultMachinePreset
+        engineResult.execution.machine ?? defaultMachinePreset,
+        engineResult.run.taskEventStore
       );
 
       return json({
@@ -77,7 +78,8 @@ const { action } = createActionApiRoute(
 async function getEnvVars(
   environment: RuntimeEnvironment,
   runId: string,
-  machinePreset: MachinePreset
+  machinePreset: MachinePreset,
+  taskEventStore?: string
 ): Promise<Record<string, string>> {
   const variables = await resolveVariablesForEnvironment(environment);
 
@@ -94,6 +96,19 @@ async function getEnvVars(
     ]
   );
 
+  if (taskEventStore) {
+    const resourceAttributes = JSON.stringify({
+      [SemanticInternalAttributes.TASK_EVENT_STORE]: taskEventStore,
+    });
+
+    variables.push(
+      ...[
+        { key: "OTEL_RESOURCE_ATTRIBUTES", value: resourceAttributes },
+        { key: "TRIGGER_OTEL_RESOURCE_ATTRIBUTES", value: resourceAttributes },
+      ]
+    );
+  }
+
   return variables.reduce((acc: Record<string, string>, curr) => {
     acc[curr.key] = curr.value;
     return acc;

apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.logs.debug.ts

@@ -3,7 +3,7 @@ import { RunId } from "@trigger.dev/core/v3/isomorphic";
 import { WorkerApiDebugLogBody } from "@trigger.dev/core/v3/runEngineWorker";
 import { z } from "zod";
 import { createActionWorkerApiRoute } from "~/services/routeBuilders/apiBuilder.server";
-import { recordRunDebugLog } from "~/v3/eventRepository.server";
+import { recordRunDebugLog } from "~/v3/eventRepository/index.server";
 
 export const action = createActionWorkerApiRoute(
   {

apps/webapp/app/routes/invite-accept.tsx

@@ -18,6 +18,12 @@ export async function loader({ request }: LoaderFunctionArgs) {
     );
   }
 
+  if (!user) {
+    return redirectWithSuccessMessage("/", request, "Please log in to accept the invite.", {
+      ephemeral: false,
+    });
+  }
+
   const invite = await getInviteFromToken({ token });
   if (!invite) {
     return redirectWithErrorMessage(
@@ -28,12 +34,6 @@ export async function loader({ request }: LoaderFunctionArgs) {
     );
   }
 
-  if (!user) {
-    return redirectWithSuccessMessage("/", request, "Please log in to accept the invite.", {
-      ephemeral: false,
-    });
-  }
-
   if (invite.email !== user.email) {
     return redirectWithErrorMessage(
       "/",

apps/webapp/app/routes/invite-resend.tsx

@@ -1,5 +1,5 @@
 import { parse } from "@conform-to/zod";
-import { ActionFunction, json } from "@remix-run/server-runtime";
+import { type ActionFunction, json } from "@remix-run/server-runtime";
 import { env } from "process";
 import { z } from "zod";
 import { resendInvite } from "~/models/member.server";
@@ -25,6 +25,7 @@ export const action: ActionFunction = async ({ request }) => {
   try {
     const invite = await resendInvite({
       inviteId: submission.value.inviteId,
+      userId,
     });
 
     try {

apps/webapp/app/routes/invite-revoke.tsx

@@ -1,5 +1,5 @@
 import { parse } from "@conform-to/zod";
-import { ActionFunction, json } from "@remix-run/server-runtime";
+import { type ActionFunction, json } from "@remix-run/server-runtime";
 import { z } from "zod";
 import { revokeInvite } from "~/models/member.server";
 import { redirectWithSuccessMessage } from "~/models/message.server";
@@ -24,7 +24,7 @@ export const action: ActionFunction = async ({ request }) => {
   try {
     const { email, organization } = await revokeInvite({
       userId,
-      slug: submission.value.slug,
+      orgSlug: submission.value.slug,
       inviteId: submission.value.inviteId,
     });
 

apps/webapp/app/routes/invites.tsx

@@ -1,6 +1,6 @@
 import { conform, useForm } from "@conform-to/react";
 import { parse } from "@conform-to/zod";
-import { ActionFunction, LoaderFunctionArgs, json, redirect } from "@remix-run/node";
+import { type ActionFunction, type LoaderFunctionArgs, json, redirect } from "@remix-run/node";
 import { Form, useActionData } from "@remix-run/react";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { z } from "zod";
@@ -36,7 +36,7 @@ const schema = z.object({
 });
 
 export const action: ActionFunction = async ({ request }) => {
-  const userId = await requireUserId(request);
+  const user = await requireUser(request);
 
   const formData = await request.formData();
   const submission = parse(formData, { schema });
@@ -49,7 +49,7 @@ export const action: ActionFunction = async ({ request }) => {
     if (submission.intent === "accept") {
       const { remainingInvites, organization } = await acceptInvite({
         inviteId: submission.value.inviteId,
-        userId,
+        user: { id: user.id, email: user.email },
       });
 
       if (remainingInvites.length === 0) {
@@ -64,7 +64,7 @@ export const action: ActionFunction = async ({ request }) => {
     } else if (submission.intent === "decline") {
       const { remainingInvites, organization } = await declineInvite({
         inviteId: submission.value.inviteId,
-        userId,
+        user: { id: user.id, email: user.email },
       });
       if (remainingInvites.length === 0) {
         return redirectWithSuccessMessage(

apps/webapp/app/routes/login.magic/route.tsx

@@ -1,7 +1,11 @@
 import { ArrowLeftIcon, EnvelopeIcon } from "@heroicons/react/20/solid";
 import { InboxArrowDownIcon } from "@heroicons/react/24/solid";
-import type { ActionFunctionArgs, LoaderFunctionArgs, MetaFunction } from "@remix-run/node";
-import { redirect } from "@remix-run/node";
+import {
+  redirect,
+  type ActionFunctionArgs,
+  type LoaderFunctionArgs,
+  type MetaFunction,
+} from "@remix-run/node";
 import { Form, useNavigation } from "@remix-run/react";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { z } from "zod";
@@ -18,6 +22,14 @@ import { Spinner } from "~/components/primitives/Spinner";
 import { TextLink } from "~/components/primitives/TextLink";
 import { authenticator } from "~/services/auth.server";
 import { commitSession, getUserSession } from "~/services/sessionStorage.server";
+import {
+  checkMagicLinkEmailRateLimit,
+  checkMagicLinkEmailDailyRateLimit,
+  MagicLinkRateLimitError,
+  checkMagicLinkIpRateLimit,
+} from "~/services/magicLinkRateLimiter.server";
+import { logger, tryCatch } from "@trigger.dev/core/v3";
+import { env } from "~/env.server";
 
 export const meta: MetaFunction = ({ matches }) => {
   const parentMeta = matches
@@ -71,29 +83,99 @@ export async function action({ request }: ActionFunctionArgs) {
 
   const payload = Object.fromEntries(await clonedRequest.formData());
 
-  const { action } = z
-    .object({
-      action: z.enum(["send", "reset"]),
-    })
+  const data = z
+    .discriminatedUnion("action", [
+      z.object({
+        action: z.literal("send"),
+        email: z.string().trim().toLowerCase(),
+      }),
+      z.object({
+        action: z.literal("reset"),
+      }),
+    ])
     .parse(payload);
 
-  if (action === "send") {
-    return authenticator.authenticate("email-link", request, {
-      successRedirect: "/login/magic",
-      failureRedirect: "/login/magic",
-    });
-  } else {
-    const session = await getUserSession(request);
-    session.unset("triggerdotdev:magiclink");
+  switch (data.action) {
+    case "send": {
+      if (!env.LOGIN_RATE_LIMITS_ENABLED) {
+        return authenticator.authenticate("email-link", request, {
+          successRedirect: "/login/magic",
+          failureRedirect: "/login/magic",
+        });
+      }
 
-    return redirect("/login/magic", {
-      headers: {
-        "Set-Cookie": await commitSession(session),
-      },
-    });
+      const { email } = data;
+      const xff = request.headers.get("x-forwarded-for");
+      const clientIp = extractClientIp(xff);
+
+      const [error] = await tryCatch(
+        Promise.all([
+          clientIp ? checkMagicLinkIpRateLimit(clientIp) : Promise.resolve(),
+          checkMagicLinkEmailRateLimit(email),
+          checkMagicLinkEmailDailyRateLimit(email),
+        ])
+      );
+
+      if (error) {
+        if (error instanceof MagicLinkRateLimitError) {
+          logger.warn("Login magic link rate limit exceeded", {
+            clientIp,
+            email,
+            error,
+          });
+        } else {
+          logger.error("Failed sending login magic link", {
+            clientIp,
+            email,
+            error,
+          });
+        }
+
+        const errorMessage =
+          error instanceof MagicLinkRateLimitError
+            ? "Too many magic link requests. Please try again shortly."
+            : "Failed sending magic link. Please try again shortly.";
+
+        const session = await getUserSession(request);
+        session.set("auth:error", {
+          message: errorMessage,
+        });
+
+        return redirect("/login/magic", {
+          headers: {
+            "Set-Cookie": await commitSession(session),
+          },
+        });
+      }
+
+      return authenticator.authenticate("email-link", request, {
+        successRedirect: "/login/magic",
+        failureRedirect: "/login/magic",
+      });
+    }
+    case "reset":
+    default: {
+      data.action satisfies "reset";
+
+      const session = await getUserSession(request);
+      session.unset("triggerdotdev:magiclink");
+
+      return redirect("/login/magic", {
+        headers: {
+          "Set-Cookie": await commitSession(session),
+        },
+      });
+    }
   }
 }
 
+const extractClientIp = (xff: string | null) => {
+  if (!xff) return null;
+
+  const parts = xff.split(",").map((p) => p.trim());
+  return parts[parts.length - 1]; // take last item, ALB appends the real client IP by default
+};
+
 export default function LoginMagicLinkPage() {
   const { magicLinkSent, magicLinkError } = useTypedLoaderData<typeof loader>();
   const navigate = useNavigation();

apps/webapp/app/routes/orgs.$organizationSlug.projects.$projectParam.runs.$runParam.ts

@@ -3,14 +3,14 @@ import { type LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { z } from "zod";
 import { prisma } from "~/db.server";
 import { requireUserId } from "~/services/session.server";
-import { ProjectParamSchema, v3DeploymentPath, v3RunPath } from "~/utils/pathBuilder";
+import { ProjectParamSchema, v3RunPath } from "~/utils/pathBuilder";
 
 const ParamSchema = ProjectParamSchema.extend({
   runParam: z.string(),
 });
 
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
-  await requireUserId(request);
+  const userId = await requireUserId(request);
   const { organizationSlug, projectParam, runParam } = ParamSchema.parse(params);
 
   const run = await prisma.taskRun.findFirst({
@@ -18,6 +18,14 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
       friendlyId: runParam,
       project: {
         slug: projectParam,
+        organization: {
+          slug: organizationSlug,
+          members: {
+            some: {
+              userId,
+            },
+          },
+        },
       },
     },
     select: {

apps/webapp/app/routes/otel.v1.logs.ts

@@ -4,15 +4,15 @@ import { otlpExporter } from "~/v3/otlpExporter.server";
 
 export async function action({ request }: ActionFunctionArgs) {
   try {
-    const contentType = request.headers.get("content-type");
+    const contentType = request.headers.get("content-type")?.toLowerCase() ?? "";
 
-    if (contentType === "application/json") {
+    if (contentType.startsWith("application/json")) {
       const body = await request.json();
 
-      const exportResponse = await otlpExporter.exportLogs(body as ExportLogsServiceRequest, false);
+      const exportResponse = await otlpExporter.exportLogs(body as ExportLogsServiceRequest);
 
       return json(exportResponse, { status: 200 });
-    } else if (contentType === "application/x-protobuf") {
+    } else if (contentType.startsWith("application/x-protobuf")) {
       const buffer = await request.arrayBuffer();
 
       const exportRequest = ExportLogsServiceRequest.decode(new Uint8Array(buffer));

apps/webapp/app/routes/otel.v1.traces.ts

@@ -4,18 +4,15 @@ import { otlpExporter } from "~/v3/otlpExporter.server";
 
 export async function action({ request }: ActionFunctionArgs) {
   try {
-    const contentType = request.headers.get("content-type");
+    const contentType = request.headers.get("content-type")?.toLowerCase() ?? "";
 
-    if (contentType === "application/json") {
+    if (contentType.startsWith("application/json")) {
       const body = await request.json();
 
-      const exportResponse = await otlpExporter.exportTraces(
-        body as ExportTraceServiceRequest,
-        false
-      );
+      const exportResponse = await otlpExporter.exportTraces(body as ExportTraceServiceRequest);
 
       return json(exportResponse, { status: 200 });
-    } else if (contentType === "application/x-protobuf") {
+    } else if (contentType.startsWith("application/x-protobuf")) {
       const buffer = await request.arrayBuffer();
 
       const exportRequest = ExportTraceServiceRequest.decode(new Uint8Array(buffer));

apps/webapp/app/routes/resources.$projectId.deployments.$deploymentShortCode.cancel.ts

@@ -0,0 +1,133 @@
+import { parse } from "@conform-to/zod";
+import { type ActionFunction, json } from "@remix-run/node";
+import { errAsync, fromPromise, okAsync } from "neverthrow";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
+import { logger } from "~/services/logger.server";
+import { requireUserId } from "~/services/session.server";
+import { DeploymentService } from "~/v3/services/deployment.server";
+
+export const cancelSchema = z.object({
+  redirectUrl: z.string(),
+});
+
+const ParamSchema = z.object({
+  projectId: z.string(),
+  deploymentShortCode: z.string(),
+});
+
+export const action: ActionFunction = async ({ request, params }) => {
+  const userId = await requireUserId(request);
+  const { projectId, deploymentShortCode } = ParamSchema.parse(params);
+
+  const formData = await request.formData();
+  const submission = parse(formData, { schema: cancelSchema });
+
+  if (!submission.value) {
+    return json(submission);
+  }
+
+  const verifyProjectMembership = () =>
+    fromPromise(
+      prisma.project.findFirst({
+        where: {
+          id: projectId,
+          organization: {
+            members: {
+              some: {
+                userId,
+              },
+            },
+          },
+        },
+        select: {
+          id: true,
+        },
+      }),
+      (error) => ({ type: "other" as const, cause: error })
+    ).andThen((project) => {
+      if (!project) {
+        return errAsync({ type: "project_not_found" as const });
+      }
+      return okAsync(project);
+    });
+
+  const findDeploymentFriendlyId = ({ id }: { id: string }) =>
+    fromPromise(
+      prisma.workerDeployment.findUnique({
+        select: {
+          friendlyId: true,
+          projectId: true,
+        },
+        where: {
+          projectId_shortCode: {
+            projectId: id,
+            shortCode: deploymentShortCode,
+          },
+        },
+      }),
+      (error) => ({ type: "other" as const, cause: error })
+    ).andThen((deployment) => {
+      if (!deployment) {
+        return errAsync({ type: "deployment_not_found" as const });
+      }
+      return okAsync(deployment);
+    });
+
+  const deploymentService = new DeploymentService();
+  const result = await verifyProjectMembership()
+    .andThen(findDeploymentFriendlyId)
+    .andThen((deployment) =>
+      deploymentService.cancelDeployment({ projectId: deployment.projectId }, deployment.friendlyId)
+    );
+
+  if (result.isErr()) {
+    logger.error(
+      `Failed to cancel deployment: ${result.error.type}`,
+      result.error.type === "other"
+        ? {
+            cause: result.error.cause,
+          }
+        : undefined
+    );
+
+    switch (result.error.type) {
+      case "project_not_found":
+        return redirectWithErrorMessage(submission.value.redirectUrl, request, "Project not found");
+      case "deployment_not_found":
+        return redirectWithErrorMessage(
+          submission.value.redirectUrl,
+          request,
+          "Deployment not found"
+        );
+      case "deployment_cannot_be_cancelled":
+        return redirectWithErrorMessage(
+          submission.value.redirectUrl,
+          request,
+          "Deployment is already in a final state and cannot be canceled"
+        );
+      case "failed_to_delete_deployment_timeout":
+        // not a critical error, ignore
+        return redirectWithSuccessMessage(
+          submission.value.redirectUrl,
+          request,
+          `Canceled deployment ${deploymentShortCode}.`
+        );
+      case "other":
+      default:
+        result.error.type satisfies "other";
+        return redirectWithErrorMessage(
+          submission.value.redirectUrl,
+          request,
+          "Internal server error"
+        );
+    }
+  }
+
+  return redirectWithSuccessMessage(
+    submission.value.redirectUrl,
+    request,
+    `Canceled deployment ${deploymentShortCode}.`
+  );
+};

apps/webapp/app/routes/resources.environments.$envId.runs.tags.tsx

@@ -0,0 +1,66 @@
+import { type LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { timeFilters } from "~/components/runs/v3/SharedFilters";
+import { $replica } from "~/db.server";
+import { RunTagListPresenter } from "~/presenters/v3/RunTagListPresenter.server";
+import { requireUserId } from "~/services/session.server";
+
+const Params = z.object({
+  envId: z.string(),
+});
+
+const SearchParams = z.object({
+  name: z.string().optional(),
+  period: z.preprocess((value) => (value === "all" ? undefined : value), z.string().optional()),
+  from: z.coerce.number().optional(),
+  to: z.coerce.number().optional(),
+});
+
+export async function loader({ request, params }: LoaderFunctionArgs) {
+  const userId = await requireUserId(request);
+  const { envId } = Params.parse(params);
+
+  const environment = await $replica.runtimeEnvironment.findFirst({
+    select: {
+      id: true,
+      projectId: true,
+      organizationId: true,
+    },
+    where: { id: envId, organization: { members: { some: { userId } } } },
+  });
+
+  if (!environment) {
+    throw new Response("Not Found", { status: 404 });
+  }
+
+  const search = new URL(request.url).searchParams;
+
+  const parsedSearchParams = SearchParams.safeParse({
+    name: search.get("name") ?? undefined,
+    period: search.get("period") ?? undefined,
+    from: search.get("from") ?? undefined,
+    to: search.get("to") ?? undefined,
+  });
+
+  if (!parsedSearchParams.success) {
+    throw new Response("Invalid search params", { status: 400 });
+  }
+
+  const { period, from, to } = timeFilters({
+    period: parsedSearchParams.data.period,
+    from: parsedSearchParams.data.from,
+    to: parsedSearchParams.data.to,
+  });
+
+  const presenter = new RunTagListPresenter();
+  const result = await presenter.call({
+    environmentId: environment.id,
+    projectId: environment.projectId,
+    organizationId: environment.organizationId,
+    name: parsedSearchParams.data.name,
+    period,
+    from,
+    to,
+  });
+  return result;
+}

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues.stream.tsx

@@ -1,64 +0,0 @@
-import { $replica } from "~/db.server";
-import { env } from "~/env.server";
-import { logger } from "~/services/logger.server";
-import { requireUserId } from "~/services/session.server";
-import { EnvironmentParamSchema } from "~/utils/pathBuilder";
-import { createSSELoader } from "~/utils/sse";
-
-export const loader = createSSELoader({
-  timeout: env.QUEUE_SSE_AUTORELOAD_TIMEOUT_MS,
-  interval: env.QUEUE_SSE_AUTORELOAD_INTERVAL_MS,
-  debug: false,
-  handler: async ({ request, params }) => {
-    const userId = await requireUserId(request);
-    const { projectParam, envParam } = EnvironmentParamSchema.parse(params);
-
-    const environment = await $replica.runtimeEnvironment.findFirst({
-      where: {
-        slug: envParam,
-        OR: [
-          {
-            type: {
-              in: ["PREVIEW", "STAGING", "PRODUCTION"],
-            },
-          },
-          {
-            type: "DEVELOPMENT",
-            orgMember: {
-              userId,
-            },
-          },
-        ],
-        project: {
-          slug: projectParam,
-        },
-      },
-    });
-
-    if (!environment) {
-      throw new Response("Not Found", { status: 404 });
-    }
-
-    return {
-      beforeStream: async () => {
-        logger.debug("Start queue page SSE session", {
-          environmentId: environment.id,
-        });
-      },
-      initStream: async ({ send }) => {
-        send({ event: "time", data: new Date().toISOString() });
-      },
-      iterator: async ({ send }) => {
-        send({
-          event: "update",
-          data: new Date().toISOString(),
-        });
-      },
-      cleanup: async () => {
-        logger.debug("End queue page SSE session", {
-          environmentId: environment.id,
-        });
-      },
-    };
-  },
-});

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx

@@ -14,6 +14,7 @@ import { assertNever } from "assert-never";
 import { useEffect } from "react";
 import { typedjson, useTypedFetcher } from "remix-typedjson";
 import { ExitIcon } from "~/assets/icons/ExitIcon";
+import { FlagIcon } from "~/assets/icons/RegionIcons";
 import { AdminDebugRun } from "~/components/admin/debugRun";
 import { CodeBlock } from "~/components/code/CodeBlock";
 import { EnvironmentCombo } from "~/components/environments/EnvironmentLabel";
@@ -22,6 +23,7 @@ import { MachineLabelCombo } from "~/components/MachineLabelCombo";
 import { MachineTooltipInfo } from "~/components/MachineTooltipInfo";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
 import { Callout } from "~/components/primitives/Callout";
+import { CopyableText } from "~/components/primitives/CopyableText";
 import { DateTime, DateTimeAccurate } from "~/components/primitives/DateTime";
 import { Header2, Header3 } from "~/components/primitives/Headers";
 import { Paragraph } from "~/components/primitives/Paragraph";
@@ -76,9 +78,11 @@ import {
 } from "~/utils/pathBuilder";
 import { createTimelineSpanEventsFromSpanEvents } from "~/utils/timelineSpanEvents";
 import { CompleteWaitpointForm } from "../resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.$waitpointFriendlyId.complete/route";
-import { FlagIcon } from "~/assets/icons/RegionIcons";
+import { requireUserId } from "~/services/session.server";
+import type { SpanOverride } from "~/v3/eventRepository/eventRepository.types";
 
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
+  const userId = await requireUserId(request);
   const { projectParam, organizationSlug, envParam, runParam, spanParam } =
     v3SpanParamsSchema.parse(params);
 
@@ -89,6 +93,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
       projectSlug: projectParam,
       spanId: spanParam,
       runFriendlyId: runParam,
+      userId,
     });
 
     return typedjson(result);
@@ -116,10 +121,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
 export function SpanView({
   runParam,
   spanId,
+  spanOverrides,
   closePanel,
 }: {
   runParam: string;
   spanId: string | undefined;
+  spanOverrides?: SpanOverride;
   closePanel?: () => void;
 }) {
   const organization = useOrganization();
@@ -170,17 +177,26 @@ export function SpanView({
       );
     }
     case "span": {
-      return <SpanBody span={fetcher.data.span} runParam={runParam} closePanel={closePanel} />;
+      return (
+        <SpanBody
+          span={fetcher.data.span}
+          spanOverrides={spanOverrides}
+          runParam={runParam}
+          closePanel={closePanel}
+        />
+      );
     }
   }
 }
 
 function SpanBody({
   span,
+  spanOverrides,
   runParam,
   closePanel,
 }: {
   span: Span;
+  spanOverrides?: SpanOverride;
   runParam?: string;
   closePanel?: () => void;
 }) {
@@ -194,6 +210,8 @@ function SpanBody({
     tab = "overview";
   }
 
+  span = applySpanOverrides(span, spanOverrides);
+
   return (
     <div className="grid h-full max-h-full grid-rows-[2.5rem_2rem_1fr] overflow-hidden bg-background-bright">
       <div className="flex items-center justify-between gap-2 overflow-x-hidden px-3">
@@ -228,88 +246,47 @@ function SpanBody({
           >
             Overview
           </TabButton>
-          <TabButton
-            isActive={tab === "detail"}
-            layoutId="span-span"
-            onClick={() => {
-              replace({ tab: "detail" });
-            }}
-            shortcut={{ key: "d" }}
-          >
-            Detail
-          </TabButton>
         </TabContainer>
       </div>
       <div className="overflow-y-auto scrollbar-thin scrollbar-track-transparent scrollbar-thumb-charcoal-600">
-        {tab === "detail" ? (
-          <div className="flex flex-col gap-4 px-3 pt-3">
-            <Property.Table>
-              <Property.Item>
-                <Property.Label>Status</Property.Label>
-                <Property.Value>
-                  <TaskRunAttemptStatusCombo
-                    status={
-                      span.isCancelled
-                        ? "CANCELED"
-                        : span.isError
-                        ? "FAILED"
-                        : span.isPartial
-                        ? "EXECUTING"
-                        : "COMPLETED"
-                    }
-                    className="text-sm"
-                  />
-                </Property.Value>
-              </Property.Item>
-              <Property.Item>
-                <Property.Label>Task</Property.Label>
-                <Property.Value>
-                  <SimpleTooltip
-                    button={
-                      <TextLink
-                        to={v3RunsPath(organization, project, environment, {
-                          tasks: [span.taskSlug],
-                        })}
-                      >
-                        {span.taskSlug}
-                      </TextLink>
-                    }
-                    content={`Filter runs by ${span.taskSlug}`}
-                  />
-                </Property.Value>
-              </Property.Item>
-              {span.idempotencyKey && (
-                <Property.Item>
-                  <Property.Label>Idempotency key</Property.Label>
-                  <Property.Value>{span.idempotencyKey}</Property.Value>
-                </Property.Item>
-              )}
-              <Property.Item>
-                <Property.Label>Version</Property.Label>
-                <Property.Value>
-                  {span.workerVersion ? (
-                    span.workerVersion
-                  ) : (
-                    <span className="flex items-center gap-1">
-                      <span>Never started</span>
-                      <InfoIconTooltip
-                        content={"Runs get locked to the latest version when they start."}
-                        contentClassName="normal-case tracking-normal"
-                      />
-                    </span>
-                  )}
-                </Property.Value>
-              </Property.Item>
-            </Property.Table>
-          </div>
-        ) : (
-          <SpanEntity span={span} />
-        )}
+        <SpanEntity span={span} />
       </div>
     </div>
   );
 }
 
+function applySpanOverrides(span: Span, spanOverrides?: SpanOverride): Span {
+  if (!spanOverrides) {
+    return span;
+  }
+
+  const newSpan = { ...span };
+
+  if (spanOverrides.isCancelled) {
+    newSpan.isCancelled = true;
+    newSpan.isPartial = false;
+    newSpan.isError = false;
+  } else if (spanOverrides.isError) {
+    newSpan.isError = true;
+    newSpan.isPartial = false;
+    newSpan.isCancelled = false;
+  }
+
+  if (typeof spanOverrides.duration !== "undefined") {
+    newSpan.duration = spanOverrides.duration;
+  }
+
+  if (spanOverrides.events) {
+    if (newSpan.events) {
+      newSpan.events = [...newSpan.events, ...spanOverrides.events];
+    } else {
+      newSpan.events = spanOverrides.events;
+    }
+  }
+
+  return newSpan;
+}
+
 function RunBody({
   run,
   runParam,
@@ -409,6 +386,7 @@ function RunBody({
                     <SimpleTooltip
                       button={<TaskRunStatusCombo status={run.status} />}
                       content={descriptionForTaskRunStatus(run.status)}
+                      disableHoverableContent
                     />
                   </Property.Value>
                 </Property.Item>
@@ -422,82 +400,129 @@ function RunBody({
                             tasks: [run.taskIdentifier],
                           })}
                         >
-                          {run.taskIdentifier}
+                          <CopyableText
+                            value={run.taskIdentifier}
+                            copyValue={run.taskIdentifier}
+                            asChild
+                          />
                         </TextLink>
                       }
-                      content={`Filter runs by ${run.taskIdentifier}`}
+                      content={`View runs filtered by ${run.taskIdentifier}`}
+                      disableHoverableContent
                     />
                   </Property.Value>
                 </Property.Item>
                 {run.relationships.root ? (
                   run.relationships.root.isParent ? (
                     <Property.Item>
-                      <Property.Label>Root & Parent</Property.Label>
+                      <Property.Label>Root & Parent run</Property.Label>
                       <Property.Value>
-                        <TextLink
-                          to={v3RunSpanPath(
-                            organization,
-                            project,
-                            environment,
-                            {
-                              friendlyId: run.relationships.root.friendlyId,
-                            },
-                            { spanId: run.relationships.root.spanId }
-                          )}
-                          className="group flex flex-wrap items-center gap-x-1 gap-y-0"
-                        >
-                          {run.relationships.root.taskIdentifier}
-                          <span className="break-all text-text-dimmed transition-colors group-hover:text-text-bright/80">
-                            ({run.relationships.root.friendlyId})
-                          </span>
-                        </TextLink>
-                      </Property.Value>
-                    </Property.Item>
-                  ) : (
-                    <>
-                      <Property.Item>
-                        <Property.Label>Root</Property.Label>
-                        <Property.Value>
-                          <TextLink
-                            to={v3RunSpanPath(
-                              organization,
-                              project,
-                              environment,
-                              {
-                                friendlyId: run.relationships.root.friendlyId,
-                              },
-                              { spanId: run.relationships.root.spanId }
-                            )}
-                            className="group flex flex-wrap items-center gap-x-1 gap-y-0"
-                          >
-                            {run.relationships.root.taskIdentifier}
-                            <span className="break-all text-text-dimmed transition-colors group-hover:text-text-bright/80">
-                              ({run.relationships.root.friendlyId})
-                            </span>
-                          </TextLink>
-                        </Property.Value>
-                      </Property.Item>
-                      {run.relationships.parent ? (
-                        <Property.Item>
-                          <Property.Label>Parent</Property.Label>
-                          <Property.Value>
+                        <SimpleTooltip
+                          button={
                             <TextLink
                               to={v3RunSpanPath(
                                 organization,
                                 project,
                                 environment,
                                 {
-                                  friendlyId: run.relationships.parent.friendlyId,
+                                  friendlyId: run.relationships.root.friendlyId,
                                 },
-                                { spanId: run.relationships.parent.spanId }
+                                { spanId: run.relationships.root.spanId }
                               )}
                               className="group flex flex-wrap items-center gap-x-1 gap-y-0"
                             >
-                              {run.relationships.parent.taskIdentifier}
+                              <CopyableText
+                                value={run.relationships.root.taskIdentifier}
+                                copyValue={run.relationships.root.taskIdentifier}
+                                asChild
+                              />
                               <span className="break-all text-text-dimmed transition-colors group-hover:text-text-bright/80">
-                                ({run.relationships.parent.friendlyId})
+                                <CopyableText
+                                  value={run.relationships.root.friendlyId}
+                                  copyValue={run.relationships.root.friendlyId}
+                                  asChild
+                                />
                               </span>
                             </TextLink>
+                          }
+                          content={`Jump to root/parent run`}
+                          disableHoverableContent
+                        />
+                      </Property.Value>
+                    </Property.Item>
+                  ) : (
+                    <>
+                      <Property.Item>
+                        <Property.Label>Root run</Property.Label>
+                        <Property.Value>
+                          <SimpleTooltip
+                            button={
+                              <TextLink
+                                to={v3RunSpanPath(
+                                  organization,
+                                  project,
+                                  environment,
+                                  {
+                                    friendlyId: run.relationships.root.friendlyId,
+                                  },
+                                  { spanId: run.relationships.root.spanId }
+                                )}
+                                className="group flex flex-wrap items-center gap-x-1 gap-y-0"
+                              >
+                                <CopyableText
+                                  value={run.relationships.root.taskIdentifier}
+                                  copyValue={run.relationships.root.taskIdentifier}
+                                  asChild
+                                />
+                                <span className="break-all text-text-dimmed transition-colors group-hover:text-text-bright/80">
+                                  <CopyableText
+                                    value={run.relationships.root.friendlyId}
+                                    copyValue={run.relationships.root.friendlyId}
+                                    asChild
+                                  />
+                                </span>
+                              </TextLink>
+                            }
+                            content={`Jump to root run`}
+                            disableHoverableContent
+                          />
+                        </Property.Value>
+                      </Property.Item>
+                      {run.relationships.parent ? (
+                        <Property.Item>
+                          <Property.Label>Parent run</Property.Label>
+                          <Property.Value>
+                            <SimpleTooltip
+                              button={
+                                <TextLink
+                                  to={v3RunSpanPath(
+                                    organization,
+                                    project,
+                                    environment,
+                                    {
+                                      friendlyId: run.relationships.parent.friendlyId,
+                                    },
+                                    { spanId: run.relationships.parent.spanId }
+                                  )}
+                                  className="group flex flex-wrap items-center gap-x-1 gap-y-0"
+                                >
+                                  <CopyableText
+                                    value={run.relationships.parent.taskIdentifier}
+                                    copyValue={run.relationships.parent.taskIdentifier}
+                                    asChild
+                                  />
+                                  <span className="break-all text-text-dimmed transition-colors group-hover:text-text-bright/80">
+                                    <CopyableText
+                                      value={run.relationships.parent.friendlyId}
+                                      copyValue={run.relationships.parent.friendlyId}
+                                      asChild
+                                    />
+                                  </span>
+                                </TextLink>
+                              }
+                              content={`Jump to parent run`}
+                              disableHoverableContent
+                            />
                           </Property.Value>
                         </Property.Item>
                       ) : null}
@@ -511,10 +536,15 @@ function RunBody({
                       <SimpleTooltip
                         button={
                           <TextLink to={v3BatchPath(organization, project, environment, run.batch)}>
-                            {run.batch.friendlyId}
+                            <CopyableText
+                              value={run.batch.friendlyId}
+                              copyValue={run.batch.friendlyId}
+                              asChild
+                            />
                           </TextLink>
                         }
-                        content={`Jump to ${run.batch.friendlyId}`}
+                        content={`View batches filtered by ${run.batch.friendlyId}`}
+                        disableHoverableContent
                       />
                     </Property.Value>
                   </Property.Item>
@@ -540,7 +570,7 @@ function RunBody({
                   <Property.Value>
                     {run.version ? (
                       environment.type === "DEVELOPMENT" ? (
-                        run.version
+                        <CopyableText value={run.version} copyValue={run.version} asChild />
                       ) : (
                         <SimpleTooltip
                           button={
@@ -553,7 +583,7 @@ function RunBody({
                               )}
                               className="group flex flex-wrap items-center gap-x-1 gap-y-0"
                             >
-                              {run.version}
+                              <CopyableText value={run.version} copyValue={run.version} asChild />
                             </TextLink>
                           }
                           content={"Jump to deployment"}
@@ -606,13 +636,23 @@ function RunBody({
                   <Property.Item>
                     <Property.Label>Replayed from</Property.Label>
                     <Property.Value>
-                      <TextLink
-                        to={v3RunRedirectPath(organization, project, {
-                          friendlyId: run.replayedFromTaskRunFriendlyId,
-                        })}
-                      >
-                        {run.replayedFromTaskRunFriendlyId}
-                      </TextLink>
+                      <SimpleTooltip
+                        button={
+                          <TextLink
+                            to={v3RunRedirectPath(organization, project, {
+                              friendlyId: run.replayedFromTaskRunFriendlyId,
+                            })}
+                          >
+                            <CopyableText
+                              value={run.replayedFromTaskRunFriendlyId}
+                              copyValue={run.replayedFromTaskRunFriendlyId}
+                              asChild
+                            />
+                          </TextLink>
+                        }
+                        content={`Jump to replayed run`}
+                        disableHoverableContent
+                      />
                     </Property.Value>
                   </Property.Item>
                 )}
@@ -747,11 +787,15 @@ function RunBody({
                 </Property.Item>
                 <Property.Item>
                   <Property.Label>Run ID</Property.Label>
-                  <Property.Value>{run.friendlyId}</Property.Value>
+                  <Property.Value>
+                    <CopyableText value={run.friendlyId} copyValue={run.friendlyId} asChild />
+                  </Property.Value>
                 </Property.Item>
                 <Property.Item>
                   <Property.Label>Internal ID</Property.Label>
-                  <Property.Value>{run.id}</Property.Value>
+                  <Property.Value>
+                    <CopyableText value={run.id} copyValue={run.id} asChild />
+                  </Property.Value>
                 </Property.Item>
                 <Property.Item>
                   <Property.Label>Run Engine</Property.Label>
@@ -772,6 +816,14 @@ function RunBody({
                       <Property.Label>Worker queue</Property.Label>
                       <Property.Value>{run.workerQueue}</Property.Value>
                     </Property.Item>
+                    <Property.Item>
+                      <Property.Label>Trace ID</Property.Label>
+                      <Property.Value>{run.traceId}</Property.Value>
+                    </Property.Item>
+                    <Property.Item>
+                      <Property.Label>Span ID</Property.Label>
+                      <Property.Value>{run.spanId}</Property.Value>
+                    </Property.Item>
                   </div>
                 )}
               </Property.Table>
@@ -1002,7 +1054,7 @@ function SpanEntity({ span }: { span: Span }) {
                             {run.taskIdentifier}
                           </TableCell>
                           <TableCell to={path} actionClassName="py-1.5" rowHoverStyle="bright">
-                            {run.lockedToVersion?.version ?? "–"}
+                            {run.taskVersion ?? "–"}
                           </TableCell>
                           <TableCell to={path} actionClassName="py-1.5" rowHoverStyle="bright">
                             <DateTime date={run.createdAt} />

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.ai-filter.tsx

@@ -65,13 +65,16 @@ export async function action({ request, params }: ActionFunctionArgs) {
     query: async (search) => {
       const tagPresenter = new RunTagListPresenter();
       const tags = await tagPresenter.call({
+        organizationId: environment.organizationId,
         projectId: environment.projectId,
+        environmentId: environment.id,
         name: search,
         page: 1,
         pageSize: 50,
+        period: "30d",
       });
       return {
-        tags: tags.tags.map((t) => t.name),
+        tags: tags.tags,
       };
     },
   };

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.bulkaction.$bulkActionParam.stream.tsx

@@ -1,103 +0,0 @@
-import { z } from "zod";
-import { $replica } from "~/db.server";
-import { env } from "~/env.server";
-import { logger } from "~/services/logger.server";
-import { requireUserId } from "~/services/session.server";
-import { EnvironmentParamSchema } from "~/utils/pathBuilder";
-import { createSSELoader, type SendFunction } from "~/utils/sse";
-
-const Params = EnvironmentParamSchema.extend({
-  bulkActionParam: z.string(),
-});
-
-export const loader = createSSELoader({
-  timeout: env.DEV_PRESENCE_SSE_TIMEOUT,
-  interval: env.DEV_PRESENCE_POLL_MS,
-  debug: false,
-  handler: async ({ id, controller, debug, request, params }) => {
-    const userId = await requireUserId(request);
-    const { organizationSlug, projectParam, envParam, bulkActionParam } = Params.parse(params);
-
-    const environment = await $replica.runtimeEnvironment.findFirst({
-      where: {
-        id: envParam,
-        project: {
-          slug: projectParam,
-          organization: {
-            members: {
-              some: {
-                userId,
-              },
-            },
-          },
-        },
-      },
-    });
-
-    if (!environment) {
-      throw new Response("Not Found", { status: 404 });
-    }
-
-    const getBulkActionProgress = async (send: SendFunction) => {
-      try {
-        const bulkAction = await $replica.bulkActionGroup.findFirst({
-          select: {
-            status: true,
-            successCount: true,
-            failureCount: true,
-          },
-          where: {
-            friendlyId: bulkActionParam,
-            environmentId: environment.id,
-          },
-        });
-
-        send({
-          event: "progress",
-          data: JSON.stringify({
-            status: bulkAction?.status,
-            successCount: bulkAction?.successCount,
-            failureCount: bulkAction?.failureCount,
-          }),
-        });
-
-        return bulkAction;
-      } catch (error) {
-        // Handle the case where the controller is closed
-        logger.debug("Failed to send bulk action progress data, stream might be closed", { error });
-        return null;
-      }
-    };
-
-    return {
-      beforeStream: async () => {
-        logger.debug("Start dev presence listening SSE session", {
-          environmentId: environment.id,
-        });
-      },
-      initStream: async ({ send }) => {
-        const bulkAction = await getBulkActionProgress(send);
-
-        send({ event: "time", data: new Date().toISOString() });
-
-        if (bulkAction?.status !== "PENDING") {
-          return false;
-        }
-
-        return true;
-      },
-      iterator: async ({ send, date }) => {
-        const bulkAction = await getBulkActionProgress(send);
-
-        if (bulkAction?.status !== "PENDING") {
-          return false;
-        }
-
-        return true;
-      },
-      cleanup: async ({ send }) => {
-        await getBulkActionProgress(send);
-      },
-    };
-  },
-});

apps/webapp/app/routes/resources.orgs.$organizationSlug.select-plan.tsx

@@ -7,16 +7,16 @@ import {
 } from "@heroicons/react/20/solid";
 import { ArrowDownCircleIcon, ArrowUpCircleIcon } from "@heroicons/react/24/outline";
 import { Form, useLocation, useNavigation } from "@remix-run/react";
-import { ActionFunctionArgs } from "@remix-run/server-runtime";
+import { type ActionFunctionArgs } from "@remix-run/server-runtime";
 import { uiComponent } from "@team-plain/typescript-sdk";
 import { GitHubLightIcon } from "@trigger.dev/companyicons";
 import {
-  FreePlanDefinition,
-  Limits,
-  PaidPlanDefinition,
-  Plans,
-  SetPlanBody,
-  SubscriptionResult,
+  type FreePlanDefinition,
+  type Limits,
+  type PaidPlanDefinition,
+  type Plans,
+  type SetPlanBody,
+  type SubscriptionResult,
 } from "@trigger.dev/platform";
 import React, { useEffect, useState } from "react";
 import { z } from "zod";
@@ -75,8 +75,8 @@ export async function action({ request, params }: ActionFunctionArgs) {
     message: message || undefined,
   });
 
-  const organization = await prisma.organization.findUnique({
-    where: { slug: organizationSlug },
+  const organization = await prisma.organization.findFirst({
+    where: { slug: organizationSlug, members: { some: { userId: user.id } } },
   });
 
   if (!organization) {

apps/webapp/app/routes/resources.projects.$projectParam.runs.tags.tsx

@@ -1,32 +0,0 @@
-import { type LoaderFunctionArgs } from "@remix-run/server-runtime";
-import { z } from "zod";
-import { $replica } from "~/db.server";
-import { RunTagListPresenter } from "~/presenters/v3/RunTagListPresenter.server";
-import { requireUserId } from "~/services/session.server";
-
-const Params = z.object({
-  projectParam: z.string(),
-});
-
-export async function loader({ request, params }: LoaderFunctionArgs) {
-  const userId = await requireUserId(request);
-  const { projectParam } = Params.parse(params);
-
-  const project = await $replica.project.findFirst({
-    where: { slug: projectParam, deletedAt: null, organization: { members: { some: { userId } } } },
-  });
-
-  if (!project) {
-    throw new Response("Not Found", { status: 404 });
-  }
-
-  const search = new URL(request.url).searchParams;
-  const name = search.get("name");
-
-  const presenter = new RunTagListPresenter();
-  const result = await presenter.call({
-    projectId: project.id,
-    name: name ? decodeURIComponent(name) : undefined,
-  });
-  return result;
-}

apps/webapp/app/routes/resources.runs.$runParam.logs.download.ts

@@ -2,13 +2,13 @@ import { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { prisma } from "~/db.server";
 import { requireUser } from "~/services/session.server";
 import { v3RunParamsSchema } from "~/utils/pathBuilder";
-import { RunPreparedEvent, eventRepository } from "~/v3/eventRepository.server";
+import type { RunPreparedEvent } from "~/v3/eventRepository/eventRepository.types";
 import { createGzip } from "zlib";
 import { Readable } from "stream";
 import { formatDurationMilliseconds } from "@trigger.dev/core/v3/utils/durations";
-import { getDateFromNanoseconds } from "~/utils/taskEvent";
 import { getTaskEventStoreTableForRun } from "~/v3/taskEventStore.server";
 import { TaskEventKind } from "@trigger.dev/database";
+import { resolveEventRepositoryForStore } from "~/v3/eventRepository/index.server";
 
 export async function loader({ params, request }: LoaderFunctionArgs) {
   const user = await requireUser(request);
@@ -33,8 +33,12 @@ export async function loader({ params, request }: LoaderFunctionArgs) {
     return new Response("Not found", { status: 404 });
   }
 
+  const eventRepository = resolveEventRepositoryForStore(run.taskEventStore);
+
   const runEvents = await eventRepository.getRunEvents(
     getTaskEventStoreTableForRun(run),
+    run.runtimeEnvironmentId,
+    run.traceId,
     run.friendlyId,
     run.createdAt,
     run.completedAt ?? undefined
@@ -117,3 +121,7 @@ function formatRunEvent(event: RunPreparedEvent): string {
 
   return entries.join("\n");
 }
+
+function getDateFromNanoseconds(nanoseconds: bigint) {
+  return new Date(Number(nanoseconds) / 1_000_000);
+}

apps/webapp/app/routes/resources.taskruns.$runParam.cancel.ts

@@ -1,9 +1,10 @@
 import { parse } from "@conform-to/zod";
-import { ActionFunction, json } from "@remix-run/node";
+import { type ActionFunction, json } from "@remix-run/node";
 import { z } from "zod";
 import { prisma } from "~/db.server";
 import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
 import { logger } from "~/services/logger.server";
+import { requireUserId } from "~/services/session.server";
 import { CancelTaskRunService } from "~/v3/services/cancelTaskRun.server";
 
 export const cancelSchema = z.object({
@@ -15,6 +16,7 @@ const ParamSchema = z.object({
 });
 
 export const action: ActionFunction = async ({ request, params }) => {
+  const userId = await requireUserId(request);
   const { runParam } = ParamSchema.parse(params);
 
   const formData = await request.formData();
@@ -25,9 +27,18 @@ export const action: ActionFunction = async ({ request, params }) => {
   }
 
   try {
-    const taskRun = await prisma.taskRun.findUnique({
+    const taskRun = await prisma.taskRun.findFirst({
       where: {
         friendlyId: runParam,
+        project: {
+          organization: {
+            members: {
+              some: {
+                userId,
+              },
+            },
+          },
+        },
       },
     });
 

apps/webapp/app/routes/resources.taskruns.$runParam.debug.ts

@@ -1,4 +1,4 @@
-import { LoaderFunctionArgs } from "@remix-run/node";
+import { type LoaderFunctionArgs } from "@remix-run/node";
 import { typedjson } from "remix-typedjson";
 import { z } from "zod";
 import { $replica } from "~/db.server";

apps/webapp/app/runEngine/concerns/idempotencyKeys.server.ts

@@ -17,7 +17,10 @@ export class IdempotencyKeyConcern {
     private readonly traceEventConcern: TraceEventConcern
   ) {}
 
-  async handleTriggerRequest(request: TriggerTaskRequest): Promise<IdempotencyKeyConcernResult> {
+  async handleTriggerRequest(
+    request: TriggerTaskRequest,
+    parentStore: string | undefined
+  ): Promise<IdempotencyKeyConcernResult> {
     const idempotencyKey = request.options?.idempotencyKey ?? request.body.options?.idempotencyKey;
     const idempotencyKeyExpiresAt =
       request.options?.idempotencyKeyExpiresAt ??
@@ -83,6 +86,7 @@ export class IdempotencyKeyConcern {
       if (associatedWaitpoint && resumeParentOnCompletion && parentRunId) {
         await this.traceEventConcern.traceIdempotentRun(
           request,
+          parentStore,
           {
             existingRun,
             idempotencyKey,
@@ -90,11 +94,18 @@ export class IdempotencyKeyConcern {
             isError: associatedWaitpoint.outputIsError,
           },
           async (event) => {
+            const spanId =
+              request.options?.parentAsLinkType === "replay"
+                ? event.spanId
+                : event.traceparent?.spanId
+                ? `${event.traceparent.spanId}:${event.spanId}`
+                : event.spanId;
+
             //block run with waitpoint
             await this.engine.blockRunWithWaitpoint({
               runId: RunId.fromFriendlyId(parentRunId),
               waitpoints: associatedWaitpoint.id,
-              spanIdToComplete: event.spanId,
+              spanIdToComplete: spanId,
               batch: request.options?.batchId
                 ? {
                     id: request.options.batchId,

apps/webapp/app/runEngine/concerns/traceEvents.server.ts

@@ -1,41 +1,40 @@
-import { EventRepository } from "~/v3/eventRepository.server";
-import { TracedEventSpan, TraceEventConcern, TriggerTaskRequest } from "../types";
 import { SemanticInternalAttributes } from "@trigger.dev/core/v3/semanticInternalAttributes";
-import { BatchId } from "@trigger.dev/core/v3/isomorphic";
 import { TaskRun } from "@trigger.dev/database";
+import { IEventRepository } from "~/v3/eventRepository/eventRepository.types";
+import { getEventRepository } from "~/v3/eventRepository/index.server";
+import { TracedEventSpan, TraceEventConcern, TriggerTaskRequest } from "../types";
 
 export class DefaultTraceEventsConcern implements TraceEventConcern {
-  private readonly eventRepository: EventRepository;
-
-  constructor(eventRepository: EventRepository) {
-    this.eventRepository = eventRepository;
+  async #getEventRepository(
+    request: TriggerTaskRequest,
+    parentStore: string | undefined
+  ): Promise<{ repository: IEventRepository; store: string }> {
+    return await getEventRepository(
+      request.environment.organization.featureFlags as Record<string, unknown>,
+      parentStore
+    );
   }
 
   async traceRun<T>(
     request: TriggerTaskRequest,
-    callback: (span: TracedEventSpan) => Promise<T>
+    parentStore: string | undefined,
+    callback: (span: TracedEventSpan, store: string) => Promise<T>
   ): Promise<T> {
-    return await this.eventRepository.traceEvent(
+    const { repository, store } = await this.#getEventRepository(request, parentStore);
+
+    return await repository.traceEvent(
       request.taskId,
       {
         context: request.options?.traceContext,
         spanParentAsLink: request.options?.spanParentAsLink,
-        parentAsLinkType: request.options?.parentAsLinkType,
         kind: "SERVER",
         environment: request.environment,
         taskSlug: request.taskId,
         attributes: {
-          properties: {
-            [SemanticInternalAttributes.SHOW_ACTIONS]: true,
-          },
+          properties: {},
           style: {
             icon: request.options?.customIcon ?? "task",
           },
-          runIsTest: request.body.options?.test ?? false,
-          batchId: request.options?.batchId
-            ? BatchId.toFriendlyId(request.options.batchId)
-            : undefined,
-          idempotencyKey: request.options?.idempotencyKey,
         },
         incomplete: true,
         immediate: true,
@@ -44,52 +43,50 @@ export class DefaultTraceEventsConcern implements TraceEventConcern {
           : undefined,
       },
       async (event, traceContext, traceparent) => {
-        return await callback({
-          traceId: event.traceId,
-          spanId: event.spanId,
-          traceContext,
-          traceparent,
-          setAttribute: (key, value) => event.setAttribute(key as any, value),
-          failWithError: event.failWithError.bind(event),
-        });
+        return await callback(
+          {
+            traceId: event.traceId,
+            spanId: event.spanId,
+            traceContext,
+            traceparent,
+            setAttribute: (key, value) => event.setAttribute(key as any, value),
+            failWithError: event.failWithError.bind(event),
+          },
+          store
+        );
       }
     );
   }
 
   async traceIdempotentRun<T>(
     request: TriggerTaskRequest,
+    parentStore: string | undefined,
     options: {
       existingRun: TaskRun;
       idempotencyKey: string;
       incomplete: boolean;
       isError: boolean;
     },
-    callback: (span: TracedEventSpan) => Promise<T>
+    callback: (span: TracedEventSpan, store: string) => Promise<T>
   ): Promise<T> {
     const { existingRun, idempotencyKey, incomplete, isError } = options;
+    const { repository, store } = await this.#getEventRepository(request, parentStore);
 
-    return await this.eventRepository.traceEvent(
+    return await repository.traceEvent(
       `${request.taskId} (cached)`,
       {
         context: request.options?.traceContext,
         spanParentAsLink: request.options?.spanParentAsLink,
-        parentAsLinkType: request.options?.parentAsLinkType,
         kind: "SERVER",
         environment: request.environment,
         taskSlug: request.taskId,
         attributes: {
           properties: {
-            [SemanticInternalAttributes.SHOW_ACTIONS]: true,
             [SemanticInternalAttributes.ORIGINAL_RUN_ID]: existingRun.friendlyId,
           },
           style: {
             icon: "task-cached",
           },
-          runIsTest: request.body.options?.test ?? false,
-          batchId: request.options?.batchId
-            ? BatchId.toFriendlyId(request.options.batchId)
-            : undefined,
-          idempotencyKey,
           runId: existingRun.friendlyId,
         },
         incomplete,
@@ -98,7 +95,7 @@ export class DefaultTraceEventsConcern implements TraceEventConcern {
       },
       async (event, traceContext, traceparent) => {
         //log a message
-        await this.eventRepository.recordEvent(
+        await repository.recordEvent(
           `There's an existing run for idempotencyKey: ${idempotencyKey}`,
           {
             taskSlug: request.taskId,
@@ -111,14 +108,17 @@ export class DefaultTraceEventsConcern implements TraceEventConcern {
           }
         );
 
-        return await callback({
-          traceId: event.traceId,
-          spanId: event.spanId,
-          traceContext,
-          traceparent,
-          setAttribute: (key, value) => event.setAttribute(key as any, value),
-          failWithError: event.failWithError.bind(event),
-        });
+        return await callback(
+          {
+            traceId: event.traceId,
+            spanId: event.spanId,
+            traceContext,
+            traceparent,
+            setAttribute: (key, value) => event.setAttribute(key as any, value),
+            failWithError: event.failWithError.bind(event),
+          },
+          store
+        );
       }
     );
   }

apps/webapp/app/runEngine/services/triggerTask.server.ts

@@ -29,7 +29,6 @@ import type {
   TriggerTaskServiceOptions,
   TriggerTaskServiceResult,
 } from "../../v3/services/triggerTask.server";
-import { getTaskEventStore } from "../../v3/taskEventStore.server";
 import { clampMaxDuration } from "../../v3/utils/maxDuration";
 import { IdempotencyKeyConcern } from "../concerns/idempotencyKeys.server";
 import type {
@@ -198,7 +197,8 @@ export class RunEngineTriggerTaskService {
       }
 
       const idempotencyKeyConcernResult = await this.idempotencyKeyConcern.handleTriggerRequest(
-        triggerRequest
+        triggerRequest,
+        parentRun?.taskEventStore
       );
 
       if (idempotencyKeyConcernResult.isCached) {
@@ -267,105 +267,109 @@ export class RunEngineTriggerTaskService {
       const workerQueue = await this.queueConcern.getWorkerQueue(environment, body.options?.region);
 
       try {
-        return await this.traceEventConcern.traceRun(triggerRequest, async (event) => {
-          const result = await this.runNumberIncrementer.incrementRunNumber(
-            triggerRequest,
-            async (num) => {
-              event.setAttribute("queueName", queueName);
-              span.setAttribute("queueName", queueName);
-              event.setAttribute("runId", runFriendlyId);
-              span.setAttribute("runId", runFriendlyId);
+        return await this.traceEventConcern.traceRun(
+          triggerRequest,
+          parentRun?.taskEventStore,
+          async (event, store) => {
+            const result = await this.runNumberIncrementer.incrementRunNumber(
+              triggerRequest,
+              async (num) => {
+                event.setAttribute("queueName", queueName);
+                span.setAttribute("queueName", queueName);
+                event.setAttribute("runId", runFriendlyId);
+                span.setAttribute("runId", runFriendlyId);
 
-              const payloadPacket = await this.payloadProcessor.process(triggerRequest);
+                const payloadPacket = await this.payloadProcessor.process(triggerRequest);
 
-              const taskRun = await this.engine.trigger(
-                {
-                  number: num,
-                  friendlyId: runFriendlyId,
-                  environment: environment,
-                  idempotencyKey,
-                  idempotencyKeyExpiresAt: idempotencyKey ? idempotencyKeyExpiresAt : undefined,
-                  taskIdentifier: taskId,
-                  payload: payloadPacket.data ?? "",
-                  payloadType: payloadPacket.dataType,
-                  context: body.context,
-                  traceContext: this.#propagateExternalTraceContext(
-                    event.traceContext,
-                    parentRun?.traceContext,
-                    event.traceparent?.spanId
-                  ),
-                  traceId: event.traceId,
-                  spanId: event.spanId,
-                  parentSpanId:
-                    options.parentAsLinkType === "replay" ? undefined : event.traceparent?.spanId,
-                  replayedFromTaskRunFriendlyId: options.replayedFromTaskRunFriendlyId,
-                  lockedToVersionId: lockedToBackgroundWorker?.id,
-                  taskVersion: lockedToBackgroundWorker?.version,
-                  sdkVersion: lockedToBackgroundWorker?.sdkVersion,
-                  cliVersion: lockedToBackgroundWorker?.cliVersion,
-                  concurrencyKey: body.options?.concurrencyKey,
-                  queue: queueName,
-                  lockedQueueId,
-                  workerQueue,
-                  isTest: body.options?.test ?? false,
-                  delayUntil,
-                  queuedAt: delayUntil ? undefined : new Date(),
-                  maxAttempts: body.options?.maxAttempts,
-                  taskEventStore: getTaskEventStore(),
-                  ttl,
-                  tags,
-                  oneTimeUseToken: options.oneTimeUseToken,
-                  parentTaskRunId: parentRun?.id,
-                  rootTaskRunId: parentRun?.rootTaskRunId ?? parentRun?.id,
-                  batch: options?.batchId
-                    ? {
-                        id: options.batchId,
-                        index: options.batchIndex ?? 0,
-                      }
-                    : undefined,
-                  resumeParentOnCompletion: body.options?.resumeParentOnCompletion,
-                  depth,
-                  metadata: metadataPacket?.data,
-                  metadataType: metadataPacket?.dataType,
-                  seedMetadata: metadataPacket?.data,
-                  seedMetadataType: metadataPacket?.dataType,
-                  maxDurationInSeconds: body.options?.maxDuration
-                    ? clampMaxDuration(body.options.maxDuration)
-                    : undefined,
-                  machine: body.options?.machine,
-                  priorityMs: body.options?.priority ? body.options.priority * 1_000 : undefined,
-                  queueTimestamp:
-                    options.queueTimestamp ??
-                    (parentRun && body.options?.resumeParentOnCompletion
-                      ? parentRun.queueTimestamp ?? undefined
-                      : undefined),
-                  scheduleId: options.scheduleId,
-                  scheduleInstanceId: options.scheduleInstanceId,
-                  createdAt: options.overrideCreatedAt,
-                  bulkActionId: body.options?.bulkActionId,
-                  planType,
-                },
-                this.prisma
-              );
+                const taskRun = await this.engine.trigger(
+                  {
+                    number: num,
+                    friendlyId: runFriendlyId,
+                    environment: environment,
+                    idempotencyKey,
+                    idempotencyKeyExpiresAt: idempotencyKey ? idempotencyKeyExpiresAt : undefined,
+                    taskIdentifier: taskId,
+                    payload: payloadPacket.data ?? "",
+                    payloadType: payloadPacket.dataType,
+                    context: body.context,
+                    traceContext: this.#propagateExternalTraceContext(
+                      event.traceContext,
+                      parentRun?.traceContext,
+                      event.traceparent?.spanId
+                    ),
+                    traceId: event.traceId,
+                    spanId: event.spanId,
+                    parentSpanId:
+                      options.parentAsLinkType === "replay" ? undefined : event.traceparent?.spanId,
+                    replayedFromTaskRunFriendlyId: options.replayedFromTaskRunFriendlyId,
+                    lockedToVersionId: lockedToBackgroundWorker?.id,
+                    taskVersion: lockedToBackgroundWorker?.version,
+                    sdkVersion: lockedToBackgroundWorker?.sdkVersion,
+                    cliVersion: lockedToBackgroundWorker?.cliVersion,
+                    concurrencyKey: body.options?.concurrencyKey,
+                    queue: queueName,
+                    lockedQueueId,
+                    workerQueue,
+                    isTest: body.options?.test ?? false,
+                    delayUntil,
+                    queuedAt: delayUntil ? undefined : new Date(),
+                    maxAttempts: body.options?.maxAttempts,
+                    taskEventStore: store,
+                    ttl,
+                    tags,
+                    oneTimeUseToken: options.oneTimeUseToken,
+                    parentTaskRunId: parentRun?.id,
+                    rootTaskRunId: parentRun?.rootTaskRunId ?? parentRun?.id,
+                    batch: options?.batchId
+                      ? {
+                          id: options.batchId,
+                          index: options.batchIndex ?? 0,
+                        }
+                      : undefined,
+                    resumeParentOnCompletion: body.options?.resumeParentOnCompletion,
+                    depth,
+                    metadata: metadataPacket?.data,
+                    metadataType: metadataPacket?.dataType,
+                    seedMetadata: metadataPacket?.data,
+                    seedMetadataType: metadataPacket?.dataType,
+                    maxDurationInSeconds: body.options?.maxDuration
+                      ? clampMaxDuration(body.options.maxDuration)
+                      : undefined,
+                    machine: body.options?.machine,
+                    priorityMs: body.options?.priority ? body.options.priority * 1_000 : undefined,
+                    queueTimestamp:
+                      options.queueTimestamp ??
+                      (parentRun && body.options?.resumeParentOnCompletion
+                        ? parentRun.queueTimestamp ?? undefined
+                        : undefined),
+                    scheduleId: options.scheduleId,
+                    scheduleInstanceId: options.scheduleInstanceId,
+                    createdAt: options.overrideCreatedAt,
+                    bulkActionId: body.options?.bulkActionId,
+                    planType,
+                  },
+                  this.prisma
+                );
 
-              const error = taskRun.error ? TaskRunError.parse(taskRun.error) : undefined;
+                const error = taskRun.error ? TaskRunError.parse(taskRun.error) : undefined;
 
-              if (error) {
-                event.failWithError(error);
+                if (error) {
+                  event.failWithError(error);
+                }
+
+                return { run: taskRun, error, isCached: false };
               }
-
-              return { run: taskRun, error, isCached: false };
-            }
-          );
-
-          if (result?.error) {
-            throw new ServiceValidationError(
-              taskRunErrorToString(taskRunErrorEnhancer(result.error))
             );
-          }
 
-          return result;
-        });
+            if (result?.error) {
+              throw new ServiceValidationError(
+                taskRunErrorToString(taskRunErrorEnhancer(result.error))
+              );
+            }
+
+            return result;
+          }
+        );
       } catch (error) {
         if (error instanceof RunDuplicateIdempotencyKeyError) {
           //retry calling this function, because this time it will return the idempotent run

apps/webapp/app/runEngine/types.ts

@@ -143,17 +143,19 @@ export type TracedEventSpan = {
 export interface TraceEventConcern {
   traceRun<T>(
     request: TriggerTaskRequest,
-    callback: (span: TracedEventSpan) => Promise<T>
+    parentStore: string | undefined,
+    callback: (span: TracedEventSpan, store: string) => Promise<T>
   ): Promise<T>;
   traceIdempotentRun<T>(
     request: TriggerTaskRequest,
+    parentStore: string | undefined,
     options: {
       existingRun: TaskRun;
       idempotencyKey: string;
       incomplete: boolean;
       isError: boolean;
     },
-    callback: (span: TracedEventSpan) => Promise<T>
+    callback: (span: TracedEventSpan, store: string) => Promise<T>
   ): Promise<T>;
 }
 

apps/webapp/app/services/apiRateLimit.server.ts

@@ -22,6 +22,7 @@ export const apiRateLimiter = authorizationRateLimitMiddleware({
   limiterCache: {
     fresh: 60_000 * 10, // Data is fresh for 10 minutes
     stale: 60_000 * 20, // Date is stale after 20 minutes
+    maxItems: 1000,
   },
   limiterConfigOverride: async (authorizationValue) => {
     const authenticatedEnv = await authenticateAuthorizationHeader(authorizationValue, {

apps/webapp/app/services/authorizationRateLimitMiddleware.server.ts

@@ -2,14 +2,13 @@ import { createCache, DefaultStatefulContext, Namespace, Cache as UnkeyCache } f
 import { MemoryStore } from "@unkey/cache/stores";
 import { Ratelimit } from "@upstash/ratelimit";
 import { Request as ExpressRequest, Response as ExpressResponse, NextFunction } from "express";
-import { RedisOptions } from "ioredis";
 import { createHash } from "node:crypto";
 import { z } from "zod";
 import { env } from "~/env.server";
+import { RedisWithClusterOptions } from "~/redis.server";
 import { logger } from "./logger.server";
 import { createRedisRateLimitClient, Duration, RateLimiter } from "./rateLimiter.server";
 import { RedisCacheStore } from "./unkey/redisCacheStore.server";
-import { RedisWithClusterOptions } from "~/redis.server";
 
 const DurationSchema = z.custom<Duration>((value) => {
   if (typeof value !== "string") {
@@ -64,6 +63,7 @@ type Options = {
   limiterCache?: {
     fresh: number;
     stale: number;
+    maxItems: number;
   };
   log?: {
     requests?: boolean;
@@ -145,7 +145,10 @@ export function authorizationRateLimitMiddleware({
   limiterConfigOverride,
 }: Options) {
   const ctx = new DefaultStatefulContext();
-  const memory = new MemoryStore({ persistentMap: new Map() });
+  const memory = new MemoryStore({
+    persistentMap: new Map(),
+    unstableEvictOnSet: { frequency: 0.001, maxItems: limiterCache?.maxItems ?? 1000 },
+  });
   const redisCacheStore = new RedisCacheStore({
     connection: {
       keyPrefix: `cache:${keyPrefix}:rate-limit-cache:`,

apps/webapp/app/services/engineRateLimit.server.ts

@@ -1,5 +1,4 @@
 import { env } from "~/env.server";
-import { authenticateAuthorizationHeader } from "./apiAuth.server";
 import { authorizationRateLimitMiddleware } from "./authorizationRateLimitMiddleware.server";
 import { Duration } from "./rateLimiter.server";
 
@@ -22,6 +21,7 @@ export const engineRateLimiter = authorizationRateLimitMiddleware({
   limiterCache: {
     fresh: 60_000 * 10, // Data is fresh for 10 minutes
     stale: 60_000 * 20, // Date is stale after 20 minutes
+    maxItems: 1000,
   },
   pathMatchers: [/^\/engine/],
   // Regex allow any path starting with /engine/v1/worker-actions/

apps/webapp/app/services/magicLinkRateLimiter.server.ts

@@ -0,0 +1,96 @@
+import { Ratelimit } from "@upstash/ratelimit";
+import { env } from "~/env.server";
+import { createRedisRateLimitClient, RateLimiter } from "~/services/rateLimiter.server";
+import { singleton } from "~/utils/singleton";
+
+export class MagicLinkRateLimitError extends Error {
+  public readonly retryAfter: number;
+
+  constructor(retryAfter: number) {
+    super("Magic link request rate limit exceeded.");
+    this.retryAfter = retryAfter;
+  }
+}
+
+function getRedisClient() {
+  return createRedisRateLimitClient({
+    port: env.RATE_LIMIT_REDIS_PORT,
+    host: env.RATE_LIMIT_REDIS_HOST,
+    username: env.RATE_LIMIT_REDIS_USERNAME,
+    password: env.RATE_LIMIT_REDIS_PASSWORD,
+    tlsDisabled: env.RATE_LIMIT_REDIS_TLS_DISABLED === "true",
+    clusterMode: env.RATE_LIMIT_REDIS_CLUSTER_MODE_ENABLED === "1",
+  });
+}
+
+const magicLinkEmailRateLimiter = singleton(
+  "magicLinkEmailRateLimiter",
+  initializeMagicLinkEmailRateLimiter
+);
+
+function initializeMagicLinkEmailRateLimiter() {
+  return new RateLimiter({
+    redisClient: getRedisClient(),
+    keyPrefix: "auth:magiclink:email",
+    limiter: Ratelimit.slidingWindow(3, "1 m"), // 3 requests per minute per email
+    logSuccess: false,
+    logFailure: true,
+  });
+}
+
+const magicLinkEmailDailyRateLimiter = singleton(
+  "magicLinkEmailDailyRateLimiter",
+  initializeMagicLinkEmailDailyRateLimiter
+);
+
+function initializeMagicLinkEmailDailyRateLimiter() {
+  return new RateLimiter({
+    redisClient: getRedisClient(),
+    keyPrefix: "auth:magiclink:email:daily",
+    limiter: Ratelimit.slidingWindow(30, "1 d"), // 30 requests per day per email
+    logSuccess: false,
+    logFailure: true,
+  });
+}
+
+const magicLinkIpRateLimiter = singleton(
+  "magicLinkIpRateLimiter",
+  initializeMagicLinkIpRateLimiter
+);
+
+function initializeMagicLinkIpRateLimiter() {
+  return new RateLimiter({
+    redisClient: getRedisClient(),
+    keyPrefix: "auth:magiclink:ip",
+    limiter: Ratelimit.slidingWindow(10, "1 m"), // 10 requests per minute per IP
+    logSuccess: false,
+    logFailure: true,
+  });
+}
+
+export async function checkMagicLinkEmailRateLimit(identifier: string): Promise<void> {
+  const result = await magicLinkEmailRateLimiter.limit(identifier);
+
+  if (!result.success) {
+    const retryAfter = new Date(result.reset).getTime() - Date.now();
+    throw new MagicLinkRateLimitError(retryAfter);
+  }
+}
+
+export async function checkMagicLinkEmailDailyRateLimit(identifier: string): Promise<void> {
+  const result = await magicLinkEmailDailyRateLimiter.limit(identifier);
+
+  if (!result.success) {
+    const retryAfter = new Date(result.reset).getTime() - Date.now();
+    throw new MagicLinkRateLimitError(retryAfter);
+  }
+}
+
+export async function checkMagicLinkIpRateLimit(ip: string): Promise<void> {
+  const result = await magicLinkIpRateLimiter.limit(ip);
+
+  if (!result.success) {
+    const retryAfter = new Date(result.reset).getTime() - Date.now();
+    throw new MagicLinkRateLimitError(retryAfter);
+  }
+}

apps/webapp/app/services/personalAccessToken.server.ts

@@ -79,15 +79,20 @@ export async function getPersonalAccessTokenFromAuthorizationCode(authorizationC
   };
 }
 
-export async function revokePersonalAccessToken(tokenId: string) {
-  await prisma.personalAccessToken.update({
+export async function revokePersonalAccessToken(tokenId: string, userId: string) {
+  const result = await prisma.personalAccessToken.updateMany({
     where: {
       id: tokenId,
+      userId,
     },
     data: {
       revokedAt: new Date(),
     },
   });
+
+  if (result.count === 0) {
+    throw new Error("PAT not found or already revoked");
+  }
 }
 
 export type PersonalAccessTokenAuthenticationResult = {

apps/webapp/app/services/platform.v3.server.ts

@@ -44,7 +44,13 @@ const client = singleton("billingClient", initializeClient);
 
 function initializePlatformCache() {
   const ctx = new DefaultStatefulContext();
-  const memory = new MemoryStore({ persistentMap: new Map() });
+  const memory = new MemoryStore({
+    persistentMap: new Map(),
+    unstableEvictOnSet: {
+      frequency: 0.01,
+      maxItems: 1000,
+    },
+  });
   const redisCacheStore = new RedisCacheStore({
     connection: {
       keyPrefix: "tr:cache:platform:v3",
@@ -199,7 +205,7 @@ export async function getCurrentPlan(orgId: string) {
     firstDayOfNextMonth.setUTCHours(0, 0, 0, 0);
 
     if (!result.success) {
-      logger.error("Error getting current plan", { orgId, error: result.error });
+      logger.error("Error getting current plan - no success", { orgId, error: result.error });
       return undefined;
     }
 
@@ -215,7 +221,7 @@ export async function getCurrentPlan(orgId: string) {
 
     return { ...result, usage };
   } catch (e) {
-    logger.error("Error getting current plan", { orgId, error: e });
+    logger.error("Error getting current plan - caught error", { orgId, error: e });
     return undefined;
   }
 }
@@ -226,13 +232,13 @@ export async function getLimits(orgId: string) {
   try {
     const result = await client.currentPlan(orgId);
     if (!result.success) {
-      logger.error("Error getting limits", { orgId, error: result.error });
+      logger.error("Error getting limits - no success", { orgId, error: result.error });
       return undefined;
     }
 
     return result.v3Subscription?.plan?.limits;
   } catch (e) {
-    logger.error("Error getting limits", { orgId, error: e });
+    logger.error("Error getting limits - caught error", { orgId, error: e });
     return undefined;
   }
 }
@@ -273,12 +279,12 @@ export async function getPlans() {
   try {
     const result = await client.plans();
     if (!result.success) {
-      logger.error("Error getting plans", { error: result.error });
+      logger.error("Error getting plans - no success", { error: result.error });
       return undefined;
     }
     return result;
   } catch (e) {
-    logger.error("Error getting plans", { error: e });
+    logger.error("Error getting plans - caught error", { error: e });
     return undefined;
   }
 }
@@ -356,12 +362,12 @@ export async function getUsage(organizationId: string, { from, to }: { from: Dat
   try {
     const result = await client.usage(organizationId, { from, to });
     if (!result.success) {
-      logger.error("Error getting usage", { error: result.error });
+      logger.error("Error getting usage - no success", { error: result.error });
       return undefined;
     }
     return result;
   } catch (e) {
-    logger.error("Error getting usage", { error: e });
+    logger.error("Error getting usage - caught error", { error: e });
     return undefined;
   }
 }
@@ -390,12 +396,12 @@ export async function getUsageSeries(organizationId: string, params: UsageSeries
   try {
     const result = await client.usageSeries(organizationId, params);
     if (!result.success) {
-      logger.error("Error getting usage series", { error: result.error });
+      logger.error("Error getting usage series - no success", { error: result.error });
       return undefined;
     }
     return result;
   } catch (e) {
-    logger.error("Error getting usage series", { error: e });
+    logger.error("Error getting usage series - caught error", { error: e });
     return undefined;
   }
 }
@@ -414,12 +420,12 @@ export async function reportInvocationUsage(
       additionalData,
     });
     if (!result.success) {
-      logger.error("Error reporting invocation", { error: result.error });
+      logger.error("Error reporting invocation - no success", { error: result.error });
       return undefined;
     }
     return result;
   } catch (e) {
-    logger.error("Error reporting invocation", { error: e });
+    logger.error("Error reporting invocation - caught error", { error: e });
     return undefined;
   }
 }
@@ -442,14 +448,14 @@ export async function getEntitlement(
   try {
     const result = await client.getEntitlement(organizationId);
     if (!result.success) {
-      logger.error("Error getting entitlement", { error: result.error });
+      logger.error("Error getting entitlement - no success", { error: result.error });
       return {
         hasAccess: true as const,
       };
     }
     return result;
   } catch (e) {
-    logger.error("Error getting entitlement", { error: e });
+    logger.error("Error getting entitlement - caught error", { error: e });
     return {
       hasAccess: true as const,
     };
@@ -505,6 +511,24 @@ export async function setBillingAlert(
   return result;
 }
 
+export async function generateRegistryCredentials(
+  projectId: string,
+  region: "us-east-1" | "eu-central-1"
+) {
+  if (!client) return undefined;
+  const result = await client.generateRegistryCredentials(projectId, region);
+  if (!result.success) {
+    logger.error("Error generating registry credentials", {
+      error: result.error,
+      projectId,
+      region,
+    });
+    throw new Error("Failed to generate registry credentials");
+  }
+
+  return result;
+}
+
 function isCloud(): boolean {
   const acceptableHosts = [
     "https://cloud.trigger.dev",

apps/webapp/app/services/projectSettings.server.ts

@@ -4,7 +4,7 @@ import { DeleteProjectService } from "~/services/deleteProject.server";
 import { BranchTrackingConfigSchema, type BranchTrackingConfig } from "~/v3/github";
 import { checkGitHubBranchExists } from "~/services/gitHub.server";
 import { errAsync, fromPromise, okAsync, ResultAsync } from "neverthrow";
-import { BuildSettings } from "~/v3/buildSettings";
+import { type BuildSettings } from "~/v3/buildSettings";
 
 export class ProjectSettingsService {
   #prismaClient: PrismaClient;
@@ -82,7 +82,7 @@ export class ProjectSettingsService {
         (error) => ({ type: "other" as const, cause: error })
       );
 
-    const createConnectedRepo = (defaultBranch: string) =>
+    const createConnectedRepo = (defaultBranch: string, previewDeploymentsEnabled: boolean) =>
       fromPromise(
         this.#prismaClient.connectedGithubRepository.create({
           data: {
@@ -90,23 +90,25 @@ export class ProjectSettingsService {
             repositoryId: repositoryId,
             branchTracking: {
               prod: { branch: defaultBranch },
-              staging: { branch: defaultBranch },
+              staging: {},
             } satisfies BranchTrackingConfig,
-            previewDeploymentsEnabled: true,
+            previewDeploymentsEnabled,
           },
         }),
         (error) => ({ type: "other" as const, cause: error })
       );
 
-    return ResultAsync.combine([getRepository(), findExistingConnection()]).andThen(
-      ([repository, existingConnection]) => {
-        if (existingConnection) {
-          return errAsync({ type: "project_already_has_connected_repository" as const });
-        }
-
-        return createConnectedRepo(repository.defaultBranch);
+    return ResultAsync.combine([
+      getRepository(),
+      findExistingConnection(),
+      this.isPreviewEnvironmentEnabled(projectId),
+    ]).andThen(([repository, existingConnection, previewEnvironmentEnabled]) => {
+      if (existingConnection) {
+        return errAsync({ type: "project_already_has_connected_repository" as const });
       }
-    );
+
+      return createConnectedRepo(repository.defaultBranch, previewEnvironmentEnabled);
+    });
   }
 
   disconnectGitHubRepo(projectId: string) {
@@ -208,7 +210,11 @@ export class ProjectSettingsService {
       return okAsync(stagingBranch);
     };
 
-    const updateConnectedRepo = () =>
+    const updateConnectedRepo = (data: {
+      productionBranch: string | undefined;
+      stagingBranch: string | undefined;
+      previewDeploymentsEnabled: boolean | undefined;
+    }) =>
       fromPromise(
         this.#prismaClient.connectedGithubRepository.update({
           where: {
@@ -216,10 +222,10 @@ export class ProjectSettingsService {
           },
           data: {
             branchTracking: {
-              prod: productionBranch ? { branch: productionBranch } : {},
-              staging: stagingBranch ? { branch: stagingBranch } : {},
+              prod: data.productionBranch ? { branch: data.productionBranch } : {},
+              staging: data.stagingBranch ? { branch: data.stagingBranch } : {},
             } satisfies BranchTrackingConfig,
-            previewDeploymentsEnabled: previewDeploymentsEnabled,
+            previewDeploymentsEnabled: data.previewDeploymentsEnabled,
           },
         }),
         (error) => ({ type: "other" as const, cause: error })
@@ -240,8 +246,14 @@ export class ProjectSettingsService {
             fullRepoName: connectedRepo.repository.fullName,
             oldStagingBranch: connectedRepo.branchTracking?.staging?.branch,
           }),
+          this.isPreviewEnvironmentEnabled(projectId),
         ]);
       })
+      .map(([productionBranch, stagingBranch, previewEnvironmentEnabled]) => ({
+        productionBranch,
+        stagingBranch,
+        previewDeploymentsEnabled: previewDeploymentsEnabled && previewEnvironmentEnabled,
+      }))
       .andThen(updateConnectedRepo);
   }
 
@@ -296,4 +308,22 @@ export class ProjectSettingsService {
       });
     });
   }
+
+  private isPreviewEnvironmentEnabled(projectId: string) {
+    return fromPromise(
+      this.#prismaClient.runtimeEnvironment.findFirst({
+        select: {
+          id: true,
+        },
+        where: {
+          projectId: projectId,
+          slug: "preview",
+        },
+      }),
+      (error) => ({
+        type: "other" as const,
+        cause: error,
+      })
+    ).map((previewEnvironment) => previewEnvironment !== null);
+  }
 }

apps/webapp/app/services/projectSettingsPresenter.server.ts

@@ -3,7 +3,7 @@ import { prisma } from "~/db.server";
 import { BranchTrackingConfigSchema } from "~/v3/github";
 import { env } from "~/env.server";
 import { findProjectBySlug } from "~/models/project.server";
-import { err, fromPromise, ok, okAsync } from "neverthrow";
+import { err, fromPromise, ok, ResultAsync } from "neverthrow";
 import { BuildSettingsSchema } from "~/v3/buildSettings";
 
 export class ProjectSettingsPresenter {
@@ -20,33 +20,31 @@ export class ProjectSettingsPresenter {
       fromPromise(findProjectBySlug(organizationSlug, projectSlug, userId), (error) => ({
         type: "other" as const,
         cause: error,
-      })).andThen((project) => {
-        if (!project) {
-          return err({ type: "project_not_found" as const });
-        }
-        return ok(project);
-      });
+      }))
+        .andThen((project) => {
+          if (!project) {
+            return err({ type: "project_not_found" as const });
+          }
+          return ok(project);
+        })
+        .map((project) => {
+          const buildSettingsOrFailure = BuildSettingsSchema.safeParse(project.buildSettings);
+          const buildSettings = buildSettingsOrFailure.success
+            ? buildSettingsOrFailure.data
+            : undefined;
+          return { ...project, buildSettings };
+        });
 
     if (!githubAppEnabled) {
-      return getProject().andThen((project) => {
-        if (!project) {
-          return err({ type: "project_not_found" as const });
-        }
-
-        const buildSettingsOrFailure = BuildSettingsSchema.safeParse(project.buildSettings);
-        const buildSettings = buildSettingsOrFailure.success
-          ? buildSettingsOrFailure.data
-          : undefined;
-
-        return ok({
-          gitHubApp: {
-            enabled: false,
-            connectedRepository: undefined,
-            installations: undefined,
-          },
-          buildSettings,
-        });
-      });
+      return getProject().map(({ buildSettings }) => ({
+        gitHubApp: {
+          enabled: false,
+          connectedRepository: undefined,
+          installations: undefined,
+          isPreviewEnvironmentEnabled: undefined,
+        },
+        buildSettings,
+      }));
     }
 
     const findConnectedGithubRepository = (projectId: string) =>
@@ -54,6 +52,12 @@ export class ProjectSettingsPresenter {
         this.#prismaClient.connectedGithubRepository.findFirst({
           where: {
             projectId,
+            repository: {
+              installation: {
+                deletedAt: null,
+                suspendedAt: null,
+              },
+            },
           },
           select: {
             branchTracking: true,
@@ -130,37 +134,39 @@ export class ProjectSettingsPresenter {
         })
       );
 
+    const isPreviewEnvironmentEnabled = (projectId: string) =>
+      fromPromise(
+        this.#prismaClient.runtimeEnvironment.findFirst({
+          select: {
+            id: true,
+          },
+          where: {
+            projectId: projectId,
+            slug: "preview",
+          },
+        }),
+        (error) => ({
+          type: "other" as const,
+          cause: error,
+        })
+      ).map((previewEnvironment) => previewEnvironment !== null);
+
     return getProject().andThen((project) =>
-      findConnectedGithubRepository(project.id).andThen((connectedGithubRepository) => {
-        const buildSettingsOrFailure = BuildSettingsSchema.safeParse(project.buildSettings);
-        const buildSettings = buildSettingsOrFailure.success
-          ? buildSettingsOrFailure.data
-          : undefined;
-
-        if (connectedGithubRepository) {
-          return okAsync({
-            gitHubApp: {
-              enabled: true,
-              connectedRepository: connectedGithubRepository,
-              // skip loading installations if there is a connected repository
-              // a project can have only a single connected repository
-              installations: undefined,
-            },
-            buildSettings,
-          });
-        }
-
-        return listGithubAppInstallations(project.organizationId).map((githubAppInstallations) => {
-          return {
-            gitHubApp: {
-              enabled: true,
-              connectedRepository: undefined,
-              installations: githubAppInstallations,
-            },
-            buildSettings,
-          };
-        });
-      })
+      ResultAsync.combine([
+        isPreviewEnvironmentEnabled(project.id),
+        findConnectedGithubRepository(project.id),
+        listGithubAppInstallations(project.organizationId),
+      ]).map(
+        ([isPreviewEnvironmentEnabled, connectedGithubRepository, githubAppInstallations]) => ({
+          gitHubApp: {
+            enabled: true,
+            connectedRepository: connectedGithubRepository,
+            installations: githubAppInstallations,
+            isPreviewEnvironmentEnabled,
+          },
+          buildSettings: project.buildSettings,
+        })
+      )
     );
   }
 }

apps/webapp/app/services/realtimeClient.server.ts

@@ -83,7 +83,10 @@ export class RealtimeClient {
     this.#registerCommands();
 
     const ctx = new DefaultStatefulContext();
-    const memory = new MemoryStore({ persistentMap: new Map() });
+    const memory = new MemoryStore({
+      persistentMap: new Map(),
+      unstableEvictOnSet: { frequency: 0.01, maxItems: 1000 },
+    });
     const redisCacheStore = new RedisCacheStore({
       connection: {
         keyPrefix: "tr:cache:realtime",

apps/webapp/app/services/requestIdempotency.server.ts

@@ -33,7 +33,13 @@ export class RequestIdempotencyService<TTypes extends string> {
       : "request-idempotency:";
 
     const ctx = new DefaultStatefulContext();
-    const memory = new MemoryStore({ persistentMap: new Map() });
+    const memory = new MemoryStore({
+      persistentMap: new Map(),
+      unstableEvictOnSet: {
+        frequency: 0.001,
+        maxItems: 1000,
+      },
+    });
     const redisCacheStore = new RedisCacheStore({
       name: "request-idempotency",
       connection: {

apps/webapp/app/services/resourceMonitor.server.ts

@@ -0,0 +1,11 @@
+import { ResourceMonitor } from "@trigger.dev/core/v3/serverOnly";
+import { singleton } from "~/utils/singleton";
+
+export const resourceMonitor = singleton("resourceMonitor", initializeResourceMonitor);
+
+function initializeResourceMonitor() {
+  return new ResourceMonitor({
+    ctx: {},
+    verbose: false,
+  });
+}

apps/webapp/app/services/runsReplicationInstance.server.ts

@@ -66,6 +66,7 @@ function initializeRunsReplicationInstance() {
     insertBaseDelayMs: env.RUN_REPLICATION_INSERT_BASE_DELAY_MS,
     insertMaxDelayMs: env.RUN_REPLICATION_INSERT_MAX_DELAY_MS,
     insertStrategy: env.RUN_REPLICATION_INSERT_STRATEGY,
+    disablePayloadInsert: env.RUN_REPLICATION_DISABLE_PAYLOAD_INSERT === "1",
   });
 
   if (env.RUN_REPLICATION_ENABLED === "1") {

apps/webapp/app/services/runsReplicationService.server.ts

@@ -57,6 +57,7 @@ export type RunsReplicationServiceOptions = {
   insertMaxRetries?: number;
   insertBaseDelayMs?: number;
   insertMaxDelayMs?: number;
+  disablePayloadInsert?: boolean;
 };
 
 type PostgresTaskRun = TaskRun & { masterQueue: string };
@@ -100,6 +101,7 @@ export class RunsReplicationService {
   private _insertBaseDelayMs: number;
   private _insertMaxDelayMs: number;
   private _insertStrategy: "insert" | "insert_async";
+  private _disablePayloadInsert: boolean;
 
   public readonly events: EventEmitter<RunsReplicationServiceEvents>;
 
@@ -112,6 +114,7 @@ export class RunsReplicationService {
     this._acknowledgeTimeoutMs = options.acknowledgeTimeoutMs ?? 1_000;
 
     this._insertStrategy = options.insertStrategy ?? "insert";
+    this._disablePayloadInsert = options.disablePayloadInsert ?? false;
 
     this._replicationClient = new LogicalReplicationClient({
       pgConfig: {
@@ -750,7 +753,7 @@ export class RunsReplicationService {
       };
     }
 
-    if (event === "update" || event === "delete") {
+    if (event === "update" || event === "delete" || this._disablePayloadInsert) {
       const taskRunInsert = await this.#prepareTaskRunInsert(
         run,
         run.organizationId,

apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts

@@ -6,8 +6,10 @@ import {
   type ListRunsOptions,
   type RunListInputOptions,
   type RunsRepositoryOptions,
+  type TagListOptions,
   convertRunListInputOptionsToFilterRunsOptions,
 } from "./runsRepository.server";
+import parseDuration from "parse-duration";
 
 export class ClickHouseRunsRepository implements IRunsRepository {
   constructor(private readonly options: RunsRepositoryOptions) {}
@@ -162,6 +164,57 @@ export class ClickHouseRunsRepository implements IRunsRepository {
 
     return result[0].count;
   }
+
+  async listTags(options: TagListOptions) {
+    const queryBuilder = this.options.clickhouse.taskRuns
+      .tagQueryBuilder()
+      .where("organization_id = {organizationId: String}", {
+        organizationId: options.organizationId,
+      })
+      .where("project_id = {projectId: String}", {
+        projectId: options.projectId,
+      })
+      .where("environment_id = {environmentId: String}", {
+        environmentId: options.environmentId,
+      });
+
+    const periodMs = options.period ? parseDuration(options.period) ?? undefined : undefined;
+    if (periodMs) {
+      queryBuilder.where("created_at >= fromUnixTimestamp64Milli({period: Int64})", {
+        period: new Date(Date.now() - periodMs).getTime(),
+      });
+    }
+
+    if (options.from) {
+      queryBuilder.where("created_at >= fromUnixTimestamp64Milli({from: Int64})", {
+        from: options.from,
+      });
+    }
+
+    if (options.to) {
+      queryBuilder.where("created_at <= fromUnixTimestamp64Milli({to: Int64})", { to: options.to });
+    }
+
+    // Filter by query (case-insensitive contains search)
+    if (options.query && options.query.trim().length > 0) {
+      queryBuilder.where("positionCaseInsensitiveUTF8(tag, {query: String}) > 0", {
+        query: options.query,
+      });
+    }
+
+    // Add ordering and pagination
+    queryBuilder.orderBy("tag ASC").limit(options.limit);
+
+    const [queryError, result] = await queryBuilder.execute();
+
+    if (queryError) {
+      throw queryError;
+    }
+
+    return {
+      tags: result.map((row) => row.tag),
+    };
+  }
 }
 
 function applyRunFiltersToQueryBuilder<T>(

apps/webapp/app/services/runsRepository/postgresRunsRepository.server.ts

@@ -8,6 +8,7 @@ import {
   type ListedRun,
   type RunListInputOptions,
   type RunsRepositoryOptions,
+  type TagListOptions,
   convertRunListInputOptionsToFilterRunsOptions,
 } from "./runsRepository.server";
 
@@ -104,6 +105,32 @@ export class PostgresRunsRepository implements IRunsRepository {
     return Number(result[0].count);
   }
 
+  async listTags({ projectId, query, offset, limit }: TagListOptions) {
+    const tags = await this.options.prisma.taskRunTag.findMany({
+      select: {
+        name: true,
+      },
+      where: {
+        projectId,
+        name: query
+          ? {
+              startsWith: query,
+              mode: "insensitive",
+            }
+          : undefined,
+      },
+      orderBy: {
+        id: "desc",
+      },
+      take: limit + 1,
+      skip: offset,
+    });
+
+    return {
+      tags: tags.map((tag) => tag.name),
+    };
+  }
+
   #buildRunIdsQuery(
     filterOptions: FilterRunsOptions,
     page: { size: number; cursor?: string; direction?: "forward" | "backward" }

apps/webapp/app/services/runsRepository/runsRepository.server.ts

@@ -69,6 +69,11 @@ type Pagination = {
   };
 };
 
+type OffsetPagination = {
+  offset: number;
+  limit: number;
+};
+
 export type ListedRun = Prisma.TaskRunGetPayload<{
   select: {
     id: true;
@@ -104,6 +109,21 @@ export type ListedRun = Prisma.TaskRunGetPayload<{
 
 export type ListRunsOptions = RunListInputOptions & Pagination;
 
+export type TagListOptions = {
+  organizationId: string;
+  projectId: string;
+  environmentId: string;
+  period?: string;
+  from?: number;
+  to?: number;
+  /** Performs a case insensitive contains search on the tag name */
+  query?: string;
+} & OffsetPagination;
+
+export type TagList = {
+  tags: string[];
+};
+
 export interface IRunsRepository {
   name: string;
   listRunIds(options: ListRunsOptions): Promise<string[]>;
@@ -115,6 +135,7 @@ export interface IRunsRepository {
     };
   }>;
   countRuns(options: RunListInputOptions): Promise<number>;
+  listTags(options: TagListOptions): Promise<TagList>;
 }
 
 export class RunsRepository implements IRunsRepository {
@@ -291,6 +312,24 @@ export class RunsRepository implements IRunsRepository {
       }
     );
   }
+
+  async listTags(options: TagListOptions): Promise<TagList> {
+    const repository = await this.#getRepository();
+    return startActiveSpan(
+      "runsRepository.listTags",
+      async () => {
+        return await repository.listTags(options);
+      },
+      {
+        attributes: {
+          "repository.name": repository.name,
+          organizationId: options.organizationId,
+          projectId: options.projectId,
+          environmentId: options.environmentId,
+        },
+      }
+    );
+  }
 }
 
 export function parseRunListInputOptions(data: any): RunListInputOptions {

apps/webapp/app/utils/taskEvent.ts

@@ -1,572 +0,0 @@
-import { Attributes, Link } from "@opentelemetry/api";
-import {
-  correctErrorStackTrace,
-  ExceptionEventProperties,
-  isExceptionSpanEvent,
-  millisecondsToNanoseconds,
-  NULL_SENTINEL,
-  SemanticInternalAttributes,
-  SpanEvent,
-  SpanEvents,
-  SpanMessagingEvent,
-  TaskEventStyle,
-  unflattenAttributes,
-} from "@trigger.dev/core/v3";
-import { Prisma, TaskEvent, TaskEventKind } from "@trigger.dev/database";
-import { createTreeFromFlatItems, flattenTree } from "~/components/primitives/TreeView/TreeView";
-import type {
-  PreparedEvent,
-  SpanLink,
-  SpanSummary,
-  TraceSummary,
-} from "~/v3/eventRepository.server";
-
-export type TraceSpan = NonNullable<ReturnType<typeof createSpanFromEvents>>;
-
-export function prepareTrace(events: TaskEvent[]): TraceSummary | undefined {
-  let preparedEvents: Array<PreparedEvent> = [];
-  let rootSpanId: string | undefined;
-  const eventsBySpanId = new Map<string, PreparedEvent>();
-
-  for (const event of events) {
-    preparedEvents.push(prepareEvent(event));
-
-    if (!rootSpanId && !event.parentId) {
-      rootSpanId = event.spanId;
-    }
-  }
-
-  for (const event of preparedEvents) {
-    const existingEvent = eventsBySpanId.get(event.spanId);
-
-    if (!existingEvent) {
-      eventsBySpanId.set(event.spanId, event);
-      continue;
-    }
-
-    if (event.isCancelled || !event.isPartial) {
-      eventsBySpanId.set(event.spanId, event);
-    }
-  }
-
-  preparedEvents = Array.from(eventsBySpanId.values());
-
-  const spansBySpanId = new Map<string, SpanSummary>();
-
-  const spans = preparedEvents.map((event) => {
-    const ancestorCancelled = isAncestorCancelled(eventsBySpanId, event.spanId);
-    const duration = calculateDurationIfAncestorIsCancelled(
-      eventsBySpanId,
-      event.spanId,
-      event.duration
-    );
-
-    const span = {
-      id: event.spanId,
-      parentId: event.parentId ?? undefined,
-      runId: event.runId,
-      data: {
-        message: event.message,
-        style: event.style,
-        duration,
-        isError: event.isError,
-        isPartial: ancestorCancelled ? false : event.isPartial,
-        isCancelled: event.isCancelled === true ? true : event.isPartial && ancestorCancelled,
-        startTime: getDateFromNanoseconds(event.startTime),
-        level: event.level,
-        events: event.events,
-        environmentType: event.environmentType,
-        isDebug: event.kind === TaskEventKind.LOG,
-      },
-    } satisfies SpanSummary;
-
-    spansBySpanId.set(event.spanId, span);
-
-    return span;
-  });
-
-  if (!rootSpanId) {
-    return;
-  }
-
-  const rootSpan = spansBySpanId.get(rootSpanId);
-
-  if (!rootSpan) {
-    return;
-  }
-
-  return {
-    rootSpan,
-    spans,
-  };
-}
-
-export function createTraceTreeFromEvents(traceSummary: TraceSummary, spanId: string) {
-  //this tree starts at the passed in span (hides parent elements if there are any)
-  const tree = createTreeFromFlatItems(traceSummary.spans, spanId);
-
-  //we need the start offset for each item, and the total duration of the entire tree
-  const treeRootStartTimeMs = tree ? tree?.data.startTime.getTime() : 0;
-  let totalDuration = tree?.data.duration ?? 0;
-  const events = tree
-    ? flattenTree(tree).map((n) => {
-        const offset = millisecondsToNanoseconds(n.data.startTime.getTime() - treeRootStartTimeMs);
-        totalDuration = Math.max(totalDuration, offset + n.data.duration);
-        return {
-          ...n,
-          data: {
-            ...n.data,
-            //set partial nodes to null duration
-            duration: n.data.isPartial ? null : n.data.duration,
-            offset,
-            isRoot: n.id === traceSummary.rootSpan.id,
-          },
-        };
-      })
-    : [];
-
-  //total duration should be a minimum of 1ms
-  totalDuration = Math.max(totalDuration, millisecondsToNanoseconds(1));
-
-  let rootSpanStatus: "executing" | "completed" | "failed" = "executing";
-  if (events[0]) {
-    if (events[0].data.isError) {
-      rootSpanStatus = "failed";
-    } else if (!events[0].data.isPartial) {
-      rootSpanStatus = "completed";
-    }
-  }
-
-  return {
-    rootSpanStatus,
-    events: events,
-    parentRunFriendlyId:
-      tree?.id === traceSummary.rootSpan.id ? undefined : traceSummary.rootSpan.runId,
-    duration: totalDuration,
-    rootStartedAt: tree?.data.startTime,
-  };
-}
-
-export function createSpanFromEvents(events: TaskEvent[], spanId: string) {
-  const spanEvent = getSpanEvent(events, spanId);
-
-  if (!spanEvent) {
-    return;
-  }
-
-  const preparedEvent = prepareEvent(spanEvent);
-  const span = createSpanFromEvent(events, preparedEvent);
-
-  const output = rehydrateJson(spanEvent.output);
-  const payload = rehydrateJson(spanEvent.payload);
-
-  const show = rehydrateShow(spanEvent.properties);
-
-  const properties = sanitizedAttributes(spanEvent.properties);
-
-  const messagingEvent = SpanMessagingEvent.optional().safeParse((properties as any)?.messaging);
-
-  const links: SpanLink[] = [];
-
-  if (messagingEvent.success && messagingEvent.data) {
-    if (messagingEvent.data.message && "id" in messagingEvent.data.message) {
-      if (messagingEvent.data.message.id.startsWith("run_")) {
-        links.push({
-          type: "run",
-          icon: "runs",
-          title: `Run ${messagingEvent.data.message.id}`,
-          runId: messagingEvent.data.message.id,
-        });
-      }
-    }
-  }
-
-  const backLinks = spanEvent.links as any as Link[] | undefined;
-
-  if (backLinks && backLinks.length > 0) {
-    backLinks.forEach((l) => {
-      const title = String(l.attributes?.[SemanticInternalAttributes.LINK_TITLE] ?? "Triggered by");
-
-      links.push({
-        type: "span",
-        icon: "trigger",
-        title,
-        traceId: l.context.traceId,
-        spanId: l.context.spanId,
-      });
-    });
-  }
-
-  const spanEvents = transformEvents(
-    preparedEvent.events,
-    spanEvent.metadata as Attributes,
-    spanEvent.environmentType === "DEVELOPMENT"
-  );
-
-  return {
-    ...spanEvent,
-    ...span.data,
-    payload,
-    output,
-    events: spanEvents,
-    show,
-    links,
-    properties: properties ? JSON.stringify(properties, null, 2) : undefined,
-    showActionBar: show?.actions === true,
-  };
-}
-
-export function createSpanFromEvent(events: TaskEvent[], event: PreparedEvent) {
-  let ancestorCancelled = false;
-  let duration = event.duration;
-
-  if (!event.isCancelled && event.isPartial) {
-    walkSpanAncestors(events, event, (ancestorEvent, level) => {
-      if (level >= 8) {
-        return { stop: true };
-      }
-
-      if (ancestorEvent.isCancelled) {
-        ancestorCancelled = true;
-
-        // We need to get the cancellation time from the cancellation span event
-        const cancellationEvent = ancestorEvent.events.find(
-          (event) => event.name === "cancellation"
-        );
-
-        if (cancellationEvent) {
-          duration = calculateDurationFromStart(event.startTime, cancellationEvent.time);
-        }
-
-        return { stop: true };
-      }
-
-      return { stop: false };
-    });
-  }
-
-  const span = {
-    id: event.spanId,
-    parentId: event.parentId ?? undefined,
-    runId: event.runId,
-    idempotencyKey: event.idempotencyKey,
-    data: {
-      message: event.message,
-      style: event.style,
-      duration,
-      isError: event.isError,
-      isPartial: ancestorCancelled ? false : event.isPartial,
-      isCancelled: event.isCancelled === true ? true : event.isPartial && ancestorCancelled,
-      startTime: getDateFromNanoseconds(event.startTime),
-      level: event.level,
-      events: event.events,
-      environmentType: event.environmentType,
-    },
-  };
-
-  return span;
-}
-
-function walkSpanAncestors(
-  events: TaskEvent[],
-  event: PreparedEvent,
-  callback: (event: PreparedEvent, level: number) => { stop: boolean }
-) {
-  const parentId = event.parentId;
-  if (!parentId) {
-    return;
-  }
-
-  let parentEvent = getSpanEvent(events, parentId);
-  let level = 1;
-
-  while (parentEvent) {
-    const preparedParentEvent = prepareEvent(parentEvent);
-
-    const result = callback(preparedParentEvent, level);
-
-    if (result.stop) {
-      return;
-    }
-
-    if (!preparedParentEvent.parentId) {
-      return;
-    }
-
-    parentEvent = getSpanEvent(events, preparedParentEvent.parentId);
-
-    level++;
-  }
-}
-
-function getSpanEvent(events: TaskEvent[], spanId: string) {
-  const spans = events.filter((e) => e.spanId === spanId);
-  const completedSpan = spans.find((s) => !s.isPartial);
-
-  if (completedSpan) {
-    return completedSpan;
-  }
-
-  return spans.at(0);
-}
-
-export function prepareEvent(event: TaskEvent): PreparedEvent {
-  return {
-    ...event,
-    duration: Number(event.duration),
-    events: parseEventsField(event.events),
-    style: parseStyleField(event.style),
-  };
-}
-
-function parseEventsField(events: Prisma.JsonValue): SpanEvents {
-  const unsafe = events
-    ? (events as any[]).map((e) => ({
-        ...e,
-        properties: unflattenAttributes(e.properties as Attributes),
-      }))
-    : undefined;
-
-  return unsafe as SpanEvents;
-}
-
-function parseStyleField(style: Prisma.JsonValue): TaskEventStyle {
-  const unsafe = unflattenAttributes(style as Attributes);
-
-  if (!unsafe) {
-    return {};
-  }
-
-  if (typeof unsafe === "object") {
-    return Object.assign(
-      {
-        icon: undefined,
-        variant: undefined,
-      },
-      unsafe
-    ) as TaskEventStyle;
-  }
-
-  return {};
-}
-
-export function isAncestorCancelled(events: Map<string, PreparedEvent>, spanId: string) {
-  const event = events.get(spanId);
-
-  if (!event) {
-    return false;
-  }
-
-  if (event.isCancelled) {
-    return true;
-  }
-
-  if (event.parentId) {
-    return isAncestorCancelled(events, event.parentId);
-  }
-
-  return false;
-}
-
-function calculateDurationIfAncestorIsCancelled(
-  events: Map<string, PreparedEvent>,
-  spanId: string,
-  defaultDuration: number
-) {
-  const event = events.get(spanId);
-
-  if (!event) {
-    return defaultDuration;
-  }
-
-  if (event.isCancelled) {
-    return defaultDuration;
-  }
-
-  if (!event.isPartial) {
-    return defaultDuration;
-  }
-
-  if (event.parentId) {
-    const cancelledAncestor = findFirstCancelledAncestor(events, event.parentId);
-
-    if (cancelledAncestor) {
-      // We need to get the cancellation time from the cancellation span event
-      const cancellationEvent = cancelledAncestor.events.find(
-        (event) => event.name === "cancellation"
-      );
-
-      if (cancellationEvent) {
-        return calculateDurationFromStart(event.startTime, cancellationEvent.time);
-      }
-    }
-  }
-
-  return defaultDuration;
-}
-
-export function calculateDurationFromStart(startTime: bigint, endTime: Date = new Date()) {
-  const $endtime = typeof endTime === "string" ? new Date(endTime) : endTime;
-
-  return Number(BigInt($endtime.getTime() * 1_000_000) - startTime);
-}
-
-function findFirstCancelledAncestor(events: Map<string, PreparedEvent>, spanId: string) {
-  const event = events.get(spanId);
-
-  if (!event) {
-    return;
-  }
-
-  if (event.isCancelled) {
-    return event;
-  }
-
-  if (event.parentId) {
-    return findFirstCancelledAncestor(events, event.parentId);
-  }
-
-  return;
-}
-
-export function getDateFromNanoseconds(nanoseconds: bigint) {
-  return new Date(Number(nanoseconds) / 1_000_000);
-}
-
-export function getNowInNanoseconds(): bigint {
-  return BigInt(new Date().getTime() * 1_000_000);
-}
-
-export function rehydrateJson(json: Prisma.JsonValue): any {
-  if (json === null) {
-    return undefined;
-  }
-
-  if (json === NULL_SENTINEL) {
-    return null;
-  }
-
-  if (typeof json === "string") {
-    return json;
-  }
-
-  if (typeof json === "number") {
-    return json;
-  }
-
-  if (typeof json === "boolean") {
-    return json;
-  }
-
-  if (Array.isArray(json)) {
-    return json.map((item) => rehydrateJson(item));
-  }
-
-  if (typeof json === "object") {
-    return unflattenAttributes(json as Attributes);
-  }
-
-  return null;
-}
-
-export function rehydrateShow(properties: Prisma.JsonValue): { actions?: boolean } | undefined {
-  if (properties === null || properties === undefined) {
-    return;
-  }
-
-  if (typeof properties !== "object") {
-    return;
-  }
-
-  if (Array.isArray(properties)) {
-    return;
-  }
-
-  const actions = properties[SemanticInternalAttributes.SHOW_ACTIONS];
-
-  if (typeof actions === "boolean") {
-    return { actions };
-  }
-
-  return;
-}
-
-export function sanitizedAttributes(json: Prisma.JsonValue) {
-  if (json === null || json === undefined) {
-    return;
-  }
-
-  const withoutPrivateProperties = removePrivateProperties(json as Attributes);
-  if (!withoutPrivateProperties) {
-    return;
-  }
-
-  return unflattenAttributes(withoutPrivateProperties);
-}
-// removes keys that start with a $ sign. If there are no keys left, return undefined
-function removePrivateProperties(
-  attributes: Attributes | undefined | null
-): Attributes | undefined {
-  if (!attributes) {
-    return undefined;
-  }
-
-  const result: Attributes = {};
-
-  for (const [key, value] of Object.entries(attributes)) {
-    if (key.startsWith("$")) {
-      continue;
-    }
-
-    result[key] = value;
-  }
-
-  if (Object.keys(result).length === 0) {
-    return undefined;
-  }
-
-  return result;
-}
-
-export function transformEvents(
-  events: SpanEvents,
-  properties: Attributes,
-  isDev: boolean
-): SpanEvents {
-  return (events ?? []).map((event) => transformEvent(event, properties, isDev));
-}
-
-function transformEvent(event: SpanEvent, properties: Attributes, isDev: boolean): SpanEvent {
-  if (isExceptionSpanEvent(event)) {
-    return {
-      ...event,
-      properties: {
-        exception: transformException(event.properties.exception, properties, isDev),
-      },
-    };
-  }
-
-  return event;
-}
-
-function transformException(
-  exception: ExceptionEventProperties,
-  properties: Attributes,
-  isDev: boolean
-): ExceptionEventProperties {
-  const projectDirAttributeValue = properties[SemanticInternalAttributes.PROJECT_DIR];
-
-  if (projectDirAttributeValue !== undefined && typeof projectDirAttributeValue !== "string") {
-    return exception;
-  }
-
-  return {
-    ...exception,
-    stacktrace: exception.stacktrace
-      ? correctErrorStackTrace(exception.stacktrace, projectDirAttributeValue, {
-          removeFirstLine: true,
-          isDev,
-        })
-      : undefined,
-  };
-}

apps/webapp/app/v3/buildSettings.ts

@@ -2,8 +2,8 @@ import { z } from "zod";
 
 export const BuildSettingsSchema = z.object({
   triggerConfigFilePath: z.string().optional(),
-  installDirectory: z.string().optional(),
   installCommand: z.string().optional(),
+  preBuildCommand: z.string().optional(),
 });
 
 export type BuildSettings = z.infer<typeof BuildSettingsSchema>;

apps/webapp/app/v3/dynamicFlushScheduler.server.ts

@@ -1,4 +1,5 @@
 import { Logger } from "@trigger.dev/core/logger";
+import { tryCatch } from "@trigger.dev/core/utils";
 import { nanoid } from "nanoid";
 import pLimit from "p-limit";
 import { signalsEmitter } from "~/services/signals.server";
@@ -195,55 +196,72 @@ export class DynamicFlushScheduler<T> {
     // Schedule all batches for concurrent processing
     const flushPromises = batchesToFlush.map((batch) =>
       this.limiter(async () => {
-        const flushId = nanoid();
         const itemCount = batch.length;
 
-        try {
-          const startTime = Date.now();
-          await this.callback(flushId, batch);
+        const self = this;
 
-          const duration = Date.now() - startTime;
-          this.totalQueuedItems -= itemCount;
-          this.consecutiveFlushFailures = 0;
-          this.lastFlushTime = Date.now();
-          this.metrics.flushedBatches++;
-          this.metrics.totalItemsFlushed += itemCount;
+        async function tryFlush(flushId: string, batchToFlush: T[], attempt: number = 1) {
+          try {
+            const startTime = Date.now();
+            await self.callback(flushId, batchToFlush);
 
-          this.logger.debug("Batch flushed successfully", {
-            flushId,
-            itemCount,
-            duration,
-            remainingQueueDepth: this.totalQueuedItems,
-            activeConcurrency: this.limiter.activeCount,
-            pendingConcurrency: this.limiter.pendingCount,
-          });
-        } catch (error) {
-          this.consecutiveFlushFailures++;
-          this.metrics.failedBatches++;
+            const duration = Date.now() - startTime;
+            self.totalQueuedItems -= itemCount;
+            self.consecutiveFlushFailures = 0;
+            self.lastFlushTime = Date.now();
+            self.metrics.flushedBatches++;
+            self.metrics.totalItemsFlushed += itemCount;
 
-          this.logger.error("Error flushing batch", {
-            flushId,
-            itemCount,
-            error,
-            consecutiveFailures: this.consecutiveFlushFailures,
-          });
+            self.logger.debug("Batch flushed successfully", {
+              flushId,
+              itemCount,
+              duration,
+              remainingQueueDepth: self.totalQueuedItems,
+              activeConcurrency: self.limiter.activeCount,
+              pendingConcurrency: self.limiter.pendingCount,
+            });
+          } catch (error) {
+            self.consecutiveFlushFailures++;
+            self.metrics.failedBatches++;
 
-          // Re-queue the batch at the front if it fails
-          this.batchQueue.unshift(batch);
-          this.totalQueuedItems += itemCount;
+            self.logger.error("Error attempting to flush batch", {
+              flushId,
+              itemCount,
+              error,
+              consecutiveFailures: self.consecutiveFlushFailures,
+              attempt,
+            });
 
-          // Back off on failures
-          if (this.consecutiveFlushFailures > 3) {
-            this.adjustConcurrency(true);
+            // Back off on failures
+            if (self.consecutiveFlushFailures > 5) {
+              self.adjustConcurrency(true);
+            }
+
+            if (attempt <= 3) {
+              await new Promise((resolve) => setTimeout(resolve, 500));
+              return await tryFlush(flushId, batchToFlush, attempt + 1);
+            } else {
+              throw error;
+            }
           }
         }
+
+        const [flushError] = await tryCatch(tryFlush(nanoid(), batch));
+
+        if (flushError) {
+          this.logger.error("Error flushing batch", {
+            error: flushError,
+          });
+        }
       })
     );
 
     // Don't await here - let them run concurrently
     Promise.allSettled(flushPromises).then(() => {
+      const shouldContinueFlushing =
+        this.batchQueue.length > 0 && (this.consecutiveFlushFailures < 3 || this.isShuttingDown);
       // After flush completes, check if we need to flush more
-      if (this.batchQueue.length > 0) {
+      if (shouldContinueFlushing) {
         this.flushBatches();
       }
     });