Merge branch 'main' into nicer-app-emails

apps/supervisor/src/env.ts

@@ -35,8 +35,16 @@ const Env = z.object({
   TRIGGER_DEQUEUE_ENABLED: BoolEnv.default(true),
   TRIGGER_DEQUEUE_INTERVAL_MS: z.coerce.number().int().default(250),
   TRIGGER_DEQUEUE_IDLE_INTERVAL_MS: z.coerce.number().int().default(1000),
-  TRIGGER_DEQUEUE_MAX_RUN_COUNT: z.coerce.number().int().default(10),
-  TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT: z.coerce.number().int().default(1),
+  TRIGGER_DEQUEUE_MAX_RUN_COUNT: z.coerce.number().int().default(1),
+  TRIGGER_DEQUEUE_MIN_CONSUMER_COUNT: z.coerce.number().int().default(1),
+  TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT: z.coerce.number().int().default(10),
+  TRIGGER_DEQUEUE_SCALING_STRATEGY: z.enum(["none", "smooth", "aggressive"]).default("none"),
+  TRIGGER_DEQUEUE_SCALING_UP_COOLDOWN_MS: z.coerce.number().int().default(5000), // 5 seconds
+  TRIGGER_DEQUEUE_SCALING_DOWN_COOLDOWN_MS: z.coerce.number().int().default(30000), // 30 seconds
+  TRIGGER_DEQUEUE_SCALING_TARGET_RATIO: z.coerce.number().default(1.0), // Target ratio of queue items to consumers (1.0 = 1 item per consumer)
+  TRIGGER_DEQUEUE_SCALING_EWMA_ALPHA: z.coerce.number().min(0).max(1).default(0.3), // Smooths queue length measurements (0=historical, 1=current)
+  TRIGGER_DEQUEUE_SCALING_BATCH_WINDOW_MS: z.coerce.number().int().positive().default(1000), // Batch window for metrics processing (ms)
+  TRIGGER_DEQUEUE_SCALING_DAMPING_FACTOR: z.coerce.number().min(0).max(1).default(0.7), // Smooths consumer count changes after EWMA (0=no scaling, 1=immediate)
 
   // Optional services
   TRIGGER_WARM_START_URL: z.string().optional(),
@@ -77,6 +85,10 @@ const Env = z.object({
   KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT: z.string().default("10Gi"),
   KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST: z.string().default("2Gi"),
   KUBERNETES_STRIP_IMAGE_DIGEST: BoolEnv.default(false),
+  KUBERNETES_CPU_REQUEST_MIN_CORES: z.coerce.number().min(0).default(0),
+  KUBERNETES_CPU_REQUEST_RATIO: z.coerce.number().min(0).max(1).default(0.75), // Ratio of CPU limit, so 0.75 = 75% of CPU limit
+  KUBERNETES_MEMORY_REQUEST_MIN_GB: z.coerce.number().min(0).default(0),
+  KUBERNETES_MEMORY_REQUEST_RATIO: z.coerce.number().min(0).max(1).default(1), // Ratio of memory limit, so 1 = 100% of memory limit
 
   // Placement tags settings
   PLACEMENT_TAGS_ENABLED: BoolEnv.default(false),

apps/supervisor/src/index.ts

@@ -128,7 +128,18 @@ class ManagedSupervisor {
       dequeueIdleIntervalMs: env.TRIGGER_DEQUEUE_IDLE_INTERVAL_MS,
       queueConsumerEnabled: env.TRIGGER_DEQUEUE_ENABLED,
       maxRunCount: env.TRIGGER_DEQUEUE_MAX_RUN_COUNT,
-      maxConsumerCount: env.TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT,
+      metricsRegistry: register,
+      scaling: {
+        strategy: env.TRIGGER_DEQUEUE_SCALING_STRATEGY,
+        minConsumerCount: env.TRIGGER_DEQUEUE_MIN_CONSUMER_COUNT,
+        maxConsumerCount: env.TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT,
+        scaleUpCooldownMs: env.TRIGGER_DEQUEUE_SCALING_UP_COOLDOWN_MS,
+        scaleDownCooldownMs: env.TRIGGER_DEQUEUE_SCALING_DOWN_COOLDOWN_MS,
+        targetRatio: env.TRIGGER_DEQUEUE_SCALING_TARGET_RATIO,
+        ewmaAlpha: env.TRIGGER_DEQUEUE_SCALING_EWMA_ALPHA,
+        batchWindowMs: env.TRIGGER_DEQUEUE_SCALING_BATCH_WINDOW_MS,
+        dampingFactor: env.TRIGGER_DEQUEUE_SCALING_DAMPING_FACTOR,
+      },
       runNotificationsEnabled: env.TRIGGER_WORKLOAD_API_ENABLED,
       heartbeatIntervalSeconds: env.TRIGGER_WORKER_HEARTBEAT_INTERVAL_SECONDS,
       sendRunDebugLogs: env.SEND_RUN_DEBUG_LOGS,

apps/supervisor/src/workloadManager/kubernetes.ts

@@ -20,6 +20,12 @@ export class KubernetesWorkloadManager implements WorkloadManager {
   private namespace = env.KUBERNETES_NAMESPACE;
   private placementTagProcessor: PlacementTagProcessor;
 
+  // Resource settings
+  private readonly cpuRequestMinCores = env.KUBERNETES_CPU_REQUEST_MIN_CORES;
+  private readonly cpuRequestRatio = env.KUBERNETES_CPU_REQUEST_RATIO;
+  private readonly memoryRequestMinGb = env.KUBERNETES_MEMORY_REQUEST_MIN_GB;
+  private readonly memoryRequestRatio = env.KUBERNETES_MEMORY_REQUEST_RATIO;
+
   constructor(private opts: WorkloadManagerOptions) {
     this.k8s = createK8sApi();
     this.placementTagProcessor = new PlacementTagProcessor({
@@ -63,6 +69,10 @@ export class KubernetesWorkloadManager implements WorkloadManager {
     return imageRef.substring(0, atIndex);
   }
 
+  private clamp(value: number, min: number, max: number): number {
+    return Math.min(Math.max(value, min), max);
+  }
+
   async create(opts: WorkloadManagerCreateOptions) {
     this.logger.log("[KubernetesWorkloadManager] Creating container", { opts });
 
@@ -295,9 +305,16 @@ export class KubernetesWorkloadManager implements WorkloadManager {
   }
 
   #getResourceRequestsForMachine(preset: MachinePreset): ResourceQuantities {
+    const cpuRequest = preset.cpu * this.cpuRequestRatio;
+    const memoryRequest = preset.memory * this.memoryRequestRatio;
+
+    // Clamp between min and max
+    const clampedCpu = this.clamp(cpuRequest, this.cpuRequestMinCores, preset.cpu);
+    const clampedMemory = this.clamp(memoryRequest, this.memoryRequestMinGb, preset.memory);
+
     return {
-      cpu: `${preset.cpu * 0.75}`,
-      memory: `${preset.memory}G`,
+      cpu: `${clampedCpu}`,
+      memory: `${clampedMemory}G`,
     };
   }
 

apps/webapp/app/eventLoopMonitor.server.ts

@@ -3,6 +3,9 @@ import { singleton } from "./utils/singleton";
 import { tracer } from "./v3/tracer.server";
 import { env } from "./env.server";
 import { context, Context } from "@opentelemetry/api";
+import { performance } from "node:perf_hooks";
+import { logger } from "./services/logger.server";
+import { signalsEmitter } from "./services/signals.server";
 
 const THRESHOLD_NS = env.EVENT_LOOP_MONITOR_THRESHOLD_MS * 1e6;
 
@@ -69,16 +72,53 @@ function after(asyncId: number) {
 export const eventLoopMonitor = singleton("eventLoopMonitor", () => {
   const hook = createHook({ init, before, after, destroy });
 
+  let stopEventLoopUtilizationMonitoring: () => void;
+
   return {
     enable: () => {
       console.log("🥸  Initializing event loop monitor");
 
       hook.enable();
+
+      stopEventLoopUtilizationMonitoring = startEventLoopUtilizationMonitoring();
     },
     disable: () => {
       console.log("🥸  Disabling event loop monitor");
 
       hook.disable();
+
+      stopEventLoopUtilizationMonitoring?.();
     },
   };
 });
+
+function startEventLoopUtilizationMonitoring() {
+  let lastEventLoopUtilization = performance.eventLoopUtilization();
+
+  const interval = setInterval(() => {
+    const currentEventLoopUtilization = performance.eventLoopUtilization();
+
+    const diff = performance.eventLoopUtilization(
+      currentEventLoopUtilization,
+      lastEventLoopUtilization
+    );
+    const utilization = Number.isFinite(diff.utilization) ? diff.utilization : 0;
+
+    if (Math.random() < env.EVENT_LOOP_MONITOR_UTILIZATION_SAMPLE_RATE) {
+      logger.info("nodejs.event_loop.utilization", { utilization });
+    }
+
+    lastEventLoopUtilization = currentEventLoopUtilization;
+  }, env.EVENT_LOOP_MONITOR_UTILIZATION_INTERVAL_MS);
+
+  signalsEmitter.on("SIGTERM", () => {
+    clearInterval(interval);
+  });
+  signalsEmitter.on("SIGINT", () => {
+    clearInterval(interval);
+  });
+
+  return () => {
+    clearInterval(interval);
+  };
+}

apps/webapp/app/routes/_app.github.callback/route.tsx

@@ -0,0 +1,121 @@
+import { type LoaderFunctionArgs } from "@remix-run/node";
+import { z } from "zod";
+import { validateGitHubAppInstallSession } from "~/services/gitHubSession.server";
+import { linkGitHubAppInstallation, updateGitHubAppInstallation } from "~/services/gitHub.server";
+import { logger } from "~/services/logger.server";
+import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
+import { tryCatch } from "@trigger.dev/core";
+import { $replica } from "~/db.server";
+import { requireUser } from "~/services/session.server";
+import { sanitizeRedirectPath } from "~/utils";
+
+const QuerySchema = z.discriminatedUnion("setup_action", [
+  z.object({
+    setup_action: z.literal("install"),
+    installation_id: z.coerce.number(),
+    state: z.string(),
+  }),
+  z.object({
+    setup_action: z.literal("update"),
+    installation_id: z.coerce.number(),
+    state: z.string(),
+  }),
+  z.object({
+    setup_action: z.literal("request"),
+    state: z.string(),
+  }),
+]);
+
+export async function loader({ request }: LoaderFunctionArgs) {
+  const url = new URL(request.url);
+  const queryParams = Object.fromEntries(url.searchParams);
+  const cookieHeader = request.headers.get("Cookie");
+
+  const result = QuerySchema.safeParse(queryParams);
+
+  if (!result.success) {
+    logger.warn("GitHub App callback with invalid params", {
+      queryParams,
+    });
+    return redirectWithErrorMessage("/", request, "Failed to install GitHub App");
+  }
+
+  const callbackData = result.data;
+
+  const sessionResult = await validateGitHubAppInstallSession(cookieHeader, callbackData.state);
+
+  if (!sessionResult.valid) {
+    logger.error("GitHub App callback with invalid session", {
+      callbackData,
+      error: sessionResult.error,
+    });
+
+    return redirectWithErrorMessage("/", request, "Failed to install GitHub App");
+  }
+
+  const { organizationId, redirectTo: unsafeRedirectTo } = sessionResult;
+  const redirectTo = sanitizeRedirectPath(unsafeRedirectTo);
+
+  const user = await requireUser(request);
+  const org = await $replica.organization.findFirst({
+    where: { id: organizationId, members: { some: { userId: user.id } }, deletedAt: null },
+    orderBy: { createdAt: "desc" },
+    select: {
+      id: true,
+    },
+  });
+
+  if (!org) {
+    // the secure cookie approach should already protect against this
+    // just an additional check
+    logger.error("GitHub app installation attempt on unauthenticated org", {
+      userId: user.id,
+      organizationId,
+    });
+    return redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub App");
+  }
+
+  switch (callbackData.setup_action) {
+    case "install": {
+      const [error] = await tryCatch(
+        linkGitHubAppInstallation(callbackData.installation_id, organizationId)
+      );
+
+      if (error) {
+        logger.error("Failed to link GitHub App installation", {
+          error,
+        });
+        return redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub App");
+      }
+
+      return redirectWithSuccessMessage(redirectTo, request, "GitHub App installed successfully");
+    }
+
+    case "update": {
+      const [error] = await tryCatch(updateGitHubAppInstallation(callbackData.installation_id));
+
+      if (error) {
+        logger.error("Failed to update GitHub App installation", {
+          error,
+        });
+        return redirectWithErrorMessage(redirectTo, request, "Failed to update GitHub App");
+      }
+
+      return redirectWithSuccessMessage(redirectTo, request, "GitHub App updated successfully");
+    }
+
+    case "request": {
+      // This happens when a non-admin user requests installation
+      // The installation_id won't be available until an admin approves
+      logger.info("GitHub App installation requested, awaiting approval", {
+        callbackData,
+      });
+
+      return redirectWithSuccessMessage(redirectTo, request, "GitHub App installation requested");
+    }
+
+    default:
+      callbackData satisfies never;
+      return redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub App");
+  }
+}

apps/webapp/app/routes/_app.github.install/route.tsx

@@ -0,0 +1,52 @@
+import type { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { redirect } from "remix-typedjson";
+import { z } from "zod";
+import { $replica } from "~/db.server";
+import { createGitHubAppInstallSession } from "~/services/gitHubSession.server";
+import { requireUser } from "~/services/session.server";
+import { newOrganizationPath } from "~/utils/pathBuilder";
+import { logger } from "~/services/logger.server";
+import { sanitizeRedirectPath } from "~/utils";
+
+const QuerySchema = z.object({
+  org_slug: z.string(),
+  redirect_to: z.string().refine((value) => value === sanitizeRedirectPath(value), {
+    message: "Invalid redirect path",
+  }),
+});
+
+export const loader = async ({ request }: LoaderFunctionArgs) => {
+  const searchParams = new URL(request.url).searchParams;
+  const parsed = QuerySchema.safeParse(Object.fromEntries(searchParams));
+
+  if (!parsed.success) {
+    logger.warn("GitHub App installation redirect with invalid params", {
+      searchParams,
+      error: parsed.error,
+    });
+    throw redirect("/");
+  }
+
+  const { org_slug, redirect_to } = parsed.data;
+  const user = await requireUser(request);
+
+  const org = await $replica.organization.findFirst({
+    where: { slug: org_slug, members: { some: { userId: user.id } }, deletedAt: null },
+    orderBy: { createdAt: "desc" },
+    select: {
+      id: true,
+    },
+  });
+
+  if (!org) {
+    throw redirect(newOrganizationPath());
+  }
+
+  const { url, cookieHeader } = await createGitHubAppInstallSession(org.id, redirect_to);
+
+  return redirect(url, {
+    headers: {
+      "Set-Cookie": cookieHeader,
+    },
+  });
+};

apps/webapp/app/routes/auth.github.callback.tsx

@@ -5,11 +5,12 @@ import { getSession, redirectWithErrorMessage } from "~/models/message.server";
 import { authenticator } from "~/services/auth.server";
 import { commitSession } from "~/services/sessionStorage.server";
 import { redirectCookie } from "./auth.github";
+import { sanitizeRedirectPath } from "~/utils";
 
 export let loader: LoaderFunction = async ({ request }) => {
   const cookie = request.headers.get("Cookie");
   const redirectValue = await redirectCookie.parse(cookie);
-  const redirectTo = redirectValue ?? "/";
+  const redirectTo = sanitizeRedirectPath(redirectValue);
 
   const auth = await authenticator.authenticate("github", request, {
     failureRedirect: "/login", // If auth fails, the failureRedirect will be thrown as a Response

apps/webapp/app/routes/auth.github.ts

@@ -1,6 +1,4 @@
-import type { ActionFunction, LoaderFunction } from "@remix-run/node";
-import { createCookie } from "@remix-run/node";
-import { redirect } from "@remix-run/node";
+import { type ActionFunction, type LoaderFunction, redirect, createCookie } from "@remix-run/node";
 import { authenticator } from "~/services/auth.server";
 
 export let loader: LoaderFunction = () => redirect("/login");

apps/webapp/app/services/gitHub.server.ts

@@ -0,0 +1,135 @@
+import { App, type Octokit } from "octokit";
+import { env } from "../env.server";
+import { prisma } from "~/db.server";
+import { logger } from "./logger.server";
+
+export const githubApp =
+  env.GITHUB_APP_ENABLED === "1"
+    ? new App({
+        appId: env.GITHUB_APP_ID,
+        privateKey: env.GITHUB_APP_PRIVATE_KEY,
+        webhooks: {
+          secret: env.GITHUB_APP_WEBHOOK_SECRET,
+        },
+      })
+    : null;
+
+/**
+ * Links a GitHub App installation to a Trigger organization
+ */
+export async function linkGitHubAppInstallation(
+  installationId: number,
+  organizationId: string
+): Promise<void> {
+  if (!githubApp) {
+    throw new Error("GitHub App is not enabled");
+  }
+
+  const octokit = await githubApp.getInstallationOctokit(installationId);
+  const { data: installation } = await octokit.rest.apps.getInstallation({
+    installation_id: installationId,
+  });
+
+  const repositories = await fetchInstallationRepositories(octokit, installationId);
+
+  const repositorySelection = installation.repository_selection === "all" ? "ALL" : "SELECTED";
+
+  await prisma.githubAppInstallation.create({
+    data: {
+      appInstallationId: installationId,
+      organizationId,
+      targetId: installation.target_id,
+      targetType: installation.target_type,
+      accountHandle: installation.account
+        ? "login" in installation.account
+          ? installation.account.login
+          : "slug" in installation.account
+          ? installation.account.slug
+          : "-"
+        : "-",
+      permissions: installation.permissions,
+      repositorySelection,
+      repositories: {
+        create: repositories,
+      },
+    },
+  });
+}
+
+/**
+ * Links a GitHub App installation to a Trigger organization
+ */
+export async function updateGitHubAppInstallation(installationId: number): Promise<void> {
+  if (!githubApp) {
+    throw new Error("GitHub App is not enabled");
+  }
+
+  const octokit = await githubApp.getInstallationOctokit(installationId);
+  const { data: installation } = await octokit.rest.apps.getInstallation({
+    installation_id: installationId,
+  });
+
+  const existingInstallation = await prisma.githubAppInstallation.findFirst({
+    where: { appInstallationId: installationId },
+  });
+
+  if (!existingInstallation) {
+    throw new Error("GitHub App installation not found");
+  }
+
+  const repositorySelection = installation.repository_selection === "all" ? "ALL" : "SELECTED";
+
+  // repos are updated asynchronously via webhook events
+  await prisma.githubAppInstallation.update({
+    where: { id: existingInstallation?.id },
+    data: {
+      appInstallationId: installationId,
+      targetId: installation.target_id,
+      targetType: installation.target_type,
+      accountHandle: installation.account
+        ? "login" in installation.account
+          ? installation.account.login
+          : "slug" in installation.account
+          ? installation.account.slug
+          : "-"
+        : "-",
+      permissions: installation.permissions,
+      suspendedAt: existingInstallation?.suspendedAt,
+      repositorySelection,
+    },
+  });
+}
+
+async function fetchInstallationRepositories(octokit: Octokit, installationId: number) {
+  const iterator = octokit.paginate.iterator(octokit.rest.apps.listReposAccessibleToInstallation, {
+    installation_id: installationId,
+    per_page: 100,
+  });
+
+  const allRepos = [];
+  const maxPages = 3;
+  let pageCount = 0;
+
+  for await (const { data } of iterator) {
+    pageCount++;
+    allRepos.push(...data);
+
+    if (maxPages && pageCount >= maxPages) {
+      logger.warn("GitHub installation repository fetch truncated", {
+        installationId,
+        maxPages,
+        totalReposFetched: allRepos.length,
+      });
+      break;
+    }
+  }
+
+  return allRepos.map((repo) => ({
+    githubId: repo.id,
+    name: repo.name,
+    fullName: repo.full_name,
+    htmlUrl: repo.html_url,
+    private: repo.private,
+    defaultBranch: repo.default_branch,
+  }));
+}

apps/webapp/app/services/gitHubSession.server.ts

@@ -0,0 +1,124 @@
+import { createCookieSessionStorage } from "@remix-run/node";
+import { randomBytes } from "crypto";
+import { env } from "../env.server";
+import { logger } from "./logger.server";
+
+const sessionStorage = createCookieSessionStorage({
+  cookie: {
+    name: "__github_app_install",
+    httpOnly: true,
+    maxAge: 60 * 60, // 1 hour
+    path: "/",
+    sameSite: "lax",
+    secrets: [env.SESSION_SECRET],
+    secure: env.NODE_ENV === "production",
+  },
+});
+
+/**
+ * Creates a secure session for GitHub App installation with organization tracking
+ */
+export async function createGitHubAppInstallSession(
+  organizationId: string,
+  redirectTo: string
+): Promise<{ url: string; cookieHeader: string }> {
+  if (env.GITHUB_APP_ENABLED !== "1") {
+    throw new Error("GitHub App is not enabled");
+  }
+
+  const state = randomBytes(32).toString("hex");
+
+  const session = await sessionStorage.getSession();
+  session.set("organizationId", organizationId);
+  session.set("redirectTo", redirectTo);
+  session.set("state", state);
+  session.set("createdAt", Date.now());
+
+  const githubAppSlug = env.GITHUB_APP_SLUG;
+
+  // the state query param gets passed through to the installation callback
+  const url = `https://github.com/apps/${githubAppSlug}/installations/new?state=${state}`;
+
+  const cookieHeader = await sessionStorage.commitSession(session);
+
+  return { url, cookieHeader };
+}
+
+/**
+ * Validates and retrieves the GitHub App installation session
+ */
+export async function validateGitHubAppInstallSession(
+  cookieHeader: string | null,
+  state: string
+): Promise<
+  { valid: true; organizationId: string; redirectTo: string } | { valid: false; error?: string }
+> {
+  if (!cookieHeader) {
+    return {
+      valid: false,
+      error: "No installation session cookie found",
+    };
+  }
+
+  const session = await sessionStorage.getSession(cookieHeader);
+
+  const sessionState = session.get("state");
+  const organizationId = session.get("organizationId");
+  const redirectTo = session.get("redirectTo");
+  const createdAt = session.get("createdAt");
+
+  if (!sessionState || !organizationId || !createdAt || !redirectTo) {
+    logger.warn("GitHub App installation session missing required fields", {
+      hasState: !!sessionState,
+      hasOrgId: !!organizationId,
+      hasCreatedAt: !!createdAt,
+      hasRedirectTo: !!redirectTo,
+    });
+
+    return {
+      valid: false,
+      error: "invalid_session_data",
+    };
+  }
+
+  if (sessionState !== state) {
+    logger.warn("GitHub App installation state mismatch", {
+      expectedState: sessionState,
+      receivedState: state,
+    });
+    return {
+      valid: false,
+      error: "state_mismatch",
+    };
+  }
+
+  const expirationTime = createdAt + 60 * 60 * 1000;
+  if (Date.now() > expirationTime) {
+    logger.warn("GitHub App installation session expired", {
+      createdAt: new Date(createdAt),
+      now: new Date(),
+    });
+    return {
+      valid: false,
+      error: "session_expired",
+    };
+  }
+
+  return {
+    valid: true,
+    organizationId,
+    redirectTo,
+  };
+}
+
+/**
+ * Destroys the GitHub App installation cookie session
+ */
+export async function destroyGitHubAppInstallSession(cookieHeader: string | null): Promise<string> {
+  if (!cookieHeader) {
+    return "";
+  }
+
+  const session = await sessionStorage.getSession(cookieHeader);
+  return await sessionStorage.destroySession(session);
+}

apps/webapp/app/services/realtime/relayRealtimeStreams.server.ts

@@ -1,5 +1,6 @@
 import { AuthenticatedEnvironment } from "../apiAuth.server";
 import { logger } from "../logger.server";
+import { signalsEmitter } from "../signals.server";
 import { StreamIngestor, StreamResponder } from "./types";
 import { LineTransformStream } from "./utils.server";
 import { v1RealtimeStreams } from "./v1StreamsGlobal.server";
@@ -243,12 +244,17 @@ export class RelayRealtimeStreams implements StreamIngestor, StreamResponder {
 }
 
 function initializeRelayRealtimeStreams() {
-  return new RelayRealtimeStreams({
+  const service = new RelayRealtimeStreams({
     ttl: 1000 * 60 * 5, // 5 minutes
     cleanupInterval: 1000 * 60, // 1 minute
     fallbackIngestor: v1RealtimeStreams,
     fallbackResponder: v1RealtimeStreams,
   });
+
+  signalsEmitter.on("SIGTERM", service.close.bind(service));
+  signalsEmitter.on("SIGINT", service.close.bind(service));
+
+  return service;
 }
 
 export const relayRealtimeStreams = singleton(

apps/webapp/app/services/runsReplicationInstance.server.ts

@@ -3,8 +3,8 @@ import invariant from "tiny-invariant";
 import { env } from "~/env.server";
 import { singleton } from "~/utils/singleton";
 import { provider } from "~/v3/tracer.server";
-import { logger } from "./logger.server";
 import { RunsReplicationService } from "./runsReplicationService.server";
+import { signalsEmitter } from "./signals.server";
 
 export const runsReplicationInstance = singleton(
   "runsReplicationInstance",
@@ -80,8 +80,8 @@ function initializeRunsReplicationInstance() {
         });
       });
 
-    process.on("SIGTERM", service.shutdown.bind(service));
-    process.on("SIGINT", service.shutdown.bind(service));
+    signalsEmitter.on("SIGTERM", service.shutdown.bind(service));
+    signalsEmitter.on("SIGINT", service.shutdown.bind(service));
   }
 
   return service;

apps/webapp/app/services/runsReplicationService.server.ts

@@ -204,6 +204,8 @@ export class RunsReplicationService {
   }
 
   public async shutdown() {
+    if (this._isShuttingDown) return;
+
     this._isShuttingDown = true;
 
     this.logger.info("Initiating shutdown of runs replication service");

apps/webapp/app/services/signals.server.ts

@@ -0,0 +1,32 @@
+import { EventEmitter } from "events";
+import { singleton } from "~/utils/singleton";
+
+export type SignalsEvents = {
+  SIGTERM: [
+    {
+      time: Date;
+      signal: NodeJS.Signals;
+    }
+  ];
+  SIGINT: [
+    {
+      time: Date;
+      signal: NodeJS.Signals;
+    }
+  ];
+};
+
+export type SignalsEventArgs<T extends keyof SignalsEvents> = SignalsEvents[T];
+
+export type SignalsEmitter = EventEmitter<SignalsEvents>;
+
+function initializeSignalsEmitter() {
+  const emitter = new EventEmitter<SignalsEvents>();
+
+  process.on("SIGTERM", () => emitter.emit("SIGTERM", { time: new Date(), signal: "SIGTERM" }));
+  process.on("SIGINT", () => emitter.emit("SIGINT", { time: new Date(), signal: "SIGINT" }));
+
+  return emitter;
+}
+
+export const signalsEmitter = singleton("signalsEmitter", initializeSignalsEmitter);

apps/webapp/app/utils.ts

@@ -7,22 +7,38 @@ const DEFAULT_REDIRECT = "/";
  * This should be used any time the redirect path is user-provided
  * (Like the query string on our login/signup pages). This avoids
  * open-redirect vulnerabilities.
- * @param {string} to The redirect destination
+ * @param {string} path The redirect destination
  * @param {string} defaultRedirect The redirect to use if the to is unsafe.
  */
-export function safeRedirect(
-  to: FormDataEntryValue | string | null | undefined,
+export function sanitizeRedirectPath(
+  path: string | undefined | null,
   defaultRedirect: string = DEFAULT_REDIRECT
-) {
-  if (!to || typeof to !== "string") {
+): string {
+  if (!path || typeof path !== "string") {
     return defaultRedirect;
   }
 
-  if (!to.startsWith("/") || to.startsWith("//")) {
+  if (!path.startsWith("/") || path.startsWith("//")) {
     return defaultRedirect;
   }
 
-  return to;
+  try {
+    // should not parse as a full URL
+    new URL(path);
+    return defaultRedirect;
+  } catch {}
+
+  try {
+    // ensure it's a valid relative path
+    const url = new URL(path, "https://example.com");
+    if (url.hostname !== "example.com") {
+      return defaultRedirect;
+    }
+  } catch {
+    return defaultRedirect;
+  }
+
+  return path;
 }
 
 /**

apps/webapp/app/v3/dynamicFlushScheduler.server.ts

@@ -1,6 +1,7 @@
 import { Logger } from "@trigger.dev/core/logger";
 import { nanoid } from "nanoid";
 import pLimit from "p-limit";
+import { signalsEmitter } from "~/services/signals.server";
 
 export type DynamicFlushSchedulerConfig<T> = {
   batchSize: number;
@@ -22,6 +23,7 @@ export class DynamicFlushScheduler<T> {
   private readonly BATCH_SIZE: number;
   private readonly FLUSH_INTERVAL: number;
   private flushTimer: NodeJS.Timeout | null;
+  private metricsReporterTimer: NodeJS.Timeout | undefined;
   private readonly callback: (flushId: string, batch: T[]) => Promise<void>;
 
   // New properties for dynamic scaling
@@ -41,6 +43,7 @@ export class DynamicFlushScheduler<T> {
     droppedEvents: 0,
     droppedEventsByKind: new Map<string, number>(),
   };
+  private isShuttingDown: boolean = false;
 
   // New properties for load shedding
   private readonly loadSheddingThreshold: number;
@@ -75,6 +78,7 @@ export class DynamicFlushScheduler<T> {
 
     this.startFlushTimer();
     this.startMetricsReporter();
+    this.setupShutdownHandlers();
   }
 
   addToBatch(items: T[]): void {
@@ -119,8 +123,8 @@ export class DynamicFlushScheduler<T> {
     this.currentBatch.push(...itemsToAdd);
     this.totalQueuedItems += itemsToAdd.length;
 
-    // Check if we need to create a batch
-    if (this.currentBatch.length >= this.currentBatchSize) {
+    // Check if we need to create a batch (if we are shutting down, create a batch immediately because the flush timer is stopped)
+    if (this.currentBatch.length >= this.currentBatchSize || this.isShuttingDown) {
       this.createBatch();
     }
 
@@ -137,6 +141,23 @@ export class DynamicFlushScheduler<T> {
     this.resetFlushTimer();
   }
 
+  private setupShutdownHandlers(): void {
+    signalsEmitter.on("SIGTERM", () =>
+      this.shutdown().catch((error) => {
+        this.logger.error("Error shutting down dynamic flush scheduler", {
+          error,
+        });
+      })
+    );
+    signalsEmitter.on("SIGINT", () =>
+      this.shutdown().catch((error) => {
+        this.logger.error("Error shutting down dynamic flush scheduler", {
+          error,
+        });
+      })
+    );
+  }
+
   private startFlushTimer(): void {
     this.flushTimer = setInterval(() => this.checkAndFlush(), this.FLUSH_INTERVAL);
   }
@@ -145,6 +166,9 @@ export class DynamicFlushScheduler<T> {
     if (this.flushTimer) {
       clearInterval(this.flushTimer);
     }
+
+    if (this.isShuttingDown) return;
+
     this.startFlushTimer();
   }
 
@@ -226,7 +250,7 @@ export class DynamicFlushScheduler<T> {
   }
 
   private lastConcurrencyAdjustment: number = Date.now();
-  
+
   private adjustConcurrency(backOff: boolean = false): void {
     const currentConcurrency = this.limiter.concurrency;
     let newConcurrency = currentConcurrency;
@@ -281,7 +305,7 @@ export class DynamicFlushScheduler<T> {
 
   private startMetricsReporter(): void {
     // Report metrics every 30 seconds
-    setInterval(() => {
+    this.metricsReporterTimer = setInterval(() => {
       const droppedByKind: Record<string, number> = {};
       this.metrics.droppedEventsByKind.forEach((count, kind) => {
         droppedByKind[kind] = count;
@@ -356,10 +380,18 @@ export class DynamicFlushScheduler<T> {
 
   // Graceful shutdown
   async shutdown(): Promise<void> {
+    if (this.isShuttingDown) return;
+
+    this.isShuttingDown = true;
+
     if (this.flushTimer) {
       clearInterval(this.flushTimer);
     }
 
+    if (this.metricsReporterTimer) {
+      clearInterval(this.metricsReporterTimer);
+    }
+
     // Flush any remaining items
     if (this.currentBatch.length > 0) {
       this.createBatch();

apps/webapp/app/v3/getDeploymentImageRef.server.ts

@@ -11,6 +11,7 @@ import { STSClient, AssumeRoleCommand } from "@aws-sdk/client-sts";
 import { tryCatch } from "@trigger.dev/core";
 import { logger } from "~/services/logger.server";
 import { type RegistryConfig } from "./registryConfig.server";
+import type { EnvironmentType } from "@trigger.dev/core/v3";
 
 // Optional configuration for cross-account access
 export type AssumeRoleConfig = {
@@ -101,19 +102,22 @@ export async function getDeploymentImageRef({
   registry,
   projectRef,
   nextVersion,
-  environmentSlug,
+  environmentType,
+  deploymentShortCode,
 }: {
   registry: RegistryConfig;
   projectRef: string;
   nextVersion: string;
-  environmentSlug: string;
+  environmentType: EnvironmentType;
+  deploymentShortCode: string;
 }): Promise<{
   imageRef: string;
   isEcr: boolean;
   repoCreated: boolean;
 }> {
   const repositoryName = `${registry.namespace}/${projectRef}`;
-  const imageRef = `${registry.host}/${repositoryName}:${nextVersion}.${environmentSlug}`;
+  const envType = environmentType.toLowerCase();
+  const imageRef = `${registry.host}/${repositoryName}:${nextVersion}.${envType}.${deploymentShortCode}`;
 
   if (!isEcrRegistry(registry.host)) {
     return {

apps/webapp/app/v3/marqs/index.server.ts

@@ -24,6 +24,7 @@ import z from "zod";
 import { env } from "~/env.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
+import { signalsEmitter } from "~/services/signals.server";
 import { singleton } from "~/utils/singleton";
 import { legacyRunEngineWorker } from "../legacyRunEngineWorker.server";
 import { concurrencyTracker } from "../services/taskRunConcurrencyTracker.server";
@@ -112,6 +113,7 @@ export class MarQS {
   private queueDequeueCooloffPeriod: Map<string, number> = new Map();
   private queueDequeueCooloffCounts: Map<string, number> = new Map();
   private clearCooloffPeriodInterval: NodeJS.Timeout;
+  isShuttingDown: boolean = false;
 
   constructor(private readonly options: MarQSOptions) {
     this.redis = options.redis;
@@ -151,11 +153,14 @@ export class MarQS {
   }
 
   #setupShutdownHandlers() {
-    process.on("SIGTERM", () => this.shutdown("SIGTERM"));
-    process.on("SIGINT", () => this.shutdown("SIGINT"));
+    signalsEmitter.on("SIGTERM", () => this.shutdown("SIGTERM"));
+    signalsEmitter.on("SIGINT", () => this.shutdown("SIGINT"));
   }
 
   async shutdown(signal: NodeJS.Signals) {
+    if (this.isShuttingDown) return;
+    this.isShuttingDown = true;
+
     console.log("👇 Shutting down marqs", this.name, signal);
     clearInterval(this.clearCooloffPeriodInterval);
     this.#rebalanceWorkers.forEach((worker) => worker.stop());

apps/webapp/app/v3/services/initializeDeployment.server.ts

@@ -73,12 +73,15 @@ export class InitializeDeploymentService extends BaseService {
       const isV4Deployment = payload.type === "MANAGED";
       const registryConfig = getRegistryConfig(isV4Deployment);
 
+      const deploymentShortCode = nanoid(8);
+
       const [imageRefError, imageRefResult] = await tryCatch(
         getDeploymentImageRef({
           registry: registryConfig,
           projectRef: environment.project.externalRef,
           nextVersion,
-          environmentSlug: environment.slug,
+          environmentType: environment.type,
+          deploymentShortCode,
         })
       );
 
@@ -111,7 +114,7 @@ export class InitializeDeploymentService extends BaseService {
         data: {
           friendlyId: generateFriendlyId("deployment"),
           contentHash: payload.contentHash,
-          shortCode: nanoid(8),
+          shortCode: deploymentShortCode,
           version: nextVersion,
           status: "BUILDING",
           environmentId: environment.id,

apps/webapp/app/v3/tracer.server.ts

@@ -57,6 +57,7 @@ import { flattenAttributes } from "@trigger.dev/core/v3";
 import { prisma } from "~/db.server";
 import { metricsRegister } from "~/metrics.server";
 import type { Prisma } from "@trigger.dev/database";
+import { performance } from "node:perf_hooks";
 
 export const SEMINTATTRS_FORCE_RECORDING = "forceRecording";
 
@@ -602,10 +603,17 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) {
     description: "Event loop 99th percentile delay",
     unit: "s",
   });
+  // ELU observable gauge (unit is a ratio, 0..1)
+  const eluGauge = meter.createObservableGauge("nodejs.event_loop.utilization", {
+    description: "Event loop utilization over the last collection interval",
+    unit: "1", // OpenTelemetry convention for ratios
+  });
 
   // Get UV threadpool size (defaults to 4 if not set)
   const uvThreadpoolSize = parseInt(process.env.UV_THREADPOOL_SIZE || "4", 10);
 
+  let lastEventLoopUtilization = performance.eventLoopUtilization();
+
   // Single helper to read metrics from prom-client
   async function readNodeMetrics() {
     const metrics = await metricsRegister.getMetricsAsJSON();
@@ -648,6 +656,16 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) {
       }
     }
 
+    const currentEventLoopUtilization = performance.eventLoopUtilization();
+    // Diff over [lastSnapshot, current]
+    const diff = performance.eventLoopUtilization(
+      currentEventLoopUtilization,
+      lastEventLoopUtilization
+    );
+
+    // diff.utilization is between 0 and 1 (fraction of time "active")
+    const utilization = Number.isFinite(diff.utilization) ? diff.utilization : 0;
+
     return {
       threadpoolSize: uvThreadpoolSize,
       handlesByType,
@@ -661,6 +679,7 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) {
         p50: eventLoopLagP50?.values?.[0]?.value ?? 0,
         p90: eventLoopLagP90?.values?.[0]?.value ?? 0,
         p99: eventLoopLagP99?.values?.[0]?.value ?? 0,
+        utilization,
       },
     };
   }
@@ -698,6 +717,7 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) {
       res.observe(eventLoopLagP50Gauge, eventLoop.p50);
       res.observe(eventLoopLagP90Gauge, eventLoop.p90);
       res.observe(eventLoopLagP99Gauge, eventLoop.p99);
+      res.observe(eluGauge, eventLoop.utilization);
     },
     [
       uvThreadpoolSizeGauge,
@@ -711,6 +731,7 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) {
       eventLoopLagP50Gauge,
       eventLoopLagP90Gauge,
       eventLoopLagP99Gauge,
+      eluGauge,
     ]
   );
 }

apps/webapp/app/v3/tracing.server.ts

@@ -41,33 +41,14 @@ export async function startSpanWithEnv<T>(
   fn: (span: Span) => Promise<T>,
   options?: SpanOptions
 ): Promise<T> {
-  return startSpan(
-    tracer,
-    name,
-    async (span) => {
-      try {
-        return await fn(span);
-      } catch (e) {
-        if (e instanceof Error) {
-          span.recordException(e);
-        } else {
-          span.recordException(new Error(String(e)));
-        }
-
-        throw e;
-      } finally {
-        span.end();
-      }
+  return startSpan(tracer, name, fn, {
+    ...options,
+    attributes: {
+      ...attributesFromAuthenticatedEnv(env),
+      ...options?.attributes,
     },
-    {
-      attributes: {
-        ...attributesFromAuthenticatedEnv(env),
-        ...options?.attributes,
-      },
-      kind: SpanKind.SERVER,
-      ...options,
-    }
-  );
+    kind: SpanKind.SERVER,
+  });
 }
 
 export async function emitDebugLog(

apps/webapp/package.json

@@ -160,6 +160,7 @@
     "morgan": "^1.10.0",
     "nanoid": "3.3.8",
     "non.geist": "^1.0.2",
+    "octokit": "^3.2.1",
     "ohash": "^1.1.3",
     "openai": "^4.33.1",
     "p-limit": "^6.2.0",

apps/webapp/server.ts

@@ -12,168 +12,257 @@ import type { Server as IoServer } from "socket.io";
 import { WebSocketServer } from "ws";
 import { RateLimitMiddleware } from "~/services/apiRateLimit.server";
 import { type RunWithHttpContextFunction } from "~/services/httpAsyncStorage.server";
+import cluster from "node:cluster";
+import os from "node:os";
 
-const app = express();
+const ENABLE_CLUSTER = process.env.ENABLE_CLUSTER === "1";
+const cpuCount = os.availableParallelism();
+const WORKERS =
+  Number.parseInt(process.env.WEB_CONCURRENCY || process.env.CLUSTER_WORKERS || "", 10) || cpuCount;
 
-if (process.env.DISABLE_COMPRESSION !== "1") {
-  app.use(compression());
+function forkWorkers() {
+  for (let i = 0; i < WORKERS; i++) {
+    cluster.fork();
+  }
 }
 
-// http://expressjs.com/en/advanced/best-practice-security.html#at-a-minimum-disable-x-powered-by-header
-app.disable("x-powered-by");
+function installPrimarySignalHandlers() {
+  let didHandleSigterm = false;
+  let didHandleSigint = false;
+  let didGracefulExit = false;
 
-// Remix fingerprints its assets so we can cache forever.
-app.use("/build", express.static("public/build", { immutable: true, maxAge: "1y" }));
-
-// Everything else (like favicon.ico) is cached for an hour. You may want to be
-// more aggressive with this caching.
-app.use(express.static("public", { maxAge: "1h" }));
-
-app.use(morgan("tiny"));
-
-process.title = "node webapp-server";
-
-const MODE = process.env.NODE_ENV;
-const BUILD_DIR = path.join(process.cwd(), "build");
-const build = require(BUILD_DIR);
-
-const port = process.env.REMIX_APP_PORT || process.env.PORT || 3000;
-
-if (process.env.HTTP_SERVER_DISABLED !== "true") {
-  const socketIo: { io: IoServer } | undefined = build.entry.module.socketIo;
-  const wss: WebSocketServer | undefined = build.entry.module.wss;
-  const apiRateLimiter: RateLimitMiddleware = build.entry.module.apiRateLimiter;
-  const engineRateLimiter: RateLimitMiddleware = build.entry.module.engineRateLimiter;
-  const runWithHttpContext: RunWithHttpContextFunction = build.entry.module.runWithHttpContext;
-
-  app.use((req, res, next) => {
-    // helpful headers:
-    res.set("Strict-Transport-Security", `max-age=${60 * 60 * 24 * 365 * 100}`);
-
-    // Add X-Robots-Tag header for test-cloud.trigger.dev
-    if (req.hostname !== "cloud.trigger.dev") {
-      res.set("X-Robots-Tag", "noindex, nofollow");
+  const forward = (signal: NodeJS.Signals) => {
+    for (const id in cluster.workers) {
+      const w = cluster.workers[id];
+      if (w?.process?.pid) {
+        try {
+          process.kill(w.process.pid, signal);
+        } catch {}
+      }
     }
+  };
 
-    // /clean-urls/ -> /clean-urls
-    if (req.path.endsWith("/") && req.path.length > 1) {
-      const query = req.url.slice(req.path.length);
-      const safepath = req.path.slice(0, -1).replace(/\/+/g, "/");
-      res.redirect(301, safepath + query);
-      return;
-    }
-    next();
+  const gracefulExit = () => {
+    if (didGracefulExit) return;
+    didGracefulExit = true;
+
+    const timeoutMs = Number(process.env.GRACEFUL_SHUTDOWN_TIMEOUT || 30_000);
+    // wait for workers to exit, then exit the primary too
+    const maybeExit = () => {
+      const alive = Object.values(cluster.workers || {}).some((w) => w && !w.isDead());
+      if (!alive) process.exit(0);
+    };
+    setInterval(maybeExit, 1000);
+    setTimeout(() => process.exit(0), timeoutMs);
+  };
+
+  process.on("SIGTERM", () => {
+    if (didHandleSigterm) return;
+    didHandleSigterm = true;
+    forward("SIGTERM");
+    gracefulExit();
+  });
+  process.on("SIGINT", () => {
+    if (didHandleSigint) return;
+    didHandleSigint = true;
+    forward("SIGINT");
+    gracefulExit();
+  });
+}
+
+if (ENABLE_CLUSTER && cluster.isPrimary) {
+  process.title = `node webapp-server primary`;
+  console.log(`[cluster] Primary ${process.pid} is starting with ${WORKERS} workers`);
+  forkWorkers();
+
+  cluster.on("exit", (worker, code, signal) => {
+    const intentional =
+      // If we sent "shutdown", the worker will exit with code 0 after closing.
+      code === 0 || worker.exitedAfterDisconnect;
+    console.log(
+      `[cluster] worker ${worker.process.pid} exited (code=${code}, signal=${signal}, intentional=${intentional})`
+    );
+    // If it wasn't during a shutdown, replace the worker.
+    if (!intentional) cluster.fork();
   });
 
-  app.use((req, res, next) => {
-    // Generate a unique request ID for each request
-    const requestId = nanoid();
+  installPrimarySignalHandlers();
+} else {
+  const app = express();
 
-    runWithHttpContext({ requestId, path: req.url, host: req.hostname, method: req.method }, next);
-  });
+  if (process.env.DISABLE_COMPRESSION !== "1") {
+    app.use(compression());
+  }
 
-  if (process.env.DASHBOARD_AND_API_DISABLED !== "true") {
-    if (process.env.ALLOW_ONLY_REALTIME_API === "true") {
-      // Block all requests that do not start with /realtime
-      app.use((req, res, next) => {
-        // Make sure /healthcheck is still accessible
-        if (!req.url.startsWith("/realtime") && req.url !== "/healthcheck") {
-          res.status(404).send("Not Found");
-          return;
-        }
+  // http://expressjs.com/en/advanced/best-practice-security.html#at-a-minimum-disable-x-powered-by-header
+  app.disable("x-powered-by");
 
-        next();
+  // Remix fingerprints its assets so we can cache forever.
+  app.use("/build", express.static("public/build", { immutable: true, maxAge: "1y" }));
+
+  // Everything else (like favicon.ico) is cached for an hour. You may want to be
+  // more aggressive with this caching.
+  app.use(express.static("public", { maxAge: "1h" }));
+
+  app.use(morgan("tiny"));
+
+  process.title = ENABLE_CLUSTER
+    ? `node webapp-worker-${cluster.isWorker ? cluster.worker?.id : "solo"}`
+    : "node webapp-server";
+
+  const MODE = process.env.NODE_ENV;
+  const BUILD_DIR = path.join(process.cwd(), "build");
+  const build = require(BUILD_DIR);
+
+  const port = process.env.REMIX_APP_PORT || process.env.PORT || 3000;
+
+  if (process.env.HTTP_SERVER_DISABLED !== "true") {
+    const socketIo: { io: IoServer } | undefined = build.entry.module.socketIo;
+    const wss: WebSocketServer | undefined = build.entry.module.wss;
+    const apiRateLimiter: RateLimitMiddleware = build.entry.module.apiRateLimiter;
+    const engineRateLimiter: RateLimitMiddleware = build.entry.module.engineRateLimiter;
+    const runWithHttpContext: RunWithHttpContextFunction = build.entry.module.runWithHttpContext;
+
+    app.use((req, res, next) => {
+      // helpful headers:
+      res.set("Strict-Transport-Security", `max-age=${60 * 60 * 24 * 365 * 100}`);
+
+      // Add X-Robots-Tag header for test-cloud.trigger.dev
+      if (req.hostname !== "cloud.trigger.dev") {
+        res.set("X-Robots-Tag", "noindex, nofollow");
+      }
+
+      // /clean-urls/ -> /clean-urls
+      if (req.path.endsWith("/") && req.path.length > 1) {
+        const query = req.url.slice(req.path.length);
+        const safepath = req.path.slice(0, -1).replace(/\/+/g, "/");
+        res.redirect(301, safepath + query);
+        return;
+      }
+      next();
+    });
+
+    app.use((req, res, next) => {
+      // Generate a unique request ID for each request
+      const requestId = nanoid();
+
+      runWithHttpContext(
+        { requestId, path: req.url, host: req.hostname, method: req.method },
+        next
+      );
+    });
+
+    if (process.env.DASHBOARD_AND_API_DISABLED !== "true") {
+      if (process.env.ALLOW_ONLY_REALTIME_API === "true") {
+        // Block all requests that do not start with /realtime
+        app.use((req, res, next) => {
+          // Make sure /healthcheck is still accessible
+          if (!req.url.startsWith("/realtime") && req.url !== "/healthcheck") {
+            res.status(404).send("Not Found");
+            return;
+          }
+
+          next();
+        });
+      }
+
+      app.use(apiRateLimiter);
+      app.use(engineRateLimiter);
+
+      app.all(
+        "*",
+        // @ts-ignore
+        createRequestHandler({
+          build,
+          mode: MODE,
+        })
+      );
+    } else {
+      // we need to do the health check here at /healthcheck
+      app.get("/healthcheck", (req, res) => {
+        res.status(200).send("OK");
       });
     }
 
-    app.use(apiRateLimiter);
-    app.use(engineRateLimiter);
+    const server = app.listen(port, () => {
+      console.log(
+        `✅ server ready: http://localhost:${port} [NODE_ENV: ${MODE}]${
+          ENABLE_CLUSTER && cluster.isWorker ? ` [worker ${cluster.worker?.id}/${process.pid}]` : ""
+        }`
+      );
 
-    app.all(
-      "*",
-      // @ts-ignore
-      createRequestHandler({
-        build,
-        mode: MODE,
-      })
-    );
-  } else {
-    // we need to do the health check here at /healthcheck
-    app.get("/healthcheck", (req, res) => {
-      res.status(200).send("OK");
-    });
-  }
-
-  const server = app.listen(port, () => {
-    console.log(`✅ server ready: http://localhost:${port} [NODE_ENV: ${MODE}]`);
-
-    if (MODE === "development") {
-      broadcastDevReady(build)
-        .then(() => logDevReady(build))
-        .catch(console.error);
-    }
-  });
-
-  server.keepAliveTimeout = 65 * 1000;
-  // Mitigate against https://github.com/triggerdotdev/trigger.dev/security/dependabot/128
-  // by not allowing 2000+ headers to be sent and causing a DoS
-  // headers will instead be limited by the maxHeaderSize
-  server.maxHeadersCount = 0;
-
-  process.on("SIGTERM", () => {
-    server.close((err) => {
-      if (err) {
-        console.error("Error closing express server:", err);
-      } else {
-        console.log("Express server closed gracefully.");
+      if (MODE === "development") {
+        broadcastDevReady(build)
+          .then(() => logDevReady(build))
+          .catch(console.error);
       }
     });
-  });
 
-  socketIo?.io.attach(server);
-  server.removeAllListeners("upgrade"); // prevent duplicate upgrades from listeners created by io.attach()
+    server.keepAliveTimeout = 65 * 1000;
+    // Mitigate against https://github.com/triggerdotdev/trigger.dev/security/dependabot/128
+    // by not allowing 2000+ headers to be sent and causing a DoS
+    // headers will instead be limited by the maxHeaderSize
+    server.maxHeadersCount = 0;
 
-  server.on("upgrade", async (req, socket, head) => {
-    console.log(
-      `Attemping to upgrade connection at url ${req.url} with headers: ${JSON.stringify(
-        req.headers
-      )}`
-    );
+    let didCloseServer = false;
 
-    socket.on("error", (err) => {
-      console.error("Connection upgrade error:", err);
-    });
+    function closeServer(signal: NodeJS.Signals) {
+      if (didCloseServer) return;
+      didCloseServer = true;
 
-    const url = new URL(req.url ?? "", "http://localhost");
-
-    // Upgrade socket.io connection
-    if (url.pathname.startsWith("/socket.io/")) {
-      console.log(`Socket.io client connected, upgrading their connection...`);
-
-      // https://github.com/socketio/socket.io/issues/4693
-      (socketIo?.io.engine as EngineServer).handleUpgrade(req, socket, head);
-      return;
+      server.close((err) => {
+        if (err) {
+          console.error("Error closing express server:", err);
+        } else {
+          console.log("Express server closed gracefully.");
+        }
+      });
     }
 
-    // Only upgrade the connecting if the path is `/ws`
-    if (url.pathname !== "/ws") {
-      // Setting the socket.destroy() error param causes an error event to be emitted which needs to be handled with socket.on("error") to prevent uncaught exceptions.
-      socket.destroy(
-        new Error(
-          "Cannot connect because of invalid path: Please include `/ws` in the path of your upgrade request."
-        )
-      );
-      return;
-    }
+    process.on("SIGTERM", closeServer);
+    process.on("SIGINT", closeServer);
 
-    console.log(`Client connected, upgrading their connection...`);
+    socketIo?.io.attach(server);
+    server.removeAllListeners("upgrade"); // prevent duplicate upgrades from listeners created by io.attach()
 
-    // Handle the WebSocket connection
-    wss?.handleUpgrade(req, socket, head, (ws) => {
-      wss?.emit("connection", ws, req);
+    server.on("upgrade", async (req, socket, head) => {
+      console.log(`Attemping to upgrade connection at url ${req.url}`);
+
+      socket.on("error", (err) => {
+        console.error("Connection upgrade error:", err);
+      });
+
+      const url = new URL(req.url ?? "", "http://localhost");
+
+      // Upgrade socket.io connection
+      if (url.pathname.startsWith("/socket.io/")) {
+        console.log(`Socket.io client connected, upgrading their connection...`);
+
+        // https://github.com/socketio/socket.io/issues/4693
+        (socketIo?.io.engine as EngineServer).handleUpgrade(req, socket, head);
+        return;
+      }
+
+      // Only upgrade the connecting if the path is `/ws`
+      if (url.pathname !== "/ws") {
+        // Setting the socket.destroy() error param causes an error event to be emitted which needs to be handled with socket.on("error") to prevent uncaught exceptions.
+        socket.destroy(
+          new Error(
+            "Cannot connect because of invalid path: Please include `/ws` in the path of your upgrade request."
+          )
+        );
+        return;
+      }
+
+      console.log(`Client connected, upgrading their connection...`);
+
+      // Handle the WebSocket connection
+      wss?.handleUpgrade(req, socket, head, (ws) => {
+        wss?.emit("connection", ws, req);
+      });
     });
-  });
-} else {
-  require(BUILD_DIR);
-  console.log(`✅ app ready (skipping http server)`);
+  } else {
+    require(BUILD_DIR);
+    console.log(`✅ app ready (skipping http server)`);
+  }
 }

apps/webapp/test/getDeploymentImageRef.test.ts

@@ -8,7 +8,7 @@ import {
 } from "../app/v3/getDeploymentImageRef.server";
 import { DeleteRepositoryCommand } from "@aws-sdk/client-ecr";
 
-describe.skipIf(process.env.RUN_REGISTRY_TESTS !== "1")("getDeploymentImageRef", () => {
+describe("getDeploymentImageRef", () => {
   const testHost =
     process.env.DEPLOY_REGISTRY_HOST || "123456789012.dkr.ecr.us-east-1.amazonaws.com";
   const testNamespace = process.env.DEPLOY_REGISTRY_NAMESPACE || "test-namespace";
@@ -25,7 +25,7 @@ describe.skipIf(process.env.RUN_REGISTRY_TESTS !== "1")("getDeploymentImageRef",
 
   // Clean up test repository after tests
   afterAll(async () => {
-    if (process.env.KEEP_TEST_REPO === "1") {
+    if (process.env.KEEP_TEST_REPO === "1" || process.env.RUN_ECR_TESTS !== "1") {
       return;
     }
 
@@ -57,7 +57,7 @@ describe.skipIf(process.env.RUN_REGISTRY_TESTS !== "1")("getDeploymentImageRef",
   it("should return the correct image ref for non-ECR registry", async () => {
     const imageRef = await getDeploymentImageRef({
       registry: {
-        host: "registry.digitalocean.com",
+        host: "registry.example.com",
         namespace: testNamespace,
         username: "test-user",
         password: "test-pass",
@@ -67,60 +67,67 @@ describe.skipIf(process.env.RUN_REGISTRY_TESTS !== "1")("getDeploymentImageRef",
       },
       projectRef: testProjectRef,
       nextVersion: "20250630.1",
-      environmentSlug: "test",
+      environmentType: "DEVELOPMENT",
+      deploymentShortCode: "test1234",
     });
 
+    // Check the image ref structure and that it contains expected parts
     expect(imageRef.imageRef).toBe(
-      `registry.digitalocean.com/${testNamespace}/${testProjectRef}:20250630.1.test`
+      `registry.example.com/${testNamespace}/${testProjectRef}:20250630.1.development.test1234`
     );
     expect(imageRef.isEcr).toBe(false);
   });
 
-  it("should create ECR repository and return correct image ref", async () => {
-    const imageRef1 = await getDeploymentImageRef({
-      registry: {
-        host: testHost,
-        namespace: testNamespace,
-        username: "test-user",
-        password: "test-pass",
-        ecrTags: registryTags,
-        ecrAssumeRoleArn: roleArn,
-        ecrAssumeRoleExternalId: externalId,
-      },
-      projectRef: testProjectRef2,
-      nextVersion: "20250630.1",
-      environmentSlug: "test",
-    });
+  it.skipIf(process.env.RUN_ECR_TESTS !== "1")(
+    "should create ECR repository and return correct image ref",
+    async () => {
+      const imageRef1 = await getDeploymentImageRef({
+        registry: {
+          host: testHost,
+          namespace: testNamespace,
+          username: "test-user",
+          password: "test-pass",
+          ecrTags: registryTags,
+          ecrAssumeRoleArn: roleArn,
+          ecrAssumeRoleExternalId: externalId,
+        },
+        projectRef: testProjectRef2,
+        nextVersion: "20250630.1",
+        environmentType: "DEVELOPMENT",
+        deploymentShortCode: "test1234",
+      });
 
-    expect(imageRef1.imageRef).toBe(
-      `${testHost}/${testNamespace}/${testProjectRef2}:20250630.1.test`
-    );
-    expect(imageRef1.isEcr).toBe(true);
-    expect(imageRef1.repoCreated).toBe(true);
+      expect(imageRef1.imageRef).toBe(
+        `${testHost}/${testNamespace}/${testProjectRef2}:20250630.1.development.test1234`
+      );
+      expect(imageRef1.isEcr).toBe(true);
+      expect(imageRef1.repoCreated).toBe(true);
 
-    const imageRef2 = await getDeploymentImageRef({
-      registry: {
-        host: testHost,
-        namespace: testNamespace,
-        username: "test-user",
-        password: "test-pass",
-        ecrTags: registryTags,
-        ecrAssumeRoleArn: roleArn,
-        ecrAssumeRoleExternalId: externalId,
-      },
-      projectRef: testProjectRef2,
-      nextVersion: "20250630.2",
-      environmentSlug: "test",
-    });
+      const imageRef2 = await getDeploymentImageRef({
+        registry: {
+          host: testHost,
+          namespace: testNamespace,
+          username: "test-user",
+          password: "test-pass",
+          ecrTags: registryTags,
+          ecrAssumeRoleArn: roleArn,
+          ecrAssumeRoleExternalId: externalId,
+        },
+        projectRef: testProjectRef2,
+        nextVersion: "20250630.2",
+        environmentType: "DEVELOPMENT",
+        deploymentShortCode: "test1234",
+      });
 
-    expect(imageRef2.imageRef).toBe(
-      `${testHost}/${testNamespace}/${testProjectRef2}:20250630.2.test`
-    );
-    expect(imageRef2.isEcr).toBe(true);
-    expect(imageRef2.repoCreated).toBe(false);
-  });
+      expect(imageRef2.imageRef).toBe(
+        `${testHost}/${testNamespace}/${testProjectRef2}:20250630.2.development.test1234`
+      );
+      expect(imageRef2.isEcr).toBe(true);
+      expect(imageRef2.repoCreated).toBe(false);
+    }
+  );
 
-  it("should reuse existing ECR repository", async () => {
+  it.skipIf(process.env.RUN_ECR_TESTS !== "1")("should reuse existing ECR repository", async () => {
     // This should use the repository created in the previous test
     const imageRef = await getDeploymentImageRef({
       registry: {
@@ -134,36 +141,65 @@ describe.skipIf(process.env.RUN_REGISTRY_TESTS !== "1")("getDeploymentImageRef",
       },
       projectRef: testProjectRef,
       nextVersion: "20250630.2",
-      environmentSlug: "prod",
+      environmentType: "PRODUCTION",
+      deploymentShortCode: "test1234",
     });
 
     expect(imageRef.imageRef).toBe(
-      `${testHost}/${testNamespace}/${testProjectRef}:20250630.2.prod`
+      `${testHost}/${testNamespace}/${testProjectRef}:20250630.2.production.test1234`
     );
     expect(imageRef.isEcr).toBe(true);
   });
 
-  it("should throw error for invalid ECR host", async () => {
-    await expect(
-      getDeploymentImageRef({
-        registry: {
-          host: "invalid.ecr.amazonaws.com",
-          namespace: testNamespace,
-          username: "test-user",
-          password: "test-pass",
-          ecrTags: registryTags,
-          ecrAssumeRoleArn: roleArn,
-          ecrAssumeRoleExternalId: externalId,
-        },
-        projectRef: testProjectRef,
-        nextVersion: "20250630.1",
-        environmentSlug: "test",
-      })
-    ).rejects.toThrow("Invalid ECR registry host: invalid.ecr.amazonaws.com");
+  it("should generate unique image tags for different deployments with same environment type", async () => {
+    // Simulates the scenario where multiple deployments happen to the same environment type
+    const sameEnvironmentType = "PREVIEW";
+    const sameVersion = "20250630.1";
+
+    const firstImageRef = await getDeploymentImageRef({
+      registry: {
+        host: "registry.example.com",
+        namespace: testNamespace,
+        username: "test-user",
+        password: "test-pass",
+        ecrTags: registryTags,
+        ecrAssumeRoleArn: roleArn,
+        ecrAssumeRoleExternalId: externalId,
+      },
+      projectRef: testProjectRef,
+      nextVersion: sameVersion,
+      environmentType: sameEnvironmentType,
+      deploymentShortCode: "test1234",
+    });
+
+    const secondImageRef = await getDeploymentImageRef({
+      registry: {
+        host: "registry.example.com",
+        namespace: testNamespace,
+        username: "test-user",
+        password: "test-pass",
+        ecrTags: registryTags,
+        ecrAssumeRoleArn: roleArn,
+        ecrAssumeRoleExternalId: externalId,
+      },
+      projectRef: testProjectRef,
+      nextVersion: sameVersion,
+      environmentType: sameEnvironmentType,
+      deploymentShortCode: "test4321",
+    });
+
+    // Even with the same environment type and version, the image refs should be different due to deployment short codes
+    expect(firstImageRef.imageRef).toBe(
+      `registry.example.com/${testNamespace}/${testProjectRef}:${sameVersion}.preview.test1234`
+    );
+    expect(secondImageRef.imageRef).toBe(
+      `registry.example.com/${testNamespace}/${testProjectRef}:${sameVersion}.preview.test4321`
+    );
+    expect(firstImageRef.imageRef).not.toBe(secondImageRef.imageRef);
   });
 });
 
-describe.skipIf(process.env.RUN_REGISTRY_AUTH_TESTS !== "1")("getEcrAuthToken", () => {
+describe.skipIf(process.env.RUN_ECR_TESTS !== "1")("getEcrAuthToken", () => {
   const testHost =
     process.env.DEPLOY_REGISTRY_HOST || "123456789012.dkr.ecr.us-east-1.amazonaws.com";
 
@@ -188,8 +224,7 @@ describe.skipIf(process.env.RUN_REGISTRY_AUTH_TESTS !== "1")("getEcrAuthToken",
     expect(auth.password.length).toBeGreaterThan(0);
 
     // Verify the token format (should be a base64-encoded string)
-    const base64Regex = /^[A-Za-z0-9+/=]+$/;
-    expect(base64Regex.test(auth.password)).toBe(true);
+    expect(auth.password).toMatch(/^[A-Za-z0-9+/=]+$/);
   });
 
   it("should throw error for invalid region", async () => {

docs/config/config-file.mdx

@@ -99,16 +99,16 @@ import { defineConfig } from "@trigger.dev/sdk";
 export default defineConfig({
   project: "<project ref>",
   // Your other config settings...
-  onSuccess: async (payload, output, { ctx }) => {
+  onSuccess: async ({ payload, output, ctx }) => {
     console.log("Task succeeded", ctx.task.id);
   },
-  onFailure: async (payload, error, { ctx }) => {
+  onFailure: async ({ payload, error, ctx }) => {
     console.log("Task failed", ctx.task.id);
   },
-  onStart: async (payload, { ctx }) => {
+  onStart: async ({ payload, ctx }) => {
     console.log("Task started", ctx.task.id);
   },
-  init: async (payload, { ctx }) => {
+  init: async ({ payload, ctx }) => {
     console.log("I run before any task is run");
   },
 });

docs/errors-retrying.mdx

@@ -180,7 +180,7 @@ export const taskWithFetchRetries = task({
 
 ## Advanced error handling and retrying
 
-We provide a `handleError` callback on the task and in your `trigger.config` file. This gets called when an uncaught error is thrown in your task.
+We provide a `catchError` callback on the task and in your `trigger.config` file. This gets called when an uncaught error is thrown in your task.
 
 You can
 
@@ -219,7 +219,7 @@ export const openaiTask = task({
 
     return chatCompletion.choices[0].message.content;
   },
-  handleError: async (payload, error, { ctx, retryAt }) => {
+  catchError: async ({ payload, error, ctx, retryAt }) => {
     if (error instanceof OpenAI.APIError) {
       if (!error.status) {
         return {

docs/github-actions.mdx

@@ -102,13 +102,16 @@ If you already have a GitHub action file, you can just add the final step "🚀
 ## CLI Version pinning
 
 The CLI and `@trigger.dev/*` package versions need to be in sync with the `trigger.dev` CLI, otherwise there will be errors and unpredictable behavior. Hence, the `deploy` command will automatically fail during CI on any version mismatches.
-Tip: add the deploy command to your `package.json` file to keep versions managed in the same place. For example:
+Tip: add the `trigger.dev` CLI to your `devDependencies` and the deploy command to your `package.json` file to keep versions managed in the same place. For example:
 
 ```json
 {
   "scripts": {
-    "deploy:trigger-prod": "npx trigger.dev@3.0.0 deploy",
-    "deploy:trigger": "npx trigger.dev@3.0.0 deploy --env staging"
+    "deploy:trigger-prod": "trigger deploy",
+    "deploy:trigger": "trigger deploy --env staging"
+  },
+  "devDependencies": {
+    "trigger.dev": "4.0.2"
   }
 }
 ```
@@ -134,9 +137,7 @@ When self-hosting, you will have to take a few additional steps:
 - Add your registry credentials to the GitHub secrets.
 - Use the `--self-hosted` and `--push` flags when deploying.
 
-<Tip>
-  If you're self-hosting v4, the `--self-hosted` and `--push` flags are **NOT** needed.
-</Tip>
+<Tip>If you're self-hosting v4, the `--self-hosted` and `--push` flags are **NOT** needed.</Tip>
 
 Other than that, your GitHub action file will look very similar to the one above:
 

docs/guides/examples/sentry-error-tracking.mdx

@@ -56,7 +56,7 @@ export default defineConfig({
       environment: process.env.NODE_ENV === "production" ? "production" : "development",
     });
   },
-  onFailure: async (payload, error, { ctx }) => {
+  onFailure: async ({ payload, error, ctx }) => {
     Sentry.captureException(error, {
       extra: {
         payload,

docs/migrating-from-v3.mdx

@@ -232,7 +232,7 @@ import { task } from "@trigger.dev/sdk";
 
 export const myTask = task({
   id: "my-task",
-  onStart: (payload, { ctx }) => {},
+  onStart: ({ payload, ctx }) => {},
   run: async (payload, { ctx }) => {},
 });
 ```

docs/tasks/overview.mdx

@@ -181,7 +181,7 @@ This function is called before a run attempt:
 ```ts /trigger/init.ts
 export const taskWithInit = task({
   id: "task-with-init",
-  init: async (payload, { ctx }) => {
+  init: async ({ payload, ctx }) => {
     //...
   },
   run: async (payload: any, { ctx }) => {
@@ -195,7 +195,7 @@ You can also return data from the `init` function that will be available in the
 ```ts /trigger/init-return.ts
 export const taskWithInitReturn = task({
   id: "task-with-init-return",
-  init: async (payload, { ctx }) => {
+  init: async ({ payload, ctx }) => {
     return { someData: "someValue" };
   },
   run: async (payload: any, { ctx, init }) => {
@@ -213,7 +213,7 @@ This function is called after the `run` function is executed, regardless of whet
 ```ts /trigger/cleanup.ts
 export const taskWithCleanup = task({
   id: "task-with-cleanup",
-  cleanup: async (payload, { ctx }) => {
+  cleanup: async ({ payload, ctx }) => {
     //...
   },
   run: async (payload: any, { ctx }) => {
@@ -230,7 +230,7 @@ Our task middleware system runs at the top level, executing before and after all
 
 <Info>
   An error thrown in `middleware` is just like an uncaught error in the run function: it will
-  propagate through to `handleError()` and then will fail the attempt (causing a retry).
+  propagate through to `catchError()` function and then will fail the attempt (causing a retry).
 </Info>
 
 The `locals` API allows you to share data between middleware and hooks.
@@ -303,7 +303,7 @@ When a task run starts, the `onStart` function is called. It's useful for sendin
 ```ts /trigger/on-start.ts
 export const taskWithOnStart = task({
   id: "task-with-on-start",
-  onStart: async (payload, { ctx }) => {
+  onStart: async ({ payload, ctx }) => {
     //...
   },
   run: async (payload: any, { ctx }) => {
@@ -319,7 +319,7 @@ import { defineConfig } from "@trigger.dev/sdk";
 
 export default defineConfig({
   project: "proj_1234",
-  onStart: async (payload, { ctx }) => {
+  onStart: async ({ payload, ctx }) => {
     console.log("Task started", ctx.task.id);
   },
 });
@@ -357,7 +357,7 @@ When a task run succeeds, the `onSuccess` function is called. It's useful for se
 ```ts /trigger/on-success.ts
 export const taskWithOnSuccess = task({
   id: "task-with-on-success",
-  onSuccess: async (payload, output, { ctx }) => {
+  onSuccess: async ({ payload, output, ctx }) => {
     //...
   },
   run: async (payload: any, { ctx }) => {
@@ -373,7 +373,7 @@ import { defineConfig } from "@trigger.dev/sdk";
 
 export default defineConfig({
   project: "proj_1234",
-  onSuccess: async (payload, output, { ctx }) => {
+  onSuccess: async ({ payload, output, ctx }) => {
     console.log("Task succeeded", ctx.task.id);
   },
 });
@@ -388,7 +388,7 @@ This hook is executed when a run completes, regardless of whether it succeeded o
 ```ts /trigger/on-complete.ts
 export const taskWithOnComplete = task({
   id: "task-with-on-complete",
-  onComplete: async (payload, output, { ctx }) => {
+  onComplete: async ({ payload, output, ctx }) => {
   if (result.ok) {
     console.log("Run succeeded", result.data);
   } else {
@@ -404,7 +404,7 @@ When a task run fails, the `onFailure` function is called. It's useful for sendi
 ```ts /trigger/on-failure.ts
 export const taskWithOnFailure = task({
   id: "task-with-on-failure",
-  onFailure: async (payload, error, { ctx }) => {
+  onFailure: async ({ payload, error, ctx }) => {
     //...
   },
   run: async (payload: any, { ctx }) => {
@@ -420,7 +420,7 @@ import { defineConfig } from "@trigger.dev/sdk";
 
 export default defineConfig({
   project: "proj_1234",
-  onFailure: async (payload, error, { ctx }) => {
+  onFailure: async ({ payload, error, ctx }) => {
     console.log("Task failed", ctx.task.id);
   },
 });
@@ -429,14 +429,15 @@ export default defineConfig({
 <Info>Errors thrown in the `onFailure` function are ignored.</Info>
 
 <Note>
-  `onFailure` doesn’t fire for some of the run statuses like `Crashed`, `System failures`, and `Canceled`.
+  `onFailure` doesn’t fire for some of the run statuses like `Crashed`, `System failures`, and
+  `Canceled`.
 </Note>
 
-### `handleError` functions
+### `catchError` functions
 
 You can define a function that will be called when an error is thrown in the `run` function, that allows you to control how the error is handled and whether the task should be retried.
 
-Read more about `handleError` in our [Errors and Retrying guide](/errors-retrying).
+Read more about `catchError` in our [Errors and Retrying guide](/errors-retrying).
 
 <Info>Uncaught errors will throw a special internal error of the type `HANDLE_ERROR_ERROR`.</Info>
 

docs/wait-for-token.mdx

@@ -263,24 +263,6 @@ Wait for a token to be completed.
   The token to wait for.
 </ParamField>
 
-<ParamField query="options" type="object" optional>
-  Options for the wait.
-
-  <Expandable title="properties">
-    <ParamField query="releaseConcurrency" type="boolean" optional>
-      If set to true, this will cause the waitpoint to release the current run from the queue's concurrency.
-
-      This is useful if you want to allow other runs to execute while waiting
-
-      Note: It's possible that this run will not be able to resume when the waitpoint is complete if this is set to true.
-      It will go back in the queue and will resume once concurrency becomes available.
-
-      The default is `false`.
-    </ParamField>
-
-  </Expandable>
-</ParamField>
-
 ### returns
 
 The `forToken` function returns a result object with the following properties:

internal-packages/database/prisma/migrations/20250902112516_add_locked_retry_config_to_task_run/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "public"."TaskRun" ADD COLUMN     "lockedRetryConfig" JSONB;

internal-packages/database/prisma/migrations/20250902133449_add_gh_installation_schema/migration.sql

@@ -0,0 +1,46 @@
+CREATE TYPE "public"."GithubRepositorySelection" AS ENUM ('ALL', 'SELECTED');
+
+CREATE TABLE "public"."GithubAppInstallation" (
+    "id" TEXT NOT NULL,
+    "appInstallationId" BIGINT NOT NULL,
+    "targetId" BIGINT NOT NULL,
+    "targetType" TEXT NOT NULL,
+    "accountHandle" TEXT NOT NULL,
+    "permissions" JSONB,
+    "repositorySelection" "public"."GithubRepositorySelection" NOT NULL,
+    "installedBy" TEXT,
+    "organizationId" TEXT NOT NULL,
+    "deletedAt" TIMESTAMP(3),
+    "suspendedAt" TIMESTAMP(3),
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "GithubAppInstallation_pkey" PRIMARY KEY ("id")
+);
+
+CREATE TABLE "public"."GithubRepository" (
+    "id" TEXT NOT NULL,
+    "githubId" BIGINT NOT NULL,
+    "name" TEXT NOT NULL,
+    "fullName" TEXT NOT NULL,
+    "htmlUrl" TEXT NOT NULL,
+    "private" BOOLEAN NOT NULL,
+    "defaultBranch" TEXT NOT NULL,
+    "installationId" TEXT NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "GithubRepository_pkey" PRIMARY KEY ("id")
+);
+
+CREATE UNIQUE INDEX "GithubAppInstallation_appInstallationId_key" ON "public"."GithubAppInstallation"("appInstallationId");
+
+CREATE INDEX "GithubAppInstallation_organizationId_idx" ON "public"."GithubAppInstallation"("organizationId");
+
+CREATE INDEX "GithubRepository_installationId_idx" ON "public"."GithubRepository"("installationId");
+
+CREATE UNIQUE INDEX "GithubRepository_installationId_githubId_key" ON "public"."GithubRepository"("installationId", "githubId");
+
+ALTER TABLE "public"."GithubAppInstallation" ADD CONSTRAINT "GithubAppInstallation_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "public"."Organization"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
+
+ALTER TABLE "public"."GithubRepository" ADD CONSTRAINT "GithubRepository_installationId_fkey" FOREIGN KEY ("installationId") REFERENCES "public"."GithubAppInstallation"("id") ON DELETE CASCADE ON UPDATE CASCADE;

internal-packages/database/prisma/migrations/migration_lock.toml

@@ -1,3 +1,3 @@
 # Please do not edit this file manually
-# It should be added in your version-control system (i.e. Git)
-provider = "postgresql"
+# It should be added in your version-control system (e.g., Git)
+provider = "postgresql"

internal-packages/database/prisma/schema.prisma

@@ -151,7 +151,7 @@ model OrganizationAccessToken {
   /// This is used to find the token in the database
   hashedToken String @unique
 
-  organization   Organization   @relation(fields: [organizationId], references: [id])
+  organization   Organization @relation(fields: [organizationId], references: [id])
   organizationId String
 
   /// Optional expiration date for the token
@@ -210,6 +210,7 @@ model Organization {
   workerGroups             WorkerInstanceGroup[]
   workerInstances          WorkerInstance[]
   executionSnapshots       TaskRunExecutionSnapshot[]
+  githubAppInstallations   GithubAppInstallation[]
 }
 
 model OrgMember {
@@ -648,11 +649,12 @@ model TaskRun {
 
   concurrencyKey String?
 
-  delayUntil  DateTime?
-  queuedAt    DateTime?
-  ttl         String?
-  expiredAt   DateTime?
-  maxAttempts Int?
+  delayUntil        DateTime?
+  queuedAt          DateTime?
+  ttl               String?
+  expiredAt         DateTime?
+  maxAttempts       Int?
+  lockedRetryConfig Json?
 
   /// optional token that can be used to authenticate the task run
   oneTimeUseToken String?
@@ -2237,3 +2239,52 @@ model TaskEventPartitioned {
   // Used for getting all logs for a run
   @@index([runId])
 }
+
+enum GithubRepositorySelection {
+  ALL
+  SELECTED
+}
+
+model GithubAppInstallation {
+  id String @id @default(cuid())
+
+  appInstallationId   BigInt                    @unique
+  targetId            BigInt
+  targetType          String
+  accountHandle       String
+  permissions         Json?
+  repositorySelection GithubRepositorySelection
+  installedBy         String?
+
+  organization   Organization @relation(fields: [organizationId], references: [id])
+  organizationId String
+
+  repositories GithubRepository[]
+
+  deletedAt   DateTime?
+  suspendedAt DateTime?
+  createdAt   DateTime  @default(now())
+  updatedAt   DateTime  @updatedAt
+
+  @@index([organizationId])
+}
+
+model GithubRepository {
+  id String @id @default(cuid())
+
+  githubId      BigInt
+  name          String
+  fullName      String
+  htmlUrl       String
+  private       Boolean
+  defaultBranch String
+
+  installation   GithubAppInstallation @relation(fields: [installationId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  installationId String
+
+  createdAt DateTime @default(now())
+  updatedAt DateTime @updatedAt
+
+  @@unique([installationId, githubId])
+  @@index([installationId])
+}

internal-packages/run-engine/src/engine/retrying.ts

@@ -3,6 +3,7 @@ import {
   isOOMRunError,
   RetryOptions,
   sanitizeError,
+  shouldLookupRetrySettings,
   shouldRetryError,
   TaskRunError,
   taskRunErrorEnhancer,
@@ -72,13 +73,11 @@ export async function retryOutcomeFromCompletion(
     };
   }
 
-  // No retry settings
-  if (!retrySettings) {
-    return { outcome: "fail_run", sanitizedError };
-  }
+  const enhancedError = taskRunErrorEnhancer(error);
 
   // Not a retriable error: fail
-  const retriableError = shouldRetryError(taskRunErrorEnhancer(error));
+  const retriableError = shouldRetryError(enhancedError);
+
   if (!retriableError) {
     return { outcome: "fail_run", sanitizedError };
   }
@@ -95,6 +94,7 @@ export async function retryOutcomeFromCompletion(
     },
     select: {
       maxAttempts: true,
+      lockedRetryConfig: true,
     },
   });
 
@@ -112,6 +112,48 @@ export async function retryOutcomeFromCompletion(
     return { outcome: "fail_run", sanitizedError };
   }
 
+  // No retry settings
+  if (!retrySettings) {
+    const shouldLookup = shouldLookupRetrySettings(enhancedError);
+
+    if (!shouldLookup) {
+      return { outcome: "fail_run", sanitizedError };
+    }
+
+    const retryConfig = run.lockedRetryConfig;
+
+    if (!retryConfig) {
+      return { outcome: "fail_run", sanitizedError };
+    }
+
+    const parsedRetryConfig = RetryOptions.nullish().safeParse(retryConfig);
+
+    if (!parsedRetryConfig.success) {
+      return { outcome: "fail_run", sanitizedError };
+    }
+
+    if (!parsedRetryConfig.data) {
+      return { outcome: "fail_run", sanitizedError };
+    }
+
+    const nextDelay = calculateNextRetryDelay(parsedRetryConfig.data, attemptNumber ?? 1);
+
+    if (!nextDelay) {
+      return { outcome: "fail_run", sanitizedError };
+    }
+
+    const retrySettings = {
+      timestamp: Date.now() + nextDelay,
+      delay: nextDelay,
+    };
+
+    return {
+      outcome: "retry",
+      method: "queue", // we'll always retry on the queue because usually having no settings means something bad happened
+      settings: retrySettings,
+    };
+  }
+
   return {
     outcome: "retry",
     method: retryUsingQueue ? "queue" : "immediate",
@@ -130,19 +172,15 @@ async function retryOOMOnMachine(
       },
       select: {
         machinePreset: true,
-        lockedBy: {
-          select: {
-            retryConfig: true,
-          },
-        },
+        lockedRetryConfig: true,
       },
     });
 
-    if (!run || !run.lockedBy || !run.machinePreset) {
+    if (!run || !run.lockedRetryConfig || !run.machinePreset) {
       return;
     }
 
-    const retryConfig = run.lockedBy?.retryConfig;
+    const retryConfig = run.lockedRetryConfig;
     const parsedRetryConfig = RetryOptions.nullish().safeParse(retryConfig);
 
     if (!parsedRetryConfig.success) {

internal-packages/run-engine/src/engine/systems/dequeueSystem.ts

@@ -403,6 +403,9 @@ export class DequeueSystem {
                 result.run.maxDurationInSeconds,
                 result.task.maxDurationInSeconds
               );
+              const lockedRetryConfig = result.run.lockedRetryConfig
+                ? undefined
+                : result.task.retryConfig;
 
               const lockedTaskRun = await prisma.taskRun.update({
                 where: {
@@ -413,6 +416,7 @@ export class DequeueSystem {
                   lockedById: result.task.id,
                   lockedToVersionId: result.worker.id,
                   lockedQueueId: result.queue.id,
+                  lockedRetryConfig: lockedRetryConfig ?? undefined,
                   status: "DEQUEUED",
                   startedAt,
                   baseCostInCents: this.options.machines.baseCostInCents,

internal-packages/run-engine/src/run-queue/index.ts

@@ -42,6 +42,7 @@ import {
   RunQueueKeyProducer,
   RunQueueSelectionStrategy,
 } from "./types.js";
+import { WorkerQueueResolver } from "./workerQueueResolver.js";
 
 const SemanticAttributes = {
   QUEUE: "runqueue.queue",
@@ -169,6 +170,7 @@ export class RunQueue {
   private shardCount: number;
   private abortController: AbortController;
   private worker: Worker<typeof workerCatalog>;
+  private workerQueueResolver: WorkerQueueResolver;
   private _observableWorkerQueues: Set<string> = new Set();
   private _meter: Meter;
   private _queueCooloffStates: Map<string, QueueCooloffState> = new Map();
@@ -185,6 +187,8 @@ export class RunQueue {
       },
     });
     this.logger = options.logger ?? new Logger("RunQueue", options.logLevel ?? "info");
+
+    this.workerQueueResolver = new WorkerQueueResolver({ logger: this.logger });
     this._meter = options.meter ?? getMeter("run-queue");
 
     const workerQueueObservableGauge = this._meter.createObservableGauge(
@@ -1845,19 +1849,8 @@ export class RunQueue {
     );
   }
 
-  #getWorkerQueueFromMessage(message: OutputPayload) {
-    if (message.version === "2") {
-      return message.workerQueue;
-    }
-
-    // In v2, if the environment is development, the worker queue is the environment id.
-    if (message.environmentType === "DEVELOPMENT") {
-      return message.environmentId;
-    }
-
-    // In v1, the master queue is something like us-nyc-3,
-    // which in v2 is the worker queue.
-    return message.masterQueues[0];
+  #getWorkerQueueFromMessage(message: OutputPayload): string {
+    return this.workerQueueResolver.getWorkerQueueFromMessage(message);
   }
 
   #createBlockingDequeueClient() {

internal-packages/run-engine/src/run-queue/tests/workerQueueResolver.test.ts

@@ -0,0 +1,484 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { Logger } from "@trigger.dev/core/logger";
+import { WorkerQueueResolver, type WorkerQueueOverrides } from "../workerQueueResolver.js";
+import { OutputPayload, OutputPayloadV1, OutputPayloadV2 } from "../types.js";
+import { RuntimeEnvironmentType } from "@trigger.dev/core/v3";
+
+vi.setConfig({ testTimeout: 5_000 });
+
+describe("WorkerQueueOverrideResolver", () => {
+  const createTestMessage = (overrides?: Partial<OutputPayloadV2>): OutputPayloadV2 => ({
+    version: "2",
+    runId: "run_123",
+    taskIdentifier: "task_123",
+    orgId: "org_123",
+    projectId: "proj_123",
+    environmentId: "env_123",
+    environmentType: RuntimeEnvironmentType.PRODUCTION,
+    queue: "test-queue",
+    timestamp: Date.now(),
+    attempt: 0,
+    workerQueue: "default-queue",
+    ...overrides,
+  });
+
+  describe("No overrides", () => {
+    it("should return original workerQueue when no overrides are set", () => {
+      const logger = new Logger("test", "error");
+      const resolver = new WorkerQueueResolver({ logger });
+      const message = createTestMessage({ workerQueue: "original-queue" });
+
+      const result = resolver.getWorkerQueueFromMessage(message);
+
+      expect(result).toBe("original-queue");
+    });
+  });
+
+  describe("Environment ID overrides", () => {
+    it("should override based on environmentId", () => {
+      const overrideConfig = JSON.stringify({
+        environmentId: {
+          env_special: "special-env-queue",
+        },
+      } satisfies WorkerQueueOverrides);
+
+      const logger = new Logger("test", "error");
+      const resolver = new WorkerQueueResolver({ logger, overrideConfig });
+      const message = createTestMessage({
+        environmentId: "env_special",
+        workerQueue: "original-queue",
+      });
+
+      const result = resolver.getWorkerQueueFromMessage(message);
+
+      expect(result).toBe("special-env-queue");
+    });
+
+    it("should not override when environmentId doesn't match", () => {
+      const overrideConfig = JSON.stringify({
+        environmentId: {
+          env_other: "other-queue",
+        },
+      } satisfies WorkerQueueOverrides);
+
+      const logger = new Logger("test", "error");
+      const resolver = new WorkerQueueResolver({ logger, overrideConfig });
+      const message = createTestMessage({
+        environmentId: "env_123",
+        workerQueue: "original-queue",
+      });
+
+      const result = resolver.getWorkerQueueFromMessage(message);
+
+      expect(result).toBe("original-queue");
+    });
+  });
+
+  describe("Project ID overrides", () => {
+    it("should override based on projectId", () => {
+      const overrideConfig = JSON.stringify({
+        projectId: {
+          proj_special: "special-project-queue",
+        },
+      } satisfies WorkerQueueOverrides);
+
+      const logger = new Logger("test", "error");
+      const resolver = new WorkerQueueResolver({ logger, overrideConfig });
+      const message = createTestMessage({
+        projectId: "proj_special",
+        workerQueue: "original-queue",
+      });
+
+      const result = resolver.getWorkerQueueFromMessage(message);
+
+      expect(result).toBe("special-project-queue");
+    });
+  });
+
+  describe("Organization ID overrides", () => {
+    it("should override based on orgId", () => {
+      const overrideConfig = JSON.stringify({
+        orgId: {
+          org_special: "special-org-queue",
+        },
+      } satisfies WorkerQueueOverrides);
+
+      const logger = new Logger("test", "error");
+      const resolver = new WorkerQueueResolver({ logger, overrideConfig });
+      const message = createTestMessage({
+        orgId: "org_special",
+        workerQueue: "original-queue",
+      });
+
+      const result = resolver.getWorkerQueueFromMessage(message);
+
+      expect(result).toBe("special-org-queue");
+    });
+  });
+
+  describe("Worker Queue overrides", () => {
+    it("should override based on workerQueue", () => {
+      const overrideConfig = JSON.stringify({
+        workerQueue: {
+          "us-east-1": "us-west-1",
+        },
+      } satisfies WorkerQueueOverrides);
+
+      const logger = new Logger("test", "error");
+      const resolver = new WorkerQueueResolver({ logger, overrideConfig });
+      const message = createTestMessage({
+        workerQueue: "us-east-1",
+      });
+
+      const result = resolver.getWorkerQueueFromMessage(message);
+
+      expect(result).toBe("us-west-1");
+    });
+  });
+
+  describe("Priority order", () => {
+    it("should prioritize environmentId over projectId", () => {
+      const overrideConfig = JSON.stringify({
+        environmentId: {
+          env_123: "env-queue",
+        },
+        projectId: {
+          proj_123: "project-queue",
+        },
+      } satisfies WorkerQueueOverrides);
+
+      const logger = new Logger("test", "error");
+      const resolver = new WorkerQueueResolver({ logger, overrideConfig });
+      const message = createTestMessage();
+
+      const result = resolver.getWorkerQueueFromMessage(message);
+
+      expect(result).toBe("env-queue");
+    });
+
+    it("should prioritize projectId over orgId", () => {
+      const overrideConfig = JSON.stringify({
+        projectId: {
+          proj_123: "project-queue",
+        },
+        orgId: {
+          org_123: "org-queue",
+        },
+      } satisfies WorkerQueueOverrides);
+
+      const logger = new Logger("test", "error");
+      const resolver = new WorkerQueueResolver({ logger, overrideConfig });
+      const message = createTestMessage();
+
+      const result = resolver.getWorkerQueueFromMessage(message);
+
+      expect(result).toBe("project-queue");
+    });
+
+    it("should prioritize orgId over workerQueue", () => {
+      const overrideConfig = JSON.stringify({
+        orgId: {
+          org_123: "org-queue",
+        },
+        workerQueue: {
+          "default-queue": "worker-override-queue",
+        },
+      } satisfies WorkerQueueOverrides);
+
+      const logger = new Logger("test", "error");
+      const resolver = new WorkerQueueResolver({ logger, overrideConfig });
+      const message = createTestMessage();
+
+      const result = resolver.getWorkerQueueFromMessage(message);
+
+      expect(result).toBe("org-queue");
+    });
+  });
+
+  describe("Configuration parsing", () => {
+    it("should handle invalid JSON gracefully", () => {
+      const loggerSpy = vi.spyOn(Logger.prototype, "error");
+
+      const overrideConfig = "invalid json {";
+
+      const logger = new Logger("test", "error");
+      const resolver = new WorkerQueueResolver({ logger, overrideConfig });
+      const message = createTestMessage();
+
+      const result = resolver.getWorkerQueueFromMessage(message);
+
+      expect(result).toBe("default-queue");
+      expect(loggerSpy).toHaveBeenCalledWith(
+        "Failed to parse worker queue overrides json",
+        expect.any(Object)
+      );
+
+      loggerSpy.mockRestore();
+    });
+
+    it("should handle non-object JSON gracefully", () => {
+      const loggerSpy = vi.spyOn(Logger.prototype, "error");
+
+      const overrideConfig = JSON.stringify("not an object");
+
+      const logger = new Logger("test", "error");
+      const resolver = new WorkerQueueResolver({ logger, overrideConfig });
+      const message = createTestMessage();
+
+      const result = resolver.getWorkerQueueFromMessage(message);
+
+      expect(result).toBe("default-queue");
+      expect(loggerSpy).toHaveBeenCalledWith(
+        "Invalid worker queue overrides format",
+        expect.any(Object)
+      );
+
+      loggerSpy.mockRestore();
+    });
+
+    it("should handle null JSON gracefully", () => {
+      const loggerSpy = vi.spyOn(Logger.prototype, "error");
+
+      const overrideConfig = JSON.stringify(null);
+
+      const logger = new Logger("test", "error");
+      const resolver = new WorkerQueueResolver({ logger, overrideConfig });
+      const message = createTestMessage();
+
+      const result = resolver.getWorkerQueueFromMessage(message);
+
+      expect(result).toBe("default-queue");
+      expect(loggerSpy).toHaveBeenCalledWith(
+        "Invalid worker queue overrides format",
+        expect.any(Object)
+      );
+
+      loggerSpy.mockRestore();
+    });
+
+    it("should log when overrides are enabled", () => {
+      const loggerSpy = vi.spyOn(Logger.prototype, "info");
+
+      const overrides: WorkerQueueOverrides = {
+        orgId: { org_123: "dedicated-queue" },
+      };
+
+      const overrideConfig = JSON.stringify(overrides);
+
+      const logger = new Logger("test", "info");
+      new WorkerQueueResolver({ logger, overrideConfig });
+
+      expect(loggerSpy).toHaveBeenCalledWith("🎯 Worker queue overrides enabled", { overrides });
+
+      loggerSpy.mockRestore();
+    });
+
+    it("should validate schema and reject invalid structure", () => {
+      const loggerSpy = vi.spyOn(Logger.prototype, "error");
+
+      // Invalid structure - numbers instead of strings in the record
+      const overrideConfig = JSON.stringify({
+        orgId: {
+          org_123: 12345, // Should be string, not number
+        },
+      });
+
+      const logger = new Logger("test", "error");
+      const resolver = new WorkerQueueResolver({ logger, overrideConfig });
+      const message = createTestMessage();
+
+      const result = resolver.getWorkerQueueFromMessage(message);
+
+      expect(result).toBe("default-queue");
+      expect(loggerSpy).toHaveBeenCalledWith(
+        "Invalid worker queue overrides format",
+        expect.any(Object)
+      );
+
+      loggerSpy.mockRestore();
+    });
+  });
+
+  describe("Complex scenarios", () => {
+    it("should handle multiple override types simultaneously", () => {
+      const overrideConfig = JSON.stringify({
+        environmentId: {
+          env_special: "special-env-queue",
+        },
+        projectId: {
+          proj_other: "other-project-queue",
+        },
+        orgId: {
+          org_123: "org-queue",
+        },
+        workerQueue: {
+          "fallback-queue": "redirected-queue",
+        },
+      });
+
+      const logger = new Logger("test", "error");
+      const resolver = new WorkerQueueResolver({ logger, overrideConfig });
+
+      // Should use orgId override since env and project don't match
+      const message1 = createTestMessage({
+        environmentId: "env_123",
+        projectId: "proj_123",
+        orgId: "org_123",
+        workerQueue: "original-queue",
+      });
+
+      const result1 = resolver.getWorkerQueueFromMessage(message1);
+      expect(result1).toBe("org-queue");
+
+      // Should use environmentId override since it matches
+      const message2 = createTestMessage({
+        environmentId: "env_special",
+        projectId: "proj_123",
+        orgId: "org_456",
+        workerQueue: "original-queue",
+      });
+
+      const result2 = resolver.getWorkerQueueFromMessage(message2);
+      expect(result2).toBe("special-env-queue");
+
+      // Should use workerQueue override as fallback
+      const message3 = createTestMessage({
+        environmentId: "env_unknown",
+        projectId: "proj_unknown",
+        orgId: "org_unknown",
+        workerQueue: "fallback-queue",
+      });
+
+      const result3 = resolver.getWorkerQueueFromMessage(message3);
+      expect(result3).toBe("redirected-queue");
+    });
+
+    it("should handle empty override sections", () => {
+      const overrideConfig = JSON.stringify({
+        environmentId: {},
+        projectId: {},
+        orgId: {
+          org_123: "org-queue",
+        },
+        workerQueue: {},
+      });
+
+      const logger = new Logger("test", "error");
+      const resolver = new WorkerQueueResolver({ logger, overrideConfig });
+      const message = createTestMessage();
+
+      const result = resolver.getWorkerQueueFromMessage(message);
+
+      expect(result).toBe("org-queue");
+    });
+  });
+
+  describe("V1 message handling", () => {
+    it("should handle v1 development messages", () => {
+      const logger = new Logger("test", "error");
+      const resolver = new WorkerQueueResolver({ logger });
+
+      const v1DevMessage: OutputPayloadV1 = {
+        version: "1",
+        runId: "run_123",
+        taskIdentifier: "task_123",
+        orgId: "org_123",
+        projectId: "proj_123",
+        environmentId: "env_dev",
+        environmentType: RuntimeEnvironmentType.DEVELOPMENT,
+        queue: "test-queue",
+        timestamp: Date.now(),
+        attempt: 0,
+        masterQueues: ["us-east-1", "us-west-1"],
+      };
+
+      const result = resolver.getWorkerQueueFromMessage(v1DevMessage);
+
+      expect(result).toBe("env_dev");
+    });
+
+    it("should handle v1 production messages", () => {
+      const logger = new Logger("test", "error");
+      const resolver = new WorkerQueueResolver({ logger });
+
+      const v1ProdMessage: OutputPayloadV1 = {
+        version: "1",
+        runId: "run_123",
+        taskIdentifier: "task_123",
+        orgId: "org_123",
+        projectId: "proj_123",
+        environmentId: "env_prod",
+        environmentType: RuntimeEnvironmentType.PRODUCTION,
+        queue: "test-queue",
+        timestamp: Date.now(),
+        attempt: 0,
+        masterQueues: ["us-east-1", "us-west-1"],
+      };
+
+      const result = resolver.getWorkerQueueFromMessage(v1ProdMessage);
+
+      expect(result).toBe("us-east-1");
+    });
+  });
+
+  describe("Environment variable fallback", () => {
+    let originalEnv: string | undefined;
+
+    beforeEach(() => {
+      originalEnv = process.env.RUN_ENGINE_WORKER_QUEUE_OVERRIDES;
+    });
+
+    afterEach(() => {
+      if (originalEnv === undefined) {
+        delete process.env.RUN_ENGINE_WORKER_QUEUE_OVERRIDES;
+      } else {
+        process.env.RUN_ENGINE_WORKER_QUEUE_OVERRIDES = originalEnv;
+      }
+    });
+
+    it("should fall back to environment variable when no overrideConfig provided", () => {
+      // Set environment variable
+      process.env.RUN_ENGINE_WORKER_QUEUE_OVERRIDES = JSON.stringify({
+        orgId: {
+          org_from_env: "env-based-queue",
+        },
+      });
+
+      const logger = new Logger("test", "error");
+      const resolver = new WorkerQueueResolver({ logger }); // No overrideConfig
+      const message = createTestMessage({
+        orgId: "org_from_env",
+      });
+
+      const result = resolver.getWorkerQueueFromMessage(message);
+
+      expect(result).toBe("env-based-queue");
+    });
+
+    it("should prioritize overrideConfig over environment variable", () => {
+      // Set environment variable
+      process.env.RUN_ENGINE_WORKER_QUEUE_OVERRIDES = JSON.stringify({
+        orgId: {
+          org_123: "env-queue",
+        },
+      });
+
+      // Pass config directly (should take precedence)
+      const overrideConfig = JSON.stringify({
+        orgId: {
+          org_123: "config-queue",
+        },
+      });
+
+      const logger = new Logger("test", "error");
+      const resolver = new WorkerQueueResolver({ logger, overrideConfig });
+      const message = createTestMessage({
+        orgId: "org_123",
+      });
+
+      const result = resolver.getWorkerQueueFromMessage(message);
+
+      expect(result).toBe("config-queue");
+    });
+  });
+});

internal-packages/run-engine/src/run-queue/workerQueueResolver.ts

@@ -0,0 +1,100 @@
+import type { Logger } from "@trigger.dev/core/logger";
+import type { OutputPayload, OutputPayloadV2 } from "./types.js";
+import { z } from "zod";
+
+const WorkerQueueOverrides = z.object({
+  environmentId: z.record(z.string(), z.string()).optional(),
+  projectId: z.record(z.string(), z.string()).optional(),
+  orgId: z.record(z.string(), z.string()).optional(),
+  workerQueue: z.record(z.string(), z.string()).optional(),
+});
+
+export type WorkerQueueOverrides = z.infer<typeof WorkerQueueOverrides>;
+
+export type WorkerQueueResolverOptions = {
+  logger: Logger;
+  overrideConfig?: string;
+};
+
+export class WorkerQueueResolver {
+  private overrides: WorkerQueueOverrides | null;
+  private logger: Logger;
+
+  constructor(opts: WorkerQueueResolverOptions) {
+    this.logger = opts.logger;
+    this.overrides = this.parseOverrides(opts.overrideConfig);
+  }
+
+  private parseOverrides(overrideConfig?: string): WorkerQueueOverrides | null {
+    const overridesJson = overrideConfig ?? process.env.RUN_ENGINE_WORKER_QUEUE_OVERRIDES;
+
+    if (!overridesJson) {
+      return null;
+    }
+
+    try {
+      const parsed = JSON.parse(overridesJson);
+      const result = WorkerQueueOverrides.safeParse(parsed);
+
+      if (!result.success) {
+        this.logger.error("Invalid worker queue overrides format", {
+          error: result.error.format(),
+        });
+        return null;
+      }
+
+      this.logger.info("🎯 Worker queue overrides enabled", { overrides: result.data });
+
+      return result.data;
+    } catch (error) {
+      this.logger.error("Failed to parse worker queue overrides json", {
+        error,
+      });
+      return null;
+    }
+  }
+
+  public getWorkerQueueFromMessage(message: OutputPayload): string {
+    if (message.version === "2") {
+      // Check overrides in priority order
+      const override = this.#getOverride(message);
+      if (override) return override;
+
+      return message.workerQueue;
+    }
+
+    // In v2, if the environment is development, the worker queue is the environment id.
+    if (message.environmentType === "DEVELOPMENT") {
+      return message.environmentId;
+    }
+
+    // In v1, the master queue is something like us-nyc-3,
+    // which in v2 is the worker queue.
+    return message.masterQueues[0];
+  }
+
+  #getOverride(message: OutputPayloadV2): string | null {
+    if (!this.overrides) {
+      return null;
+    }
+
+    // Priority: environmentId > projectId > orgId > workerQueue
+    if (this.overrides.environmentId?.[message.environmentId]) {
+      return this.overrides.environmentId[message.environmentId];
+    }
+
+    if (this.overrides.projectId?.[message.projectId]) {
+      return this.overrides.projectId[message.projectId];
+    }
+
+    if (this.overrides.orgId?.[message.orgId]) {
+      return this.overrides.orgId[message.orgId];
+    }
+
+    if (this.overrides.workerQueue?.[message.workerQueue]) {
+      return this.overrides.workerQueue[message.workerQueue];
+    }
+
+    return null;
+  }
+}

packages/core/src/v3/errors.ts

@@ -346,6 +346,33 @@ export function shouldRetryError(error: TaskRunError): boolean {
   }
 }
 
+export function shouldLookupRetrySettings(error: TaskRunError): boolean {
+  switch (error.type) {
+    case "INTERNAL_ERROR": {
+      switch (error.code) {
+        case "TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE":
+        case "TASK_PROCESS_SIGTERM":
+          return true;
+
+        default:
+          return false;
+      }
+    }
+    case "STRING_ERROR": {
+      return false;
+    }
+    case "BUILT_IN_ERROR": {
+      return false;
+    }
+    case "CUSTOM_ERROR": {
+      return false;
+    }
+    default: {
+      assertExhaustive(error);
+    }
+  }
+}
+
 export function correctErrorStackTrace(
   stackTrace: string,
   projectDir?: string,

packages/core/src/v3/runEngineWorker/index.ts

@@ -2,6 +2,7 @@ export * from "./consts.js";
 export * from "./supervisor/http.js";
 export * from "./supervisor/schemas.js";
 export * from "./supervisor/session.js";
+export * from "./supervisor/consumerPool.js";
 export * from "./workload/http.js";
 export * from "./workload/schemas.js";
 export * from "./types.js";

packages/core/src/v3/runEngineWorker/supervisor/consumerPool.test.ts

@@ -0,0 +1,721 @@
+import { describe, it, expect, beforeEach, afterEach, vi, Mock } from "vitest";
+import {
+  RunQueueConsumerPool,
+  type ConsumerPoolOptions,
+  type QueueConsumerFactory,
+} from "./consumerPool.js";
+import { SupervisorHttpClient } from "./http.js";
+import type { WorkerApiDequeueResponseBody } from "./schemas.js";
+import type { QueueConsumer } from "./queueConsumer.js";
+
+// Mock only the logger
+vi.mock("../../utils/structuredLogger.js");
+
+// Test implementation of QueueConsumer
+class TestQueueConsumer implements QueueConsumer {
+  public started = false;
+  public stopped = false;
+  public onDequeue?: (messages: WorkerApiDequeueResponseBody) => Promise<void>;
+
+  constructor(opts: any) {
+    this.onDequeue = opts.onDequeue;
+  }
+
+  start(): void {
+    this.started = true;
+    this.stopped = false;
+  }
+
+  stop(): void {
+    this.stopped = true;
+    this.started = false;
+  }
+}
+
+describe("RunQueueConsumerPool", () => {
+  let mockClient: SupervisorHttpClient;
+  let mockOnDequeue: Mock;
+  let pool: RunQueueConsumerPool;
+  let defaultOptions: Omit<ConsumerPoolOptions, "scaling">;
+  let testConsumers: TestQueueConsumer[];
+  let testConsumerFactory: QueueConsumerFactory;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.useFakeTimers();
+
+    mockClient = {} as SupervisorHttpClient;
+    mockOnDequeue = vi.fn();
+    testConsumers = [];
+
+    testConsumerFactory = (opts) => {
+      const consumer = new TestQueueConsumer(opts);
+      testConsumers.push(consumer);
+      return consumer;
+    };
+
+    defaultOptions = {
+      consumer: {
+        client: mockClient,
+        intervalMs: 0,
+        idleIntervalMs: 1000,
+        onDequeue: mockOnDequeue,
+      },
+      consumerFactory: testConsumerFactory,
+    };
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+    if (pool) {
+      pool.stop();
+    }
+  });
+
+  function advanceTimeAndProcessMetrics(ms: number) {
+    vi.advanceTimersByTime(ms);
+
+    // Trigger batch processing if ready (without adding a sample)
+    if (pool["metricsProcessor"].shouldProcessBatch()) {
+      pool["processMetricsBatch"]();
+    }
+  }
+
+  describe("Static mode (strategy='none')", () => {
+    it("should start with maxConsumerCount in static mode", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: { strategy: "none", maxConsumerCount: 5 },
+      });
+
+      await pool.start();
+
+      expect(pool.size).toBe(5);
+      expect(testConsumers.length).toBe(5);
+    });
+
+    it("should not scale in static mode even with queue length updates", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: { strategy: "none", maxConsumerCount: 3 },
+      });
+
+      await pool.start();
+      const initialCount = pool.size;
+
+      pool.updateQueueLength(100);
+      vi.advanceTimersByTime(2000);
+
+      expect(pool.size).toBe(initialCount);
+      expect(pool.size).toBe(3);
+    });
+  });
+
+  describe("Smooth scaling strategy", () => {
+    it("should scale smoothly with damping", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "smooth",
+          minConsumerCount: 1,
+          maxConsumerCount: 10,
+          scaleUpCooldownMs: 0,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+      expect(pool.size).toBe(1);
+
+      pool.updateQueueLength(5);
+      advanceTimeAndProcessMetrics(1100);
+      expect(pool.size).toBe(4); // Damped scaling
+
+      pool.updateQueueLength(5);
+      advanceTimeAndProcessMetrics(1100);
+      expect(pool.size).toBe(5); // Gradually approaches target
+    });
+
+    it("should respect max consumer count", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "smooth",
+          minConsumerCount: 1,
+          maxConsumerCount: 5,
+          scaleUpCooldownMs: 0,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+      expect(pool.size).toBe(1);
+
+      pool.updateQueueLength(100);
+      advanceTimeAndProcessMetrics(1100);
+      expect(pool.size).toBe(5);
+
+      pool.updateQueueLength(100);
+      advanceTimeAndProcessMetrics(1100);
+      expect(pool.size).toBe(5);
+    });
+  });
+
+  describe("Aggressive scaling strategy", () => {
+    it("should scale up quickly based on queue pressure", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "aggressive",
+          minConsumerCount: 2,
+          maxConsumerCount: 10,
+          scaleUpCooldownMs: 0,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+      expect(pool.size).toBe(2);
+
+      pool.updateQueueLength(10);
+      advanceTimeAndProcessMetrics(1100);
+      expect(pool.size).toBe(3);
+
+      pool.updateQueueLength(20);
+      advanceTimeAndProcessMetrics(1100);
+      expect(pool.size).toBe(4);
+    });
+
+    it("should scale down cautiously when queue is small", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "aggressive",
+          minConsumerCount: 1,
+          maxConsumerCount: 10,
+          scaleUpCooldownMs: 0,
+          scaleDownCooldownMs: 0,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+      expect(pool.size).toBe(1);
+
+      pool.updateQueueLength(10);
+      advanceTimeAndProcessMetrics(1100);
+      expect(pool.size).toBe(2);
+
+      pool.updateQueueLength(0.5);
+      advanceTimeAndProcessMetrics(1100);
+      expect(pool.size).toBe(3); // EWMA smoothing delays scale down
+
+      pool.updateQueueLength(0.5);
+      advanceTimeAndProcessMetrics(1100);
+      expect(pool.size).toBeGreaterThanOrEqual(3); // Stays in optimal zone
+    });
+
+    it("should maintain current level in optimal zone", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "aggressive",
+          minConsumerCount: 3,
+          maxConsumerCount: 10,
+          scaleUpCooldownMs: 0,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+      expect(pool.size).toBe(3);
+
+      pool.updateQueueLength(3);
+      advanceTimeAndProcessMetrics(1100);
+      expect(pool.size).toBe(3);
+
+      pool.updateQueueLength(4);
+      advanceTimeAndProcessMetrics(1100);
+      expect(pool.size).toBe(3);
+    });
+  });
+
+  describe("Smooth scaling with EWMA", () => {
+    it("should use exponential smoothing for stable scaling", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "smooth",
+          minConsumerCount: 1,
+          maxConsumerCount: 10,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+
+      const queueLengths = [10, 2, 8, 3, 9, 1, 7];
+      for (const length of queueLengths) {
+        pool.updateQueueLength(length);
+        vi.advanceTimersByTime(200);
+      }
+      vi.advanceTimersByTime(900);
+
+      const metrics = pool.getMetrics();
+      expect(metrics.smoothedQueueLength).toBeGreaterThan(0);
+      expect(metrics.smoothedQueueLength).toBeLessThan(10);
+    });
+
+    it("should apply damping factor to avoid rapid changes", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "smooth",
+          minConsumerCount: 1,
+          maxConsumerCount: 10,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+
+      pool.updateQueueLength(2);
+      advanceTimeAndProcessMetrics(1100);
+      const metrics1 = pool.getMetrics();
+      expect(metrics1.smoothedQueueLength).toBe(2);
+
+      pool.updateQueueLength(20);
+      advanceTimeAndProcessMetrics(1100);
+      const metrics2 = pool.getMetrics();
+
+      expect(metrics2.smoothedQueueLength).toBeGreaterThan(2);
+      expect(metrics2.smoothedQueueLength).toBeLessThan(20);
+    });
+  });
+
+  describe("High throughput parallel dequeuing", () => {
+    it("should handle rapid parallel queue updates", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "aggressive",
+          minConsumerCount: 1,
+          maxConsumerCount: 20,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+
+      const updates: number[] = [];
+      for (let i = 0; i < 100; i++) {
+        updates.push(Math.floor(Math.random() * 50) + 10);
+      }
+
+      updates.forEach((length, index) => {
+        setTimeout(() => pool.updateQueueLength(length), index * 10);
+      });
+
+      advanceTimeAndProcessMetrics(1100);
+
+      const metrics = pool.getMetrics();
+      expect(metrics.queueLength).toBeDefined();
+    });
+
+    it("should batch metrics updates to avoid excessive scaling", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "smooth",
+          minConsumerCount: 1,
+          maxConsumerCount: 10,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+      const evaluateScalingSpy = vi.spyOn(pool as any, "evaluateScaling");
+
+      pool.updateQueueLength(10);
+      for (let i = 1; i < 50; i++) {
+        pool.updateQueueLength(Math.floor(Math.random() * 20) + 5);
+      }
+
+      expect(evaluateScalingSpy).not.toHaveBeenCalled();
+      advanceTimeAndProcessMetrics(1000);
+      expect(evaluateScalingSpy).toHaveBeenCalledTimes(1);
+    });
+
+    it("should use median to filter outliers in high-frequency updates", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "aggressive",
+          minConsumerCount: 1,
+          maxConsumerCount: 10,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+
+      const updates = [10, 11, 9, 12, 10, 100, 11, 10, 9, 11, 1];
+      updates.forEach((length) => pool.updateQueueLength(length));
+      advanceTimeAndProcessMetrics(1100);
+
+      const metrics = pool.getMetrics();
+      expect(metrics.queueLength).toBeGreaterThanOrEqual(9);
+      expect(metrics.queueLength).toBeLessThanOrEqual(12);
+    });
+  });
+
+  describe("Scaling cooldowns and jitter", () => {
+    it("should respect scale-up cooldown", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "smooth",
+          scaleUpCooldownMs: 5000,
+          minConsumerCount: 1,
+          maxConsumerCount: 10,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+      pool["scaleToTarget"](5);
+      const scaleToTargetSpy = vi.spyOn(pool as any, "scaleToTarget");
+
+      pool.updateQueueLength(10);
+      advanceTimeAndProcessMetrics(1100);
+      expect(scaleToTargetSpy).not.toHaveBeenCalled();
+
+      vi.advanceTimersByTime(10000);
+      pool.updateQueueLength(20);
+      advanceTimeAndProcessMetrics(1100);
+    });
+
+    it("should respect scale-down cooldown (longer than scale-up)", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "aggressive",
+          minConsumerCount: 1,
+          maxConsumerCount: 10,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+
+      for (let i = 0; i < 4; i++) {
+        pool["addConsumers"](1);
+      }
+      pool["scaleToTarget"](5);
+      pool["metrics"].lastScaleTime = new Date(Date.now() - 70000);
+
+      pool.updateQueueLength(1);
+      advanceTimeAndProcessMetrics(1100);
+
+      const metrics = pool.getMetrics();
+      expect(metrics.queueLength).toBe(1);
+    });
+
+    it("should add random jitter to prevent thundering herd", async () => {
+      const pools: RunQueueConsumerPool[] = [];
+      const scaleTimes: number[] = [];
+
+      for (let i = 0; i < 3; i++) {
+        const p = new RunQueueConsumerPool({
+          ...defaultOptions,
+          scaling: {
+            strategy: "smooth",
+            minConsumerCount: 1,
+            maxConsumerCount: 10,
+            disableJitter: true,
+          },
+        });
+
+        const originalScale = p["scaleToTarget"];
+        p["scaleToTarget"] = vi.fn(async (target: number) => {
+          scaleTimes.push(Date.now());
+          return originalScale.call(p, target);
+        });
+
+        pools.push(p);
+        await p.start();
+      }
+
+      pools.forEach((p) => p.updateQueueLength(20));
+      advanceTimeAndProcessMetrics(1100);
+      vi.advanceTimersByTime(15000);
+
+      await Promise.all(pools.map((p) => p.stop()));
+    });
+  });
+
+  describe("Consumer lifecycle management", () => {
+    it("should properly start and stop consumers", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "none",
+          maxConsumerCount: 3,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+
+      expect(pool.size).toBe(3);
+      expect(testConsumers.length).toBe(3);
+      testConsumers.forEach((consumer) => {
+        expect(consumer.started).toBe(true);
+      });
+
+      await pool.stop();
+
+      testConsumers.forEach((consumer) => {
+        expect(consumer.stopped).toBe(true);
+      });
+    });
+
+    it("should forward dequeue messages with queue length updates", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "aggressive",
+          maxConsumerCount: 2,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+
+      const messages: WorkerApiDequeueResponseBody = [{ workerQueueLength: 15 } as any];
+
+      if (testConsumers[0]?.onDequeue) {
+        await testConsumers[0].onDequeue(messages);
+      }
+
+      expect(mockOnDequeue).toHaveBeenCalledWith(messages);
+
+      advanceTimeAndProcessMetrics(1100);
+      const metrics = pool.getMetrics();
+      expect(metrics.queueLength).toBe(15);
+    });
+  });
+
+  describe("Memory leak prevention", () => {
+    it("should collect all samples within batch window without limit", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "aggressive",
+          minConsumerCount: 1,
+          maxConsumerCount: 10,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+
+      for (let i = 0; i < 100; i++) {
+        pool.updateQueueLength(i);
+      }
+
+      const metrics = pool.getMetrics();
+      expect(metrics.queueLength).toBeUndefined();
+    });
+
+    it("should clear consumer map on stop", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "none",
+          maxConsumerCount: 5,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+      expect(pool.size).toBe(5);
+
+      await pool.stop();
+      expect(pool.size).toBe(0);
+    });
+
+    it("should clear recentQueueLengths after processing batch", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: { strategy: "smooth" },
+      });
+
+      await pool.start();
+
+      for (let i = 0; i < 5; i++) {
+        pool.updateQueueLength(10 + i);
+      }
+
+      advanceTimeAndProcessMetrics(1100);
+      const metrics = pool.getMetrics();
+      expect(metrics.queueLength).toBeDefined();
+    });
+
+    it("should not accumulate scaling operations in memory", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "aggressive",
+          scaleUpCooldownMs: 100,
+          scaleDownCooldownMs: 100,
+          minConsumerCount: 1,
+          maxConsumerCount: 10,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+
+      for (let i = 0; i < 5; i++) {
+        pool["metrics"].lastScaleTime = new Date(0);
+        pool.updateQueueLength(i % 2 === 0 ? 50 : 1);
+        vi.advanceTimersByTime(1100);
+      }
+
+      expect(pool.size).toBeGreaterThanOrEqual(1);
+      expect(pool.size).toBeLessThanOrEqual(10);
+    });
+  });
+
+  describe("Edge cases", () => {
+    it("should handle empty recent queue lengths", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: { strategy: "aggressive" },
+      });
+
+      await pool.start();
+
+      const metrics = pool.getMetrics();
+      expect(metrics.queueLength).toBeUndefined();
+    });
+
+    it("should clamp consumer count to min/max bounds", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "smooth",
+          minConsumerCount: 2,
+          maxConsumerCount: 5,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+      expect(pool.size).toBe(2);
+
+      pool.updateQueueLength(100);
+      advanceTimeAndProcessMetrics(1100);
+      expect(pool.size).toBeLessThanOrEqual(5);
+    });
+
+    it("should respect custom targetRatio with smooth strategy", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "smooth",
+          targetRatio: 5,
+          scaleUpCooldownMs: 0,
+          minConsumerCount: 1,
+          maxConsumerCount: 10,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+      expect(pool.size).toBe(1);
+
+      pool.updateQueueLength(10);
+      advanceTimeAndProcessMetrics(1100);
+
+      const firstSize = pool.size;
+      expect(firstSize).toBeGreaterThanOrEqual(1);
+      expect(firstSize).toBeLessThanOrEqual(2);
+
+      pool.updateQueueLength(10);
+      advanceTimeAndProcessMetrics(1100);
+      expect(pool.size).toBeLessThanOrEqual(2);
+    });
+
+    it("should respect custom targetRatio with aggressive strategy", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "aggressive",
+          targetRatio: 5,
+          scaleUpCooldownMs: 0,
+          minConsumerCount: 1,
+          maxConsumerCount: 10,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+      expect(pool.size).toBe(1);
+
+      pool.updateQueueLength(20);
+      advanceTimeAndProcessMetrics(1100);
+
+      const sizeAfterFirstScale = pool.size;
+      expect(sizeAfterFirstScale).toBeGreaterThanOrEqual(1);
+
+      pool.updateQueueLength(20);
+      advanceTimeAndProcessMetrics(1100);
+      expect(pool.size).toBeLessThanOrEqual(6);
+    });
+
+    it("should scale down when no items are dequeued (zero queue length)", async () => {
+      pool = new RunQueueConsumerPool({
+        ...defaultOptions,
+        scaling: {
+          strategy: "smooth",
+          minConsumerCount: 1,
+          maxConsumerCount: 10,
+          scaleUpCooldownMs: 0,
+          scaleDownCooldownMs: 0,
+          disableJitter: true,
+        },
+      });
+
+      await pool.start();
+      expect(pool.size).toBe(1);
+
+      // Scale up first
+      pool.updateQueueLength(20);
+      advanceTimeAndProcessMetrics(1100);
+      expect(pool.size).toBeGreaterThan(1);
+      const sizeAfterScaleUp = pool.size;
+
+      // Now send multiple zero queue lengths to converge EWMA to 0
+      // The EWMA needs time to converge due to exponential smoothing
+      for (let i = 0; i < 5; i++) {
+        pool.updateQueueLength(0);
+        advanceTimeAndProcessMetrics(1100);
+      }
+
+      // After multiple iterations with zero queue, should scale down but not to minimum yet
+      expect(pool.size).toBeLessThan(sizeAfterScaleUp);
+      expect(pool.size).toBeGreaterThan(1);
+
+      // Continue until we reach minimum
+      for (let i = 0; i < 5; i++) {
+        pool.updateQueueLength(0);
+        advanceTimeAndProcessMetrics(1100);
+      }
+
+      // Should eventually reach minimum
+      expect(pool.size).toBe(1);
+    });
+  });
+});

packages/core/src/v3/runEngineWorker/supervisor/consumerPool.ts

@@ -0,0 +1,411 @@
+import { SimpleStructuredLogger } from "../../utils/structuredLogger.js";
+import { QueueConsumer, RunQueueConsumer, RunQueueConsumerOptions } from "./queueConsumer.js";
+import { QueueMetricsProcessor } from "./queueMetricsProcessor.js";
+import {
+  ScalingStrategy,
+  ScalingStrategyKind,
+  ScalingStrategyOptions,
+} from "./scalingStrategies.js";
+import { ConsumerPoolMetrics } from "./consumerPoolMetrics.js";
+import type { Registry } from "prom-client";
+
+export type QueueConsumerFactory = (opts: RunQueueConsumerOptions) => QueueConsumer;
+
+export type ScalingOptions = {
+  strategy?: ScalingStrategyKind;
+  minConsumerCount?: number;
+  maxConsumerCount?: number;
+  scaleUpCooldownMs?: number;
+  scaleDownCooldownMs?: number;
+  targetRatio?: number;
+  ewmaAlpha?: number;
+  batchWindowMs?: number;
+  disableJitter?: boolean;
+  dampingFactor?: number;
+};
+
+export type ConsumerPoolOptions = {
+  consumer: RunQueueConsumerOptions;
+  scaling: ScalingOptions;
+  consumerFactory?: QueueConsumerFactory;
+  metricsRegistry?: Registry;
+};
+
+type ScalingMetrics = {
+  targetConsumerCount: number;
+  queueLength?: number;
+  smoothedQueueLength: number;
+  lastScaleTime: Date;
+  lastQueueLengthUpdate: Date;
+};
+
+export class RunQueueConsumerPool {
+  private readonly consumerOptions: RunQueueConsumerOptions;
+
+  private readonly logger = new SimpleStructuredLogger("consumer-pool");
+  private readonly promMetrics?: ConsumerPoolMetrics;
+
+  private readonly minConsumerCount: number;
+  private readonly maxConsumerCount: number;
+  private readonly scalingStrategy: ScalingStrategy;
+  private readonly disableJitter: boolean;
+
+  private consumers: Map<string, QueueConsumer> = new Map();
+  private readonly consumerFactory: QueueConsumerFactory;
+  private isEnabled: boolean = false;
+  private isScaling: boolean = false;
+
+  private metrics: ScalingMetrics;
+  private readonly metricsProcessor: QueueMetricsProcessor;
+
+  // Scaling parameters
+  private readonly ewmaAlpha: number;
+  private readonly scaleUpCooldownMs: number;
+  private readonly scaleDownCooldownMs: number;
+  private readonly batchWindowMs: number;
+
+  constructor(opts: ConsumerPoolOptions) {
+    this.consumerOptions = opts.consumer;
+
+    // Initialize Prometheus metrics if registry provided
+    if (opts.metricsRegistry) {
+      this.promMetrics = new ConsumerPoolMetrics({
+        register: opts.metricsRegistry,
+      });
+    }
+
+    this.minConsumerCount = Math.max(1, opts.scaling.minConsumerCount ?? 1);
+    this.maxConsumerCount = Math.max(this.minConsumerCount, opts.scaling.maxConsumerCount ?? 10);
+    this.scaleUpCooldownMs = opts.scaling.scaleUpCooldownMs ?? 10000; // 10 seconds default
+    this.scaleDownCooldownMs = opts.scaling.scaleDownCooldownMs ?? 60000; // 60 seconds default
+    this.disableJitter = opts.scaling.disableJitter ?? false;
+
+    // Configure EWMA parameters from options
+    this.ewmaAlpha = opts.scaling.ewmaAlpha ?? 0.3;
+    this.batchWindowMs = opts.scaling.batchWindowMs ?? 1000;
+
+    // Validate EWMA parameters
+    if (this.ewmaAlpha < 0 || this.ewmaAlpha > 1) {
+      throw new Error(`ewmaAlpha must be between 0 and 1, got: ${this.ewmaAlpha}`);
+    }
+    if (this.batchWindowMs <= 0) {
+      throw new Error(`batchWindowMs must be positive, got: ${this.batchWindowMs}`);
+    }
+
+    // Initialize metrics processor
+    this.metricsProcessor = new QueueMetricsProcessor({
+      ewmaAlpha: this.ewmaAlpha,
+      batchWindowMs: this.batchWindowMs,
+    });
+
+    const targetRatio = opts.scaling.targetRatio ?? 1.0;
+    const dampingFactor = opts.scaling.dampingFactor;
+
+    // Create scaling strategy with metrics processor injected
+    this.scalingStrategy = ScalingStrategy.create(opts.scaling.strategy ?? "none", {
+      metricsProcessor: this.metricsProcessor,
+      dampingFactor,
+      targetRatio,
+      minConsumerCount: this.minConsumerCount,
+      maxConsumerCount: this.maxConsumerCount,
+    });
+
+    // Use provided factory or default to RunQueueConsumer
+    this.consumerFactory =
+      opts.consumerFactory || ((consumerOpts) => new RunQueueConsumer(consumerOpts));
+
+    this.metrics = {
+      targetConsumerCount: this.minConsumerCount,
+      queueLength: undefined,
+      smoothedQueueLength: 0,
+      lastScaleTime: new Date(0),
+      lastQueueLengthUpdate: new Date(0),
+    };
+
+    this.logger.log("Initialized consumer pool", {
+      minConsumerCount: this.minConsumerCount,
+      maxConsumerCount: this.maxConsumerCount,
+      scalingStrategy: this.scalingStrategy.name,
+      mode: this.scalingStrategy.name === "none" ? "static" : "dynamic",
+      ewmaAlpha: this.ewmaAlpha,
+      batchWindowMs: this.batchWindowMs,
+    });
+  }
+
+  async start() {
+    if (this.isEnabled) {
+      return;
+    }
+
+    this.isEnabled = true;
+
+    // For 'none' strategy, start with max consumers (static mode)
+    // For dynamic strategies, start with minimum
+    const initialCount =
+      this.scalingStrategy.name === "none" ? this.maxConsumerCount : this.minConsumerCount;
+
+    // Set initial metrics
+    this.metrics.targetConsumerCount = initialCount;
+
+    this.addConsumers(initialCount);
+
+    this.logger.log("Started dynamic consumer pool", {
+      initialConsumerCount: this.consumers.size,
+    });
+
+    // Initialize Prometheus metrics with initial state
+    this.promMetrics?.updateState({
+      consumerCount: this.consumers.size,
+      queueLength: this.metrics.queueLength,
+      smoothedQueueLength: this.metrics.smoothedQueueLength,
+      targetConsumerCount: initialCount,
+      strategy: this.scalingStrategy.name,
+    });
+  }
+
+  async stop() {
+    if (!this.isEnabled) {
+      return;
+    }
+
+    this.isEnabled = false;
+
+    // Stop all consumers
+    Array.from(this.consumers.values()).forEach((consumer) => consumer.stop());
+
+    this.consumers.clear();
+
+    this.logger.log("Stopped dynamic consumer pool");
+  }
+
+  /**
+   * Updates the queue length metric and triggers scaling decisions
+   * Uses QueueMetricsProcessor for batching and EWMA smoothing
+   */
+  updateQueueLength(queueLength: number) {
+    // Track queue length update in metrics
+    this.promMetrics?.recordQueueLengthUpdate();
+
+    // Skip metrics tracking for static mode
+    if (this.scalingStrategy.name === "none") {
+      return;
+    }
+
+    // Add sample to metrics processor
+    this.metricsProcessor.addSample(queueLength);
+
+    // Check if we should process the current batch
+    if (this.metricsProcessor.shouldProcessBatch()) {
+      this.processMetricsBatch();
+    }
+  }
+
+  private processMetricsBatch() {
+    // Process batch using the metrics processor
+    const result = this.metricsProcessor.processBatch();
+
+    if (!result) {
+      this.logger.debug("No queue length samples in batch window - skipping scaling evaluation");
+      return;
+    }
+
+    // Update metrics
+    this.metrics.queueLength = result.median;
+    this.metrics.smoothedQueueLength = result.smoothedValue;
+    this.metrics.lastQueueLengthUpdate = new Date();
+
+    this.logger.verbose("Queue metrics batch processed", {
+      samples: result.sampleCount,
+      median: result.median,
+      smoothed: result.smoothedValue,
+      currentConsumerCount: this.consumers.size,
+    });
+
+    // Make scaling decision
+    this.evaluateScaling();
+  }
+
+  private evaluateScaling() {
+    if (!this.isEnabled) {
+      return;
+    }
+
+    // No scaling in static mode
+    if (this.scalingStrategy.name === "none") {
+      return;
+    }
+
+    // Skip if already scaling
+    if (this.isScaling) {
+      this.logger.debug("Scaling blocked - operation already in progress", {
+        currentCount: this.consumers.size,
+        targetCount: this.metrics.targetConsumerCount,
+        actualCount: this.consumers.size,
+      });
+      return;
+    }
+
+    const targetCount = this.calculateTargetConsumerCount();
+
+    if (targetCount === this.consumers.size) {
+      return;
+    }
+
+    const timeSinceLastScale = Date.now() - this.metrics.lastScaleTime.getTime();
+
+    // Add random jitter to avoid thundering herd when multiple replicas exist
+    // Works without needing to know replica index or count
+    const jitterMs = this.disableJitter ? 0 : Math.random() * 3000; // 0-3 seconds random jitter
+
+    // Check cooldown periods with jitter
+    if (targetCount > this.consumers.size) {
+      // Scale up
+      const effectiveCooldown = this.scaleUpCooldownMs + jitterMs;
+      if (timeSinceLastScale < effectiveCooldown) {
+        this.logger.debug("Scale up blocked by cooldown", {
+          timeSinceLastScale,
+          cooldownMs: effectiveCooldown,
+          jitterMs,
+          remainingMs: effectiveCooldown - timeSinceLastScale,
+        });
+        this.promMetrics?.recordCooldownApplied("up");
+        return;
+      }
+    } else if (targetCount < this.consumers.size) {
+      // Scale down
+      const effectiveCooldown = this.scaleDownCooldownMs + jitterMs;
+      if (timeSinceLastScale < effectiveCooldown) {
+        this.logger.debug("Scale down blocked by cooldown", {
+          timeSinceLastScale,
+          cooldownMs: effectiveCooldown,
+          jitterMs,
+          remainingMs: effectiveCooldown - timeSinceLastScale,
+        });
+        this.promMetrics?.recordCooldownApplied("down");
+        return;
+      }
+    }
+
+    this.logger.info("Scaling consumer pool", {
+      from: this.consumers.size,
+      to: targetCount,
+      queueLength: this.metrics.queueLength,
+      smoothedQueueLength: this.metrics.smoothedQueueLength,
+      strategy: this.scalingStrategy,
+    });
+
+    // Set flag before scaling
+    this.isScaling = true;
+
+    // Update target metric for visibility
+    const previousTarget = this.metrics.targetConsumerCount;
+    this.metrics.targetConsumerCount = targetCount;
+
+    try {
+      this.scaleToTarget(targetCount);
+    } catch (error) {
+      this.logger.error("Failed to scale consumer pool", { error });
+      // Revert target on failure
+      this.metrics.targetConsumerCount = previousTarget;
+    } finally {
+      this.isScaling = false;
+    }
+  }
+
+  private calculateTargetConsumerCount(): number {
+    return this.scalingStrategy.calculateTargetCount(this.consumers.size);
+  }
+
+  private scaleToTarget(targetCount: number) {
+    const actualCurrentCount = this.consumers.size;
+
+    if (targetCount > actualCurrentCount) {
+      // Scale up
+      const count = targetCount - actualCurrentCount;
+      this.addConsumers(count);
+      this.promMetrics?.recordScalingOperation("up", this.scalingStrategy.name, count);
+    } else if (targetCount < actualCurrentCount) {
+      // Scale down
+      const count = actualCurrentCount - targetCount;
+      this.removeConsumers(count);
+      this.promMetrics?.recordScalingOperation("down", this.scalingStrategy.name, count);
+    }
+
+    this.metrics.lastScaleTime = new Date();
+
+    // Update Prometheus state metrics
+    this.promMetrics?.updateState({
+      consumerCount: this.consumers.size,
+      queueLength: this.metrics.queueLength,
+      smoothedQueueLength: this.metrics.smoothedQueueLength,
+      targetConsumerCount: targetCount,
+      strategy: this.scalingStrategy.name,
+    });
+  }
+
+  private addConsumers(count: number) {
+    const newConsumers: QueueConsumer[] = [];
+
+    for (let i = 0; i < count; i++) {
+      const consumerId = `consumer-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
+
+      const consumer = this.consumerFactory({
+        ...this.consumerOptions,
+        onDequeue: async (messages) => {
+          // Always update queue length, default to 0 for empty dequeues or missing value
+          this.updateQueueLength(messages[0]?.workerQueueLength ?? 0);
+
+          // Forward to the original handler
+          await this.consumerOptions.onDequeue(messages);
+        },
+      });
+
+      this.consumers.set(consumerId, consumer);
+      newConsumers.push(consumer);
+    }
+
+    // Start all new consumers
+    newConsumers.forEach((c) => c.start());
+
+    this.logger.info("Added consumers", {
+      count,
+      totalConsumers: this.consumers.size,
+    });
+  }
+
+  private removeConsumers(count: number) {
+    const allIds = Array.from(this.consumers.keys());
+    const consumerIds = allIds.slice(-count); // Take from the end
+    const consumersToStop: QueueConsumer[] = [];
+
+    for (const id of consumerIds) {
+      const consumer = this.consumers.get(id);
+      if (consumer) {
+        consumersToStop.push(consumer);
+        this.consumers.delete(id);
+      }
+    }
+
+    // Stop removed consumers
+    consumersToStop.forEach((c) => c.stop());
+
+    this.logger.info("Removed consumers", {
+      count: consumersToStop.length,
+      totalConsumers: this.consumers.size,
+    });
+  }
+
+  /**
+   * Get current pool metrics for monitoring
+   */
+  getMetrics(): Readonly<ScalingMetrics> {
+    return { ...this.metrics };
+  }
+
+  /**
+   * Get current number of consumers in the pool
+   */
+  get size(): number {
+    return this.consumers.size;
+  }
+}

packages/core/src/v3/runEngineWorker/supervisor/consumerPoolMetrics.ts

@@ -0,0 +1,160 @@
+import { Counter, Gauge, Histogram, Registry } from "prom-client";
+
+export interface ConsumerPoolMetricsOptions {
+  register?: Registry;
+  prefix?: string;
+}
+
+export class ConsumerPoolMetrics {
+  private readonly register: Registry;
+  private readonly prefix: string;
+
+  // Current state metrics
+  public readonly consumerCount: Gauge;
+  public readonly queueLength: Gauge;
+  public readonly smoothedQueueLength: Gauge;
+  public readonly targetConsumerCount: Gauge;
+  public readonly scalingStrategy: Gauge;
+
+  // Scaling operation metrics
+  public readonly scalingOperationsTotal: Counter;
+  public readonly consumersAddedTotal: Counter;
+  public readonly consumersRemovedTotal: Counter;
+  public readonly scalingCooldownsApplied: Counter;
+
+  // Performance metrics
+  public readonly queueLengthUpdatesTotal: Counter;
+  public readonly batchesProcessedTotal: Counter;
+
+  constructor(opts: ConsumerPoolMetricsOptions = {}) {
+    this.register = opts.register ?? new Registry();
+    this.prefix = opts.prefix ?? "queue_consumer_pool";
+
+    // Current state metrics
+    this.consumerCount = new Gauge({
+      name: `${this.prefix}_consumer_count`,
+      help: "Current number of active queue consumers",
+      labelNames: ["strategy"],
+      registers: [this.register],
+    });
+
+    this.queueLength = new Gauge({
+      name: `${this.prefix}_queue_length`,
+      help: "Current queue length (median of recent samples)",
+      registers: [this.register],
+    });
+
+    this.smoothedQueueLength = new Gauge({
+      name: `${this.prefix}_smoothed_queue_length`,
+      help: "EWMA smoothed queue length",
+      registers: [this.register],
+    });
+
+    this.targetConsumerCount = new Gauge({
+      name: `${this.prefix}_target_consumer_count`,
+      help: "Target number of consumers calculated by scaling strategy",
+      labelNames: ["strategy"],
+      registers: [this.register],
+    });
+
+    this.scalingStrategy = new Gauge({
+      name: `${this.prefix}_scaling_strategy_info`,
+      help: "Information about the active scaling strategy (1 = active, 0 = inactive)",
+      labelNames: ["strategy"],
+      registers: [this.register],
+    });
+
+    // Scaling operation metrics
+    this.scalingOperationsTotal = new Counter({
+      name: `${this.prefix}_scaling_operations_total`,
+      help: "Total number of scaling operations performed",
+      labelNames: ["direction", "strategy"],
+      registers: [this.register],
+    });
+
+    this.consumersAddedTotal = new Counter({
+      name: `${this.prefix}_consumers_added_total`,
+      help: "Total number of consumers added",
+      registers: [this.register],
+    });
+
+    this.consumersRemovedTotal = new Counter({
+      name: `${this.prefix}_consumers_removed_total`,
+      help: "Total number of consumers removed",
+      registers: [this.register],
+    });
+
+    this.scalingCooldownsApplied = new Counter({
+      name: `${this.prefix}_scaling_cooldowns_applied_total`,
+      help: "Number of times scaling was prevented due to cooldown",
+      labelNames: ["direction"],
+      registers: [this.register],
+    });
+
+    this.queueLengthUpdatesTotal = new Counter({
+      name: `${this.prefix}_queue_length_updates_total`,
+      help: "Total number of queue length updates received",
+      registers: [this.register],
+    });
+
+    this.batchesProcessedTotal = new Counter({
+      name: `${this.prefix}_batches_processed_total`,
+      help: "Total number of metric batches processed",
+      registers: [this.register],
+    });
+  }
+
+  /**
+   * Update all gauge metrics with current state
+   */
+  updateState(state: {
+    consumerCount: number;
+    queueLength?: number;
+    smoothedQueueLength: number;
+    targetConsumerCount: number;
+    strategy: string;
+  }) {
+    this.consumerCount.set({ strategy: state.strategy }, state.consumerCount);
+
+    if (state.queueLength !== undefined) {
+      this.queueLength.set(state.queueLength);
+    }
+
+    this.smoothedQueueLength.set(state.smoothedQueueLength);
+    this.targetConsumerCount.set({ strategy: state.strategy }, state.targetConsumerCount);
+
+    // Set strategy info (1 for active strategy, 0 for others)
+    ["none", "smooth", "aggressive"].forEach((s) => {
+      this.scalingStrategy.set({ strategy: s }, s === state.strategy ? 1 : 0);
+    });
+  }
+
+  /**
+   * Record a scaling operation
+   */
+  recordScalingOperation(direction: "up" | "down" | "none", strategy: string, count: number) {
+    if (direction !== "none") {
+      this.scalingOperationsTotal.inc({ direction, strategy });
+
+      if (direction === "up") {
+        this.consumersAddedTotal.inc(count);
+      } else {
+        this.consumersRemovedTotal.inc(count);
+      }
+    }
+  }
+
+  /**
+   * Record that scaling was prevented by cooldown
+   */
+  recordCooldownApplied(direction: "up" | "down") {
+    this.scalingCooldownsApplied.inc({ direction });
+  }
+
+  /**
+   * Record a queue length update
+   */
+  recordQueueLengthUpdate() {
+    this.queueLengthUpdatesTotal.inc();
+  }
+}

packages/core/src/v3/runEngineWorker/supervisor/queueConsumer.ts

@@ -3,7 +3,12 @@ import { SupervisorHttpClient } from "./http.js";
 import { WorkerApiDequeueResponseBody } from "./schemas.js";
 import { PreDequeueFn, PreSkipFn } from "./types.js";
 
-type RunQueueConsumerOptions = {
+export interface QueueConsumer {
+  start(): void;
+  stop(): void;
+}
+
+export type RunQueueConsumerOptions = {
   client: SupervisorHttpClient;
   intervalMs: number;
   idleIntervalMs: number;
@@ -13,7 +18,7 @@ type RunQueueConsumerOptions = {
   onDequeue: (messages: WorkerApiDequeueResponseBody) => Promise<void>;
 };
 
-export class RunQueueConsumer {
+export class RunQueueConsumer implements QueueConsumer {
   private readonly client: SupervisorHttpClient;
   private readonly preDequeue?: PreDequeueFn;
   private readonly preSkip?: PreSkipFn;
@@ -131,7 +136,7 @@ export class RunQueueConsumer {
     this.scheduleNextDequeue(nextIntervalMs);
   }
 
-  scheduleNextDequeue(delayMs: number) {
+  private scheduleNextDequeue(delayMs: number) {
     if (delayMs === this.idleIntervalMs && this.idleIntervalMs !== this.intervalMs) {
       this.logger.verbose("scheduled dequeue with idle interval", { delayMs });
     }

packages/core/src/v3/runEngineWorker/supervisor/queueMetricsProcessor.test.ts

@@ -0,0 +1,371 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { QueueMetricsProcessor } from "./queueMetricsProcessor.js";
+
+describe("QueueMetricsProcessor", () => {
+  let processor: QueueMetricsProcessor;
+
+  beforeEach(() => {
+    vi.useFakeTimers();
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  describe("Constructor validation", () => {
+    it("should throw error for invalid ewmaAlpha", () => {
+      expect(() => new QueueMetricsProcessor({ ewmaAlpha: -0.1, batchWindowMs: 1000 })).toThrow(
+        "ewmaAlpha must be between 0 and 1"
+      );
+
+      expect(() => new QueueMetricsProcessor({ ewmaAlpha: 1.1, batchWindowMs: 1000 })).toThrow(
+        "ewmaAlpha must be between 0 and 1"
+      );
+    });
+
+    it("should throw error for invalid batchWindowMs", () => {
+      expect(() => new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 0 })).toThrow(
+        "batchWindowMs must be positive"
+      );
+
+      expect(() => new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: -100 })).toThrow(
+        "batchWindowMs must be positive"
+      );
+    });
+
+    it("should accept valid parameters", () => {
+      expect(() => new QueueMetricsProcessor({ ewmaAlpha: 0, batchWindowMs: 1 })).not.toThrow();
+      expect(() => new QueueMetricsProcessor({ ewmaAlpha: 1, batchWindowMs: 5000 })).not.toThrow();
+    });
+  });
+
+  describe("Sample collection", () => {
+    beforeEach(() => {
+      processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 });
+    });
+
+    it("should collect samples without limit", () => {
+      for (let i = 0; i < 100; i++) {
+        processor.addSample(i);
+      }
+
+      expect(processor.getCurrentSampleCount()).toBe(100);
+      expect(processor.getCurrentSamples()).toHaveLength(100);
+    });
+
+    it("should throw error for negative queue lengths", () => {
+      expect(() => processor.addSample(-1)).toThrow("Queue length cannot be negative");
+    });
+
+    it("should accept zero queue length", () => {
+      expect(() => processor.addSample(0)).not.toThrow();
+      expect(processor.getCurrentSampleCount()).toBe(1);
+    });
+
+    it("should handle empty queue with all zero samples", () => {
+      processor.addSample(0);
+      processor.addSample(0);
+      processor.addSample(0);
+
+      const result = processor.processBatch();
+      expect(result).not.toBeNull();
+      expect(result!.median).toBe(0);
+      expect(result!.smoothedValue).toBe(0);
+      expect(processor.getSmoothedValue()).toBe(0);
+    });
+
+    it("should properly transition from zero to non-zero queue", () => {
+      // Start with empty queue
+      processor.addSample(0);
+      processor.addSample(0);
+      let result = processor.processBatch();
+      expect(result!.median).toBe(0);
+      expect(result!.smoothedValue).toBe(0);
+
+      // Queue starts filling
+      processor.addSample(10);
+      processor.addSample(15);
+      result = processor.processBatch();
+      expect(result!.median).toBeGreaterThan(0);
+      // EWMA: 0.3 * median + 0.7 * 0
+      expect(result!.smoothedValue).toBeGreaterThan(0);
+    });
+
+    it("should properly transition from non-zero to zero queue", () => {
+      // Start with non-empty queue
+      processor.addSample(10);
+      processor.addSample(15);
+      let result = processor.processBatch();
+      const initialSmoothed = result!.smoothedValue;
+      expect(initialSmoothed).toBeGreaterThan(0);
+
+      // Queue becomes empty
+      processor.addSample(0);
+      processor.addSample(0);
+      processor.addSample(0);
+      result = processor.processBatch();
+      expect(result!.median).toBe(0);
+      // EWMA should gradually decrease: 0.3 * 0 + 0.7 * initialSmoothed
+      expect(result!.smoothedValue).toBe(0.7 * initialSmoothed);
+    });
+  });
+
+  describe("Batch processing timing", () => {
+    beforeEach(() => {
+      processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 });
+    });
+
+    it("should not process batch before window expires", () => {
+      processor.addSample(10, 1000);
+
+      expect(processor.shouldProcessBatch(1500)).toBe(false); // 500ms later
+      expect(processor.shouldProcessBatch(1999)).toBe(false); // 999ms later
+    });
+
+    it("should process batch when window expires", () => {
+      processor.addSample(10, 1000);
+
+      expect(processor.shouldProcessBatch(2000)).toBe(true); // 1000ms later
+      expect(processor.shouldProcessBatch(2500)).toBe(true); // 1500ms later
+    });
+
+    it("should not process empty batch", () => {
+      expect(processor.shouldProcessBatch(5000)).toBe(false);
+    });
+  });
+
+  describe("EWMA calculation", () => {
+    it("should initialize with first value", () => {
+      processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 });
+
+      processor.addSample(10);
+      const result = processor.processBatch();
+
+      expect(result).not.toBeNull();
+      expect(result!.median).toBe(10);
+      expect(result!.smoothedValue).toBe(10);
+      expect(processor.getSmoothedValue()).toBe(10);
+    });
+
+    it("should apply EWMA formula correctly", () => {
+      processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 });
+
+      // First batch: smoothed = 10
+      processor.addSample(10);
+      processor.processBatch();
+      expect(processor.getSmoothedValue()).toBe(10);
+
+      // Second batch: smoothed = 0.3 * 20 + 0.7 * 10 = 6 + 7 = 13
+      processor.addSample(20);
+      processor.processBatch();
+      expect(processor.getSmoothedValue()).toBe(13);
+
+      // Third batch: smoothed = 0.3 * 5 + 0.7 * 13 = 1.5 + 9.1 = 10.6
+      processor.addSample(5);
+      processor.processBatch();
+      expect(processor.getSmoothedValue()).toBe(10.6);
+    });
+
+    it("should test different alpha values", () => {
+      // High alpha (0.8) - more responsive
+      const highAlphaProcessor = new QueueMetricsProcessor({ ewmaAlpha: 0.8, batchWindowMs: 1000 });
+      highAlphaProcessor.addSample(10);
+      highAlphaProcessor.processBatch();
+      highAlphaProcessor.addSample(20);
+      highAlphaProcessor.processBatch();
+
+      // Low alpha (0.1) - more smoothing
+      const lowAlphaProcessor = new QueueMetricsProcessor({ ewmaAlpha: 0.1, batchWindowMs: 1000 });
+      lowAlphaProcessor.addSample(10);
+      lowAlphaProcessor.processBatch();
+      lowAlphaProcessor.addSample(20);
+      lowAlphaProcessor.processBatch();
+
+      // High alpha should be closer to recent value (20)
+      expect(highAlphaProcessor.getSmoothedValue()).toBeCloseTo(18); // 0.8 * 20 + 0.2 * 10 = 18
+      // Low alpha should be closer to previous value (10)
+      expect(lowAlphaProcessor.getSmoothedValue()).toBeCloseTo(11); // 0.1 * 20 + 0.9 * 10 = 11
+    });
+  });
+
+  describe("Median filtering", () => {
+    beforeEach(() => {
+      processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 });
+    });
+
+    it("should calculate median of single sample", () => {
+      processor.addSample(42);
+
+      const result = processor.processBatch();
+      expect(result!.median).toBe(42);
+      expect(result!.sampleCount).toBe(1);
+      expect(result!.smoothedValue).toBe(42); // First batch initializes to median
+    });
+
+    it("should calculate median of odd number of samples", () => {
+      processor.addSample(1);
+      processor.addSample(10);
+      processor.addSample(5);
+
+      const result = processor.processBatch();
+      expect(result!.median).toBe(5);
+    });
+
+    it("should calculate median of even number of samples", () => {
+      processor.addSample(1);
+      processor.addSample(10);
+      processor.addSample(5);
+      processor.addSample(8);
+
+      const result = processor.processBatch();
+      // With even count, we average the two middle values
+      // Sorted: [1, 5, 8, 10], median = (5 + 8) / 2 = 6.5
+      expect(result!.median).toBe(6.5);
+    });
+
+    it("should filter outliers using median", () => {
+      // Add mostly low values with one outlier
+      processor.addSample(5);
+      processor.addSample(5);
+      processor.addSample(5);
+      processor.addSample(100); // outlier
+      processor.addSample(5);
+
+      const result = processor.processBatch();
+      // Sorted: [5, 5, 5, 5, 100], median = 5 (filters out outlier)
+      expect(result!.median).toBe(5);
+    });
+  });
+
+  describe("Batch result", () => {
+    beforeEach(() => {
+      processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 });
+    });
+
+    it("should return comprehensive batch result", () => {
+      processor.addSample(10);
+      processor.addSample(20);
+      processor.addSample(15);
+
+      const result = processor.processBatch();
+
+      expect(result).not.toBeNull();
+      expect(result!.median).toBe(15);
+      expect(result!.smoothedValue).toBe(15); // First batch
+      expect(result!.sampleCount).toBe(3);
+      expect(result!.samples).toEqual([10, 20, 15]);
+    });
+
+    it("should return null for empty batch", () => {
+      const result = processor.processBatch();
+      expect(result).toBeNull();
+    });
+
+    it("should clear samples after processing", () => {
+      processor.addSample(10);
+      processor.addSample(20);
+
+      expect(processor.getCurrentSampleCount()).toBe(2);
+
+      processor.processBatch();
+
+      expect(processor.getCurrentSampleCount()).toBe(0);
+      expect(processor.getCurrentSamples()).toHaveLength(0);
+    });
+  });
+
+  describe("Reset functionality", () => {
+    beforeEach(() => {
+      processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 });
+    });
+
+    it("should reset all state", () => {
+      processor.addSample(10);
+      processor.processBatch();
+      processor.addSample(20);
+
+      expect(processor.getSmoothedValue()).toBe(10);
+      expect(processor.getCurrentSampleCount()).toBe(1);
+
+      processor.reset();
+
+      expect(processor.getSmoothedValue()).toBe(0);
+      expect(processor.getCurrentSampleCount()).toBe(0);
+      expect(processor.getCurrentSamples()).toHaveLength(0);
+    });
+
+    it("should reinitialize correctly after reset", () => {
+      // Process some data
+      processor.addSample(10);
+      processor.processBatch();
+      processor.addSample(20);
+      processor.processBatch();
+
+      processor.reset();
+
+      // Should initialize with first value again
+      processor.addSample(30);
+      const result = processor.processBatch();
+
+      expect(result!.smoothedValue).toBe(30);
+      expect(processor.getSmoothedValue()).toBe(30);
+    });
+  });
+
+  describe("Configuration", () => {
+    it("should return configuration", () => {
+      processor = new QueueMetricsProcessor({ ewmaAlpha: 0.5, batchWindowMs: 2000 });
+
+      const config = processor.getConfig();
+      expect(config.ewmaAlpha).toBe(0.5);
+      expect(config.batchWindowMs).toBe(2000);
+    });
+  });
+
+  describe("Real-world simulation", () => {
+    it("should handle high-frequency samples from multiple consumers", () => {
+      processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 });
+
+      // Simulate 40 consumers reporting queue lengths within 1 second
+      const baseTime = 1000;
+      for (let i = 0; i < 40; i++) {
+        const queueLength = 100 - i * 2; // Queue decreasing as consumers work
+        processor.addSample(queueLength, baseTime + i * 25); // Spread over 1 second
+      }
+
+      const result = processor.processBatch(baseTime + 1000);
+
+      expect(result).not.toBeNull();
+      expect(result!.sampleCount).toBe(40);
+      // Median should be around middle values (queue lengths 60-80)
+      expect(result!.median).toBeGreaterThanOrEqual(60);
+      expect(result!.median).toBeLessThanOrEqual(80);
+    });
+
+    it("should demonstrate EWMA smoothing over time", () => {
+      processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 });
+
+      const results = [];
+
+      // Simulate queue spike and recovery
+      const scenarios = [
+        { samples: [5, 5, 5], expected: 5 }, // Baseline
+        { samples: [50, 50, 50], expected: 18.5 }, // Spike: 0.3 * 50 + 0.7 * 5 = 18.5
+        { samples: [5, 5, 5], expected: 10.05 }, // Recovery: 0.3 * 5 + 0.7 * 18.5 = 14.45
+      ];
+
+      for (const scenario of scenarios) {
+        for (const sample of scenario.samples) {
+          processor.addSample(sample);
+        }
+        const result = processor.processBatch();
+        results.push(result!.smoothedValue);
+      }
+
+      // Should show gradual change due to EWMA smoothing
+      expect(results[0]).toBe(5); // Initial
+      expect(results[1]).toBeCloseTo(18.5); // Spike response
+      expect(results[2]).toBeCloseTo(14.45); // Gradual recovery
+    });
+  });
+});

packages/core/src/v3/runEngineWorker/supervisor/queueMetricsProcessor.ts

@@ -0,0 +1,209 @@
+import { SimpleStructuredLogger } from "../../utils/structuredLogger.js";
+
+export interface QueueMetricsProcessorOptions {
+  /**
+   * EWMA smoothing factor (0-1)
+   * Lower values = more smoothing, less reactive
+   * Higher values = more responsive to recent changes
+   */
+  ewmaAlpha: number;
+
+  /**
+   * Batch window duration in milliseconds
+   * Samples within this window are collected and processed together
+   */
+  batchWindowMs: number;
+}
+
+export interface BatchProcessingResult {
+  /** Median of samples in the batch */
+  median: number;
+
+  /** EWMA-smoothed value after processing this batch */
+  smoothedValue: number;
+
+  /** Number of samples processed in this batch */
+  sampleCount: number;
+
+  /** Raw samples that were processed */
+  samples: readonly number[];
+}
+
+/**
+ * Processes queue length samples using exponential weighted moving average (EWMA)
+ * for smoothing and median filtering for outlier resistance.
+ *
+ * Collects samples within a batch window, calculates median to filter outliers,
+ * then applies EWMA smoothing for stable trend tracking.
+ */
+export class QueueMetricsProcessor {
+  private readonly ewmaAlpha: number;
+  private readonly batchWindowMs: number;
+  private readonly logger = new SimpleStructuredLogger("queue-metrics-processor");
+
+  private samples: number[] = [];
+  private smoothedValue: number = 0;
+  private lastBatchTime: number = 0;
+  private isInitialized: boolean = false;
+
+  constructor(options: QueueMetricsProcessorOptions) {
+    if (options.ewmaAlpha < 0 || options.ewmaAlpha > 1) {
+      throw new Error("ewmaAlpha must be between 0 and 1");
+    }
+    if (options.batchWindowMs <= 0) {
+      throw new Error("batchWindowMs must be positive");
+    }
+
+    this.ewmaAlpha = options.ewmaAlpha;
+    this.batchWindowMs = options.batchWindowMs;
+  }
+
+  /**
+   * Adds a sample to the current batch
+   */
+  addSample(value: number, timestamp: number = Date.now()): void {
+    if (value < 0) {
+      throw new Error("Queue length cannot be negative");
+    }
+
+    this.samples.push(value);
+
+    // Update last batch time on first sample
+    if (this.samples.length === 1) {
+      this.lastBatchTime = timestamp;
+    }
+  }
+
+  /**
+   * Checks if enough time has passed to process the current batch
+   */
+  shouldProcessBatch(currentTime: number = Date.now()): boolean {
+    if (this.samples.length === 0) {
+      return false;
+    }
+
+    return currentTime - this.lastBatchTime >= this.batchWindowMs;
+  }
+
+  private calculateMedian(samples: number[]): number | null {
+    const sortedSamples = [...samples].sort((a, b) => a - b);
+    const mid = Math.floor(sortedSamples.length / 2);
+
+    if (sortedSamples.length % 2 === 1) {
+      // Odd length: use middle value
+      const median = sortedSamples[mid];
+
+      if (median === undefined) {
+        this.logger.error("Invalid median calculated from odd samples", {
+          sortedSamples,
+          mid,
+          median,
+        });
+        return null;
+      }
+
+      return median;
+    } else {
+      // Even length: average two middle values
+      const lowMid = sortedSamples[mid - 1];
+      const highMid = sortedSamples[mid];
+
+      if (lowMid === undefined || highMid === undefined) {
+        this.logger.error("Invalid median calculated from even samples", {
+          sortedSamples,
+          mid,
+          lowMid,
+          highMid,
+        });
+        return null;
+      }
+
+      const median = (lowMid + highMid) / 2;
+      return median;
+    }
+  }
+
+  /**
+   * Processes the current batch of samples and returns the result.
+   * Clears the samples array and updates the smoothed value.
+   *
+   * Returns null if there are no samples to process.
+   */
+  processBatch(currentTime: number = Date.now()): BatchProcessingResult | null {
+    if (this.samples.length === 0) {
+      // No samples to process
+      return null;
+    }
+
+    // Calculate median of samples to filter outliers
+    const median = this.calculateMedian(this.samples);
+    if (median === null) {
+      // We already logged a more specific error message
+      return null;
+    }
+
+    // Update EWMA smoothed value
+    if (!this.isInitialized) {
+      // First value - initialize with median
+      this.smoothedValue = median;
+      this.isInitialized = true;
+    } else {
+      // Apply EWMA: s_t = α * x_t + (1 - α) * s_(t-1)
+      this.smoothedValue = this.ewmaAlpha * median + (1 - this.ewmaAlpha) * this.smoothedValue;
+    }
+
+    const result: BatchProcessingResult = {
+      median,
+      smoothedValue: this.smoothedValue,
+      sampleCount: this.samples.length,
+      samples: Object.freeze([...this.samples]),
+    };
+
+    // Clear samples for next batch
+    this.samples = [];
+    this.lastBatchTime = currentTime;
+
+    return result;
+  }
+
+  /**
+   * Gets the current smoothed value without processing a batch
+   */
+  getSmoothedValue(): number {
+    return this.smoothedValue;
+  }
+
+  /**
+   * Gets the number of samples in the current batch
+   */
+  getCurrentSampleCount(): number {
+    return this.samples.length;
+  }
+
+  /**
+   * Gets the current samples (for testing/debugging)
+   */
+  getCurrentSamples(): readonly number[] {
+    return Object.freeze([...this.samples]);
+  }
+
+  /**
+   * Resets the processor state
+   */
+  reset(): void {
+    this.samples = [];
+    this.smoothedValue = 0;
+    this.lastBatchTime = 0;
+    this.isInitialized = false;
+  }
+
+  /**
+   * Gets processor configuration
+   */
+  getConfig(): Readonly<QueueMetricsProcessorOptions> {
+    return {
+      ewmaAlpha: this.ewmaAlpha,
+      batchWindowMs: this.batchWindowMs,
+    };
+  }
+}

packages/core/src/v3/runEngineWorker/supervisor/scalingStrategies.test.ts

@@ -0,0 +1,293 @@
+import { describe, it, expect } from "vitest";
+import {
+  NoneScalingStrategy,
+  SmoothScalingStrategy,
+  AggressiveScalingStrategy,
+  ScalingStrategyOptions,
+} from "./scalingStrategies.js";
+import { QueueMetricsProcessor } from "./queueMetricsProcessor.js";
+
+describe("Scaling Strategies", () => {
+  const baseOptions: ScalingStrategyOptions = {
+    minConsumerCount: 1,
+    maxConsumerCount: 20,
+    targetRatio: 1.0,
+  };
+
+  function createMetricsProcessor(smoothedValue: number): QueueMetricsProcessor {
+    const processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 });
+    // Initialize processor with the target smoothed value
+    processor.addSample(smoothedValue);
+    processor.processBatch();
+    return processor;
+  }
+
+  describe("NoneScalingStrategy", () => {
+    const strategy = new NoneScalingStrategy(baseOptions);
+
+    it("should always return current count (static mode)", () => {
+      expect(strategy.calculateTargetCount(5)).toBe(5);
+      expect(strategy.calculateTargetCount(1)).toBe(1);
+      expect(strategy.calculateTargetCount(10)).toBe(10);
+      // Clamping still applies
+      expect(strategy.calculateTargetCount(25)).toBe(20); // Clamped to max
+      expect(strategy.calculateTargetCount(0)).toBe(1); // Clamped to min
+    });
+
+    it("should have correct name", () => {
+      expect(strategy.name).toBe("none");
+    });
+
+    it("should handle zero current count", () => {
+      // Should clamp to minConsumerCount
+      const result = strategy.calculateTargetCount(0);
+      expect(result).toBe(1);
+    });
+  });
+
+  describe("SmoothScalingStrategy", () => {
+    it("should calculate target based on smoothed queue length", () => {
+      const metricsProcessor = createMetricsProcessor(10); // smoothed value = 10
+      const strategy = new SmoothScalingStrategy({ ...baseOptions, metricsProcessor });
+
+      // With targetRatio=1.0, target consumers = ceil(10/1.0) = 10
+      // With dampingFactor=0.7 and currentCount=5:
+      // dampedTarget = 5 + (10 - 5) * 0.7 = 5 + 3.5 = 8.5 → 9
+      const result = strategy.calculateTargetCount(5);
+      expect(result).toBe(9);
+    });
+
+    it("should apply damping factor correctly", () => {
+      const metricsProcessor = createMetricsProcessor(20); // smoothed value = 20
+      const strategy = new SmoothScalingStrategy({
+        ...baseOptions,
+        metricsProcessor,
+        dampingFactor: 0.5,
+      }); // 50% damping
+
+      // With targetRatio=1.0, target consumers = ceil(20/1.0) = 20
+      // With dampingFactor=0.5 and currentCount=5:
+      // dampedTarget = 5 + (20 - 5) * 0.5 = 5 + 7.5 = 12.5 → 13
+      const result = strategy.calculateTargetCount(5);
+      expect(result).toBe(13);
+    });
+
+    it("should handle zero current count", () => {
+      const metricsProcessor = createMetricsProcessor(5);
+      const strategy = new SmoothScalingStrategy({ ...baseOptions, metricsProcessor });
+
+      // With smoothedQueueLength=5, targetRatio=1.0:
+      // targetConsumers = ceil(5/1.0) = 5
+      // dampedTarget = 0 + (5 - 0) * 0.7 = 3.5 → 4
+      const result = strategy.calculateTargetCount(0);
+      expect(result).toBe(4);
+    });
+
+    it("should validate damping factor", () => {
+      const metricsProcessor = createMetricsProcessor(10);
+      expect(
+        () =>
+          new SmoothScalingStrategy({
+            ...baseOptions,
+            metricsProcessor,
+            dampingFactor: -0.1,
+          })
+      ).toThrow("dampingFactor must be between 0 and 1");
+
+      expect(
+        () =>
+          new SmoothScalingStrategy({
+            ...baseOptions,
+            metricsProcessor,
+            dampingFactor: 1.1,
+          })
+      ).toThrow("dampingFactor must be between 0 and 1");
+
+      expect(
+        () =>
+          new SmoothScalingStrategy({
+            ...baseOptions,
+            metricsProcessor,
+            dampingFactor: 0,
+          })
+      ).not.toThrow();
+
+      expect(
+        () =>
+          new SmoothScalingStrategy({
+            ...baseOptions,
+            metricsProcessor,
+            dampingFactor: 1,
+          })
+      ).not.toThrow();
+    });
+
+    it("should handle zero current count", () => {
+      const metricsProcessor = createMetricsProcessor(10);
+      const strategy = new SmoothScalingStrategy({ ...baseOptions, metricsProcessor });
+
+      // With smoothedQueueLength=10, targetRatio=1.0:
+      // targetConsumers = ceil(10/1.0) = 10
+      // dampedTarget = 0 + (10 - 0) * 0.7 = 7
+      const result = strategy.calculateTargetCount(0);
+      expect(result).toBe(7);
+    });
+  });
+
+  describe("AggressiveScalingStrategy", () => {
+    it("should scale down when under-utilized", () => {
+      // queuePerConsumer = 2/5 = 0.4, scaleDownThreshold = 1.0 * 0.5 = 0.5
+      // Under-utilized since 0.4 < 0.5
+      const metricsProcessor = createMetricsProcessor(2);
+      const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor });
+
+      const result = strategy.calculateTargetCount(5);
+      expect(result).toBeLessThan(5);
+      expect(result).toBeGreaterThanOrEqual(baseOptions.minConsumerCount);
+    });
+
+    it("should maintain count when in optimal zone", () => {
+      // queuePerConsumer = 5/5 = 1.0
+      // Optimal zone: 0.5 < 1.0 < 2.0
+      const metricsProcessor = createMetricsProcessor(5);
+      const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor });
+
+      const result = strategy.calculateTargetCount(5);
+      expect(result).toBe(5);
+    });
+
+    it("should scale up when over-utilized", () => {
+      // queuePerConsumer = 15/5 = 3.0, scaleUpThreshold = 1.0 * 2.0 = 2.0
+      // Over-utilized since 3.0 > 2.0
+      const metricsProcessor = createMetricsProcessor(15);
+      const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor });
+
+      const result = strategy.calculateTargetCount(5);
+      expect(result).toBeGreaterThan(5);
+      expect(result).toBeLessThanOrEqual(baseOptions.maxConsumerCount);
+    });
+
+    it("should scale aggressively for critical load", () => {
+      // queuePerConsumer = 25/5 = 5.0 (critical: 5x target ratio)
+      const metricsProcessor = createMetricsProcessor(25);
+      const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor });
+
+      const result = strategy.calculateTargetCount(5);
+      // Should apply 50% scale factor: ceil(5 * 1.5) = 8
+      // But capped by 50% max increment: 5 + ceil(5 * 0.5) = 5 + 3 = 8
+      expect(result).toBe(8);
+    });
+
+    it("should respect max consumer count", () => {
+      const metricsProcessor = createMetricsProcessor(50); // Very high load
+      const strategy = new AggressiveScalingStrategy({
+        ...baseOptions,
+        maxConsumerCount: 6,
+        metricsProcessor,
+      });
+
+      const result = strategy.calculateTargetCount(5);
+      expect(result).toBeLessThanOrEqual(6);
+    });
+
+    it("should respect min consumer count", () => {
+      const metricsProcessor = createMetricsProcessor(0.1); // Very low load
+      const strategy = new AggressiveScalingStrategy({
+        ...baseOptions,
+        minConsumerCount: 3,
+        metricsProcessor,
+      });
+
+      const result = strategy.calculateTargetCount(5);
+      expect(result).toBeGreaterThanOrEqual(3);
+    });
+
+    it("should return thresholds", () => {
+      const metricsProcessor = createMetricsProcessor(10);
+      const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor });
+      const thresholds = strategy.getThresholds(1.0);
+      expect(thresholds).toEqual({
+        scaleDownThreshold: 0.5,
+        scaleUpThreshold: 2.0,
+        criticalThreshold: 5.0,
+        highThreshold: 3.0,
+      });
+    });
+
+    it("should handle zero current count without division by zero", () => {
+      const metricsProcessor = createMetricsProcessor(10);
+      const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor });
+
+      // Should use (currentCount || 1) to prevent division by zero
+      // queuePerConsumer = 10 / 1 = 10 (not 10 / 0)
+      // This is over-utilized (10 > 2.0), should scale up
+      const result = strategy.calculateTargetCount(0);
+      expect(result).toBeGreaterThan(0);
+      expect(result).toBeLessThanOrEqual(baseOptions.maxConsumerCount);
+    });
+
+    it("should handle zero queue with zero consumers", () => {
+      const metricsProcessor = createMetricsProcessor(0);
+      const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor });
+
+      // queuePerConsumer = 0 / 1 = 0
+      // This is under-utilized (0 < 0.5), should scale down
+      // But already at 0, so should return minConsumerCount
+      const result = strategy.calculateTargetCount(0);
+      expect(result).toBe(baseOptions.minConsumerCount);
+    });
+  });
+
+  describe("Integration scenarios", () => {
+    it("should handle gradual load increase with smooth strategy", () => {
+      const metricsProcessor = createMetricsProcessor(2);
+      const strategy = new SmoothScalingStrategy({ ...baseOptions, metricsProcessor });
+      let currentCount = 2;
+
+      // Gradual increase: 2 → 6 → 10 → 15
+      const loads = [2, 6, 10, 15];
+      const results = [];
+
+      for (const load of loads) {
+        // Update the processor with the new load
+        metricsProcessor.addSample(load);
+        metricsProcessor.processBatch();
+        const target = strategy.calculateTargetCount(currentCount);
+        results.push(target);
+        currentCount = target;
+      }
+
+      // Should show gradual increase due to damping
+      expect(results[0]).toBeLessThan(results[1]!);
+      expect(results[1]).toBeLessThan(results[2]!);
+      expect(results[2]).toBeLessThan(results[3]!);
+
+      // But not immediate jumps due to damping
+      expect(results[1]! - results[0]!).toBeLessThan(loads[1]! - loads[0]!);
+    });
+
+    it("should handle load spike with aggressive strategy", () => {
+      let currentCount = 3;
+
+      // Sudden spike from normal to critical
+      const normalLoad = 3; // queuePerConsumer = 1.0 (optimal)
+      const spikeLoad = 15; // queuePerConsumer = 5.0 (critical)
+
+      const normalProcessor = createMetricsProcessor(normalLoad);
+      const normalStrategy = new AggressiveScalingStrategy({
+        ...baseOptions,
+        metricsProcessor: normalProcessor,
+      });
+      const normalTarget = normalStrategy.calculateTargetCount(currentCount);
+      expect(normalTarget).toBe(3); // Should maintain
+
+      const spikeProcessor = createMetricsProcessor(spikeLoad);
+      const spikeStrategy = new AggressiveScalingStrategy({
+        ...baseOptions,
+        metricsProcessor: spikeProcessor,
+      });
+      const spikeTarget = spikeStrategy.calculateTargetCount(currentCount);
+      expect(spikeTarget).toBeGreaterThan(3); // Should scale up aggressively
+    });
+  });
+});

packages/core/src/v3/runEngineWorker/supervisor/scalingStrategies.ts

@@ -0,0 +1,186 @@
+import { QueueMetricsProcessor } from "./queueMetricsProcessor.js";
+
+export type ScalingStrategyKind = "none" | "smooth" | "aggressive";
+
+export interface ScalingStrategyOptions {
+  metricsProcessor?: QueueMetricsProcessor;
+  dampingFactor?: number;
+  minConsumerCount: number;
+  maxConsumerCount: number;
+  targetRatio: number;
+}
+
+export abstract class ScalingStrategy {
+  abstract readonly name: string;
+
+  private readonly minConsumerCount: number;
+  private readonly maxConsumerCount: number;
+
+  protected readonly targetRatio: number;
+
+  constructor(options?: ScalingStrategyOptions) {
+    this.minConsumerCount = options?.minConsumerCount ?? 1;
+    this.maxConsumerCount = options?.maxConsumerCount ?? 10;
+    this.targetRatio = options?.targetRatio ?? 1;
+  }
+
+  /**
+   * Calculates the target consumer count with clamping to min/max bounds
+   * Uses template method pattern to ensure consistent clamping across all strategies
+   */
+  calculateTargetCount(currentCount: number): number {
+    const targetCount = this.calculateTargetCountInternal(currentCount);
+
+    // Apply consistent clamping to all strategies
+    return Math.min(Math.max(targetCount, this.minConsumerCount), this.maxConsumerCount);
+  }
+
+  /**
+   * Internal method for subclasses to implement their specific scaling logic
+   * Should return the unclamped target count
+   */
+  protected abstract calculateTargetCountInternal(currentCount: number): number;
+
+  /**
+   * Creates a scaling strategy by name
+   */
+  static create(strategy: ScalingStrategyKind, options?: ScalingStrategyOptions): ScalingStrategy {
+    switch (strategy) {
+      case "none":
+        return new NoneScalingStrategy(options);
+
+      case "smooth":
+        return new SmoothScalingStrategy(options);
+
+      case "aggressive":
+        return new AggressiveScalingStrategy(options);
+
+      default:
+        throw new Error(`Unknown scaling strategy: ${strategy}`);
+    }
+  }
+}
+
+/**
+ * Static scaling strategy - maintains a fixed number of consumers
+ */
+export class NoneScalingStrategy extends ScalingStrategy {
+  readonly name = "none";
+
+  constructor(options?: ScalingStrategyOptions) {
+    super(options);
+  }
+
+  protected calculateTargetCountInternal(currentCount: number): number {
+    return currentCount;
+  }
+}
+
+/**
+ * Smooth scaling strategy with EWMA smoothing and damping
+ * Uses exponentially weighted moving average for queue length smoothing
+ * and applies damping to prevent rapid oscillations.
+ */
+export class SmoothScalingStrategy extends ScalingStrategy {
+  readonly name = "smooth";
+  private readonly dampingFactor: number;
+  private readonly metricsProcessor: QueueMetricsProcessor;
+
+  constructor(options?: ScalingStrategyOptions) {
+    super(options);
+    const dampingFactor = options?.dampingFactor ?? 0.7;
+    if (dampingFactor < 0 || dampingFactor > 1) {
+      throw new Error("dampingFactor must be between 0 and 1");
+    }
+    if (!options?.metricsProcessor) {
+      throw new Error("metricsProcessor is required for smooth scaling strategy");
+    }
+    this.dampingFactor = dampingFactor;
+    this.metricsProcessor = options.metricsProcessor;
+  }
+
+  protected calculateTargetCountInternal(currentCount: number): number {
+    const smoothedQueueLength = this.metricsProcessor.getSmoothedValue();
+
+    // Calculate target consumers based on the configured ratio
+    const targetConsumers = Math.ceil(smoothedQueueLength / this.targetRatio);
+
+    // Apply damping factor to smooth out changes
+    // This prevents oscillation by only moving toward the target gradually
+    const dampedTarget = currentCount + (targetConsumers - currentCount) * this.dampingFactor;
+
+    // Return rounded value without clamping (handled by base class)
+    return Math.round(dampedTarget);
+  }
+}
+
+/**
+ * Aggressive scaling strategy with threshold-based zones
+ * Uses threshold-based zones for different scaling behaviors.
+ * Scales up quickly when load increases but scales down cautiously.
+ */
+export class AggressiveScalingStrategy extends ScalingStrategy {
+  readonly name = "aggressive";
+  private readonly metricsProcessor: QueueMetricsProcessor;
+
+  constructor(options?: ScalingStrategyOptions) {
+    super(options);
+    if (!options?.metricsProcessor) {
+      throw new Error("metricsProcessor is required for aggressive scaling strategy");
+    }
+    this.metricsProcessor = options.metricsProcessor;
+  }
+
+  protected calculateTargetCountInternal(currentCount: number): number {
+    const smoothedQueueLength = this.metricsProcessor.getSmoothedValue();
+
+    // Calculate queue items per consumer,
+    const queuePerConsumer = smoothedQueueLength / (currentCount || 1);
+
+    // Define zones based on targetRatio
+    // Optimal zone: 0.5x to 2x the target ratio
+    const scaleDownThreshold = this.targetRatio * 0.5;
+    const scaleUpThreshold = this.targetRatio * 2.0;
+
+    if (queuePerConsumer < scaleDownThreshold) {
+      // Zone 1: Under-utilized (< 0.5x target ratio)
+      // Scale down gradually to avoid removing too many consumers
+      const reductionFactor = Math.max(0.9, 1 - (scaleDownThreshold - queuePerConsumer) * 0.1);
+      // Return without min clamping (handled by base class)
+      return Math.floor(currentCount * reductionFactor);
+    } else if (queuePerConsumer > scaleUpThreshold) {
+      // Zone 3: Over-utilized (> 2x target ratio)
+      // Scale up aggressively based on queue pressure
+      let scaleFactor: number;
+      if (queuePerConsumer >= this.targetRatio * 5) {
+        // Critical: Queue is 5x target ratio or higher
+        scaleFactor = 1.5; // 50% increase
+      } else if (queuePerConsumer >= this.targetRatio * 3) {
+        // High: Queue is 3x target ratio
+        scaleFactor = 1.3; // 30% increase
+      } else {
+        // Moderate: Queue is 2x target ratio
+        scaleFactor = 1.1; // 10% increase
+      }
+
+      const targetCount = Math.ceil(currentCount * scaleFactor);
+      // Cap increase at 50% to prevent overshooting
+      const maxIncrement = Math.ceil(currentCount * 0.5);
+      // Return without max clamping (handled by base class)
+      return Math.min(currentCount + maxIncrement, targetCount);
+    } else {
+      // Zone 2: Optimal (0.5x - 2x target ratio)
+      // Maintain current consumer count
+      return currentCount;
+    }
+  }
+
+  getThresholds(targetRatio: number) {
+    return {
+      scaleDownThreshold: targetRatio * 0.5,
+      scaleUpThreshold: targetRatio * 2.0,
+      criticalThreshold: targetRatio * 5.0,
+      highThreshold: targetRatio * 3.0,
+    };
+  }
+}

packages/core/src/v3/runEngineWorker/supervisor/session.ts

@@ -1,7 +1,7 @@
 import { SupervisorHttpClient } from "./http.js";
 import { PreDequeueFn, PreSkipFn, SupervisorClientCommonOptions } from "./types.js";
 import { WorkerApiDequeueResponseBody, WorkerApiHeartbeatRequestBody } from "./schemas.js";
-import { RunQueueConsumer } from "./queueConsumer.js";
+import { RunQueueConsumerPool, ScalingOptions } from "./consumerPool.js";
 import { WorkerEvents } from "./events.js";
 import EventEmitter from "events";
 import { VERSION } from "../../../version.js";
@@ -10,6 +10,7 @@ import { WorkerClientToServerEvents, WorkerServerToClientEvents } from "../types
 import { getDefaultWorkerHeaders } from "./util.js";
 import { IntervalService } from "../../utils/interval.js";
 import { SimpleStructuredLogger } from "../../utils/structuredLogger.js";
+import type { Registry } from "prom-client";
 
 type SupervisorSessionOptions = SupervisorClientCommonOptions & {
   queueConsumerEnabled?: boolean;
@@ -20,8 +21,9 @@ type SupervisorSessionOptions = SupervisorClientCommonOptions & {
   preDequeue?: PreDequeueFn;
   preSkip?: PreSkipFn;
   maxRunCount?: number;
-  maxConsumerCount?: number;
   sendRunDebugLogs?: boolean;
+  scaling: ScalingOptions;
+  metricsRegistry?: Registry;
 };
 
 export class SupervisorSession extends EventEmitter<WorkerEvents> {
@@ -33,7 +35,7 @@ export class SupervisorSession extends EventEmitter<WorkerEvents> {
   private runNotificationsSocket?: Socket<WorkerServerToClientEvents, WorkerClientToServerEvents>;
 
   private readonly queueConsumerEnabled: boolean;
-  private readonly queueConsumers: RunQueueConsumer[];
+  private readonly consumerPool: RunQueueConsumerPool;
 
   private readonly heartbeat: IntervalService;
 
@@ -44,8 +46,9 @@ export class SupervisorSession extends EventEmitter<WorkerEvents> {
     this.queueConsumerEnabled = opts.queueConsumerEnabled ?? true;
 
     this.httpClient = new SupervisorHttpClient(opts);
-    this.queueConsumers = Array.from({ length: opts.maxConsumerCount ?? 1 }, () => {
-      return new RunQueueConsumer({
+
+    this.consumerPool = new RunQueueConsumerPool({
+      consumer: {
         client: this.httpClient,
         preDequeue: opts.preDequeue,
         preSkip: opts.preSkip,
@@ -53,7 +56,9 @@ export class SupervisorSession extends EventEmitter<WorkerEvents> {
         intervalMs: opts.dequeueIntervalMs,
         idleIntervalMs: opts.dequeueIdleIntervalMs,
         maxRunCount: opts.maxRunCount,
-      });
+      },
+      scaling: opts.scaling,
+      metricsRegistry: opts.metricsRegistry,
     });
 
     this.heartbeat = new IntervalService({
@@ -179,8 +184,13 @@ export class SupervisorSession extends EventEmitter<WorkerEvents> {
     });
 
     if (this.queueConsumerEnabled) {
-      this.logger.log("Queue consumer enabled");
-      await Promise.allSettled(this.queueConsumers.map(async (q) => q.start()));
+      this.logger.log("Queue consumer enabled", {
+        scalingStrategy: this.consumerPool["scalingStrategy"],
+        minConsumers: this.consumerPool["minConsumerCount"],
+        maxConsumers: this.consumerPool["maxConsumerCount"],
+      });
+
+      await this.consumerPool.start();
       this.heartbeat.start();
     } else {
       this.logger.warn("Queue consumer disabled");
@@ -195,7 +205,7 @@ export class SupervisorSession extends EventEmitter<WorkerEvents> {
   }
 
   async stop() {
-    await Promise.allSettled(this.queueConsumers.map(async (q) => q.stop()));
+    await this.consumerPool.stop();
     this.heartbeat.stop();
     this.runNotificationsSocket?.disconnect();
   }

pnpm-lock.yaml

@@ -587,6 +587,9 @@ importers:
       non.geist:
         specifier: ^1.0.2
         version: 1.0.2
+      octokit:
+        specifier: ^3.2.1
+        version: 3.2.2
       ohash:
         specifier: ^1.1.3
         version: 1.1.3
@@ -8546,6 +8549,262 @@ packages:
       which: 3.0.1
     dev: true
 
+  /@octokit/app@14.1.0:
+    resolution: {integrity: sha512-g3uEsGOQCBl1+W1rgfwoRFUIR6PtvB2T1E4RpygeUU5LrLvlOqcxrt5lfykIeRpUPpupreGJUYl70fqMDXdTpw==}
+    engines: {node: '>= 18'}
+    dependencies:
+      '@octokit/auth-app': 6.1.4
+      '@octokit/auth-unauthenticated': 5.0.1
+      '@octokit/core': 5.2.2
+      '@octokit/oauth-app': 6.1.0
+      '@octokit/plugin-paginate-rest': 9.2.2(@octokit/core@5.2.2)
+      '@octokit/types': 12.6.0
+      '@octokit/webhooks': 12.3.2
+    dev: false
+
+  /@octokit/auth-app@6.1.4:
+    resolution: {integrity: sha512-QkXkSOHZK4dA5oUqY5Dk3S+5pN2s1igPjEASNQV8/vgJgW034fQWR16u7VsNOK/EljA00eyjYF5mWNxWKWhHRQ==}
+    engines: {node: '>= 18'}
+    dependencies:
+      '@octokit/auth-oauth-app': 7.1.0
+      '@octokit/auth-oauth-user': 4.1.0
+      '@octokit/request': 8.4.1
+      '@octokit/request-error': 5.1.1
+      '@octokit/types': 13.10.0
+      deprecation: 2.3.1
+      lru-cache: /@wolfy1339/lru-cache@11.0.2-patch.1
+      universal-github-app-jwt: 1.2.0
+      universal-user-agent: 6.0.1
+    dev: false
+
+  /@octokit/auth-oauth-app@7.1.0:
+    resolution: {integrity: sha512-w+SyJN/b0l/HEb4EOPRudo7uUOSW51jcK1jwLa+4r7PA8FPFpoxEnHBHMITqCsc/3Vo2qqFjgQfz/xUUvsSQnA==}
+    engines: {node: '>= 18'}
+    dependencies:
+      '@octokit/auth-oauth-device': 6.1.0
+      '@octokit/auth-oauth-user': 4.1.0
+      '@octokit/request': 8.4.1
+      '@octokit/types': 13.10.0
+      '@types/btoa-lite': 1.0.2
+      btoa-lite: 1.0.0
+      universal-user-agent: 6.0.1
+    dev: false
+
+  /@octokit/auth-oauth-device@6.1.0:
+    resolution: {integrity: sha512-FNQ7cb8kASufd6Ej4gnJ3f1QB5vJitkoV1O0/g6e6lUsQ7+VsSNRHRmFScN2tV4IgKA12frrr/cegUs0t+0/Lw==}
+    engines: {node: '>= 18'}
+    dependencies:
+      '@octokit/oauth-methods': 4.1.0
+      '@octokit/request': 8.4.1
+      '@octokit/types': 13.10.0
+      universal-user-agent: 6.0.1
+    dev: false
+
+  /@octokit/auth-oauth-user@4.1.0:
+    resolution: {integrity: sha512-FrEp8mtFuS/BrJyjpur+4GARteUCrPeR/tZJzD8YourzoVhRics7u7we/aDcKv+yywRNwNi/P4fRi631rG/OyQ==}
+    engines: {node: '>= 18'}
+    dependencies:
+      '@octokit/auth-oauth-device': 6.1.0
+      '@octokit/oauth-methods': 4.1.0
+      '@octokit/request': 8.4.1
+      '@octokit/types': 13.10.0
+      btoa-lite: 1.0.0
+      universal-user-agent: 6.0.1
+    dev: false
+
+  /@octokit/auth-token@4.0.0:
+    resolution: {integrity: sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==}
+    engines: {node: '>= 18'}
+    dev: false
+
+  /@octokit/auth-unauthenticated@5.0.1:
+    resolution: {integrity: sha512-oxeWzmBFxWd+XolxKTc4zr+h3mt+yofn4r7OfoIkR/Cj/o70eEGmPsFbueyJE2iBAGpjgTnEOKM3pnuEGVmiqg==}
+    engines: {node: '>= 18'}
+    dependencies:
+      '@octokit/request-error': 5.1.1
+      '@octokit/types': 12.6.0
+    dev: false
+
+  /@octokit/core@5.2.2:
+    resolution: {integrity: sha512-/g2d4sW9nUDJOMz3mabVQvOGhVa4e/BN/Um7yca9Bb2XTzPPnfTWHWQg+IsEYO7M3Vx+EXvaM/I2pJWIMun1bg==}
+    engines: {node: '>= 18'}
+    dependencies:
+      '@octokit/auth-token': 4.0.0
+      '@octokit/graphql': 7.1.1
+      '@octokit/request': 8.4.1
+      '@octokit/request-error': 5.1.1
+      '@octokit/types': 13.10.0
+      before-after-hook: 2.2.3
+      universal-user-agent: 6.0.1
+    dev: false
+
+  /@octokit/endpoint@9.0.6:
+    resolution: {integrity: sha512-H1fNTMA57HbkFESSt3Y9+FBICv+0jFceJFPWDePYlR/iMGrwM5ph+Dd4XRQs+8X+PUFURLQgX9ChPfhJ/1uNQw==}
+    engines: {node: '>= 18'}
+    dependencies:
+      '@octokit/types': 13.10.0
+      universal-user-agent: 6.0.1
+    dev: false
+
+  /@octokit/graphql@7.1.1:
+    resolution: {integrity: sha512-3mkDltSfcDUoa176nlGoA32RGjeWjl3K7F/BwHwRMJUW/IteSa4bnSV8p2ThNkcIcZU2umkZWxwETSSCJf2Q7g==}
+    engines: {node: '>= 18'}
+    dependencies:
+      '@octokit/request': 8.4.1
+      '@octokit/types': 13.10.0
+      universal-user-agent: 6.0.1
+    dev: false
+
+  /@octokit/oauth-app@6.1.0:
+    resolution: {integrity: sha512-nIn/8eUJ/BKUVzxUXd5vpzl1rwaVxMyYbQkNZjHrF7Vk/yu98/YDF/N2KeWO7uZ0g3b5EyiFXFkZI8rJ+DH1/g==}
+    engines: {node: '>= 18'}
+    dependencies:
+      '@octokit/auth-oauth-app': 7.1.0
+      '@octokit/auth-oauth-user': 4.1.0
+      '@octokit/auth-unauthenticated': 5.0.1
+      '@octokit/core': 5.2.2
+      '@octokit/oauth-authorization-url': 6.0.2
+      '@octokit/oauth-methods': 4.1.0
+      '@types/aws-lambda': 8.10.152
+      universal-user-agent: 6.0.1
+    dev: false
+
+  /@octokit/oauth-authorization-url@6.0.2:
+    resolution: {integrity: sha512-CdoJukjXXxqLNK4y/VOiVzQVjibqoj/xHgInekviUJV73y/BSIcwvJ/4aNHPBPKcPWFnd4/lO9uqRV65jXhcLA==}
+    engines: {node: '>= 18'}
+    dev: false
+
+  /@octokit/oauth-methods@4.1.0:
+    resolution: {integrity: sha512-4tuKnCRecJ6CG6gr0XcEXdZtkTDbfbnD5oaHBmLERTjTMZNi2CbfEHZxPU41xXLDG4DfKf+sonu00zvKI9NSbw==}
+    engines: {node: '>= 18'}
+    dependencies:
+      '@octokit/oauth-authorization-url': 6.0.2
+      '@octokit/request': 8.4.1
+      '@octokit/request-error': 5.1.1
+      '@octokit/types': 13.10.0
+      btoa-lite: 1.0.0
+    dev: false
+
+  /@octokit/openapi-types@20.0.0:
+    resolution: {integrity: sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==}
+    dev: false
+
+  /@octokit/openapi-types@24.2.0:
+    resolution: {integrity: sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==}
+    dev: false
+
+  /@octokit/plugin-paginate-graphql@4.0.1(@octokit/core@5.2.2):
+    resolution: {integrity: sha512-R8ZQNmrIKKpHWC6V2gum4x9LG2qF1RxRjo27gjQcG3j+vf2tLsEfE7I/wRWEPzYMaenr1M+qDAtNcwZve1ce1A==}
+    engines: {node: '>= 18'}
+    peerDependencies:
+      '@octokit/core': '>=5'
+    dependencies:
+      '@octokit/core': 5.2.2
+    dev: false
+
+  /@octokit/plugin-paginate-rest@11.4.4-cjs.2(@octokit/core@5.2.2):
+    resolution: {integrity: sha512-2dK6z8fhs8lla5PaOTgqfCGBxgAv/le+EhPs27KklPhm1bKObpu6lXzwfUEQ16ajXzqNrKMujsFyo9K2eaoISw==}
+    engines: {node: '>= 18'}
+    peerDependencies:
+      '@octokit/core': '5'
+    dependencies:
+      '@octokit/core': 5.2.2
+      '@octokit/types': 13.10.0
+    dev: false
+
+  /@octokit/plugin-paginate-rest@9.2.2(@octokit/core@5.2.2):
+    resolution: {integrity: sha512-u3KYkGF7GcZnSD/3UP0S7K5XUFT2FkOQdcfXZGZQPGv3lm4F2Xbf71lvjldr8c1H3nNbF+33cLEkWYbokGWqiQ==}
+    engines: {node: '>= 18'}
+    peerDependencies:
+      '@octokit/core': '5'
+    dependencies:
+      '@octokit/core': 5.2.2
+      '@octokit/types': 12.6.0
+    dev: false
+
+  /@octokit/plugin-rest-endpoint-methods@13.3.2-cjs.1(@octokit/core@5.2.2):
+    resolution: {integrity: sha512-VUjIjOOvF2oELQmiFpWA1aOPdawpyaCUqcEBc/UOUnj3Xp6DJGrJ1+bjUIIDzdHjnFNO6q57ODMfdEZnoBkCwQ==}
+    engines: {node: '>= 18'}
+    peerDependencies:
+      '@octokit/core': ^5
+    dependencies:
+      '@octokit/core': 5.2.2
+      '@octokit/types': 13.10.0
+    dev: false
+
+  /@octokit/plugin-retry@6.1.0(@octokit/core@5.2.2):
+    resolution: {integrity: sha512-WrO3bvq4E1Xh1r2mT9w6SDFg01gFmP81nIG77+p/MqW1JeXXgL++6umim3t6x0Zj5pZm3rXAN+0HEjmmdhIRig==}
+    engines: {node: '>= 18'}
+    peerDependencies:
+      '@octokit/core': '5'
+    dependencies:
+      '@octokit/core': 5.2.2
+      '@octokit/request-error': 5.1.1
+      '@octokit/types': 13.10.0
+      bottleneck: 2.19.5
+    dev: false
+
+  /@octokit/plugin-throttling@8.2.0(@octokit/core@5.2.2):
+    resolution: {integrity: sha512-nOpWtLayKFpgqmgD0y3GqXafMFuKcA4tRPZIfu7BArd2lEZeb1988nhWhwx4aZWmjDmUfdgVf7W+Tt4AmvRmMQ==}
+    engines: {node: '>= 18'}
+    peerDependencies:
+      '@octokit/core': ^5.0.0
+    dependencies:
+      '@octokit/core': 5.2.2
+      '@octokit/types': 12.6.0
+      bottleneck: 2.19.5
+    dev: false
+
+  /@octokit/request-error@5.1.1:
+    resolution: {integrity: sha512-v9iyEQJH6ZntoENr9/yXxjuezh4My67CBSu9r6Ve/05Iu5gNgnisNWOsoJHTP6k0Rr0+HQIpnH+kyammu90q/g==}
+    engines: {node: '>= 18'}
+    dependencies:
+      '@octokit/types': 13.10.0
+      deprecation: 2.3.1
+      once: 1.4.0
+    dev: false
+
+  /@octokit/request@8.4.1:
+    resolution: {integrity: sha512-qnB2+SY3hkCmBxZsR/MPCybNmbJe4KAlfWErXq+rBKkQJlbjdJeS85VI9r8UqeLYLvnAenU8Q1okM/0MBsAGXw==}
+    engines: {node: '>= 18'}
+    dependencies:
+      '@octokit/endpoint': 9.0.6
+      '@octokit/request-error': 5.1.1
+      '@octokit/types': 13.10.0
+      universal-user-agent: 6.0.1
+    dev: false
+
+  /@octokit/types@12.6.0:
+    resolution: {integrity: sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==}
+    dependencies:
+      '@octokit/openapi-types': 20.0.0
+    dev: false
+
+  /@octokit/types@13.10.0:
+    resolution: {integrity: sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==}
+    dependencies:
+      '@octokit/openapi-types': 24.2.0
+    dev: false
+
+  /@octokit/webhooks-methods@4.1.0:
+    resolution: {integrity: sha512-zoQyKw8h9STNPqtm28UGOYFE7O6D4Il8VJwhAtMHFt2C4L0VQT1qGKLeefUOqHNs1mNRYSadVv7x0z8U2yyeWQ==}
+    engines: {node: '>= 18'}
+    dev: false
+
+  /@octokit/webhooks-types@7.6.1:
+    resolution: {integrity: sha512-S8u2cJzklBC0FgTwWVLaM8tMrDuDMVE4xiTK4EYXM9GntyvrdbSoxqDQa+Fh57CCNApyIpyeqPhhFEmHPfrXgw==}
+    dev: false
+
+  /@octokit/webhooks@12.3.2:
+    resolution: {integrity: sha512-exj1MzVXoP7xnAcAB3jZ97pTvVPkQF9y6GA/dvYC47HV7vLv+24XRS6b/v/XnyikpEuvMhugEXdGtAlU086WkQ==}
+    engines: {node: '>= 18'}
+    dependencies:
+      '@octokit/request-error': 5.1.1
+      '@octokit/webhooks-methods': 4.1.0
+      '@octokit/webhooks-types': 7.6.1
+      aggregate-error: 3.1.0
+    dev: false
+
   /@one-ini/wasm@0.1.1:
     resolution: {integrity: sha512-XuySG1E38YScSJoMlqovLru4KTUNSjgVTIjyh7qMX6aNN5HY5Ct5LhRJdxO79JtTzKfzV/bnWpz+zquYrISsvw==}
     dev: false
@@ -17538,6 +17797,10 @@ packages:
     resolution: {integrity: sha512-XTIieEY+gvJ39ChLcB4If5zHtPxt3Syj5rgZR+e1ctpmK8NjPf0zFqsz4JpLJT0xla9GFDKjy8Cpu331nrmE1Q==}
     dev: true
 
+  /@types/aws-lambda@8.10.152:
+    resolution: {integrity: sha512-soT/c2gYBnT5ygwiHPmd9a1bftj462NWVk2tKCc1PYHSIacB2UwbTS2zYG4jzag1mRDuzg/OjtxQjQ2NKRB6Rw==}
+    dev: false
+
   /@types/bcryptjs@2.4.2:
     resolution: {integrity: sha512-LiMQ6EOPob/4yUL66SZzu6Yh77cbzJFYll+ZfaPiPPFswtIlA/Fs1MzdKYA7JApHU49zQTbJGX3PDmCpIdDBRQ==}
     dev: true
@@ -17549,6 +17812,10 @@ packages:
       '@types/node': 20.14.14
     dev: true
 
+  /@types/btoa-lite@1.0.2:
+    resolution: {integrity: sha512-ZYbcE2x7yrvNFJiU7xJGrpF/ihpkM7zKgw8bha3LNJSesvTtUNxbpzaT7WXBIryf6jovisrxTBvymxMeLLj1Mg==}
+    dev: false
+
   /@types/bun@1.1.6:
     resolution: {integrity: sha512-uJgKjTdX0GkWEHZzQzFsJkWp5+43ZS7HC8sZPFnOwnSo1AsNl2q9o2bFeS23disNDqbggEgyFkKCHl/w8iZsMA==}
     dependencies:
@@ -17808,6 +18075,13 @@ packages:
     resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==}
     dev: true
 
+  /@types/jsonwebtoken@9.0.10:
+    resolution: {integrity: sha512-asx5hIG9Qmf/1oStypjanR7iKTv0gXQ1Ov/jfrX6kS/EO0OFni8orbmGCn0672NHR3kXHwpAwR+B368ZGN/2rA==}
+    dependencies:
+      '@types/ms': 0.7.31
+      '@types/node': 20.14.14
+    dev: false
+
   /@types/keyv@3.1.4:
     resolution: {integrity: sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg==}
     dependencies:
@@ -17926,7 +18200,6 @@ packages:
     resolution: {integrity: sha512-acBjXdRJ3A6Pb3tqnw9HZmyR3Fiol3aGxRCK1x3d+6CDAMjl7I649wpSd+yNURCjbOUGu9tqtLKnTGxmK6CyGw==}
     dependencies:
       undici-types: 6.20.0
-    dev: false
 
   /@types/nodemailer@6.4.17:
     resolution: {integrity: sha512-I9CCaIp6DTldEg7vyUTZi8+9Vo0hi1/T8gv3C89yk1rSAAzoKQ8H8ki/jBYJSFoH/BisgLP8tkZMlQ91CIquww==}
@@ -19012,6 +19285,11 @@ packages:
       xstate: 5.18.1
     dev: false
 
+  /@wolfy1339/lru-cache@11.0.2-patch.1:
+    resolution: {integrity: sha512-BgYZfL2ADCXKOw2wJtkM3slhHotawWkgIRRxq4wEybnZQPjvAp71SPX35xepMykTw8gXlzWcWPTY31hlbnRsDA==}
+    engines: {node: 18 >=18.20 || 20 || >=22}
+    dev: false
+
   /@xobotyi/scrollbar-width@1.9.5:
     resolution: {integrity: sha512-N8tkAACJx2ww8vFMneJmaAgmjAG1tnVBZJRLRcx061tmsLRZHSEZSLuGWnwPtunsSLvSqXQ2wfp7Mgqg1I+2dQ==}
     dev: false
@@ -19181,7 +19459,6 @@ packages:
     dependencies:
       clean-stack: 2.2.0
       indent-string: 4.0.0
-    dev: true
 
   /aggregate-error@4.0.1:
     resolution: {integrity: sha512-0poP0T7el6Vq3rstR8Mn4V/IQrpBLO6POkUSrN7RhyY+GF/InCFShQzsQ39T25gkHhLgSLByyAz+Kjb+c2L98w==}
@@ -19878,6 +20155,10 @@ packages:
     dependencies:
       tweetnacl: 0.14.5
 
+  /before-after-hook@2.2.3:
+    resolution: {integrity: sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==}
+    dev: false
+
   /better-path-resolve@1.0.0:
     resolution: {integrity: sha512-pbnl5XzGBdrFU/wT4jqmJVPn2B6UHPBOhzMQkY/SPUPB6QtUXtmBHBIwCbXJol93mOpGMnQyP/+BB19q04xj7g==}
     engines: {node: '>=4'}
@@ -19958,6 +20239,10 @@ packages:
       - supports-color
     dev: false
 
+  /bottleneck@2.19.5:
+    resolution: {integrity: sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==}
+    dev: false
+
   /bowser@2.11.0:
     resolution: {integrity: sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==}
     dev: false
@@ -20023,6 +20308,10 @@ packages:
       update-browserslist-db: 1.1.3(browserslist@4.25.0)
     dev: true
 
+  /btoa-lite@1.0.0:
+    resolution: {integrity: sha512-gvW7InbIyF8AicrqWoptdW08pUxuhq8BEgowNajy9RhiE86fmGAGl+bLKo6oB8QP0CkqHLowfN0oJdKC/J6LbA==}
+    dev: false
+
   /buffer-crc32@0.2.13:
     resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==}
     dev: false
@@ -20032,6 +20321,10 @@ packages:
     engines: {node: '>=8.0.0'}
     dev: true
 
+  /buffer-equal-constant-time@1.0.1:
+    resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==}
+    dev: false
+
   /buffer-from@1.1.2:
     resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==}
 
@@ -20453,7 +20746,6 @@ packages:
   /clean-stack@2.2.0:
     resolution: {integrity: sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==}
     engines: {node: '>=6'}
-    dev: true
 
   /clean-stack@4.2.0:
     resolution: {integrity: sha512-LYv6XPxoyODi36Dp976riBtSY27VmFo+MKqEU9QCCWyTrdEPDog+RWA7xQWHi6Vbp61j5c4cdzzX1NidnwtUWg==}
@@ -21440,6 +21732,10 @@ packages:
     resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==}
     engines: {node: '>= 0.8'}
 
+  /deprecation@2.3.1:
+    resolution: {integrity: sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==}
+    dev: false
+
   /dequal@2.0.3:
     resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==}
     engines: {node: '>=6'}
@@ -21702,6 +21998,12 @@ packages:
       safer-buffer: 2.1.2
     dev: false
 
+  /ecdsa-sig-formatter@1.0.11:
+    resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==}
+    dependencies:
+      safe-buffer: 5.2.1
+    dev: false
+
   /editorconfig@1.0.4:
     resolution: {integrity: sha512-L9Qe08KWTlqYMVvMcTIvMAdl1cDUubzRNYL+WfA4bLDMHe4nemKkpmYzkznE1FwLKu0EEmy6obgQKzMJrg4x9Q==}
     engines: {node: '>=14'}
@@ -21834,7 +22136,7 @@ packages:
     engines: {node: '>=10.13.0'}
     dependencies:
       graceful-fs: 4.2.11
-      tapable: 2.2.1
+      tapable: 2.2.2
 
   /enquirer@2.3.6:
     resolution: {integrity: sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==}
@@ -25124,7 +25426,7 @@ packages:
     resolution: {integrity: sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==}
     engines: {node: '>= 10.13.0'}
     dependencies:
-      '@types/node': 20.14.14
+      '@types/node': 22.13.9
       merge-stream: 2.0.0
       supports-color: 8.1.1
 
@@ -25338,6 +25640,22 @@ packages:
     engines: {node: '>=0.10.0'}
     dev: false
 
+  /jsonwebtoken@9.0.2:
+    resolution: {integrity: sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==}
+    engines: {node: '>=12', npm: '>=6'}
+    dependencies:
+      jws: 3.2.2
+      lodash.includes: 4.3.0
+      lodash.isboolean: 3.0.3
+      lodash.isinteger: 4.0.4
+      lodash.isnumber: 3.0.3
+      lodash.isplainobject: 4.0.6
+      lodash.isstring: 4.0.1
+      lodash.once: 4.1.1
+      ms: 2.1.3
+      semver: 7.7.2
+    dev: false
+
   /jsprim@1.4.2:
     resolution: {integrity: sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==}
     engines: {node: '>=0.6.0'}
@@ -25361,6 +25679,21 @@ packages:
     engines: {node: '>=12.20'}
     dev: true
 
+  /jwa@1.4.2:
+    resolution: {integrity: sha512-eeH5JO+21J78qMvTIDdBXidBd6nG2kZjg5Ohz/1fpa28Z4CcsWUzJ1ZZyFq/3z3N17aZy+ZuBoHljASbL1WfOw==}
+    dependencies:
+      buffer-equal-constant-time: 1.0.1
+      ecdsa-sig-formatter: 1.0.11
+      safe-buffer: 5.2.1
+    dev: false
+
+  /jws@3.2.2:
+    resolution: {integrity: sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==}
+    dependencies:
+      jwa: 1.4.2
+      safe-buffer: 5.2.1
+    dev: false
+
   /keyv@3.1.0:
     resolution: {integrity: sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==}
     dependencies:
@@ -25742,21 +26075,40 @@ packages:
     resolution: {integrity: sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==}
     dev: false
 
+  /lodash.includes@4.3.0:
+    resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==}
+    dev: false
+
   /lodash.isarguments@3.1.0:
     resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==}
     dev: false
 
+  /lodash.isboolean@3.0.3:
+    resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==}
+    dev: false
+
   /lodash.isfunction@3.0.9:
     resolution: {integrity: sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==}
     dev: false
 
+  /lodash.isinteger@4.0.4:
+    resolution: {integrity: sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==}
+    dev: false
+
   /lodash.isnil@4.0.0:
     resolution: {integrity: sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==}
     dev: false
 
+  /lodash.isnumber@3.0.3:
+    resolution: {integrity: sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==}
+    dev: false
+
   /lodash.isplainobject@4.0.6:
     resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==}
-    dev: true
+
+  /lodash.isstring@4.0.1:
+    resolution: {integrity: sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==}
+    dev: false
 
   /lodash.isundefined@3.0.1:
     resolution: {integrity: sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==}
@@ -25769,6 +26121,10 @@ packages:
     resolution: {integrity: sha512-XeqSp49hNGmlkj2EJlfrQFIzQ6lXdNro9sddtQzcJY8QaoC2GO0DT7xaIokHeyM+mIT0mPMlPvkYzg2xCuHdZg==}
     dev: false
 
+  /lodash.once@4.1.1:
+    resolution: {integrity: sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==}
+    dev: false
+
   /lodash.sortby@4.7.0:
     resolution: {integrity: sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA==}
     dev: true
@@ -27631,6 +27987,23 @@ packages:
   /obuf@1.1.2:
     resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==}
 
+  /octokit@3.2.2:
+    resolution: {integrity: sha512-7Abo3nADdja8l/aglU6Y3lpnHSfv0tw7gFPiqzry/yCU+2gTAX7R1roJ8hJrxIK+S1j+7iqRJXtmuHJ/UDsBhQ==}
+    engines: {node: '>= 18'}
+    dependencies:
+      '@octokit/app': 14.1.0
+      '@octokit/core': 5.2.2
+      '@octokit/oauth-app': 6.1.0
+      '@octokit/plugin-paginate-graphql': 4.0.1(@octokit/core@5.2.2)
+      '@octokit/plugin-paginate-rest': 11.4.4-cjs.2(@octokit/core@5.2.2)
+      '@octokit/plugin-rest-endpoint-methods': 13.3.2-cjs.1(@octokit/core@5.2.2)
+      '@octokit/plugin-retry': 6.1.0(@octokit/core@5.2.2)
+      '@octokit/plugin-throttling': 8.2.0(@octokit/core@5.2.2)
+      '@octokit/request-error': 5.1.1
+      '@octokit/types': 13.10.0
+      '@octokit/webhooks': 12.3.2
+    dev: false
+
   /ohash@1.1.3:
     resolution: {integrity: sha512-zuHHiGTYTA1sYJ/wZN+t5HKZaH23i4yI1HMwbuXm24Nid7Dv0KcuRlKoNKS9UNfAVSBlnGLcuQrnOKWOZoEGaw==}
     dev: false
@@ -32014,7 +32387,6 @@ packages:
   /tapable@2.2.2:
     resolution: {integrity: sha512-Re10+NauLTMCudc7T5WLFLAwDhQ0JWdrMK+9B2M8zR5hRExKmsRDCBA7/aV/pNJFltmBFO5BAMlQFi/vq3nKOg==}
     engines: {node: '>=6'}
-    dev: true
 
   /tar-fs@2.1.3:
     resolution: {integrity: sha512-090nwYJDmlhwFwEW3QQl+vaNnxsO2yVsd45eTKRBzSzu+hlb1w2K9inVq5b0ngXuLVqQ4ApvsUHHnu/zQNkWAg==}
@@ -33000,7 +33372,6 @@ packages:
 
   /undici-types@6.20.0:
     resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==}
-    dev: false
 
   /undici@5.28.4:
     resolution: {integrity: sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==}
@@ -33152,6 +33523,17 @@ packages:
       cookie: 0.6.0
     dev: false
 
+  /universal-github-app-jwt@1.2.0:
+    resolution: {integrity: sha512-dncpMpnsKBk0eetwfN8D8OUHGfiDhhJ+mtsbMl+7PfW7mYjiH8LIcqRmYMtzYLgSh47HjfdBtrBwIQ/gizKR3g==}
+    dependencies:
+      '@types/jsonwebtoken': 9.0.10
+      jsonwebtoken: 9.0.2
+    dev: false
+
+  /universal-user-agent@6.0.1:
+    resolution: {integrity: sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==}
+    dev: false
+
   /universalify@0.1.2:
     resolution: {integrity: sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==}
     engines: {node: '>= 4.0.0'}