From 0d1eac9406e5bc2c4a5971f3f429440846e6aa92 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Tue, 2 Sep 2025 13:52:42 +0100 Subject: [PATCH 01/13] feat(supervisor): dynamic queue consumer pool (#2461) * feat(supervisor): dynamic queue consumer pool * add changeset * fix: correctly handle zero median and even samples * feat(supervisor): consumer pool metrics * fix tests * more tests and fixes * decrease default scaling cooldowns * don't treat initial pool size as scale up * handle scale down when queue length drops to zero * remove changeset, supervisor changes only * add damping factor env var --- apps/supervisor/src/env.ts | 12 +- apps/supervisor/src/index.ts | 13 +- packages/core/src/v3/runEngineWorker/index.ts | 1 + .../supervisor/consumerPool.test.ts | 721 ++++++++++++++++++ .../supervisor/consumerPool.ts | 411 ++++++++++ .../supervisor/consumerPoolMetrics.ts | 160 ++++ .../supervisor/queueConsumer.ts | 11 +- .../supervisor/queueMetricsProcessor.test.ts | 371 +++++++++ .../supervisor/queueMetricsProcessor.ts | 209 +++++ .../supervisor/scalingStrategies.test.ts | 293 +++++++ .../supervisor/scalingStrategies.ts | 186 +++++ .../v3/runEngineWorker/supervisor/session.ts | 28 +- 12 files changed, 2401 insertions(+), 15 deletions(-) create mode 100644 packages/core/src/v3/runEngineWorker/supervisor/consumerPool.test.ts create mode 100644 packages/core/src/v3/runEngineWorker/supervisor/consumerPool.ts create mode 100644 packages/core/src/v3/runEngineWorker/supervisor/consumerPoolMetrics.ts create mode 100644 packages/core/src/v3/runEngineWorker/supervisor/queueMetricsProcessor.test.ts create mode 100644 packages/core/src/v3/runEngineWorker/supervisor/queueMetricsProcessor.ts create mode 100644 packages/core/src/v3/runEngineWorker/supervisor/scalingStrategies.test.ts create mode 100644 packages/core/src/v3/runEngineWorker/supervisor/scalingStrategies.ts diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index a225af5ea..074c18ad7 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -35,8 +35,16 @@ const Env = z.object({ TRIGGER_DEQUEUE_ENABLED: BoolEnv.default(true), TRIGGER_DEQUEUE_INTERVAL_MS: z.coerce.number().int().default(250), TRIGGER_DEQUEUE_IDLE_INTERVAL_MS: z.coerce.number().int().default(1000), - TRIGGER_DEQUEUE_MAX_RUN_COUNT: z.coerce.number().int().default(10), - TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT: z.coerce.number().int().default(1), + TRIGGER_DEQUEUE_MAX_RUN_COUNT: z.coerce.number().int().default(1), + TRIGGER_DEQUEUE_MIN_CONSUMER_COUNT: z.coerce.number().int().default(1), + TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT: z.coerce.number().int().default(10), + TRIGGER_DEQUEUE_SCALING_STRATEGY: z.enum(["none", "smooth", "aggressive"]).default("none"), + TRIGGER_DEQUEUE_SCALING_UP_COOLDOWN_MS: z.coerce.number().int().default(5000), // 5 seconds + TRIGGER_DEQUEUE_SCALING_DOWN_COOLDOWN_MS: z.coerce.number().int().default(30000), // 30 seconds + TRIGGER_DEQUEUE_SCALING_TARGET_RATIO: z.coerce.number().default(1.0), // Target ratio of queue items to consumers (1.0 = 1 item per consumer) + TRIGGER_DEQUEUE_SCALING_EWMA_ALPHA: z.coerce.number().min(0).max(1).default(0.3), // Smooths queue length measurements (0=historical, 1=current) + TRIGGER_DEQUEUE_SCALING_BATCH_WINDOW_MS: z.coerce.number().int().positive().default(1000), // Batch window for metrics processing (ms) + TRIGGER_DEQUEUE_SCALING_DAMPING_FACTOR: z.coerce.number().min(0).max(1).default(0.7), // Smooths consumer count changes after EWMA (0=no scaling, 1=immediate) // Optional services TRIGGER_WARM_START_URL: z.string().optional(), diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts index 1ed00edad..b958897ad 100644 --- a/apps/supervisor/src/index.ts +++ b/apps/supervisor/src/index.ts @@ -128,7 +128,18 @@ class ManagedSupervisor { dequeueIdleIntervalMs: env.TRIGGER_DEQUEUE_IDLE_INTERVAL_MS, queueConsumerEnabled: env.TRIGGER_DEQUEUE_ENABLED, maxRunCount: env.TRIGGER_DEQUEUE_MAX_RUN_COUNT, - maxConsumerCount: env.TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT, + metricsRegistry: register, + scaling: { + strategy: env.TRIGGER_DEQUEUE_SCALING_STRATEGY, + minConsumerCount: env.TRIGGER_DEQUEUE_MIN_CONSUMER_COUNT, + maxConsumerCount: env.TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT, + scaleUpCooldownMs: env.TRIGGER_DEQUEUE_SCALING_UP_COOLDOWN_MS, + scaleDownCooldownMs: env.TRIGGER_DEQUEUE_SCALING_DOWN_COOLDOWN_MS, + targetRatio: env.TRIGGER_DEQUEUE_SCALING_TARGET_RATIO, + ewmaAlpha: env.TRIGGER_DEQUEUE_SCALING_EWMA_ALPHA, + batchWindowMs: env.TRIGGER_DEQUEUE_SCALING_BATCH_WINDOW_MS, + dampingFactor: env.TRIGGER_DEQUEUE_SCALING_DAMPING_FACTOR, + }, runNotificationsEnabled: env.TRIGGER_WORKLOAD_API_ENABLED, heartbeatIntervalSeconds: env.TRIGGER_WORKER_HEARTBEAT_INTERVAL_SECONDS, sendRunDebugLogs: env.SEND_RUN_DEBUG_LOGS, diff --git a/packages/core/src/v3/runEngineWorker/index.ts b/packages/core/src/v3/runEngineWorker/index.ts index 98566ae22..50617df11 100644 --- a/packages/core/src/v3/runEngineWorker/index.ts +++ b/packages/core/src/v3/runEngineWorker/index.ts @@ -2,6 +2,7 @@ export * from "./consts.js"; export * from "./supervisor/http.js"; export * from "./supervisor/schemas.js"; export * from "./supervisor/session.js"; +export * from "./supervisor/consumerPool.js"; export * from "./workload/http.js"; export * from "./workload/schemas.js"; export * from "./types.js"; diff --git a/packages/core/src/v3/runEngineWorker/supervisor/consumerPool.test.ts b/packages/core/src/v3/runEngineWorker/supervisor/consumerPool.test.ts new file mode 100644 index 000000000..5f515b95b --- /dev/null +++ b/packages/core/src/v3/runEngineWorker/supervisor/consumerPool.test.ts @@ -0,0 +1,721 @@ +import { describe, it, expect, beforeEach, afterEach, vi, Mock } from "vitest"; +import { + RunQueueConsumerPool, + type ConsumerPoolOptions, + type QueueConsumerFactory, +} from "./consumerPool.js"; +import { SupervisorHttpClient } from "./http.js"; +import type { WorkerApiDequeueResponseBody } from "./schemas.js"; +import type { QueueConsumer } from "./queueConsumer.js"; + +// Mock only the logger +vi.mock("../../utils/structuredLogger.js"); + +// Test implementation of QueueConsumer +class TestQueueConsumer implements QueueConsumer { + public started = false; + public stopped = false; + public onDequeue?: (messages: WorkerApiDequeueResponseBody) => Promise; + + constructor(opts: any) { + this.onDequeue = opts.onDequeue; + } + + start(): void { + this.started = true; + this.stopped = false; + } + + stop(): void { + this.stopped = true; + this.started = false; + } +} + +describe("RunQueueConsumerPool", () => { + let mockClient: SupervisorHttpClient; + let mockOnDequeue: Mock; + let pool: RunQueueConsumerPool; + let defaultOptions: Omit; + let testConsumers: TestQueueConsumer[]; + let testConsumerFactory: QueueConsumerFactory; + + beforeEach(() => { + vi.clearAllMocks(); + vi.useFakeTimers(); + + mockClient = {} as SupervisorHttpClient; + mockOnDequeue = vi.fn(); + testConsumers = []; + + testConsumerFactory = (opts) => { + const consumer = new TestQueueConsumer(opts); + testConsumers.push(consumer); + return consumer; + }; + + defaultOptions = { + consumer: { + client: mockClient, + intervalMs: 0, + idleIntervalMs: 1000, + onDequeue: mockOnDequeue, + }, + consumerFactory: testConsumerFactory, + }; + }); + + afterEach(() => { + vi.useRealTimers(); + if (pool) { + pool.stop(); + } + }); + + function advanceTimeAndProcessMetrics(ms: number) { + vi.advanceTimersByTime(ms); + + // Trigger batch processing if ready (without adding a sample) + if (pool["metricsProcessor"].shouldProcessBatch()) { + pool["processMetricsBatch"](); + } + } + + describe("Static mode (strategy='none')", () => { + it("should start with maxConsumerCount in static mode", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { strategy: "none", maxConsumerCount: 5 }, + }); + + await pool.start(); + + expect(pool.size).toBe(5); + expect(testConsumers.length).toBe(5); + }); + + it("should not scale in static mode even with queue length updates", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { strategy: "none", maxConsumerCount: 3 }, + }); + + await pool.start(); + const initialCount = pool.size; + + pool.updateQueueLength(100); + vi.advanceTimersByTime(2000); + + expect(pool.size).toBe(initialCount); + expect(pool.size).toBe(3); + }); + }); + + describe("Smooth scaling strategy", () => { + it("should scale smoothly with damping", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "smooth", + minConsumerCount: 1, + maxConsumerCount: 10, + scaleUpCooldownMs: 0, + disableJitter: true, + }, + }); + + await pool.start(); + expect(pool.size).toBe(1); + + pool.updateQueueLength(5); + advanceTimeAndProcessMetrics(1100); + expect(pool.size).toBe(4); // Damped scaling + + pool.updateQueueLength(5); + advanceTimeAndProcessMetrics(1100); + expect(pool.size).toBe(5); // Gradually approaches target + }); + + it("should respect max consumer count", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "smooth", + minConsumerCount: 1, + maxConsumerCount: 5, + scaleUpCooldownMs: 0, + disableJitter: true, + }, + }); + + await pool.start(); + expect(pool.size).toBe(1); + + pool.updateQueueLength(100); + advanceTimeAndProcessMetrics(1100); + expect(pool.size).toBe(5); + + pool.updateQueueLength(100); + advanceTimeAndProcessMetrics(1100); + expect(pool.size).toBe(5); + }); + }); + + describe("Aggressive scaling strategy", () => { + it("should scale up quickly based on queue pressure", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "aggressive", + minConsumerCount: 2, + maxConsumerCount: 10, + scaleUpCooldownMs: 0, + disableJitter: true, + }, + }); + + await pool.start(); + expect(pool.size).toBe(2); + + pool.updateQueueLength(10); + advanceTimeAndProcessMetrics(1100); + expect(pool.size).toBe(3); + + pool.updateQueueLength(20); + advanceTimeAndProcessMetrics(1100); + expect(pool.size).toBe(4); + }); + + it("should scale down cautiously when queue is small", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "aggressive", + minConsumerCount: 1, + maxConsumerCount: 10, + scaleUpCooldownMs: 0, + scaleDownCooldownMs: 0, + disableJitter: true, + }, + }); + + await pool.start(); + expect(pool.size).toBe(1); + + pool.updateQueueLength(10); + advanceTimeAndProcessMetrics(1100); + expect(pool.size).toBe(2); + + pool.updateQueueLength(0.5); + advanceTimeAndProcessMetrics(1100); + expect(pool.size).toBe(3); // EWMA smoothing delays scale down + + pool.updateQueueLength(0.5); + advanceTimeAndProcessMetrics(1100); + expect(pool.size).toBeGreaterThanOrEqual(3); // Stays in optimal zone + }); + + it("should maintain current level in optimal zone", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "aggressive", + minConsumerCount: 3, + maxConsumerCount: 10, + scaleUpCooldownMs: 0, + disableJitter: true, + }, + }); + + await pool.start(); + expect(pool.size).toBe(3); + + pool.updateQueueLength(3); + advanceTimeAndProcessMetrics(1100); + expect(pool.size).toBe(3); + + pool.updateQueueLength(4); + advanceTimeAndProcessMetrics(1100); + expect(pool.size).toBe(3); + }); + }); + + describe("Smooth scaling with EWMA", () => { + it("should use exponential smoothing for stable scaling", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "smooth", + minConsumerCount: 1, + maxConsumerCount: 10, + disableJitter: true, + }, + }); + + await pool.start(); + + const queueLengths = [10, 2, 8, 3, 9, 1, 7]; + for (const length of queueLengths) { + pool.updateQueueLength(length); + vi.advanceTimersByTime(200); + } + vi.advanceTimersByTime(900); + + const metrics = pool.getMetrics(); + expect(metrics.smoothedQueueLength).toBeGreaterThan(0); + expect(metrics.smoothedQueueLength).toBeLessThan(10); + }); + + it("should apply damping factor to avoid rapid changes", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "smooth", + minConsumerCount: 1, + maxConsumerCount: 10, + disableJitter: true, + }, + }); + + await pool.start(); + + pool.updateQueueLength(2); + advanceTimeAndProcessMetrics(1100); + const metrics1 = pool.getMetrics(); + expect(metrics1.smoothedQueueLength).toBe(2); + + pool.updateQueueLength(20); + advanceTimeAndProcessMetrics(1100); + const metrics2 = pool.getMetrics(); + + expect(metrics2.smoothedQueueLength).toBeGreaterThan(2); + expect(metrics2.smoothedQueueLength).toBeLessThan(20); + }); + }); + + describe("High throughput parallel dequeuing", () => { + it("should handle rapid parallel queue updates", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "aggressive", + minConsumerCount: 1, + maxConsumerCount: 20, + disableJitter: true, + }, + }); + + await pool.start(); + + const updates: number[] = []; + for (let i = 0; i < 100; i++) { + updates.push(Math.floor(Math.random() * 50) + 10); + } + + updates.forEach((length, index) => { + setTimeout(() => pool.updateQueueLength(length), index * 10); + }); + + advanceTimeAndProcessMetrics(1100); + + const metrics = pool.getMetrics(); + expect(metrics.queueLength).toBeDefined(); + }); + + it("should batch metrics updates to avoid excessive scaling", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "smooth", + minConsumerCount: 1, + maxConsumerCount: 10, + disableJitter: true, + }, + }); + + await pool.start(); + const evaluateScalingSpy = vi.spyOn(pool as any, "evaluateScaling"); + + pool.updateQueueLength(10); + for (let i = 1; i < 50; i++) { + pool.updateQueueLength(Math.floor(Math.random() * 20) + 5); + } + + expect(evaluateScalingSpy).not.toHaveBeenCalled(); + advanceTimeAndProcessMetrics(1000); + expect(evaluateScalingSpy).toHaveBeenCalledTimes(1); + }); + + it("should use median to filter outliers in high-frequency updates", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "aggressive", + minConsumerCount: 1, + maxConsumerCount: 10, + disableJitter: true, + }, + }); + + await pool.start(); + + const updates = [10, 11, 9, 12, 10, 100, 11, 10, 9, 11, 1]; + updates.forEach((length) => pool.updateQueueLength(length)); + advanceTimeAndProcessMetrics(1100); + + const metrics = pool.getMetrics(); + expect(metrics.queueLength).toBeGreaterThanOrEqual(9); + expect(metrics.queueLength).toBeLessThanOrEqual(12); + }); + }); + + describe("Scaling cooldowns and jitter", () => { + it("should respect scale-up cooldown", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "smooth", + scaleUpCooldownMs: 5000, + minConsumerCount: 1, + maxConsumerCount: 10, + disableJitter: true, + }, + }); + + await pool.start(); + pool["scaleToTarget"](5); + const scaleToTargetSpy = vi.spyOn(pool as any, "scaleToTarget"); + + pool.updateQueueLength(10); + advanceTimeAndProcessMetrics(1100); + expect(scaleToTargetSpy).not.toHaveBeenCalled(); + + vi.advanceTimersByTime(10000); + pool.updateQueueLength(20); + advanceTimeAndProcessMetrics(1100); + }); + + it("should respect scale-down cooldown (longer than scale-up)", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "aggressive", + minConsumerCount: 1, + maxConsumerCount: 10, + disableJitter: true, + }, + }); + + await pool.start(); + + for (let i = 0; i < 4; i++) { + pool["addConsumers"](1); + } + pool["scaleToTarget"](5); + pool["metrics"].lastScaleTime = new Date(Date.now() - 70000); + + pool.updateQueueLength(1); + advanceTimeAndProcessMetrics(1100); + + const metrics = pool.getMetrics(); + expect(metrics.queueLength).toBe(1); + }); + + it("should add random jitter to prevent thundering herd", async () => { + const pools: RunQueueConsumerPool[] = []; + const scaleTimes: number[] = []; + + for (let i = 0; i < 3; i++) { + const p = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "smooth", + minConsumerCount: 1, + maxConsumerCount: 10, + disableJitter: true, + }, + }); + + const originalScale = p["scaleToTarget"]; + p["scaleToTarget"] = vi.fn(async (target: number) => { + scaleTimes.push(Date.now()); + return originalScale.call(p, target); + }); + + pools.push(p); + await p.start(); + } + + pools.forEach((p) => p.updateQueueLength(20)); + advanceTimeAndProcessMetrics(1100); + vi.advanceTimersByTime(15000); + + await Promise.all(pools.map((p) => p.stop())); + }); + }); + + describe("Consumer lifecycle management", () => { + it("should properly start and stop consumers", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "none", + maxConsumerCount: 3, + disableJitter: true, + }, + }); + + await pool.start(); + + expect(pool.size).toBe(3); + expect(testConsumers.length).toBe(3); + testConsumers.forEach((consumer) => { + expect(consumer.started).toBe(true); + }); + + await pool.stop(); + + testConsumers.forEach((consumer) => { + expect(consumer.stopped).toBe(true); + }); + }); + + it("should forward dequeue messages with queue length updates", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "aggressive", + maxConsumerCount: 2, + disableJitter: true, + }, + }); + + await pool.start(); + + const messages: WorkerApiDequeueResponseBody = [{ workerQueueLength: 15 } as any]; + + if (testConsumers[0]?.onDequeue) { + await testConsumers[0].onDequeue(messages); + } + + expect(mockOnDequeue).toHaveBeenCalledWith(messages); + + advanceTimeAndProcessMetrics(1100); + const metrics = pool.getMetrics(); + expect(metrics.queueLength).toBe(15); + }); + }); + + describe("Memory leak prevention", () => { + it("should collect all samples within batch window without limit", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "aggressive", + minConsumerCount: 1, + maxConsumerCount: 10, + disableJitter: true, + }, + }); + + await pool.start(); + + for (let i = 0; i < 100; i++) { + pool.updateQueueLength(i); + } + + const metrics = pool.getMetrics(); + expect(metrics.queueLength).toBeUndefined(); + }); + + it("should clear consumer map on stop", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "none", + maxConsumerCount: 5, + disableJitter: true, + }, + }); + + await pool.start(); + expect(pool.size).toBe(5); + + await pool.stop(); + expect(pool.size).toBe(0); + }); + + it("should clear recentQueueLengths after processing batch", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { strategy: "smooth" }, + }); + + await pool.start(); + + for (let i = 0; i < 5; i++) { + pool.updateQueueLength(10 + i); + } + + advanceTimeAndProcessMetrics(1100); + const metrics = pool.getMetrics(); + expect(metrics.queueLength).toBeDefined(); + }); + + it("should not accumulate scaling operations in memory", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "aggressive", + scaleUpCooldownMs: 100, + scaleDownCooldownMs: 100, + minConsumerCount: 1, + maxConsumerCount: 10, + disableJitter: true, + }, + }); + + await pool.start(); + + for (let i = 0; i < 5; i++) { + pool["metrics"].lastScaleTime = new Date(0); + pool.updateQueueLength(i % 2 === 0 ? 50 : 1); + vi.advanceTimersByTime(1100); + } + + expect(pool.size).toBeGreaterThanOrEqual(1); + expect(pool.size).toBeLessThanOrEqual(10); + }); + }); + + describe("Edge cases", () => { + it("should handle empty recent queue lengths", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { strategy: "aggressive" }, + }); + + await pool.start(); + + const metrics = pool.getMetrics(); + expect(metrics.queueLength).toBeUndefined(); + }); + + it("should clamp consumer count to min/max bounds", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "smooth", + minConsumerCount: 2, + maxConsumerCount: 5, + disableJitter: true, + }, + }); + + await pool.start(); + expect(pool.size).toBe(2); + + pool.updateQueueLength(100); + advanceTimeAndProcessMetrics(1100); + expect(pool.size).toBeLessThanOrEqual(5); + }); + + it("should respect custom targetRatio with smooth strategy", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "smooth", + targetRatio: 5, + scaleUpCooldownMs: 0, + minConsumerCount: 1, + maxConsumerCount: 10, + disableJitter: true, + }, + }); + + await pool.start(); + expect(pool.size).toBe(1); + + pool.updateQueueLength(10); + advanceTimeAndProcessMetrics(1100); + + const firstSize = pool.size; + expect(firstSize).toBeGreaterThanOrEqual(1); + expect(firstSize).toBeLessThanOrEqual(2); + + pool.updateQueueLength(10); + advanceTimeAndProcessMetrics(1100); + expect(pool.size).toBeLessThanOrEqual(2); + }); + + it("should respect custom targetRatio with aggressive strategy", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "aggressive", + targetRatio: 5, + scaleUpCooldownMs: 0, + minConsumerCount: 1, + maxConsumerCount: 10, + disableJitter: true, + }, + }); + + await pool.start(); + expect(pool.size).toBe(1); + + pool.updateQueueLength(20); + advanceTimeAndProcessMetrics(1100); + + const sizeAfterFirstScale = pool.size; + expect(sizeAfterFirstScale).toBeGreaterThanOrEqual(1); + + pool.updateQueueLength(20); + advanceTimeAndProcessMetrics(1100); + expect(pool.size).toBeLessThanOrEqual(6); + }); + + it("should scale down when no items are dequeued (zero queue length)", async () => { + pool = new RunQueueConsumerPool({ + ...defaultOptions, + scaling: { + strategy: "smooth", + minConsumerCount: 1, + maxConsumerCount: 10, + scaleUpCooldownMs: 0, + scaleDownCooldownMs: 0, + disableJitter: true, + }, + }); + + await pool.start(); + expect(pool.size).toBe(1); + + // Scale up first + pool.updateQueueLength(20); + advanceTimeAndProcessMetrics(1100); + expect(pool.size).toBeGreaterThan(1); + const sizeAfterScaleUp = pool.size; + + // Now send multiple zero queue lengths to converge EWMA to 0 + // The EWMA needs time to converge due to exponential smoothing + for (let i = 0; i < 5; i++) { + pool.updateQueueLength(0); + advanceTimeAndProcessMetrics(1100); + } + + // After multiple iterations with zero queue, should scale down but not to minimum yet + expect(pool.size).toBeLessThan(sizeAfterScaleUp); + expect(pool.size).toBeGreaterThan(1); + + // Continue until we reach minimum + for (let i = 0; i < 5; i++) { + pool.updateQueueLength(0); + advanceTimeAndProcessMetrics(1100); + } + + // Should eventually reach minimum + expect(pool.size).toBe(1); + }); + }); +}); diff --git a/packages/core/src/v3/runEngineWorker/supervisor/consumerPool.ts b/packages/core/src/v3/runEngineWorker/supervisor/consumerPool.ts new file mode 100644 index 000000000..2dd3d1b89 --- /dev/null +++ b/packages/core/src/v3/runEngineWorker/supervisor/consumerPool.ts @@ -0,0 +1,411 @@ +import { SimpleStructuredLogger } from "../../utils/structuredLogger.js"; +import { QueueConsumer, RunQueueConsumer, RunQueueConsumerOptions } from "./queueConsumer.js"; +import { QueueMetricsProcessor } from "./queueMetricsProcessor.js"; +import { + ScalingStrategy, + ScalingStrategyKind, + ScalingStrategyOptions, +} from "./scalingStrategies.js"; +import { ConsumerPoolMetrics } from "./consumerPoolMetrics.js"; +import type { Registry } from "prom-client"; + +export type QueueConsumerFactory = (opts: RunQueueConsumerOptions) => QueueConsumer; + +export type ScalingOptions = { + strategy?: ScalingStrategyKind; + minConsumerCount?: number; + maxConsumerCount?: number; + scaleUpCooldownMs?: number; + scaleDownCooldownMs?: number; + targetRatio?: number; + ewmaAlpha?: number; + batchWindowMs?: number; + disableJitter?: boolean; + dampingFactor?: number; +}; + +export type ConsumerPoolOptions = { + consumer: RunQueueConsumerOptions; + scaling: ScalingOptions; + consumerFactory?: QueueConsumerFactory; + metricsRegistry?: Registry; +}; + +type ScalingMetrics = { + targetConsumerCount: number; + queueLength?: number; + smoothedQueueLength: number; + lastScaleTime: Date; + lastQueueLengthUpdate: Date; +}; + +export class RunQueueConsumerPool { + private readonly consumerOptions: RunQueueConsumerOptions; + + private readonly logger = new SimpleStructuredLogger("consumer-pool"); + private readonly promMetrics?: ConsumerPoolMetrics; + + private readonly minConsumerCount: number; + private readonly maxConsumerCount: number; + private readonly scalingStrategy: ScalingStrategy; + private readonly disableJitter: boolean; + + private consumers: Map = new Map(); + private readonly consumerFactory: QueueConsumerFactory; + private isEnabled: boolean = false; + private isScaling: boolean = false; + + private metrics: ScalingMetrics; + private readonly metricsProcessor: QueueMetricsProcessor; + + // Scaling parameters + private readonly ewmaAlpha: number; + private readonly scaleUpCooldownMs: number; + private readonly scaleDownCooldownMs: number; + private readonly batchWindowMs: number; + + constructor(opts: ConsumerPoolOptions) { + this.consumerOptions = opts.consumer; + + // Initialize Prometheus metrics if registry provided + if (opts.metricsRegistry) { + this.promMetrics = new ConsumerPoolMetrics({ + register: opts.metricsRegistry, + }); + } + + this.minConsumerCount = Math.max(1, opts.scaling.minConsumerCount ?? 1); + this.maxConsumerCount = Math.max(this.minConsumerCount, opts.scaling.maxConsumerCount ?? 10); + this.scaleUpCooldownMs = opts.scaling.scaleUpCooldownMs ?? 10000; // 10 seconds default + this.scaleDownCooldownMs = opts.scaling.scaleDownCooldownMs ?? 60000; // 60 seconds default + this.disableJitter = opts.scaling.disableJitter ?? false; + + // Configure EWMA parameters from options + this.ewmaAlpha = opts.scaling.ewmaAlpha ?? 0.3; + this.batchWindowMs = opts.scaling.batchWindowMs ?? 1000; + + // Validate EWMA parameters + if (this.ewmaAlpha < 0 || this.ewmaAlpha > 1) { + throw new Error(`ewmaAlpha must be between 0 and 1, got: ${this.ewmaAlpha}`); + } + if (this.batchWindowMs <= 0) { + throw new Error(`batchWindowMs must be positive, got: ${this.batchWindowMs}`); + } + + // Initialize metrics processor + this.metricsProcessor = new QueueMetricsProcessor({ + ewmaAlpha: this.ewmaAlpha, + batchWindowMs: this.batchWindowMs, + }); + + const targetRatio = opts.scaling.targetRatio ?? 1.0; + const dampingFactor = opts.scaling.dampingFactor; + + // Create scaling strategy with metrics processor injected + this.scalingStrategy = ScalingStrategy.create(opts.scaling.strategy ?? "none", { + metricsProcessor: this.metricsProcessor, + dampingFactor, + targetRatio, + minConsumerCount: this.minConsumerCount, + maxConsumerCount: this.maxConsumerCount, + }); + + // Use provided factory or default to RunQueueConsumer + this.consumerFactory = + opts.consumerFactory || ((consumerOpts) => new RunQueueConsumer(consumerOpts)); + + this.metrics = { + targetConsumerCount: this.minConsumerCount, + queueLength: undefined, + smoothedQueueLength: 0, + lastScaleTime: new Date(0), + lastQueueLengthUpdate: new Date(0), + }; + + this.logger.log("Initialized consumer pool", { + minConsumerCount: this.minConsumerCount, + maxConsumerCount: this.maxConsumerCount, + scalingStrategy: this.scalingStrategy.name, + mode: this.scalingStrategy.name === "none" ? "static" : "dynamic", + ewmaAlpha: this.ewmaAlpha, + batchWindowMs: this.batchWindowMs, + }); + } + + async start() { + if (this.isEnabled) { + return; + } + + this.isEnabled = true; + + // For 'none' strategy, start with max consumers (static mode) + // For dynamic strategies, start with minimum + const initialCount = + this.scalingStrategy.name === "none" ? this.maxConsumerCount : this.minConsumerCount; + + // Set initial metrics + this.metrics.targetConsumerCount = initialCount; + + this.addConsumers(initialCount); + + this.logger.log("Started dynamic consumer pool", { + initialConsumerCount: this.consumers.size, + }); + + // Initialize Prometheus metrics with initial state + this.promMetrics?.updateState({ + consumerCount: this.consumers.size, + queueLength: this.metrics.queueLength, + smoothedQueueLength: this.metrics.smoothedQueueLength, + targetConsumerCount: initialCount, + strategy: this.scalingStrategy.name, + }); + } + + async stop() { + if (!this.isEnabled) { + return; + } + + this.isEnabled = false; + + // Stop all consumers + Array.from(this.consumers.values()).forEach((consumer) => consumer.stop()); + + this.consumers.clear(); + + this.logger.log("Stopped dynamic consumer pool"); + } + + /** + * Updates the queue length metric and triggers scaling decisions + * Uses QueueMetricsProcessor for batching and EWMA smoothing + */ + updateQueueLength(queueLength: number) { + // Track queue length update in metrics + this.promMetrics?.recordQueueLengthUpdate(); + + // Skip metrics tracking for static mode + if (this.scalingStrategy.name === "none") { + return; + } + + // Add sample to metrics processor + this.metricsProcessor.addSample(queueLength); + + // Check if we should process the current batch + if (this.metricsProcessor.shouldProcessBatch()) { + this.processMetricsBatch(); + } + } + + private processMetricsBatch() { + // Process batch using the metrics processor + const result = this.metricsProcessor.processBatch(); + + if (!result) { + this.logger.debug("No queue length samples in batch window - skipping scaling evaluation"); + return; + } + + // Update metrics + this.metrics.queueLength = result.median; + this.metrics.smoothedQueueLength = result.smoothedValue; + this.metrics.lastQueueLengthUpdate = new Date(); + + this.logger.verbose("Queue metrics batch processed", { + samples: result.sampleCount, + median: result.median, + smoothed: result.smoothedValue, + currentConsumerCount: this.consumers.size, + }); + + // Make scaling decision + this.evaluateScaling(); + } + + private evaluateScaling() { + if (!this.isEnabled) { + return; + } + + // No scaling in static mode + if (this.scalingStrategy.name === "none") { + return; + } + + // Skip if already scaling + if (this.isScaling) { + this.logger.debug("Scaling blocked - operation already in progress", { + currentCount: this.consumers.size, + targetCount: this.metrics.targetConsumerCount, + actualCount: this.consumers.size, + }); + return; + } + + const targetCount = this.calculateTargetConsumerCount(); + + if (targetCount === this.consumers.size) { + return; + } + + const timeSinceLastScale = Date.now() - this.metrics.lastScaleTime.getTime(); + + // Add random jitter to avoid thundering herd when multiple replicas exist + // Works without needing to know replica index or count + const jitterMs = this.disableJitter ? 0 : Math.random() * 3000; // 0-3 seconds random jitter + + // Check cooldown periods with jitter + if (targetCount > this.consumers.size) { + // Scale up + const effectiveCooldown = this.scaleUpCooldownMs + jitterMs; + if (timeSinceLastScale < effectiveCooldown) { + this.logger.debug("Scale up blocked by cooldown", { + timeSinceLastScale, + cooldownMs: effectiveCooldown, + jitterMs, + remainingMs: effectiveCooldown - timeSinceLastScale, + }); + this.promMetrics?.recordCooldownApplied("up"); + return; + } + } else if (targetCount < this.consumers.size) { + // Scale down + const effectiveCooldown = this.scaleDownCooldownMs + jitterMs; + if (timeSinceLastScale < effectiveCooldown) { + this.logger.debug("Scale down blocked by cooldown", { + timeSinceLastScale, + cooldownMs: effectiveCooldown, + jitterMs, + remainingMs: effectiveCooldown - timeSinceLastScale, + }); + this.promMetrics?.recordCooldownApplied("down"); + return; + } + } + + this.logger.info("Scaling consumer pool", { + from: this.consumers.size, + to: targetCount, + queueLength: this.metrics.queueLength, + smoothedQueueLength: this.metrics.smoothedQueueLength, + strategy: this.scalingStrategy, + }); + + // Set flag before scaling + this.isScaling = true; + + // Update target metric for visibility + const previousTarget = this.metrics.targetConsumerCount; + this.metrics.targetConsumerCount = targetCount; + + try { + this.scaleToTarget(targetCount); + } catch (error) { + this.logger.error("Failed to scale consumer pool", { error }); + // Revert target on failure + this.metrics.targetConsumerCount = previousTarget; + } finally { + this.isScaling = false; + } + } + + private calculateTargetConsumerCount(): number { + return this.scalingStrategy.calculateTargetCount(this.consumers.size); + } + + private scaleToTarget(targetCount: number) { + const actualCurrentCount = this.consumers.size; + + if (targetCount > actualCurrentCount) { + // Scale up + const count = targetCount - actualCurrentCount; + this.addConsumers(count); + this.promMetrics?.recordScalingOperation("up", this.scalingStrategy.name, count); + } else if (targetCount < actualCurrentCount) { + // Scale down + const count = actualCurrentCount - targetCount; + this.removeConsumers(count); + this.promMetrics?.recordScalingOperation("down", this.scalingStrategy.name, count); + } + + this.metrics.lastScaleTime = new Date(); + + // Update Prometheus state metrics + this.promMetrics?.updateState({ + consumerCount: this.consumers.size, + queueLength: this.metrics.queueLength, + smoothedQueueLength: this.metrics.smoothedQueueLength, + targetConsumerCount: targetCount, + strategy: this.scalingStrategy.name, + }); + } + + private addConsumers(count: number) { + const newConsumers: QueueConsumer[] = []; + + for (let i = 0; i < count; i++) { + const consumerId = `consumer-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`; + + const consumer = this.consumerFactory({ + ...this.consumerOptions, + onDequeue: async (messages) => { + // Always update queue length, default to 0 for empty dequeues or missing value + this.updateQueueLength(messages[0]?.workerQueueLength ?? 0); + + // Forward to the original handler + await this.consumerOptions.onDequeue(messages); + }, + }); + + this.consumers.set(consumerId, consumer); + newConsumers.push(consumer); + } + + // Start all new consumers + newConsumers.forEach((c) => c.start()); + + this.logger.info("Added consumers", { + count, + totalConsumers: this.consumers.size, + }); + } + + private removeConsumers(count: number) { + const allIds = Array.from(this.consumers.keys()); + const consumerIds = allIds.slice(-count); // Take from the end + const consumersToStop: QueueConsumer[] = []; + + for (const id of consumerIds) { + const consumer = this.consumers.get(id); + if (consumer) { + consumersToStop.push(consumer); + this.consumers.delete(id); + } + } + + // Stop removed consumers + consumersToStop.forEach((c) => c.stop()); + + this.logger.info("Removed consumers", { + count: consumersToStop.length, + totalConsumers: this.consumers.size, + }); + } + + /** + * Get current pool metrics for monitoring + */ + getMetrics(): Readonly { + return { ...this.metrics }; + } + + /** + * Get current number of consumers in the pool + */ + get size(): number { + return this.consumers.size; + } +} diff --git a/packages/core/src/v3/runEngineWorker/supervisor/consumerPoolMetrics.ts b/packages/core/src/v3/runEngineWorker/supervisor/consumerPoolMetrics.ts new file mode 100644 index 000000000..8f65fa077 --- /dev/null +++ b/packages/core/src/v3/runEngineWorker/supervisor/consumerPoolMetrics.ts @@ -0,0 +1,160 @@ +import { Counter, Gauge, Histogram, Registry } from "prom-client"; + +export interface ConsumerPoolMetricsOptions { + register?: Registry; + prefix?: string; +} + +export class ConsumerPoolMetrics { + private readonly register: Registry; + private readonly prefix: string; + + // Current state metrics + public readonly consumerCount: Gauge; + public readonly queueLength: Gauge; + public readonly smoothedQueueLength: Gauge; + public readonly targetConsumerCount: Gauge; + public readonly scalingStrategy: Gauge; + + // Scaling operation metrics + public readonly scalingOperationsTotal: Counter; + public readonly consumersAddedTotal: Counter; + public readonly consumersRemovedTotal: Counter; + public readonly scalingCooldownsApplied: Counter; + + // Performance metrics + public readonly queueLengthUpdatesTotal: Counter; + public readonly batchesProcessedTotal: Counter; + + constructor(opts: ConsumerPoolMetricsOptions = {}) { + this.register = opts.register ?? new Registry(); + this.prefix = opts.prefix ?? "queue_consumer_pool"; + + // Current state metrics + this.consumerCount = new Gauge({ + name: `${this.prefix}_consumer_count`, + help: "Current number of active queue consumers", + labelNames: ["strategy"], + registers: [this.register], + }); + + this.queueLength = new Gauge({ + name: `${this.prefix}_queue_length`, + help: "Current queue length (median of recent samples)", + registers: [this.register], + }); + + this.smoothedQueueLength = new Gauge({ + name: `${this.prefix}_smoothed_queue_length`, + help: "EWMA smoothed queue length", + registers: [this.register], + }); + + this.targetConsumerCount = new Gauge({ + name: `${this.prefix}_target_consumer_count`, + help: "Target number of consumers calculated by scaling strategy", + labelNames: ["strategy"], + registers: [this.register], + }); + + this.scalingStrategy = new Gauge({ + name: `${this.prefix}_scaling_strategy_info`, + help: "Information about the active scaling strategy (1 = active, 0 = inactive)", + labelNames: ["strategy"], + registers: [this.register], + }); + + // Scaling operation metrics + this.scalingOperationsTotal = new Counter({ + name: `${this.prefix}_scaling_operations_total`, + help: "Total number of scaling operations performed", + labelNames: ["direction", "strategy"], + registers: [this.register], + }); + + this.consumersAddedTotal = new Counter({ + name: `${this.prefix}_consumers_added_total`, + help: "Total number of consumers added", + registers: [this.register], + }); + + this.consumersRemovedTotal = new Counter({ + name: `${this.prefix}_consumers_removed_total`, + help: "Total number of consumers removed", + registers: [this.register], + }); + + this.scalingCooldownsApplied = new Counter({ + name: `${this.prefix}_scaling_cooldowns_applied_total`, + help: "Number of times scaling was prevented due to cooldown", + labelNames: ["direction"], + registers: [this.register], + }); + + this.queueLengthUpdatesTotal = new Counter({ + name: `${this.prefix}_queue_length_updates_total`, + help: "Total number of queue length updates received", + registers: [this.register], + }); + + this.batchesProcessedTotal = new Counter({ + name: `${this.prefix}_batches_processed_total`, + help: "Total number of metric batches processed", + registers: [this.register], + }); + } + + /** + * Update all gauge metrics with current state + */ + updateState(state: { + consumerCount: number; + queueLength?: number; + smoothedQueueLength: number; + targetConsumerCount: number; + strategy: string; + }) { + this.consumerCount.set({ strategy: state.strategy }, state.consumerCount); + + if (state.queueLength !== undefined) { + this.queueLength.set(state.queueLength); + } + + this.smoothedQueueLength.set(state.smoothedQueueLength); + this.targetConsumerCount.set({ strategy: state.strategy }, state.targetConsumerCount); + + // Set strategy info (1 for active strategy, 0 for others) + ["none", "smooth", "aggressive"].forEach((s) => { + this.scalingStrategy.set({ strategy: s }, s === state.strategy ? 1 : 0); + }); + } + + /** + * Record a scaling operation + */ + recordScalingOperation(direction: "up" | "down" | "none", strategy: string, count: number) { + if (direction !== "none") { + this.scalingOperationsTotal.inc({ direction, strategy }); + + if (direction === "up") { + this.consumersAddedTotal.inc(count); + } else { + this.consumersRemovedTotal.inc(count); + } + } + } + + /** + * Record that scaling was prevented by cooldown + */ + recordCooldownApplied(direction: "up" | "down") { + this.scalingCooldownsApplied.inc({ direction }); + } + + /** + * Record a queue length update + */ + recordQueueLengthUpdate() { + this.queueLengthUpdatesTotal.inc(); + } +} diff --git a/packages/core/src/v3/runEngineWorker/supervisor/queueConsumer.ts b/packages/core/src/v3/runEngineWorker/supervisor/queueConsumer.ts index 6eb5572bf..4379eb54f 100644 --- a/packages/core/src/v3/runEngineWorker/supervisor/queueConsumer.ts +++ b/packages/core/src/v3/runEngineWorker/supervisor/queueConsumer.ts @@ -3,7 +3,12 @@ import { SupervisorHttpClient } from "./http.js"; import { WorkerApiDequeueResponseBody } from "./schemas.js"; import { PreDequeueFn, PreSkipFn } from "./types.js"; -type RunQueueConsumerOptions = { +export interface QueueConsumer { + start(): void; + stop(): void; +} + +export type RunQueueConsumerOptions = { client: SupervisorHttpClient; intervalMs: number; idleIntervalMs: number; @@ -13,7 +18,7 @@ type RunQueueConsumerOptions = { onDequeue: (messages: WorkerApiDequeueResponseBody) => Promise; }; -export class RunQueueConsumer { +export class RunQueueConsumer implements QueueConsumer { private readonly client: SupervisorHttpClient; private readonly preDequeue?: PreDequeueFn; private readonly preSkip?: PreSkipFn; @@ -131,7 +136,7 @@ export class RunQueueConsumer { this.scheduleNextDequeue(nextIntervalMs); } - scheduleNextDequeue(delayMs: number) { + private scheduleNextDequeue(delayMs: number) { if (delayMs === this.idleIntervalMs && this.idleIntervalMs !== this.intervalMs) { this.logger.verbose("scheduled dequeue with idle interval", { delayMs }); } diff --git a/packages/core/src/v3/runEngineWorker/supervisor/queueMetricsProcessor.test.ts b/packages/core/src/v3/runEngineWorker/supervisor/queueMetricsProcessor.test.ts new file mode 100644 index 000000000..ac6a19a04 --- /dev/null +++ b/packages/core/src/v3/runEngineWorker/supervisor/queueMetricsProcessor.test.ts @@ -0,0 +1,371 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { QueueMetricsProcessor } from "./queueMetricsProcessor.js"; + +describe("QueueMetricsProcessor", () => { + let processor: QueueMetricsProcessor; + + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + describe("Constructor validation", () => { + it("should throw error for invalid ewmaAlpha", () => { + expect(() => new QueueMetricsProcessor({ ewmaAlpha: -0.1, batchWindowMs: 1000 })).toThrow( + "ewmaAlpha must be between 0 and 1" + ); + + expect(() => new QueueMetricsProcessor({ ewmaAlpha: 1.1, batchWindowMs: 1000 })).toThrow( + "ewmaAlpha must be between 0 and 1" + ); + }); + + it("should throw error for invalid batchWindowMs", () => { + expect(() => new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 0 })).toThrow( + "batchWindowMs must be positive" + ); + + expect(() => new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: -100 })).toThrow( + "batchWindowMs must be positive" + ); + }); + + it("should accept valid parameters", () => { + expect(() => new QueueMetricsProcessor({ ewmaAlpha: 0, batchWindowMs: 1 })).not.toThrow(); + expect(() => new QueueMetricsProcessor({ ewmaAlpha: 1, batchWindowMs: 5000 })).not.toThrow(); + }); + }); + + describe("Sample collection", () => { + beforeEach(() => { + processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 }); + }); + + it("should collect samples without limit", () => { + for (let i = 0; i < 100; i++) { + processor.addSample(i); + } + + expect(processor.getCurrentSampleCount()).toBe(100); + expect(processor.getCurrentSamples()).toHaveLength(100); + }); + + it("should throw error for negative queue lengths", () => { + expect(() => processor.addSample(-1)).toThrow("Queue length cannot be negative"); + }); + + it("should accept zero queue length", () => { + expect(() => processor.addSample(0)).not.toThrow(); + expect(processor.getCurrentSampleCount()).toBe(1); + }); + + it("should handle empty queue with all zero samples", () => { + processor.addSample(0); + processor.addSample(0); + processor.addSample(0); + + const result = processor.processBatch(); + expect(result).not.toBeNull(); + expect(result!.median).toBe(0); + expect(result!.smoothedValue).toBe(0); + expect(processor.getSmoothedValue()).toBe(0); + }); + + it("should properly transition from zero to non-zero queue", () => { + // Start with empty queue + processor.addSample(0); + processor.addSample(0); + let result = processor.processBatch(); + expect(result!.median).toBe(0); + expect(result!.smoothedValue).toBe(0); + + // Queue starts filling + processor.addSample(10); + processor.addSample(15); + result = processor.processBatch(); + expect(result!.median).toBeGreaterThan(0); + // EWMA: 0.3 * median + 0.7 * 0 + expect(result!.smoothedValue).toBeGreaterThan(0); + }); + + it("should properly transition from non-zero to zero queue", () => { + // Start with non-empty queue + processor.addSample(10); + processor.addSample(15); + let result = processor.processBatch(); + const initialSmoothed = result!.smoothedValue; + expect(initialSmoothed).toBeGreaterThan(0); + + // Queue becomes empty + processor.addSample(0); + processor.addSample(0); + processor.addSample(0); + result = processor.processBatch(); + expect(result!.median).toBe(0); + // EWMA should gradually decrease: 0.3 * 0 + 0.7 * initialSmoothed + expect(result!.smoothedValue).toBe(0.7 * initialSmoothed); + }); + }); + + describe("Batch processing timing", () => { + beforeEach(() => { + processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 }); + }); + + it("should not process batch before window expires", () => { + processor.addSample(10, 1000); + + expect(processor.shouldProcessBatch(1500)).toBe(false); // 500ms later + expect(processor.shouldProcessBatch(1999)).toBe(false); // 999ms later + }); + + it("should process batch when window expires", () => { + processor.addSample(10, 1000); + + expect(processor.shouldProcessBatch(2000)).toBe(true); // 1000ms later + expect(processor.shouldProcessBatch(2500)).toBe(true); // 1500ms later + }); + + it("should not process empty batch", () => { + expect(processor.shouldProcessBatch(5000)).toBe(false); + }); + }); + + describe("EWMA calculation", () => { + it("should initialize with first value", () => { + processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 }); + + processor.addSample(10); + const result = processor.processBatch(); + + expect(result).not.toBeNull(); + expect(result!.median).toBe(10); + expect(result!.smoothedValue).toBe(10); + expect(processor.getSmoothedValue()).toBe(10); + }); + + it("should apply EWMA formula correctly", () => { + processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 }); + + // First batch: smoothed = 10 + processor.addSample(10); + processor.processBatch(); + expect(processor.getSmoothedValue()).toBe(10); + + // Second batch: smoothed = 0.3 * 20 + 0.7 * 10 = 6 + 7 = 13 + processor.addSample(20); + processor.processBatch(); + expect(processor.getSmoothedValue()).toBe(13); + + // Third batch: smoothed = 0.3 * 5 + 0.7 * 13 = 1.5 + 9.1 = 10.6 + processor.addSample(5); + processor.processBatch(); + expect(processor.getSmoothedValue()).toBe(10.6); + }); + + it("should test different alpha values", () => { + // High alpha (0.8) - more responsive + const highAlphaProcessor = new QueueMetricsProcessor({ ewmaAlpha: 0.8, batchWindowMs: 1000 }); + highAlphaProcessor.addSample(10); + highAlphaProcessor.processBatch(); + highAlphaProcessor.addSample(20); + highAlphaProcessor.processBatch(); + + // Low alpha (0.1) - more smoothing + const lowAlphaProcessor = new QueueMetricsProcessor({ ewmaAlpha: 0.1, batchWindowMs: 1000 }); + lowAlphaProcessor.addSample(10); + lowAlphaProcessor.processBatch(); + lowAlphaProcessor.addSample(20); + lowAlphaProcessor.processBatch(); + + // High alpha should be closer to recent value (20) + expect(highAlphaProcessor.getSmoothedValue()).toBeCloseTo(18); // 0.8 * 20 + 0.2 * 10 = 18 + // Low alpha should be closer to previous value (10) + expect(lowAlphaProcessor.getSmoothedValue()).toBeCloseTo(11); // 0.1 * 20 + 0.9 * 10 = 11 + }); + }); + + describe("Median filtering", () => { + beforeEach(() => { + processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 }); + }); + + it("should calculate median of single sample", () => { + processor.addSample(42); + + const result = processor.processBatch(); + expect(result!.median).toBe(42); + expect(result!.sampleCount).toBe(1); + expect(result!.smoothedValue).toBe(42); // First batch initializes to median + }); + + it("should calculate median of odd number of samples", () => { + processor.addSample(1); + processor.addSample(10); + processor.addSample(5); + + const result = processor.processBatch(); + expect(result!.median).toBe(5); + }); + + it("should calculate median of even number of samples", () => { + processor.addSample(1); + processor.addSample(10); + processor.addSample(5); + processor.addSample(8); + + const result = processor.processBatch(); + // With even count, we average the two middle values + // Sorted: [1, 5, 8, 10], median = (5 + 8) / 2 = 6.5 + expect(result!.median).toBe(6.5); + }); + + it("should filter outliers using median", () => { + // Add mostly low values with one outlier + processor.addSample(5); + processor.addSample(5); + processor.addSample(5); + processor.addSample(100); // outlier + processor.addSample(5); + + const result = processor.processBatch(); + // Sorted: [5, 5, 5, 5, 100], median = 5 (filters out outlier) + expect(result!.median).toBe(5); + }); + }); + + describe("Batch result", () => { + beforeEach(() => { + processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 }); + }); + + it("should return comprehensive batch result", () => { + processor.addSample(10); + processor.addSample(20); + processor.addSample(15); + + const result = processor.processBatch(); + + expect(result).not.toBeNull(); + expect(result!.median).toBe(15); + expect(result!.smoothedValue).toBe(15); // First batch + expect(result!.sampleCount).toBe(3); + expect(result!.samples).toEqual([10, 20, 15]); + }); + + it("should return null for empty batch", () => { + const result = processor.processBatch(); + expect(result).toBeNull(); + }); + + it("should clear samples after processing", () => { + processor.addSample(10); + processor.addSample(20); + + expect(processor.getCurrentSampleCount()).toBe(2); + + processor.processBatch(); + + expect(processor.getCurrentSampleCount()).toBe(0); + expect(processor.getCurrentSamples()).toHaveLength(0); + }); + }); + + describe("Reset functionality", () => { + beforeEach(() => { + processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 }); + }); + + it("should reset all state", () => { + processor.addSample(10); + processor.processBatch(); + processor.addSample(20); + + expect(processor.getSmoothedValue()).toBe(10); + expect(processor.getCurrentSampleCount()).toBe(1); + + processor.reset(); + + expect(processor.getSmoothedValue()).toBe(0); + expect(processor.getCurrentSampleCount()).toBe(0); + expect(processor.getCurrentSamples()).toHaveLength(0); + }); + + it("should reinitialize correctly after reset", () => { + // Process some data + processor.addSample(10); + processor.processBatch(); + processor.addSample(20); + processor.processBatch(); + + processor.reset(); + + // Should initialize with first value again + processor.addSample(30); + const result = processor.processBatch(); + + expect(result!.smoothedValue).toBe(30); + expect(processor.getSmoothedValue()).toBe(30); + }); + }); + + describe("Configuration", () => { + it("should return configuration", () => { + processor = new QueueMetricsProcessor({ ewmaAlpha: 0.5, batchWindowMs: 2000 }); + + const config = processor.getConfig(); + expect(config.ewmaAlpha).toBe(0.5); + expect(config.batchWindowMs).toBe(2000); + }); + }); + + describe("Real-world simulation", () => { + it("should handle high-frequency samples from multiple consumers", () => { + processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 }); + + // Simulate 40 consumers reporting queue lengths within 1 second + const baseTime = 1000; + for (let i = 0; i < 40; i++) { + const queueLength = 100 - i * 2; // Queue decreasing as consumers work + processor.addSample(queueLength, baseTime + i * 25); // Spread over 1 second + } + + const result = processor.processBatch(baseTime + 1000); + + expect(result).not.toBeNull(); + expect(result!.sampleCount).toBe(40); + // Median should be around middle values (queue lengths 60-80) + expect(result!.median).toBeGreaterThanOrEqual(60); + expect(result!.median).toBeLessThanOrEqual(80); + }); + + it("should demonstrate EWMA smoothing over time", () => { + processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 }); + + const results = []; + + // Simulate queue spike and recovery + const scenarios = [ + { samples: [5, 5, 5], expected: 5 }, // Baseline + { samples: [50, 50, 50], expected: 18.5 }, // Spike: 0.3 * 50 + 0.7 * 5 = 18.5 + { samples: [5, 5, 5], expected: 10.05 }, // Recovery: 0.3 * 5 + 0.7 * 18.5 = 14.45 + ]; + + for (const scenario of scenarios) { + for (const sample of scenario.samples) { + processor.addSample(sample); + } + const result = processor.processBatch(); + results.push(result!.smoothedValue); + } + + // Should show gradual change due to EWMA smoothing + expect(results[0]).toBe(5); // Initial + expect(results[1]).toBeCloseTo(18.5); // Spike response + expect(results[2]).toBeCloseTo(14.45); // Gradual recovery + }); + }); +}); diff --git a/packages/core/src/v3/runEngineWorker/supervisor/queueMetricsProcessor.ts b/packages/core/src/v3/runEngineWorker/supervisor/queueMetricsProcessor.ts new file mode 100644 index 000000000..5f628f432 --- /dev/null +++ b/packages/core/src/v3/runEngineWorker/supervisor/queueMetricsProcessor.ts @@ -0,0 +1,209 @@ +import { SimpleStructuredLogger } from "../../utils/structuredLogger.js"; + +export interface QueueMetricsProcessorOptions { + /** + * EWMA smoothing factor (0-1) + * Lower values = more smoothing, less reactive + * Higher values = more responsive to recent changes + */ + ewmaAlpha: number; + + /** + * Batch window duration in milliseconds + * Samples within this window are collected and processed together + */ + batchWindowMs: number; +} + +export interface BatchProcessingResult { + /** Median of samples in the batch */ + median: number; + + /** EWMA-smoothed value after processing this batch */ + smoothedValue: number; + + /** Number of samples processed in this batch */ + sampleCount: number; + + /** Raw samples that were processed */ + samples: readonly number[]; +} + +/** + * Processes queue length samples using exponential weighted moving average (EWMA) + * for smoothing and median filtering for outlier resistance. + * + * Collects samples within a batch window, calculates median to filter outliers, + * then applies EWMA smoothing for stable trend tracking. + */ +export class QueueMetricsProcessor { + private readonly ewmaAlpha: number; + private readonly batchWindowMs: number; + private readonly logger = new SimpleStructuredLogger("queue-metrics-processor"); + + private samples: number[] = []; + private smoothedValue: number = 0; + private lastBatchTime: number = 0; + private isInitialized: boolean = false; + + constructor(options: QueueMetricsProcessorOptions) { + if (options.ewmaAlpha < 0 || options.ewmaAlpha > 1) { + throw new Error("ewmaAlpha must be between 0 and 1"); + } + if (options.batchWindowMs <= 0) { + throw new Error("batchWindowMs must be positive"); + } + + this.ewmaAlpha = options.ewmaAlpha; + this.batchWindowMs = options.batchWindowMs; + } + + /** + * Adds a sample to the current batch + */ + addSample(value: number, timestamp: number = Date.now()): void { + if (value < 0) { + throw new Error("Queue length cannot be negative"); + } + + this.samples.push(value); + + // Update last batch time on first sample + if (this.samples.length === 1) { + this.lastBatchTime = timestamp; + } + } + + /** + * Checks if enough time has passed to process the current batch + */ + shouldProcessBatch(currentTime: number = Date.now()): boolean { + if (this.samples.length === 0) { + return false; + } + + return currentTime - this.lastBatchTime >= this.batchWindowMs; + } + + private calculateMedian(samples: number[]): number | null { + const sortedSamples = [...samples].sort((a, b) => a - b); + const mid = Math.floor(sortedSamples.length / 2); + + if (sortedSamples.length % 2 === 1) { + // Odd length: use middle value + const median = sortedSamples[mid]; + + if (median === undefined) { + this.logger.error("Invalid median calculated from odd samples", { + sortedSamples, + mid, + median, + }); + return null; + } + + return median; + } else { + // Even length: average two middle values + const lowMid = sortedSamples[mid - 1]; + const highMid = sortedSamples[mid]; + + if (lowMid === undefined || highMid === undefined) { + this.logger.error("Invalid median calculated from even samples", { + sortedSamples, + mid, + lowMid, + highMid, + }); + return null; + } + + const median = (lowMid + highMid) / 2; + return median; + } + } + + /** + * Processes the current batch of samples and returns the result. + * Clears the samples array and updates the smoothed value. + * + * Returns null if there are no samples to process. + */ + processBatch(currentTime: number = Date.now()): BatchProcessingResult | null { + if (this.samples.length === 0) { + // No samples to process + return null; + } + + // Calculate median of samples to filter outliers + const median = this.calculateMedian(this.samples); + if (median === null) { + // We already logged a more specific error message + return null; + } + + // Update EWMA smoothed value + if (!this.isInitialized) { + // First value - initialize with median + this.smoothedValue = median; + this.isInitialized = true; + } else { + // Apply EWMA: s_t = α * x_t + (1 - α) * s_(t-1) + this.smoothedValue = this.ewmaAlpha * median + (1 - this.ewmaAlpha) * this.smoothedValue; + } + + const result: BatchProcessingResult = { + median, + smoothedValue: this.smoothedValue, + sampleCount: this.samples.length, + samples: Object.freeze([...this.samples]), + }; + + // Clear samples for next batch + this.samples = []; + this.lastBatchTime = currentTime; + + return result; + } + + /** + * Gets the current smoothed value without processing a batch + */ + getSmoothedValue(): number { + return this.smoothedValue; + } + + /** + * Gets the number of samples in the current batch + */ + getCurrentSampleCount(): number { + return this.samples.length; + } + + /** + * Gets the current samples (for testing/debugging) + */ + getCurrentSamples(): readonly number[] { + return Object.freeze([...this.samples]); + } + + /** + * Resets the processor state + */ + reset(): void { + this.samples = []; + this.smoothedValue = 0; + this.lastBatchTime = 0; + this.isInitialized = false; + } + + /** + * Gets processor configuration + */ + getConfig(): Readonly { + return { + ewmaAlpha: this.ewmaAlpha, + batchWindowMs: this.batchWindowMs, + }; + } +} diff --git a/packages/core/src/v3/runEngineWorker/supervisor/scalingStrategies.test.ts b/packages/core/src/v3/runEngineWorker/supervisor/scalingStrategies.test.ts new file mode 100644 index 000000000..3d6f4062e --- /dev/null +++ b/packages/core/src/v3/runEngineWorker/supervisor/scalingStrategies.test.ts @@ -0,0 +1,293 @@ +import { describe, it, expect } from "vitest"; +import { + NoneScalingStrategy, + SmoothScalingStrategy, + AggressiveScalingStrategy, + ScalingStrategyOptions, +} from "./scalingStrategies.js"; +import { QueueMetricsProcessor } from "./queueMetricsProcessor.js"; + +describe("Scaling Strategies", () => { + const baseOptions: ScalingStrategyOptions = { + minConsumerCount: 1, + maxConsumerCount: 20, + targetRatio: 1.0, + }; + + function createMetricsProcessor(smoothedValue: number): QueueMetricsProcessor { + const processor = new QueueMetricsProcessor({ ewmaAlpha: 0.3, batchWindowMs: 1000 }); + // Initialize processor with the target smoothed value + processor.addSample(smoothedValue); + processor.processBatch(); + return processor; + } + + describe("NoneScalingStrategy", () => { + const strategy = new NoneScalingStrategy(baseOptions); + + it("should always return current count (static mode)", () => { + expect(strategy.calculateTargetCount(5)).toBe(5); + expect(strategy.calculateTargetCount(1)).toBe(1); + expect(strategy.calculateTargetCount(10)).toBe(10); + // Clamping still applies + expect(strategy.calculateTargetCount(25)).toBe(20); // Clamped to max + expect(strategy.calculateTargetCount(0)).toBe(1); // Clamped to min + }); + + it("should have correct name", () => { + expect(strategy.name).toBe("none"); + }); + + it("should handle zero current count", () => { + // Should clamp to minConsumerCount + const result = strategy.calculateTargetCount(0); + expect(result).toBe(1); + }); + }); + + describe("SmoothScalingStrategy", () => { + it("should calculate target based on smoothed queue length", () => { + const metricsProcessor = createMetricsProcessor(10); // smoothed value = 10 + const strategy = new SmoothScalingStrategy({ ...baseOptions, metricsProcessor }); + + // With targetRatio=1.0, target consumers = ceil(10/1.0) = 10 + // With dampingFactor=0.7 and currentCount=5: + // dampedTarget = 5 + (10 - 5) * 0.7 = 5 + 3.5 = 8.5 → 9 + const result = strategy.calculateTargetCount(5); + expect(result).toBe(9); + }); + + it("should apply damping factor correctly", () => { + const metricsProcessor = createMetricsProcessor(20); // smoothed value = 20 + const strategy = new SmoothScalingStrategy({ + ...baseOptions, + metricsProcessor, + dampingFactor: 0.5, + }); // 50% damping + + // With targetRatio=1.0, target consumers = ceil(20/1.0) = 20 + // With dampingFactor=0.5 and currentCount=5: + // dampedTarget = 5 + (20 - 5) * 0.5 = 5 + 7.5 = 12.5 → 13 + const result = strategy.calculateTargetCount(5); + expect(result).toBe(13); + }); + + it("should handle zero current count", () => { + const metricsProcessor = createMetricsProcessor(5); + const strategy = new SmoothScalingStrategy({ ...baseOptions, metricsProcessor }); + + // With smoothedQueueLength=5, targetRatio=1.0: + // targetConsumers = ceil(5/1.0) = 5 + // dampedTarget = 0 + (5 - 0) * 0.7 = 3.5 → 4 + const result = strategy.calculateTargetCount(0); + expect(result).toBe(4); + }); + + it("should validate damping factor", () => { + const metricsProcessor = createMetricsProcessor(10); + expect( + () => + new SmoothScalingStrategy({ + ...baseOptions, + metricsProcessor, + dampingFactor: -0.1, + }) + ).toThrow("dampingFactor must be between 0 and 1"); + + expect( + () => + new SmoothScalingStrategy({ + ...baseOptions, + metricsProcessor, + dampingFactor: 1.1, + }) + ).toThrow("dampingFactor must be between 0 and 1"); + + expect( + () => + new SmoothScalingStrategy({ + ...baseOptions, + metricsProcessor, + dampingFactor: 0, + }) + ).not.toThrow(); + + expect( + () => + new SmoothScalingStrategy({ + ...baseOptions, + metricsProcessor, + dampingFactor: 1, + }) + ).not.toThrow(); + }); + + it("should handle zero current count", () => { + const metricsProcessor = createMetricsProcessor(10); + const strategy = new SmoothScalingStrategy({ ...baseOptions, metricsProcessor }); + + // With smoothedQueueLength=10, targetRatio=1.0: + // targetConsumers = ceil(10/1.0) = 10 + // dampedTarget = 0 + (10 - 0) * 0.7 = 7 + const result = strategy.calculateTargetCount(0); + expect(result).toBe(7); + }); + }); + + describe("AggressiveScalingStrategy", () => { + it("should scale down when under-utilized", () => { + // queuePerConsumer = 2/5 = 0.4, scaleDownThreshold = 1.0 * 0.5 = 0.5 + // Under-utilized since 0.4 < 0.5 + const metricsProcessor = createMetricsProcessor(2); + const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor }); + + const result = strategy.calculateTargetCount(5); + expect(result).toBeLessThan(5); + expect(result).toBeGreaterThanOrEqual(baseOptions.minConsumerCount); + }); + + it("should maintain count when in optimal zone", () => { + // queuePerConsumer = 5/5 = 1.0 + // Optimal zone: 0.5 < 1.0 < 2.0 + const metricsProcessor = createMetricsProcessor(5); + const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor }); + + const result = strategy.calculateTargetCount(5); + expect(result).toBe(5); + }); + + it("should scale up when over-utilized", () => { + // queuePerConsumer = 15/5 = 3.0, scaleUpThreshold = 1.0 * 2.0 = 2.0 + // Over-utilized since 3.0 > 2.0 + const metricsProcessor = createMetricsProcessor(15); + const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor }); + + const result = strategy.calculateTargetCount(5); + expect(result).toBeGreaterThan(5); + expect(result).toBeLessThanOrEqual(baseOptions.maxConsumerCount); + }); + + it("should scale aggressively for critical load", () => { + // queuePerConsumer = 25/5 = 5.0 (critical: 5x target ratio) + const metricsProcessor = createMetricsProcessor(25); + const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor }); + + const result = strategy.calculateTargetCount(5); + // Should apply 50% scale factor: ceil(5 * 1.5) = 8 + // But capped by 50% max increment: 5 + ceil(5 * 0.5) = 5 + 3 = 8 + expect(result).toBe(8); + }); + + it("should respect max consumer count", () => { + const metricsProcessor = createMetricsProcessor(50); // Very high load + const strategy = new AggressiveScalingStrategy({ + ...baseOptions, + maxConsumerCount: 6, + metricsProcessor, + }); + + const result = strategy.calculateTargetCount(5); + expect(result).toBeLessThanOrEqual(6); + }); + + it("should respect min consumer count", () => { + const metricsProcessor = createMetricsProcessor(0.1); // Very low load + const strategy = new AggressiveScalingStrategy({ + ...baseOptions, + minConsumerCount: 3, + metricsProcessor, + }); + + const result = strategy.calculateTargetCount(5); + expect(result).toBeGreaterThanOrEqual(3); + }); + + it("should return thresholds", () => { + const metricsProcessor = createMetricsProcessor(10); + const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor }); + const thresholds = strategy.getThresholds(1.0); + expect(thresholds).toEqual({ + scaleDownThreshold: 0.5, + scaleUpThreshold: 2.0, + criticalThreshold: 5.0, + highThreshold: 3.0, + }); + }); + + it("should handle zero current count without division by zero", () => { + const metricsProcessor = createMetricsProcessor(10); + const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor }); + + // Should use (currentCount || 1) to prevent division by zero + // queuePerConsumer = 10 / 1 = 10 (not 10 / 0) + // This is over-utilized (10 > 2.0), should scale up + const result = strategy.calculateTargetCount(0); + expect(result).toBeGreaterThan(0); + expect(result).toBeLessThanOrEqual(baseOptions.maxConsumerCount); + }); + + it("should handle zero queue with zero consumers", () => { + const metricsProcessor = createMetricsProcessor(0); + const strategy = new AggressiveScalingStrategy({ ...baseOptions, metricsProcessor }); + + // queuePerConsumer = 0 / 1 = 0 + // This is under-utilized (0 < 0.5), should scale down + // But already at 0, so should return minConsumerCount + const result = strategy.calculateTargetCount(0); + expect(result).toBe(baseOptions.minConsumerCount); + }); + }); + + describe("Integration scenarios", () => { + it("should handle gradual load increase with smooth strategy", () => { + const metricsProcessor = createMetricsProcessor(2); + const strategy = new SmoothScalingStrategy({ ...baseOptions, metricsProcessor }); + let currentCount = 2; + + // Gradual increase: 2 → 6 → 10 → 15 + const loads = [2, 6, 10, 15]; + const results = []; + + for (const load of loads) { + // Update the processor with the new load + metricsProcessor.addSample(load); + metricsProcessor.processBatch(); + const target = strategy.calculateTargetCount(currentCount); + results.push(target); + currentCount = target; + } + + // Should show gradual increase due to damping + expect(results[0]).toBeLessThan(results[1]!); + expect(results[1]).toBeLessThan(results[2]!); + expect(results[2]).toBeLessThan(results[3]!); + + // But not immediate jumps due to damping + expect(results[1]! - results[0]!).toBeLessThan(loads[1]! - loads[0]!); + }); + + it("should handle load spike with aggressive strategy", () => { + let currentCount = 3; + + // Sudden spike from normal to critical + const normalLoad = 3; // queuePerConsumer = 1.0 (optimal) + const spikeLoad = 15; // queuePerConsumer = 5.0 (critical) + + const normalProcessor = createMetricsProcessor(normalLoad); + const normalStrategy = new AggressiveScalingStrategy({ + ...baseOptions, + metricsProcessor: normalProcessor, + }); + const normalTarget = normalStrategy.calculateTargetCount(currentCount); + expect(normalTarget).toBe(3); // Should maintain + + const spikeProcessor = createMetricsProcessor(spikeLoad); + const spikeStrategy = new AggressiveScalingStrategy({ + ...baseOptions, + metricsProcessor: spikeProcessor, + }); + const spikeTarget = spikeStrategy.calculateTargetCount(currentCount); + expect(spikeTarget).toBeGreaterThan(3); // Should scale up aggressively + }); + }); +}); diff --git a/packages/core/src/v3/runEngineWorker/supervisor/scalingStrategies.ts b/packages/core/src/v3/runEngineWorker/supervisor/scalingStrategies.ts new file mode 100644 index 000000000..665c689d6 --- /dev/null +++ b/packages/core/src/v3/runEngineWorker/supervisor/scalingStrategies.ts @@ -0,0 +1,186 @@ +import { QueueMetricsProcessor } from "./queueMetricsProcessor.js"; + +export type ScalingStrategyKind = "none" | "smooth" | "aggressive"; + +export interface ScalingStrategyOptions { + metricsProcessor?: QueueMetricsProcessor; + dampingFactor?: number; + minConsumerCount: number; + maxConsumerCount: number; + targetRatio: number; +} + +export abstract class ScalingStrategy { + abstract readonly name: string; + + private readonly minConsumerCount: number; + private readonly maxConsumerCount: number; + + protected readonly targetRatio: number; + + constructor(options?: ScalingStrategyOptions) { + this.minConsumerCount = options?.minConsumerCount ?? 1; + this.maxConsumerCount = options?.maxConsumerCount ?? 10; + this.targetRatio = options?.targetRatio ?? 1; + } + + /** + * Calculates the target consumer count with clamping to min/max bounds + * Uses template method pattern to ensure consistent clamping across all strategies + */ + calculateTargetCount(currentCount: number): number { + const targetCount = this.calculateTargetCountInternal(currentCount); + + // Apply consistent clamping to all strategies + return Math.min(Math.max(targetCount, this.minConsumerCount), this.maxConsumerCount); + } + + /** + * Internal method for subclasses to implement their specific scaling logic + * Should return the unclamped target count + */ + protected abstract calculateTargetCountInternal(currentCount: number): number; + + /** + * Creates a scaling strategy by name + */ + static create(strategy: ScalingStrategyKind, options?: ScalingStrategyOptions): ScalingStrategy { + switch (strategy) { + case "none": + return new NoneScalingStrategy(options); + + case "smooth": + return new SmoothScalingStrategy(options); + + case "aggressive": + return new AggressiveScalingStrategy(options); + + default: + throw new Error(`Unknown scaling strategy: ${strategy}`); + } + } +} + +/** + * Static scaling strategy - maintains a fixed number of consumers + */ +export class NoneScalingStrategy extends ScalingStrategy { + readonly name = "none"; + + constructor(options?: ScalingStrategyOptions) { + super(options); + } + + protected calculateTargetCountInternal(currentCount: number): number { + return currentCount; + } +} + +/** + * Smooth scaling strategy with EWMA smoothing and damping + * Uses exponentially weighted moving average for queue length smoothing + * and applies damping to prevent rapid oscillations. + */ +export class SmoothScalingStrategy extends ScalingStrategy { + readonly name = "smooth"; + private readonly dampingFactor: number; + private readonly metricsProcessor: QueueMetricsProcessor; + + constructor(options?: ScalingStrategyOptions) { + super(options); + const dampingFactor = options?.dampingFactor ?? 0.7; + if (dampingFactor < 0 || dampingFactor > 1) { + throw new Error("dampingFactor must be between 0 and 1"); + } + if (!options?.metricsProcessor) { + throw new Error("metricsProcessor is required for smooth scaling strategy"); + } + this.dampingFactor = dampingFactor; + this.metricsProcessor = options.metricsProcessor; + } + + protected calculateTargetCountInternal(currentCount: number): number { + const smoothedQueueLength = this.metricsProcessor.getSmoothedValue(); + + // Calculate target consumers based on the configured ratio + const targetConsumers = Math.ceil(smoothedQueueLength / this.targetRatio); + + // Apply damping factor to smooth out changes + // This prevents oscillation by only moving toward the target gradually + const dampedTarget = currentCount + (targetConsumers - currentCount) * this.dampingFactor; + + // Return rounded value without clamping (handled by base class) + return Math.round(dampedTarget); + } +} + +/** + * Aggressive scaling strategy with threshold-based zones + * Uses threshold-based zones for different scaling behaviors. + * Scales up quickly when load increases but scales down cautiously. + */ +export class AggressiveScalingStrategy extends ScalingStrategy { + readonly name = "aggressive"; + private readonly metricsProcessor: QueueMetricsProcessor; + + constructor(options?: ScalingStrategyOptions) { + super(options); + if (!options?.metricsProcessor) { + throw new Error("metricsProcessor is required for aggressive scaling strategy"); + } + this.metricsProcessor = options.metricsProcessor; + } + + protected calculateTargetCountInternal(currentCount: number): number { + const smoothedQueueLength = this.metricsProcessor.getSmoothedValue(); + + // Calculate queue items per consumer, + const queuePerConsumer = smoothedQueueLength / (currentCount || 1); + + // Define zones based on targetRatio + // Optimal zone: 0.5x to 2x the target ratio + const scaleDownThreshold = this.targetRatio * 0.5; + const scaleUpThreshold = this.targetRatio * 2.0; + + if (queuePerConsumer < scaleDownThreshold) { + // Zone 1: Under-utilized (< 0.5x target ratio) + // Scale down gradually to avoid removing too many consumers + const reductionFactor = Math.max(0.9, 1 - (scaleDownThreshold - queuePerConsumer) * 0.1); + // Return without min clamping (handled by base class) + return Math.floor(currentCount * reductionFactor); + } else if (queuePerConsumer > scaleUpThreshold) { + // Zone 3: Over-utilized (> 2x target ratio) + // Scale up aggressively based on queue pressure + let scaleFactor: number; + if (queuePerConsumer >= this.targetRatio * 5) { + // Critical: Queue is 5x target ratio or higher + scaleFactor = 1.5; // 50% increase + } else if (queuePerConsumer >= this.targetRatio * 3) { + // High: Queue is 3x target ratio + scaleFactor = 1.3; // 30% increase + } else { + // Moderate: Queue is 2x target ratio + scaleFactor = 1.1; // 10% increase + } + + const targetCount = Math.ceil(currentCount * scaleFactor); + // Cap increase at 50% to prevent overshooting + const maxIncrement = Math.ceil(currentCount * 0.5); + // Return without max clamping (handled by base class) + return Math.min(currentCount + maxIncrement, targetCount); + } else { + // Zone 2: Optimal (0.5x - 2x target ratio) + // Maintain current consumer count + return currentCount; + } + } + + getThresholds(targetRatio: number) { + return { + scaleDownThreshold: targetRatio * 0.5, + scaleUpThreshold: targetRatio * 2.0, + criticalThreshold: targetRatio * 5.0, + highThreshold: targetRatio * 3.0, + }; + } +} diff --git a/packages/core/src/v3/runEngineWorker/supervisor/session.ts b/packages/core/src/v3/runEngineWorker/supervisor/session.ts index a458a9cd4..e5a783b8d 100644 --- a/packages/core/src/v3/runEngineWorker/supervisor/session.ts +++ b/packages/core/src/v3/runEngineWorker/supervisor/session.ts @@ -1,7 +1,7 @@ import { SupervisorHttpClient } from "./http.js"; import { PreDequeueFn, PreSkipFn, SupervisorClientCommonOptions } from "./types.js"; import { WorkerApiDequeueResponseBody, WorkerApiHeartbeatRequestBody } from "./schemas.js"; -import { RunQueueConsumer } from "./queueConsumer.js"; +import { RunQueueConsumerPool, ScalingOptions } from "./consumerPool.js"; import { WorkerEvents } from "./events.js"; import EventEmitter from "events"; import { VERSION } from "../../../version.js"; @@ -10,6 +10,7 @@ import { WorkerClientToServerEvents, WorkerServerToClientEvents } from "../types import { getDefaultWorkerHeaders } from "./util.js"; import { IntervalService } from "../../utils/interval.js"; import { SimpleStructuredLogger } from "../../utils/structuredLogger.js"; +import type { Registry } from "prom-client"; type SupervisorSessionOptions = SupervisorClientCommonOptions & { queueConsumerEnabled?: boolean; @@ -20,8 +21,9 @@ type SupervisorSessionOptions = SupervisorClientCommonOptions & { preDequeue?: PreDequeueFn; preSkip?: PreSkipFn; maxRunCount?: number; - maxConsumerCount?: number; sendRunDebugLogs?: boolean; + scaling: ScalingOptions; + metricsRegistry?: Registry; }; export class SupervisorSession extends EventEmitter { @@ -33,7 +35,7 @@ export class SupervisorSession extends EventEmitter { private runNotificationsSocket?: Socket; private readonly queueConsumerEnabled: boolean; - private readonly queueConsumers: RunQueueConsumer[]; + private readonly consumerPool: RunQueueConsumerPool; private readonly heartbeat: IntervalService; @@ -44,8 +46,9 @@ export class SupervisorSession extends EventEmitter { this.queueConsumerEnabled = opts.queueConsumerEnabled ?? true; this.httpClient = new SupervisorHttpClient(opts); - this.queueConsumers = Array.from({ length: opts.maxConsumerCount ?? 1 }, () => { - return new RunQueueConsumer({ + + this.consumerPool = new RunQueueConsumerPool({ + consumer: { client: this.httpClient, preDequeue: opts.preDequeue, preSkip: opts.preSkip, @@ -53,7 +56,9 @@ export class SupervisorSession extends EventEmitter { intervalMs: opts.dequeueIntervalMs, idleIntervalMs: opts.dequeueIdleIntervalMs, maxRunCount: opts.maxRunCount, - }); + }, + scaling: opts.scaling, + metricsRegistry: opts.metricsRegistry, }); this.heartbeat = new IntervalService({ @@ -179,8 +184,13 @@ export class SupervisorSession extends EventEmitter { }); if (this.queueConsumerEnabled) { - this.logger.log("Queue consumer enabled"); - await Promise.allSettled(this.queueConsumers.map(async (q) => q.start())); + this.logger.log("Queue consumer enabled", { + scalingStrategy: this.consumerPool["scalingStrategy"], + minConsumers: this.consumerPool["minConsumerCount"], + maxConsumers: this.consumerPool["maxConsumerCount"], + }); + + await this.consumerPool.start(); this.heartbeat.start(); } else { this.logger.warn("Queue consumer disabled"); @@ -195,7 +205,7 @@ export class SupervisorSession extends EventEmitter { } async stop() { - await Promise.allSettled(this.queueConsumers.map(async (q) => q.stop())); + await this.consumerPool.stop(); this.heartbeat.stop(); this.runNotificationsSocket?.disconnect(); } From 0b2b73fc5211f698352be333589471f9262242f6 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 2 Sep 2025 14:10:01 +0100 Subject: [PATCH 02/13] chore(docs): improve version pinning advise to use installed trigger.dev CLI (#2466) --- docs/github-actions.mdx | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/github-actions.mdx b/docs/github-actions.mdx index 9fba762fa..0fe0788cc 100644 --- a/docs/github-actions.mdx +++ b/docs/github-actions.mdx @@ -102,13 +102,16 @@ If you already have a GitHub action file, you can just add the final step "🚀 ## CLI Version pinning The CLI and `@trigger.dev/*` package versions need to be in sync with the `trigger.dev` CLI, otherwise there will be errors and unpredictable behavior. Hence, the `deploy` command will automatically fail during CI on any version mismatches. -Tip: add the deploy command to your `package.json` file to keep versions managed in the same place. For example: +Tip: add the `trigger.dev` CLI to your `devDependencies` and the deploy command to your `package.json` file to keep versions managed in the same place. For example: ```json { "scripts": { - "deploy:trigger-prod": "npx trigger.dev@3.0.0 deploy", - "deploy:trigger": "npx trigger.dev@3.0.0 deploy --env staging" + "deploy:trigger-prod": "trigger deploy", + "deploy:trigger": "trigger deploy --env staging" + }, + "devDependencies": { + "trigger.dev": "4.0.2" } } ``` @@ -134,9 +137,7 @@ When self-hosting, you will have to take a few additional steps: - Add your registry credentials to the GitHub secrets. - Use the `--self-hosted` and `--push` flags when deploying. - - If you're self-hosting v4, the `--self-hosted` and `--push` flags are **NOT** needed. - +If you're self-hosting v4, the `--self-hosted` and `--push` flags are **NOT** needed. Other than that, your GitHub action file will look very similar to the one above: From 9b1877bef25695a9d5c8b68e33ca5975f9b3206c Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 2 Sep 2025 14:12:14 +0100 Subject: [PATCH 03/13] fix(run-engine): retry non-zero exit code errors (#2467) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We’re also now saving the retryConfig from the BackgroundWorkerTask on TaskRun.lockedRetryConfig when the run is first locked to the version --- .../migration.sql | 2 + .../prisma/migrations/migration_lock.toml | 2 +- .../database/prisma/schema.prisma | 13 ++-- .../run-engine/src/engine/retrying.ts | 62 +++++++++++++++---- .../src/engine/systems/dequeueSystem.ts | 4 ++ packages/core/src/v3/errors.ts | 26 ++++++++ 6 files changed, 90 insertions(+), 19 deletions(-) create mode 100644 internal-packages/database/prisma/migrations/20250902112516_add_locked_retry_config_to_task_run/migration.sql diff --git a/internal-packages/database/prisma/migrations/20250902112516_add_locked_retry_config_to_task_run/migration.sql b/internal-packages/database/prisma/migrations/20250902112516_add_locked_retry_config_to_task_run/migration.sql new file mode 100644 index 000000000..22cde96c6 --- /dev/null +++ b/internal-packages/database/prisma/migrations/20250902112516_add_locked_retry_config_to_task_run/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "public"."TaskRun" ADD COLUMN "lockedRetryConfig" JSONB; diff --git a/internal-packages/database/prisma/migrations/migration_lock.toml b/internal-packages/database/prisma/migrations/migration_lock.toml index fbffa92c2..99e4f2009 100644 --- a/internal-packages/database/prisma/migrations/migration_lock.toml +++ b/internal-packages/database/prisma/migrations/migration_lock.toml @@ -1,3 +1,3 @@ # Please do not edit this file manually # It should be added in your version-control system (i.e. Git) -provider = "postgresql" \ No newline at end of file +provider = "postgresql" diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index ba5a8de26..be10d47f7 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -151,7 +151,7 @@ model OrganizationAccessToken { /// This is used to find the token in the database hashedToken String @unique - organization Organization @relation(fields: [organizationId], references: [id]) + organization Organization @relation(fields: [organizationId], references: [id]) organizationId String /// Optional expiration date for the token @@ -648,11 +648,12 @@ model TaskRun { concurrencyKey String? - delayUntil DateTime? - queuedAt DateTime? - ttl String? - expiredAt DateTime? - maxAttempts Int? + delayUntil DateTime? + queuedAt DateTime? + ttl String? + expiredAt DateTime? + maxAttempts Int? + lockedRetryConfig Json? /// optional token that can be used to authenticate the task run oneTimeUseToken String? diff --git a/internal-packages/run-engine/src/engine/retrying.ts b/internal-packages/run-engine/src/engine/retrying.ts index a621552e9..8001c5d8a 100644 --- a/internal-packages/run-engine/src/engine/retrying.ts +++ b/internal-packages/run-engine/src/engine/retrying.ts @@ -3,6 +3,7 @@ import { isOOMRunError, RetryOptions, sanitizeError, + shouldLookupRetrySettings, shouldRetryError, TaskRunError, taskRunErrorEnhancer, @@ -72,13 +73,11 @@ export async function retryOutcomeFromCompletion( }; } - // No retry settings - if (!retrySettings) { - return { outcome: "fail_run", sanitizedError }; - } + const enhancedError = taskRunErrorEnhancer(error); // Not a retriable error: fail - const retriableError = shouldRetryError(taskRunErrorEnhancer(error)); + const retriableError = shouldRetryError(enhancedError); + if (!retriableError) { return { outcome: "fail_run", sanitizedError }; } @@ -95,6 +94,7 @@ export async function retryOutcomeFromCompletion( }, select: { maxAttempts: true, + lockedRetryConfig: true, }, }); @@ -112,6 +112,48 @@ export async function retryOutcomeFromCompletion( return { outcome: "fail_run", sanitizedError }; } + // No retry settings + if (!retrySettings) { + const shouldLookup = shouldLookupRetrySettings(enhancedError); + + if (!shouldLookup) { + return { outcome: "fail_run", sanitizedError }; + } + + const retryConfig = run.lockedRetryConfig; + + if (!retryConfig) { + return { outcome: "fail_run", sanitizedError }; + } + + const parsedRetryConfig = RetryOptions.nullish().safeParse(retryConfig); + + if (!parsedRetryConfig.success) { + return { outcome: "fail_run", sanitizedError }; + } + + if (!parsedRetryConfig.data) { + return { outcome: "fail_run", sanitizedError }; + } + + const nextDelay = calculateNextRetryDelay(parsedRetryConfig.data, attemptNumber ?? 1); + + if (!nextDelay) { + return { outcome: "fail_run", sanitizedError }; + } + + const retrySettings = { + timestamp: Date.now() + nextDelay, + delay: nextDelay, + }; + + return { + outcome: "retry", + method: "queue", // we'll always retry on the queue because usually having no settings means something bad happened + settings: retrySettings, + }; + } + return { outcome: "retry", method: retryUsingQueue ? "queue" : "immediate", @@ -130,19 +172,15 @@ async function retryOOMOnMachine( }, select: { machinePreset: true, - lockedBy: { - select: { - retryConfig: true, - }, - }, + lockedRetryConfig: true, }, }); - if (!run || !run.lockedBy || !run.machinePreset) { + if (!run || !run.lockedRetryConfig || !run.machinePreset) { return; } - const retryConfig = run.lockedBy?.retryConfig; + const retryConfig = run.lockedRetryConfig; const parsedRetryConfig = RetryOptions.nullish().safeParse(retryConfig); if (!parsedRetryConfig.success) { diff --git a/internal-packages/run-engine/src/engine/systems/dequeueSystem.ts b/internal-packages/run-engine/src/engine/systems/dequeueSystem.ts index 6102898bc..a19c80dd7 100644 --- a/internal-packages/run-engine/src/engine/systems/dequeueSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/dequeueSystem.ts @@ -403,6 +403,9 @@ export class DequeueSystem { result.run.maxDurationInSeconds, result.task.maxDurationInSeconds ); + const lockedRetryConfig = result.run.lockedRetryConfig + ? undefined + : result.task.retryConfig; const lockedTaskRun = await prisma.taskRun.update({ where: { @@ -413,6 +416,7 @@ export class DequeueSystem { lockedById: result.task.id, lockedToVersionId: result.worker.id, lockedQueueId: result.queue.id, + lockedRetryConfig: lockedRetryConfig ?? undefined, status: "DEQUEUED", startedAt, baseCostInCents: this.options.machines.baseCostInCents, diff --git a/packages/core/src/v3/errors.ts b/packages/core/src/v3/errors.ts index b585b938c..35cbc3ea4 100644 --- a/packages/core/src/v3/errors.ts +++ b/packages/core/src/v3/errors.ts @@ -346,6 +346,32 @@ export function shouldRetryError(error: TaskRunError): boolean { } } +export function shouldLookupRetrySettings(error: TaskRunError): boolean { + switch (error.type) { + case "INTERNAL_ERROR": { + switch (error.code) { + case "TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE": + return true; + + default: + return false; + } + } + case "STRING_ERROR": { + return false; + } + case "BUILT_IN_ERROR": { + return false; + } + case "CUSTOM_ERROR": { + return false; + } + default: { + assertExhaustive(error); + } + } +} + export function correctErrorStackTrace( stackTrace: string, projectDir?: string, From cf9398b56ef647cda656145c4dc874d196d138f3 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Tue, 2 Sep 2025 15:18:16 +0100 Subject: [PATCH 04/13] fix(run-engine): retry SIGTERM errors (#2468) --- packages/core/src/v3/errors.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/src/v3/errors.ts b/packages/core/src/v3/errors.ts index 35cbc3ea4..6cf900d7b 100644 --- a/packages/core/src/v3/errors.ts +++ b/packages/core/src/v3/errors.ts @@ -351,6 +351,7 @@ export function shouldLookupRetrySettings(error: TaskRunError): boolean { case "INTERNAL_ERROR": { switch (error.code) { case "TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE": + case "TASK_PROCESS_SIGTERM": return true; default: From 436d951b65f1ae4335afdfaa8a3f613955892a72 Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Tue, 2 Sep 2025 16:35:33 +0200 Subject: [PATCH 05/13] feat(webapp): github app installation flow (#2463) * Add schemas for gh app installations * Implement gh app installation flow * Make the gh app configs optional * Add additional org check on gh app installation callback * Save account handle and repo default branch on install * Do repo hard deletes in favor of simplicity * Disable github app by default * Fix gh env schema union issue * Use octokit's iterator for paginating repos * Parse gh app install callback with a discriminated union * Remove duplicate env vars * Use bigint for github integer IDs * Sanitize redirect paths in the gh installation and auth flow * Regenerate migration after rebase on main to fix ordering * Handle gh install updates separately from new installs --- apps/webapp/app/env.server.ts | 2262 +++++++++-------- .../app/routes/_app.github.callback/route.tsx | 121 + .../app/routes/_app.github.install/route.tsx | 52 + .../app/routes/auth.github.callback.tsx | 3 +- apps/webapp/app/routes/auth.github.ts | 4 +- apps/webapp/app/services/gitHub.server.ts | 135 + .../app/services/gitHubSession.server.ts | 124 + apps/webapp/app/utils.ts | 30 +- apps/webapp/package.json | 1 + .../migration.sql | 46 + .../prisma/migrations/migration_lock.toml | 2 +- .../database/prisma/schema.prisma | 50 + pnpm-lock.yaml | 398 ++- 13 files changed, 2114 insertions(+), 1114 deletions(-) create mode 100644 apps/webapp/app/routes/_app.github.callback/route.tsx create mode 100644 apps/webapp/app/routes/_app.github.install/route.tsx create mode 100644 apps/webapp/app/services/gitHub.server.ts create mode 100644 apps/webapp/app/services/gitHubSession.server.ts create mode 100644 internal-packages/database/prisma/migrations/20250902133449_add_gh_installation_schema/migration.sql diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index f20c02f6a..350463a5b 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -3,1100 +3,1174 @@ import { BoolEnv } from "./utils/boolEnv"; import { isValidDatabaseUrl } from "./utils/db"; import { isValidRegex } from "./utils/regex"; -const EnvironmentSchema = z.object({ - NODE_ENV: z.union([z.literal("development"), z.literal("production"), z.literal("test")]), - DATABASE_URL: z - .string() - .refine( - isValidDatabaseUrl, - "DATABASE_URL is invalid, for details please check the additional output above this message." - ), - DATABASE_CONNECTION_LIMIT: z.coerce.number().int().default(10), - DATABASE_POOL_TIMEOUT: z.coerce.number().int().default(60), - DATABASE_CONNECTION_TIMEOUT: z.coerce.number().int().default(20), - DIRECT_URL: z - .string() - .refine( - isValidDatabaseUrl, - "DIRECT_URL is invalid, for details please check the additional output above this message." - ), - DATABASE_READ_REPLICA_URL: z.string().optional(), - SESSION_SECRET: z.string(), - MAGIC_LINK_SECRET: z.string(), - ENCRYPTION_KEY: z - .string() - .refine( - (val) => Buffer.from(val, "utf8").length === 32, - "ENCRYPTION_KEY must be exactly 32 bytes" - ), - WHITELISTED_EMAILS: z - .string() - .refine(isValidRegex, "WHITELISTED_EMAILS must be a valid regex.") - .optional(), - ADMIN_EMAILS: z.string().refine(isValidRegex, "ADMIN_EMAILS must be a valid regex.").optional(), - REMIX_APP_PORT: z.string().optional(), - LOGIN_ORIGIN: z.string().default("http://localhost:3030"), - APP_ORIGIN: z.string().default("http://localhost:3030"), - API_ORIGIN: z.string().optional(), - STREAM_ORIGIN: z.string().optional(), - ELECTRIC_ORIGIN: z.string().default("http://localhost:3060"), - // A comma separated list of electric origins to shard into different electric instances by environmentId - // example: "http://localhost:3060,http://localhost:3061,http://localhost:3062" - ELECTRIC_ORIGIN_SHARDS: z.string().optional(), - APP_ENV: z.string().default(process.env.NODE_ENV), - SERVICE_NAME: z.string().default("trigger.dev webapp"), - POSTHOG_PROJECT_KEY: z.string().default("phc_LFH7kJiGhdIlnO22hTAKgHpaKhpM8gkzWAFvHmf5vfS"), - TRIGGER_TELEMETRY_DISABLED: z.string().optional(), - AUTH_GITHUB_CLIENT_ID: z.string().optional(), - AUTH_GITHUB_CLIENT_SECRET: z.string().optional(), - EMAIL_TRANSPORT: z.enum(["resend", "smtp", "aws-ses"]).optional(), - FROM_EMAIL: z.string().optional(), - REPLY_TO_EMAIL: z.string().optional(), - RESEND_API_KEY: z.string().optional(), - SMTP_HOST: z.string().optional(), - SMTP_PORT: z.coerce.number().optional(), - SMTP_SECURE: BoolEnv.optional(), - SMTP_USER: z.string().optional(), - SMTP_PASSWORD: z.string().optional(), - - PLAIN_API_KEY: z.string().optional(), - WORKER_SCHEMA: z.string().default("graphile_worker"), - WORKER_CONCURRENCY: z.coerce.number().int().default(10), - WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000), - WORKER_ENABLED: z.string().default("true"), - GRACEFUL_SHUTDOWN_TIMEOUT: z.coerce.number().int().default(60000), - DISABLE_SSE: z.string().optional(), - OPENAI_API_KEY: z.string().optional(), - - // Redis options - REDIS_HOST: z.string().optional(), - REDIS_READER_HOST: z.string().optional(), - REDIS_READER_PORT: z.coerce.number().optional(), - REDIS_PORT: z.coerce.number().optional(), - REDIS_USERNAME: z.string().optional(), - REDIS_PASSWORD: z.string().optional(), - REDIS_TLS_DISABLED: z.string().optional(), - - RATE_LIMIT_REDIS_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_HOST), - RATE_LIMIT_REDIS_READER_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_READER_HOST), - RATE_LIMIT_REDIS_READER_PORT: z.coerce - .number() - .optional() - .transform( - (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) - ), - RATE_LIMIT_REDIS_PORT: z.coerce - .number() - .optional() - .transform((v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)), - RATE_LIMIT_REDIS_USERNAME: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_USERNAME), - RATE_LIMIT_REDIS_PASSWORD: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_PASSWORD), - RATE_LIMIT_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"), - RATE_LIMIT_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), - - CACHE_REDIS_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_HOST), - CACHE_REDIS_READER_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_READER_HOST), - CACHE_REDIS_READER_PORT: z.coerce - .number() - .optional() - .transform( - (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) - ), - CACHE_REDIS_PORT: z.coerce - .number() - .optional() - .transform((v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)), - CACHE_REDIS_USERNAME: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_USERNAME), - CACHE_REDIS_PASSWORD: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_PASSWORD), - CACHE_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"), - CACHE_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), - - REALTIME_STREAMS_REDIS_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_HOST), - REALTIME_STREAMS_REDIS_READER_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_READER_HOST), - REALTIME_STREAMS_REDIS_READER_PORT: z.coerce - .number() - .optional() - .transform( - (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) - ), - REALTIME_STREAMS_REDIS_PORT: z.coerce - .number() - .optional() - .transform((v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)), - REALTIME_STREAMS_REDIS_USERNAME: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_USERNAME), - REALTIME_STREAMS_REDIS_PASSWORD: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_PASSWORD), - REALTIME_STREAMS_REDIS_TLS_DISABLED: z - .string() - .default(process.env.REDIS_TLS_DISABLED ?? "false"), - REALTIME_STREAMS_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), - - REALTIME_MAXIMUM_CREATED_AT_FILTER_AGE_IN_MS: z.coerce - .number() - .int() - .default(24 * 60 * 60 * 1000), // 1 day in milliseconds - - PUBSUB_REDIS_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_HOST), - PUBSUB_REDIS_READER_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_READER_HOST), - PUBSUB_REDIS_READER_PORT: z.coerce - .number() - .optional() - .transform( - (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) - ), - PUBSUB_REDIS_PORT: z.coerce - .number() - .optional() - .transform((v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)), - PUBSUB_REDIS_USERNAME: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_USERNAME), - PUBSUB_REDIS_PASSWORD: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_PASSWORD), - PUBSUB_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"), - PUBSUB_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), - - DEFAULT_ENV_EXECUTION_CONCURRENCY_LIMIT: z.coerce.number().int().default(100), - DEFAULT_ENV_EXECUTION_CONCURRENCY_BURST_FACTOR: z.coerce.number().default(1.0), - DEFAULT_ORG_EXECUTION_CONCURRENCY_LIMIT: z.coerce.number().int().default(300), - DEFAULT_DEV_ENV_EXECUTION_ATTEMPTS: z.coerce.number().int().positive().default(1), - - //API Rate limiting - /** - * @example "60s" - * @example "1m" - * @example "1h" - * @example "1d" - * @example "1000ms" - * @example "1000s" - */ - API_RATE_LIMIT_REFILL_INTERVAL: z.string().default("10s"), // refill 250 tokens every 10 seconds - API_RATE_LIMIT_MAX: z.coerce.number().int().default(750), // allow bursts of 750 requests - API_RATE_LIMIT_REFILL_RATE: z.coerce.number().int().default(250), // refix 250 tokens every 10 seconds - API_RATE_LIMIT_REQUEST_LOGS_ENABLED: z.string().default("0"), - API_RATE_LIMIT_REJECTION_LOGS_ENABLED: z.string().default("1"), - API_RATE_LIMIT_LIMITER_LOGS_ENABLED: z.string().default("0"), - - API_RATE_LIMIT_JWT_WINDOW: z.string().default("1m"), - API_RATE_LIMIT_JWT_TOKENS: z.coerce.number().int().default(60), - - //v3 - PROVIDER_SECRET: z.string().default("provider-secret"), - COORDINATOR_SECRET: z.string().default("coordinator-secret"), - DEPOT_TOKEN: z.string().optional(), - DEPOT_ORG_ID: z.string().optional(), - DEPOT_REGION: z.string().default("us-east-1"), - - // Deployment registry (v3) - DEPLOY_REGISTRY_HOST: z.string().min(1), - DEPLOY_REGISTRY_USERNAME: z.string().optional(), - DEPLOY_REGISTRY_PASSWORD: z.string().optional(), - DEPLOY_REGISTRY_NAMESPACE: z.string().min(1).default("trigger"), - DEPLOY_REGISTRY_ECR_TAGS: z.string().optional(), // csv, for example: "key1=value1,key2=value2" - DEPLOY_REGISTRY_ECR_ASSUME_ROLE_ARN: z.string().optional(), - DEPLOY_REGISTRY_ECR_ASSUME_ROLE_EXTERNAL_ID: z.string().optional(), - - // Deployment registry (v4) - falls back to v3 registry if not specified - V4_DEPLOY_REGISTRY_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.DEPLOY_REGISTRY_HOST) - .pipe(z.string().min(1)), // Ensure final type is required string - V4_DEPLOY_REGISTRY_USERNAME: z - .string() - .optional() - .transform((v) => v ?? process.env.DEPLOY_REGISTRY_USERNAME), - V4_DEPLOY_REGISTRY_PASSWORD: z - .string() - .optional() - .transform((v) => v ?? process.env.DEPLOY_REGISTRY_PASSWORD), - V4_DEPLOY_REGISTRY_NAMESPACE: z - .string() - .optional() - .transform((v) => v ?? process.env.DEPLOY_REGISTRY_NAMESPACE) - .pipe(z.string().min(1).default("trigger")), // Ensure final type is required string - V4_DEPLOY_REGISTRY_ECR_TAGS: z - .string() - .optional() - .transform((v) => v ?? process.env.DEPLOY_REGISTRY_ECR_TAGS), - V4_DEPLOY_REGISTRY_ECR_ASSUME_ROLE_ARN: z - .string() - .optional() - .transform((v) => v ?? process.env.DEPLOY_REGISTRY_ECR_ASSUME_ROLE_ARN), - V4_DEPLOY_REGISTRY_ECR_ASSUME_ROLE_EXTERNAL_ID: z - .string() - .optional() - .transform((v) => v ?? process.env.DEPLOY_REGISTRY_ECR_ASSUME_ROLE_EXTERNAL_ID), - - DEPLOY_IMAGE_PLATFORM: z.string().default("linux/amd64"), - DEPLOY_TIMEOUT_MS: z.coerce - .number() - .int() - .default(60 * 1000 * 8), // 8 minutes - - OBJECT_STORE_BASE_URL: z.string().optional(), - OBJECT_STORE_ACCESS_KEY_ID: z.string().optional(), - OBJECT_STORE_SECRET_ACCESS_KEY: z.string().optional(), - OBJECT_STORE_REGION: z.string().optional(), - OBJECT_STORE_SERVICE: z.string().default("s3"), - EVENTS_BATCH_SIZE: z.coerce.number().int().default(100), - EVENTS_BATCH_INTERVAL: z.coerce.number().int().default(1000), - EVENTS_DEFAULT_LOG_RETENTION: z.coerce.number().int().default(7), - EVENTS_MIN_CONCURRENCY: z.coerce.number().int().default(1), - EVENTS_MAX_CONCURRENCY: z.coerce.number().int().default(10), - EVENTS_MAX_BATCH_SIZE: z.coerce.number().int().default(500), - EVENTS_MEMORY_PRESSURE_THRESHOLD: z.coerce.number().int().default(5000), - EVENTS_LOAD_SHEDDING_THRESHOLD: z.coerce.number().int().default(100000), - EVENTS_LOAD_SHEDDING_ENABLED: z.string().default("1"), - SHARED_QUEUE_CONSUMER_POOL_SIZE: z.coerce.number().int().default(10), - SHARED_QUEUE_CONSUMER_INTERVAL_MS: z.coerce.number().int().default(100), - SHARED_QUEUE_CONSUMER_NEXT_TICK_INTERVAL_MS: z.coerce.number().int().default(100), - SHARED_QUEUE_CONSUMER_EMIT_RESUME_DEPENDENCY_TIMEOUT_MS: z.coerce.number().int().default(1000), - SHARED_QUEUE_CONSUMER_RESOLVE_PAYLOADS_BATCH_SIZE: z.coerce.number().int().default(25), - - MANAGED_WORKER_SECRET: z.string().default("managed-secret"), - - // Development OTEL environment variables - DEV_OTEL_EXPORTER_OTLP_ENDPOINT: z.string().optional(), - // If this is set to 1, then the below variables are used to configure the batch processor for spans and logs - DEV_OTEL_BATCH_PROCESSING_ENABLED: z.string().default("0"), - DEV_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE: z.string().default("64"), - DEV_OTEL_SPAN_SCHEDULED_DELAY_MILLIS: z.string().default("200"), - DEV_OTEL_SPAN_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"), - DEV_OTEL_SPAN_MAX_QUEUE_SIZE: z.string().default("512"), - DEV_OTEL_LOG_MAX_EXPORT_BATCH_SIZE: z.string().default("64"), - DEV_OTEL_LOG_SCHEDULED_DELAY_MILLIS: z.string().default("200"), - DEV_OTEL_LOG_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"), - DEV_OTEL_LOG_MAX_QUEUE_SIZE: z.string().default("512"), - - PROD_OTEL_BATCH_PROCESSING_ENABLED: z.string().default("0"), - PROD_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE: z.string().default("64"), - PROD_OTEL_SPAN_SCHEDULED_DELAY_MILLIS: z.string().default("200"), - PROD_OTEL_SPAN_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"), - PROD_OTEL_SPAN_MAX_QUEUE_SIZE: z.string().default("512"), - PROD_OTEL_LOG_MAX_EXPORT_BATCH_SIZE: z.string().default("64"), - PROD_OTEL_LOG_SCHEDULED_DELAY_MILLIS: z.string().default("200"), - PROD_OTEL_LOG_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"), - PROD_OTEL_LOG_MAX_QUEUE_SIZE: z.string().default("512"), - - TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT: z.string().default("1024"), - TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT: z.string().default("1024"), - TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT: z.string().default("131072"), - TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT: z.string().default("131072"), - TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT: z.string().default("10"), - TRIGGER_OTEL_LINK_COUNT_LIMIT: z.string().default("2"), - TRIGGER_OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT: z.string().default("10"), - TRIGGER_OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT: z.string().default("10"), - - CHECKPOINT_THRESHOLD_IN_MS: z.coerce.number().int().default(30000), - - // Internal OTEL environment variables - INTERNAL_OTEL_TRACE_EXPORTER_URL: z.string().optional(), - INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADERS: z.string().optional(), - INTERNAL_OTEL_TRACE_LOGGING_ENABLED: z.string().default("1"), - // this means 1/20 traces or 5% of traces will be sampled (sampled = recorded) - INTERNAL_OTEL_TRACE_SAMPLING_RATE: z.string().default("20"), - INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED: z.string().default("0"), - INTERNAL_OTEL_TRACE_DISABLED: z.string().default("0"), - - INTERNAL_OTEL_LOG_EXPORTER_URL: z.string().optional(), - INTERNAL_OTEL_METRIC_EXPORTER_URL: z.string().optional(), - INTERNAL_OTEL_METRIC_EXPORTER_AUTH_HEADERS: z.string().optional(), - INTERNAL_OTEL_METRIC_EXPORTER_ENABLED: z.string().default("0"), - INTERNAL_OTEL_METRIC_EXPORTER_INTERVAL_MS: z.coerce.number().int().default(30_000), - INTERNAL_OTEL_HOST_METRICS_ENABLED: BoolEnv.default(true), - INTERNAL_OTEL_NODEJS_METRICS_ENABLED: BoolEnv.default(true), - INTERNAL_OTEL_ADDITIONAL_DETECTORS_ENABLED: BoolEnv.default(true), - - ORG_SLACK_INTEGRATION_CLIENT_ID: z.string().optional(), - ORG_SLACK_INTEGRATION_CLIENT_SECRET: z.string().optional(), - - /** These enable the alerts feature in v3 */ - ALERT_EMAIL_TRANSPORT: z.enum(["resend", "smtp", "aws-ses"]).optional(), - ALERT_FROM_EMAIL: z.string().optional(), - ALERT_REPLY_TO_EMAIL: z.string().optional(), - ALERT_RESEND_API_KEY: z.string().optional(), - ALERT_SMTP_HOST: z.string().optional(), - ALERT_SMTP_PORT: z.coerce.number().optional(), - ALERT_SMTP_SECURE: BoolEnv.optional(), - ALERT_SMTP_USER: z.string().optional(), - ALERT_SMTP_PASSWORD: z.string().optional(), - ALERT_RATE_LIMITER_EMISSION_INTERVAL: z.coerce.number().int().default(2_500), - ALERT_RATE_LIMITER_BURST_TOLERANCE: z.coerce.number().int().default(10_000), - ALERT_RATE_LIMITER_REDIS_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_HOST), - ALERT_RATE_LIMITER_REDIS_READER_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_READER_HOST), - ALERT_RATE_LIMITER_REDIS_READER_PORT: z.coerce - .number() - .optional() - .transform( - (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) - ), - ALERT_RATE_LIMITER_REDIS_PORT: z.coerce - .number() - .optional() - .transform((v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)), - ALERT_RATE_LIMITER_REDIS_USERNAME: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_USERNAME), - ALERT_RATE_LIMITER_REDIS_PASSWORD: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_PASSWORD), - ALERT_RATE_LIMITER_REDIS_TLS_DISABLED: z - .string() - .default(process.env.REDIS_TLS_DISABLED ?? "false"), - ALERT_RATE_LIMITER_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), - - LOOPS_API_KEY: z.string().optional(), - MARQS_DISABLE_REBALANCING: BoolEnv.default(false), - MARQS_VISIBILITY_TIMEOUT_MS: z.coerce - .number() - .int() - .default(60 * 1000 * 15), - MARQS_SHARED_QUEUE_LIMIT: z.coerce.number().int().default(1000), - MARQS_MAXIMUM_QUEUE_PER_ENV_COUNT: z.coerce.number().int().default(50), - MARQS_DEV_QUEUE_LIMIT: z.coerce.number().int().default(1000), - MARQS_MAXIMUM_NACK_COUNT: z.coerce.number().int().default(64), - MARQS_CONCURRENCY_LIMIT_BIAS: z.coerce.number().default(0.75), - MARQS_AVAILABLE_CAPACITY_BIAS: z.coerce.number().default(0.3), - MARQS_QUEUE_AGE_RANDOMIZATION_BIAS: z.coerce.number().default(0.25), - MARQS_REUSE_SNAPSHOT_COUNT: z.coerce.number().int().default(0), - MARQS_MAXIMUM_ENV_COUNT: z.coerce.number().int().optional(), - MARQS_SHARED_WORKER_QUEUE_CONSUMER_INTERVAL_MS: z.coerce.number().int().default(250), - MARQS_SHARED_WORKER_QUEUE_MAX_MESSAGE_COUNT: z.coerce.number().int().default(10), - - MARQS_SHARED_WORKER_QUEUE_EAGER_DEQUEUE_ENABLED: z.string().default("0"), - MARQS_WORKER_ENABLED: z.string().default("0"), - MARQS_WORKER_COUNT: z.coerce.number().int().default(2), - MARQS_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(50), - MARQS_WORKER_CONCURRENCY_TASKS_PER_WORKER: z.coerce.number().int().default(5), - MARQS_WORKER_POLL_INTERVAL_MS: z.coerce.number().int().default(100), - MARQS_WORKER_IMMEDIATE_POLL_INTERVAL_MS: z.coerce.number().int().default(100), - MARQS_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000), - MARQS_SHARED_WORKER_QUEUE_COOLOFF_COUNT_THRESHOLD: z.coerce.number().int().default(10), - MARQS_SHARED_WORKER_QUEUE_COOLOFF_PERIOD_MS: z.coerce.number().int().default(5_000), - - PROD_TASK_HEARTBEAT_INTERVAL_MS: z.coerce.number().int().optional(), - - VERBOSE_GRAPHILE_LOGGING: z.string().default("false"), - V2_MARQS_ENABLED: z.string().default("0"), - V2_MARQS_CONSUMER_POOL_ENABLED: z.string().default("0"), - V2_MARQS_CONSUMER_POOL_SIZE: z.coerce.number().int().default(10), - V2_MARQS_CONSUMER_POLL_INTERVAL_MS: z.coerce.number().int().default(1000), - V2_MARQS_QUEUE_SELECTION_COUNT: z.coerce.number().int().default(36), - V2_MARQS_VISIBILITY_TIMEOUT_MS: z.coerce - .number() - .int() - .default(60 * 1000 * 15), - V2_MARQS_DEFAULT_ENV_CONCURRENCY: z.coerce.number().int().default(100), - V2_MARQS_VERBOSE: z.string().default("0"), - V3_MARQS_CONCURRENCY_MONITOR_ENABLED: z.string().default("0"), - V2_MARQS_CONCURRENCY_MONITOR_ENABLED: z.string().default("0"), - /* Usage settings */ - USAGE_EVENT_URL: z.string().optional(), - PROD_USAGE_HEARTBEAT_INTERVAL_MS: z.coerce.number().int().optional(), - - CENTS_PER_RUN: z.coerce.number().default(0), - - EVENT_LOOP_MONITOR_ENABLED: z.string().default("1"), - MAXIMUM_LIVE_RELOADING_EVENTS: z.coerce.number().int().default(1000), - MAXIMUM_TRACE_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(25_000), - MAXIMUM_TRACE_DETAILED_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(10_000), - TASK_PAYLOAD_OFFLOAD_THRESHOLD: z.coerce.number().int().default(524_288), // 512KB - TASK_PAYLOAD_MAXIMUM_SIZE: z.coerce.number().int().default(3_145_728), // 3MB - BATCH_TASK_PAYLOAD_MAXIMUM_SIZE: z.coerce.number().int().default(1_000_000), // 1MB - TASK_RUN_METADATA_MAXIMUM_SIZE: z.coerce.number().int().default(262_144), // 256KB - - MAXIMUM_DEV_QUEUE_SIZE: z.coerce.number().int().optional(), - MAXIMUM_DEPLOYED_QUEUE_SIZE: z.coerce.number().int().optional(), - MAX_BATCH_V2_TRIGGER_ITEMS: z.coerce.number().int().default(500), - MAX_BATCH_AND_WAIT_V2_TRIGGER_ITEMS: z.coerce.number().int().default(500), - - REALTIME_STREAM_VERSION: z.enum(["v1", "v2"]).default("v1"), - REALTIME_STREAM_MAX_LENGTH: z.coerce.number().int().default(1000), - REALTIME_STREAM_TTL: z.coerce - .number() - .int() - .default(60 * 60 * 24), // 1 day in seconds - BATCH_METADATA_OPERATIONS_FLUSH_INTERVAL_MS: z.coerce.number().int().default(1000), - BATCH_METADATA_OPERATIONS_FLUSH_ENABLED: z.string().default("1"), - BATCH_METADATA_OPERATIONS_FLUSH_LOGGING_ENABLED: z.string().default("1"), - - // Run Engine 2.0 - RUN_ENGINE_WORKER_COUNT: z.coerce.number().int().default(4), - RUN_ENGINE_TASKS_PER_WORKER: z.coerce.number().int().default(10), - RUN_ENGINE_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(10), - RUN_ENGINE_WORKER_POLL_INTERVAL: z.coerce.number().int().default(100), - RUN_ENGINE_WORKER_IMMEDIATE_POLL_INTERVAL: z.coerce.number().int().default(100), - RUN_ENGINE_TIMEOUT_PENDING_EXECUTING: z.coerce.number().int().default(60_000), - RUN_ENGINE_TIMEOUT_PENDING_CANCEL: z.coerce.number().int().default(60_000), - RUN_ENGINE_TIMEOUT_EXECUTING: z.coerce.number().int().default(60_000), - RUN_ENGINE_TIMEOUT_EXECUTING_WITH_WAITPOINTS: z.coerce.number().int().default(60_000), - RUN_ENGINE_TIMEOUT_SUSPENDED: z.coerce - .number() - .int() - .default(60_000 * 10), - RUN_ENGINE_DEBUG_WORKER_NOTIFICATIONS: BoolEnv.default(false), - RUN_ENGINE_PARENT_QUEUE_LIMIT: z.coerce.number().int().default(1000), - RUN_ENGINE_CONCURRENCY_LIMIT_BIAS: z.coerce.number().default(0.75), - RUN_ENGINE_AVAILABLE_CAPACITY_BIAS: z.coerce.number().default(0.3), - RUN_ENGINE_QUEUE_AGE_RANDOMIZATION_BIAS: z.coerce.number().default(0.25), - RUN_ENGINE_REUSE_SNAPSHOT_COUNT: z.coerce.number().int().default(0), - RUN_ENGINE_MAXIMUM_ENV_COUNT: z.coerce.number().int().optional(), - RUN_ENGINE_RUN_QUEUE_SHARD_COUNT: z.coerce.number().int().default(4), - RUN_ENGINE_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000), - RUN_ENGINE_RETRY_WARM_START_THRESHOLD_MS: z.coerce.number().int().default(30_000), - RUN_ENGINE_PROCESS_WORKER_QUEUE_DEBOUNCE_MS: z.coerce.number().int().default(200), - RUN_ENGINE_DEQUEUE_BLOCKING_TIMEOUT_SECONDS: z.coerce.number().int().default(10), - RUN_ENGINE_MASTER_QUEUE_CONSUMERS_INTERVAL_MS: z.coerce.number().int().default(1000), - RUN_ENGINE_MASTER_QUEUE_COOLOFF_PERIOD_MS: z.coerce.number().int().default(10_000), - RUN_ENGINE_MASTER_QUEUE_COOLOFF_COUNT_THRESHOLD: z.coerce.number().int().default(10), - RUN_ENGINE_MASTER_QUEUE_CONSUMER_DEQUEUE_COUNT: z.coerce.number().int().default(10), - RUN_ENGINE_CONCURRENCY_SWEEPER_SCAN_SCHEDULE: z.string().optional(), - RUN_ENGINE_CONCURRENCY_SWEEPER_PROCESS_MARKED_SCHEDULE: z.string().optional(), - RUN_ENGINE_CONCURRENCY_SWEEPER_SCAN_JITTER_IN_MS: z.coerce.number().int().optional(), - RUN_ENGINE_CONCURRENCY_SWEEPER_PROCESS_MARKED_JITTER_IN_MS: z.coerce.number().int().optional(), - - RUN_ENGINE_RUN_LOCK_DURATION: z.coerce.number().int().default(5000), - RUN_ENGINE_RUN_LOCK_AUTOMATIC_EXTENSION_THRESHOLD: z.coerce.number().int().default(1000), - RUN_ENGINE_RUN_LOCK_MAX_RETRIES: z.coerce.number().int().default(10), - RUN_ENGINE_RUN_LOCK_BASE_DELAY: z.coerce.number().int().default(100), - RUN_ENGINE_RUN_LOCK_MAX_DELAY: z.coerce.number().int().default(3000), - RUN_ENGINE_RUN_LOCK_BACKOFF_MULTIPLIER: z.coerce.number().default(1.8), - RUN_ENGINE_RUN_LOCK_JITTER_FACTOR: z.coerce.number().default(0.15), - RUN_ENGINE_RUN_LOCK_MAX_TOTAL_WAIT_TIME: z.coerce.number().int().default(15000), - - RUN_ENGINE_SUSPENDED_HEARTBEAT_RETRIES_MAX_COUNT: z.coerce.number().int().default(12), - RUN_ENGINE_SUSPENDED_HEARTBEAT_RETRIES_MAX_DELAY_MS: z.coerce - .number() - .int() - .default(60_000 * 60 * 6), - RUN_ENGINE_SUSPENDED_HEARTBEAT_RETRIES_INITIAL_DELAY_MS: z.coerce.number().int().default(60_000), - RUN_ENGINE_SUSPENDED_HEARTBEAT_RETRIES_FACTOR: z.coerce.number().default(2), - - RUN_ENGINE_WORKER_REDIS_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_HOST), - RUN_ENGINE_WORKER_REDIS_READER_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_READER_HOST), - RUN_ENGINE_WORKER_REDIS_READER_PORT: z.coerce - .number() - .optional() - .transform( - (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) - ), - RUN_ENGINE_WORKER_REDIS_PORT: z.coerce - .number() - .optional() - .transform((v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)), - RUN_ENGINE_WORKER_REDIS_USERNAME: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_USERNAME), - RUN_ENGINE_WORKER_REDIS_PASSWORD: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_PASSWORD), - RUN_ENGINE_WORKER_REDIS_TLS_DISABLED: z - .string() - .default(process.env.REDIS_TLS_DISABLED ?? "false"), - - RUN_ENGINE_RUN_QUEUE_REDIS_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_HOST), - RUN_ENGINE_RUN_QUEUE_REDIS_READER_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_READER_HOST), - RUN_ENGINE_RUN_QUEUE_REDIS_READER_PORT: z.coerce - .number() - .optional() - .transform( - (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) - ), - RUN_ENGINE_RUN_QUEUE_REDIS_PORT: z.coerce - .number() - .optional() - .transform((v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)), - RUN_ENGINE_RUN_QUEUE_REDIS_USERNAME: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_USERNAME), - RUN_ENGINE_RUN_QUEUE_REDIS_PASSWORD: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_PASSWORD), - RUN_ENGINE_RUN_QUEUE_REDIS_TLS_DISABLED: z - .string() - .default(process.env.REDIS_TLS_DISABLED ?? "false"), - - RUN_ENGINE_RUN_LOCK_REDIS_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_HOST), - RUN_ENGINE_RUN_LOCK_REDIS_READER_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_READER_HOST), - RUN_ENGINE_RUN_LOCK_REDIS_READER_PORT: z.coerce - .number() - .optional() - .transform( - (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) - ), - RUN_ENGINE_RUN_LOCK_REDIS_PORT: z.coerce - .number() - .optional() - .transform((v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)), - RUN_ENGINE_RUN_LOCK_REDIS_USERNAME: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_USERNAME), - RUN_ENGINE_RUN_LOCK_REDIS_PASSWORD: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_PASSWORD), - RUN_ENGINE_RUN_LOCK_REDIS_TLS_DISABLED: z - .string() - .default(process.env.REDIS_TLS_DISABLED ?? "false"), - - RUN_ENGINE_DEV_PRESENCE_REDIS_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_HOST), - RUN_ENGINE_DEV_PRESENCE_REDIS_READER_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_READER_HOST), - RUN_ENGINE_DEV_PRESENCE_REDIS_READER_PORT: z.coerce - .number() - .optional() - .transform( - (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) - ), - RUN_ENGINE_DEV_PRESENCE_REDIS_PORT: z.coerce - .number() - .optional() - .transform((v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)), - RUN_ENGINE_DEV_PRESENCE_REDIS_USERNAME: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_USERNAME), - RUN_ENGINE_DEV_PRESENCE_REDIS_PASSWORD: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_PASSWORD), - RUN_ENGINE_DEV_PRESENCE_REDIS_TLS_DISABLED: z - .string() - .default(process.env.REDIS_TLS_DISABLED ?? "false"), - - //API Rate limiting - /** - * @example "60s" - * @example "1m" - * @example "1h" - * @example "1d" - * @example "1000ms" - * @example "1000s" - */ - RUN_ENGINE_RATE_LIMIT_REFILL_INTERVAL: z.string().default("10s"), // refill 250 tokens every 10 seconds - RUN_ENGINE_RATE_LIMIT_MAX: z.coerce.number().int().default(1200), // allow bursts of 750 requests - RUN_ENGINE_RATE_LIMIT_REFILL_RATE: z.coerce.number().int().default(400), // refix 250 tokens every 10 seconds - RUN_ENGINE_RATE_LIMIT_REQUEST_LOGS_ENABLED: z.string().default("0"), - RUN_ENGINE_RATE_LIMIT_REJECTION_LOGS_ENABLED: z.string().default("1"), - RUN_ENGINE_RATE_LIMIT_LIMITER_LOGS_ENABLED: z.string().default("0"), - - RUN_ENGINE_RELEASE_CONCURRENCY_ENABLED: z.string().default("0"), - RUN_ENGINE_RELEASE_CONCURRENCY_DISABLE_CONSUMERS: z.string().default("0"), - RUN_ENGINE_RELEASE_CONCURRENCY_MAX_TOKENS_RATIO: z.coerce.number().default(1), - RUN_ENGINE_RELEASE_CONCURRENCY_RELEASINGS_MAX_AGE: z.coerce - .number() - .int() - .default(60_000 * 30), - RUN_ENGINE_RELEASE_CONCURRENCY_RELEASINGS_POLL_INTERVAL: z.coerce.number().int().default(60_000), - RUN_ENGINE_RELEASE_CONCURRENCY_MAX_RETRIES: z.coerce.number().int().default(3), - RUN_ENGINE_RELEASE_CONCURRENCY_CONSUMERS_COUNT: z.coerce.number().int().default(1), - RUN_ENGINE_RELEASE_CONCURRENCY_POLL_INTERVAL: z.coerce.number().int().default(500), - RUN_ENGINE_RELEASE_CONCURRENCY_BATCH_SIZE: z.coerce.number().int().default(10), - - RUN_ENGINE_WORKER_ENABLED: z.string().default("1"), - RUN_ENGINE_WORKER_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"), - RUN_ENGINE_RUN_QUEUE_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"), - - /** How long should the presence ttl last */ - DEV_PRESENCE_SSE_TIMEOUT: z.coerce.number().int().default(30_000), - DEV_PRESENCE_TTL_MS: z.coerce.number().int().default(5_000), - DEV_PRESENCE_POLL_MS: z.coerce.number().int().default(1_000), - /** How many ms to wait until dequeuing again, if there was a run last time */ - DEV_DEQUEUE_INTERVAL_WITH_RUN: z.coerce.number().int().default(250), - /** How many ms to wait until dequeuing again, if there was no run last time */ - DEV_DEQUEUE_INTERVAL_WITHOUT_RUN: z.coerce.number().int().default(1_000), - /** The max number of runs per API call that we'll dequeue in DEV */ - DEV_DEQUEUE_MAX_RUNS_PER_PULL: z.coerce.number().int().default(10), - - /** The maximum concurrent local run processes executing at once in dev */ - DEV_MAX_CONCURRENT_RUNS: z.coerce.number().int().default(25), - - /** The CLI should connect to this for dev runs */ - DEV_ENGINE_URL: z.string().default(process.env.APP_ORIGIN ?? "http://localhost:3030"), - - LEGACY_RUN_ENGINE_WORKER_ENABLED: z.string().default(process.env.WORKER_ENABLED ?? "true"), - LEGACY_RUN_ENGINE_WORKER_CONCURRENCY_WORKERS: z.coerce.number().int().default(2), - LEGACY_RUN_ENGINE_WORKER_CONCURRENCY_TASKS_PER_WORKER: z.coerce.number().int().default(1), - LEGACY_RUN_ENGINE_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000), - LEGACY_RUN_ENGINE_WORKER_IMMEDIATE_POLL_INTERVAL: z.coerce.number().int().default(50), - LEGACY_RUN_ENGINE_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(50), - LEGACY_RUN_ENGINE_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000), - LEGACY_RUN_ENGINE_WORKER_LOG_LEVEL: z - .enum(["log", "error", "warn", "info", "debug"]) - .default("info"), - - LEGACY_RUN_ENGINE_WORKER_REDIS_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_HOST), - LEGACY_RUN_ENGINE_WORKER_REDIS_READER_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_READER_HOST), - LEGACY_RUN_ENGINE_WORKER_REDIS_READER_PORT: z.coerce - .number() - .optional() - .transform( - (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) - ), - LEGACY_RUN_ENGINE_WORKER_REDIS_PORT: z.coerce - .number() - .optional() - .transform((v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)), - LEGACY_RUN_ENGINE_WORKER_REDIS_USERNAME: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_USERNAME), - LEGACY_RUN_ENGINE_WORKER_REDIS_PASSWORD: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_PASSWORD), - LEGACY_RUN_ENGINE_WORKER_REDIS_TLS_DISABLED: z - .string() - .default(process.env.REDIS_TLS_DISABLED ?? "false"), - LEGACY_RUN_ENGINE_WORKER_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), - - LEGACY_RUN_ENGINE_WAITING_FOR_DEPLOY_BATCH_SIZE: z.coerce.number().int().default(100), - LEGACY_RUN_ENGINE_WAITING_FOR_DEPLOY_BATCH_STAGGER_MS: z.coerce.number().int().default(1_000), - - COMMON_WORKER_ENABLED: z.string().default(process.env.WORKER_ENABLED ?? "true"), - COMMON_WORKER_CONCURRENCY_WORKERS: z.coerce.number().int().default(2), - COMMON_WORKER_CONCURRENCY_TASKS_PER_WORKER: z.coerce.number().int().default(10), - COMMON_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000), - COMMON_WORKER_IMMEDIATE_POLL_INTERVAL: z.coerce.number().int().default(50), - COMMON_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(50), - COMMON_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000), - COMMON_WORKER_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"), - - COMMON_WORKER_REDIS_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_HOST), - COMMON_WORKER_REDIS_READER_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_READER_HOST), - COMMON_WORKER_REDIS_READER_PORT: z.coerce - .number() - .optional() - .transform( - (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) - ), - COMMON_WORKER_REDIS_PORT: z.coerce - .number() - .optional() - .transform((v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)), - COMMON_WORKER_REDIS_USERNAME: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_USERNAME), - COMMON_WORKER_REDIS_PASSWORD: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_PASSWORD), - COMMON_WORKER_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"), - COMMON_WORKER_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), - - BATCH_TRIGGER_PROCESS_JOB_VISIBILITY_TIMEOUT_MS: z.coerce - .number() - .int() - .default(60_000 * 5), // 5 minutes - - BATCH_TRIGGER_CACHED_RUNS_CHECK_ENABLED: BoolEnv.default(false), - - BATCH_TRIGGER_WORKER_ENABLED: z.string().default(process.env.WORKER_ENABLED ?? "true"), - BATCH_TRIGGER_WORKER_CONCURRENCY_WORKERS: z.coerce.number().int().default(2), - BATCH_TRIGGER_WORKER_CONCURRENCY_TASKS_PER_WORKER: z.coerce.number().int().default(10), - BATCH_TRIGGER_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000), - BATCH_TRIGGER_WORKER_IMMEDIATE_POLL_INTERVAL: z.coerce.number().int().default(50), - BATCH_TRIGGER_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(20), - BATCH_TRIGGER_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000), - BATCH_TRIGGER_WORKER_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"), - - BATCH_TRIGGER_WORKER_REDIS_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_HOST), - BATCH_TRIGGER_WORKER_REDIS_READER_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_READER_HOST), - BATCH_TRIGGER_WORKER_REDIS_READER_PORT: z.coerce - .number() - .optional() - .transform( - (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) - ), - BATCH_TRIGGER_WORKER_REDIS_PORT: z.coerce - .number() - .optional() - .transform((v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)), - BATCH_TRIGGER_WORKER_REDIS_USERNAME: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_USERNAME), - BATCH_TRIGGER_WORKER_REDIS_PASSWORD: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_PASSWORD), - BATCH_TRIGGER_WORKER_REDIS_TLS_DISABLED: z - .string() - .default(process.env.REDIS_TLS_DISABLED ?? "false"), - BATCH_TRIGGER_WORKER_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), - - ADMIN_WORKER_ENABLED: z.string().default(process.env.WORKER_ENABLED ?? "true"), - ADMIN_WORKER_CONCURRENCY_WORKERS: z.coerce.number().int().default(2), - ADMIN_WORKER_CONCURRENCY_TASKS_PER_WORKER: z.coerce.number().int().default(10), - ADMIN_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000), - ADMIN_WORKER_IMMEDIATE_POLL_INTERVAL: z.coerce.number().int().default(50), - ADMIN_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(20), - ADMIN_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000), - ADMIN_WORKER_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"), - - ADMIN_WORKER_REDIS_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_HOST), - ADMIN_WORKER_REDIS_READER_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_READER_HOST), - ADMIN_WORKER_REDIS_READER_PORT: z.coerce - .number() - .optional() - .transform( - (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) - ), - ADMIN_WORKER_REDIS_PORT: z.coerce - .number() - .optional() - .transform((v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)), - ADMIN_WORKER_REDIS_USERNAME: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_USERNAME), - ADMIN_WORKER_REDIS_PASSWORD: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_PASSWORD), - ADMIN_WORKER_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"), - ADMIN_WORKER_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), - - ALERTS_WORKER_ENABLED: z.string().default(process.env.WORKER_ENABLED ?? "true"), - ALERTS_WORKER_CONCURRENCY_WORKERS: z.coerce.number().int().default(2), - ALERTS_WORKER_CONCURRENCY_TASKS_PER_WORKER: z.coerce.number().int().default(10), - ALERTS_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000), - ALERTS_WORKER_IMMEDIATE_POLL_INTERVAL: z.coerce.number().int().default(100), - ALERTS_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(50), - ALERTS_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000), - ALERTS_WORKER_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"), - - ALERTS_WORKER_REDIS_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_HOST), - ALERTS_WORKER_REDIS_READER_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_READER_HOST), - ALERTS_WORKER_REDIS_READER_PORT: z.coerce - .number() - .optional() - .transform( - (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) - ), - ALERTS_WORKER_REDIS_PORT: z.coerce - .number() - .optional() - .transform((v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)), - ALERTS_WORKER_REDIS_USERNAME: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_USERNAME), - ALERTS_WORKER_REDIS_PASSWORD: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_PASSWORD), - ALERTS_WORKER_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"), - ALERTS_WORKER_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), - - SCHEDULE_ENGINE_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"), - SCHEDULE_WORKER_ENABLED: z.string().default(process.env.WORKER_ENABLED ?? "true"), - SCHEDULE_WORKER_CONCURRENCY_WORKERS: z.coerce.number().int().default(2), - SCHEDULE_WORKER_CONCURRENCY_TASKS_PER_WORKER: z.coerce.number().int().default(10), - SCHEDULE_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000), - SCHEDULE_WORKER_IMMEDIATE_POLL_INTERVAL: z.coerce.number().int().default(50), - SCHEDULE_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(50), - SCHEDULE_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(30_000), - SCHEDULE_WORKER_DISTRIBUTION_WINDOW_SECONDS: z.coerce.number().int().default(30), - - SCHEDULE_WORKER_REDIS_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_HOST), - SCHEDULE_WORKER_REDIS_READER_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_READER_HOST), - SCHEDULE_WORKER_REDIS_READER_PORT: z.coerce - .number() - .optional() - .transform( - (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) - ), - SCHEDULE_WORKER_REDIS_PORT: z.coerce - .number() - .optional() - .transform((v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)), - SCHEDULE_WORKER_REDIS_USERNAME: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_USERNAME), - SCHEDULE_WORKER_REDIS_PASSWORD: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_PASSWORD), - SCHEDULE_WORKER_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"), - SCHEDULE_WORKER_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), - - TASK_EVENT_PARTITIONING_ENABLED: z.string().default("0"), - TASK_EVENT_PARTITIONED_WINDOW_IN_SECONDS: z.coerce.number().int().default(60), // 1 minute - - QUEUE_SSE_AUTORELOAD_INTERVAL_MS: z.coerce.number().int().default(5_000), - QUEUE_SSE_AUTORELOAD_TIMEOUT_MS: z.coerce.number().int().default(60_000), - - SLACK_BOT_TOKEN: z.string().optional(), - SLACK_SIGNUP_REASON_CHANNEL_ID: z.string().optional(), - - // kapa.ai - KAPA_AI_WEBSITE_ID: z.string().optional(), - - // BetterStack - BETTERSTACK_API_KEY: z.string().optional(), - BETTERSTACK_STATUS_PAGE_ID: z.string().optional(), - - RUN_REPLICATION_REDIS_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_HOST), - RUN_REPLICATION_REDIS_READER_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_READER_HOST), - RUN_REPLICATION_REDIS_READER_PORT: z.coerce - .number() - .optional() - .transform( - (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) - ), - RUN_REPLICATION_REDIS_PORT: z.coerce - .number() - .optional() - .transform((v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)), - RUN_REPLICATION_REDIS_USERNAME: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_USERNAME), - RUN_REPLICATION_REDIS_PASSWORD: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_PASSWORD), - RUN_REPLICATION_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"), - - RUN_REPLICATION_CLICKHOUSE_URL: z.string().optional(), - RUN_REPLICATION_ENABLED: z.string().default("0"), - RUN_REPLICATION_SLOT_NAME: z.string().default("task_runs_to_clickhouse_v1"), - RUN_REPLICATION_PUBLICATION_NAME: z.string().default("task_runs_to_clickhouse_v1_publication"), - RUN_REPLICATION_MAX_FLUSH_CONCURRENCY: z.coerce.number().int().default(2), - RUN_REPLICATION_FLUSH_INTERVAL_MS: z.coerce.number().int().default(1000), - RUN_REPLICATION_FLUSH_BATCH_SIZE: z.coerce.number().int().default(100), - RUN_REPLICATION_LEADER_LOCK_TIMEOUT_MS: z.coerce.number().int().default(30_000), - RUN_REPLICATION_LEADER_LOCK_EXTEND_INTERVAL_MS: z.coerce.number().int().default(10_000), - RUN_REPLICATION_ACK_INTERVAL_SECONDS: z.coerce.number().int().default(10), - RUN_REPLICATION_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"), - RUN_REPLICATION_CLICKHOUSE_LOG_LEVEL: z - .enum(["log", "error", "warn", "info", "debug"]) - .default("info"), - RUN_REPLICATION_LEADER_LOCK_ADDITIONAL_TIME_MS: z.coerce.number().int().default(10_000), - RUN_REPLICATION_LEADER_LOCK_RETRY_INTERVAL_MS: z.coerce.number().int().default(500), - RUN_REPLICATION_WAIT_FOR_ASYNC_INSERT: z.string().default("0"), - RUN_REPLICATION_KEEP_ALIVE_ENABLED: z.string().default("0"), - RUN_REPLICATION_KEEP_ALIVE_IDLE_SOCKET_TTL_MS: z.coerce.number().int().optional(), - RUN_REPLICATION_MAX_OPEN_CONNECTIONS: z.coerce.number().int().default(10), - // Retry configuration for insert operations - RUN_REPLICATION_INSERT_MAX_RETRIES: z.coerce.number().int().default(3), - RUN_REPLICATION_INSERT_BASE_DELAY_MS: z.coerce.number().int().default(100), - RUN_REPLICATION_INSERT_MAX_DELAY_MS: z.coerce.number().int().default(2000), - RUN_REPLICATION_INSERT_STRATEGY: z.enum(["insert", "insert_async"]).default("insert"), - - // Clickhouse - CLICKHOUSE_URL: z.string(), - CLICKHOUSE_KEEP_ALIVE_ENABLED: z.string().default("1"), - CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS: z.coerce.number().int().optional(), - CLICKHOUSE_MAX_OPEN_CONNECTIONS: z.coerce.number().int().default(10), - CLICKHOUSE_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"), - CLICKHOUSE_COMPRESSION_REQUEST: z.string().default("1"), - - // Bootstrap - TRIGGER_BOOTSTRAP_ENABLED: z.string().default("0"), - TRIGGER_BOOTSTRAP_WORKER_GROUP_NAME: z.string().optional(), - TRIGGER_BOOTSTRAP_WORKER_TOKEN_PATH: z.string().optional(), - - // Machine presets - MACHINE_PRESETS_OVERRIDE_PATH: z.string().optional(), - - // CLI package tag (e.g. "latest", "v4-beta", "4.0.0") - used for setup commands - TRIGGER_CLI_TAG: z.string().default("latest"), - - HEALTHCHECK_DATABASE_DISABLED: z.string().default("0"), - - REQUEST_IDEMPOTENCY_REDIS_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_HOST), - REQUEST_IDEMPOTENCY_REDIS_READER_HOST: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_READER_HOST), - REQUEST_IDEMPOTENCY_REDIS_READER_PORT: z.coerce - .number() - .optional() - .transform( - (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) - ), - REQUEST_IDEMPOTENCY_REDIS_PORT: z.coerce - .number() - .optional() - .transform((v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)), - REQUEST_IDEMPOTENCY_REDIS_USERNAME: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_USERNAME), - REQUEST_IDEMPOTENCY_REDIS_PASSWORD: z - .string() - .optional() - .transform((v) => v ?? process.env.REDIS_PASSWORD), - REQUEST_IDEMPOTENCY_REDIS_TLS_DISABLED: z - .string() - .default(process.env.REDIS_TLS_DISABLED ?? "false"), - - REQUEST_IDEMPOTENCY_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"), - - REQUEST_IDEMPOTENCY_TTL_IN_MS: z.coerce - .number() - .int() - .default(60_000 * 60 * 24), - - // Bulk action - BULK_ACTION_BATCH_SIZE: z.coerce.number().int().default(100), - BULK_ACTION_BATCH_DELAY_MS: z.coerce.number().int().default(200), - BULK_ACTION_SUBBATCH_CONCURRENCY: z.coerce.number().int().default(5), - - // AI Run Filter - AI_RUN_FILTER_MODEL: z.string().optional(), - - EVENT_LOOP_MONITOR_THRESHOLD_MS: z.coerce.number().int().default(100), - - VERY_SLOW_QUERY_THRESHOLD_MS: z.coerce.number().int().optional(), -}); +const GithubAppEnvSchema = z.preprocess( + (val) => { + const obj = val as any; + if (!obj || !obj.GITHUB_APP_ENABLED) { + return { ...obj, GITHUB_APP_ENABLED: "0" }; + } + return obj; + }, + z.discriminatedUnion("GITHUB_APP_ENABLED", [ + z.object({ + GITHUB_APP_ENABLED: z.literal("1"), + GITHUB_APP_ID: z.string(), + GITHUB_APP_PRIVATE_KEY: z.string(), + GITHUB_APP_WEBHOOK_SECRET: z.string(), + GITHUB_APP_SLUG: z.string(), + }), + z.object({ + GITHUB_APP_ENABLED: z.literal("0"), + }), + ]) +); + +const EnvironmentSchema = z + .object({ + NODE_ENV: z.union([z.literal("development"), z.literal("production"), z.literal("test")]), + DATABASE_URL: z + .string() + .refine( + isValidDatabaseUrl, + "DATABASE_URL is invalid, for details please check the additional output above this message." + ), + DATABASE_CONNECTION_LIMIT: z.coerce.number().int().default(10), + DATABASE_POOL_TIMEOUT: z.coerce.number().int().default(60), + DATABASE_CONNECTION_TIMEOUT: z.coerce.number().int().default(20), + DIRECT_URL: z + .string() + .refine( + isValidDatabaseUrl, + "DIRECT_URL is invalid, for details please check the additional output above this message." + ), + DATABASE_READ_REPLICA_URL: z.string().optional(), + SESSION_SECRET: z.string(), + MAGIC_LINK_SECRET: z.string(), + ENCRYPTION_KEY: z + .string() + .refine( + (val) => Buffer.from(val, "utf8").length === 32, + "ENCRYPTION_KEY must be exactly 32 bytes" + ), + WHITELISTED_EMAILS: z + .string() + .refine(isValidRegex, "WHITELISTED_EMAILS must be a valid regex.") + .optional(), + ADMIN_EMAILS: z.string().refine(isValidRegex, "ADMIN_EMAILS must be a valid regex.").optional(), + REMIX_APP_PORT: z.string().optional(), + LOGIN_ORIGIN: z.string().default("http://localhost:3030"), + APP_ORIGIN: z.string().default("http://localhost:3030"), + API_ORIGIN: z.string().optional(), + STREAM_ORIGIN: z.string().optional(), + ELECTRIC_ORIGIN: z.string().default("http://localhost:3060"), + // A comma separated list of electric origins to shard into different electric instances by environmentId + // example: "http://localhost:3060,http://localhost:3061,http://localhost:3062" + ELECTRIC_ORIGIN_SHARDS: z.string().optional(), + APP_ENV: z.string().default(process.env.NODE_ENV), + SERVICE_NAME: z.string().default("trigger.dev webapp"), + POSTHOG_PROJECT_KEY: z.string().default("phc_LFH7kJiGhdIlnO22hTAKgHpaKhpM8gkzWAFvHmf5vfS"), + TRIGGER_TELEMETRY_DISABLED: z.string().optional(), + AUTH_GITHUB_CLIENT_ID: z.string().optional(), + AUTH_GITHUB_CLIENT_SECRET: z.string().optional(), + EMAIL_TRANSPORT: z.enum(["resend", "smtp", "aws-ses"]).optional(), + FROM_EMAIL: z.string().optional(), + REPLY_TO_EMAIL: z.string().optional(), + RESEND_API_KEY: z.string().optional(), + SMTP_HOST: z.string().optional(), + SMTP_PORT: z.coerce.number().optional(), + SMTP_SECURE: BoolEnv.optional(), + SMTP_USER: z.string().optional(), + SMTP_PASSWORD: z.string().optional(), + + PLAIN_API_KEY: z.string().optional(), + WORKER_SCHEMA: z.string().default("graphile_worker"), + WORKER_CONCURRENCY: z.coerce.number().int().default(10), + WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000), + WORKER_ENABLED: z.string().default("true"), + GRACEFUL_SHUTDOWN_TIMEOUT: z.coerce.number().int().default(60000), + DISABLE_SSE: z.string().optional(), + OPENAI_API_KEY: z.string().optional(), + + // Redis options + REDIS_HOST: z.string().optional(), + REDIS_READER_HOST: z.string().optional(), + REDIS_READER_PORT: z.coerce.number().optional(), + REDIS_PORT: z.coerce.number().optional(), + REDIS_USERNAME: z.string().optional(), + REDIS_PASSWORD: z.string().optional(), + REDIS_TLS_DISABLED: z.string().optional(), + + RATE_LIMIT_REDIS_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_HOST), + RATE_LIMIT_REDIS_READER_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_READER_HOST), + RATE_LIMIT_REDIS_READER_PORT: z.coerce + .number() + .optional() + .transform( + (v) => + v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + ), + RATE_LIMIT_REDIS_PORT: z.coerce + .number() + .optional() + .transform( + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + ), + RATE_LIMIT_REDIS_USERNAME: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_USERNAME), + RATE_LIMIT_REDIS_PASSWORD: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_PASSWORD), + RATE_LIMIT_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"), + RATE_LIMIT_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), + + CACHE_REDIS_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_HOST), + CACHE_REDIS_READER_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_READER_HOST), + CACHE_REDIS_READER_PORT: z.coerce + .number() + .optional() + .transform( + (v) => + v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + ), + CACHE_REDIS_PORT: z.coerce + .number() + .optional() + .transform( + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + ), + CACHE_REDIS_USERNAME: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_USERNAME), + CACHE_REDIS_PASSWORD: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_PASSWORD), + CACHE_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"), + CACHE_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), + + REALTIME_STREAMS_REDIS_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_HOST), + REALTIME_STREAMS_REDIS_READER_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_READER_HOST), + REALTIME_STREAMS_REDIS_READER_PORT: z.coerce + .number() + .optional() + .transform( + (v) => + v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + ), + REALTIME_STREAMS_REDIS_PORT: z.coerce + .number() + .optional() + .transform( + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + ), + REALTIME_STREAMS_REDIS_USERNAME: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_USERNAME), + REALTIME_STREAMS_REDIS_PASSWORD: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_PASSWORD), + REALTIME_STREAMS_REDIS_TLS_DISABLED: z + .string() + .default(process.env.REDIS_TLS_DISABLED ?? "false"), + REALTIME_STREAMS_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), + + REALTIME_MAXIMUM_CREATED_AT_FILTER_AGE_IN_MS: z.coerce + .number() + .int() + .default(24 * 60 * 60 * 1000), // 1 day in milliseconds + + PUBSUB_REDIS_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_HOST), + PUBSUB_REDIS_READER_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_READER_HOST), + PUBSUB_REDIS_READER_PORT: z.coerce + .number() + .optional() + .transform( + (v) => + v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + ), + PUBSUB_REDIS_PORT: z.coerce + .number() + .optional() + .transform( + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + ), + PUBSUB_REDIS_USERNAME: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_USERNAME), + PUBSUB_REDIS_PASSWORD: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_PASSWORD), + PUBSUB_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"), + PUBSUB_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), + + DEFAULT_ENV_EXECUTION_CONCURRENCY_LIMIT: z.coerce.number().int().default(100), + DEFAULT_ENV_EXECUTION_CONCURRENCY_BURST_FACTOR: z.coerce.number().default(1.0), + DEFAULT_ORG_EXECUTION_CONCURRENCY_LIMIT: z.coerce.number().int().default(300), + DEFAULT_DEV_ENV_EXECUTION_ATTEMPTS: z.coerce.number().int().positive().default(1), + + //API Rate limiting + /** + * @example "60s" + * @example "1m" + * @example "1h" + * @example "1d" + * @example "1000ms" + * @example "1000s" + */ + API_RATE_LIMIT_REFILL_INTERVAL: z.string().default("10s"), // refill 250 tokens every 10 seconds + API_RATE_LIMIT_MAX: z.coerce.number().int().default(750), // allow bursts of 750 requests + API_RATE_LIMIT_REFILL_RATE: z.coerce.number().int().default(250), // refix 250 tokens every 10 seconds + API_RATE_LIMIT_REQUEST_LOGS_ENABLED: z.string().default("0"), + API_RATE_LIMIT_REJECTION_LOGS_ENABLED: z.string().default("1"), + API_RATE_LIMIT_LIMITER_LOGS_ENABLED: z.string().default("0"), + + API_RATE_LIMIT_JWT_WINDOW: z.string().default("1m"), + API_RATE_LIMIT_JWT_TOKENS: z.coerce.number().int().default(60), + + //v3 + PROVIDER_SECRET: z.string().default("provider-secret"), + COORDINATOR_SECRET: z.string().default("coordinator-secret"), + DEPOT_TOKEN: z.string().optional(), + DEPOT_ORG_ID: z.string().optional(), + DEPOT_REGION: z.string().default("us-east-1"), + + // Deployment registry (v3) + DEPLOY_REGISTRY_HOST: z.string().min(1), + DEPLOY_REGISTRY_USERNAME: z.string().optional(), + DEPLOY_REGISTRY_PASSWORD: z.string().optional(), + DEPLOY_REGISTRY_NAMESPACE: z.string().min(1).default("trigger"), + DEPLOY_REGISTRY_ECR_TAGS: z.string().optional(), // csv, for example: "key1=value1,key2=value2" + DEPLOY_REGISTRY_ECR_ASSUME_ROLE_ARN: z.string().optional(), + DEPLOY_REGISTRY_ECR_ASSUME_ROLE_EXTERNAL_ID: z.string().optional(), + + // Deployment registry (v4) - falls back to v3 registry if not specified + V4_DEPLOY_REGISTRY_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.DEPLOY_REGISTRY_HOST) + .pipe(z.string().min(1)), // Ensure final type is required string + V4_DEPLOY_REGISTRY_USERNAME: z + .string() + .optional() + .transform((v) => v ?? process.env.DEPLOY_REGISTRY_USERNAME), + V4_DEPLOY_REGISTRY_PASSWORD: z + .string() + .optional() + .transform((v) => v ?? process.env.DEPLOY_REGISTRY_PASSWORD), + V4_DEPLOY_REGISTRY_NAMESPACE: z + .string() + .optional() + .transform((v) => v ?? process.env.DEPLOY_REGISTRY_NAMESPACE) + .pipe(z.string().min(1).default("trigger")), // Ensure final type is required string + V4_DEPLOY_REGISTRY_ECR_TAGS: z + .string() + .optional() + .transform((v) => v ?? process.env.DEPLOY_REGISTRY_ECR_TAGS), + V4_DEPLOY_REGISTRY_ECR_ASSUME_ROLE_ARN: z + .string() + .optional() + .transform((v) => v ?? process.env.DEPLOY_REGISTRY_ECR_ASSUME_ROLE_ARN), + V4_DEPLOY_REGISTRY_ECR_ASSUME_ROLE_EXTERNAL_ID: z + .string() + .optional() + .transform((v) => v ?? process.env.DEPLOY_REGISTRY_ECR_ASSUME_ROLE_EXTERNAL_ID), + + DEPLOY_IMAGE_PLATFORM: z.string().default("linux/amd64"), + DEPLOY_TIMEOUT_MS: z.coerce + .number() + .int() + .default(60 * 1000 * 8), // 8 minutes + + OBJECT_STORE_BASE_URL: z.string().optional(), + OBJECT_STORE_ACCESS_KEY_ID: z.string().optional(), + OBJECT_STORE_SECRET_ACCESS_KEY: z.string().optional(), + OBJECT_STORE_REGION: z.string().optional(), + OBJECT_STORE_SERVICE: z.string().default("s3"), + EVENTS_BATCH_SIZE: z.coerce.number().int().default(100), + EVENTS_BATCH_INTERVAL: z.coerce.number().int().default(1000), + EVENTS_DEFAULT_LOG_RETENTION: z.coerce.number().int().default(7), + EVENTS_MIN_CONCURRENCY: z.coerce.number().int().default(1), + EVENTS_MAX_CONCURRENCY: z.coerce.number().int().default(10), + EVENTS_MAX_BATCH_SIZE: z.coerce.number().int().default(500), + EVENTS_MEMORY_PRESSURE_THRESHOLD: z.coerce.number().int().default(5000), + EVENTS_LOAD_SHEDDING_THRESHOLD: z.coerce.number().int().default(100000), + EVENTS_LOAD_SHEDDING_ENABLED: z.string().default("1"), + SHARED_QUEUE_CONSUMER_POOL_SIZE: z.coerce.number().int().default(10), + SHARED_QUEUE_CONSUMER_INTERVAL_MS: z.coerce.number().int().default(100), + SHARED_QUEUE_CONSUMER_NEXT_TICK_INTERVAL_MS: z.coerce.number().int().default(100), + SHARED_QUEUE_CONSUMER_EMIT_RESUME_DEPENDENCY_TIMEOUT_MS: z.coerce.number().int().default(1000), + SHARED_QUEUE_CONSUMER_RESOLVE_PAYLOADS_BATCH_SIZE: z.coerce.number().int().default(25), + + MANAGED_WORKER_SECRET: z.string().default("managed-secret"), + + // Development OTEL environment variables + DEV_OTEL_EXPORTER_OTLP_ENDPOINT: z.string().optional(), + // If this is set to 1, then the below variables are used to configure the batch processor for spans and logs + DEV_OTEL_BATCH_PROCESSING_ENABLED: z.string().default("0"), + DEV_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE: z.string().default("64"), + DEV_OTEL_SPAN_SCHEDULED_DELAY_MILLIS: z.string().default("200"), + DEV_OTEL_SPAN_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"), + DEV_OTEL_SPAN_MAX_QUEUE_SIZE: z.string().default("512"), + DEV_OTEL_LOG_MAX_EXPORT_BATCH_SIZE: z.string().default("64"), + DEV_OTEL_LOG_SCHEDULED_DELAY_MILLIS: z.string().default("200"), + DEV_OTEL_LOG_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"), + DEV_OTEL_LOG_MAX_QUEUE_SIZE: z.string().default("512"), + + PROD_OTEL_BATCH_PROCESSING_ENABLED: z.string().default("0"), + PROD_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE: z.string().default("64"), + PROD_OTEL_SPAN_SCHEDULED_DELAY_MILLIS: z.string().default("200"), + PROD_OTEL_SPAN_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"), + PROD_OTEL_SPAN_MAX_QUEUE_SIZE: z.string().default("512"), + PROD_OTEL_LOG_MAX_EXPORT_BATCH_SIZE: z.string().default("64"), + PROD_OTEL_LOG_SCHEDULED_DELAY_MILLIS: z.string().default("200"), + PROD_OTEL_LOG_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"), + PROD_OTEL_LOG_MAX_QUEUE_SIZE: z.string().default("512"), + + TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT: z.string().default("1024"), + TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT: z.string().default("1024"), + TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT: z.string().default("131072"), + TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT: z.string().default("131072"), + TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT: z.string().default("10"), + TRIGGER_OTEL_LINK_COUNT_LIMIT: z.string().default("2"), + TRIGGER_OTEL_ATTRIBUTE_PER_LINK_COUNT_LIMIT: z.string().default("10"), + TRIGGER_OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT: z.string().default("10"), + + CHECKPOINT_THRESHOLD_IN_MS: z.coerce.number().int().default(30000), + + // Internal OTEL environment variables + INTERNAL_OTEL_TRACE_EXPORTER_URL: z.string().optional(), + INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADERS: z.string().optional(), + INTERNAL_OTEL_TRACE_LOGGING_ENABLED: z.string().default("1"), + // this means 1/20 traces or 5% of traces will be sampled (sampled = recorded) + INTERNAL_OTEL_TRACE_SAMPLING_RATE: z.string().default("20"), + INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED: z.string().default("0"), + INTERNAL_OTEL_TRACE_DISABLED: z.string().default("0"), + + INTERNAL_OTEL_LOG_EXPORTER_URL: z.string().optional(), + INTERNAL_OTEL_METRIC_EXPORTER_URL: z.string().optional(), + INTERNAL_OTEL_METRIC_EXPORTER_AUTH_HEADERS: z.string().optional(), + INTERNAL_OTEL_METRIC_EXPORTER_ENABLED: z.string().default("0"), + INTERNAL_OTEL_METRIC_EXPORTER_INTERVAL_MS: z.coerce.number().int().default(30_000), + INTERNAL_OTEL_HOST_METRICS_ENABLED: BoolEnv.default(true), + INTERNAL_OTEL_NODEJS_METRICS_ENABLED: BoolEnv.default(true), + INTERNAL_OTEL_ADDITIONAL_DETECTORS_ENABLED: BoolEnv.default(true), + + ORG_SLACK_INTEGRATION_CLIENT_ID: z.string().optional(), + ORG_SLACK_INTEGRATION_CLIENT_SECRET: z.string().optional(), + + /** These enable the alerts feature in v3 */ + ALERT_EMAIL_TRANSPORT: z.enum(["resend", "smtp", "aws-ses"]).optional(), + ALERT_FROM_EMAIL: z.string().optional(), + ALERT_REPLY_TO_EMAIL: z.string().optional(), + ALERT_RESEND_API_KEY: z.string().optional(), + ALERT_SMTP_HOST: z.string().optional(), + ALERT_SMTP_PORT: z.coerce.number().optional(), + ALERT_SMTP_SECURE: BoolEnv.optional(), + ALERT_SMTP_USER: z.string().optional(), + ALERT_SMTP_PASSWORD: z.string().optional(), + ALERT_RATE_LIMITER_EMISSION_INTERVAL: z.coerce.number().int().default(2_500), + ALERT_RATE_LIMITER_BURST_TOLERANCE: z.coerce.number().int().default(10_000), + ALERT_RATE_LIMITER_REDIS_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_HOST), + ALERT_RATE_LIMITER_REDIS_READER_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_READER_HOST), + ALERT_RATE_LIMITER_REDIS_READER_PORT: z.coerce + .number() + .optional() + .transform( + (v) => + v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + ), + ALERT_RATE_LIMITER_REDIS_PORT: z.coerce + .number() + .optional() + .transform( + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + ), + ALERT_RATE_LIMITER_REDIS_USERNAME: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_USERNAME), + ALERT_RATE_LIMITER_REDIS_PASSWORD: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_PASSWORD), + ALERT_RATE_LIMITER_REDIS_TLS_DISABLED: z + .string() + .default(process.env.REDIS_TLS_DISABLED ?? "false"), + ALERT_RATE_LIMITER_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), + + LOOPS_API_KEY: z.string().optional(), + MARQS_DISABLE_REBALANCING: BoolEnv.default(false), + MARQS_VISIBILITY_TIMEOUT_MS: z.coerce + .number() + .int() + .default(60 * 1000 * 15), + MARQS_SHARED_QUEUE_LIMIT: z.coerce.number().int().default(1000), + MARQS_MAXIMUM_QUEUE_PER_ENV_COUNT: z.coerce.number().int().default(50), + MARQS_DEV_QUEUE_LIMIT: z.coerce.number().int().default(1000), + MARQS_MAXIMUM_NACK_COUNT: z.coerce.number().int().default(64), + MARQS_CONCURRENCY_LIMIT_BIAS: z.coerce.number().default(0.75), + MARQS_AVAILABLE_CAPACITY_BIAS: z.coerce.number().default(0.3), + MARQS_QUEUE_AGE_RANDOMIZATION_BIAS: z.coerce.number().default(0.25), + MARQS_REUSE_SNAPSHOT_COUNT: z.coerce.number().int().default(0), + MARQS_MAXIMUM_ENV_COUNT: z.coerce.number().int().optional(), + MARQS_SHARED_WORKER_QUEUE_CONSUMER_INTERVAL_MS: z.coerce.number().int().default(250), + MARQS_SHARED_WORKER_QUEUE_MAX_MESSAGE_COUNT: z.coerce.number().int().default(10), + + MARQS_SHARED_WORKER_QUEUE_EAGER_DEQUEUE_ENABLED: z.string().default("0"), + MARQS_WORKER_ENABLED: z.string().default("0"), + MARQS_WORKER_COUNT: z.coerce.number().int().default(2), + MARQS_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(50), + MARQS_WORKER_CONCURRENCY_TASKS_PER_WORKER: z.coerce.number().int().default(5), + MARQS_WORKER_POLL_INTERVAL_MS: z.coerce.number().int().default(100), + MARQS_WORKER_IMMEDIATE_POLL_INTERVAL_MS: z.coerce.number().int().default(100), + MARQS_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000), + MARQS_SHARED_WORKER_QUEUE_COOLOFF_COUNT_THRESHOLD: z.coerce.number().int().default(10), + MARQS_SHARED_WORKER_QUEUE_COOLOFF_PERIOD_MS: z.coerce.number().int().default(5_000), + + PROD_TASK_HEARTBEAT_INTERVAL_MS: z.coerce.number().int().optional(), + + VERBOSE_GRAPHILE_LOGGING: z.string().default("false"), + V2_MARQS_ENABLED: z.string().default("0"), + V2_MARQS_CONSUMER_POOL_ENABLED: z.string().default("0"), + V2_MARQS_CONSUMER_POOL_SIZE: z.coerce.number().int().default(10), + V2_MARQS_CONSUMER_POLL_INTERVAL_MS: z.coerce.number().int().default(1000), + V2_MARQS_QUEUE_SELECTION_COUNT: z.coerce.number().int().default(36), + V2_MARQS_VISIBILITY_TIMEOUT_MS: z.coerce + .number() + .int() + .default(60 * 1000 * 15), + V2_MARQS_DEFAULT_ENV_CONCURRENCY: z.coerce.number().int().default(100), + V2_MARQS_VERBOSE: z.string().default("0"), + V3_MARQS_CONCURRENCY_MONITOR_ENABLED: z.string().default("0"), + V2_MARQS_CONCURRENCY_MONITOR_ENABLED: z.string().default("0"), + /* Usage settings */ + USAGE_EVENT_URL: z.string().optional(), + PROD_USAGE_HEARTBEAT_INTERVAL_MS: z.coerce.number().int().optional(), + + CENTS_PER_RUN: z.coerce.number().default(0), + + EVENT_LOOP_MONITOR_ENABLED: z.string().default("1"), + MAXIMUM_LIVE_RELOADING_EVENTS: z.coerce.number().int().default(1000), + MAXIMUM_TRACE_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(25_000), + MAXIMUM_TRACE_DETAILED_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(10_000), + TASK_PAYLOAD_OFFLOAD_THRESHOLD: z.coerce.number().int().default(524_288), // 512KB + TASK_PAYLOAD_MAXIMUM_SIZE: z.coerce.number().int().default(3_145_728), // 3MB + BATCH_TASK_PAYLOAD_MAXIMUM_SIZE: z.coerce.number().int().default(1_000_000), // 1MB + TASK_RUN_METADATA_MAXIMUM_SIZE: z.coerce.number().int().default(262_144), // 256KB + + MAXIMUM_DEV_QUEUE_SIZE: z.coerce.number().int().optional(), + MAXIMUM_DEPLOYED_QUEUE_SIZE: z.coerce.number().int().optional(), + MAX_BATCH_V2_TRIGGER_ITEMS: z.coerce.number().int().default(500), + MAX_BATCH_AND_WAIT_V2_TRIGGER_ITEMS: z.coerce.number().int().default(500), + + REALTIME_STREAM_VERSION: z.enum(["v1", "v2"]).default("v1"), + REALTIME_STREAM_MAX_LENGTH: z.coerce.number().int().default(1000), + REALTIME_STREAM_TTL: z.coerce + .number() + .int() + .default(60 * 60 * 24), // 1 day in seconds + BATCH_METADATA_OPERATIONS_FLUSH_INTERVAL_MS: z.coerce.number().int().default(1000), + BATCH_METADATA_OPERATIONS_FLUSH_ENABLED: z.string().default("1"), + BATCH_METADATA_OPERATIONS_FLUSH_LOGGING_ENABLED: z.string().default("1"), + + // Run Engine 2.0 + RUN_ENGINE_WORKER_COUNT: z.coerce.number().int().default(4), + RUN_ENGINE_TASKS_PER_WORKER: z.coerce.number().int().default(10), + RUN_ENGINE_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(10), + RUN_ENGINE_WORKER_POLL_INTERVAL: z.coerce.number().int().default(100), + RUN_ENGINE_WORKER_IMMEDIATE_POLL_INTERVAL: z.coerce.number().int().default(100), + RUN_ENGINE_TIMEOUT_PENDING_EXECUTING: z.coerce.number().int().default(60_000), + RUN_ENGINE_TIMEOUT_PENDING_CANCEL: z.coerce.number().int().default(60_000), + RUN_ENGINE_TIMEOUT_EXECUTING: z.coerce.number().int().default(60_000), + RUN_ENGINE_TIMEOUT_EXECUTING_WITH_WAITPOINTS: z.coerce.number().int().default(60_000), + RUN_ENGINE_TIMEOUT_SUSPENDED: z.coerce + .number() + .int() + .default(60_000 * 10), + RUN_ENGINE_DEBUG_WORKER_NOTIFICATIONS: BoolEnv.default(false), + RUN_ENGINE_PARENT_QUEUE_LIMIT: z.coerce.number().int().default(1000), + RUN_ENGINE_CONCURRENCY_LIMIT_BIAS: z.coerce.number().default(0.75), + RUN_ENGINE_AVAILABLE_CAPACITY_BIAS: z.coerce.number().default(0.3), + RUN_ENGINE_QUEUE_AGE_RANDOMIZATION_BIAS: z.coerce.number().default(0.25), + RUN_ENGINE_REUSE_SNAPSHOT_COUNT: z.coerce.number().int().default(0), + RUN_ENGINE_MAXIMUM_ENV_COUNT: z.coerce.number().int().optional(), + RUN_ENGINE_RUN_QUEUE_SHARD_COUNT: z.coerce.number().int().default(4), + RUN_ENGINE_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000), + RUN_ENGINE_RETRY_WARM_START_THRESHOLD_MS: z.coerce.number().int().default(30_000), + RUN_ENGINE_PROCESS_WORKER_QUEUE_DEBOUNCE_MS: z.coerce.number().int().default(200), + RUN_ENGINE_DEQUEUE_BLOCKING_TIMEOUT_SECONDS: z.coerce.number().int().default(10), + RUN_ENGINE_MASTER_QUEUE_CONSUMERS_INTERVAL_MS: z.coerce.number().int().default(1000), + RUN_ENGINE_MASTER_QUEUE_COOLOFF_PERIOD_MS: z.coerce.number().int().default(10_000), + RUN_ENGINE_MASTER_QUEUE_COOLOFF_COUNT_THRESHOLD: z.coerce.number().int().default(10), + RUN_ENGINE_MASTER_QUEUE_CONSUMER_DEQUEUE_COUNT: z.coerce.number().int().default(10), + RUN_ENGINE_CONCURRENCY_SWEEPER_SCAN_SCHEDULE: z.string().optional(), + RUN_ENGINE_CONCURRENCY_SWEEPER_PROCESS_MARKED_SCHEDULE: z.string().optional(), + RUN_ENGINE_CONCURRENCY_SWEEPER_SCAN_JITTER_IN_MS: z.coerce.number().int().optional(), + RUN_ENGINE_CONCURRENCY_SWEEPER_PROCESS_MARKED_JITTER_IN_MS: z.coerce.number().int().optional(), + + RUN_ENGINE_RUN_LOCK_DURATION: z.coerce.number().int().default(5000), + RUN_ENGINE_RUN_LOCK_AUTOMATIC_EXTENSION_THRESHOLD: z.coerce.number().int().default(1000), + RUN_ENGINE_RUN_LOCK_MAX_RETRIES: z.coerce.number().int().default(10), + RUN_ENGINE_RUN_LOCK_BASE_DELAY: z.coerce.number().int().default(100), + RUN_ENGINE_RUN_LOCK_MAX_DELAY: z.coerce.number().int().default(3000), + RUN_ENGINE_RUN_LOCK_BACKOFF_MULTIPLIER: z.coerce.number().default(1.8), + RUN_ENGINE_RUN_LOCK_JITTER_FACTOR: z.coerce.number().default(0.15), + RUN_ENGINE_RUN_LOCK_MAX_TOTAL_WAIT_TIME: z.coerce.number().int().default(15000), + + RUN_ENGINE_SUSPENDED_HEARTBEAT_RETRIES_MAX_COUNT: z.coerce.number().int().default(12), + RUN_ENGINE_SUSPENDED_HEARTBEAT_RETRIES_MAX_DELAY_MS: z.coerce + .number() + .int() + .default(60_000 * 60 * 6), + RUN_ENGINE_SUSPENDED_HEARTBEAT_RETRIES_INITIAL_DELAY_MS: z.coerce + .number() + .int() + .default(60_000), + RUN_ENGINE_SUSPENDED_HEARTBEAT_RETRIES_FACTOR: z.coerce.number().default(2), + + RUN_ENGINE_WORKER_REDIS_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_HOST), + RUN_ENGINE_WORKER_REDIS_READER_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_READER_HOST), + RUN_ENGINE_WORKER_REDIS_READER_PORT: z.coerce + .number() + .optional() + .transform( + (v) => + v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + ), + RUN_ENGINE_WORKER_REDIS_PORT: z.coerce + .number() + .optional() + .transform( + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + ), + RUN_ENGINE_WORKER_REDIS_USERNAME: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_USERNAME), + RUN_ENGINE_WORKER_REDIS_PASSWORD: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_PASSWORD), + RUN_ENGINE_WORKER_REDIS_TLS_DISABLED: z + .string() + .default(process.env.REDIS_TLS_DISABLED ?? "false"), + + RUN_ENGINE_RUN_QUEUE_REDIS_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_HOST), + RUN_ENGINE_RUN_QUEUE_REDIS_READER_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_READER_HOST), + RUN_ENGINE_RUN_QUEUE_REDIS_READER_PORT: z.coerce + .number() + .optional() + .transform( + (v) => + v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + ), + RUN_ENGINE_RUN_QUEUE_REDIS_PORT: z.coerce + .number() + .optional() + .transform( + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + ), + RUN_ENGINE_RUN_QUEUE_REDIS_USERNAME: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_USERNAME), + RUN_ENGINE_RUN_QUEUE_REDIS_PASSWORD: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_PASSWORD), + RUN_ENGINE_RUN_QUEUE_REDIS_TLS_DISABLED: z + .string() + .default(process.env.REDIS_TLS_DISABLED ?? "false"), + + RUN_ENGINE_RUN_LOCK_REDIS_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_HOST), + RUN_ENGINE_RUN_LOCK_REDIS_READER_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_READER_HOST), + RUN_ENGINE_RUN_LOCK_REDIS_READER_PORT: z.coerce + .number() + .optional() + .transform( + (v) => + v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + ), + RUN_ENGINE_RUN_LOCK_REDIS_PORT: z.coerce + .number() + .optional() + .transform( + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + ), + RUN_ENGINE_RUN_LOCK_REDIS_USERNAME: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_USERNAME), + RUN_ENGINE_RUN_LOCK_REDIS_PASSWORD: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_PASSWORD), + RUN_ENGINE_RUN_LOCK_REDIS_TLS_DISABLED: z + .string() + .default(process.env.REDIS_TLS_DISABLED ?? "false"), + + RUN_ENGINE_DEV_PRESENCE_REDIS_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_HOST), + RUN_ENGINE_DEV_PRESENCE_REDIS_READER_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_READER_HOST), + RUN_ENGINE_DEV_PRESENCE_REDIS_READER_PORT: z.coerce + .number() + .optional() + .transform( + (v) => + v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + ), + RUN_ENGINE_DEV_PRESENCE_REDIS_PORT: z.coerce + .number() + .optional() + .transform( + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + ), + RUN_ENGINE_DEV_PRESENCE_REDIS_USERNAME: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_USERNAME), + RUN_ENGINE_DEV_PRESENCE_REDIS_PASSWORD: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_PASSWORD), + RUN_ENGINE_DEV_PRESENCE_REDIS_TLS_DISABLED: z + .string() + .default(process.env.REDIS_TLS_DISABLED ?? "false"), + + //API Rate limiting + /** + * @example "60s" + * @example "1m" + * @example "1h" + * @example "1d" + * @example "1000ms" + * @example "1000s" + */ + RUN_ENGINE_RATE_LIMIT_REFILL_INTERVAL: z.string().default("10s"), // refill 250 tokens every 10 seconds + RUN_ENGINE_RATE_LIMIT_MAX: z.coerce.number().int().default(1200), // allow bursts of 750 requests + RUN_ENGINE_RATE_LIMIT_REFILL_RATE: z.coerce.number().int().default(400), // refix 250 tokens every 10 seconds + RUN_ENGINE_RATE_LIMIT_REQUEST_LOGS_ENABLED: z.string().default("0"), + RUN_ENGINE_RATE_LIMIT_REJECTION_LOGS_ENABLED: z.string().default("1"), + RUN_ENGINE_RATE_LIMIT_LIMITER_LOGS_ENABLED: z.string().default("0"), + + RUN_ENGINE_RELEASE_CONCURRENCY_ENABLED: z.string().default("0"), + RUN_ENGINE_RELEASE_CONCURRENCY_DISABLE_CONSUMERS: z.string().default("0"), + RUN_ENGINE_RELEASE_CONCURRENCY_MAX_TOKENS_RATIO: z.coerce.number().default(1), + RUN_ENGINE_RELEASE_CONCURRENCY_RELEASINGS_MAX_AGE: z.coerce + .number() + .int() + .default(60_000 * 30), + RUN_ENGINE_RELEASE_CONCURRENCY_RELEASINGS_POLL_INTERVAL: z.coerce + .number() + .int() + .default(60_000), + RUN_ENGINE_RELEASE_CONCURRENCY_MAX_RETRIES: z.coerce.number().int().default(3), + RUN_ENGINE_RELEASE_CONCURRENCY_CONSUMERS_COUNT: z.coerce.number().int().default(1), + RUN_ENGINE_RELEASE_CONCURRENCY_POLL_INTERVAL: z.coerce.number().int().default(500), + RUN_ENGINE_RELEASE_CONCURRENCY_BATCH_SIZE: z.coerce.number().int().default(10), + + RUN_ENGINE_WORKER_ENABLED: z.string().default("1"), + RUN_ENGINE_WORKER_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"), + RUN_ENGINE_RUN_QUEUE_LOG_LEVEL: z + .enum(["log", "error", "warn", "info", "debug"]) + .default("info"), + + /** How long should the presence ttl last */ + DEV_PRESENCE_SSE_TIMEOUT: z.coerce.number().int().default(30_000), + DEV_PRESENCE_TTL_MS: z.coerce.number().int().default(5_000), + DEV_PRESENCE_POLL_MS: z.coerce.number().int().default(1_000), + /** How many ms to wait until dequeuing again, if there was a run last time */ + DEV_DEQUEUE_INTERVAL_WITH_RUN: z.coerce.number().int().default(250), + /** How many ms to wait until dequeuing again, if there was no run last time */ + DEV_DEQUEUE_INTERVAL_WITHOUT_RUN: z.coerce.number().int().default(1_000), + /** The max number of runs per API call that we'll dequeue in DEV */ + DEV_DEQUEUE_MAX_RUNS_PER_PULL: z.coerce.number().int().default(10), + + /** The maximum concurrent local run processes executing at once in dev */ + DEV_MAX_CONCURRENT_RUNS: z.coerce.number().int().default(25), + + /** The CLI should connect to this for dev runs */ + DEV_ENGINE_URL: z.string().default(process.env.APP_ORIGIN ?? "http://localhost:3030"), + + LEGACY_RUN_ENGINE_WORKER_ENABLED: z.string().default(process.env.WORKER_ENABLED ?? "true"), + LEGACY_RUN_ENGINE_WORKER_CONCURRENCY_WORKERS: z.coerce.number().int().default(2), + LEGACY_RUN_ENGINE_WORKER_CONCURRENCY_TASKS_PER_WORKER: z.coerce.number().int().default(1), + LEGACY_RUN_ENGINE_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000), + LEGACY_RUN_ENGINE_WORKER_IMMEDIATE_POLL_INTERVAL: z.coerce.number().int().default(50), + LEGACY_RUN_ENGINE_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(50), + LEGACY_RUN_ENGINE_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000), + LEGACY_RUN_ENGINE_WORKER_LOG_LEVEL: z + .enum(["log", "error", "warn", "info", "debug"]) + .default("info"), + + LEGACY_RUN_ENGINE_WORKER_REDIS_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_HOST), + LEGACY_RUN_ENGINE_WORKER_REDIS_READER_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_READER_HOST), + LEGACY_RUN_ENGINE_WORKER_REDIS_READER_PORT: z.coerce + .number() + .optional() + .transform( + (v) => + v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + ), + LEGACY_RUN_ENGINE_WORKER_REDIS_PORT: z.coerce + .number() + .optional() + .transform( + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + ), + LEGACY_RUN_ENGINE_WORKER_REDIS_USERNAME: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_USERNAME), + LEGACY_RUN_ENGINE_WORKER_REDIS_PASSWORD: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_PASSWORD), + LEGACY_RUN_ENGINE_WORKER_REDIS_TLS_DISABLED: z + .string() + .default(process.env.REDIS_TLS_DISABLED ?? "false"), + LEGACY_RUN_ENGINE_WORKER_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), + + LEGACY_RUN_ENGINE_WAITING_FOR_DEPLOY_BATCH_SIZE: z.coerce.number().int().default(100), + LEGACY_RUN_ENGINE_WAITING_FOR_DEPLOY_BATCH_STAGGER_MS: z.coerce.number().int().default(1_000), + + COMMON_WORKER_ENABLED: z.string().default(process.env.WORKER_ENABLED ?? "true"), + COMMON_WORKER_CONCURRENCY_WORKERS: z.coerce.number().int().default(2), + COMMON_WORKER_CONCURRENCY_TASKS_PER_WORKER: z.coerce.number().int().default(10), + COMMON_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000), + COMMON_WORKER_IMMEDIATE_POLL_INTERVAL: z.coerce.number().int().default(50), + COMMON_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(50), + COMMON_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000), + COMMON_WORKER_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"), + + COMMON_WORKER_REDIS_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_HOST), + COMMON_WORKER_REDIS_READER_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_READER_HOST), + COMMON_WORKER_REDIS_READER_PORT: z.coerce + .number() + .optional() + .transform( + (v) => + v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + ), + COMMON_WORKER_REDIS_PORT: z.coerce + .number() + .optional() + .transform( + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + ), + COMMON_WORKER_REDIS_USERNAME: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_USERNAME), + COMMON_WORKER_REDIS_PASSWORD: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_PASSWORD), + COMMON_WORKER_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"), + COMMON_WORKER_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), + + BATCH_TRIGGER_PROCESS_JOB_VISIBILITY_TIMEOUT_MS: z.coerce + .number() + .int() + .default(60_000 * 5), // 5 minutes + + BATCH_TRIGGER_CACHED_RUNS_CHECK_ENABLED: BoolEnv.default(false), + + BATCH_TRIGGER_WORKER_ENABLED: z.string().default(process.env.WORKER_ENABLED ?? "true"), + BATCH_TRIGGER_WORKER_CONCURRENCY_WORKERS: z.coerce.number().int().default(2), + BATCH_TRIGGER_WORKER_CONCURRENCY_TASKS_PER_WORKER: z.coerce.number().int().default(10), + BATCH_TRIGGER_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000), + BATCH_TRIGGER_WORKER_IMMEDIATE_POLL_INTERVAL: z.coerce.number().int().default(50), + BATCH_TRIGGER_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(20), + BATCH_TRIGGER_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000), + BATCH_TRIGGER_WORKER_LOG_LEVEL: z + .enum(["log", "error", "warn", "info", "debug"]) + .default("info"), + + BATCH_TRIGGER_WORKER_REDIS_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_HOST), + BATCH_TRIGGER_WORKER_REDIS_READER_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_READER_HOST), + BATCH_TRIGGER_WORKER_REDIS_READER_PORT: z.coerce + .number() + .optional() + .transform( + (v) => + v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + ), + BATCH_TRIGGER_WORKER_REDIS_PORT: z.coerce + .number() + .optional() + .transform( + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + ), + BATCH_TRIGGER_WORKER_REDIS_USERNAME: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_USERNAME), + BATCH_TRIGGER_WORKER_REDIS_PASSWORD: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_PASSWORD), + BATCH_TRIGGER_WORKER_REDIS_TLS_DISABLED: z + .string() + .default(process.env.REDIS_TLS_DISABLED ?? "false"), + BATCH_TRIGGER_WORKER_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), + + ADMIN_WORKER_ENABLED: z.string().default(process.env.WORKER_ENABLED ?? "true"), + ADMIN_WORKER_CONCURRENCY_WORKERS: z.coerce.number().int().default(2), + ADMIN_WORKER_CONCURRENCY_TASKS_PER_WORKER: z.coerce.number().int().default(10), + ADMIN_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000), + ADMIN_WORKER_IMMEDIATE_POLL_INTERVAL: z.coerce.number().int().default(50), + ADMIN_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(20), + ADMIN_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000), + ADMIN_WORKER_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"), + + ADMIN_WORKER_REDIS_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_HOST), + ADMIN_WORKER_REDIS_READER_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_READER_HOST), + ADMIN_WORKER_REDIS_READER_PORT: z.coerce + .number() + .optional() + .transform( + (v) => + v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + ), + ADMIN_WORKER_REDIS_PORT: z.coerce + .number() + .optional() + .transform( + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + ), + ADMIN_WORKER_REDIS_USERNAME: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_USERNAME), + ADMIN_WORKER_REDIS_PASSWORD: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_PASSWORD), + ADMIN_WORKER_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"), + ADMIN_WORKER_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), + + ALERTS_WORKER_ENABLED: z.string().default(process.env.WORKER_ENABLED ?? "true"), + ALERTS_WORKER_CONCURRENCY_WORKERS: z.coerce.number().int().default(2), + ALERTS_WORKER_CONCURRENCY_TASKS_PER_WORKER: z.coerce.number().int().default(10), + ALERTS_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000), + ALERTS_WORKER_IMMEDIATE_POLL_INTERVAL: z.coerce.number().int().default(100), + ALERTS_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(50), + ALERTS_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000), + ALERTS_WORKER_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"), + + ALERTS_WORKER_REDIS_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_HOST), + ALERTS_WORKER_REDIS_READER_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_READER_HOST), + ALERTS_WORKER_REDIS_READER_PORT: z.coerce + .number() + .optional() + .transform( + (v) => + v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + ), + ALERTS_WORKER_REDIS_PORT: z.coerce + .number() + .optional() + .transform( + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + ), + ALERTS_WORKER_REDIS_USERNAME: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_USERNAME), + ALERTS_WORKER_REDIS_PASSWORD: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_PASSWORD), + ALERTS_WORKER_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"), + ALERTS_WORKER_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), + + SCHEDULE_ENGINE_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"), + SCHEDULE_WORKER_ENABLED: z.string().default(process.env.WORKER_ENABLED ?? "true"), + SCHEDULE_WORKER_CONCURRENCY_WORKERS: z.coerce.number().int().default(2), + SCHEDULE_WORKER_CONCURRENCY_TASKS_PER_WORKER: z.coerce.number().int().default(10), + SCHEDULE_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000), + SCHEDULE_WORKER_IMMEDIATE_POLL_INTERVAL: z.coerce.number().int().default(50), + SCHEDULE_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(50), + SCHEDULE_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(30_000), + SCHEDULE_WORKER_DISTRIBUTION_WINDOW_SECONDS: z.coerce.number().int().default(30), + + SCHEDULE_WORKER_REDIS_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_HOST), + SCHEDULE_WORKER_REDIS_READER_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_READER_HOST), + SCHEDULE_WORKER_REDIS_READER_PORT: z.coerce + .number() + .optional() + .transform( + (v) => + v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + ), + SCHEDULE_WORKER_REDIS_PORT: z.coerce + .number() + .optional() + .transform( + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + ), + SCHEDULE_WORKER_REDIS_USERNAME: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_USERNAME), + SCHEDULE_WORKER_REDIS_PASSWORD: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_PASSWORD), + SCHEDULE_WORKER_REDIS_TLS_DISABLED: z + .string() + .default(process.env.REDIS_TLS_DISABLED ?? "false"), + SCHEDULE_WORKER_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), + + TASK_EVENT_PARTITIONING_ENABLED: z.string().default("0"), + TASK_EVENT_PARTITIONED_WINDOW_IN_SECONDS: z.coerce.number().int().default(60), // 1 minute + + QUEUE_SSE_AUTORELOAD_INTERVAL_MS: z.coerce.number().int().default(5_000), + QUEUE_SSE_AUTORELOAD_TIMEOUT_MS: z.coerce.number().int().default(60_000), + + SLACK_BOT_TOKEN: z.string().optional(), + SLACK_SIGNUP_REASON_CHANNEL_ID: z.string().optional(), + + // kapa.ai + KAPA_AI_WEBSITE_ID: z.string().optional(), + + // BetterStack + BETTERSTACK_API_KEY: z.string().optional(), + BETTERSTACK_STATUS_PAGE_ID: z.string().optional(), + + RUN_REPLICATION_REDIS_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_HOST), + RUN_REPLICATION_REDIS_READER_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_READER_HOST), + RUN_REPLICATION_REDIS_READER_PORT: z.coerce + .number() + .optional() + .transform( + (v) => + v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + ), + RUN_REPLICATION_REDIS_PORT: z.coerce + .number() + .optional() + .transform( + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + ), + RUN_REPLICATION_REDIS_USERNAME: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_USERNAME), + RUN_REPLICATION_REDIS_PASSWORD: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_PASSWORD), + RUN_REPLICATION_REDIS_TLS_DISABLED: z + .string() + .default(process.env.REDIS_TLS_DISABLED ?? "false"), + + RUN_REPLICATION_CLICKHOUSE_URL: z.string().optional(), + RUN_REPLICATION_ENABLED: z.string().default("0"), + RUN_REPLICATION_SLOT_NAME: z.string().default("task_runs_to_clickhouse_v1"), + RUN_REPLICATION_PUBLICATION_NAME: z.string().default("task_runs_to_clickhouse_v1_publication"), + RUN_REPLICATION_MAX_FLUSH_CONCURRENCY: z.coerce.number().int().default(2), + RUN_REPLICATION_FLUSH_INTERVAL_MS: z.coerce.number().int().default(1000), + RUN_REPLICATION_FLUSH_BATCH_SIZE: z.coerce.number().int().default(100), + RUN_REPLICATION_LEADER_LOCK_TIMEOUT_MS: z.coerce.number().int().default(30_000), + RUN_REPLICATION_LEADER_LOCK_EXTEND_INTERVAL_MS: z.coerce.number().int().default(10_000), + RUN_REPLICATION_ACK_INTERVAL_SECONDS: z.coerce.number().int().default(10), + RUN_REPLICATION_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"), + RUN_REPLICATION_CLICKHOUSE_LOG_LEVEL: z + .enum(["log", "error", "warn", "info", "debug"]) + .default("info"), + RUN_REPLICATION_LEADER_LOCK_ADDITIONAL_TIME_MS: z.coerce.number().int().default(10_000), + RUN_REPLICATION_LEADER_LOCK_RETRY_INTERVAL_MS: z.coerce.number().int().default(500), + RUN_REPLICATION_WAIT_FOR_ASYNC_INSERT: z.string().default("0"), + RUN_REPLICATION_KEEP_ALIVE_ENABLED: z.string().default("0"), + RUN_REPLICATION_KEEP_ALIVE_IDLE_SOCKET_TTL_MS: z.coerce.number().int().optional(), + RUN_REPLICATION_MAX_OPEN_CONNECTIONS: z.coerce.number().int().default(10), + // Retry configuration for insert operations + RUN_REPLICATION_INSERT_MAX_RETRIES: z.coerce.number().int().default(3), + RUN_REPLICATION_INSERT_BASE_DELAY_MS: z.coerce.number().int().default(100), + RUN_REPLICATION_INSERT_MAX_DELAY_MS: z.coerce.number().int().default(2000), + RUN_REPLICATION_INSERT_STRATEGY: z.enum(["insert", "insert_async"]).default("insert"), + + // Clickhouse + CLICKHOUSE_URL: z.string(), + CLICKHOUSE_KEEP_ALIVE_ENABLED: z.string().default("1"), + CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS: z.coerce.number().int().optional(), + CLICKHOUSE_MAX_OPEN_CONNECTIONS: z.coerce.number().int().default(10), + CLICKHOUSE_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"), + CLICKHOUSE_COMPRESSION_REQUEST: z.string().default("1"), + + // Bootstrap + TRIGGER_BOOTSTRAP_ENABLED: z.string().default("0"), + TRIGGER_BOOTSTRAP_WORKER_GROUP_NAME: z.string().optional(), + TRIGGER_BOOTSTRAP_WORKER_TOKEN_PATH: z.string().optional(), + + // Machine presets + MACHINE_PRESETS_OVERRIDE_PATH: z.string().optional(), + + // CLI package tag (e.g. "latest", "v4-beta", "4.0.0") - used for setup commands + TRIGGER_CLI_TAG: z.string().default("latest"), + + HEALTHCHECK_DATABASE_DISABLED: z.string().default("0"), + + REQUEST_IDEMPOTENCY_REDIS_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_HOST), + REQUEST_IDEMPOTENCY_REDIS_READER_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_READER_HOST), + REQUEST_IDEMPOTENCY_REDIS_READER_PORT: z.coerce + .number() + .optional() + .transform( + (v) => + v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + ), + REQUEST_IDEMPOTENCY_REDIS_PORT: z.coerce + .number() + .optional() + .transform( + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + ), + REQUEST_IDEMPOTENCY_REDIS_USERNAME: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_USERNAME), + REQUEST_IDEMPOTENCY_REDIS_PASSWORD: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_PASSWORD), + REQUEST_IDEMPOTENCY_REDIS_TLS_DISABLED: z + .string() + .default(process.env.REDIS_TLS_DISABLED ?? "false"), + + REQUEST_IDEMPOTENCY_LOG_LEVEL: z + .enum(["log", "error", "warn", "info", "debug"]) + .default("info"), + + REQUEST_IDEMPOTENCY_TTL_IN_MS: z.coerce + .number() + .int() + .default(60_000 * 60 * 24), + + // Bulk action + BULK_ACTION_BATCH_SIZE: z.coerce.number().int().default(100), + BULK_ACTION_BATCH_DELAY_MS: z.coerce.number().int().default(200), + BULK_ACTION_SUBBATCH_CONCURRENCY: z.coerce.number().int().default(5), + + // AI Run Filter + AI_RUN_FILTER_MODEL: z.string().optional(), + + EVENT_LOOP_MONITOR_THRESHOLD_MS: z.coerce.number().int().default(100), + + VERY_SLOW_QUERY_THRESHOLD_MS: z.coerce.number().int().optional(), + }) + .and(GithubAppEnvSchema); export type Environment = z.infer; export const env = EnvironmentSchema.parse(process.env); diff --git a/apps/webapp/app/routes/_app.github.callback/route.tsx b/apps/webapp/app/routes/_app.github.callback/route.tsx new file mode 100644 index 000000000..44c7f37c1 --- /dev/null +++ b/apps/webapp/app/routes/_app.github.callback/route.tsx @@ -0,0 +1,121 @@ +import { type LoaderFunctionArgs } from "@remix-run/node"; +import { z } from "zod"; +import { validateGitHubAppInstallSession } from "~/services/gitHubSession.server"; +import { linkGitHubAppInstallation, updateGitHubAppInstallation } from "~/services/gitHub.server"; +import { logger } from "~/services/logger.server"; +import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server"; +import { tryCatch } from "@trigger.dev/core"; +import { $replica } from "~/db.server"; +import { requireUser } from "~/services/session.server"; +import { sanitizeRedirectPath } from "~/utils"; + +const QuerySchema = z.discriminatedUnion("setup_action", [ + z.object({ + setup_action: z.literal("install"), + installation_id: z.coerce.number(), + state: z.string(), + }), + z.object({ + setup_action: z.literal("update"), + installation_id: z.coerce.number(), + state: z.string(), + }), + z.object({ + setup_action: z.literal("request"), + state: z.string(), + }), +]); + +export async function loader({ request }: LoaderFunctionArgs) { + const url = new URL(request.url); + const queryParams = Object.fromEntries(url.searchParams); + const cookieHeader = request.headers.get("Cookie"); + + const result = QuerySchema.safeParse(queryParams); + + if (!result.success) { + logger.warn("GitHub App callback with invalid params", { + queryParams, + }); + return redirectWithErrorMessage("/", request, "Failed to install GitHub App"); + } + + const callbackData = result.data; + + const sessionResult = await validateGitHubAppInstallSession(cookieHeader, callbackData.state); + + if (!sessionResult.valid) { + logger.error("GitHub App callback with invalid session", { + callbackData, + error: sessionResult.error, + }); + + return redirectWithErrorMessage("/", request, "Failed to install GitHub App"); + } + + const { organizationId, redirectTo: unsafeRedirectTo } = sessionResult; + const redirectTo = sanitizeRedirectPath(unsafeRedirectTo); + + const user = await requireUser(request); + const org = await $replica.organization.findFirst({ + where: { id: organizationId, members: { some: { userId: user.id } }, deletedAt: null }, + orderBy: { createdAt: "desc" }, + select: { + id: true, + }, + }); + + if (!org) { + // the secure cookie approach should already protect against this + // just an additional check + logger.error("GitHub app installation attempt on unauthenticated org", { + userId: user.id, + organizationId, + }); + return redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub App"); + } + + switch (callbackData.setup_action) { + case "install": { + const [error] = await tryCatch( + linkGitHubAppInstallation(callbackData.installation_id, organizationId) + ); + + if (error) { + logger.error("Failed to link GitHub App installation", { + error, + }); + return redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub App"); + } + + return redirectWithSuccessMessage(redirectTo, request, "GitHub App installed successfully"); + } + + case "update": { + const [error] = await tryCatch(updateGitHubAppInstallation(callbackData.installation_id)); + + if (error) { + logger.error("Failed to update GitHub App installation", { + error, + }); + return redirectWithErrorMessage(redirectTo, request, "Failed to update GitHub App"); + } + + return redirectWithSuccessMessage(redirectTo, request, "GitHub App updated successfully"); + } + + case "request": { + // This happens when a non-admin user requests installation + // The installation_id won't be available until an admin approves + logger.info("GitHub App installation requested, awaiting approval", { + callbackData, + }); + + return redirectWithSuccessMessage(redirectTo, request, "GitHub App installation requested"); + } + + default: + callbackData satisfies never; + return redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub App"); + } +} diff --git a/apps/webapp/app/routes/_app.github.install/route.tsx b/apps/webapp/app/routes/_app.github.install/route.tsx new file mode 100644 index 000000000..42d68e5be --- /dev/null +++ b/apps/webapp/app/routes/_app.github.install/route.tsx @@ -0,0 +1,52 @@ +import type { LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { redirect } from "remix-typedjson"; +import { z } from "zod"; +import { $replica } from "~/db.server"; +import { createGitHubAppInstallSession } from "~/services/gitHubSession.server"; +import { requireUser } from "~/services/session.server"; +import { newOrganizationPath } from "~/utils/pathBuilder"; +import { logger } from "~/services/logger.server"; +import { sanitizeRedirectPath } from "~/utils"; + +const QuerySchema = z.object({ + org_slug: z.string(), + redirect_to: z.string().refine((value) => value === sanitizeRedirectPath(value), { + message: "Invalid redirect path", + }), +}); + +export const loader = async ({ request }: LoaderFunctionArgs) => { + const searchParams = new URL(request.url).searchParams; + const parsed = QuerySchema.safeParse(Object.fromEntries(searchParams)); + + if (!parsed.success) { + logger.warn("GitHub App installation redirect with invalid params", { + searchParams, + error: parsed.error, + }); + throw redirect("/"); + } + + const { org_slug, redirect_to } = parsed.data; + const user = await requireUser(request); + + const org = await $replica.organization.findFirst({ + where: { slug: org_slug, members: { some: { userId: user.id } }, deletedAt: null }, + orderBy: { createdAt: "desc" }, + select: { + id: true, + }, + }); + + if (!org) { + throw redirect(newOrganizationPath()); + } + + const { url, cookieHeader } = await createGitHubAppInstallSession(org.id, redirect_to); + + return redirect(url, { + headers: { + "Set-Cookie": cookieHeader, + }, + }); +}; diff --git a/apps/webapp/app/routes/auth.github.callback.tsx b/apps/webapp/app/routes/auth.github.callback.tsx index d85a68a47..ee8776ae8 100644 --- a/apps/webapp/app/routes/auth.github.callback.tsx +++ b/apps/webapp/app/routes/auth.github.callback.tsx @@ -5,11 +5,12 @@ import { getSession, redirectWithErrorMessage } from "~/models/message.server"; import { authenticator } from "~/services/auth.server"; import { commitSession } from "~/services/sessionStorage.server"; import { redirectCookie } from "./auth.github"; +import { sanitizeRedirectPath } from "~/utils"; export let loader: LoaderFunction = async ({ request }) => { const cookie = request.headers.get("Cookie"); const redirectValue = await redirectCookie.parse(cookie); - const redirectTo = redirectValue ?? "/"; + const redirectTo = sanitizeRedirectPath(redirectValue); const auth = await authenticator.authenticate("github", request, { failureRedirect: "/login", // If auth fails, the failureRedirect will be thrown as a Response diff --git a/apps/webapp/app/routes/auth.github.ts b/apps/webapp/app/routes/auth.github.ts index ec376153f..a4adc7f28 100644 --- a/apps/webapp/app/routes/auth.github.ts +++ b/apps/webapp/app/routes/auth.github.ts @@ -1,6 +1,4 @@ -import type { ActionFunction, LoaderFunction } from "@remix-run/node"; -import { createCookie } from "@remix-run/node"; -import { redirect } from "@remix-run/node"; +import { type ActionFunction, type LoaderFunction, redirect, createCookie } from "@remix-run/node"; import { authenticator } from "~/services/auth.server"; export let loader: LoaderFunction = () => redirect("/login"); diff --git a/apps/webapp/app/services/gitHub.server.ts b/apps/webapp/app/services/gitHub.server.ts new file mode 100644 index 000000000..9e4c26a55 --- /dev/null +++ b/apps/webapp/app/services/gitHub.server.ts @@ -0,0 +1,135 @@ +import { App, type Octokit } from "octokit"; +import { env } from "../env.server"; +import { prisma } from "~/db.server"; +import { logger } from "./logger.server"; + +export const githubApp = + env.GITHUB_APP_ENABLED === "1" + ? new App({ + appId: env.GITHUB_APP_ID, + privateKey: env.GITHUB_APP_PRIVATE_KEY, + webhooks: { + secret: env.GITHUB_APP_WEBHOOK_SECRET, + }, + }) + : null; + +/** + * Links a GitHub App installation to a Trigger organization + */ +export async function linkGitHubAppInstallation( + installationId: number, + organizationId: string +): Promise { + if (!githubApp) { + throw new Error("GitHub App is not enabled"); + } + + const octokit = await githubApp.getInstallationOctokit(installationId); + const { data: installation } = await octokit.rest.apps.getInstallation({ + installation_id: installationId, + }); + + const repositories = await fetchInstallationRepositories(octokit, installationId); + + const repositorySelection = installation.repository_selection === "all" ? "ALL" : "SELECTED"; + + await prisma.githubAppInstallation.create({ + data: { + appInstallationId: installationId, + organizationId, + targetId: installation.target_id, + targetType: installation.target_type, + accountHandle: installation.account + ? "login" in installation.account + ? installation.account.login + : "slug" in installation.account + ? installation.account.slug + : "-" + : "-", + permissions: installation.permissions, + repositorySelection, + repositories: { + create: repositories, + }, + }, + }); +} + +/** + * Links a GitHub App installation to a Trigger organization + */ +export async function updateGitHubAppInstallation(installationId: number): Promise { + if (!githubApp) { + throw new Error("GitHub App is not enabled"); + } + + const octokit = await githubApp.getInstallationOctokit(installationId); + const { data: installation } = await octokit.rest.apps.getInstallation({ + installation_id: installationId, + }); + + const existingInstallation = await prisma.githubAppInstallation.findFirst({ + where: { appInstallationId: installationId }, + }); + + if (!existingInstallation) { + throw new Error("GitHub App installation not found"); + } + + const repositorySelection = installation.repository_selection === "all" ? "ALL" : "SELECTED"; + + // repos are updated asynchronously via webhook events + await prisma.githubAppInstallation.update({ + where: { id: existingInstallation?.id }, + data: { + appInstallationId: installationId, + targetId: installation.target_id, + targetType: installation.target_type, + accountHandle: installation.account + ? "login" in installation.account + ? installation.account.login + : "slug" in installation.account + ? installation.account.slug + : "-" + : "-", + permissions: installation.permissions, + suspendedAt: existingInstallation?.suspendedAt, + repositorySelection, + }, + }); +} + +async function fetchInstallationRepositories(octokit: Octokit, installationId: number) { + const iterator = octokit.paginate.iterator(octokit.rest.apps.listReposAccessibleToInstallation, { + installation_id: installationId, + per_page: 100, + }); + + const allRepos = []; + const maxPages = 3; + let pageCount = 0; + + for await (const { data } of iterator) { + pageCount++; + allRepos.push(...data); + + if (maxPages && pageCount >= maxPages) { + logger.warn("GitHub installation repository fetch truncated", { + installationId, + maxPages, + totalReposFetched: allRepos.length, + }); + break; + } + } + + return allRepos.map((repo) => ({ + githubId: repo.id, + name: repo.name, + fullName: repo.full_name, + htmlUrl: repo.html_url, + private: repo.private, + defaultBranch: repo.default_branch, + })); +} diff --git a/apps/webapp/app/services/gitHubSession.server.ts b/apps/webapp/app/services/gitHubSession.server.ts new file mode 100644 index 000000000..499c1dff1 --- /dev/null +++ b/apps/webapp/app/services/gitHubSession.server.ts @@ -0,0 +1,124 @@ +import { createCookieSessionStorage } from "@remix-run/node"; +import { randomBytes } from "crypto"; +import { env } from "../env.server"; +import { logger } from "./logger.server"; + +const sessionStorage = createCookieSessionStorage({ + cookie: { + name: "__github_app_install", + httpOnly: true, + maxAge: 60 * 60, // 1 hour + path: "/", + sameSite: "lax", + secrets: [env.SESSION_SECRET], + secure: env.NODE_ENV === "production", + }, +}); + +/** + * Creates a secure session for GitHub App installation with organization tracking + */ +export async function createGitHubAppInstallSession( + organizationId: string, + redirectTo: string +): Promise<{ url: string; cookieHeader: string }> { + if (env.GITHUB_APP_ENABLED !== "1") { + throw new Error("GitHub App is not enabled"); + } + + const state = randomBytes(32).toString("hex"); + + const session = await sessionStorage.getSession(); + session.set("organizationId", organizationId); + session.set("redirectTo", redirectTo); + session.set("state", state); + session.set("createdAt", Date.now()); + + const githubAppSlug = env.GITHUB_APP_SLUG; + + // the state query param gets passed through to the installation callback + const url = `https://github.com/apps/${githubAppSlug}/installations/new?state=${state}`; + + const cookieHeader = await sessionStorage.commitSession(session); + + return { url, cookieHeader }; +} + +/** + * Validates and retrieves the GitHub App installation session + */ +export async function validateGitHubAppInstallSession( + cookieHeader: string | null, + state: string +): Promise< + { valid: true; organizationId: string; redirectTo: string } | { valid: false; error?: string } +> { + if (!cookieHeader) { + return { + valid: false, + error: "No installation session cookie found", + }; + } + + const session = await sessionStorage.getSession(cookieHeader); + + const sessionState = session.get("state"); + const organizationId = session.get("organizationId"); + const redirectTo = session.get("redirectTo"); + const createdAt = session.get("createdAt"); + + if (!sessionState || !organizationId || !createdAt || !redirectTo) { + logger.warn("GitHub App installation session missing required fields", { + hasState: !!sessionState, + hasOrgId: !!organizationId, + hasCreatedAt: !!createdAt, + hasRedirectTo: !!redirectTo, + }); + + return { + valid: false, + error: "invalid_session_data", + }; + } + + if (sessionState !== state) { + logger.warn("GitHub App installation state mismatch", { + expectedState: sessionState, + receivedState: state, + }); + return { + valid: false, + error: "state_mismatch", + }; + } + + const expirationTime = createdAt + 60 * 60 * 1000; + if (Date.now() > expirationTime) { + logger.warn("GitHub App installation session expired", { + createdAt: new Date(createdAt), + now: new Date(), + }); + return { + valid: false, + error: "session_expired", + }; + } + + return { + valid: true, + organizationId, + redirectTo, + }; +} + +/** + * Destroys the GitHub App installation cookie session + */ +export async function destroyGitHubAppInstallSession(cookieHeader: string | null): Promise { + if (!cookieHeader) { + return ""; + } + + const session = await sessionStorage.getSession(cookieHeader); + return await sessionStorage.destroySession(session); +} diff --git a/apps/webapp/app/utils.ts b/apps/webapp/app/utils.ts index 7dc407cec..adb182928 100644 --- a/apps/webapp/app/utils.ts +++ b/apps/webapp/app/utils.ts @@ -7,22 +7,38 @@ const DEFAULT_REDIRECT = "/"; * This should be used any time the redirect path is user-provided * (Like the query string on our login/signup pages). This avoids * open-redirect vulnerabilities. - * @param {string} to The redirect destination + * @param {string} path The redirect destination * @param {string} defaultRedirect The redirect to use if the to is unsafe. */ -export function safeRedirect( - to: FormDataEntryValue | string | null | undefined, +export function sanitizeRedirectPath( + path: string | undefined | null, defaultRedirect: string = DEFAULT_REDIRECT -) { - if (!to || typeof to !== "string") { +): string { + if (!path || typeof path !== "string") { return defaultRedirect; } - if (!to.startsWith("/") || to.startsWith("//")) { + if (!path.startsWith("/") || path.startsWith("//")) { return defaultRedirect; } - return to; + try { + // should not parse as a full URL + new URL(path); + return defaultRedirect; + } catch {} + + try { + // ensure it's a valid relative path + const url = new URL(path, "https://example.com"); + if (url.hostname !== "example.com") { + return defaultRedirect; + } + } catch { + return defaultRedirect; + } + + return path; } /** diff --git a/apps/webapp/package.json b/apps/webapp/package.json index a09bcc5a6..a02949424 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -160,6 +160,7 @@ "morgan": "^1.10.0", "nanoid": "3.3.8", "non.geist": "^1.0.2", + "octokit": "^3.2.1", "ohash": "^1.1.3", "openai": "^4.33.1", "p-limit": "^6.2.0", diff --git a/internal-packages/database/prisma/migrations/20250902133449_add_gh_installation_schema/migration.sql b/internal-packages/database/prisma/migrations/20250902133449_add_gh_installation_schema/migration.sql new file mode 100644 index 000000000..78b5e6bf2 --- /dev/null +++ b/internal-packages/database/prisma/migrations/20250902133449_add_gh_installation_schema/migration.sql @@ -0,0 +1,46 @@ +CREATE TYPE "public"."GithubRepositorySelection" AS ENUM ('ALL', 'SELECTED'); + +CREATE TABLE "public"."GithubAppInstallation" ( + "id" TEXT NOT NULL, + "appInstallationId" BIGINT NOT NULL, + "targetId" BIGINT NOT NULL, + "targetType" TEXT NOT NULL, + "accountHandle" TEXT NOT NULL, + "permissions" JSONB, + "repositorySelection" "public"."GithubRepositorySelection" NOT NULL, + "installedBy" TEXT, + "organizationId" TEXT NOT NULL, + "deletedAt" TIMESTAMP(3), + "suspendedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "GithubAppInstallation_pkey" PRIMARY KEY ("id") +); + +CREATE TABLE "public"."GithubRepository" ( + "id" TEXT NOT NULL, + "githubId" BIGINT NOT NULL, + "name" TEXT NOT NULL, + "fullName" TEXT NOT NULL, + "htmlUrl" TEXT NOT NULL, + "private" BOOLEAN NOT NULL, + "defaultBranch" TEXT NOT NULL, + "installationId" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "GithubRepository_pkey" PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "GithubAppInstallation_appInstallationId_key" ON "public"."GithubAppInstallation"("appInstallationId"); + +CREATE INDEX "GithubAppInstallation_organizationId_idx" ON "public"."GithubAppInstallation"("organizationId"); + +CREATE INDEX "GithubRepository_installationId_idx" ON "public"."GithubRepository"("installationId"); + +CREATE UNIQUE INDEX "GithubRepository_installationId_githubId_key" ON "public"."GithubRepository"("installationId", "githubId"); + +ALTER TABLE "public"."GithubAppInstallation" ADD CONSTRAINT "GithubAppInstallation_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "public"."Organization"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +ALTER TABLE "public"."GithubRepository" ADD CONSTRAINT "GithubRepository_installationId_fkey" FOREIGN KEY ("installationId") REFERENCES "public"."GithubAppInstallation"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/internal-packages/database/prisma/migrations/migration_lock.toml b/internal-packages/database/prisma/migrations/migration_lock.toml index 99e4f2009..044d57cdb 100644 --- a/internal-packages/database/prisma/migrations/migration_lock.toml +++ b/internal-packages/database/prisma/migrations/migration_lock.toml @@ -1,3 +1,3 @@ # Please do not edit this file manually -# It should be added in your version-control system (i.e. Git) +# It should be added in your version-control system (e.g., Git) provider = "postgresql" diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index be10d47f7..edcc82f53 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -210,6 +210,7 @@ model Organization { workerGroups WorkerInstanceGroup[] workerInstances WorkerInstance[] executionSnapshots TaskRunExecutionSnapshot[] + githubAppInstallations GithubAppInstallation[] } model OrgMember { @@ -2238,3 +2239,52 @@ model TaskEventPartitioned { // Used for getting all logs for a run @@index([runId]) } + +enum GithubRepositorySelection { + ALL + SELECTED +} + +model GithubAppInstallation { + id String @id @default(cuid()) + + appInstallationId BigInt @unique + targetId BigInt + targetType String + accountHandle String + permissions Json? + repositorySelection GithubRepositorySelection + installedBy String? + + organization Organization @relation(fields: [organizationId], references: [id]) + organizationId String + + repositories GithubRepository[] + + deletedAt DateTime? + suspendedAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@index([organizationId]) +} + +model GithubRepository { + id String @id @default(cuid()) + + githubId BigInt + name String + fullName String + htmlUrl String + private Boolean + defaultBranch String + + installation GithubAppInstallation @relation(fields: [installationId], references: [id], onDelete: Cascade, onUpdate: Cascade) + installationId String + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([installationId, githubId]) + @@index([installationId]) +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c4ddc7298..a2c6c7954 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -587,6 +587,9 @@ importers: non.geist: specifier: ^1.0.2 version: 1.0.2 + octokit: + specifier: ^3.2.1 + version: 3.2.2 ohash: specifier: ^1.1.3 version: 1.1.3 @@ -8546,6 +8549,262 @@ packages: which: 3.0.1 dev: true + /@octokit/app@14.1.0: + resolution: {integrity: sha512-g3uEsGOQCBl1+W1rgfwoRFUIR6PtvB2T1E4RpygeUU5LrLvlOqcxrt5lfykIeRpUPpupreGJUYl70fqMDXdTpw==} + engines: {node: '>= 18'} + dependencies: + '@octokit/auth-app': 6.1.4 + '@octokit/auth-unauthenticated': 5.0.1 + '@octokit/core': 5.2.2 + '@octokit/oauth-app': 6.1.0 + '@octokit/plugin-paginate-rest': 9.2.2(@octokit/core@5.2.2) + '@octokit/types': 12.6.0 + '@octokit/webhooks': 12.3.2 + dev: false + + /@octokit/auth-app@6.1.4: + resolution: {integrity: sha512-QkXkSOHZK4dA5oUqY5Dk3S+5pN2s1igPjEASNQV8/vgJgW034fQWR16u7VsNOK/EljA00eyjYF5mWNxWKWhHRQ==} + engines: {node: '>= 18'} + dependencies: + '@octokit/auth-oauth-app': 7.1.0 + '@octokit/auth-oauth-user': 4.1.0 + '@octokit/request': 8.4.1 + '@octokit/request-error': 5.1.1 + '@octokit/types': 13.10.0 + deprecation: 2.3.1 + lru-cache: /@wolfy1339/lru-cache@11.0.2-patch.1 + universal-github-app-jwt: 1.2.0 + universal-user-agent: 6.0.1 + dev: false + + /@octokit/auth-oauth-app@7.1.0: + resolution: {integrity: sha512-w+SyJN/b0l/HEb4EOPRudo7uUOSW51jcK1jwLa+4r7PA8FPFpoxEnHBHMITqCsc/3Vo2qqFjgQfz/xUUvsSQnA==} + engines: {node: '>= 18'} + dependencies: + '@octokit/auth-oauth-device': 6.1.0 + '@octokit/auth-oauth-user': 4.1.0 + '@octokit/request': 8.4.1 + '@octokit/types': 13.10.0 + '@types/btoa-lite': 1.0.2 + btoa-lite: 1.0.0 + universal-user-agent: 6.0.1 + dev: false + + /@octokit/auth-oauth-device@6.1.0: + resolution: {integrity: sha512-FNQ7cb8kASufd6Ej4gnJ3f1QB5vJitkoV1O0/g6e6lUsQ7+VsSNRHRmFScN2tV4IgKA12frrr/cegUs0t+0/Lw==} + engines: {node: '>= 18'} + dependencies: + '@octokit/oauth-methods': 4.1.0 + '@octokit/request': 8.4.1 + '@octokit/types': 13.10.0 + universal-user-agent: 6.0.1 + dev: false + + /@octokit/auth-oauth-user@4.1.0: + resolution: {integrity: sha512-FrEp8mtFuS/BrJyjpur+4GARteUCrPeR/tZJzD8YourzoVhRics7u7we/aDcKv+yywRNwNi/P4fRi631rG/OyQ==} + engines: {node: '>= 18'} + dependencies: + '@octokit/auth-oauth-device': 6.1.0 + '@octokit/oauth-methods': 4.1.0 + '@octokit/request': 8.4.1 + '@octokit/types': 13.10.0 + btoa-lite: 1.0.0 + universal-user-agent: 6.0.1 + dev: false + + /@octokit/auth-token@4.0.0: + resolution: {integrity: sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==} + engines: {node: '>= 18'} + dev: false + + /@octokit/auth-unauthenticated@5.0.1: + resolution: {integrity: sha512-oxeWzmBFxWd+XolxKTc4zr+h3mt+yofn4r7OfoIkR/Cj/o70eEGmPsFbueyJE2iBAGpjgTnEOKM3pnuEGVmiqg==} + engines: {node: '>= 18'} + dependencies: + '@octokit/request-error': 5.1.1 + '@octokit/types': 12.6.0 + dev: false + + /@octokit/core@5.2.2: + resolution: {integrity: sha512-/g2d4sW9nUDJOMz3mabVQvOGhVa4e/BN/Um7yca9Bb2XTzPPnfTWHWQg+IsEYO7M3Vx+EXvaM/I2pJWIMun1bg==} + engines: {node: '>= 18'} + dependencies: + '@octokit/auth-token': 4.0.0 + '@octokit/graphql': 7.1.1 + '@octokit/request': 8.4.1 + '@octokit/request-error': 5.1.1 + '@octokit/types': 13.10.0 + before-after-hook: 2.2.3 + universal-user-agent: 6.0.1 + dev: false + + /@octokit/endpoint@9.0.6: + resolution: {integrity: sha512-H1fNTMA57HbkFESSt3Y9+FBICv+0jFceJFPWDePYlR/iMGrwM5ph+Dd4XRQs+8X+PUFURLQgX9ChPfhJ/1uNQw==} + engines: {node: '>= 18'} + dependencies: + '@octokit/types': 13.10.0 + universal-user-agent: 6.0.1 + dev: false + + /@octokit/graphql@7.1.1: + resolution: {integrity: sha512-3mkDltSfcDUoa176nlGoA32RGjeWjl3K7F/BwHwRMJUW/IteSa4bnSV8p2ThNkcIcZU2umkZWxwETSSCJf2Q7g==} + engines: {node: '>= 18'} + dependencies: + '@octokit/request': 8.4.1 + '@octokit/types': 13.10.0 + universal-user-agent: 6.0.1 + dev: false + + /@octokit/oauth-app@6.1.0: + resolution: {integrity: sha512-nIn/8eUJ/BKUVzxUXd5vpzl1rwaVxMyYbQkNZjHrF7Vk/yu98/YDF/N2KeWO7uZ0g3b5EyiFXFkZI8rJ+DH1/g==} + engines: {node: '>= 18'} + dependencies: + '@octokit/auth-oauth-app': 7.1.0 + '@octokit/auth-oauth-user': 4.1.0 + '@octokit/auth-unauthenticated': 5.0.1 + '@octokit/core': 5.2.2 + '@octokit/oauth-authorization-url': 6.0.2 + '@octokit/oauth-methods': 4.1.0 + '@types/aws-lambda': 8.10.152 + universal-user-agent: 6.0.1 + dev: false + + /@octokit/oauth-authorization-url@6.0.2: + resolution: {integrity: sha512-CdoJukjXXxqLNK4y/VOiVzQVjibqoj/xHgInekviUJV73y/BSIcwvJ/4aNHPBPKcPWFnd4/lO9uqRV65jXhcLA==} + engines: {node: '>= 18'} + dev: false + + /@octokit/oauth-methods@4.1.0: + resolution: {integrity: sha512-4tuKnCRecJ6CG6gr0XcEXdZtkTDbfbnD5oaHBmLERTjTMZNi2CbfEHZxPU41xXLDG4DfKf+sonu00zvKI9NSbw==} + engines: {node: '>= 18'} + dependencies: + '@octokit/oauth-authorization-url': 6.0.2 + '@octokit/request': 8.4.1 + '@octokit/request-error': 5.1.1 + '@octokit/types': 13.10.0 + btoa-lite: 1.0.0 + dev: false + + /@octokit/openapi-types@20.0.0: + resolution: {integrity: sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==} + dev: false + + /@octokit/openapi-types@24.2.0: + resolution: {integrity: sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==} + dev: false + + /@octokit/plugin-paginate-graphql@4.0.1(@octokit/core@5.2.2): + resolution: {integrity: sha512-R8ZQNmrIKKpHWC6V2gum4x9LG2qF1RxRjo27gjQcG3j+vf2tLsEfE7I/wRWEPzYMaenr1M+qDAtNcwZve1ce1A==} + engines: {node: '>= 18'} + peerDependencies: + '@octokit/core': '>=5' + dependencies: + '@octokit/core': 5.2.2 + dev: false + + /@octokit/plugin-paginate-rest@11.4.4-cjs.2(@octokit/core@5.2.2): + resolution: {integrity: sha512-2dK6z8fhs8lla5PaOTgqfCGBxgAv/le+EhPs27KklPhm1bKObpu6lXzwfUEQ16ajXzqNrKMujsFyo9K2eaoISw==} + engines: {node: '>= 18'} + peerDependencies: + '@octokit/core': '5' + dependencies: + '@octokit/core': 5.2.2 + '@octokit/types': 13.10.0 + dev: false + + /@octokit/plugin-paginate-rest@9.2.2(@octokit/core@5.2.2): + resolution: {integrity: sha512-u3KYkGF7GcZnSD/3UP0S7K5XUFT2FkOQdcfXZGZQPGv3lm4F2Xbf71lvjldr8c1H3nNbF+33cLEkWYbokGWqiQ==} + engines: {node: '>= 18'} + peerDependencies: + '@octokit/core': '5' + dependencies: + '@octokit/core': 5.2.2 + '@octokit/types': 12.6.0 + dev: false + + /@octokit/plugin-rest-endpoint-methods@13.3.2-cjs.1(@octokit/core@5.2.2): + resolution: {integrity: sha512-VUjIjOOvF2oELQmiFpWA1aOPdawpyaCUqcEBc/UOUnj3Xp6DJGrJ1+bjUIIDzdHjnFNO6q57ODMfdEZnoBkCwQ==} + engines: {node: '>= 18'} + peerDependencies: + '@octokit/core': ^5 + dependencies: + '@octokit/core': 5.2.2 + '@octokit/types': 13.10.0 + dev: false + + /@octokit/plugin-retry@6.1.0(@octokit/core@5.2.2): + resolution: {integrity: sha512-WrO3bvq4E1Xh1r2mT9w6SDFg01gFmP81nIG77+p/MqW1JeXXgL++6umim3t6x0Zj5pZm3rXAN+0HEjmmdhIRig==} + engines: {node: '>= 18'} + peerDependencies: + '@octokit/core': '5' + dependencies: + '@octokit/core': 5.2.2 + '@octokit/request-error': 5.1.1 + '@octokit/types': 13.10.0 + bottleneck: 2.19.5 + dev: false + + /@octokit/plugin-throttling@8.2.0(@octokit/core@5.2.2): + resolution: {integrity: sha512-nOpWtLayKFpgqmgD0y3GqXafMFuKcA4tRPZIfu7BArd2lEZeb1988nhWhwx4aZWmjDmUfdgVf7W+Tt4AmvRmMQ==} + engines: {node: '>= 18'} + peerDependencies: + '@octokit/core': ^5.0.0 + dependencies: + '@octokit/core': 5.2.2 + '@octokit/types': 12.6.0 + bottleneck: 2.19.5 + dev: false + + /@octokit/request-error@5.1.1: + resolution: {integrity: sha512-v9iyEQJH6ZntoENr9/yXxjuezh4My67CBSu9r6Ve/05Iu5gNgnisNWOsoJHTP6k0Rr0+HQIpnH+kyammu90q/g==} + engines: {node: '>= 18'} + dependencies: + '@octokit/types': 13.10.0 + deprecation: 2.3.1 + once: 1.4.0 + dev: false + + /@octokit/request@8.4.1: + resolution: {integrity: sha512-qnB2+SY3hkCmBxZsR/MPCybNmbJe4KAlfWErXq+rBKkQJlbjdJeS85VI9r8UqeLYLvnAenU8Q1okM/0MBsAGXw==} + engines: {node: '>= 18'} + dependencies: + '@octokit/endpoint': 9.0.6 + '@octokit/request-error': 5.1.1 + '@octokit/types': 13.10.0 + universal-user-agent: 6.0.1 + dev: false + + /@octokit/types@12.6.0: + resolution: {integrity: sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==} + dependencies: + '@octokit/openapi-types': 20.0.0 + dev: false + + /@octokit/types@13.10.0: + resolution: {integrity: sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==} + dependencies: + '@octokit/openapi-types': 24.2.0 + dev: false + + /@octokit/webhooks-methods@4.1.0: + resolution: {integrity: sha512-zoQyKw8h9STNPqtm28UGOYFE7O6D4Il8VJwhAtMHFt2C4L0VQT1qGKLeefUOqHNs1mNRYSadVv7x0z8U2yyeWQ==} + engines: {node: '>= 18'} + dev: false + + /@octokit/webhooks-types@7.6.1: + resolution: {integrity: sha512-S8u2cJzklBC0FgTwWVLaM8tMrDuDMVE4xiTK4EYXM9GntyvrdbSoxqDQa+Fh57CCNApyIpyeqPhhFEmHPfrXgw==} + dev: false + + /@octokit/webhooks@12.3.2: + resolution: {integrity: sha512-exj1MzVXoP7xnAcAB3jZ97pTvVPkQF9y6GA/dvYC47HV7vLv+24XRS6b/v/XnyikpEuvMhugEXdGtAlU086WkQ==} + engines: {node: '>= 18'} + dependencies: + '@octokit/request-error': 5.1.1 + '@octokit/webhooks-methods': 4.1.0 + '@octokit/webhooks-types': 7.6.1 + aggregate-error: 3.1.0 + dev: false + /@one-ini/wasm@0.1.1: resolution: {integrity: sha512-XuySG1E38YScSJoMlqovLru4KTUNSjgVTIjyh7qMX6aNN5HY5Ct5LhRJdxO79JtTzKfzV/bnWpz+zquYrISsvw==} dev: false @@ -17538,6 +17797,10 @@ packages: resolution: {integrity: sha512-XTIieEY+gvJ39ChLcB4If5zHtPxt3Syj5rgZR+e1ctpmK8NjPf0zFqsz4JpLJT0xla9GFDKjy8Cpu331nrmE1Q==} dev: true + /@types/aws-lambda@8.10.152: + resolution: {integrity: sha512-soT/c2gYBnT5ygwiHPmd9a1bftj462NWVk2tKCc1PYHSIacB2UwbTS2zYG4jzag1mRDuzg/OjtxQjQ2NKRB6Rw==} + dev: false + /@types/bcryptjs@2.4.2: resolution: {integrity: sha512-LiMQ6EOPob/4yUL66SZzu6Yh77cbzJFYll+ZfaPiPPFswtIlA/Fs1MzdKYA7JApHU49zQTbJGX3PDmCpIdDBRQ==} dev: true @@ -17549,6 +17812,10 @@ packages: '@types/node': 20.14.14 dev: true + /@types/btoa-lite@1.0.2: + resolution: {integrity: sha512-ZYbcE2x7yrvNFJiU7xJGrpF/ihpkM7zKgw8bha3LNJSesvTtUNxbpzaT7WXBIryf6jovisrxTBvymxMeLLj1Mg==} + dev: false + /@types/bun@1.1.6: resolution: {integrity: sha512-uJgKjTdX0GkWEHZzQzFsJkWp5+43ZS7HC8sZPFnOwnSo1AsNl2q9o2bFeS23disNDqbggEgyFkKCHl/w8iZsMA==} dependencies: @@ -17808,6 +18075,13 @@ packages: resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==} dev: true + /@types/jsonwebtoken@9.0.10: + resolution: {integrity: sha512-asx5hIG9Qmf/1oStypjanR7iKTv0gXQ1Ov/jfrX6kS/EO0OFni8orbmGCn0672NHR3kXHwpAwR+B368ZGN/2rA==} + dependencies: + '@types/ms': 0.7.31 + '@types/node': 20.14.14 + dev: false + /@types/keyv@3.1.4: resolution: {integrity: sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg==} dependencies: @@ -17926,7 +18200,6 @@ packages: resolution: {integrity: sha512-acBjXdRJ3A6Pb3tqnw9HZmyR3Fiol3aGxRCK1x3d+6CDAMjl7I649wpSd+yNURCjbOUGu9tqtLKnTGxmK6CyGw==} dependencies: undici-types: 6.20.0 - dev: false /@types/nodemailer@6.4.17: resolution: {integrity: sha512-I9CCaIp6DTldEg7vyUTZi8+9Vo0hi1/T8gv3C89yk1rSAAzoKQ8H8ki/jBYJSFoH/BisgLP8tkZMlQ91CIquww==} @@ -19012,6 +19285,11 @@ packages: xstate: 5.18.1 dev: false + /@wolfy1339/lru-cache@11.0.2-patch.1: + resolution: {integrity: sha512-BgYZfL2ADCXKOw2wJtkM3slhHotawWkgIRRxq4wEybnZQPjvAp71SPX35xepMykTw8gXlzWcWPTY31hlbnRsDA==} + engines: {node: 18 >=18.20 || 20 || >=22} + dev: false + /@xobotyi/scrollbar-width@1.9.5: resolution: {integrity: sha512-N8tkAACJx2ww8vFMneJmaAgmjAG1tnVBZJRLRcx061tmsLRZHSEZSLuGWnwPtunsSLvSqXQ2wfp7Mgqg1I+2dQ==} dev: false @@ -19181,7 +19459,6 @@ packages: dependencies: clean-stack: 2.2.0 indent-string: 4.0.0 - dev: true /aggregate-error@4.0.1: resolution: {integrity: sha512-0poP0T7el6Vq3rstR8Mn4V/IQrpBLO6POkUSrN7RhyY+GF/InCFShQzsQ39T25gkHhLgSLByyAz+Kjb+c2L98w==} @@ -19878,6 +20155,10 @@ packages: dependencies: tweetnacl: 0.14.5 + /before-after-hook@2.2.3: + resolution: {integrity: sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==} + dev: false + /better-path-resolve@1.0.0: resolution: {integrity: sha512-pbnl5XzGBdrFU/wT4jqmJVPn2B6UHPBOhzMQkY/SPUPB6QtUXtmBHBIwCbXJol93mOpGMnQyP/+BB19q04xj7g==} engines: {node: '>=4'} @@ -19958,6 +20239,10 @@ packages: - supports-color dev: false + /bottleneck@2.19.5: + resolution: {integrity: sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==} + dev: false + /bowser@2.11.0: resolution: {integrity: sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==} dev: false @@ -20023,6 +20308,10 @@ packages: update-browserslist-db: 1.1.3(browserslist@4.25.0) dev: true + /btoa-lite@1.0.0: + resolution: {integrity: sha512-gvW7InbIyF8AicrqWoptdW08pUxuhq8BEgowNajy9RhiE86fmGAGl+bLKo6oB8QP0CkqHLowfN0oJdKC/J6LbA==} + dev: false + /buffer-crc32@0.2.13: resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} dev: false @@ -20032,6 +20321,10 @@ packages: engines: {node: '>=8.0.0'} dev: true + /buffer-equal-constant-time@1.0.1: + resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + dev: false + /buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} @@ -20453,7 +20746,6 @@ packages: /clean-stack@2.2.0: resolution: {integrity: sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==} engines: {node: '>=6'} - dev: true /clean-stack@4.2.0: resolution: {integrity: sha512-LYv6XPxoyODi36Dp976riBtSY27VmFo+MKqEU9QCCWyTrdEPDog+RWA7xQWHi6Vbp61j5c4cdzzX1NidnwtUWg==} @@ -21440,6 +21732,10 @@ packages: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} + /deprecation@2.3.1: + resolution: {integrity: sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==} + dev: false + /dequal@2.0.3: resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} engines: {node: '>=6'} @@ -21702,6 +21998,12 @@ packages: safer-buffer: 2.1.2 dev: false + /ecdsa-sig-formatter@1.0.11: + resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} + dependencies: + safe-buffer: 5.2.1 + dev: false + /editorconfig@1.0.4: resolution: {integrity: sha512-L9Qe08KWTlqYMVvMcTIvMAdl1cDUubzRNYL+WfA4bLDMHe4nemKkpmYzkznE1FwLKu0EEmy6obgQKzMJrg4x9Q==} engines: {node: '>=14'} @@ -21834,7 +22136,7 @@ packages: engines: {node: '>=10.13.0'} dependencies: graceful-fs: 4.2.11 - tapable: 2.2.1 + tapable: 2.2.2 /enquirer@2.3.6: resolution: {integrity: sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==} @@ -25124,7 +25426,7 @@ packages: resolution: {integrity: sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==} engines: {node: '>= 10.13.0'} dependencies: - '@types/node': 20.14.14 + '@types/node': 22.13.9 merge-stream: 2.0.0 supports-color: 8.1.1 @@ -25338,6 +25640,22 @@ packages: engines: {node: '>=0.10.0'} dev: false + /jsonwebtoken@9.0.2: + resolution: {integrity: sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==} + engines: {node: '>=12', npm: '>=6'} + dependencies: + jws: 3.2.2 + lodash.includes: 4.3.0 + lodash.isboolean: 3.0.3 + lodash.isinteger: 4.0.4 + lodash.isnumber: 3.0.3 + lodash.isplainobject: 4.0.6 + lodash.isstring: 4.0.1 + lodash.once: 4.1.1 + ms: 2.1.3 + semver: 7.7.2 + dev: false + /jsprim@1.4.2: resolution: {integrity: sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==} engines: {node: '>=0.6.0'} @@ -25361,6 +25679,21 @@ packages: engines: {node: '>=12.20'} dev: true + /jwa@1.4.2: + resolution: {integrity: sha512-eeH5JO+21J78qMvTIDdBXidBd6nG2kZjg5Ohz/1fpa28Z4CcsWUzJ1ZZyFq/3z3N17aZy+ZuBoHljASbL1WfOw==} + dependencies: + buffer-equal-constant-time: 1.0.1 + ecdsa-sig-formatter: 1.0.11 + safe-buffer: 5.2.1 + dev: false + + /jws@3.2.2: + resolution: {integrity: sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==} + dependencies: + jwa: 1.4.2 + safe-buffer: 5.2.1 + dev: false + /keyv@3.1.0: resolution: {integrity: sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==} dependencies: @@ -25742,21 +26075,40 @@ packages: resolution: {integrity: sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==} dev: false + /lodash.includes@4.3.0: + resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==} + dev: false + /lodash.isarguments@3.1.0: resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} dev: false + /lodash.isboolean@3.0.3: + resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==} + dev: false + /lodash.isfunction@3.0.9: resolution: {integrity: sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==} dev: false + /lodash.isinteger@4.0.4: + resolution: {integrity: sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==} + dev: false + /lodash.isnil@4.0.0: resolution: {integrity: sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==} dev: false + /lodash.isnumber@3.0.3: + resolution: {integrity: sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==} + dev: false + /lodash.isplainobject@4.0.6: resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==} - dev: true + + /lodash.isstring@4.0.1: + resolution: {integrity: sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==} + dev: false /lodash.isundefined@3.0.1: resolution: {integrity: sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==} @@ -25769,6 +26121,10 @@ packages: resolution: {integrity: sha512-XeqSp49hNGmlkj2EJlfrQFIzQ6lXdNro9sddtQzcJY8QaoC2GO0DT7xaIokHeyM+mIT0mPMlPvkYzg2xCuHdZg==} dev: false + /lodash.once@4.1.1: + resolution: {integrity: sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==} + dev: false + /lodash.sortby@4.7.0: resolution: {integrity: sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA==} dev: true @@ -27631,6 +27987,23 @@ packages: /obuf@1.1.2: resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==} + /octokit@3.2.2: + resolution: {integrity: sha512-7Abo3nADdja8l/aglU6Y3lpnHSfv0tw7gFPiqzry/yCU+2gTAX7R1roJ8hJrxIK+S1j+7iqRJXtmuHJ/UDsBhQ==} + engines: {node: '>= 18'} + dependencies: + '@octokit/app': 14.1.0 + '@octokit/core': 5.2.2 + '@octokit/oauth-app': 6.1.0 + '@octokit/plugin-paginate-graphql': 4.0.1(@octokit/core@5.2.2) + '@octokit/plugin-paginate-rest': 11.4.4-cjs.2(@octokit/core@5.2.2) + '@octokit/plugin-rest-endpoint-methods': 13.3.2-cjs.1(@octokit/core@5.2.2) + '@octokit/plugin-retry': 6.1.0(@octokit/core@5.2.2) + '@octokit/plugin-throttling': 8.2.0(@octokit/core@5.2.2) + '@octokit/request-error': 5.1.1 + '@octokit/types': 13.10.0 + '@octokit/webhooks': 12.3.2 + dev: false + /ohash@1.1.3: resolution: {integrity: sha512-zuHHiGTYTA1sYJ/wZN+t5HKZaH23i4yI1HMwbuXm24Nid7Dv0KcuRlKoNKS9UNfAVSBlnGLcuQrnOKWOZoEGaw==} dev: false @@ -32014,7 +32387,6 @@ packages: /tapable@2.2.2: resolution: {integrity: sha512-Re10+NauLTMCudc7T5WLFLAwDhQ0JWdrMK+9B2M8zR5hRExKmsRDCBA7/aV/pNJFltmBFO5BAMlQFi/vq3nKOg==} engines: {node: '>=6'} - dev: true /tar-fs@2.1.3: resolution: {integrity: sha512-090nwYJDmlhwFwEW3QQl+vaNnxsO2yVsd45eTKRBzSzu+hlb1w2K9inVq5b0ngXuLVqQ4ApvsUHHnu/zQNkWAg==} @@ -33000,7 +33372,6 @@ packages: /undici-types@6.20.0: resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==} - dev: false /undici@5.28.4: resolution: {integrity: sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==} @@ -33152,6 +33523,17 @@ packages: cookie: 0.6.0 dev: false + /universal-github-app-jwt@1.2.0: + resolution: {integrity: sha512-dncpMpnsKBk0eetwfN8D8OUHGfiDhhJ+mtsbMl+7PfW7mYjiH8LIcqRmYMtzYLgSh47HjfdBtrBwIQ/gizKR3g==} + dependencies: + '@types/jsonwebtoken': 9.0.10 + jsonwebtoken: 9.0.2 + dev: false + + /universal-user-agent@6.0.1: + resolution: {integrity: sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==} + dev: false + /universalify@0.1.2: resolution: {integrity: sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==} engines: {node: '>= 4.0.0'} From ed23615aa444212affae9004356bbd92f34f7090 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Wed, 3 Sep 2025 11:39:03 +0100 Subject: [PATCH 06/13] perf(webapp): add event loop utilization metric (#2471) * perf(webapp): add event loop utilization metric * add event loop utilization logging as well --- apps/webapp/app/env.server.ts | 2 ++ apps/webapp/app/eventLoopMonitor.server.ts | 32 ++++++++++++++++++++++ apps/webapp/app/v3/tracer.server.ts | 21 ++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 350463a5b..059195d54 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -1167,6 +1167,8 @@ const EnvironmentSchema = z AI_RUN_FILTER_MODEL: z.string().optional(), EVENT_LOOP_MONITOR_THRESHOLD_MS: z.coerce.number().int().default(100), + EVENT_LOOP_MONITOR_UTILIZATION_INTERVAL_MS: z.coerce.number().int().default(1000), + EVENT_LOOP_MONITOR_UTILIZATION_SAMPLE_RATE: z.coerce.number().default(0.05), VERY_SLOW_QUERY_THRESHOLD_MS: z.coerce.number().int().optional(), }) diff --git a/apps/webapp/app/eventLoopMonitor.server.ts b/apps/webapp/app/eventLoopMonitor.server.ts index 42e982bdb..1d8603a4a 100644 --- a/apps/webapp/app/eventLoopMonitor.server.ts +++ b/apps/webapp/app/eventLoopMonitor.server.ts @@ -3,6 +3,8 @@ import { singleton } from "./utils/singleton"; import { tracer } from "./v3/tracer.server"; import { env } from "./env.server"; import { context, Context } from "@opentelemetry/api"; +import { performance } from "node:perf_hooks"; +import { logger } from "./services/logger.server"; const THRESHOLD_NS = env.EVENT_LOOP_MONITOR_THRESHOLD_MS * 1e6; @@ -69,16 +71,46 @@ function after(asyncId: number) { export const eventLoopMonitor = singleton("eventLoopMonitor", () => { const hook = createHook({ init, before, after, destroy }); + let stopEventLoopUtilizationMonitoring: () => void; + return { enable: () => { console.log("🥸 Initializing event loop monitor"); hook.enable(); + + stopEventLoopUtilizationMonitoring = startEventLoopUtilizationMonitoring(); }, disable: () => { console.log("🥸 Disabling event loop monitor"); hook.disable(); + + stopEventLoopUtilizationMonitoring?.(); }, }; }); + +function startEventLoopUtilizationMonitoring() { + let lastEventLoopUtilization = performance.eventLoopUtilization(); + + const interval = setInterval(() => { + const currentEventLoopUtilization = performance.eventLoopUtilization(); + + const diff = performance.eventLoopUtilization( + currentEventLoopUtilization, + lastEventLoopUtilization + ); + const utilization = Number.isFinite(diff.utilization) ? diff.utilization : 0; + + if (Math.random() < env.EVENT_LOOP_MONITOR_UTILIZATION_SAMPLE_RATE) { + logger.info("nodejs.event_loop.utilization", { utilization }); + } + + lastEventLoopUtilization = currentEventLoopUtilization; + }, env.EVENT_LOOP_MONITOR_UTILIZATION_INTERVAL_MS); + + return () => { + clearInterval(interval); + }; +} diff --git a/apps/webapp/app/v3/tracer.server.ts b/apps/webapp/app/v3/tracer.server.ts index d084f78a7..71e14521e 100644 --- a/apps/webapp/app/v3/tracer.server.ts +++ b/apps/webapp/app/v3/tracer.server.ts @@ -57,6 +57,7 @@ import { flattenAttributes } from "@trigger.dev/core/v3"; import { prisma } from "~/db.server"; import { metricsRegister } from "~/metrics.server"; import type { Prisma } from "@trigger.dev/database"; +import { performance } from "node:perf_hooks"; export const SEMINTATTRS_FORCE_RECORDING = "forceRecording"; @@ -602,10 +603,17 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) { description: "Event loop 99th percentile delay", unit: "s", }); + // ELU observable gauge (unit is a ratio, 0..1) + const eluGauge = meter.createObservableGauge("nodejs.event_loop.utilization", { + description: "Event loop utilization over the last collection interval", + unit: "1", // OpenTelemetry convention for ratios + }); // Get UV threadpool size (defaults to 4 if not set) const uvThreadpoolSize = parseInt(process.env.UV_THREADPOOL_SIZE || "4", 10); + let lastEventLoopUtilization = performance.eventLoopUtilization(); + // Single helper to read metrics from prom-client async function readNodeMetrics() { const metrics = await metricsRegister.getMetricsAsJSON(); @@ -648,6 +656,16 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) { } } + const currentEventLoopUtilization = performance.eventLoopUtilization(); + // Diff over [lastSnapshot, current] + const diff = performance.eventLoopUtilization( + currentEventLoopUtilization, + lastEventLoopUtilization + ); + + // diff.utilization is between 0 and 1 (fraction of time "active") + const utilization = Number.isFinite(diff.utilization) ? diff.utilization : 0; + return { threadpoolSize: uvThreadpoolSize, handlesByType, @@ -661,6 +679,7 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) { p50: eventLoopLagP50?.values?.[0]?.value ?? 0, p90: eventLoopLagP90?.values?.[0]?.value ?? 0, p99: eventLoopLagP99?.values?.[0]?.value ?? 0, + utilization, }, }; } @@ -698,6 +717,7 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) { res.observe(eventLoopLagP50Gauge, eventLoop.p50); res.observe(eventLoopLagP90Gauge, eventLoop.p90); res.observe(eventLoopLagP99Gauge, eventLoop.p99); + res.observe(eluGauge, eventLoop.utilization); }, [ uvThreadpoolSizeGauge, @@ -711,6 +731,7 @@ function configureNodejsMetrics({ meter }: { meter: Meter }) { eventLoopLagP50Gauge, eventLoopLagP90Gauge, eventLoopLagP99Gauge, + eluGauge, ] ); } From 59c17e04e95d4014a24ca8c194a73609b43ca59f Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Thu, 4 Sep 2025 09:27:06 +0100 Subject: [PATCH 07/13] feat(run-engine): worker queue resolver (#2476) --- .../run-engine/src/run-queue/index.ts | 19 +- .../tests/workerQueueResolver.test.ts | 484 ++++++++++++++++++ .../src/run-queue/workerQueueResolver.ts | 100 ++++ 3 files changed, 590 insertions(+), 13 deletions(-) create mode 100644 internal-packages/run-engine/src/run-queue/tests/workerQueueResolver.test.ts create mode 100644 internal-packages/run-engine/src/run-queue/workerQueueResolver.ts diff --git a/internal-packages/run-engine/src/run-queue/index.ts b/internal-packages/run-engine/src/run-queue/index.ts index ec12c19f1..81cbb0379 100644 --- a/internal-packages/run-engine/src/run-queue/index.ts +++ b/internal-packages/run-engine/src/run-queue/index.ts @@ -42,6 +42,7 @@ import { RunQueueKeyProducer, RunQueueSelectionStrategy, } from "./types.js"; +import { WorkerQueueResolver } from "./workerQueueResolver.js"; const SemanticAttributes = { QUEUE: "runqueue.queue", @@ -169,6 +170,7 @@ export class RunQueue { private shardCount: number; private abortController: AbortController; private worker: Worker; + private workerQueueResolver: WorkerQueueResolver; private _observableWorkerQueues: Set = new Set(); private _meter: Meter; private _queueCooloffStates: Map = new Map(); @@ -185,6 +187,8 @@ export class RunQueue { }, }); this.logger = options.logger ?? new Logger("RunQueue", options.logLevel ?? "info"); + + this.workerQueueResolver = new WorkerQueueResolver({ logger: this.logger }); this._meter = options.meter ?? getMeter("run-queue"); const workerQueueObservableGauge = this._meter.createObservableGauge( @@ -1845,19 +1849,8 @@ export class RunQueue { ); } - #getWorkerQueueFromMessage(message: OutputPayload) { - if (message.version === "2") { - return message.workerQueue; - } - - // In v2, if the environment is development, the worker queue is the environment id. - if (message.environmentType === "DEVELOPMENT") { - return message.environmentId; - } - - // In v1, the master queue is something like us-nyc-3, - // which in v2 is the worker queue. - return message.masterQueues[0]; + #getWorkerQueueFromMessage(message: OutputPayload): string { + return this.workerQueueResolver.getWorkerQueueFromMessage(message); } #createBlockingDequeueClient() { diff --git a/internal-packages/run-engine/src/run-queue/tests/workerQueueResolver.test.ts b/internal-packages/run-engine/src/run-queue/tests/workerQueueResolver.test.ts new file mode 100644 index 000000000..3ea8f0ce7 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/tests/workerQueueResolver.test.ts @@ -0,0 +1,484 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { Logger } from "@trigger.dev/core/logger"; +import { WorkerQueueResolver, type WorkerQueueOverrides } from "../workerQueueResolver.js"; +import { OutputPayload, OutputPayloadV1, OutputPayloadV2 } from "../types.js"; +import { RuntimeEnvironmentType } from "@trigger.dev/core/v3"; + +vi.setConfig({ testTimeout: 5_000 }); + +describe("WorkerQueueOverrideResolver", () => { + const createTestMessage = (overrides?: Partial): OutputPayloadV2 => ({ + version: "2", + runId: "run_123", + taskIdentifier: "task_123", + orgId: "org_123", + projectId: "proj_123", + environmentId: "env_123", + environmentType: RuntimeEnvironmentType.PRODUCTION, + queue: "test-queue", + timestamp: Date.now(), + attempt: 0, + workerQueue: "default-queue", + ...overrides, + }); + + describe("No overrides", () => { + it("should return original workerQueue when no overrides are set", () => { + const logger = new Logger("test", "error"); + const resolver = new WorkerQueueResolver({ logger }); + const message = createTestMessage({ workerQueue: "original-queue" }); + + const result = resolver.getWorkerQueueFromMessage(message); + + expect(result).toBe("original-queue"); + }); + }); + + describe("Environment ID overrides", () => { + it("should override based on environmentId", () => { + const overrideConfig = JSON.stringify({ + environmentId: { + env_special: "special-env-queue", + }, + } satisfies WorkerQueueOverrides); + + const logger = new Logger("test", "error"); + const resolver = new WorkerQueueResolver({ logger, overrideConfig }); + const message = createTestMessage({ + environmentId: "env_special", + workerQueue: "original-queue", + }); + + const result = resolver.getWorkerQueueFromMessage(message); + + expect(result).toBe("special-env-queue"); + }); + + it("should not override when environmentId doesn't match", () => { + const overrideConfig = JSON.stringify({ + environmentId: { + env_other: "other-queue", + }, + } satisfies WorkerQueueOverrides); + + const logger = new Logger("test", "error"); + const resolver = new WorkerQueueResolver({ logger, overrideConfig }); + const message = createTestMessage({ + environmentId: "env_123", + workerQueue: "original-queue", + }); + + const result = resolver.getWorkerQueueFromMessage(message); + + expect(result).toBe("original-queue"); + }); + }); + + describe("Project ID overrides", () => { + it("should override based on projectId", () => { + const overrideConfig = JSON.stringify({ + projectId: { + proj_special: "special-project-queue", + }, + } satisfies WorkerQueueOverrides); + + const logger = new Logger("test", "error"); + const resolver = new WorkerQueueResolver({ logger, overrideConfig }); + const message = createTestMessage({ + projectId: "proj_special", + workerQueue: "original-queue", + }); + + const result = resolver.getWorkerQueueFromMessage(message); + + expect(result).toBe("special-project-queue"); + }); + }); + + describe("Organization ID overrides", () => { + it("should override based on orgId", () => { + const overrideConfig = JSON.stringify({ + orgId: { + org_special: "special-org-queue", + }, + } satisfies WorkerQueueOverrides); + + const logger = new Logger("test", "error"); + const resolver = new WorkerQueueResolver({ logger, overrideConfig }); + const message = createTestMessage({ + orgId: "org_special", + workerQueue: "original-queue", + }); + + const result = resolver.getWorkerQueueFromMessage(message); + + expect(result).toBe("special-org-queue"); + }); + }); + + describe("Worker Queue overrides", () => { + it("should override based on workerQueue", () => { + const overrideConfig = JSON.stringify({ + workerQueue: { + "us-east-1": "us-west-1", + }, + } satisfies WorkerQueueOverrides); + + const logger = new Logger("test", "error"); + const resolver = new WorkerQueueResolver({ logger, overrideConfig }); + const message = createTestMessage({ + workerQueue: "us-east-1", + }); + + const result = resolver.getWorkerQueueFromMessage(message); + + expect(result).toBe("us-west-1"); + }); + }); + + describe("Priority order", () => { + it("should prioritize environmentId over projectId", () => { + const overrideConfig = JSON.stringify({ + environmentId: { + env_123: "env-queue", + }, + projectId: { + proj_123: "project-queue", + }, + } satisfies WorkerQueueOverrides); + + const logger = new Logger("test", "error"); + const resolver = new WorkerQueueResolver({ logger, overrideConfig }); + const message = createTestMessage(); + + const result = resolver.getWorkerQueueFromMessage(message); + + expect(result).toBe("env-queue"); + }); + + it("should prioritize projectId over orgId", () => { + const overrideConfig = JSON.stringify({ + projectId: { + proj_123: "project-queue", + }, + orgId: { + org_123: "org-queue", + }, + } satisfies WorkerQueueOverrides); + + const logger = new Logger("test", "error"); + const resolver = new WorkerQueueResolver({ logger, overrideConfig }); + const message = createTestMessage(); + + const result = resolver.getWorkerQueueFromMessage(message); + + expect(result).toBe("project-queue"); + }); + + it("should prioritize orgId over workerQueue", () => { + const overrideConfig = JSON.stringify({ + orgId: { + org_123: "org-queue", + }, + workerQueue: { + "default-queue": "worker-override-queue", + }, + } satisfies WorkerQueueOverrides); + + const logger = new Logger("test", "error"); + const resolver = new WorkerQueueResolver({ logger, overrideConfig }); + const message = createTestMessage(); + + const result = resolver.getWorkerQueueFromMessage(message); + + expect(result).toBe("org-queue"); + }); + }); + + describe("Configuration parsing", () => { + it("should handle invalid JSON gracefully", () => { + const loggerSpy = vi.spyOn(Logger.prototype, "error"); + + const overrideConfig = "invalid json {"; + + const logger = new Logger("test", "error"); + const resolver = new WorkerQueueResolver({ logger, overrideConfig }); + const message = createTestMessage(); + + const result = resolver.getWorkerQueueFromMessage(message); + + expect(result).toBe("default-queue"); + expect(loggerSpy).toHaveBeenCalledWith( + "Failed to parse worker queue overrides json", + expect.any(Object) + ); + + loggerSpy.mockRestore(); + }); + + it("should handle non-object JSON gracefully", () => { + const loggerSpy = vi.spyOn(Logger.prototype, "error"); + + const overrideConfig = JSON.stringify("not an object"); + + const logger = new Logger("test", "error"); + const resolver = new WorkerQueueResolver({ logger, overrideConfig }); + const message = createTestMessage(); + + const result = resolver.getWorkerQueueFromMessage(message); + + expect(result).toBe("default-queue"); + expect(loggerSpy).toHaveBeenCalledWith( + "Invalid worker queue overrides format", + expect.any(Object) + ); + + loggerSpy.mockRestore(); + }); + + it("should handle null JSON gracefully", () => { + const loggerSpy = vi.spyOn(Logger.prototype, "error"); + + const overrideConfig = JSON.stringify(null); + + const logger = new Logger("test", "error"); + const resolver = new WorkerQueueResolver({ logger, overrideConfig }); + const message = createTestMessage(); + + const result = resolver.getWorkerQueueFromMessage(message); + + expect(result).toBe("default-queue"); + expect(loggerSpy).toHaveBeenCalledWith( + "Invalid worker queue overrides format", + expect.any(Object) + ); + + loggerSpy.mockRestore(); + }); + + it("should log when overrides are enabled", () => { + const loggerSpy = vi.spyOn(Logger.prototype, "info"); + + const overrides: WorkerQueueOverrides = { + orgId: { org_123: "dedicated-queue" }, + }; + + const overrideConfig = JSON.stringify(overrides); + + const logger = new Logger("test", "info"); + new WorkerQueueResolver({ logger, overrideConfig }); + + expect(loggerSpy).toHaveBeenCalledWith("🎯 Worker queue overrides enabled", { overrides }); + + loggerSpy.mockRestore(); + }); + + it("should validate schema and reject invalid structure", () => { + const loggerSpy = vi.spyOn(Logger.prototype, "error"); + + // Invalid structure - numbers instead of strings in the record + const overrideConfig = JSON.stringify({ + orgId: { + org_123: 12345, // Should be string, not number + }, + }); + + const logger = new Logger("test", "error"); + const resolver = new WorkerQueueResolver({ logger, overrideConfig }); + const message = createTestMessage(); + + const result = resolver.getWorkerQueueFromMessage(message); + + expect(result).toBe("default-queue"); + expect(loggerSpy).toHaveBeenCalledWith( + "Invalid worker queue overrides format", + expect.any(Object) + ); + + loggerSpy.mockRestore(); + }); + }); + + describe("Complex scenarios", () => { + it("should handle multiple override types simultaneously", () => { + const overrideConfig = JSON.stringify({ + environmentId: { + env_special: "special-env-queue", + }, + projectId: { + proj_other: "other-project-queue", + }, + orgId: { + org_123: "org-queue", + }, + workerQueue: { + "fallback-queue": "redirected-queue", + }, + }); + + const logger = new Logger("test", "error"); + const resolver = new WorkerQueueResolver({ logger, overrideConfig }); + + // Should use orgId override since env and project don't match + const message1 = createTestMessage({ + environmentId: "env_123", + projectId: "proj_123", + orgId: "org_123", + workerQueue: "original-queue", + }); + + const result1 = resolver.getWorkerQueueFromMessage(message1); + expect(result1).toBe("org-queue"); + + // Should use environmentId override since it matches + const message2 = createTestMessage({ + environmentId: "env_special", + projectId: "proj_123", + orgId: "org_456", + workerQueue: "original-queue", + }); + + const result2 = resolver.getWorkerQueueFromMessage(message2); + expect(result2).toBe("special-env-queue"); + + // Should use workerQueue override as fallback + const message3 = createTestMessage({ + environmentId: "env_unknown", + projectId: "proj_unknown", + orgId: "org_unknown", + workerQueue: "fallback-queue", + }); + + const result3 = resolver.getWorkerQueueFromMessage(message3); + expect(result3).toBe("redirected-queue"); + }); + + it("should handle empty override sections", () => { + const overrideConfig = JSON.stringify({ + environmentId: {}, + projectId: {}, + orgId: { + org_123: "org-queue", + }, + workerQueue: {}, + }); + + const logger = new Logger("test", "error"); + const resolver = new WorkerQueueResolver({ logger, overrideConfig }); + const message = createTestMessage(); + + const result = resolver.getWorkerQueueFromMessage(message); + + expect(result).toBe("org-queue"); + }); + }); + + describe("V1 message handling", () => { + it("should handle v1 development messages", () => { + const logger = new Logger("test", "error"); + const resolver = new WorkerQueueResolver({ logger }); + + const v1DevMessage: OutputPayloadV1 = { + version: "1", + runId: "run_123", + taskIdentifier: "task_123", + orgId: "org_123", + projectId: "proj_123", + environmentId: "env_dev", + environmentType: RuntimeEnvironmentType.DEVELOPMENT, + queue: "test-queue", + timestamp: Date.now(), + attempt: 0, + masterQueues: ["us-east-1", "us-west-1"], + }; + + const result = resolver.getWorkerQueueFromMessage(v1DevMessage); + + expect(result).toBe("env_dev"); + }); + + it("should handle v1 production messages", () => { + const logger = new Logger("test", "error"); + const resolver = new WorkerQueueResolver({ logger }); + + const v1ProdMessage: OutputPayloadV1 = { + version: "1", + runId: "run_123", + taskIdentifier: "task_123", + orgId: "org_123", + projectId: "proj_123", + environmentId: "env_prod", + environmentType: RuntimeEnvironmentType.PRODUCTION, + queue: "test-queue", + timestamp: Date.now(), + attempt: 0, + masterQueues: ["us-east-1", "us-west-1"], + }; + + const result = resolver.getWorkerQueueFromMessage(v1ProdMessage); + + expect(result).toBe("us-east-1"); + }); + }); + + describe("Environment variable fallback", () => { + let originalEnv: string | undefined; + + beforeEach(() => { + originalEnv = process.env.RUN_ENGINE_WORKER_QUEUE_OVERRIDES; + }); + + afterEach(() => { + if (originalEnv === undefined) { + delete process.env.RUN_ENGINE_WORKER_QUEUE_OVERRIDES; + } else { + process.env.RUN_ENGINE_WORKER_QUEUE_OVERRIDES = originalEnv; + } + }); + + it("should fall back to environment variable when no overrideConfig provided", () => { + // Set environment variable + process.env.RUN_ENGINE_WORKER_QUEUE_OVERRIDES = JSON.stringify({ + orgId: { + org_from_env: "env-based-queue", + }, + }); + + const logger = new Logger("test", "error"); + const resolver = new WorkerQueueResolver({ logger }); // No overrideConfig + const message = createTestMessage({ + orgId: "org_from_env", + }); + + const result = resolver.getWorkerQueueFromMessage(message); + + expect(result).toBe("env-based-queue"); + }); + + it("should prioritize overrideConfig over environment variable", () => { + // Set environment variable + process.env.RUN_ENGINE_WORKER_QUEUE_OVERRIDES = JSON.stringify({ + orgId: { + org_123: "env-queue", + }, + }); + + // Pass config directly (should take precedence) + const overrideConfig = JSON.stringify({ + orgId: { + org_123: "config-queue", + }, + }); + + const logger = new Logger("test", "error"); + const resolver = new WorkerQueueResolver({ logger, overrideConfig }); + const message = createTestMessage({ + orgId: "org_123", + }); + + const result = resolver.getWorkerQueueFromMessage(message); + + expect(result).toBe("config-queue"); + }); + }); +}); diff --git a/internal-packages/run-engine/src/run-queue/workerQueueResolver.ts b/internal-packages/run-engine/src/run-queue/workerQueueResolver.ts new file mode 100644 index 000000000..5c83d3de6 --- /dev/null +++ b/internal-packages/run-engine/src/run-queue/workerQueueResolver.ts @@ -0,0 +1,100 @@ +import type { Logger } from "@trigger.dev/core/logger"; +import type { OutputPayload, OutputPayloadV2 } from "./types.js"; +import { z } from "zod"; + +const WorkerQueueOverrides = z.object({ + environmentId: z.record(z.string(), z.string()).optional(), + projectId: z.record(z.string(), z.string()).optional(), + orgId: z.record(z.string(), z.string()).optional(), + workerQueue: z.record(z.string(), z.string()).optional(), +}); + +export type WorkerQueueOverrides = z.infer; + +export type WorkerQueueResolverOptions = { + logger: Logger; + overrideConfig?: string; +}; + +export class WorkerQueueResolver { + private overrides: WorkerQueueOverrides | null; + private logger: Logger; + + constructor(opts: WorkerQueueResolverOptions) { + this.logger = opts.logger; + this.overrides = this.parseOverrides(opts.overrideConfig); + } + + private parseOverrides(overrideConfig?: string): WorkerQueueOverrides | null { + const overridesJson = overrideConfig ?? process.env.RUN_ENGINE_WORKER_QUEUE_OVERRIDES; + + if (!overridesJson) { + return null; + } + + try { + const parsed = JSON.parse(overridesJson); + const result = WorkerQueueOverrides.safeParse(parsed); + + if (!result.success) { + this.logger.error("Invalid worker queue overrides format", { + error: result.error.format(), + }); + return null; + } + + this.logger.info("🎯 Worker queue overrides enabled", { overrides: result.data }); + + return result.data; + } catch (error) { + this.logger.error("Failed to parse worker queue overrides json", { + error, + }); + return null; + } + } + + public getWorkerQueueFromMessage(message: OutputPayload): string { + if (message.version === "2") { + // Check overrides in priority order + const override = this.#getOverride(message); + if (override) return override; + + return message.workerQueue; + } + + // In v2, if the environment is development, the worker queue is the environment id. + if (message.environmentType === "DEVELOPMENT") { + return message.environmentId; + } + + // In v1, the master queue is something like us-nyc-3, + // which in v2 is the worker queue. + return message.masterQueues[0]; + } + + #getOverride(message: OutputPayloadV2): string | null { + if (!this.overrides) { + return null; + } + + // Priority: environmentId > projectId > orgId > workerQueue + if (this.overrides.environmentId?.[message.environmentId]) { + return this.overrides.environmentId[message.environmentId]; + } + + if (this.overrides.projectId?.[message.projectId]) { + return this.overrides.projectId[message.projectId]; + } + + if (this.overrides.orgId?.[message.orgId]) { + return this.overrides.orgId[message.orgId]; + } + + if (this.overrides.workerQueue?.[message.workerQueue]) { + return this.overrides.workerQueue[message.workerQueue]; + } + + return null; + } +} From a1e9738faaa3341c87861418214a01a9af5a20fd Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Thu, 4 Sep 2025 09:37:22 +0100 Subject: [PATCH 08/13] fix(webapp): prevent duplicate preview env image tags (#2475) * fix(webapp): prevent duplicate preview env image tags * use deploy shortcode instead of new nanoid * replace regexps in tests --- .../app/v3/getDeploymentImageRef.server.ts | 10 +- .../services/initializeDeployment.server.ts | 7 +- .../webapp/test/getDeploymentImageRef.test.ts | 171 +++++++++++------- 3 files changed, 115 insertions(+), 73 deletions(-) diff --git a/apps/webapp/app/v3/getDeploymentImageRef.server.ts b/apps/webapp/app/v3/getDeploymentImageRef.server.ts index e01f340a2..a456791e9 100644 --- a/apps/webapp/app/v3/getDeploymentImageRef.server.ts +++ b/apps/webapp/app/v3/getDeploymentImageRef.server.ts @@ -11,6 +11,7 @@ import { STSClient, AssumeRoleCommand } from "@aws-sdk/client-sts"; import { tryCatch } from "@trigger.dev/core"; import { logger } from "~/services/logger.server"; import { type RegistryConfig } from "./registryConfig.server"; +import type { EnvironmentType } from "@trigger.dev/core/v3"; // Optional configuration for cross-account access export type AssumeRoleConfig = { @@ -101,19 +102,22 @@ export async function getDeploymentImageRef({ registry, projectRef, nextVersion, - environmentSlug, + environmentType, + deploymentShortCode, }: { registry: RegistryConfig; projectRef: string; nextVersion: string; - environmentSlug: string; + environmentType: EnvironmentType; + deploymentShortCode: string; }): Promise<{ imageRef: string; isEcr: boolean; repoCreated: boolean; }> { const repositoryName = `${registry.namespace}/${projectRef}`; - const imageRef = `${registry.host}/${repositoryName}:${nextVersion}.${environmentSlug}`; + const envType = environmentType.toLowerCase(); + const imageRef = `${registry.host}/${repositoryName}:${nextVersion}.${envType}.${deploymentShortCode}`; if (!isEcrRegistry(registry.host)) { return { diff --git a/apps/webapp/app/v3/services/initializeDeployment.server.ts b/apps/webapp/app/v3/services/initializeDeployment.server.ts index cdd174ed9..234cfa127 100644 --- a/apps/webapp/app/v3/services/initializeDeployment.server.ts +++ b/apps/webapp/app/v3/services/initializeDeployment.server.ts @@ -73,12 +73,15 @@ export class InitializeDeploymentService extends BaseService { const isV4Deployment = payload.type === "MANAGED"; const registryConfig = getRegistryConfig(isV4Deployment); + const deploymentShortCode = nanoid(8); + const [imageRefError, imageRefResult] = await tryCatch( getDeploymentImageRef({ registry: registryConfig, projectRef: environment.project.externalRef, nextVersion, - environmentSlug: environment.slug, + environmentType: environment.type, + deploymentShortCode, }) ); @@ -111,7 +114,7 @@ export class InitializeDeploymentService extends BaseService { data: { friendlyId: generateFriendlyId("deployment"), contentHash: payload.contentHash, - shortCode: nanoid(8), + shortCode: deploymentShortCode, version: nextVersion, status: "BUILDING", environmentId: environment.id, diff --git a/apps/webapp/test/getDeploymentImageRef.test.ts b/apps/webapp/test/getDeploymentImageRef.test.ts index 2810c1443..a05236596 100644 --- a/apps/webapp/test/getDeploymentImageRef.test.ts +++ b/apps/webapp/test/getDeploymentImageRef.test.ts @@ -8,7 +8,7 @@ import { } from "../app/v3/getDeploymentImageRef.server"; import { DeleteRepositoryCommand } from "@aws-sdk/client-ecr"; -describe.skipIf(process.env.RUN_REGISTRY_TESTS !== "1")("getDeploymentImageRef", () => { +describe("getDeploymentImageRef", () => { const testHost = process.env.DEPLOY_REGISTRY_HOST || "123456789012.dkr.ecr.us-east-1.amazonaws.com"; const testNamespace = process.env.DEPLOY_REGISTRY_NAMESPACE || "test-namespace"; @@ -25,7 +25,7 @@ describe.skipIf(process.env.RUN_REGISTRY_TESTS !== "1")("getDeploymentImageRef", // Clean up test repository after tests afterAll(async () => { - if (process.env.KEEP_TEST_REPO === "1") { + if (process.env.KEEP_TEST_REPO === "1" || process.env.RUN_ECR_TESTS !== "1") { return; } @@ -57,7 +57,7 @@ describe.skipIf(process.env.RUN_REGISTRY_TESTS !== "1")("getDeploymentImageRef", it("should return the correct image ref for non-ECR registry", async () => { const imageRef = await getDeploymentImageRef({ registry: { - host: "registry.digitalocean.com", + host: "registry.example.com", namespace: testNamespace, username: "test-user", password: "test-pass", @@ -67,60 +67,67 @@ describe.skipIf(process.env.RUN_REGISTRY_TESTS !== "1")("getDeploymentImageRef", }, projectRef: testProjectRef, nextVersion: "20250630.1", - environmentSlug: "test", + environmentType: "DEVELOPMENT", + deploymentShortCode: "test1234", }); + // Check the image ref structure and that it contains expected parts expect(imageRef.imageRef).toBe( - `registry.digitalocean.com/${testNamespace}/${testProjectRef}:20250630.1.test` + `registry.example.com/${testNamespace}/${testProjectRef}:20250630.1.development.test1234` ); expect(imageRef.isEcr).toBe(false); }); - it("should create ECR repository and return correct image ref", async () => { - const imageRef1 = await getDeploymentImageRef({ - registry: { - host: testHost, - namespace: testNamespace, - username: "test-user", - password: "test-pass", - ecrTags: registryTags, - ecrAssumeRoleArn: roleArn, - ecrAssumeRoleExternalId: externalId, - }, - projectRef: testProjectRef2, - nextVersion: "20250630.1", - environmentSlug: "test", - }); + it.skipIf(process.env.RUN_ECR_TESTS !== "1")( + "should create ECR repository and return correct image ref", + async () => { + const imageRef1 = await getDeploymentImageRef({ + registry: { + host: testHost, + namespace: testNamespace, + username: "test-user", + password: "test-pass", + ecrTags: registryTags, + ecrAssumeRoleArn: roleArn, + ecrAssumeRoleExternalId: externalId, + }, + projectRef: testProjectRef2, + nextVersion: "20250630.1", + environmentType: "DEVELOPMENT", + deploymentShortCode: "test1234", + }); - expect(imageRef1.imageRef).toBe( - `${testHost}/${testNamespace}/${testProjectRef2}:20250630.1.test` - ); - expect(imageRef1.isEcr).toBe(true); - expect(imageRef1.repoCreated).toBe(true); + expect(imageRef1.imageRef).toBe( + `${testHost}/${testNamespace}/${testProjectRef2}:20250630.1.development.test1234` + ); + expect(imageRef1.isEcr).toBe(true); + expect(imageRef1.repoCreated).toBe(true); - const imageRef2 = await getDeploymentImageRef({ - registry: { - host: testHost, - namespace: testNamespace, - username: "test-user", - password: "test-pass", - ecrTags: registryTags, - ecrAssumeRoleArn: roleArn, - ecrAssumeRoleExternalId: externalId, - }, - projectRef: testProjectRef2, - nextVersion: "20250630.2", - environmentSlug: "test", - }); + const imageRef2 = await getDeploymentImageRef({ + registry: { + host: testHost, + namespace: testNamespace, + username: "test-user", + password: "test-pass", + ecrTags: registryTags, + ecrAssumeRoleArn: roleArn, + ecrAssumeRoleExternalId: externalId, + }, + projectRef: testProjectRef2, + nextVersion: "20250630.2", + environmentType: "DEVELOPMENT", + deploymentShortCode: "test1234", + }); - expect(imageRef2.imageRef).toBe( - `${testHost}/${testNamespace}/${testProjectRef2}:20250630.2.test` - ); - expect(imageRef2.isEcr).toBe(true); - expect(imageRef2.repoCreated).toBe(false); - }); + expect(imageRef2.imageRef).toBe( + `${testHost}/${testNamespace}/${testProjectRef2}:20250630.2.development.test1234` + ); + expect(imageRef2.isEcr).toBe(true); + expect(imageRef2.repoCreated).toBe(false); + } + ); - it("should reuse existing ECR repository", async () => { + it.skipIf(process.env.RUN_ECR_TESTS !== "1")("should reuse existing ECR repository", async () => { // This should use the repository created in the previous test const imageRef = await getDeploymentImageRef({ registry: { @@ -134,36 +141,65 @@ describe.skipIf(process.env.RUN_REGISTRY_TESTS !== "1")("getDeploymentImageRef", }, projectRef: testProjectRef, nextVersion: "20250630.2", - environmentSlug: "prod", + environmentType: "PRODUCTION", + deploymentShortCode: "test1234", }); expect(imageRef.imageRef).toBe( - `${testHost}/${testNamespace}/${testProjectRef}:20250630.2.prod` + `${testHost}/${testNamespace}/${testProjectRef}:20250630.2.production.test1234` ); expect(imageRef.isEcr).toBe(true); }); - it("should throw error for invalid ECR host", async () => { - await expect( - getDeploymentImageRef({ - registry: { - host: "invalid.ecr.amazonaws.com", - namespace: testNamespace, - username: "test-user", - password: "test-pass", - ecrTags: registryTags, - ecrAssumeRoleArn: roleArn, - ecrAssumeRoleExternalId: externalId, - }, - projectRef: testProjectRef, - nextVersion: "20250630.1", - environmentSlug: "test", - }) - ).rejects.toThrow("Invalid ECR registry host: invalid.ecr.amazonaws.com"); + it("should generate unique image tags for different deployments with same environment type", async () => { + // Simulates the scenario where multiple deployments happen to the same environment type + const sameEnvironmentType = "PREVIEW"; + const sameVersion = "20250630.1"; + + const firstImageRef = await getDeploymentImageRef({ + registry: { + host: "registry.example.com", + namespace: testNamespace, + username: "test-user", + password: "test-pass", + ecrTags: registryTags, + ecrAssumeRoleArn: roleArn, + ecrAssumeRoleExternalId: externalId, + }, + projectRef: testProjectRef, + nextVersion: sameVersion, + environmentType: sameEnvironmentType, + deploymentShortCode: "test1234", + }); + + const secondImageRef = await getDeploymentImageRef({ + registry: { + host: "registry.example.com", + namespace: testNamespace, + username: "test-user", + password: "test-pass", + ecrTags: registryTags, + ecrAssumeRoleArn: roleArn, + ecrAssumeRoleExternalId: externalId, + }, + projectRef: testProjectRef, + nextVersion: sameVersion, + environmentType: sameEnvironmentType, + deploymentShortCode: "test4321", + }); + + // Even with the same environment type and version, the image refs should be different due to deployment short codes + expect(firstImageRef.imageRef).toBe( + `registry.example.com/${testNamespace}/${testProjectRef}:${sameVersion}.preview.test1234` + ); + expect(secondImageRef.imageRef).toBe( + `registry.example.com/${testNamespace}/${testProjectRef}:${sameVersion}.preview.test4321` + ); + expect(firstImageRef.imageRef).not.toBe(secondImageRef.imageRef); }); }); -describe.skipIf(process.env.RUN_REGISTRY_AUTH_TESTS !== "1")("getEcrAuthToken", () => { +describe.skipIf(process.env.RUN_ECR_TESTS !== "1")("getEcrAuthToken", () => { const testHost = process.env.DEPLOY_REGISTRY_HOST || "123456789012.dkr.ecr.us-east-1.amazonaws.com"; @@ -188,8 +224,7 @@ describe.skipIf(process.env.RUN_REGISTRY_AUTH_TESTS !== "1")("getEcrAuthToken", expect(auth.password.length).toBeGreaterThan(0); // Verify the token format (should be a base64-encoded string) - const base64Regex = /^[A-Za-z0-9+/=]+$/; - expect(base64Regex.test(auth.password)).toBe(true); + expect(auth.password).toMatch(/^[A-Za-z0-9+/=]+$/); }); it("should throw error for invalid region", async () => { From 00d32ed4eeda732e3a2aaf886e52145801e40ca2 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Thu, 4 Sep 2025 09:53:15 +0100 Subject: [PATCH 09/13] feat(webapp): add support for running web services (api, engine, webapp) in cluster mode for better perf (#2472) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(webapp): add support for running web services (api, engine, webapp) in cluster mode for better perf * cleaned up signal handling and resolved some valid 🐇 issues --- apps/webapp/app/eventLoopMonitor.server.ts | 8 + .../realtime/relayRealtimeStreams.server.ts | 8 +- .../runsReplicationInstance.server.ts | 6 +- .../services/runsReplicationService.server.ts | 2 + apps/webapp/app/services/signals.server.ts | 32 ++ .../app/v3/dynamicFlushScheduler.server.ts | 40 +- apps/webapp/app/v3/marqs/index.server.ts | 9 +- apps/webapp/app/v3/tracing.server.ts | 33 +- apps/webapp/server.ts | 363 +++++++++++------- 9 files changed, 328 insertions(+), 173 deletions(-) create mode 100644 apps/webapp/app/services/signals.server.ts diff --git a/apps/webapp/app/eventLoopMonitor.server.ts b/apps/webapp/app/eventLoopMonitor.server.ts index 1d8603a4a..b86ea3d31 100644 --- a/apps/webapp/app/eventLoopMonitor.server.ts +++ b/apps/webapp/app/eventLoopMonitor.server.ts @@ -5,6 +5,7 @@ import { env } from "./env.server"; import { context, Context } from "@opentelemetry/api"; import { performance } from "node:perf_hooks"; import { logger } from "./services/logger.server"; +import { signalsEmitter } from "./services/signals.server"; const THRESHOLD_NS = env.EVENT_LOOP_MONITOR_THRESHOLD_MS * 1e6; @@ -110,6 +111,13 @@ function startEventLoopUtilizationMonitoring() { lastEventLoopUtilization = currentEventLoopUtilization; }, env.EVENT_LOOP_MONITOR_UTILIZATION_INTERVAL_MS); + signalsEmitter.on("SIGTERM", () => { + clearInterval(interval); + }); + signalsEmitter.on("SIGINT", () => { + clearInterval(interval); + }); + return () => { clearInterval(interval); }; diff --git a/apps/webapp/app/services/realtime/relayRealtimeStreams.server.ts b/apps/webapp/app/services/realtime/relayRealtimeStreams.server.ts index 46e16ff5e..99a82199d 100644 --- a/apps/webapp/app/services/realtime/relayRealtimeStreams.server.ts +++ b/apps/webapp/app/services/realtime/relayRealtimeStreams.server.ts @@ -1,5 +1,6 @@ import { AuthenticatedEnvironment } from "../apiAuth.server"; import { logger } from "../logger.server"; +import { signalsEmitter } from "../signals.server"; import { StreamIngestor, StreamResponder } from "./types"; import { LineTransformStream } from "./utils.server"; import { v1RealtimeStreams } from "./v1StreamsGlobal.server"; @@ -243,12 +244,17 @@ export class RelayRealtimeStreams implements StreamIngestor, StreamResponder { } function initializeRelayRealtimeStreams() { - return new RelayRealtimeStreams({ + const service = new RelayRealtimeStreams({ ttl: 1000 * 60 * 5, // 5 minutes cleanupInterval: 1000 * 60, // 1 minute fallbackIngestor: v1RealtimeStreams, fallbackResponder: v1RealtimeStreams, }); + + signalsEmitter.on("SIGTERM", service.close.bind(service)); + signalsEmitter.on("SIGINT", service.close.bind(service)); + + return service; } export const relayRealtimeStreams = singleton( diff --git a/apps/webapp/app/services/runsReplicationInstance.server.ts b/apps/webapp/app/services/runsReplicationInstance.server.ts index 45b7b7a97..2c9aafb1c 100644 --- a/apps/webapp/app/services/runsReplicationInstance.server.ts +++ b/apps/webapp/app/services/runsReplicationInstance.server.ts @@ -3,8 +3,8 @@ import invariant from "tiny-invariant"; import { env } from "~/env.server"; import { singleton } from "~/utils/singleton"; import { provider } from "~/v3/tracer.server"; -import { logger } from "./logger.server"; import { RunsReplicationService } from "./runsReplicationService.server"; +import { signalsEmitter } from "./signals.server"; export const runsReplicationInstance = singleton( "runsReplicationInstance", @@ -80,8 +80,8 @@ function initializeRunsReplicationInstance() { }); }); - process.on("SIGTERM", service.shutdown.bind(service)); - process.on("SIGINT", service.shutdown.bind(service)); + signalsEmitter.on("SIGTERM", service.shutdown.bind(service)); + signalsEmitter.on("SIGINT", service.shutdown.bind(service)); } return service; diff --git a/apps/webapp/app/services/runsReplicationService.server.ts b/apps/webapp/app/services/runsReplicationService.server.ts index b9eeeab25..41170fadc 100644 --- a/apps/webapp/app/services/runsReplicationService.server.ts +++ b/apps/webapp/app/services/runsReplicationService.server.ts @@ -204,6 +204,8 @@ export class RunsReplicationService { } public async shutdown() { + if (this._isShuttingDown) return; + this._isShuttingDown = true; this.logger.info("Initiating shutdown of runs replication service"); diff --git a/apps/webapp/app/services/signals.server.ts b/apps/webapp/app/services/signals.server.ts new file mode 100644 index 000000000..308f16fde --- /dev/null +++ b/apps/webapp/app/services/signals.server.ts @@ -0,0 +1,32 @@ +import { EventEmitter } from "events"; +import { singleton } from "~/utils/singleton"; + +export type SignalsEvents = { + SIGTERM: [ + { + time: Date; + signal: NodeJS.Signals; + } + ]; + SIGINT: [ + { + time: Date; + signal: NodeJS.Signals; + } + ]; +}; + +export type SignalsEventArgs = SignalsEvents[T]; + +export type SignalsEmitter = EventEmitter; + +function initializeSignalsEmitter() { + const emitter = new EventEmitter(); + + process.on("SIGTERM", () => emitter.emit("SIGTERM", { time: new Date(), signal: "SIGTERM" })); + process.on("SIGINT", () => emitter.emit("SIGINT", { time: new Date(), signal: "SIGINT" })); + + return emitter; +} + +export const signalsEmitter = singleton("signalsEmitter", initializeSignalsEmitter); diff --git a/apps/webapp/app/v3/dynamicFlushScheduler.server.ts b/apps/webapp/app/v3/dynamicFlushScheduler.server.ts index 88e6a1024..30c508d03 100644 --- a/apps/webapp/app/v3/dynamicFlushScheduler.server.ts +++ b/apps/webapp/app/v3/dynamicFlushScheduler.server.ts @@ -1,6 +1,7 @@ import { Logger } from "@trigger.dev/core/logger"; import { nanoid } from "nanoid"; import pLimit from "p-limit"; +import { signalsEmitter } from "~/services/signals.server"; export type DynamicFlushSchedulerConfig = { batchSize: number; @@ -22,6 +23,7 @@ export class DynamicFlushScheduler { private readonly BATCH_SIZE: number; private readonly FLUSH_INTERVAL: number; private flushTimer: NodeJS.Timeout | null; + private metricsReporterTimer: NodeJS.Timeout | undefined; private readonly callback: (flushId: string, batch: T[]) => Promise; // New properties for dynamic scaling @@ -41,6 +43,7 @@ export class DynamicFlushScheduler { droppedEvents: 0, droppedEventsByKind: new Map(), }; + private isShuttingDown: boolean = false; // New properties for load shedding private readonly loadSheddingThreshold: number; @@ -75,6 +78,7 @@ export class DynamicFlushScheduler { this.startFlushTimer(); this.startMetricsReporter(); + this.setupShutdownHandlers(); } addToBatch(items: T[]): void { @@ -119,8 +123,8 @@ export class DynamicFlushScheduler { this.currentBatch.push(...itemsToAdd); this.totalQueuedItems += itemsToAdd.length; - // Check if we need to create a batch - if (this.currentBatch.length >= this.currentBatchSize) { + // Check if we need to create a batch (if we are shutting down, create a batch immediately because the flush timer is stopped) + if (this.currentBatch.length >= this.currentBatchSize || this.isShuttingDown) { this.createBatch(); } @@ -137,6 +141,23 @@ export class DynamicFlushScheduler { this.resetFlushTimer(); } + private setupShutdownHandlers(): void { + signalsEmitter.on("SIGTERM", () => + this.shutdown().catch((error) => { + this.logger.error("Error shutting down dynamic flush scheduler", { + error, + }); + }) + ); + signalsEmitter.on("SIGINT", () => + this.shutdown().catch((error) => { + this.logger.error("Error shutting down dynamic flush scheduler", { + error, + }); + }) + ); + } + private startFlushTimer(): void { this.flushTimer = setInterval(() => this.checkAndFlush(), this.FLUSH_INTERVAL); } @@ -145,6 +166,9 @@ export class DynamicFlushScheduler { if (this.flushTimer) { clearInterval(this.flushTimer); } + + if (this.isShuttingDown) return; + this.startFlushTimer(); } @@ -226,7 +250,7 @@ export class DynamicFlushScheduler { } private lastConcurrencyAdjustment: number = Date.now(); - + private adjustConcurrency(backOff: boolean = false): void { const currentConcurrency = this.limiter.concurrency; let newConcurrency = currentConcurrency; @@ -281,7 +305,7 @@ export class DynamicFlushScheduler { private startMetricsReporter(): void { // Report metrics every 30 seconds - setInterval(() => { + this.metricsReporterTimer = setInterval(() => { const droppedByKind: Record = {}; this.metrics.droppedEventsByKind.forEach((count, kind) => { droppedByKind[kind] = count; @@ -356,10 +380,18 @@ export class DynamicFlushScheduler { // Graceful shutdown async shutdown(): Promise { + if (this.isShuttingDown) return; + + this.isShuttingDown = true; + if (this.flushTimer) { clearInterval(this.flushTimer); } + if (this.metricsReporterTimer) { + clearInterval(this.metricsReporterTimer); + } + // Flush any remaining items if (this.currentBatch.length > 0) { this.createBatch(); diff --git a/apps/webapp/app/v3/marqs/index.server.ts b/apps/webapp/app/v3/marqs/index.server.ts index 89dfa1e3f..5348f228a 100644 --- a/apps/webapp/app/v3/marqs/index.server.ts +++ b/apps/webapp/app/v3/marqs/index.server.ts @@ -24,6 +24,7 @@ import z from "zod"; import { env } from "~/env.server"; import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; +import { signalsEmitter } from "~/services/signals.server"; import { singleton } from "~/utils/singleton"; import { legacyRunEngineWorker } from "../legacyRunEngineWorker.server"; import { concurrencyTracker } from "../services/taskRunConcurrencyTracker.server"; @@ -112,6 +113,7 @@ export class MarQS { private queueDequeueCooloffPeriod: Map = new Map(); private queueDequeueCooloffCounts: Map = new Map(); private clearCooloffPeriodInterval: NodeJS.Timeout; + isShuttingDown: boolean = false; constructor(private readonly options: MarQSOptions) { this.redis = options.redis; @@ -151,11 +153,14 @@ export class MarQS { } #setupShutdownHandlers() { - process.on("SIGTERM", () => this.shutdown("SIGTERM")); - process.on("SIGINT", () => this.shutdown("SIGINT")); + signalsEmitter.on("SIGTERM", () => this.shutdown("SIGTERM")); + signalsEmitter.on("SIGINT", () => this.shutdown("SIGINT")); } async shutdown(signal: NodeJS.Signals) { + if (this.isShuttingDown) return; + this.isShuttingDown = true; + console.log("👇 Shutting down marqs", this.name, signal); clearInterval(this.clearCooloffPeriodInterval); this.#rebalanceWorkers.forEach((worker) => worker.stop()); diff --git a/apps/webapp/app/v3/tracing.server.ts b/apps/webapp/app/v3/tracing.server.ts index 936cf9a57..b02fc5ec6 100644 --- a/apps/webapp/app/v3/tracing.server.ts +++ b/apps/webapp/app/v3/tracing.server.ts @@ -41,33 +41,14 @@ export async function startSpanWithEnv( fn: (span: Span) => Promise, options?: SpanOptions ): Promise { - return startSpan( - tracer, - name, - async (span) => { - try { - return await fn(span); - } catch (e) { - if (e instanceof Error) { - span.recordException(e); - } else { - span.recordException(new Error(String(e))); - } - - throw e; - } finally { - span.end(); - } + return startSpan(tracer, name, fn, { + ...options, + attributes: { + ...attributesFromAuthenticatedEnv(env), + ...options?.attributes, }, - { - attributes: { - ...attributesFromAuthenticatedEnv(env), - ...options?.attributes, - }, - kind: SpanKind.SERVER, - ...options, - } - ); + kind: SpanKind.SERVER, + }); } export async function emitDebugLog( diff --git a/apps/webapp/server.ts b/apps/webapp/server.ts index 455fded7a..b2cc93873 100644 --- a/apps/webapp/server.ts +++ b/apps/webapp/server.ts @@ -12,168 +12,257 @@ import type { Server as IoServer } from "socket.io"; import { WebSocketServer } from "ws"; import { RateLimitMiddleware } from "~/services/apiRateLimit.server"; import { type RunWithHttpContextFunction } from "~/services/httpAsyncStorage.server"; +import cluster from "node:cluster"; +import os from "node:os"; -const app = express(); +const ENABLE_CLUSTER = process.env.ENABLE_CLUSTER === "1"; +const cpuCount = os.availableParallelism(); +const WORKERS = + Number.parseInt(process.env.WEB_CONCURRENCY || process.env.CLUSTER_WORKERS || "", 10) || cpuCount; -if (process.env.DISABLE_COMPRESSION !== "1") { - app.use(compression()); +function forkWorkers() { + for (let i = 0; i < WORKERS; i++) { + cluster.fork(); + } } -// http://expressjs.com/en/advanced/best-practice-security.html#at-a-minimum-disable-x-powered-by-header -app.disable("x-powered-by"); +function installPrimarySignalHandlers() { + let didHandleSigterm = false; + let didHandleSigint = false; + let didGracefulExit = false; -// Remix fingerprints its assets so we can cache forever. -app.use("/build", express.static("public/build", { immutable: true, maxAge: "1y" })); - -// Everything else (like favicon.ico) is cached for an hour. You may want to be -// more aggressive with this caching. -app.use(express.static("public", { maxAge: "1h" })); - -app.use(morgan("tiny")); - -process.title = "node webapp-server"; - -const MODE = process.env.NODE_ENV; -const BUILD_DIR = path.join(process.cwd(), "build"); -const build = require(BUILD_DIR); - -const port = process.env.REMIX_APP_PORT || process.env.PORT || 3000; - -if (process.env.HTTP_SERVER_DISABLED !== "true") { - const socketIo: { io: IoServer } | undefined = build.entry.module.socketIo; - const wss: WebSocketServer | undefined = build.entry.module.wss; - const apiRateLimiter: RateLimitMiddleware = build.entry.module.apiRateLimiter; - const engineRateLimiter: RateLimitMiddleware = build.entry.module.engineRateLimiter; - const runWithHttpContext: RunWithHttpContextFunction = build.entry.module.runWithHttpContext; - - app.use((req, res, next) => { - // helpful headers: - res.set("Strict-Transport-Security", `max-age=${60 * 60 * 24 * 365 * 100}`); - - // Add X-Robots-Tag header for test-cloud.trigger.dev - if (req.hostname !== "cloud.trigger.dev") { - res.set("X-Robots-Tag", "noindex, nofollow"); + const forward = (signal: NodeJS.Signals) => { + for (const id in cluster.workers) { + const w = cluster.workers[id]; + if (w?.process?.pid) { + try { + process.kill(w.process.pid, signal); + } catch {} + } } + }; - // /clean-urls/ -> /clean-urls - if (req.path.endsWith("/") && req.path.length > 1) { - const query = req.url.slice(req.path.length); - const safepath = req.path.slice(0, -1).replace(/\/+/g, "/"); - res.redirect(301, safepath + query); - return; - } - next(); + const gracefulExit = () => { + if (didGracefulExit) return; + didGracefulExit = true; + + const timeoutMs = Number(process.env.GRACEFUL_SHUTDOWN_TIMEOUT || 30_000); + // wait for workers to exit, then exit the primary too + const maybeExit = () => { + const alive = Object.values(cluster.workers || {}).some((w) => w && !w.isDead()); + if (!alive) process.exit(0); + }; + setInterval(maybeExit, 1000); + setTimeout(() => process.exit(0), timeoutMs); + }; + + process.on("SIGTERM", () => { + if (didHandleSigterm) return; + didHandleSigterm = true; + forward("SIGTERM"); + gracefulExit(); + }); + process.on("SIGINT", () => { + if (didHandleSigint) return; + didHandleSigint = true; + forward("SIGINT"); + gracefulExit(); + }); +} + +if (ENABLE_CLUSTER && cluster.isPrimary) { + process.title = `node webapp-server primary`; + console.log(`[cluster] Primary ${process.pid} is starting with ${WORKERS} workers`); + forkWorkers(); + + cluster.on("exit", (worker, code, signal) => { + const intentional = + // If we sent "shutdown", the worker will exit with code 0 after closing. + code === 0 || worker.exitedAfterDisconnect; + console.log( + `[cluster] worker ${worker.process.pid} exited (code=${code}, signal=${signal}, intentional=${intentional})` + ); + // If it wasn't during a shutdown, replace the worker. + if (!intentional) cluster.fork(); }); - app.use((req, res, next) => { - // Generate a unique request ID for each request - const requestId = nanoid(); + installPrimarySignalHandlers(); +} else { + const app = express(); - runWithHttpContext({ requestId, path: req.url, host: req.hostname, method: req.method }, next); - }); + if (process.env.DISABLE_COMPRESSION !== "1") { + app.use(compression()); + } - if (process.env.DASHBOARD_AND_API_DISABLED !== "true") { - if (process.env.ALLOW_ONLY_REALTIME_API === "true") { - // Block all requests that do not start with /realtime - app.use((req, res, next) => { - // Make sure /healthcheck is still accessible - if (!req.url.startsWith("/realtime") && req.url !== "/healthcheck") { - res.status(404).send("Not Found"); - return; - } + // http://expressjs.com/en/advanced/best-practice-security.html#at-a-minimum-disable-x-powered-by-header + app.disable("x-powered-by"); - next(); + // Remix fingerprints its assets so we can cache forever. + app.use("/build", express.static("public/build", { immutable: true, maxAge: "1y" })); + + // Everything else (like favicon.ico) is cached for an hour. You may want to be + // more aggressive with this caching. + app.use(express.static("public", { maxAge: "1h" })); + + app.use(morgan("tiny")); + + process.title = ENABLE_CLUSTER + ? `node webapp-worker-${cluster.isWorker ? cluster.worker?.id : "solo"}` + : "node webapp-server"; + + const MODE = process.env.NODE_ENV; + const BUILD_DIR = path.join(process.cwd(), "build"); + const build = require(BUILD_DIR); + + const port = process.env.REMIX_APP_PORT || process.env.PORT || 3000; + + if (process.env.HTTP_SERVER_DISABLED !== "true") { + const socketIo: { io: IoServer } | undefined = build.entry.module.socketIo; + const wss: WebSocketServer | undefined = build.entry.module.wss; + const apiRateLimiter: RateLimitMiddleware = build.entry.module.apiRateLimiter; + const engineRateLimiter: RateLimitMiddleware = build.entry.module.engineRateLimiter; + const runWithHttpContext: RunWithHttpContextFunction = build.entry.module.runWithHttpContext; + + app.use((req, res, next) => { + // helpful headers: + res.set("Strict-Transport-Security", `max-age=${60 * 60 * 24 * 365 * 100}`); + + // Add X-Robots-Tag header for test-cloud.trigger.dev + if (req.hostname !== "cloud.trigger.dev") { + res.set("X-Robots-Tag", "noindex, nofollow"); + } + + // /clean-urls/ -> /clean-urls + if (req.path.endsWith("/") && req.path.length > 1) { + const query = req.url.slice(req.path.length); + const safepath = req.path.slice(0, -1).replace(/\/+/g, "/"); + res.redirect(301, safepath + query); + return; + } + next(); + }); + + app.use((req, res, next) => { + // Generate a unique request ID for each request + const requestId = nanoid(); + + runWithHttpContext( + { requestId, path: req.url, host: req.hostname, method: req.method }, + next + ); + }); + + if (process.env.DASHBOARD_AND_API_DISABLED !== "true") { + if (process.env.ALLOW_ONLY_REALTIME_API === "true") { + // Block all requests that do not start with /realtime + app.use((req, res, next) => { + // Make sure /healthcheck is still accessible + if (!req.url.startsWith("/realtime") && req.url !== "/healthcheck") { + res.status(404).send("Not Found"); + return; + } + + next(); + }); + } + + app.use(apiRateLimiter); + app.use(engineRateLimiter); + + app.all( + "*", + // @ts-ignore + createRequestHandler({ + build, + mode: MODE, + }) + ); + } else { + // we need to do the health check here at /healthcheck + app.get("/healthcheck", (req, res) => { + res.status(200).send("OK"); }); } - app.use(apiRateLimiter); - app.use(engineRateLimiter); + const server = app.listen(port, () => { + console.log( + `✅ server ready: http://localhost:${port} [NODE_ENV: ${MODE}]${ + ENABLE_CLUSTER && cluster.isWorker ? ` [worker ${cluster.worker?.id}/${process.pid}]` : "" + }` + ); - app.all( - "*", - // @ts-ignore - createRequestHandler({ - build, - mode: MODE, - }) - ); - } else { - // we need to do the health check here at /healthcheck - app.get("/healthcheck", (req, res) => { - res.status(200).send("OK"); - }); - } - - const server = app.listen(port, () => { - console.log(`✅ server ready: http://localhost:${port} [NODE_ENV: ${MODE}]`); - - if (MODE === "development") { - broadcastDevReady(build) - .then(() => logDevReady(build)) - .catch(console.error); - } - }); - - server.keepAliveTimeout = 65 * 1000; - // Mitigate against https://github.com/triggerdotdev/trigger.dev/security/dependabot/128 - // by not allowing 2000+ headers to be sent and causing a DoS - // headers will instead be limited by the maxHeaderSize - server.maxHeadersCount = 0; - - process.on("SIGTERM", () => { - server.close((err) => { - if (err) { - console.error("Error closing express server:", err); - } else { - console.log("Express server closed gracefully."); + if (MODE === "development") { + broadcastDevReady(build) + .then(() => logDevReady(build)) + .catch(console.error); } }); - }); - socketIo?.io.attach(server); - server.removeAllListeners("upgrade"); // prevent duplicate upgrades from listeners created by io.attach() + server.keepAliveTimeout = 65 * 1000; + // Mitigate against https://github.com/triggerdotdev/trigger.dev/security/dependabot/128 + // by not allowing 2000+ headers to be sent and causing a DoS + // headers will instead be limited by the maxHeaderSize + server.maxHeadersCount = 0; - server.on("upgrade", async (req, socket, head) => { - console.log( - `Attemping to upgrade connection at url ${req.url} with headers: ${JSON.stringify( - req.headers - )}` - ); + let didCloseServer = false; - socket.on("error", (err) => { - console.error("Connection upgrade error:", err); - }); + function closeServer(signal: NodeJS.Signals) { + if (didCloseServer) return; + didCloseServer = true; - const url = new URL(req.url ?? "", "http://localhost"); - - // Upgrade socket.io connection - if (url.pathname.startsWith("/socket.io/")) { - console.log(`Socket.io client connected, upgrading their connection...`); - - // https://github.com/socketio/socket.io/issues/4693 - (socketIo?.io.engine as EngineServer).handleUpgrade(req, socket, head); - return; + server.close((err) => { + if (err) { + console.error("Error closing express server:", err); + } else { + console.log("Express server closed gracefully."); + } + }); } - // Only upgrade the connecting if the path is `/ws` - if (url.pathname !== "/ws") { - // Setting the socket.destroy() error param causes an error event to be emitted which needs to be handled with socket.on("error") to prevent uncaught exceptions. - socket.destroy( - new Error( - "Cannot connect because of invalid path: Please include `/ws` in the path of your upgrade request." - ) - ); - return; - } + process.on("SIGTERM", closeServer); + process.on("SIGINT", closeServer); - console.log(`Client connected, upgrading their connection...`); + socketIo?.io.attach(server); + server.removeAllListeners("upgrade"); // prevent duplicate upgrades from listeners created by io.attach() - // Handle the WebSocket connection - wss?.handleUpgrade(req, socket, head, (ws) => { - wss?.emit("connection", ws, req); + server.on("upgrade", async (req, socket, head) => { + console.log(`Attemping to upgrade connection at url ${req.url}`); + + socket.on("error", (err) => { + console.error("Connection upgrade error:", err); + }); + + const url = new URL(req.url ?? "", "http://localhost"); + + // Upgrade socket.io connection + if (url.pathname.startsWith("/socket.io/")) { + console.log(`Socket.io client connected, upgrading their connection...`); + + // https://github.com/socketio/socket.io/issues/4693 + (socketIo?.io.engine as EngineServer).handleUpgrade(req, socket, head); + return; + } + + // Only upgrade the connecting if the path is `/ws` + if (url.pathname !== "/ws") { + // Setting the socket.destroy() error param causes an error event to be emitted which needs to be handled with socket.on("error") to prevent uncaught exceptions. + socket.destroy( + new Error( + "Cannot connect because of invalid path: Please include `/ws` in the path of your upgrade request." + ) + ); + return; + } + + console.log(`Client connected, upgrading their connection...`); + + // Handle the WebSocket connection + wss?.handleUpgrade(req, socket, head, (ws) => { + wss?.emit("connection", ws, req); + }); }); - }); -} else { - require(BUILD_DIR); - console.log(`✅ app ready (skipping http server)`); + } else { + require(BUILD_DIR); + console.log(`✅ app ready (skipping http server)`); + } } From e6586d3c1a140a2f08555456d5c2218bd0036aaf Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 4 Sep 2025 14:57:17 +0100 Subject: [PATCH 10/13] Remove deprecated releaseConcurrency from wait.forToken() docs (#2477) --- docs/wait-for-token.mdx | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/docs/wait-for-token.mdx b/docs/wait-for-token.mdx index 4f5a8faed..9e050a343 100644 --- a/docs/wait-for-token.mdx +++ b/docs/wait-for-token.mdx @@ -263,24 +263,6 @@ Wait for a token to be completed. The token to wait for. - - Options for the wait. - - - - If set to true, this will cause the waitpoint to release the current run from the queue's concurrency. - - This is useful if you want to allow other runs to execute while waiting - - Note: It's possible that this run will not be able to resume when the waitpoint is complete if this is set to true. - It will go back in the queue and will resume once concurrency becomes available. - - The default is `false`. - - - - - ### returns The `forToken` function returns a result object with the following properties: From 99660112bd6494211b17be6cdbf216d9a2b48588 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Thu, 4 Sep 2025 17:09:34 +0100 Subject: [PATCH 11/13] feat(supervisor): add configurable resource requests (#2474) --- apps/supervisor/src/env.ts | 4 ++++ .../src/workloadManager/kubernetes.ts | 21 +++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index 074c18ad7..99cb16425 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -85,6 +85,10 @@ const Env = z.object({ KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT: z.string().default("10Gi"), KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST: z.string().default("2Gi"), KUBERNETES_STRIP_IMAGE_DIGEST: BoolEnv.default(false), + KUBERNETES_CPU_REQUEST_MIN_CORES: z.coerce.number().min(0).default(0), + KUBERNETES_CPU_REQUEST_RATIO: z.coerce.number().min(0).max(1).default(0.75), // Ratio of CPU limit, so 0.75 = 75% of CPU limit + KUBERNETES_MEMORY_REQUEST_MIN_GB: z.coerce.number().min(0).default(0), + KUBERNETES_MEMORY_REQUEST_RATIO: z.coerce.number().min(0).max(1).default(1), // Ratio of memory limit, so 1 = 100% of memory limit // Placement tags settings PLACEMENT_TAGS_ENABLED: BoolEnv.default(false), diff --git a/apps/supervisor/src/workloadManager/kubernetes.ts b/apps/supervisor/src/workloadManager/kubernetes.ts index b38e6c5b4..55281c56a 100644 --- a/apps/supervisor/src/workloadManager/kubernetes.ts +++ b/apps/supervisor/src/workloadManager/kubernetes.ts @@ -20,6 +20,12 @@ export class KubernetesWorkloadManager implements WorkloadManager { private namespace = env.KUBERNETES_NAMESPACE; private placementTagProcessor: PlacementTagProcessor; + // Resource settings + private readonly cpuRequestMinCores = env.KUBERNETES_CPU_REQUEST_MIN_CORES; + private readonly cpuRequestRatio = env.KUBERNETES_CPU_REQUEST_RATIO; + private readonly memoryRequestMinGb = env.KUBERNETES_MEMORY_REQUEST_MIN_GB; + private readonly memoryRequestRatio = env.KUBERNETES_MEMORY_REQUEST_RATIO; + constructor(private opts: WorkloadManagerOptions) { this.k8s = createK8sApi(); this.placementTagProcessor = new PlacementTagProcessor({ @@ -63,6 +69,10 @@ export class KubernetesWorkloadManager implements WorkloadManager { return imageRef.substring(0, atIndex); } + private clamp(value: number, min: number, max: number): number { + return Math.min(Math.max(value, min), max); + } + async create(opts: WorkloadManagerCreateOptions) { this.logger.log("[KubernetesWorkloadManager] Creating container", { opts }); @@ -295,9 +305,16 @@ export class KubernetesWorkloadManager implements WorkloadManager { } #getResourceRequestsForMachine(preset: MachinePreset): ResourceQuantities { + const cpuRequest = preset.cpu * this.cpuRequestRatio; + const memoryRequest = preset.memory * this.memoryRequestRatio; + + // Clamp between min and max + const clampedCpu = this.clamp(cpuRequest, this.cpuRequestMinCores, preset.cpu); + const clampedMemory = this.clamp(memoryRequest, this.memoryRequestMinGb, preset.memory); + return { - cpu: `${preset.cpu * 0.75}`, - memory: `${preset.memory}G`, + cpu: `${clampedCpu}`, + memory: `${clampedMemory}G`, }; } From 97015ba8c81ccf71f25ce5cabac4b6465d610348 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 5 Sep 2025 11:13:13 +0100 Subject: [PATCH 12/13] fix(docs): update sentry error tracking guide for v4 (#2479) --- docs/guides/examples/sentry-error-tracking.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/examples/sentry-error-tracking.mdx b/docs/guides/examples/sentry-error-tracking.mdx index 04eef6bf8..d861e93e7 100644 --- a/docs/guides/examples/sentry-error-tracking.mdx +++ b/docs/guides/examples/sentry-error-tracking.mdx @@ -56,7 +56,7 @@ export default defineConfig({ environment: process.env.NODE_ENV === "production" ? "production" : "development", }); }, - onFailure: async (payload, error, { ctx }) => { + onFailure: async ({ payload, error, ctx }) => { Sentry.captureException(error, { extra: { payload, From 89b1d8ba139efe67cf0e24c3308be324f0ce41b2 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 5 Sep 2025 11:17:38 +0100 Subject: [PATCH 13/13] fix all lifecycle hooks (#2480) --- docs/config/config-file.mdx | 8 ++++---- docs/errors-retrying.mdx | 4 ++-- docs/migrating-from-v3.mdx | 2 +- docs/tasks/overview.mdx | 29 +++++++++++++++-------------- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/docs/config/config-file.mdx b/docs/config/config-file.mdx index d11704d30..ef79bc862 100644 --- a/docs/config/config-file.mdx +++ b/docs/config/config-file.mdx @@ -99,16 +99,16 @@ import { defineConfig } from "@trigger.dev/sdk"; export default defineConfig({ project: "", // Your other config settings... - onSuccess: async (payload, output, { ctx }) => { + onSuccess: async ({ payload, output, ctx }) => { console.log("Task succeeded", ctx.task.id); }, - onFailure: async (payload, error, { ctx }) => { + onFailure: async ({ payload, error, ctx }) => { console.log("Task failed", ctx.task.id); }, - onStart: async (payload, { ctx }) => { + onStart: async ({ payload, ctx }) => { console.log("Task started", ctx.task.id); }, - init: async (payload, { ctx }) => { + init: async ({ payload, ctx }) => { console.log("I run before any task is run"); }, }); diff --git a/docs/errors-retrying.mdx b/docs/errors-retrying.mdx index 7bd2deae3..246db2b82 100644 --- a/docs/errors-retrying.mdx +++ b/docs/errors-retrying.mdx @@ -180,7 +180,7 @@ export const taskWithFetchRetries = task({ ## Advanced error handling and retrying -We provide a `handleError` callback on the task and in your `trigger.config` file. This gets called when an uncaught error is thrown in your task. +We provide a `catchError` callback on the task and in your `trigger.config` file. This gets called when an uncaught error is thrown in your task. You can @@ -219,7 +219,7 @@ export const openaiTask = task({ return chatCompletion.choices[0].message.content; }, - handleError: async (payload, error, { ctx, retryAt }) => { + catchError: async ({ payload, error, ctx, retryAt }) => { if (error instanceof OpenAI.APIError) { if (!error.status) { return { diff --git a/docs/migrating-from-v3.mdx b/docs/migrating-from-v3.mdx index e06cd0806..03f0cae57 100644 --- a/docs/migrating-from-v3.mdx +++ b/docs/migrating-from-v3.mdx @@ -232,7 +232,7 @@ import { task } from "@trigger.dev/sdk"; export const myTask = task({ id: "my-task", - onStart: (payload, { ctx }) => {}, + onStart: ({ payload, ctx }) => {}, run: async (payload, { ctx }) => {}, }); ``` diff --git a/docs/tasks/overview.mdx b/docs/tasks/overview.mdx index aa305f2f8..2b3452c87 100644 --- a/docs/tasks/overview.mdx +++ b/docs/tasks/overview.mdx @@ -181,7 +181,7 @@ This function is called before a run attempt: ```ts /trigger/init.ts export const taskWithInit = task({ id: "task-with-init", - init: async (payload, { ctx }) => { + init: async ({ payload, ctx }) => { //... }, run: async (payload: any, { ctx }) => { @@ -195,7 +195,7 @@ You can also return data from the `init` function that will be available in the ```ts /trigger/init-return.ts export const taskWithInitReturn = task({ id: "task-with-init-return", - init: async (payload, { ctx }) => { + init: async ({ payload, ctx }) => { return { someData: "someValue" }; }, run: async (payload: any, { ctx, init }) => { @@ -213,7 +213,7 @@ This function is called after the `run` function is executed, regardless of whet ```ts /trigger/cleanup.ts export const taskWithCleanup = task({ id: "task-with-cleanup", - cleanup: async (payload, { ctx }) => { + cleanup: async ({ payload, ctx }) => { //... }, run: async (payload: any, { ctx }) => { @@ -230,7 +230,7 @@ Our task middleware system runs at the top level, executing before and after all An error thrown in `middleware` is just like an uncaught error in the run function: it will - propagate through to `handleError()` and then will fail the attempt (causing a retry). + propagate through to `catchError()` function and then will fail the attempt (causing a retry). The `locals` API allows you to share data between middleware and hooks. @@ -303,7 +303,7 @@ When a task run starts, the `onStart` function is called. It's useful for sendin ```ts /trigger/on-start.ts export const taskWithOnStart = task({ id: "task-with-on-start", - onStart: async (payload, { ctx }) => { + onStart: async ({ payload, ctx }) => { //... }, run: async (payload: any, { ctx }) => { @@ -319,7 +319,7 @@ import { defineConfig } from "@trigger.dev/sdk"; export default defineConfig({ project: "proj_1234", - onStart: async (payload, { ctx }) => { + onStart: async ({ payload, ctx }) => { console.log("Task started", ctx.task.id); }, }); @@ -357,7 +357,7 @@ When a task run succeeds, the `onSuccess` function is called. It's useful for se ```ts /trigger/on-success.ts export const taskWithOnSuccess = task({ id: "task-with-on-success", - onSuccess: async (payload, output, { ctx }) => { + onSuccess: async ({ payload, output, ctx }) => { //... }, run: async (payload: any, { ctx }) => { @@ -373,7 +373,7 @@ import { defineConfig } from "@trigger.dev/sdk"; export default defineConfig({ project: "proj_1234", - onSuccess: async (payload, output, { ctx }) => { + onSuccess: async ({ payload, output, ctx }) => { console.log("Task succeeded", ctx.task.id); }, }); @@ -388,7 +388,7 @@ This hook is executed when a run completes, regardless of whether it succeeded o ```ts /trigger/on-complete.ts export const taskWithOnComplete = task({ id: "task-with-on-complete", - onComplete: async (payload, output, { ctx }) => { + onComplete: async ({ payload, output, ctx }) => { if (result.ok) { console.log("Run succeeded", result.data); } else { @@ -404,7 +404,7 @@ When a task run fails, the `onFailure` function is called. It's useful for sendi ```ts /trigger/on-failure.ts export const taskWithOnFailure = task({ id: "task-with-on-failure", - onFailure: async (payload, error, { ctx }) => { + onFailure: async ({ payload, error, ctx }) => { //... }, run: async (payload: any, { ctx }) => { @@ -420,7 +420,7 @@ import { defineConfig } from "@trigger.dev/sdk"; export default defineConfig({ project: "proj_1234", - onFailure: async (payload, error, { ctx }) => { + onFailure: async ({ payload, error, ctx }) => { console.log("Task failed", ctx.task.id); }, }); @@ -429,14 +429,15 @@ export default defineConfig({ Errors thrown in the `onFailure` function are ignored. - `onFailure` doesn’t fire for some of the run statuses like `Crashed`, `System failures`, and `Canceled`. + `onFailure` doesn’t fire for some of the run statuses like `Crashed`, `System failures`, and + `Canceled`. -### `handleError` functions +### `catchError` functions You can define a function that will be called when an error is thrown in the `run` function, that allows you to control how the error is handled and whether the task should be retried. -Read more about `handleError` in our [Errors and Retrying guide](/errors-retrying). +Read more about `catchError` in our [Errors and Retrying guide](/errors-retrying). Uncaught errors will throw a special internal error of the type `HANDLE_ERROR_ERROR`.