diff --git a/.server-changes/mollifier-burst-protection.md b/.server-changes/mollifier-burst-protection.md index be3c3f3b8..182811d68 100644 --- a/.server-changes/mollifier-burst-protection.md +++ b/.server-changes/mollifier-burst-protection.md @@ -3,4 +3,4 @@ area: webapp type: feature --- -Lay the groundwork for an opt-in burst-protection layer on the trigger hot path. This release ships **monitoring only** — operators can observe per-env trigger storms via two opt-in modes, but no trigger calls are diverted or rate-limited yet (active burst smoothing follows in a later release). All new env vars default off, so existing deployments see no behaviour change. With `MOLLIFIER_SHADOW_MODE=1`, each trigger evaluates a per-env rate counter and logs `mollifier.would_mollify` when the threshold is crossed. With `MOLLIFIER_ENABLED=1` plus a per-org `mollifierEnabled` flag, over-threshold triggers are also recorded in a Redis audit buffer alongside the normal `engine.trigger` call, drained by a background no-op consumer. Emits the `mollifier.decisions` OTel counter for per-env rate visibility. +Lay the groundwork for an opt-in burst-protection layer on the trigger hot path. This release ships **monitoring only** — operators can observe per-env trigger storms via two opt-in modes, but no trigger calls are diverted or rate-limited yet (active burst smoothing follows in a later release). All new env vars are prefixed `TRIGGER_MOLLIFIER_*` and default off, so existing deployments see no behaviour change. With `TRIGGER_MOLLIFIER_SHADOW_MODE=1`, each trigger evaluates a per-env rate counter and logs `mollifier.would_mollify` when the threshold is crossed. With `TRIGGER_MOLLIFIER_ENABLED=1` plus a per-org `mollifierEnabled` flag, over-threshold triggers are also recorded in a Redis audit buffer alongside the normal `engine.trigger` call, drained by a background no-op consumer. The drainer has its own switch (`TRIGGER_MOLLIFIER_DRAINER_ENABLED`) so multi-replica deployments can pin the polling loop to a single worker service while every replica still produces into the buffer; unset, it inherits `TRIGGER_MOLLIFIER_ENABLED` so single-container self-hosters need only one flag. Drainer misconfiguration (shutdown-timeout reconciliation against `GRACEFUL_SHUTDOWN_TIMEOUT`, or `TRIGGER_MOLLIFIER_ENABLED=1` with no buffer Redis) now throws `MollifierConfigurationError` at boot and crashes the process, so the misconfig surfaces to the orchestrator instead of disappearing into a log line; transient init failures (Redis blip) are still logged-and-swallowed. Emits the `mollifier.decisions` OTel counter for per-env rate visibility. diff --git a/apps/webapp/app/v3/mollifierDrainerWorker.server.ts b/apps/webapp/app/v3/mollifierDrainerWorker.server.ts index 8c7032d1f..313e9af67 100644 --- a/apps/webapp/app/v3/mollifierDrainerWorker.server.ts +++ b/apps/webapp/app/v3/mollifierDrainerWorker.server.ts @@ -43,13 +43,25 @@ declare global { * master kill switch; the new flag only controls WHICH replicas * run the drainer when the system is on. */ -export function initMollifierDrainerWorker(): void { - if (env.TRIGGER_MOLLIFIER_DRAINER_ENABLED !== "1") { +export function initMollifierDrainerWorker( + opts: { + // Test seams. Production callers pass nothing; the defaults read the + // live env and resolve the live singleton. Tests inject overrides so + // the misconfig-rethrow / transient-swallow branches can be driven + // without manipulating module-level env state. + isEnabled?: () => boolean; + getDrainer?: typeof getMollifierDrainer; + } = {}, +): void { + const isEnabled = opts.isEnabled ?? (() => env.TRIGGER_MOLLIFIER_DRAINER_ENABLED === "1"); + const getDrainer = opts.getDrainer ?? getMollifierDrainer; + + if (!isEnabled()) { return; } try { - const drainer = getMollifierDrainer(); + const drainer = getDrainer(); if (drainer && !global.__mollifierShutdownRegistered__) { // `__mollifierShutdownRegistered__` guards against double-register // on dev hot-reloads (this bootstrap is called from diff --git a/apps/webapp/test/mollifierDrainerWorker.test.ts b/apps/webapp/test/mollifierDrainerWorker.test.ts new file mode 100644 index 000000000..e5f38229d --- /dev/null +++ b/apps/webapp/test/mollifierDrainerWorker.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vitest"; +import { MollifierConfigurationError } from "~/v3/mollifier/mollifierDrainer.server"; +import { initMollifierDrainerWorker } from "~/v3/mollifierDrainerWorker.server"; + +// Pins the error-classification policy inside the bootstrap's catch: +// deterministic misconfig errors propagate (so a deploy fails loud +// rather than silently disabling the drainer), and anything else is +// logged-and-swallowed (so a transient Redis blip during boot doesn't +// take the whole webapp down). The corresponding production-path +// integration is the call at `entry.server.tsx`: a sync throw out of +// `initMollifierDrainerWorker` propagates to the module top level +// BEFORE `process.on("uncaughtException", ...)` is registered, so Node +// crashes with a stack trace and exit code 1 — which is exactly what we +// want from the orchestrator's health-check perspective. +describe("initMollifierDrainerWorker error classification", () => { + it("rethrows MollifierConfigurationError so the process can crash on misconfig", () => { + const misconfig = new MollifierConfigurationError( + "TRIGGER_MOLLIFIER_DRAIN_SHUTDOWN_TIMEOUT_MS must be at least 1000ms below GRACEFUL_SHUTDOWN_TIMEOUT", + ); + + expect(() => + initMollifierDrainerWorker({ + isEnabled: () => true, + getDrainer: () => { + throw misconfig; + }, + }), + ).toThrow(MollifierConfigurationError); + }); + + it("rethrows when the error carries the marker name even if instanceof fails (dev-realm hot-reload fallback)", () => { + // Simulate the cross-realm case where the consumer's instanceof + // check sees a different class instance from the one the throw + // site used. The bootstrap's `.name === "MollifierConfigurationError"` + // fallback must catch this so dev hot-reload doesn't silently + // suppress misconfig errors. + const cousin = new Error("buffer not initialised"); + cousin.name = "MollifierConfigurationError"; + + expect(() => + initMollifierDrainerWorker({ + isEnabled: () => true, + getDrainer: () => { + throw cousin; + }, + }), + ).toThrow(cousin); + }); + + it("swallows non-configuration errors so transient init failures don't take the webapp down", () => { + expect(() => + initMollifierDrainerWorker({ + isEnabled: () => true, + getDrainer: () => { + throw new Error("transient redis blip during buffer init"); + }, + }), + ).not.toThrow(); + }); + + it("is a no-op when the drainer is disabled for this replica", () => { + let factoryCalled = false; + initMollifierDrainerWorker({ + isEnabled: () => false, + getDrainer: () => { + factoryCalled = true; + return null; + }, + }); + expect(factoryCalled).toBe(false); + }); +});