diff --git a/internal-packages/run-engine/.gitignore b/internal-packages/run-engine/.gitignore deleted file mode 100644 index cb5b3f39f..000000000 --- a/internal-packages/run-engine/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Micro-benchmark output (generated by ckMicroBench.bench.test.ts) -bench-results/ diff --git a/internal-packages/run-engine/design/benchmarks/2026-07-26-ck-vtime-benchmark.md b/internal-packages/run-engine/design/benchmarks/2026-07-26-ck-vtime-benchmark.md deleted file mode 100644 index 72146a88c..000000000 --- a/internal-packages/run-engine/design/benchmarks/2026-07-26-ck-vtime-benchmark.md +++ /dev/null @@ -1,253 +0,0 @@ -# CK virtual-time scheduling: prod-like A/B benchmark plan - -A method for producing defensible A/B numbers for the concurrency-key -virtual-time (SFQ) scheduling change, run on a single prod-shaped box. The two -arms are flag OFF (today's age-ordered CK dequeue) and flag ON (virtual-time -ordering), under identical load. - -## What the change is (grounded in the branch) - -The concurrency-key dequeue used to serve variants of a base queue in head-message -age order. Behind `RUN_ENGINE_CK_VTIME_SCHEDULING_ENABLED` (off by default) it now -orders them by start-time fair queueing (SFQ) virtual time: - -- Each CK variant carries a virtual clock in a parallel `:ckVtime` ZSET; a - monotonic floor lives in `:ckVtimeFloor`. Both sit under the base queue's - `{org}` hash tag, so one atomic Lua script touches all of a queue's state. -- The dequeue runs two passes: pass 1 serves the lowest virtual clocks and - advances each served variant by `quantum / weight` (weight fixed at 1 today); - pass 2 fills any leftover batch slots in today's age order. Pass 2 makes the - new command a strict superset of the old one, so it is work-conserving and can - never serve fewer runs than today. -- Enqueue and nack register a variant into `:ckVtime` at the floor with `NX`, so - a brand-new key is reachable from its first enqueue and cannot be parked behind - a backlog. This is the case a per-key concurrency cap cannot fix: one tenant - sharding work across many keys. -- Flag off is byte-identical: the pre-existing Lua scripts run unchanged and no - vtime keys are created. The behaviour lives only in new command names. - -Tuning knobs (real env var names, all positive integers, re-clamped in the -`RunQueue` constructor): - -| Env var | Default | Meaning | -| --- | --- | --- | -| `RUN_ENGINE_CK_VTIME_SCHEDULING_ENABLED` | off | master flag | -| `RUN_ENGINE_CK_VTIME_QUANTUM` | 1 | virtual-time advance per serve | -| `RUN_ENGINE_CK_VTIME_WINDOW_MULTIPLIER` | 3 | pass-1 window = `maxCount * this` | -| `RUN_ENGINE_CK_VTIME_STATE_TTL_SECONDS` | 86400 | EXPIRE on the vtime keys | - -Stated limitations this benchmark deliberately probes (from -`../../src/run-queue/CK_VTIME_KNOWN_LIMITATIONS.md`): bounded tombstone drift on -ack/TTL/DLQ paths; member-name tie-break at equal tags; future-scheduled or -retry-backoff variants occupying pass-1 window slots; and the pass-1 window being -narrower than the live variant cardinality. - -## Validity: the numbers are RELATIVE, not absolute - -The target is one node on nested/slow storage. Absolute throughput here is NOT -prod-scale and must never be reported as such. Every result is a ratio between -flag OFF and flag ON, measured under identical load on the same box in the same -session. That relative signal is what isolates the scheduler. Two guards keep the -arms comparable: - -- Same enqueue set, same timestamps, same step loop / same load generator across - OFF and ON. -- Fresh queue state between arms (the micro arm FLUSHes a dedicated Redis; the - end-to-end arm drains and uses a fresh batch tag), a warmup, and N trials. - -## Two arms - -### Arm 1 (PRIMARY): queue-level micro-benchmark - -Drives the real `RunQueue` directly against a dedicated Redis, comparing OFF vs -ON on identical synthetic load. This isolates the scheduler and is where the -defensible numbers come from. Because the flag is a `RunQueue` constructor -option, one bench process runs both arms in-process: no webapp, no redeploy, no -worker clusters. It reuses the existing fairness harness -(`../../src/run-queue/tests/ckVtimeFairness.test.ts`): same step loop, same -scenario shapes, same conservation checks, plus wall-clock latency, a Redis -op-count, N trials, and file output. - -Harness: `../../src/run-queue/bench/ckMicroBench.bench.test.ts`. It is inert in CI -(only runs when `CK_BENCH_REDIS_URL` is set) and FLUSHes its target Redis between -arms, so it must point only at a dedicated throwaway instance. - -Each step makes one `maxCount = 10` dequeue call, records `(step, key, messageId, -wallMs)` per served message, then acks in-flight messages whose logical hold has -elapsed. Wait per message = the step it was served at (all load is pre-enqueued -at step 0). The logical schedule is deterministic, so step-based metrics are -identical across trials (the harness asserts this); trials exist to stabilise the -wall-clock latency and op-count. - -### Arm 2 (END-TO-END): deployed tasks on the worker clusters - -The realism check on top of arm 1. A deployed task on a shared base queue with -per-run concurrency keys, driven by a noisy-neighbor load generator: tenant A -floods across many keys, tenant B sends a few. Each run carries a per-run region -so the load also spreads across the three managed worker groups -(`trigger-regiona/b/c`), exercising multi-cluster placement. Latency is read back -per run as `startedAt - createdAt`. - -Project: `e2e-tasks/` (deployable, secret-free). Contention is forced by pinning -the environment concurrency ceiling low (so many keys contend for a few slots and -the CK dequeue order decides who starts first); the per-key lane width is 1 in the -task config for reproducibility. - -Because the flag is server-side here, arm 2 is a manual OFF-then-ON: set the flag, -redeploy the control plane, run the load, collect; flip the flag, redeploy, run -the load again, collect. The exact toggle + redeploy belongs to the operator -runbook. - -## Hypotheses (tied to the change) - -1. **Bounded wait behind a backlog.** A light key arriving behind a big backlog - waits O(number of active keys) under vtime, versus O(backlog size) under the - baseline (which drains the backlog first). Micro: `ckSkew`, `ckTrickle` - victim wait p95/p99 drops sharply ON vs OFF. E2E: tenant B start-latency p95 - stays bounded as tenant A's backlog grows. -2. **Sharding across many keys cannot starve others.** A tenant fanning out over - many concurrency keys (the case a per-key cap cannot fix) does not starve a - light key, because the light key registers at the floor and is reachable in - pass 1. Micro: `ckSybil` victim first-serve step is small ON (near-immediate) - and its wait ratio drops; `ckManyKeys` shows no permanent starvation even - when cardinality exceeds the pass-1 window. E2E: tenant B (few keys) is not - starved by tenant A's many-key flood. -3. **Work conservation.** A lone backlogged tenant still drains at full rate; - the fair order adds no idle time when nothing else contends. Micro: - `ckHeavyIdle` drain step ON equals OFF exactly. No-harm corollary: the - symmetric `ckBalanced` case is not made worse. - -## Scenarios - -### Arm 1 (micro), all ported from the fairness-spike shapes - -| scenario | shape | env limit | hold | probes | -| --- | --- | --- | --- | --- | -| `ckSkew` | heavy 120 backlog + 4 light x 10 | 1 | 3 | starvation (H1) | -| `ckTrickle` | bulk 120 + 2 trickle x 15 | 1 | 3 | starvation (H1) | -| `ckSybil` | 20 attacker x 8 + 1 light x 10 | 25 | 3 | sharding/sybil (H2) | -| `ckManyKeys` | 60 attacker x 8 (tied head) + 1 light x 10 | 25 | 3 | window limitation, no permanent starvation (H2) | -| `ckBalanced` | 4 symmetric x 25 | 4 | 3 | no-harm (H3) | -| `ckHeavyIdle` | 1 key x 60 | 25 | 3 | work conservation (H3) | - -### Arm 2 (end-to-end), noisy-neighbor - -- Tenant A: `A_KEYS` (default 40) keys x `A_PER_KEY` (default 5) runs = the flood. -- Tenant B: `B_KEYS` (default 2) keys x `B_PER_KEY` (default 5) runs = the victim. -- Per-run hold `HOLD_MS` (default 1500). Runs round-robined across the three - regions. Environment concurrency ceiling pinned low (e.g. 5) so the keys - actually contend. -- Optional placement variant: pin tenant A to one region and tenant B to another - to separate scheduler effects from cross-cluster effects. - -## Metrics and collection - -| metric | what it shows | source | -| --- | --- | --- | -| victim wait p50/p95/p99 | starvation relief | micro: serve step; e2e: `startedAt - createdAt` per tenant | -| victim first-serve (starvation bound) | reachability at the floor | micro: first serve step for the victim key | -| drain step / total served | work conservation, no loss/dup | micro: last serve step + unique messageId count | -| Jain's fairness index | share fairness during contention | micro: over per-key contention-window serves | -| dequeue call p95 (ms) | scheduler op cost (relative) | micro: wall-clock around each dequeue call | -| redis ops (dequeue+ack) | per-dequeue overhead | micro: `CONFIG RESETSTAT` then `INFO commandstats` | - -Jain's index over per-key served counts `x_i`: `J = (sum x_i)^2 / (n * sum -x_i^2)`. 1.0 is perfectly fair; `1/n` means one key took everything. - -The micro harness writes `ck-micro-results.json` and `ck-micro-results.md` -(the results table below) to `CK_BENCH_OUT`. - -For the end-to-end arm, the primary source is the Runs API by tag -(`startedAt - createdAt`), which `collect.ts` reads. Two alternatives give a -tighter dequeue-only timestamp if the API delta looks noisy: - -- TRQL `runs` (verify column names with the query schema first): per-run - `createdAt` and `startedAt`, filtered by the batch/arm tag. -- The run-engine Postgres `TaskRun` timestamps directly (createdAt and the first - attempt/started timestamp), if API round-trips add too much jitter. - -## Reproducible A/B procedure - -### Arm 1 (micro) - -1. Stand up a dedicated throwaway Redis reachable from the harness host (a local - forward is fine). Nothing else may use it. -2. From the run-engine package: - - ```bash - CK_BENCH_REDIS_URL=redis://127.0.0.1:6399 \ - CK_BENCH_TRIALS=5 CK_BENCH_OUT=./bench-results \ - pnpm exec vitest run src/run-queue/bench/ckMicroBench.bench.test.ts - ``` - - Both arms (OFF, then ON) run in one process per scenario, FLUSHing between - arms. Knob sweep: add `CK_BENCH_QUANTUM` / `CK_BENCH_WINDOW_MULT`. Scenario - subset: `CK_BENCH_SCENARIOS=ckSybil,ckSkew`. -3. Read `bench-results/ck-micro-results.md`. The harness fails the run if either - arm loses or double-serves a message, so a green run means the comparison is - sound. - -### Arm 2 (end-to-end) - -1. Deploy `e2e-tasks/` to the bench project's PROD environment (dev - short-circuits worker-group routing, so it must be prod). Pin the prod env - concurrency ceiling low. -2. Warmup: trigger a handful of runs, confirm they start on each region, discard. -3. **Arm OFF:** ensure the flag is off and the control plane is redeployed; then - `ARM=off BATCH= pnpm loadgen`, wait for drain, `ARM=off BATCH= pnpm - collect`. -4. **Arm ON:** flip the flag on, redeploy the control plane, drain/clear queue - state; then `ARM=on BATCH= pnpm loadgen`, wait for drain, `ARM=on - BATCH= pnpm collect`. -5. Repeat both arms N times with fresh batch ids; compare per-tenant latency. - -The exact deploy, flag-toggle, and secret-key retrieval steps for the specific -box are in the operator runbook (kept out of this repo). - -## Results template (paste into the PR) - -Fill from `ck-micro-results.md` (arm 1) and `e2e-summary.md` (arm 2). Keep the -"relative only" caveat in the PR text. - -### Arm 1 (micro), quantum 1 / window x3, N trials, dedicated Redis on the box - -| scenario | metric | baseline (OFF) | vtime (ON) | delta | -| --- | --- | --- | --- | --- | -| **ckSkew** | victim wait p95 (steps) | | | | -| | victim wait p99 | | | | -| | victim first-serve | | | | -| | drain step | | | | -| | dequeue call p95 (ms) | | | | -| **ckTrickle** | victim wait p95 | | | | -| | victim first-serve | | | | -| **ckSybil** | victim wait p95 | | | | -| | victim first-serve | | | | -| | Jain index (contention) | | | | -| **ckManyKeys** | victim first-serve | | | | -| | drain step | | | | -| **ckBalanced** | worst-key wait p95 | | | | -| **ckHeavyIdle** | drain step | | | | -| (all) | redis ops (dequeue+ack) | | | | - -### Arm 2 (end-to-end), noisy-neighbor across regiona/b/c, env cap N - -| tenant | metric | baseline (OFF) | vtime (ON) | delta | -| --- | --- | --- | --- | --- | -| B (victim, few keys) | start latency p50 (ms) | | | | -| B | start latency p95 | | | | -| B | start latency p99 | | | | -| A (flood, many keys) | start latency p95 | | | | - -Expected direction: B's p95/p99 drop substantially ON; A's is similar or slightly -higher ON (it stops jumping the queue); `ckHeavyIdle` drain step is exactly equal; -`ckBalanced` worst-key wait is within noise. - -## How to reproduce on the box (short) - -1. Dedicated Redis up, forwarded locally. Run the arm-1 vitest command above; - collect `ck-micro-results.md`. -2. Deploy `e2e-tasks/` to prod, pin the env concurrency ceiling, warm up. -3. Flag OFF: redeploy control plane, loadgen + collect. Flag ON: redeploy, - loadgen + collect. N trials. -4. Paste both tables into the PR under a "relative numbers on a single - prod-shaped box" heading. diff --git a/internal-packages/run-engine/design/benchmarks/2026-07-28-ck-vtime-resource-cardinality-plan.md b/internal-packages/run-engine/design/benchmarks/2026-07-28-ck-vtime-resource-cardinality-plan.md deleted file mode 100644 index 48825d66c..000000000 --- a/internal-packages/run-engine/design/benchmarks/2026-07-28-ck-vtime-resource-cardinality-plan.md +++ /dev/null @@ -1,156 +0,0 @@ -# CK virtual-time scheduling: Redis CPU + memory vs cardinality test plan - -Answers the review question: how do these changes affect the run-queue Redis -CPU and memory, and how do both react as concurrency-key cardinality grows (e.g. -a base queue that suddenly has 10k distinct concurrency keys)? - -This is a cost/scaling plan, separate from the fairness A/B -(`results-2026-07-27.md`). Same environment constraints: run on a single box, all -numbers **relative** (flag OFF vs ON, identical load, same box), against a -dedicated throwaway Redis. - -## What the change adds to Redis (grounded in the branch) - -Per base queue, flag ON adds: - -- `:ckVtime`, a ZSET whose members are the exact same full CK-variant queue-name - strings the existing `:ckIndex` ZSET already holds, each with an 8-byte double - score. So it is effectively a second copy of `ckIndex`'s membership. -- `:ckVtimeFloor`, a STRING holding one number. Negligible. - -Both are GC'd from the dequeue path when a variant drains, carry a 24h TTL, and -live under the base queue's `{org}` hash tag. The per-call dequeue scan window is -`maxCount * windowMultiplier` (default 30) and does **not** grow with cardinality; -the cardinality-sensitive operations are the ZSET writes/reads (`ZADD` `NX`, -`ZSCORE`, `ZRANGE` by rank, `ZRANGE 0 0`), which are O(log N) on the `ckVtime` -skiplist. - -## Hypotheses - -1. **Memory grows linearly with cardinality, adding roughly one `ckIndex`-sized - ZSET per base queue.** Incremental `used_memory` under ON minus OFF should - track `cardinality x per-member cost`, and `MEMORY USAGE :ckVtime` should be - close to `MEMORY USAGE :ckIndex` for the same queue (same members, one extra - double score). At 10k keys on one queue this is a low single-digit MB for that - queue, bounded and TTL-reclaimed. There is a one-step jump at the - listpack->skiplist encoding boundary (128 entries by default). -2. **Redis CPU per operation grows sub-linearly (about O(log cardinality)), not - linearly.** The fixed 30-entry window scan dominates the vtime-specific work - and does not change with N; the ZSET ops add a `log N` term. So dequeue/enqueue - `usec_per_call` should rise only mildly from 100 to 10k keys, and the ON/OFF - `usec_per_call` ratio should stay roughly flat across the sweep. -3. **Tombstone drift stays bounded under high-cardinality churn.** `ack`, TTL - expiry, and DLQ drain a variant without removing it from `ckVtime` (documented - in `CK_VTIME_KNOWN_LIMITATIONS.md`). Under churn where variants drain via those - paths rather than a vtime dequeue, `ckVtime` may transiently exceed `ckIndex`, - but it should self-heal (next vtime pass GCs empties) or expire (24h TTL), so - `size(ckVtime) / size(ckIndex)` stays bounded and does not grow without limit. - -## Scenarios - -All on a single base queue (worst case for one queue's ZSETs), dedicated Redis, -flag OFF then ON with identical load. - -- **Cardinality sweep (memory).** Enqueue N distinct concurrency keys, one - message each, N in {100, 1_000, 10_000, 50_000}. Measure the Redis memory - footprint at rest for each N, OFF vs ON. This isolates the storage cost with no - dequeue activity. -- **Steady-state load (CPU).** At each N, after building cardinality, run a fixed - 60s workload of enqueue + batched dequeue (`maxCount 10`) + ack at a capped - concurrency, so keys are continuously served and re-registered. Measure Redis - CPU and per-command time over the window, OFF vs ON. -- **Churn / tombstone (memory under adversarial drain).** Build 10k keys, then - drain them via `ack` (not via vtime dequeue) while enqueuing new keys, for a - fixed duration. Sample `size(ckVtime)` and `size(ckIndex)` over time and confirm - the ratio stays bounded (self-heal + TTL), not monotonically growing. - -## Metrics and collection (exact commands) - -Use a second plain Redis client for measurement so it does not perturb the -harness. `redis-cli` shown; the harness issues the same via ioredis. - -Memory: - -- Totals: `INFO memory` -> `used_memory`, `used_memory_dataset`. Delta ON vs OFF - at each N is the incremental footprint. -- Per structure (exact bytes): `MEMORY USAGE {org...}:queue::ckIndex` and - `MEMORY USAGE {org...}:queue::ckVtime`. Report both and the ratio. -- Encoding: `OBJECT ENCODING ` (listpack vs skiplist) at each N, to - mark the transition. - -CPU: - -- Process CPU over the load window: `INFO cpu` -> `used_cpu_user` + - `used_cpu_sys`, sampled before and after the fixed 60s load; the delta is Redis - CPU-seconds consumed. Divide by op count for CPU-per-op. -- Per-command time: `CONFIG RESETSTAT` before the window, then `INFO commandstats` - after -> `cmdstat_zadd`, `cmdstat_zrange`, `cmdstat_zrangebyscore`, - `cmdstat_zscore`, `cmdstat_get`, `cmdstat_set`, `cmdstat_expire` - (`calls`, `usec`, `usec_per_call`). Compare OFF vs ON. -- Cross-check (optional, OS-level): `pidstat -p 1` over the window, or - `redis-cli --latency` / `--latency-history` for command latency. - -Derived: - -- Memory vs N curve (expect linear, slope ~ per-member bytes), OFF and ON. -- CPU-per-op vs N curve (expect flat-ish / log), OFF/ON ratio. -- `size(ckVtime)/size(ckIndex)` over time in the churn scenario (expect bounded). - -## A/B procedure - -1. Dedicated throwaway Redis; `FLUSHDB` between arms and between cardinality - points. Warm up once. -2. For each N in the sweep, for each arm (OFF, ON): - - Build N keys (enqueue N distinct concurrency keys). - - Snapshot memory (`used_memory`, `MEMORY USAGE` of both ZSETs, `OBJECT - ENCODING`). - - `CONFIG RESETSTAT`; run the fixed 60s steady load; snapshot `INFO cpu` and - `INFO commandstats`. - - Record, then `FLUSHDB`. -3. N trials of the load window per point; report median for CPU (memory at rest is - near-deterministic). -4. Run the churn scenario once per arm at N = 10k. - -Only the OFF-vs-ON delta and the shape of the growth curves are reported; absolute -throughput on a single box is not prod scale. - -## Results template - -### Memory at rest, per cardinality (single base queue) - -| keys (N) | used_memory OFF | used_memory ON | delta | MEMORY USAGE ckIndex | MEMORY USAGE ckVtime | ckVtime/ckIndex | ckVtime encoding | -| --- | --- | --- | --- | --- | --- | --- | --- | -| 100 | | | | | | | | -| 1,000 | | | | | | | | -| 10,000 | | | | | | | | -| 50,000 | | | | | | | | - -### Redis CPU under 60s steady load, per cardinality - -| keys (N) | CPU-sec OFF | CPU-sec ON | delta | dequeue usec/call OFF | dequeue usec/call ON | delta | total redis calls OFF/ON | -| --- | --- | --- | --- | --- | --- | --- | --- | -| 100 | | | | | | | | -| 1,000 | | | | | | | | -| 10,000 | | | | | | | | - -### Tombstone drift (N=10k, drain via ack) - -| elapsed | size(ckIndex) | size(ckVtime) | ratio | -| --- | --- | --- | --- | -| 0s | | | | -| 30s | | | | -| 60s | | | | - -## Harness - -Extend the existing micro-benchmark -(`../../src/run-queue/bench/ckMicroBench.bench.test.ts`), which already drives a -real `RunQueue` against an external Redis and reads `INFO commandstats`. Add a -resource/cardinality mode that: builds N variants, snapshots `used_memory` + -`MEMORY USAGE` of the base queue's `ckIndex`/`ckVtime` + `OBJECT ENCODING`, runs a -fixed-duration steady load while sampling `INFO cpu`/`commandstats`, and emits the -tables above. The churn scenario reuses the same enqueue/ack primitives with a -drain-by-ack loop and periodic `ZCARD` sampling of both ZSETs. - -Reuse the same dedicated Redis and the OFF-vs-ON constructor-flag pattern, so this -arm, like the micro-benchmark, needs no webapp or redeploy. diff --git a/internal-packages/run-engine/design/benchmarks/e2e-tasks/.gitignore b/internal-packages/run-engine/design/benchmarks/e2e-tasks/.gitignore deleted file mode 100644 index ebeab39cd..000000000 --- a/internal-packages/run-engine/design/benchmarks/e2e-tasks/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -# Standalone deploy/run artifacts (this project is installed + run outside the monorepo) -node_modules/ -.trigger/ -e2e-results/ -.env.bench -pnpm-lock.yaml -package-lock.json diff --git a/internal-packages/run-engine/design/benchmarks/e2e-tasks/README.md b/internal-packages/run-engine/design/benchmarks/e2e-tasks/README.md deleted file mode 100644 index afa3cfc70..000000000 --- a/internal-packages/run-engine/design/benchmarks/e2e-tasks/README.md +++ /dev/null @@ -1,72 +0,0 @@ -# CK virtual-time end-to-end bench tasks - -A tiny self-contained trigger.dev project used as the END-TO-END arm of the CK -virtual-time A/B (see `../2026-07-26-ck-vtime-benchmark.md`). It deploys to a -self-hosted instance and is not part of the monorepo build. - -- `src/trigger/ckBench.ts` — one `ck-bench` task on a shared base queue; per-run - `concurrencyKey` makes the CK variants; a slot hold forces contention. -- `src/loadgen.ts` — noisy-neighbor load: tenant A floods across many keys, - tenant B sends a few; each run carries a per-run `region`. Captures every run - id at trigger time and writes a manifest (`e2e-results/manifest-.json`). -- `src/waitdrain.ts` — polls the manifest's run ids until all are terminal. -- `src/collect.ts` — reads each run's `createdAt`/`startedAt` by id and reports - per-tenant enqueue->start latency p50/p95/p99. -- `src/preflight.ts` — fires one run per region to validate routing + access. - -Everything reads credentials from the environment (`TRIGGER_API_URL`, -`TRIGGER_SECRET_KEY`); nothing is hard-coded. The feature flag is server-side and -is flipped by the operator between arms, not by this project. Exact instance -coordinates and the toggle live in the operator runbook, kept outside this repo. - -## Deploy from a standalone checkout, not inside the monorepo - -This project must be installed and deployed from OUTSIDE the pnpm monorepo tree -(copy it somewhere with no `pnpm-workspace.yaml` ancestor). Inside the workspace, -`pnpm install` binds to the workspace and links the local SDK instead of the -pinned published one. Use npm for the standalone copy (it avoids pnpm's -build-script policy on esbuild): - -```bash -cp -r ~/ck-vtime-e2e && cd ~/ck-vtime-e2e -npm install # CLI + SDK pinned to the same version (4.5.7) -``` - -## Deploy - -The 4.5.7 CLI has no `--self-hosted` flag; self-hosted is implicit from the -profile's API URL. Do NOT pass `--local-build` (it routes to a cloud-only ECR -credential endpoint and fails on self-hosted). The push then rides the host's -existing docker login to the registry. `--network host` is required so the -in-build indexer step can reach the instance API (e.g. over a tailnet): - -```bash -TRIGGER_PROJECT_REF= \ - ./node_modules/.bin/trigger deploy -e prod --network host --profile -p -``` - -Deploy to the `prod` environment: the `dev` environment short-circuits -worker-group routing, so dev runs never reach the managed regions. - -## Run one A/B arm - -The flag is flipped + redeployed server-side by the operator; run this once per -arm with the matching `ARM`. This instance's `runs.list` (ClickHouse-backed) can -be empty, so the harness enumerates runs from the trigger-time id manifest, not -by tag. - -```bash -export TRIGGER_API_URL= -export TRIGGER_SECRET_KEY= - -# validate region routing once -./node_modules/.bin/tsx src/preflight.ts - -# one arm (env knobs: HOLD_MS, A_KEYS, A_PER_KEY, B_KEYS, B_PER_KEY, REGIONS) -ARM=off BATCH=off-1 ./node_modules/.bin/tsx src/loadgen.ts -BATCH=off-1 ./node_modules/.bin/tsx src/waitdrain.ts -ARM=off BATCH=off-1 ./node_modules/.bin/tsx src/collect.ts -``` - -Results land in `e2e-results/` (`manifest-.json`, `e2e--.json`, -`e2e-summary.md`). diff --git a/internal-packages/run-engine/design/benchmarks/e2e-tasks/package.json b/internal-packages/run-engine/design/benchmarks/e2e-tasks/package.json deleted file mode 100644 index 63ec78ff5..000000000 --- a/internal-packages/run-engine/design/benchmarks/e2e-tasks/package.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "ck-vtime-e2e-bench", - "version": "0.0.0", - "private": true, - "type": "module", - "description": "End-to-end noisy-neighbor load harness for the CK virtual-time scheduling A/B. Deployed to a self-hosted trigger.dev instance; not part of the monorepo build.", - "scripts": { - "loadgen": "tsx src/loadgen.ts", - "collect": "tsx src/collect.ts" - }, - "dependencies": { - "@trigger.dev/sdk": "4.5.7" - }, - "devDependencies": { - "trigger.dev": "4.5.7", - "tsx": "^4.19.0", - "typescript": "^5.5.0" - } -} diff --git a/internal-packages/run-engine/design/benchmarks/e2e-tasks/src/collect.ts b/internal-packages/run-engine/design/benchmarks/e2e-tasks/src/collect.ts deleted file mode 100644 index ed299755d..000000000 --- a/internal-packages/run-engine/design/benchmarks/e2e-tasks/src/collect.ts +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Collect per-tenant enqueue->start latency for one END-TO-END arm, from a batch - * manifest (retrieve by id; runs.list is unreliable on this instance). - * - * Metric per run = startedAt - createdAt (run-start latency: queue wait + dequeue - * + worker pickup). Headline is tenant B (few keys): bounded under vtime, grows - * with A's backlog under baseline. - * - * Env: TRIGGER_API_URL, TRIGGER_SECRET_KEY. Args (env): BATCH ARM OUT POLL_CONCURRENCY - */ -import { configure, runs } from "@trigger.dev/sdk"; -import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; - -function pct(sorted: number[], p: number): number { - if (sorted.length === 0) return NaN; - const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1)); - return sorted[idx]!; -} -function summarize(xs: number[]) { - const s = [...xs].sort((a, b) => a - b); - const mean = xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : NaN; - return { count: xs.length, mean, p50: pct(s, 50), p95: pct(s, 95), p99: pct(s, 99) }; -} - -async function main() { - configure({ - baseURL: process.env.TRIGGER_API_URL!, - accessToken: process.env.TRIGGER_SECRET_KEY!, - }); - const batch = process.env.BATCH!; - const arm = (process.env.ARM ?? "off") as "off" | "on"; - const outDir = process.env.OUT ?? "./e2e-results"; - const conc = Number(process.env.POLL_CONCURRENCY ?? "20"); - const manifest = JSON.parse(readFileSync(`${outDir}/manifest-${batch}.json`, "utf8")); - const entries: { id: string; tenant: string }[] = manifest.runs; - - const waitsByTenant = new Map(); - let missingStart = 0; - let i = 0; - async function w() { - while (i < entries.length) { - const e = entries[i++]!; - try { - const r = await runs.retrieve(e.id); - const c = r.createdAt?.getTime(); - const s = r.startedAt?.getTime(); - if (c === undefined || s === undefined) { - missingStart++; - continue; - } - const arr = waitsByTenant.get(e.tenant) ?? waitsByTenant.set(e.tenant, []).get(e.tenant)!; - arr.push(s - c); - } catch { - missingStart++; - } - } - } - await Promise.all(Array.from({ length: Math.min(conc, entries.length) }, w)); - - const perTenant: Record> = {}; - for (const [t, xs] of waitsByTenant) perTenant[t] = summarize(xs); - const report = { - arm, - batch, - total: entries.length, - missingStart, - unit: "ms (startedAt - createdAt)", - perTenant, - }; - - mkdirSync(outDir, { recursive: true }); - writeFileSync(`${outDir}/e2e-${batch}-${arm}.json`, JSON.stringify(report, null, 2)); - - const rows = Object.entries(perTenant) - .map( - ([t, s]) => - `| ${batch} | ${arm} | ${t} | ${s.count} | ${s.mean.toFixed(0)} | ${s.p50.toFixed(0)} | ${s.p95.toFixed(0)} | ${s.p99.toFixed(0)} |` - ) - .join("\n"); - appendFileSync( - `${outDir}/e2e-summary.md`, - `\n\n| batch | arm | tenant | runs | mean ms | p50 | p95 | p99 |\n| --- | --- | --- | --- | --- | --- | --- | --- |\n${rows}\n` - ); - console.log( - `[collect] arm=${arm} batch=${batch} runs=${entries.length} missingStart=${missingStart}` - ); - console.log(JSON.stringify(perTenant, null, 2)); -} -main().catch((e) => { - console.error(e); - process.exit(1); -}); diff --git a/internal-packages/run-engine/design/benchmarks/e2e-tasks/src/loadgen.ts b/internal-packages/run-engine/design/benchmarks/e2e-tasks/src/loadgen.ts deleted file mode 100644 index ad7e686b0..000000000 --- a/internal-packages/run-engine/design/benchmarks/e2e-tasks/src/loadgen.ts +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Noisy-neighbor load generator for the CK virtual-time END-TO-END A/B arm. - * - * Tenant A floods the base queue across MANY concurrency keys (the sharding case - * a per-key cap cannot fix); tenant B sends a few runs on a couple of keys. Each - * run carries a per-run `region` so load spreads across the managed worker groups. - * - * This instance's runs.list (ClickHouse-backed) is unreliable, so we capture each - * run id at trigger time via individual tasks.trigger calls (fired with bounded - * concurrency so they still enqueue near-simultaneously as a backlog) and write a - * manifest. collect.ts / waitdrain.ts retrieve by id (the Postgres path). - * - * Auth (env): TRIGGER_API_URL, TRIGGER_SECRET_KEY (prod env secret key). - * Config (env): ARM=off|on BATCH= HOLD_MS A_KEYS A_PER_KEY B_KEYS B_PER_KEY - * REGIONS=trigger-regiona,trigger-regionb,trigger-regionc OUT=./e2e-results - * FIRE_CONCURRENCY=30 - */ -import { configure, tasks } from "@trigger.dev/sdk"; -import { mkdirSync, writeFileSync } from "node:fs"; -import type { ckBenchTask } from "./trigger/ckBench.js"; - -const envInt = (n: string, d: number) => - process.env[n] === undefined ? d : Number(process.env[n]); - -async function main() { - const apiURL = process.env.TRIGGER_API_URL; - const accessToken = process.env.TRIGGER_SECRET_KEY; - if (!apiURL || !accessToken) throw new Error("Set TRIGGER_API_URL and TRIGGER_SECRET_KEY."); - configure({ baseURL: apiURL, accessToken }); - - const arm = (process.env.ARM ?? "off") as "off" | "on"; - const batch = process.env.BATCH ?? `b${arm}`; - const holdMs = envInt("HOLD_MS", 1500); - const aKeys = envInt("A_KEYS", 30); - const aPerKey = envInt("A_PER_KEY", 8); - const bKeys = envInt("B_KEYS", 3); - const bPerKey = envInt("B_PER_KEY", 10); - const outDir = process.env.OUT ?? "./e2e-results"; - const fireConcurrency = envInt("FIRE_CONCURRENCY", 30); - const regions = (process.env.REGIONS ?? "trigger-regiona,trigger-regionb,trigger-regionc") - .split(",") - .map((s) => s.trim()) - .filter(Boolean); - - type Item = { tenant: "A" | "B"; key: string; region: string }; - const items: Item[] = []; - let n = 0; - const push = (tenant: "A" | "B", key: string) => { - items.push({ tenant, key, region: regions[n % regions.length]! }); - n++; - }; - for (let k = 0; k < aKeys; k++) for (let i = 0; i < aPerKey; i++) push("A", `A-${k}`); - for (let k = 0; k < bKeys; k++) for (let i = 0; i < bPerKey; i++) push("B", `B-${k}`); - // interleave so B does not all arrive first - items.sort((x, y) => x.key.localeCompare(y.key)); - - console.log( - `[loadgen] arm=${arm} batch=${batch} total=${items.length} (A=${aKeys}x${aPerKey}, B=${bKeys}x${bPerKey}) regions=${regions.join(",")} holdMs=${holdMs}` - ); - - const manifest: { - batch: string; - arm: string; - triggeredAt: number; - runs: { id: string; tenant: string; key: string; region: string }[]; - } = { batch, arm, triggeredAt: Date.now(), runs: [] }; - - let idx = 0; - let failed = 0; - async function worker() { - while (idx < items.length) { - const it = items[idx++]!; - try { - const h = await tasks.trigger( - "ck-bench", - { holdMs, tenant: it.tenant, key: it.key, arm, batch }, - { - concurrencyKey: it.key, - region: it.region, - tags: ["ckbench", `arm:${arm}`, `tenant:${it.tenant}`, `batch:${batch}`], - } - ); - manifest.runs.push({ id: h.id, tenant: it.tenant, key: it.key, region: it.region }); - } catch (e) { - failed++; - if (failed <= 3) console.error(`[loadgen] trigger failed:`, (e as Error).message); - } - } - } - await Promise.all(Array.from({ length: Math.min(fireConcurrency, items.length) }, worker)); - - mkdirSync(outDir, { recursive: true }); - const path = `${outDir}/manifest-${batch}.json`; - writeFileSync(path, JSON.stringify(manifest, null, 2)); - console.log( - `[loadgen] triggered ${manifest.runs.length}/${items.length} (failed ${failed}). manifest: ${path}` - ); -} - -main().catch((e) => { - console.error(e); - process.exit(1); -}); diff --git a/internal-packages/run-engine/design/benchmarks/e2e-tasks/src/preflight.ts b/internal-packages/run-engine/design/benchmarks/e2e-tasks/src/preflight.ts deleted file mode 100644 index cc7b1bf8e..000000000 --- a/internal-packages/run-engine/design/benchmarks/e2e-tasks/src/preflight.ts +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Region preflight: trigger one ck-bench run per worker group and confirm each - * starts (validates region routing + isWorkerGroupAllowedForProject before the - * real load). Reads TRIGGER_API_URL / TRIGGER_SECRET_KEY from env. - */ -import { configure, runs, tasks } from "@trigger.dev/sdk"; -import type { ckBenchTask } from "./trigger/ckBench.js"; - -const regions = ["trigger-regiona", "trigger-regionb", "trigger-regionc"]; - -async function main() { - configure({ - baseURL: process.env.TRIGGER_API_URL!, - accessToken: process.env.TRIGGER_SECRET_KEY!, - }); - - const handles: { region: string; id: string }[] = []; - for (const region of regions) { - const h = await tasks.trigger( - "ck-bench", - { holdMs: 2000, tenant: "preflight", key: `pf-${region}`, arm: "on", batch: "preflight" }, - { concurrencyKey: `pf-${region}`, region, tags: ["ckbench", "preflight", `region:${region}`] } - ); - handles.push({ region, id: h.id }); - console.log(`[preflight] triggered ${region}: ${h.id}`); - } - - // Poll up to ~90s for each to leave the queue and reach a terminal/executing state. - const deadline = Date.now() + 90_000; - const seen = new Map(); - while (Date.now() < deadline && seen.size < handles.length) { - for (const { region, id } of handles) { - if (seen.has(id)) continue; - const r = await runs.retrieve(id); - if (["EXECUTING", "COMPLETED", "FAILED", "CRASHED", "SYSTEM_FAILURE"].includes(r.status)) { - seen.set(id, r.status); - const started = r.startedAt ? r.startedAt.getTime() - r.createdAt.getTime() : undefined; - console.log( - `[preflight] ${region} ${id} -> ${r.status}${started !== undefined ? ` (start wait ${started}ms)` : ""}` - ); - } - } - if (seen.size < handles.length) await new Promise((r) => setTimeout(r, 3000)); - } - - const stuck = handles.filter((h) => !seen.has(h.id)); - if (stuck.length) { - console.log( - `[preflight] STILL QUEUED after 90s (possible access/routing issue): ${stuck.map((s) => `${s.region}:${s.id}`).join(", ")}` - ); - process.exit(2); - } - console.log("[preflight] all regions accepted + started. Routing + access OK."); -} - -main().catch((e) => { - console.error(e); - process.exit(1); -}); diff --git a/internal-packages/run-engine/design/benchmarks/e2e-tasks/src/trigger/ckBench.ts b/internal-packages/run-engine/design/benchmarks/e2e-tasks/src/trigger/ckBench.ts deleted file mode 100644 index 397061dfd..000000000 --- a/internal-packages/run-engine/design/benchmarks/e2e-tasks/src/trigger/ckBench.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { logger, queue, task } from "@trigger.dev/sdk"; - -// One shared base queue. Per-run concurrencyKey (set at trigger time) creates the -// concurrency-key variants whose dequeue ORDER the change under test governs. -// -// concurrencyLimit is the PER-KEY lane width. Set it to 1 so each key holds one -// slot at a time; cross-key contention is then forced by the ENVIRONMENT -// concurrency ceiling (pin RuntimeEnvironment.maximumConcurrencyLimit low, e.g. -// 5, on the prod env of the bench project). With N keys all wanting to run and -// only a few env slots, the CK dequeue decides who starts first: that ordering -// is exactly OFF (age) vs ON (virtual time). -export const ckBenchQueue = queue({ - name: "ck-bench", - concurrencyLimit: 1, -}); - -export type CkBenchPayload = { - // logical hold: how long the run occupies its slot, in ms - holdMs: number; - // carried through for grouping in analysis (also set as a tag by the loadgen) - tenant: string; - key: string; - arm: "off" | "on"; - batch: string; -}; - -export const ckBenchTask = task({ - id: "ck-bench", - queue: ckBenchQueue, - run: async (payload: CkBenchPayload) => { - logger.info("ck-bench start", { - tenant: payload.tenant, - key: payload.key, - arm: payload.arm, - batch: payload.batch, - }); - // Occupy the slot for the hold so concurrency actually contends. A plain - // timer is enough: this task exists only to hold a slot, not to do work. - await new Promise((resolve) => setTimeout(resolve, payload.holdMs)); - return { tenant: payload.tenant, key: payload.key, arm: payload.arm }; - }, -}); diff --git a/internal-packages/run-engine/design/benchmarks/e2e-tasks/src/waitdrain.ts b/internal-packages/run-engine/design/benchmarks/e2e-tasks/src/waitdrain.ts deleted file mode 100644 index 38851517e..000000000 --- a/internal-packages/run-engine/design/benchmarks/e2e-tasks/src/waitdrain.ts +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Wait until every run in a batch manifest reaches a terminal state (so startedAt - * is populated), by retrieving each run id (Postgres path; runs.list is unreliable - * on this instance). Env: TRIGGER_API_URL, TRIGGER_SECRET_KEY. - * Args (env): BATCH= OUT=./e2e-results TIMEOUT_S=420 POLL_CONCURRENCY=20 - */ -import { configure, runs } from "@trigger.dev/sdk"; -import { readFileSync } from "node:fs"; - -const TERMINAL = [ - "COMPLETED", - "FAILED", - "CRASHED", - "SYSTEM_FAILURE", - "CANCELED", - "TIMED_OUT", - "EXPIRED", -]; - -async function statusMap(ids: string[], conc: number): Promise> { - const out = new Map(); - let i = 0; - async function w() { - while (i < ids.length) { - const id = ids[i++]!; - try { - const r = await runs.retrieve(id); - out.set(id, r.status); - } catch { - out.set(id, "ERR_RETRIEVE"); - } - } - } - await Promise.all(Array.from({ length: Math.min(conc, ids.length) }, w)); - return out; -} - -async function main() { - configure({ - baseURL: process.env.TRIGGER_API_URL!, - accessToken: process.env.TRIGGER_SECRET_KEY!, - }); - const batch = process.env.BATCH!; - const outDir = process.env.OUT ?? "./e2e-results"; - const timeoutMs = Number(process.env.TIMEOUT_S ?? "420") * 1000; - const conc = Number(process.env.POLL_CONCURRENCY ?? "20"); - const manifest = JSON.parse(readFileSync(`${outDir}/manifest-${batch}.json`, "utf8")); - const ids: string[] = manifest.runs.map((r: any) => r.id); - const deadline = Date.now() + timeoutMs; - - while (Date.now() < deadline) { - const sm = await statusMap(ids, conc); - let terminal = 0; - for (const s of sm.values()) if (TERMINAL.includes(s)) terminal++; - console.log(`[waitdrain] ${batch}: terminal ${terminal}/${ids.length}`); - if (terminal >= ids.length) { - console.log("[waitdrain] drained."); - return; - } - await new Promise((r) => setTimeout(r, 5000)); - } - console.log("[waitdrain] TIMEOUT before full drain."); - process.exit(2); -} -main().catch((e) => { - console.error(e); - process.exit(1); -}); diff --git a/internal-packages/run-engine/design/benchmarks/e2e-tasks/trigger.config.ts b/internal-packages/run-engine/design/benchmarks/e2e-tasks/trigger.config.ts deleted file mode 100644 index 36f3572ee..000000000 --- a/internal-packages/run-engine/design/benchmarks/e2e-tasks/trigger.config.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { defineConfig } from "@trigger.dev/sdk"; - -// The project ref is passed on the CLI (`-p `) at deploy time, so it is not -// hard-coded here. This keeps the harness portable and secret-free in the repo. -export default defineConfig({ - project: process.env.TRIGGER_PROJECT_REF ?? "proj_REPLACE_ME", - runtime: "node", - logLevel: "info", - maxDuration: 120, - dirs: ["./src/trigger"], -}); diff --git a/internal-packages/run-engine/design/benchmarks/e2e-tasks/tsconfig.json b/internal-packages/run-engine/design/benchmarks/e2e-tasks/tsconfig.json deleted file mode 100644 index 282634969..000000000 --- a/internal-packages/run-engine/design/benchmarks/e2e-tasks/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "ESNext", - "moduleResolution": "Bundler", - "strict": true, - "esModuleInterop": true, - "skipLibCheck": true, - "types": ["node"], - "noEmit": true - }, - "include": ["src/**/*.ts", "trigger.config.ts"] -} diff --git a/internal-packages/run-engine/design/benchmarks/results-2026-07-27.md b/internal-packages/run-engine/design/benchmarks/results-2026-07-27.md deleted file mode 100644 index 4369e45d0..000000000 --- a/internal-packages/run-engine/design/benchmarks/results-2026-07-27.md +++ /dev/null @@ -1,80 +0,0 @@ -# CK virtual-time scheduling: A/B benchmark results (2026-07-27) - -Run on a **local homelab box, not production**. It is a *production-like* -topology only in shape: one shared control plane (Postgres + one shared run-queue -Redis + ClickHouse + the webapp/engine on this branch) and three separate managed -worker clusters registered as distinct worker groups, all as containers on a -single machine. No production data, traffic, or infrastructure was involved. -Method and harness: `2026-07-26-ck-vtime-benchmark.md`. - -All numbers are **relative** (flag OFF vs ON, identical load, same box). Absolute -throughput on this single-box homelab is nowhere near prod scale and is not -reported as such; only the OFF-vs-ON delta is meaningful. - -## Arm 1: queue-level micro-benchmark (the isolated, defensible numbers) - -Real `RunQueue` driven directly against a dedicated Redis, flag OFF (age-ordered -CK dequeue) vs ON (virtual time), 5 trials, quantum 1 / window multiplier 3. -Every arm served every message exactly once (conservation checked), so the -comparison is sound. Wait is in logical dequeue steps. - -| scenario | metric | baseline (OFF) | vtime (ON) | delta | -| --- | --- | --- | --- | --- | -| **ckSkew** (heavy backlog + light keys) | light-key first-serve step | 480 | 4 | **−99%** | -| | light-key wait p50 | 556 | 96 | −83% | -| | light-key wait p95 | 628 | 188 | −70% | -| | light-key wait p99 | 636 | 196 | −69% | -| | Jain fairness (contention) | 0.34 | 1.00 | +192% | -| | drain step (work conservation) | 636 | 636 | 0% | -| **ckTrickle** (bulk + trickle keys) | trickle first-serve step | 480 | 4 | −99% | -| | trickle wait p95 | 592 | 172 | −71% | -| | Jain fairness | 0.50 | 1.00 | +100% | -| **ckSybil** (20 keys flood + 1 light; caps can't fix) | light first-serve step | 25 | 2 | −92% | -| | light wait p95 | 34 | 27 | −21% | -| **ckManyKeys** (61 variants > pass-1 window) | light first-serve step | 72 | 9 | −88% | -| | drain step (no starvation) | 81 | 79 | drains fully | -| **ckBalanced** (4 symmetric keys, no-harm) | worst-key wait p95 | 92 | 92 | 0% | -| **ckHeavyIdle** (lone key, work conservation) | drain step | 59 | 59 | 0% (exact) | -| (all scenarios) | Redis ops per dequeue+ack | baseline | +7% to +23% | cost | - -Reading it: a light key behind a backlog goes from waiting out the whole backlog -(first-serve step 480) to being served almost immediately (4), wait p95 drops -~70%, and contention fairness goes from lopsided (Jain 0.34) to even (1.00). The -symmetric and lone-key cases are untouched, and drain steps are identical, so the -fair order is work-conserving and does no harm. The cost is a modest per-dequeue -Redis op increase. - -## Arm 2: end-to-end on the three worker regions (realism check) - -Deployed task on one shared base queue, per-run concurrency keys, environment -concurrency ceiling pinned to 5 so the CK dequeue order is the bottleneck. -Noisy-neighbor load: tenant A floods across 30 keys (240 runs), tenant B sends 30 -runs across 3 keys, runs spread across the three regions, 1.5s hold. Metric is -per-run start latency (`startedAt - createdAt`, ms). Flag flipped OFF vs ON with a -control-plane redeploy between arms; identical load each arm. - -| tenant | metric | baseline (OFF) | vtime (ON) | delta | -| --- | --- | --- | --- | --- | -| **B (victim, 3 keys)** | start latency mean | 221,476 | 136,344 | **−38%** | -| | start latency p50 | 219,901 | 133,930 | **−39%** | -| | start latency p95 | 259,092 | 221,061 | −15% | -| | start latency p99 | 259,136 | 221,078 | −15% | -| A (flood, 30 keys) | start latency mean | 95,130 | 100,136 | +5% | -| A | start latency p50 | 95,049 | 95,689 | ~0% | - -Reading it: under age order the light tenant waits behind the flood (mean ~221s); -with virtual time it takes fair turns and drops to ~136s (mean −38%, p50 −39%), -while the flood tenant is essentially unchanged (+5%, it stops jumping the queue). -The smaller p95/p99 gain reflects per-**key** fairness: tenant B's keys carry -deeper per-key backlogs here (10 runs/key vs the flood's 8), so B's last runs -stay in the fair rotation longer. Absolute latencies are the cap-5 serialization -plus per-run worker startup on a single box; only the OFF-vs-ON delta is the -signal. - -## Bottom line - -Both arms agree: the fair order removes the starvation of a light concurrency key -behind a large or sharded backlog, does not harm the balanced or lone-key cases, -stays work-conserving, and costs a small, bounded per-dequeue Redis overhead. The -micro-benchmark isolates the scheduler (the defensible numbers); the end-to-end -run confirms the same effect on real deployed runs across the worker regions. diff --git a/internal-packages/run-engine/design/benchmarks/results-cardinality-2026-07-28.md b/internal-packages/run-engine/design/benchmarks/results-cardinality-2026-07-28.md deleted file mode 100644 index e62826e8e..000000000 --- a/internal-packages/run-engine/design/benchmarks/results-cardinality-2026-07-28.md +++ /dev/null @@ -1,82 +0,0 @@ -# CK virtual-time scheduling: Redis CPU + memory vs cardinality (2026-07-28) - -Answers the review question: how do these changes affect the run-queue Redis CPU -and memory, and how do both react as concurrency-key cardinality grows? - -Run on a **local homelab box, not production**, against a dedicated Redis -configured for measurement (no RDB/AOF in the sampling windows, `maxmemory 0`, -zset listpack thresholds at their defaults so the encoding boundary sits at 128). -All numbers are **relative** (flag OFF vs ON, identical load, same box); absolute -throughput is not prod scale. Method: `2026-07-28-ck-vtime-resource-cardinality-plan.md`. -Server-side metrics (`INFO memory`/`cpu`/`commandstats`, `MEMORY USAGE`, -`OBJECT ENCODING`) are RTT-independent. - -## Memory at rest (single base queue, one queued message per key) - -| keys (N) | used_memory OFF | used_memory ON | ON-OFF delta | ckIndex | ckVtime | ckVtime/ckIndex | ckVtime encoding | -| --- | --- | --- | --- | --- | --- | --- | --- | -| 100 | 1.84 MB | 1.93 MB | +0.09 MB | 6.9 KB | 6.2 KB | 0.89 | listpack | -| 1,000 | 2.42 MB | 2.58 MB | +0.15 MB | 131 KB | 131 KB | 1.00 | skiplist | -| 10,000 | 9.12 MB | 10.4 MB | +1.26 MB | 1.42 MB | 1.42 MB | 1.00 | skiplist | -| 50,000 | 39.1 MB | 45.6 MB | +6.51 MB | 7.55 MB | 7.54 MB | 1.00 | skiplist | - -The `:ckVtime` ZSET is the whole added footprint, and it is essentially a second -copy of `:ckIndex`: same members (the full CK-variant queue names), one extra -8-byte score, so `MEMORY USAGE(ckVtime) ~= MEMORY USAGE(ckIndex)` once past the -listpack boundary. Cost is linear in cardinality at roughly **150 bytes per -concurrency key** on top of the index the queue already keeps: about +1.3 MB for -a queue with 10k keys, +6.5 MB at 50k. It is bounded by live cardinality (entries -are GC'd when a variant drains) and expires on the 24h state TTL. The -`used_memory` delta tracks the direct `MEMORY USAGE(ckVtime)` figure to within -allocator noise. The listpack->skiplist transition lands between 100 and 1,000 -keys as expected. - -## Redis CPU under an identical workload (2,000 rounds, ~42k script calls) - -Same logical workload both arms (enqueue + batched dequeue + ack), so the -`evalsha` call count is identical and the difference is pure vtime overhead. Note -the RunQueue Lua runs as `EVALSHA`, so per-`redis.call` costs inside a script are -not separable in `commandstats`; the reportable signals are total Redis CPU and -aggregate `evalsha` time per call. - -| keys (N) | CPU-sec OFF | CPU-sec ON | delta | overhead | evalsha usec/call OFF | evalsha usec/call ON | -| --- | --- | --- | --- | --- | --- | --- | -| 100 | 1.34 | 1.49 | +0.15 | +12% | 22.3 | 26.2 | -| 1,000 | 1.34 | 1.44 | +0.10 | +7% | 22.2 | 24.8 | -| 10,000 | 1.38 | 1.45 | +0.07 | +5% | 23.8 | 25.8 | - -CPU overhead does not grow with cardinality, it shrinks: +12% at 100 keys down to -+5% at 10k. Per-script cost stays roughly flat (about +2 to +4 usec/call, ~25 -usec/call at every cardinality), which is what the design predicts: the pass-1 -dequeue window is fixed (`maxCount * multiplier`, default 30) and independent of -N, and the added ZSET ops are O(log N), so `log2(10000) ~= 13` adds a negligible -constant. The overhead falls as a percentage because that fixed per-call cost is -amortised over more work as the queue grows. - -## Membership under sustained churn (N = 10,000, flag ON) - -60 rounds of continuous registration + drain (fresh keys enqueued while others -are served and acked), holding cardinality at 10k: - -| round | ckIndex card | ckVtime card | ratio | -| --- | --- | --- | --- | -| 0 | 10,000 | 10,000 | 1.0 | -| 20 | 10,000 | 10,000 | 1.0 | -| 40 | 10,000 | 10,000 | 1.0 | -| 59 | 10,000 | 10,000 | 1.0 | - -`ckVtime` membership tracks `ckIndex` exactly throughout: no tombstone -accumulation, no unbounded growth. The bounded/self-healing drift the limitations -doc calls out (ack/TTL/DLQ draining a variant without a vtime GC) stays reclaimed -by the next vtime pass and the state TTL. - -## Bottom line for the cardinality question - -- **Memory** grows linearly with concurrency-key cardinality, adding one - `ckIndex`-sized ZSET per base queue (~150 B/key): a low-single-digit MB even at - 10k keys on one queue, bounded by live cardinality and TTL-reclaimed. A sudden - 10k-key queue costs about +1.3 MB for that queue. -- **CPU** overhead is small and does not scale with cardinality: ~+5 to +12% on - an identical workload, per-script cost flat at ~+2 to +4 usec/call regardless of - N, because the dequeue scan window is fixed and the ZSET ops are O(log N). -- **No runaway state**: `ckVtime` never outgrows `ckIndex` under sustained churn. diff --git a/internal-packages/run-engine/design/plans/2026-07-23-ck-virtual-time-scheduling-plan.md b/internal-packages/run-engine/design/plans/2026-07-23-ck-virtual-time-scheduling-plan.md deleted file mode 100644 index 26a8311d4..000000000 --- a/internal-packages/run-engine/design/plans/2026-07-23-ck-virtual-time-scheduling-plan.md +++ /dev/null @@ -1,829 +0,0 @@ -# Virtual-time (SFQ) scheduling for the concurrency-key dequeue - -Implementation and testing plan for the recommendation out of three fairness -spikes. The spike harness and benchmark code are archived on the remote branch -`chore/fair-queueing-spike` (throwaway, never merged); the findings and research -they produced are kept alongside this plan as references: -`internal-packages/run-engine/design/references/run-queue-fairness-ck-findings.md`, -`.../run-queue-fairness-caps-vs-scheduling-findings.md`, and -`.../run-queue-fairness-research.md`. - -The recommendation: score the per-base-queue concurrency-key selection by -start-time fair queueing (SFQ) virtual time instead of head timestamp, inside the -real batched CK-dequeue Lua, layered UNDER the existing per-key concurrency gate -and the planned group caps. Caps bound occupancy; the fair order bounds wait under -contention (the Kubernetes-APF shape, and the Parekh-Gallager joint result). - -## Goal - -When many concurrency-key variants of one base queue are contending, the -dequeue order across variants follows SFQ virtual time (each variant advances -its own virtual clock by a quantum per serve; new variants join at a monotonic -floor). This fixes the #2617 starvation dynamic the spikes measured: a key -arriving behind a big backlog waits its fair turn instead of waiting for the -backlog to drain, and (unlike per-key caps) the fix survives a tenant sharding -its work across many keys, while staying work-conserving. - -Everything is behind a constructor feature flag. Flag off is byte-identical to -today: the exact same Lua scripts run and no new Redis keys are ever touched. - -## Architecture - -The design keeps `ckIndex` exactly as it is and adds a parallel virtual-time -ZSET. This is the load-bearing decision, so the reasoning up front: - -`ckIndex` scores are head-message timestamps, and three things depend on that -score domain staying timestamps: - -1. Time eligibility. `ZRANGEBYSCORE ckIndexKey -inf now` filters out variants - whose head message is scheduled in the future (delayed runs, nack backoff). - Virtual-time tags carry no wall-clock meaning, so they cannot express "not - available yet". -2. Master-queue rebalancing. Every CK Lua (enqueue, dequeue, ack, nack, the - sweeper) re-scores the `:ck:*` master-queue member from `ZRANGE ckIndexKey - 0 0 WITHSCORES`. The master queue is timestamp-ordered and compared against - `now`; writing virtual times there would corrupt the shard-level selection. -3. Every other writer. `enqueueMessageCkTracked`, `nackMessageCkTracked`, - `acknowledgeMessageCkTracked`, the concurrency sweeper (index.ts ~3733, - ~3847) all `ZADD ckIndexKey `. Rescoring only in the dequeue - Lua would leave a mixed score domain, and during a rolling deploy old - instances would keep writing timestamps regardless (the mixed-arity hazard - the caps plan warned about, in score-domain form). - -So: instead of changing `ckIndex`'s score domain, add per base queue - -- `{org:...}:...:queue::ckVtime`, a ZSET, member = the full CK-variant - queue name (the same member strings `ckIndex` holds), score = the variant's - next virtual start tag (the spike's `SfqCk.clock` value, i.e. start of last - serve + quantum). -- `{org:...}:...:queue::ckVtimeFloor`, a STRING holding the monotonic - floor (the CFS `min_vruntime` analogue from `disciplines.ts`). - -Both live under the same `{org:...}` hash tag as every other key of the base -queue, so cluster slotting is unchanged and one Lua script can touch all of -them atomically. - -The dequeue Lua (new command, flag-selected) runs two passes: - -- Pass 1 (fair order): take candidates from `ckVtime` by rank (lowest tag - first, `ZRANGE 0 W-1 WITHSCORES`), and for each run the existing - per-candidate logic unchanged: per-key concurrency gate, per-variant - time-eligibility check (`ZRANGEBYSCORE -inf now LIMIT 0 1`), TTL - branch, counters, `ckIndex` rebalance. On each successful serve, advance - that variant's tag (`ZADD ckVtimeKey max(tag, floor) + quantum/weight`) - before moving to the next candidate, so the state is correct per serve - within the batch. A skipped variant (at cap, or head in the future) keeps - its tag: no service, no advance, which is the SFQ rule. -- Pass 2 (fill + discovery): if pass 1 served fewer than `actualMaxCount`, - scan `ckIndex` in today's age order (`ZRANGEBYSCORE -inf now LIMIT 0 W`), - skip variants already attempted in pass 1, and serve the rest through the - same per-candidate logic, registering each served variant into `ckVtime`. - Pass 2 makes the new command a strict superset of today's: it can never - serve fewer messages than the current script would, so work conservation - and mixed-deploy discovery both hold by construction. - -Registration (how a variant gets INTO `ckVtime` before it is ever served): -every Lua that adds messages to a variant, i.e. the CK enqueue commands and -the CK nack command, gains a flag-selected variant that does -`ZADD ckVtimeKey NX ` after its existing `ckIndex` rebalance. -`NX` means registration can never rewind an advanced tag. This is what makes -the sybil case work: a brand-new light key is present in the vtime order at -the floor from its first enqueue, so it is reachable in pass 1 even when a -hundred attacker variants have older heads (which is exactly where today's -age-ordered `*3` window fails, per CAPS_FINDINGS). - -Closure argument for registration (state this as an invariant and test it): -a variant's queue becomes non-empty only via enqueue or nack, both of which -register. The sweeper and ack/release Luas only rebalance variants whose -queues are already non-empty, so they never need to register. The dequeue Lua -GCs a variant from BOTH `ckIndex` and `ckVtime` when its queue is empty, so -membership stays closed under all transitions. The one gap is old-code -enqueues during a rolling deploy, and pass 2 covers that (served via age -order, registered on serve). - -Batched-call semantics: the current Lua serves at most ONE message per variant -per call (`LIMIT 0, 1` per candidate, and ZSET members are unique in the -candidate list). The new command keeps that. Within one call the batch is -therefore one-serve-per-variant round robin over the `actualMaxCount` lowest -tags, and each serve's `ZADD` makes the NEXT call's order correct. This -deviates from pure SFQ within a single batch (pure SFQ could serve the same -far-behind variant several times in a row) but converges across calls, and -one-per-variant is itself a fair schedule. Preserving it also means zero -change to today's per-call throughput shape. - -Layering with caps: the per-key gate (`ckCurrentConcurrency < -queueConcurrencyLimit`) stays exactly where it is, ahead of the serve. The -planned Phase-1 `:groupConcurrency`/`:totalConcurrency` total cap and Phase-2 -`:ckLimits` per-key overrides slot into the same per-candidate position as -additional admission conditions when they land; virtual time only decides the -ORDER among candidates those gates admit. Nothing in this plan blocks or is -blocked by the caps work, and the fairQueue-level "queue at total" drop stays -untouched (the CK pick is below the `RunQueueSelectionStrategy` interface; -`fairQueueSelectionStrategy.ts` is not modified). - -Weights: concurrency keys carry no configured weight today, so every key gets -weight 1 (quantum advance of 1.0 per serve). The advance is written as -`quantum / weight` with `weight` a named local fixed at 1, so a future -per-key weight (e.g. a sparse `:ckWeights` HASH mirroring the Phase-2 -`:ckLimits` shape) is a one-line change at the marked site. Justification for -equal-weight first: the spikes only measured equal weights, no product surface -exists to set a weight, and SFQ's starvation fix does not depend on weights. - -State lifecycle (GC/TTL), since concurrency keys are client-chosen and -unbounded: - -- Per-variant GC: whenever the dequeue Lua finds a variant queue empty it - already `ZREM`s the variant from `ckIndex`; the new command also `ZREM`s it - from `ckVtime` at those sites. A GC'd key that returns re-registers at the - floor, which is standard SFQ flow re-entry (history is forgiven when a flow - drains; a drained flow was by definition not backlogged). -- Whole-key TTL: `ckVtime` and `ckVtimeFloor` get `EXPIRE ` - (default 86400, matching the `counterTtlSeconds` precedent) refreshed on - every write. An idle base queue's vtime state evaporates; on resumption - everyone re-enters at floor 0, which is a clean restart. If only the floor - key expires, tags in `ckVtime` still self-heal because every read applies - `max(tag, floor)` and the next dequeue re-advances the floor to the minimum - stored tag. -- Cardinality guard: tags are only created for variants that actually have - queued messages (registration happens on enqueue/nack, GC on empty), so - `ckVtime` cardinality is bounded by `ckIndex` cardinality plus transiently - stale entries awaiting scan-time GC or TTL expiry. - -Floor semantics: on each dequeue call, `floor = max(stored floor, score of -ckVtime rank 0)`, written back with the TTL. The floor never decreases (test -this), and a newly registered key's tag starts AT the floor, so it can never -be scheduled behind the accumulated backlog of long-running keys (the SFQ -property; this is the exact `SfqCk` logic from `disciplines.ts`, moved into -Lua with the Map replaced by the ZSET and the floor by the STRING). - -Numeric domain: tags are Redis doubles starting at 0 advancing by 1.0 per -serve; integer-exact to 2^53 serves per base queue, so precision is a -non-issue. - -The scan window: pass 1's window is `actualMaxCount * windowMultiplier` -(default multiplier 3, same as today, made configurable). The score domain of -the window changes meaning: today the window can hide the oldest ELIGIBLE -head behind at-cap variants with older heads; under vtime it can hide the -lowest ELIGIBLE tag behind at-cap or future-scheduled variants with lower -tags. Those clogging variants keep low tags while skipped (no serve, no -advance), so the failure shape is symmetric with today's, and it is the same -class of limitation CAPS_FINDINGS documents for the `*3` window rather than a -new one. We do not widen the default; we make the multiplier an option so an -operator can widen it if per-key caps plus heavy nack backoff ever clog a -window in practice, and pass 2 guarantees the call still finds work. - -## Tech stack - -- Redis Lua (ioredis `defineCommand`) in - `internal-packages/run-engine/src/run-queue/index.ts`, following the - existing tracked-command patterns. -- TypeScript for options plumbing and call-site selection. -- vitest + `@internal/testcontainers` (`redisTest`) for all tests. No mocks. -- Verification: `pnpm run typecheck --filter @internal/run-engine` and - `cd internal-packages/run-engine && pnpm run test --run`. - -## Global constraints - -- Flag off must be byte-identical: off-path call sites keep calling the - existing command names whose script text is not edited at all. New - behaviour lives only in NEW command names (`...Vtime...`), selected in TS. - This is why we add command variants instead of threading an `enableVtime` - ARGV through existing scripts: an ARGV-gated single script would still be a - new script body (new SHA, new arity risks) even when the flag is off. -- `ckIndex`, the master queue, `fairQueueSelectionStrategy.ts`, and all - ack/release/sweeper Luas keep their current score domain and text. -- The dead untracked `dequeueMessagesFromCkQueue` (index.ts ~3999-4141) is not - touched and gets no vtime variant. -- All vtime state mutations happen inside single Lua scripts (atomic; Redis - serialises scripts, which is the whole multi-consumer correctness story). -- No process-memory scheduling state anywhere. -- Do not import anything from `fairness-spike-ck/` or `fairness-spike/` into - production code or the new tests; those directories are throwaway (their - own headers say delete before merge). Port logic and scenario shapes by - copying, with attribution comments. -- Zod stays at the repo-pinned version; no new dependencies. -- Formatting/lint before commit: `pnpm run format && pnpm run lint:fix`. - -## File structure - -Modify: - -- `internal-packages/run-engine/src/run-queue/keyProducer.ts` - (two new key builders + constants) -- `internal-packages/run-engine/src/run-queue/types.ts` - (`RunQueueKeyProducer` interface additions) -- `internal-packages/run-engine/src/run-queue/index.ts` - (options field; four new `defineCommand`s; TS module augmentation for them; - flag switches at the enqueue/nack/dequeue call sites; span attributes) -- `internal-packages/run-engine/src/run-queue/tests/keyProducer.test.ts` - (key builder tests) -- `internal-packages/run-engine/src/engine/types.ts` - (`RunEngineOptions["queue"].ckVirtualTimeScheduling`) -- `internal-packages/run-engine/src/engine/index.ts` - (pass the option through to `new RunQueue({...})`, ~line 196) -- `apps/webapp/app/env.server.ts` - (`RUN_ENGINE_CK_VTIME_SCHEDULING_ENABLED` and friends) -- `apps/webapp/app/v3/runEngine.server.ts` - (wire env vars into the engine options, ~line 61 `queue:` block) - -Create: - -- `internal-packages/run-engine/src/run-queue/tests/ckVtime.test.ts` - (Lua behaviour: ordering, floor, tag init, advance-within-batch, GC, TTL, - registration, pass-2 fill, flag-off keyspace purity) -- `internal-packages/run-engine/src/run-queue/tests/ckVtimeFairness.test.ts` - (ported scenarios at batched maxCount, wait/share/work-conservation/sybil - assertions) -- `internal-packages/run-engine/src/run-queue/tests/ckVtimeConcurrency.test.ts` - (multi-consumer correctness, op-count budget) -- `.server-changes/2026-07-24-ck-fair-scheduling.md` (at PR time; note it - ships dark) - -## Tasks - -### Task 1: key producer additions - -Files: `keyProducer.ts`, `types.ts`, `tests/keyProducer.test.ts`. - -Test first (append to `tests/keyProducer.test.ts`, matching its existing -style): - -```ts -it("produces ckVtime keys from a CK variant queue name", () => { - const keys = new RunQueueFullKeyProducer(); - const q = "{org:o1}:proj:p1:env:e1:queue:task/my-task:ck:tenant-a"; - expect(keys.ckVtimeKeyFromQueue(q)).toBe( - "{org:o1}:proj:p1:env:e1:queue:task/my-task:ckVtime" - ); - expect(keys.ckVtimeFloorKeyFromQueue(q)).toBe( - "{org:o1}:proj:p1:env:e1:queue:task/my-task:ckVtimeFloor" - ); - // ck wildcard and base-queue inputs normalise the same way - expect(keys.ckVtimeKeyFromQueue(q.replace(":ck:tenant-a", ":ck:*"))).toBe( - keys.ckVtimeKeyFromQueue(q) - ); -}); -``` - -Implementation in `keyProducer.ts`: add to `constants` - -```ts -CK_VTIME_PART: "ckVtime", -CK_VTIME_FLOOR_PART: "ckVtimeFloor", -``` - -and the builders (next to `ckIndexKeyFromQueue`): - -```ts -ckVtimeKeyFromQueue(queue: string): string { - return `${this.baseQueueKeyFromQueue(queue)}:${constants.CK_VTIME_PART}`; -} - -ckVtimeFloorKeyFromQueue(queue: string): string { - return `${this.baseQueueKeyFromQueue(queue)}:${constants.CK_VTIME_FLOOR_PART}`; -} -``` - -Add both signatures to `RunQueueKeyProducer` in `types.ts`. - -Verify: `cd internal-packages/run-engine && pnpm run test ./src/run-queue/tests/keyProducer.test.ts --run` -(new tests pass, existing pass), then -`pnpm run typecheck --filter @internal/run-engine` (clean). - -### Task 2: options plumbing in RunQueue - -File: `index.ts` (RunQueueOptions, ~line 60). - -```ts -/** - * Fair (virtual-time / SFQ) ordering across concurrency-key variants of a - * base queue. Off by default; when off, the exact pre-existing Lua commands - * run and no vtime keys are created. See internal-packages/run-engine/design/plans/ - * 2026-07-23-ck-virtual-time-scheduling-plan.md. - */ -ckVirtualTimeScheduling?: { - enabled: boolean; - /** Virtual-time advance per serve (dimensionless). Default 1. */ - quantum?: number; - /** Pass-1 candidate window = actualMaxCount * this. Default 3. */ - scanWindowMultiplier?: number; - /** EXPIRE applied to ckVtime/ckVtimeFloor on every write. Default 86400. */ - stateTtlSeconds?: number; -}; -``` - -Store resolved values once in the constructor (private readonly fields -`#ckVtimeEnabled`, `#ckVtimeQuantum`, `#ckVtimeWindowMultiplier`, -`#ckVtimeStateTtl`) so call sites read fields, never re-derive. - -Verify: `pnpm run typecheck --filter @internal/run-engine`. - -### Task 3: the vtime dequeue Lua (the core change) - -File: `index.ts`. New command `dequeueMessagesFromCkQueueVtimeTracked`, -`numberOfKeys: 12` (the 10 keys of `dequeueMessagesFromCkQueueTracked` plus -`ckVtimeKey`, `ckVtimeFloorKey`), plus its entry in the ioredis module -augmentation (next to the existing declaration at ~line 5591, same parameter -list plus `ckVtimeKey: string, ckVtimeFloorKey: string` after -`lengthCounterKey` and `quantum: string, windowMultiplier: string, -stateTtlSeconds: string` after `maxCount`). - -Write the failing tests FIRST in `tests/ckVtime.test.ts`. Scaffold the file -from `tests/ckIndex.test.ts` (same `testOptions`, `authenticatedEnvDev`, -`createQueue`, `makeMessage` helpers), with `createQueue` extended to accept -`ckVirtualTimeScheduling` overrides. Tests to write in this task: - -1. "vtime order beats head-timestamp order": enqueue 30 messages on - `ck: heavy` with timestamps `t0 .. t0+29`, then 3 messages on `ck: light` - at `t0+1000`. Register both (enqueue registration is Task 4; until then - the test seeds `ckVtime` directly with - `queue.redis.zadd(ckVtimeKey, 0, heavyVariant, 0, lightVariant)`). - Dequeue with `maxCount: 10` repeatedly (acking between calls to free - concurrency). Assert light's 3 messages are all served within the first 3 - calls (age order alone would drain heavy first). Assert each call returns - at most one message per variant. -2. "tags advance per serve within one batched call": seed 5 variants at tag - 0, one message each; one dequeue call with `maxCount: 5`; assert all 5 - served and `ZSCORE ckVtime ` is `1` for each (advanced inside the one - call, not once per call). -3. "floor is monotonic and read-repairs": drive tags to ~20 by repeated - serve of two keys, assert `GET ckVtimeFloor` never decreased across calls - (sample after each call), and equals the min stored tag after the last. -4. "new key initialises at the floor, not zero and not behind the backlog": - after tags reach ~20, register a fresh variant with the enqueue path (or - direct ZADD NX at the current floor pre-Task-4), enqueue one message on - it, one dequeue call; assert the fresh variant is served in that first - call and its tag afterwards is `floor + quantum`, not `1`. -5. "no service, no advance": set the base queue concurrency limit to 1 via - `queue.updateQueueConcurrencyLimits`, occupy `ck: a`'s slot (dequeue one, - do not ack), then call dequeue; assert `ck: a` was skipped, its tag is - unchanged, and other variants were served. -6. "GC on empty variant": drain a variant completely; assert it is removed - from BOTH `ckIndex` and `ckVtime`. -7. "TTL is set and refreshed": after any dequeue, `PTTL ckVtime` and - `PTTL ckVtimeFloor` are in `(0, stateTtlSeconds * 1000]`. -8. "pass 2 fill serves unregistered variants and registers them": enqueue on - a variant, delete its `ckVtime` entry by hand (simulating an old-code - enqueue), dequeue; assert the message is served AND the variant now has a - `ckVtime` tag. -9. "future-scheduled variants are skipped without advance": nack a message - with a future score (or enqueue with future timestamp); dequeue; assert - the variant is not served and its tag is unchanged. - -The Lua. Full sketch (the per-candidate serve body is today's tracked body -verbatim; only the parts marked NEW differ): - -```lua -local ckIndexKey = KEYS[1] -local queueConcurrencyLimitKey = KEYS[2] -local envConcurrencyLimitKey = KEYS[3] -local envConcurrencyLimitBurstFactorKey = KEYS[4] -local envCurrentConcurrencyKey = KEYS[5] -local messageKeyPrefix = KEYS[6] -local envQueueKey = KEYS[7] -local masterQueueKey = KEYS[8] -local ttlQueueKey = KEYS[9] -local lengthCounterKey = KEYS[10] -local ckVtimeKey = KEYS[11] -- NEW -local ckVtimeFloorKey = KEYS[12] -- NEW - -local ckWildcardName = ARGV[1] -local currentTime = tonumber(ARGV[2]) -local defaultEnvConcurrencyLimit = ARGV[3] -local defaultEnvConcurrencyBurstFactor = ARGV[4] -local keyPrefix = ARGV[5] -local maxCount = tonumber(ARGV[6] or '1') -local quantum = tonumber(ARGV[7] or '1') -- NEW -local windowMultiplier = tonumber(ARGV[8] or '3') -- NEW -local stateTtl = tonumber(ARGV[9] or '86400') -- NEW - -local function decrLengthCounter() - if tonumber(redis.call('GET', lengthCounterKey) or '0') > 0 then - redis.call('DECR', lengthCounterKey) - end -end - --- env gate: identical to dequeueMessagesFromCkQueueTracked -local envCurrentConcurrency = tonumber(redis.call('SCARD', envCurrentConcurrencyKey) or '0') -local envConcurrencyLimit = tonumber(redis.call('GET', envConcurrencyLimitKey) or defaultEnvConcurrencyLimit) -local envConcurrencyLimitBurstFactor = tonumber(redis.call('GET', envConcurrencyLimitBurstFactorKey) or defaultEnvConcurrencyBurstFactor) -local envConcurrencyLimitWithBurstFactor = math.floor(envConcurrencyLimit * envConcurrencyLimitBurstFactor) -if envCurrentConcurrency >= envConcurrencyLimitWithBurstFactor then - return nil -end -local queueConcurrencyLimit = math.min(tonumber(redis.call('GET', queueConcurrencyLimitKey) or '1000000'), envConcurrencyLimit) -local envAvailableCapacity = envConcurrencyLimitWithBurstFactor - envCurrentConcurrency -local actualMaxCount = math.min(maxCount, envAvailableCapacity) -if actualMaxCount <= 0 then - return nil -end - -local window = actualMaxCount * windowMultiplier - --- NEW: monotonic floor, advanced to the minimum stored tag -local floor = tonumber(redis.call('GET', ckVtimeFloorKey) or '0') -local minEntry = redis.call('ZRANGE', ckVtimeKey, 0, 0, 'WITHSCORES') -if #minEntry > 0 then - local minTag = tonumber(minEntry[2]) - if minTag > floor then - floor = minTag - end -end - -local results = {} -local dequeuedCount = 0 -local attempted = {} - --- Per-candidate serve. Body between BEGIN/END COPY is today's tracked --- per-candidate block, unmodified except the two NEW lines. -local function tryServe(ckQueueName) - attempted[ckQueueName] = true - local fullQueueKey = keyPrefix .. ckQueueName - local ckConcurrencyKey = fullQueueKey .. ':currentConcurrency' - local ckCurrentConcurrency = tonumber(redis.call('SCARD', ckConcurrencyKey) or '0') - if ckCurrentConcurrency >= queueConcurrencyLimit then - return - end - -- BEGIN COPY (from dequeueMessagesFromCkQueueTracked, lines ~4219-4273) - local messages = redis.call('ZRANGEBYSCORE', fullQueueKey, '-inf', tostring(currentTime), 'WITHSCORES', 'LIMIT', 0, 1) - if #messages >= 2 then - -- ... TTL-expired / normal-dequeue / stale-orphan branches verbatim ... - -- in the normal-dequeue branch, after dequeuedCount = dequeuedCount + 1: - -- NEW: advance this variant's virtual time (weight hook: fixed 1 today) - -- local weight = 1 - -- local tag = tonumber(redis.call('ZSCORE', ckVtimeKey, ckQueueName) or floor) - -- if tag < floor then tag = floor end - -- redis.call('ZADD', ckVtimeKey, tag + (quantum / weight), ckQueueName) - -- rebalance ckIndex from the variant head, verbatim, plus: - -- NEW: if the variant queue is empty, also redis.call('ZREM', ckVtimeKey, ckQueueName) - else - -- empty-in-range branch verbatim, plus the same NEW ZREM when fully empty - end - -- END COPY -end - --- Pass 1: fair order (lowest virtual start tag first) -local vtimeCandidates = redis.call('ZRANGE', ckVtimeKey, 0, window - 1) -for _, ckQueueName in ipairs(vtimeCandidates) do - if dequeuedCount >= actualMaxCount then break end - tryServe(ckQueueName) -end - --- Pass 2: fill + discovery in today's age order (work conservation, --- mixed-deploy safety). Never runs when pass 1 filled the batch. -if dequeuedCount < actualMaxCount then - local ckQueues = redis.call('ZRANGEBYSCORE', ckIndexKey, '-inf', tostring(currentTime), 'LIMIT', 0, window) - for _, ckQueueName in ipairs(ckQueues) do - if dequeuedCount >= actualMaxCount then break end - if not attempted[ckQueueName] then - tryServe(ckQueueName) - end - end -end - --- NEW: persist floor and refresh TTLs -redis.call('SET', ckVtimeFloorKey, tostring(floor), 'EX', stateTtl) -if redis.call('EXISTS', ckVtimeKey) == 1 then - redis.call('EXPIRE', ckVtimeKey, stateTtl) -end - --- master queue rebalance: verbatim from the tracked command (uses ckIndex, --- which keeps its timestamp domain) -local earliestIdx = redis.call('ZRANGE', ckIndexKey, 0, 0, 'WITHSCORES') -if #earliestIdx == 0 then - redis.call('ZREM', masterQueueKey, ckWildcardName) -else - redis.call('ZADD', masterQueueKey, earliestIdx[2], ckWildcardName) -end - -return results -``` - -Note the `tryServe` extraction is inside the NEW script only; the old script -is not refactored. When writing the real script, inline today's per-candidate -block into `tryServe` exactly (including `decrLengthCounter`, the TTL-member -removal, and both rebalance branches); the sketch elides it to keep the plan -readable, and the byte-identity constraint applies to the OLD script, which -is untouched. - -Call-site switch in `#callDequeueMessagesFromCkQueue` (~line 2206): - -```ts -const result = this.#ckVtimeEnabled - ? await this.redis.dequeueMessagesFromCkQueueVtimeTracked( - ckIndexKey, queueConcurrencyLimitKey, envConcurrencyLimitKey, - envConcurrencyLimitBurstFactorKey, envCurrentConcurrencyKey, - messageKeyPrefix, envQueueKey, masterQueueKey, ttlQueueKey, - lengthCounterKey, - this.keys.ckVtimeKeyFromQueue(ckWildcardQueue), - this.keys.ckVtimeFloorKeyFromQueue(ckWildcardQueue), - ckWildcardQueue, String(Date.now()), - String(this.options.defaultEnvConcurrency), - String(this.options.defaultEnvConcurrencyBurstFactor ?? 1), - this.options.redis.keyPrefix ?? "", String(maxCount), - String(this.#ckVtimeQuantum), String(this.#ckVtimeWindowMultiplier), - String(this.#ckVtimeStateTtl) - ) - : await this.redis.dequeueMessagesFromCkQueueTracked(/* unchanged */); -``` - -Add span attributes on the vtime path: `ck_vtime_enabled: true` plus, from a -small extension of the return shape if desired later, keep it simple now and -only tag the flag. - -Verify: `cd internal-packages/run-engine && pnpm run test ./src/run-queue/tests/ckVtime.test.ts --run` -(tests 1-3, 5-9 pass; 4 passes with the direct-ZADD seeding until Task 4), -then `pnpm run typecheck --filter @internal/run-engine`. - -### Task 4: enqueue registration - -File: `index.ts`. Two new commands, `enqueueMessageCkVtimeTracked` and -`enqueueMessageWithTtlCkVtimeTracked`, `numberOfKeys: 17` (the existing 15 -plus `ckVtimeKey` as KEYS[16] and `ckVtimeFloorKey` as KEYS[17]; the WithTtl -variant is existing-16 plus 2), ARGV extended with `stateTtl`. Script body = -existing tracked script verbatim, plus, in the SLOW PATH ONLY, immediately -after the `-- Rebalance CK index` block: - -```lua --- Register this variant in the virtual-time index at the floor. NX means an --- already-advanced tag is never rewound. -local vfloor = redis.call('GET', ckVtimeFloorKey) or '0' -redis.call('ZADD', ckVtimeKey, 'NX', vfloor, queueName) -redis.call('EXPIRE', ckVtimeKey, stateTtl) -``` - -The fast path (direct-to-worker-queue when the variant is empty and capacity -is free) does NOT register or advance; see open decision 1. - -Tests first, in `tests/ckVtime.test.ts`: - -10. "enqueue registers the variant at the current floor with NX": drive the - floor to ~5 via serves, enqueue on a fresh key, assert - `ZSCORE ckVtime ` equals the floor; enqueue a second message on a - key whose tag is 9, assert the tag is still 9. -11. "test 4 now passes end-to-end without direct ZADD seeding" (remove the - seeding from test 4). -12. "fast path leaves vtime state untouched": empty variant, free capacity, - enqueue (fast path fires, returns 1); assert no `ckVtime` entry was - created for it. Then saturate capacity, enqueue again (slow path); - assert registration happened. - -Call-site switches at ~lines 1906 and 1941 pick the vtime variants when -`this.#ckVtimeEnabled`, passing the two extra keys and `stateTtl`. Add both -to the module augmentation. - -Verify: same test file command; plus -`pnpm run test ./src/run-queue/tests/enqueueMessage.test.ts --run` and -`./src/run-queue/tests/ckIndex.test.ts --run` still green (flag off). - -### Task 5: nack registration - -File: `index.ts`. New command `nackMessageCkVtimeTracked`, -`numberOfKeys: 13` (existing 11 plus the two vtime keys), ARGV plus -`stateTtl`. Body = existing verbatim plus the same NX-register block after -its `-- Rebalance CK index` section. Call-site switch at ~line 2584. - -Test first (in `tests/ckVtime.test.ts`): - -13. "nack re-registers a GC'd variant": enqueue one message on `ck: a`, - dequeue it (variant now GC'd from both indexes), nack it; assert the - variant is back in `ckIndex` AND in `ckVtime` at the floor, and a - subsequent dequeue serves it (respecting its future score if the nack - applied backoff: use a nack with an immediate retry score). - -Closure invariant test: - -14. "ckVtime membership tracks ckIndex membership": property-style loop of - ~200 random operations (enqueue on 1 of 8 keys, dequeue batch, ack or - nack a random in-flight message); after each step assert every member of - `ckIndex` is a member of `ckVtime` (the converse may transiently not - hold, which is fine; stale `ckVtime` entries GC on scan). - -Verify: `pnpm run test ./src/run-queue/tests/ckVtime.test.ts --run` and -`./src/run-queue/tests/nack.test.ts --run` (flag off, untouched). - -### Task 6: fairness scenarios on the real batched path - -File: `tests/ckVtimeFairness.test.ts` (new). This closes the spike's fidelity -gap: the spike proved the ordering at `maxCount = 1` with driver-side -rescoring; these tests drive the REAL batched Lua (`maxCount = 10`) with the -state advanced inside the script. - -Harness design (deterministic, no wall-clock sleeps, no spike imports): a -step loop against one `RunQueue` on testcontainers Redis. - -- Enqueue with explicit `timestamp` values in `InputPayload` (all in the - past so everything is time-eligible; the backlog key gets one old shared - timestamp, other keys get strictly increasing later timestamps, mirroring - the ckScenarios head-age reasoning). -- Each step: call the dequeue path once with `maxCount: 10` (via the public - dequeue API used by `ckIndex.test.ts`), record `(step, variant, messageId)` - per served message, then ack each served message after a per-key logical - hold of H steps (keep a small in-flight list and ack entries whose - `servedAt + H <= step`), which is how the env concurrency contends. -- Wait metric per message = serve step minus a per-message logical arrival - step (arrival step derived from the enqueue order). All assertions are on - ratios between flag-on and flag-off runs of the SAME scenario and seed, so - they are stable in CI; use generous factors. - -Scenarios (ported shapes from `ckScenarios.ts` and -`capsFairness.bench.test.ts`, scaled down for CI): - -- ckSkew: heavy 120 backlog msgs (old shared head), 4 light keys x 10 msgs - (later heads). env limit 4, hold 3 steps. Assert: mean light-key wait with - flag ON <= 0.3 x flag OFF (spike measured ~1100 -> ~15, so 0.3 is very - loose); heavy key's wait may rise (do not assert it down). -- ckTrickle: bulk 120, two trickle keys x 15. Same assertion. -- ckSybil (the case caps cannot fix): 20 attacker keys x 8 msgs each, all - older heads, 1 light key x 10 newer. Assert: flag ON mean light wait - <= 0.7 x flag OFF (spike: 1765 -> 1009), AND light key's first serve - happens within the first 3 steps (reachability at the floor), AND - contention-window share: over the steps where >= 2 keys have queued - backlog, light's served fraction >= 0.5 x its fair share 1/21 (directional, - per the spike's confounding caveat; wait is the headline). -- ckBalanced (no-harm check): 4 symmetric keys x 25. Assert: max per-key - mean wait with flag ON <= 1.25 x flag OFF (fair order must not make the - symmetric case worse). -- ckHeavyIdle (work conservation): single key, 60 msgs. Assert: steps to - drain with flag ON == flag OFF exactly (nothing else contends, so any - extra step is a work-conservation bug). - -Also assert in every scenario: total served ON == total served OFF == total -enqueued (no loss, no double-serve; `messageId`s unique). - -Verify: `cd internal-packages/run-engine && pnpm run test ./src/run-queue/tests/ckVtimeFairness.test.ts --run` -(all scenarios pass; target < 60s wall time total, scale message counts down -if needed before loosening assertions). - -### Task 7: multi-consumer / multi-shard correctness - -File: `tests/ckVtimeConcurrency.test.ts` (new). - -15. "two consumers, one base queue, no corruption": one `RunQueue` for - enqueues, two more instances (same Redis, same key prefix, flag on) each - running a dequeue loop with `maxCount: 5` concurrently - (`Promise.all` of two loops, acking with a short hold). 6 keys x 30 - messages. Assert: every message served exactly once across both - consumers (union of served IDs has no duplicates and equals the enqueued - set); after drain, `ckVtime` is empty and floor equals the max it ever - reached; sample the floor between iterations and assert it never - decreased. The correctness argument is that every mutation happens - inside one Lua script and Redis serialises scripts; this test is the - check that the scripts do not assume cross-call state. -16. "concurrent enqueue during dequeue cannot rewind a tag": interleave - enqueues on a hot key with dequeue batches; after each round assert - `ZSCORE ckVtime ` is non-decreasing (NX registration + advance-only - writes). - -17. "op-count budget": using a second plain Redis client, `CONFIG RESETSTAT`, - run 50 identical dequeue calls flag OFF, snapshot - `INFO commandstats` total calls; repeat flag ON with identical data. - Assert `on_total <= off_total + 50 * (6 + 2 * maxCount)` (per call the - vtime path adds at worst: GET floor, ZRANGE min, ZRANGE window, SET - floor, EXPIRE, the pass-2 ZRANGEBYSCORE, plus per serve one ZSCORE and - one ZADD). This pins the per-dequeue overhead the way the caps plan pins - the fairQueue snapshot cost. - -Verify: `cd internal-packages/run-engine && pnpm run test ./src/run-queue/tests/ckVtimeConcurrency.test.ts --run`. - -### Task 8: default-off regression proof - -Location: `tests/ckVtime.test.ts` (final describe block). - -18. "flag off creates no vtime keys and matches today's order": with - `ckVirtualTimeScheduling` absent, run a mixed sequence (enqueues across - 3 keys with distinct head ages, batched dequeues, one nack, acks), then: - `KEYS *` contains no key matching `*ckVtime*`; the dequeue order equals - the head-timestamp order (re-assert the core expectation of - `ckIndex.test.ts` inside this sequence). The stronger guarantee (same - script text, same SHA) holds by construction: the off path calls the - same command names whose `defineCommand` strings this plan never edits; - say so in a comment rather than pretending a test can diff against an - old build. -19. Run the whole existing run-queue suite with the code in place and flag - off: `cd internal-packages/run-engine && pnpm run test ./src/run-queue/ --run` - (excluding the `fairness-spike*` dirs if they are still present). All - green. - -### Task 9: engine and webapp wiring (code-dark rollout) - -Files: `engine/types.ts`, `engine/index.ts`, `apps/webapp/app/env.server.ts`, -`apps/webapp/app/v3/runEngine.server.ts`. - -- `engine/types.ts`, inside `queue:`: - `ckVirtualTimeScheduling?: RunQueueOptions["ckVirtualTimeScheduling"];` -- `engine/index.ts` (~line 196): pass - `ckVirtualTimeScheduling: options.queue?.ckVirtualTimeScheduling,`. -- `env.server.ts`: - `RUN_ENGINE_CK_VTIME_SCHEDULING_ENABLED: z.string().default("0")`, - `RUN_ENGINE_CK_VTIME_QUANTUM: z.coerce.number().default(1)`, - `RUN_ENGINE_CK_VTIME_WINDOW_MULTIPLIER: z.coerce.number().default(3)`, - `RUN_ENGINE_CK_VTIME_STATE_TTL_SECONDS: z.coerce.number().default(86400)` - (match the file's existing patterns for flag-style vars). -- `runEngine.server.ts` `queue:` block: - -```ts -ckVirtualTimeScheduling: - env.RUN_ENGINE_CK_VTIME_SCHEDULING_ENABLED === "1" - ? { - enabled: true, - quantum: env.RUN_ENGINE_CK_VTIME_QUANTUM, - scanWindowMultiplier: env.RUN_ENGINE_CK_VTIME_WINDOW_MULTIPLIER, - stateTtlSeconds: env.RUN_ENGINE_CK_VTIME_STATE_TTL_SECONDS, - } - : undefined, -``` - -Mixed-deploy analysis to record in the PR description (the analogue of the -caps plan's mixed-arity warning): old and new instances coexist safely -because ioredis registers scripts per process, so arity never mixes within a -script call; old-instance enqueues skip registration and old-instance -dequeues neither advance tags nor GC `ckVtime`. Consequences during overlap: -unregistered variants are served via pass 2 (age order, today's behaviour) -and get registered on serve; keys served by old instances gain a temporary -priority bias (tags lag), which the floor bounds and which disappears when -the rollout completes. No state leaks: `ckVtime` entries created during a -rollout that is then rolled BACK are ignored by the old script entirely and -expire via the state TTL. Turning the flag OFF after running ON is the same: -stale vtime keys are inert and expire within `stateTtlSeconds`. - -Verify: `pnpm run typecheck --filter @internal/run-engine` and -`pnpm run typecheck --filter webapp`. - -### Task 10: ship notes and cleanup - -- Add `.server-changes/2026-07-24-ck-fair-scheduling.md` per - `.server-changes/README.md` (user-facing wording; it ships dark, so the - note says the fair ordering exists behind a flag and changes nothing by - default). No changeset (no public package touched). -- `pnpm run format && pnpm run lint:fix` before committing. -- The `fairness-spike/` and `fairness-spike-ck/` directories say "delete - before any merge to main" in their own headers. Deleting them is a - separate commit/decision, not part of this implementation branch; do not - import from them (already a global constraint). - -## Rollout sequence (after merge) - -1. Deploy with the flag off (nothing changes; scripts for the new commands - are registered but never called). -2. Enable on a staging/test cell; watch dequeue latency spans and Redis op - rates against the Task-7 budget; run a manual sybil-shaped workload and - confirm the light key's wait. -3. Enable in production. During the instance-rolling window the behaviour - interpolates between age order and fair order per the mixed-deploy - analysis; both endpoints are safe. -4. Rollback at any point = flip the env var off; stale vtime keys expire via - TTL within 24h. - -## Open design decisions (flagged, with recommended defaults) - -1. Fast-path enqueue does not advance or register virtual time. - Recommended: keep it that way. The fast path fires only when the variant - queue is empty AND env and queue capacity are free, i.e. when there is no - contention, and fairness only exists under contention. Charging fast-path - serves would need vtime keys touched on the hot uncontended path for no - measurable benefit. Revisit only if a workload alternates fast-path and - queued serves on the same keys at saturation boundaries (the Task-6 - ckBalanced no-harm test would catch a regression shape here). -2. Stored tag semantics and quantum. Recommended: store the NEXT start tag - (start of last serve + quantum), quantum 1.0, matching `SfqCk` in - `disciplines.ts` exactly, since that is the vetted logic both spikes - measured. A cost-proportional quantum (e.g. by machine size) is possible - later via the same field. -3. Pass-1 window multiplier. Recommended: default 3 (today's), configurable. - The residual reachability limit (more than `window` at-cap or - future-scheduled low-tag variants hiding an eligible one) is the same - class as today's `*3` limit and pass 2 keeps the call work-conserving; - widening by default would raise per-call cost for a case not yet observed. -4. Registration sites. Recommended: enqueue and nack only, with pass 2 as - the safety net, per the closure argument (only enqueue and nack make a - variant queue non-empty). Adding registration to the sweeper/ack Luas - would touch more scripts for no covered transition. -5. State TTL default. Recommended: 86400s, matching `counterTtlSeconds`'s - precedent and rationale (periodic re-anchor bounds any drift, including - drift from rolling-deploy overlap). -6. Command variants vs ARGV-gated single script. Recommended: separate - `...Vtime...` commands. Byte-identity when off then holds by construction - instead of by test. -7. Equal weights. Recommended: yes, with the `quantum / weight` hook left in - place (weight fixed at 1, named local, comment pointing at a future - sparse `:ckWeights` HASH shaped like Phase-2's `:ckLimits`). No product - surface for weights exists today. -8. Discipline. Recommended: SFQ (stride is arithmetically the same thing - here; DRR would need a ring cursor in Redis and buys nothing per the - spike, where DRR and SFQ tracked each other within noise). Keep DRR as - the documented O(1) fallback if ZSET ops on `ckVtime` ever show up in - profiles, which the Task-7 op budget makes visible. - -## Verification summary - -```bash -pnpm run typecheck --filter @internal/run-engine -pnpm run typecheck --filter webapp -cd internal-packages/run-engine -pnpm run test ./src/run-queue/tests/keyProducer.test.ts --run -pnpm run test ./src/run-queue/tests/ckVtime.test.ts --run -pnpm run test ./src/run-queue/tests/ckVtimeFairness.test.ts --run -pnpm run test ./src/run-queue/tests/ckVtimeConcurrency.test.ts --run -pnpm run test ./src/run-queue/ --run # full regression, flag off default -``` diff --git a/internal-packages/run-engine/design/references/README.md b/internal-packages/run-engine/design/references/README.md deleted file mode 100644 index 2cd163408..000000000 --- a/internal-packages/run-engine/design/references/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Run-queue multi-tenant fairness: spike references - -Reference material for the implementation plan -`internal-packages/run-engine/design/plans/2026-07-23-ck-virtual-time-scheduling-plan.md`. These are -the findings and the queueing-theory research produced by three throwaway spikes -on RunQueue tenant fairness (#2617). The spikes' harness, bench, and results code -was throwaway and is NOT on this branch; it is archived on the remote branch -`chore/fair-queueing-spike` (never merged, delete-before-anything). Any -`internal-packages/.../fairness-spike*` paths mentioned inside these documents -refer to that archived code. - -## The documents - -- `run-queue-fairness-research.md` — queueing-theory grounding: SFQ/WFQ and DRR - delay bounds, the Parekh-Gallager result that a worst-case per-flow delay bound - needs BOTH an admission regulator and a scheduler, why CoDel is an AQM and not a - fairness scheduler, and how production systems (Kubernetes APF, YARN, SQL Server - Resource Governor, SQS fair queues) layer caps under a fair order. -- `run-queue-fairness-base-queue-findings.md` — spike 1, base-queue grain: ranked - SFQ / stride / DRR / CoDel against the production age-order baseline. SFQ and - stride fix starvation and are seed-stable; CoDel is a no-op on a fair base and - harmful on an unfair one. -- `run-queue-fairness-ck-findings.md` — spike 2, the real concurrency-key seam: - drove the production `dequeueMessagesFromCkQueueTracked` Lua via `ckIndex` - rescoring. Per-key fairness lives below the selection-strategy interface, in the - CK dequeue scoring; virtual-time ordering fixes it there. Documents the - `maxCount = 1` fidelity limit that the implementation plan's tests must close. -- `run-queue-fairness-caps-vs-scheduling-findings.md` — spike 3, the - reconciliation with the plan of record (which ships concurrency caps): caps and - scheduling are orthogonal knobs. A per-key cap fixes wait when one key floods - but gives no relief once a tenant shards across many keys (the sybil split), and - it is not work-conserving; a total cap is a cross-task knob, not a cross-key - one; fair scheduling fixes every case and stays work-conserving. Ship the caps - first, add the fair order as the general fix, layer them. - -## Why the plan follows from these - -The recommended fix (score `ckIndex` by SFQ virtual time, inside the batched CK -dequeue, layered under the caps) is the one mechanism the spikes found that -survives key-sharding and stays work-conserving, and the research says the caps -the plan of record ships cannot bound wait on their own. The plan turns that into -a flag-gated, mixed-deploy-safe engine change with a test suite that exercises the -real batched path the spikes could not. diff --git a/internal-packages/run-engine/design/references/diagrams/fairness-how-it-works.png b/internal-packages/run-engine/design/references/diagrams/fairness-how-it-works.png deleted file mode 100644 index 98e33bb88..000000000 Binary files a/internal-packages/run-engine/design/references/diagrams/fairness-how-it-works.png and /dev/null differ diff --git a/internal-packages/run-engine/design/references/diagrams/fairness-problem-and-fix.png b/internal-packages/run-engine/design/references/diagrams/fairness-problem-and-fix.png deleted file mode 100644 index 07027b76c..000000000 Binary files a/internal-packages/run-engine/design/references/diagrams/fairness-problem-and-fix.png and /dev/null differ diff --git a/internal-packages/run-engine/design/references/run-queue-fairness-base-queue-findings.md b/internal-packages/run-engine/design/references/run-queue-fairness-base-queue-findings.md deleted file mode 100644 index c360f7832..000000000 --- a/internal-packages/run-engine/design/references/run-queue-fairness-base-queue-findings.md +++ /dev/null @@ -1,168 +0,0 @@ -# Fair-queueing spike: findings - -Findings from a throwaway spike whose harness is archived and ships nothing; these -findings are retained here as a design reference that informed the change, not as code. - -Read the caveats section before quoting any single number. Two of them matter up -front: (1) the candidate selectors are handed tenant identity (parsed from the -queue name) and the exact per-tenant weights, and the fairness target is defined -by those same groups and weights, so the candidate win over the tenant-blind -baseline is closer to definitional than discovered. (2) The harness anchors -message scores far in the past, which flattens all queue ages, so the baseline's -production age bias is not exercised here; the baseline measured is closer to a -uniform-random shuffle than the real one. - -Bottom line: at the base-queue grain, virtual-time ordering (SFQ) and stride give -tight, seed-stable proportional fairness, honour weights, and cut a starved -tenant's wait hard. DRR lands within noise of them (its small shortfall is a -measurement artifact of the harness, not an intrinsic property). The baseline is -fair on average but seed-variant and has no weight concept. The CoDel wrapper is -not worth shipping as built: it is a forced no-op under bulk arrival and it -actively hurt fairness on the one trickle-arrival workload that could exercise it. -The biggest single result is architectural: per-concurrency-key fairness (the -actual #2617 grain) cannot be expressed through the `RunQueueSelectionStrategy` -interface at all; it lives below that interface, in the CK-dequeue Lua. - -Every number comes from the real `RunQueue` against a testcontainers Redis, one -selector per run, real enqueue/dequeue/ack and real concurrency gating. Each -scenario runs over 3 seeds; tables show the mean and min..max spread. Per-tenant -detail (first seed) is in `results/*.json`. - -## Grain, and why it is not the concurrency key - -A tenant is the fairness group; a tenant owns one or more base queues. The -adversarial scenario gives one tenant 30 queues and the light tenants one each, -which is how the #2617 starvation shows up at the base-queue grain: an ordering -blind to tenant identity lets the many-queue tenant win most of the selection -chances. - -The concurrency-key grain #2617 asks for is not reachable through the strategy -interface. `FairQueueSelectionStrategy` reads the master-queue members verbatim, -and CK runs enqueue a single CK-wildcard entry per base queue. The per-CK pick -runs later inside `dequeueMessagesFromCkQueueTracked`, where `ckIndexKey` is a -ZSET of CK-queues scored by head timestamp and the Lua serves them oldest-first. -That age ordering is the unfairness. Fixing it means changing that Lua or the -`ckIndex` scoring, not the selection strategy. That is the follow-on spike. - -## How fairness is measured (and its limits) - -Because the sim drains every run, final throughput share is fixed by the workload -and cannot tell selectors apart. Two measures do: - -- contention share: a tenant's share of dequeues at instants when at least two - tenants have arrived, unserved work, over its expected weighted share. - `contWorstS/W` is the least-served contender; 1.0 is fair, near 0 means starved - while others had work. Getting this right took two corrections a review caught: - it must only count a tenant once its runs have actually arrived (else poisson - arrival looks like starvation), and the virtual-time floor must be monotonic - (else a returning idle tenant monopolises and skews the window). Even so, when - tenants have very different volumes (trickleStale: 30 runs vs 300) the - low-volume tenant can legitimately be over- or under-represented in the window, - so read this metric together with wait, not alone. -- wait: dequeue time minus enqueue time, per tenant, in the JSON. This is the - clean anti-staleness signal. (Note: `worstWaitP99` in the JSON is NOT an - anti-staleness win signal; it is dominated by the highest-volume tenant, which - a fair selector deliberately delays, so a fairer selector scores worse on it. - Use per-tenant wait.) - -## Results - -`contWorstS/W` mean over 3 seeds (min..max). Higher is fairer. - -| scenario | baseline | sfq | drr | stride | codel-sfq | codel-baseline | -| --------------- | ------------------- | ----- | ------------------- | ------ | --------- | -------------- | -| balanced | 0.889 (0.774..0.954)| 0.985 | 0.954 (0.923..0.970)| 0.985 | 0.985 | 0.889 | -| adversarialSkew | 0.288 (0.261..0.310)| 1.000 | 0.978 (0.968..0.984)| 1.000 | 1.000 | 0.288 | -| weighted | 0.703 (0.679..0.719)| 1.000 | 0.990 (0.977..1.000)| 1.000 | 1.000 | 0.703 | -| burst | 0.978 (0.966..0.992)| 0.992 | 0.958 (0.941..0.975)| 0.992 | 0.992 | 0.978 | -| longHold | 0.828 (0.800..0.842)| 0.981 | 0.981 | 0.981 | 0.981 | 0.828 | -| trickleStale | 0.208 (0.179..0.235)| 0.804 (0.769..0.826)| 0.776 (0.769..0.783)| 0.804 | 0.366 | 0.195 | - -Per-tenant mean wait (seed-a, logical ms), the anti-staleness signal: - -| scenario / selector | low-volume tenant wait | heavy tenant wait | -| ------------------------ | ---------------------- | ----------------- | -| adversarialSkew baseline | 1380 | 805 | -| adversarialSkew sfq | 319 | 1324 | -| trickleStale baseline | 1359 | 1234 | -| trickleStale sfq | 19 | 1353 | -| trickleStale codel-sfq | 213 | 1315 | - -Reading these: the fair selectors cut the light tenant's wait (skew 1380 to 319, -trickle 1359 to 19) by making the heavy tenant wait its fair turn. The heavy -tenant is not punished, it stops jumping the queue. CoDel undoes part of the -trickle win (19 back up to 213). - -## Verdict per mechanism - -- SFQ (start-time virtual time, the start-tag form of WFQ): the strongest result. - Perfect contention fairness under skew and weighting, seed-stable (zero variance - across seeds), and the largest cut to the starved tenant's wait. The floor is - now monotonic (a review found the earlier version let a returning idle tenant - monopolise; fixed). Recommended as the leaf ordering. -- Stride: identical to SFQ to the decimal on every scenario. The spike does not - separate them. Stride carries slightly less state. -- DRR: within noise of SFQ. It trails by a couple of points on balanced (0.954) - and burst (0.958) and matches SFQ elsewhere. That small shortfall is a - measurement artifact, not an intrinsic property: the driver drains a whole - capacity batch from a single strategy snapshot and only advances DRR's deficit - after the batch (via `onServiced`), so DRR's current-winner group, whose queues - it fronts together, grabs several slots before its deficit updates and its - deficit runs negative. Served one-at-a-time DRR is exactly fair (see - `drr.test.ts`). Note the earlier claim that "virtual-time sorts an over-served - group's queues to the back and so avoids this" was wrong: at a tie all of a - group's queues share one clock, so SFQ fronts them together too; the schemes - only separate after their state advances. DRR is O(1) and composes weight - trivially, so it is a fine choice if per-op cost matters, subject to that - caveat. -- CoDel wrapper: do not ship as built. Under bulk arrival it is a forced no-op: - all of a queue's runs share one enqueue timestamp, so every tenant's sojourn is - identical and they all cross the target together, so hoisting everyone collapses - to the base order (this is why codel-sfq equals sfq and codel-baseline equals - baseline to the decimal on those scenarios; it is one workload shape confirming - a null result, not five independent tests). On trickleStale, the one scenario - where sojourns diverge, the sojourn-hoist overshoots: it drops SFQ from 0.804 to - 0.366 and pushes the trickle tenant's wait from 19 back to 213. A staleness - monitor may still help on top of an unfair base or behind a hard concurrency - wall, but that needs a different construction and this spike does not support it. -- Baseline (`FairQueueSelectionStrategy`): fair on average on the easy scenarios - but seed-variant (balanced 0.774..0.954), no weight concept (weighted 0.703), - and it starves a light tenant under queue-count skew (0.288) and under trickle - arrival (0.208, trickle wait 1359). Remember its age bias is not exercised here - (see caveats), so this is a floor on its unfairness, not the production picture. - -## Caveats - -- Grain is base queues, not concurrency keys. The disciplines are grain-agnostic - so the ranking should carry over, but the #2617 gap itself needs the CK-Lua - spike. adversarialSkew is a proxy for that gap, not a measurement of it. -- Definitional advantage: candidates get tenant identity and exact weights the - real interface does not carry; the baseline structurally cannot. -- Baseline age bias inert: scores are anchored ~600s in the past, so all queue - ages are near-equal and the baseline degenerates to near-uniform selection. The - production age bias (which would give a heavy tenant's older heads more weight, - i.e. make skew worse) is not measured. -- Selection-only seam: the driver feeds serviced descriptors back via an - `onServiced` hook; production would advance selector state inside the ack/dequeue - Lua. The spike proves ordering logic, not that wiring. -- Cost was not rigorously measured. `selectionRounds` is roughly equal across - selectors (646..729) but is not comparable between them (a candidate reads all - queues per call; the baseline short-circuits at capacity), and there is no load - benchmark. DRR's "O(1)" advantage is a theory claim, not a spike measurement. -- Scenario quality varies. balanced best shows the baseline's variance; - adversarialSkew and weighted carry the clear separation; longHold and burst - barely separate the candidates; trickleStale's contention number only became - meaningful after two metric fixes and should be read with wait. Per-tenant p99 - equals max for the small (20 to 30 run) tenants, so "p99" there is just the max. -- Single Redis shard; single sequential consumer (not the multi-consumer, - Redis-hash-state design the spec sketched); simulated holds on a logical clock. - Three seeds shows the baseline's variance and the virtual-time schemes' - stability but is not a statistical study. - -## Recommended direction - -Use virtual-time (SFQ, or stride) for leaf ordering and compose weight with it. -DRR is an acceptable O(1) fallback given the batch caveat. Do not adopt the CoDel -wrapper as built. Then run the follow-on spike against the CK-dequeue Lua / -`ckIndex` scoring, because that is where per-tenant fairness actually has to land -in the current design. diff --git a/internal-packages/run-engine/design/references/run-queue-fairness-caps-vs-scheduling-findings.md b/internal-packages/run-engine/design/references/run-queue-fairness-caps-vs-scheduling-findings.md deleted file mode 100644 index f89257709..000000000 --- a/internal-packages/run-engine/design/references/run-queue-fairness-caps-vs-scheduling-findings.md +++ /dev/null @@ -1,198 +0,0 @@ -# Caps vs scheduling: reconciliation findings - -Findings from a throwaway spike whose harness is archived (`chore/fair-queueing-spike`) and ships nothing; these findings are retained here as a design reference. Relative ranking -on a small simulation, not a statistical or load study: read the verdicts as "what -this harness supports", not proofs. Went through a blind two-model adversarial -review (a third stalled); the review's fixes are folded in below. - -Bottom line: the plan-of-record's concurrency CAPS and the earlier spike's fair -SCHEDULING are different knobs, and the data on the real CK-dequeue Lua lines up -with the queueing theory (see `RESEARCH.md`). A per-key cap cuts a starved key's -wait when ONE key floods, because Trigger's CK dequeue is oldest-eligible-first; -it gives no wait improvement once a tenant shards its backlog across many -concurrency keys, and it is not work-conserving. Fair scheduling (SFQ/DRR) is the -only knob here that improves the starved key on every scenario including the -sharded one, and it stays work-conserving. A total (per-task) cap is a -cross-task knob; inside one task it only lowers the ceiling and is not a fairness -lever at all. The mechanisms are complementary, and production systems that need -fairness under saturation layer them (Kubernetes APF: seats + fair queueing). - -## How the mechanisms were modelled (fidelity) - -- Per-key cap (Phase 2): the REAL Lua gate. `updateQueueConcurrencyLimits` sets - the base queue's concurrencyLimit; the CK-dequeue Lua caps each ck variant's - in-flight at it and skips an at-limit variant (oldest-eligible-first, true age - order, no rescore). Uniform across variants: Phase 2's per-key HGET override - would cap only the heavy key, but a light key never approaches the cap so the - effect is equivalent here. (So "just lower the existing per-queue concurrency - limit" already IS a per-key cap; Phase 2 makes it per-key-specific.) -- Total cap (Phase 1): driver-side. The real Lua has no group gate yet, so the - driver refuses to admit while total in-flight across all variants of the base - queue (= `:groupConcurrency` SCARD in one base queue) is at the cap. -- Ordering disciplines (baseline age order, SFQ, DRR) are unchanged from the CK - scheduling spike, driven through the same real Lua at `maxCount = 1`. -- Two fidelity limits matter for reading the numbers, both driver-independent: - - `maxCount = 1` (same as the CK spike): production dequeues in batches, so a - real per-key/total gate lives inside the batched Lua. - - The `*3` scan window: the real CK Lua reads `ZRANGEBYSCORE ckIndexKey -inf now - LIMIT 0, actualMaxCount*3` (`index.ts:4041/4193`). At `maxCount = 1` that is - the 3 oldest-scored variants per call. So a per-key cap frees the light key - only when the light head lands inside that 3-wide window after the at-cap - variants ahead of it. With one heavy key it does; with many old-headed - attacker variants it never does. This window governs the skew-works / - sharded-fails split, and it scales with `maxCount` in production (batches), - not fixed at 3, so the sharded result's exact severity would differ on the - real batched path (direction not established). - -## Results - -env=4, per-key cap=2, total cap=2, 3 seeds. Columns: `lightWait` = the starved -key's mean wait (logical ms, the headline where it is not confounded); -`worstWait` = the largest per-group mean wait (i.e. the busiest key, which a fair -discipline deliberately makes wait its turn, so higher here is often correct); -`makespan` = logical time of the last dequeue (a work-conservation signal ONLY on -ckHeavyIdle, arrival-confounded elsewhere); `contWorstS/W` = worst contention -share over weight (directional; volume-confounded and, for per-key cap on sybil, -seed-noisy). - -| scenario | treatment | lightWait | worstWait | makespan | contWorstS/W | -| ----------- | ------------------ | --------- | --------- | -------- | ------------ | -| ckSkew | baseline | 1098 | 1261 | 2083 | 0.187 | -| ckSkew | perKeyCap | 20 | 1555 | 3038 | 0.814 | -| ckSkew | totalCap | 2840 | 2974 | 3947 | 0.213 | -| ckSkew | total+perKey | 2840 | 2974 | 3947 | 0.213 | -| ckSkew | sfq | 14 | 1069 | 2083 | 0.723 | -| ckSkew | drr | 17 | 1067 | 2083 | 0.608 | -| ckSkew | perKeyCap+sfq | 7 | 1628 | 3114 | 0.800 | -| ckSkew | total+perKey+sfq | 52 | 2363 | 3939 | 0.803 | -| ckTrickle | baseline | 1107 | 1134 | 1940 | 0.279 | -| ckTrickle | perKeyCap | 19 | 1555 | 3038 | 0.922 | -| ckTrickle | totalCap | 2852 | 2861 | 3905 | 0.279 | -| ckTrickle | total+perKey | 2852 | 2861 | 3905 | 0.279 | -| ckTrickle | sfq | 17 | 1070 | 1942 | 0.909 | -| ckTrickle | drr | 23 | 1069 | 1941 | 0.790 | -| ckTrickle | perKeyCap+sfq | 8 | 1606 | 3097 | 0.658 | -| ckTrickle | total+perKey+sfq | 106 | 2337 | 3911 | 0.883 | -| ckSybil | baseline | 1765 | 1876 | 2070 | 0.000 | -| ckSybil | perKeyCap | 1776 | 1869 | 2128 | 0.403 | -| ckSybil | totalCap | 3793 | 3801 | 4147 | 0.000 | -| ckSybil | total+perKey | 3793 | 3801 | 4147 | 0.000 | -| ckSybil | sfq | 1009 | 1019 | 2065 | 1.000 | -| ckSybil | drr | 1061 | 1068 | 2055 | 0.994 | -| ckSybil | perKeyCap+sfq | 1010 | 1019 | 2072 | 1.000 | -| ckSybil | total+perKey+sfq | 2292 | 2292 | 4146 | 1.000 | -| ckHeavyIdle | baseline | 633 | 633 | 1240 | 1.000 | -| ckHeavyIdle | perKeyCap | 1291 | 1291 | 2507 | 1.000 | -| ckHeavyIdle | totalCap | 1291 | 1291 | 2507 | 1.000 | -| ckHeavyIdle | total+perKey | 1291 | 1291 | 2507 | 1.000 | -| ckHeavyIdle | sfq | 633 | 633 | 1240 | 1.000 | -| ckHeavyIdle | drr | 633 | 633 | 1240 | 1.000 | -| ckHeavyIdle | perKeyCap+sfq | 1291 | 1291 | 2507 | 1.000 | -| ckHeavyIdle | total+perKey+sfq | 1291 | 1291 | 2507 | 1.000 | - -(ckHeavyIdle is a single key, so "lightWait" is the heavy key's own wait and the -contention metric is degenerate at 1.0; makespan is the signal there. ckSybil is -20 attacker keys plus one light key.) - -## What each mechanism does, at the cross-key grain - -- Per-key cap (Phase 2). SUPPORTED for the single-heavy case; NOT a wait fix once - the tenant shards; not work-conserving. - - Single heavy key (ckSkew/ckTrickle): cuts the light key's wait like a - scheduler (1098 to 20, 1107 to 19) because capping the one heavy key frees - slots and the CK Lua serves the light head as the next eligible one. This - depends on the light head being reachable inside the Lua's 3-wide scan window; - with one at-cap variant ahead of it, it is. - - Sharded / sybil (ckSybil, 20 attacker keys): NO wait improvement (baseline - 1765, perKeyCap 1776; the difference is within the per-seed spread, baseline - 1681..1926, perKeyCap 1672..1861). Contention share nudges up (0.000 to a - mean 0.403) but that mean hides a ~10x seed swing (0.07..0.71), so it is not a - dependable improvement. The reason is structural: the sum of per-key caps is - unbounded relative to the queue when keys are client-chosen, and the 3-wide - scan window is always full of older attacker heads, so the light head is never - reached. Concurrency keys are client-chosen, so this is cheap to trigger. - - Not work-conserving: throttles the capped key even with the env idle - (ckHeavyIdle makespan 1240 to 2507, 2x, the cleanest single result in the - spike; ckSkew 2083 to 3038, though that scenario's makespan is partly arrival- - confounded). -- Total cap (Phase 1) at the cross-key grain: NOT a fairness lever, and the - in-task comparison is capacity-confounded. `totalCap=2` caps the whole task's - aggregate at half of env=4, so it simply halves throughput: light's wait rises - (ckSkew 1098 to 2840) for the same reason heavy's does (both now share half the - server), which is Little's-Law throughput loss, not a fairness effect. It is the - wrong knob for cross-key starvation, measured on a lower ceiling; do not read - the "worse" numbers as "total caps harm fairness." -- Total cap (Phase 1) at the cross-TASK grain: this IS its job, and it works. - Measured in a separate multi-base-queue bench (`crossTaskCaps.bench.test.ts`): - two keyless tasks share one env, a heavy task floods it, and capping the heavy - task (its per-queue concurrency limit, the real native gate, which for a keyless - task equals its total cap) cuts the light TASK's wait from 475 to 2 under the - production `FairQueueSelectionStrategy`. So the total cap protects a light task - from a heavy task, the reservation-isolation role the research describes. It is - still not work-conserving (makespan 2039 to 3039), and SFQ at the task grain - protects the light task too (wait 14) while staying work-conserving (2039). The - fidelity note: this models a KEYLESS task, so the per-queue limit is the total; - a task WITH concurrency keys needs the group SET to sum across variants (the - unbuilt Phase-1 gate). -- Combined total + per-key (the shipped Phase-1+2 config): in this toy the total - cap (2) is below a single per-key cap's reach, so it dominates and the per-key - cap is non-binding (`total+perKey` equals `totalCap` to the digit). This toy - therefore does not exercise the combined config's real regime (total >> per-key, - cross-task). What it does show: adding a fair order on top (`total+perKey+sfq`) - restores fair share within the throttled aggregate (contWorstS/W 0.80..1.0) but - still pays the total cap's throughput loss (makespan ~3900+). -- Scheduling (SFQ/DRR): the only knob that improves the starved key on every - scenario, and work-conserving (makespan stays at the baseline optimum - 2083/1240). On the sharded case SFQ takes the light key from fully starved to - its full fair share (contWorstS/W 0.000 to 1.000, seed-stable) and roughly - halves its wait (1765 to 1009); the residual wait is real saturation shared - fairly across 21 keys, not starvation. SFQ and DRR track each other within - noise, as in the CK spike. -- Layered per-key cap + SFQ: best light-key wait on the single-heavy case (7, 8) - and it carries the cap's occupancy bound, at the cap's makespan cost (matches or - slightly exceeds perKeyCap makespan: ckSkew 3038 to 3114). On the sharded case - the cap adds nothing and SFQ does all the work (1010, same as SFQ alone). This - is the Kubernetes-APF shape; APF avoids the work-conservation cost by making the - cap ELASTIC (borrow/lend seats), which a static cap cannot. - -## Reconciliation with the earlier spike and the plan of record - -Measured here: caps and scheduling fix different things and can be layered -(the per-key-cap+SFQ and total+perKey+sfq rows). Fair scheduling is the only -mechanism in this harness that improves the starved key on the sharded case and -stays work-conserving, which is what the earlier spike recommended (score -`ckIndex` by virtual time). - -Interpretation, NOT measured by this benchmark (it measures wait/makespan/share on -a simulation, not engineering cost or rollout risk): shipping the caps first still -reads as defensible. A per-key cap is bounded, operator-controlled, self-healing -(a Redis SET), and it fully fixes the common single-heavy-key case, which is a -smaller engine change than reworking the dequeue scoring. Its limits are real -(no help once a tenant shards its keys, not work-conserving), which is the case -for treating automatic fair scheduling as a later phase rather than never. - -The layered end state matches Kubernetes APF, SQL Server Resource Governor, YARN, -and the Parekh-Gallager result that a worst-case delay bound needs BOTH an -admission regulator AND a scheduler: keep the caps for isolation and entitlements, -add a fair dequeue order for the contended region when saturation and key-sharding -make caps alone insufficient. Not either/or. - -## Caveats - -- Relative ranking on a simulation; single shard, single base queue, single - sequential consumer; simulated holds on a logical clock; 3 seeds; equal weights. - The verdict words ("supported", "not a wait fix") are relative to this harness. -- `maxCount = 1` and the `*3` scan window (see fidelity section); the total cap is - driver-modelled, not the real (unbuilt) group gate. -- Per-key cap is modelled uniformly (real per-queue gate); a per-key-specific - Phase-2 override is equivalent here only because the light key never approaches - the cap. -- makespan is the last dequeue, not completion, and is arrival-confounded on the - poisson scenarios; trust it only on ckHeavyIdle. -- Contention share is volume-confounded for low-volume keys, and for the per-key - cap on the sharded case it is seed-noisy (0.07..0.71); wait is the trustworthy - signal, share is directional. -- Cross-task isolation (the total cap's real purpose) is now measured in - `crossTaskCaps.bench.test.ts` for KEYLESS tasks (per-queue limit = total cap). - A task with concurrency keys needs the unbuilt group-SET gate to sum across - variants; that batched, keyed path is still not exercised. diff --git a/internal-packages/run-engine/design/references/run-queue-fairness-ck-findings.md b/internal-packages/run-engine/design/references/run-queue-fairness-ck-findings.md deleted file mode 100644 index 04f615c90..000000000 --- a/internal-packages/run-engine/design/references/run-queue-fairness-ck-findings.md +++ /dev/null @@ -1,122 +0,0 @@ -# Per-concurrency-key fairness spike: findings - -Findings from a throwaway spike whose harness is archived (`chore/fair-queueing-spike`) and ships nothing; these findings are retained here as a design reference. - -Bottom line: the base-queue spike's direction carries over to the real seam. At -the concurrency-key grain the production baseline (serve the oldest-head CK -first) starves keys that arrive behind a big backlog, and virtual-time (SFQ) and -stride fix it: they cut the starved key's wait from ~1300ms to ~20ms by making -the backlog key wait its turn. DRR does the same. A CoDel wrapper on the baseline -makes it worse. This was measured by driving the real -`dequeueMessagesFromCkQueueTracked` Lua and only rewriting `ckIndex` scores to -express each discipline. That is enough to say the ordering fix is worth a design -spike, but NOT that a production implementation is proven (see the fidelity -caveat: the spike serves one key per Lua call, and production dequeues in -batches). - -## What was driven, and the two things to know before reading numbers - -Runs enqueue across many concurrency keys under one base queue via the real -`RunQueue`. The per-CK pick is `ZRANGEBYSCORE ckIndexKey -inf now` in the CK Lua -(lowest score first, score = head timestamp). Candidates rewrite those scores -each round to encode discipline order; the baseline leaves them (production age -order). Enqueue, dequeue, concurrency gating and ack all run through the real -code. - -Two caveats a review forced, both load-bearing: - -1. Lead with wait, not contention share. The contention-share metric is - volume-confounded for low-volume keys (a key with 15 runs cannot take a third - of a long window even when served instantly), so on these scenarios it lands - around 0.7 to 0.9 for a discipline that has in fact eliminated the starvation. - The per-key wait is the clean signal. -2. The scenarios must give keys genuinely different head ages. An earlier version - enqueued every run at one timestamp; with tied `ckIndex` scores the real Lua - falls back to a lexicographic member-name tie-break, so the "baseline starves - the heavy key's rivals" result was actually "Redis sorts by name" and the - heavy key only won because "heavy" sorts before "light". Fixed: the backlog - key fires at once (persistently old head) and the other keys arrive via - poisson (distinct, later heads), so the baseline now exercises real age order. - -## Results - -`contWorstS/W` mean over 3 seeds (min..max), and the worst-served key's mean wait -(seed-a, logical ms). Read the wait column as the headline. - -| scenario | discipline | contWorstS/W | worst-key wait | backlog-key wait | -| ---------- | ---------- | ------------------- | -------------- | ---------------- | -| ckSkew | baseline | 0.187 (0.186..0.188)| 1321 | 872 | -| ckSkew | sfq | 0.723 (0.655..0.769)| 16 | 1150 | -| ckSkew | drr | 0.608 (0.556..0.648)| 21 | 1149 | -| ckTrickle | baseline | 0.279 (0.254..0.291)| 1339 | 872 | -| ckTrickle | sfq | 0.909 (0.891..0.918)| 24 | 1237 | -| ckTrickle | drr | 0.790 (0.769..0.818)| 30 | 1236 | - -Full matrix (contWorstS/W mean over 3 seeds): - -| scenario | baseline | sfq | drr | stride | codel(sfq) | codel(baseline) | -| ---------- | ------------------- | ------------------- | ------------------- | ------ | ---------- | ------------------- | -| ckSkew | 0.187 | 0.723 | 0.608 | 0.723 | 0.723 | 0.104 (0.000..0.157)| -| ckBalanced | 0.515 (0.444..0.600)| 0.611 (0.462..0.800)| 0.730 (0.615..0.909)| 0.611 | 0.611 | 0.464 | -| ckTrickle | 0.279 | 0.909 | 0.790 | 0.909 | 0.909 | 0.018 (0.000..0.055)| - -## Verdict per discipline (at the concurrency-key grain) - -- SFQ / stride: fix the starvation. Contention share improves (skew 0.187 to - 0.723, trickle 0.279 to 0.909) and the starved key's wait collapses (skew 1321 - to 16, trickle 1339 to 24) because the backlog key now waits its turn (its wait - rises 872 to ~1150 to 1237). Identical to each other on every scenario. - Recommended discipline for the fix. -- DRR: fixes the wait just as well (skew 21, trickle 30) and its contention share - tracks SFQ within noise (sometimes a little lower, sometimes higher, e.g. - balanced 0.730 vs 0.611). Fine. -- CoDel(sfq): no harm, matches SFQ to the decimal. Adds nothing on top of a fair - base. -- CoDel(baseline): harmful. Hoisting stale keys on top of the age-order baseline - drove ckSkew to 0.104 (below baseline's 0.187) and ckTrickle to 0.018 (below - 0.279). A staleness monitor is not a substitute for a fair base. -- Baseline (production age order): starves keys that queue behind a backlog - (ckSkew 0.187, worst key waits 1321ms; ckTrickle 0.279, 1339ms). It is roughly - fair when keys are symmetric (ckBalanced 0.515, though seed-variant 0.444 to - 0.600). This is the #2617 dynamic at the seam where it lives. - -## Fidelity caveat (the reason this is not "proven for production") - -The driver dequeues one key per Lua call (`maxCount = 1`) and rescores `ckIndex` -before each call. Production dequeues in batches (`maxCount` default 10). Inside a -batched CK-dequeue call the Lua re-scores each served key back to its head -timestamp as it goes, so a once-per-round rescore would only steer the FIRST pick -of a batch; the rest would follow head-timestamp order again. So this spike -demonstrates the ordering fix only in a one-key-per-call regime, which is not how -production dequeues. A real fix has to advance per-key discipline state inside the -Lua on every serve (and hold that state in Redis, not process memory). This spike -does not exercise that batch path, so the correct claim is "the ordering fix is -worth a design spike", not "a production fix is viable". - -## Other caveats - -- Contention share is volume-confounded (see above); the wait column is the - trustworthy signal, and the contention numbers should be read as directional. -- The DRR contention-share gap is NOT the base-queue spike's batch-drain artifact - (that harness batched; this one serves one key per call and advances DRR's - deficit every serve). The cause of DRR's slightly lower share here is not - established; its wait result is as good as SFQ's. -- Per-CK concurrency gating never binds in these runs (no per-CK limit is set, so - it collapses to the env limit), so the spike says nothing about the per-CK - concurrency-limit-multiplication half of #2617, which is out of scope. -- A rescore discipline advances its floor/ring state on the final no-op drain - round of an instant (order() is called before the empty dequeue). It is - self-correcting and does not corrupt the event-based metrics, but it is a minor - infidelity to a production per-serve advance. -- Equal weights only; single shard, single base queue, single sequential - consumer; simulated holds on a logical clock; 3 seeds. - -## Recommended direction - -Score `ckIndex` by a fair discipline (SFQ/stride virtual time, or DRR) instead of -by head timestamp. Both spikes agree on the discipline and this one shows the -ordering fix works through the real dequeue path at `maxCount = 1`. The design -spike past this needs to: advance per-key virtual-time state inside the batched -CK-dequeue Lua (the `maxCount > 1` path this spike did not exercise), hold that -state in Redis for the multi-consumer case, and address the per-CK -concurrency-limit multiplication that is the other half of #2617. diff --git a/internal-packages/run-engine/design/references/run-queue-fairness-research.md b/internal-packages/run-engine/design/references/run-queue-fairness-research.md deleted file mode 100644 index afb831de5..000000000 --- a/internal-packages/run-engine/design/references/run-queue-fairness-research.md +++ /dev/null @@ -1,133 +0,0 @@ -# Queue-fairness research (grounding for the caps-vs-scheduling reconciliation) - -Research notes distilled from a throwaway spike, retained here as a design reference. Five Fable research passes, distilled. Citations kept so -the findings write-up and report can point at real sources. This grounds the -central claim: occupancy caps and fair scheduling are orthogonal knobs, and the -plan-of-record ships the cap knob while the earlier spike measured the scheduler -knob. - -## The orthogonality result (theory) - -Caps bound occupancy, not wait. A tenant capped at C in-flight with mean service -time S has long-run throughput <= C/S (Little's Law, L = lambda*W). That is an -upper bound on the capped tenant's share; it reserves no lower bound for anyone -else and says nothing about any tenant's waiting time (Little relates averages in -a stable system, not tails, and if the capped tenant's arrival rate exceeds C/S -its queue never stabilises so the law does not even apply to it). - -Scheduling bounds wait, not occupancy. WFQ/PGPS tracks GPS within one max job -(Parekh-Gallager finish-time bound L_max/r); SFQ gives a starved flow's head item -a hard wait bound of "one max-size job from every other active tenant" (Goyal-Vin -SFQ Theorem 2), with no server-rate assumption. But all of family A is -work-conserving: a lone backlogged tenant takes 100% of the server. Nothing in a -scheduler limits how many slots a tenant holds. - -Parekh-Gallager is the canonical joint statement: a worst-case per-flow delay -bound is the product of arrival regulation (leaky/token bucket = the admission -knob) AND a scheduling discipline (GPS/WFQ = the order knob). Neither alone yields -the bound. Cruz network-calculus caveat: plain FIFO does get a delay bound IF every -input is burstiness-constrained (arrival regulator on ingress) and aggregate rho < -C, but a concurrency cap is not an ingress regulator (it bounds in-flight, not -queue admission, and queue depth stays unbounded), so under adversarial arrival -FIFO wait is unbounded and the orthogonality holds without qualification. - -Sources: Little 1961 (Oper. Res. 9:383-387); Parekh & Gallager 1993/1994 (GPS, -IEEE/ACM ToN); Goyal, Vin & Cheng, Start-time Fair Queueing (SIGCOMM'96 / ToN'97); -Shreedhar & Varghese, DRR (SIGCOMM'95); Waldspurger & Weihl, Stride Scheduling -(MIT TM-528, 1995); Cruz, A Calculus for Network Delay (IEEE T-IT 1991); Kingman's -formula; Harchol-Balter, Performance Modeling and Design of Computer Systems (CUP -2013). - -## When a cap alone DOES cut a starved tenant's wait (the load-bearing condition) - -A per-tenant concurrency cap on heavy tenant H cuts light tenant L's wait to -near-zero iff BOTH: - -1. Slot availability: sum of caps of all backlogged tenants other than L is < N - (the binding aggregate limit), so freed slots exist that capped tenants can - never occupy; and -2. Eligibility-aware serve order: the dequeue selects the oldest ELIGIBLE item, - skipping items whose tenant is at cap, so L's head is reachable without - draining H's older items first. - -If (2) fails (single global age-ordered list with a head-blocking consumer), the -cap does NOT help L and with strict head-blocking makes L's wait WORSE: H's -backlog drains at k slots instead of N while the freed N-k slots sit idle -(head-of-line blocking; Parekh-Gallager's FCFS-gives-no-isolation remark). - -Trigger's CK dequeue is oldest-ELIGIBLE-first: the CK Lua gates each variant at -its per-key concurrencyLimit and skips a variant that is at its limit, moving to -the next-oldest eligible. So condition (2) holds structurally. That is WHY per-key -caps can work for wait here, and would not work on a head-blocking FIFO. - -## Where caps fail even with eligibility-aware order (the sybil split) - -Concurrency keys are client-chosen. A heavy tenant spreads its backlog across many -CK variants, each under its own per-key cap, all "eligible". The binding -constraint becomes the base-queue total cap (or env limit); oldest-first then -serves the adversary's older backlog across its many keys before a newcomer's -head. Per-key caps bound nothing in aggregate, because sum of per-key caps is -unbounded relative to the queue cap when keys are dynamic. The light tenant's wait -scales with the adversary's total queued backlog, which no per-key cap regulates. -Within a base queue, the fix under adversarial arrival is an order change -(round-robin / fair queueing across keys), not another cap. - -## Known static-cap failure modes (practice) - -2DFQ (Mace et al., SIGCOMM 2016): "Rate limiters, typically implemented as token -buckets, are not designed to provide fairness at short time intervals ... they can -either underutilize the system or concurrent bursts can overload it without -providing any further fairness guarantees." Their desirable-properties section -requires the scheduler be work-conserving, which "precludes the use of ad-hoc -throttling mechanisms to control misbehaving tenants." Pisces (OSDI 2012) and DRF -(NSDI 2011) both exist because static slot partitioning under/over-utilises under -skewed demand. Netflix concurrency-limits: static limits (Limit = RPS * latency, -i.e. Little's Law) "quickly go out of date"; hence adaptive. - -The price of caps when they DO give order-independent wait bounds: they degenerate -into a static partition (sum of caps <= K), which is non-work-conserving -(utilisation ceiling of sum-of-caps even when one tenant could use all K) and -needs bounded, pre-known tenant cardinality. - -## Production precedent: caps and scheduling are LAYERED, not either/or - -- Kubernetes API Priority & Fairness (the closest analog): total server - concurrency split into per-priority-level "seats" (a cap), THEN shuffle-sharded - fair queueing decides dispatch order within a level. Kubernetes hit exactly the - cap-alone failure with max-inflight before APF, and the fix ADDED fair queueing - on top of the existing cap rather than replacing it. (K8s docs; KEP-1040.) -- SQL Server Resource Governor: pool MIN/MAX/CAP percent (caps + reservation) + - workload-group IMPORTANCE biasing the scheduler's order. Same shape. -- YARN Fair/Capacity scheduler: minShare floor + maxResources cap + weighted - fair-share ordering, with preemption to reclaim the floor. -- Mesos/DRF, Borg: quota/admission caps layered with fair-share or priority order. -- Amazon SQS fair queues: fairness metric is DWELL TIME, fixed by reprioritising - delivery ORDER when a tenant's in-flight share is disproportionate, and - explicitly does NOT rate-limit per tenant. Order is the dwell-time knob; - occupancy limits alone were not it. -- Envoy / gRPC / Postgres connection caps: caps ALONE, and they claim overload - PROTECTION, not fairness. AWS token-bucket quotas claim "fairness" only in the - weaker selective-throttling sense, and rely on an elastic, rarely-saturated - fleet. - -Condition for caps-alone to suffice in practice: sum of caps comfortably below (or -elastically kept below) real capacity, and shed/rejected work acceptable, i.e. the -system never holds a contended backlog it must drain in some order. The moment you -hold a queue of admitted-but-waiting work at saturation, the serve order IS the -fairness policy. - -## CoDel (confirms the prior spike's "CoDel disproven" verdict) - -CoDel is an AQM: it bounds standing-queue delay by DROPPING packets when the -minimum sojourn over a window stays above target (5ms/100ms defaults; RFC 8289). -It is not a scheduler and gives no inter-flow fairness (RFC 7567: queue management -and scheduling are complementary, not substitutes). FQ-CoDel gets all its fairness -from a DRR scheduler; CoDel is only the per-flow AQM inside each sub-queue (RFC -8290). In a durable run queue that can never drop work, CoDel's actuator is gone: -reordering conserves total queue and total work, so hoisting one item's sojourn -down pushes others' up. Hoisting the stalest item to the front of an already -age-ordered queue re-applies the age bias the base already has, so it is a no-op -at best and a dominant-tenant amplifier at worst (a tenant that dumps a big -backlog owns the entire stale set). Facebook's server-side CoDel adaptation ("Fail -at Scale", ACM Queue 2015) keeps the drop (stale requests expire) and reorders -adaptive-LIFO (newest first), the opposite of stalest-first hoisting. diff --git a/internal-packages/run-engine/src/run-queue/CK_VTIME_KNOWN_LIMITATIONS.md b/internal-packages/run-engine/src/run-queue/CK_VTIME_KNOWN_LIMITATIONS.md deleted file mode 100644 index 43a6cd433..000000000 --- a/internal-packages/run-engine/src/run-queue/CK_VTIME_KNOWN_LIMITATIONS.md +++ /dev/null @@ -1,70 +0,0 @@ -# CK virtual-time scheduling: known limitations (read before enabling) - -The feature ships behind `RUN_ENGINE_CK_VTIME_SCHEDULING_ENABLED` (off by default). -A three-model blind adversarial review found no Critical issues; the correctness -and safety fixes it surfaced are applied. The items below are the review findings -that were deliberately NOT code-fixed because they are bounded, self-healing, or -pre-existing. They are the checklist for the "enable in production" decision. - -## Bounded state drift on paths that don't GC `ckVtime` - -The vtime dequeue command GCs a drained variant from both `ckIndex` and `ckVtime`. -But `acknowledgeMessageCkTracked`, `expireTtlRuns`, `moveToDeadLetterQueueCkTracked`, -and the flag-off dequeue command do NOT remove a drained variant from `ckVtime` -(they were left byte-identical). Consequences, all bounded: - -- A low-tag tombstone (a variant emptied by ack/TTL/DLQ without a vtime serve) is - the minimum entry, so the very next vtime dequeue visits it first, finds the - queue empty, and GCs it: self-heals in ~1 call. It can pin the floor low for - that one call. -- A high-tag tombstone (a heavily-served variant whose remaining backlog is then - removed out-of-band) lingers until the floor climbs to its tag or the 24h state - TTL fires. Pure memory drift, does not affect fairness. -- Rollback (flag on -> off): variants drained by the old command leave inert - `ckVtime` entries. Old code never reads them; they expire within `stateTtl` - (default 24h) once the base queue stops receiving writes. To reclaim sooner, - delete the `*:ckVtime` / `*:ckVtimeFloor` keys after disabling. - -A full fix (vtime-aware ack/TTL/DLQ command variants) is deferred: it adds three -more command variants for a bounded, self-healing drift on a dark feature. - -## Tie-break among equal virtual-time tags is member-name order - -When variants tie at the same tag (a fresh batch at the floor: cold start, new -deploy, or a GC'd variant re-entering), pass 1's `ZRANGE ckVtime` falls back to -Redis's lexicographic member order, i.e. the fully-qualified queue name including -the client-chosen concurrency key. A lex-early name gets a first-serve head start -in a tie. This is PRE-EXISTING (the head-timestamp baseline ties the same way) and -bounded: tags diverge after the first serve, so it affects only first-serve order, -not long-run fairness. A future improvement is to tie-break by head age instead of -member name. Do not rank fairness on an untrusted string if that head start ever -matters at scale. - -## Future-scheduled / retry-backoff variants occupy pass-1 window slots - -Enqueue and nack register a variant in `ckVtime` even when its head message is -scheduled in the future (delayed run, nack backoff). Pass 1 selects by tag with no -readiness filter, so a burst of future-headed variants can fill the pass-1 window -(`maxCount * scanWindowMultiplier`, default 3x); actual serves then come from pass -2 (today's age order). Work conservation still holds (pass 2 is a superset), so -this is fairness degradation under a retry storm, not loss. Widen -`scanWindowMultiplier` if observed. - -## Minor operational notes - -- Idle-polling a CK queue whose only work is future-scheduled now does a couple of - extra Redis writes per poll (floor SET + EXPIRE) vs the old early-return. Bounded; - visible in Redis write metrics after enabling. -- `descriptorFromQueue` positional parsing mis-splits a concurrency key containing - a literal `:` (pre-existing; not introduced here). The vtime feature uses the - full queue key as the ZSET member, which is unaffected, but any code that parses - the member back into fields inherits the pre-existing limitation. - -## Rollout (from the plan) - -1. Deploy with the flag off (new command scripts registered, never called). -2. Enable on a staging cell; watch dequeue-latency spans and Redis op rates against - the op-count budget; run a sybil-shaped workload and confirm the light key's wait. -3. Enable in production; during the instance-rolling window behaviour interpolates - between age order and fair order (both endpoints safe). -4. Rollback = flip the env var off; stale vtime keys expire via TTL within 24h. diff --git a/internal-packages/run-engine/src/run-queue/bench/ckMicroBench.bench.test.ts b/internal-packages/run-engine/src/run-queue/bench/ckMicroBench.bench.test.ts deleted file mode 100644 index 88051f71a..000000000 --- a/internal-packages/run-engine/src/run-queue/bench/ckMicroBench.bench.test.ts +++ /dev/null @@ -1,555 +0,0 @@ -import { createRedisClient } from "@internal/redis"; -import { trace } from "@internal/tracing"; -import { Logger } from "@trigger.dev/core/logger"; -import { Decimal } from "@trigger.dev/database"; -import { mkdirSync, writeFileSync } from "node:fs"; -import { performance } from "node:perf_hooks"; -import { describe, expect, it } from "vitest"; -import { FairQueueSelectionStrategy } from "../fairQueueSelectionStrategy.js"; -import { RunQueue } from "../index.js"; -import { RunQueueFullKeyProducer } from "../keyProducer.js"; -import type { InputPayload } from "../types.js"; - -// CK virtual-time PRIMARY arm: a queue-level A/B micro-benchmark. -// -// This drives the REAL RunQueue against an EXTERNAL Redis (not a testcontainer) -// and compares flag OFF (age-ordered CK dequeue) vs flag ON (SFQ virtual time) -// under identical load. The flag is a RunQueue constructor option, so one bench -// process runs both arms in-process: no webapp, no redeploy. -// -// It is a deliberate, defensible A/B: the two arms enqueue the exact same -// messages with the exact same timestamps and drive the exact same step loop. -// The isolation and reuse are inherited from tests/ckVtimeFairness.test.ts (the -// step loop, scenario shapes, conservation checks are the same); this file adds -// wall-clock dequeue latency, a Redis op-count, N trials, and file output. -// -// It is INERT in CI: the suite only runs when CK_BENCH_REDIS_URL is set, so -// `pnpm run test` collects it as a skipped describe and never touches a network. -// -// Run it (from the run-engine package, pointed at a dedicated throwaway Redis): -// CK_BENCH_REDIS_URL=redis://127.0.0.1:6399 \ -// CK_BENCH_TRIALS=5 CK_BENCH_OUT=./bench-results \ -// pnpm exec vitest run src/run-queue/bench/ckMicroBench.bench.test.ts -// -// Knob sweep (optional, defaults match production defaults 1 / 3): -// CK_BENCH_QUANTUM=1 CK_BENCH_WINDOW_MULT=3 -// -// WARNING: the bench FLUSHDBs the target Redis between arms. Point it ONLY at a -// dedicated throwaway instance, never at a shared or production Redis. - -const REDIS_URL = process.env.CK_BENCH_REDIS_URL; -const TRIALS = Math.max(1, Number(process.env.CK_BENCH_TRIALS ?? "5")); -const OUT_DIR = process.env.CK_BENCH_OUT ?? "./bench-results"; -const QUANTUM = Math.max(1, Number(process.env.CK_BENCH_QUANTUM ?? "1")); -const WINDOW_MULT = Math.max(1, Number(process.env.CK_BENCH_WINDOW_MULT ?? "3")); -const SCENARIO_FILTER = process.env.CK_BENCH_SCENARIOS?.split(",").map((s) => s.trim()); - -const keys = new RunQueueFullKeyProducer(); - -const testOptions = { - name: "rq", - tracer: trace.getTracer("rq"), - workers: 1, - defaultEnvConcurrency: 25, - logger: new Logger("RunQueue", "error"), - retryOptions: { - maxAttempts: 5, - factor: 1.1, - minTimeoutInMs: 100, - maxTimeoutInMs: 1_000, - randomize: true, - }, - keys, -}; - -const authenticatedEnvDev = { - id: "e1234", - type: "DEVELOPMENT" as const, - maximumConcurrencyLimit: 10, - concurrencyLimitBurstFactor: new Decimal(2.0), - project: { id: "p1234" }, - organization: { id: "o1234" }, -}; - -function redisConn() { - const u = new URL(REDIS_URL!); - return { host: u.hostname, port: Number(u.port || "6379") }; -} - -function createQueue(keyPrefix: string, vtimeEnabled: boolean) { - const conn = redisConn(); - return new RunQueue({ - ...testOptions, - masterQueueConsumersDisabled: true, - workerOptions: { disabled: true }, - ckVirtualTimeScheduling: { - enabled: vtimeEnabled, - quantum: QUANTUM, - scanWindowMultiplier: WINDOW_MULT, - }, - queueSelectionStrategy: new FairQueueSelectionStrategy({ - redis: { keyPrefix, host: conn.host, port: conn.port }, - keys, - }), - redis: { keyPrefix, host: conn.host, port: conn.port }, - }); -} - -function makeMessage(overrides: Partial = {}): InputPayload { - return { - runId: "r1", - taskIdentifier: "task/my-task", - orgId: "o1234", - projectId: "p1234", - environmentId: "e1234", - environmentType: "DEVELOPMENT", - queue: "task/my-task", - timestamp: Date.now(), - attempt: 0, - ...overrides, - }; -} - -type ScenarioMessage = { runId: string; ck: string; timestamp: number }; - -type Scenario = { - name: string; - // Human label for the "victim" the fairness hypothesis is about. - victimLabel: string; - // Classifies a concurrency key as the victim (the light/starved tenant). - isVictim: (ck: string) => boolean; - messages: ScenarioMessage[]; - envConcurrencyLimit: number; - holdSteps: number; - maxSteps: number; -}; - -type ServeRecord = { step: number; ck: string; messageId: string; wallMs: number }; - -type ArmResult = { - serves: ServeRecord[]; - drainStep: number; - contentionByCk: Map; - contentionTotal: number; - callLatenciesMs: number[]; - redisCalls: number; -}; - -// ---- scenario shapes (ported values from the fairness spike, same as the -// tests/ckVtimeFairness.test.ts scenarios; nothing imported from the spike) ---- - -function buildScenarios(): Scenario[] { - const t0 = Date.now() - 500_000; - const all: Scenario[] = []; - - // ckSkew (starvation): heavy 120-msg backlog on an old shared head, 4 light - // keys x 10 on later heads. Serialized contention (env limit 1) is where the - // baseline's age order starves the light keys. - { - const messages: ScenarioMessage[] = []; - for (let i = 0; i < 120; i++) - messages.push({ runId: `heavy-${i}`, ck: "heavy", timestamp: t0 }); - for (let i = 0; i < 10; i++) - for (let k = 0; k < 4; k++) - messages.push({ - runId: `light${k}-${i}`, - ck: `light${k}`, - timestamp: t0 + 10_000 + i * 4 + k, - }); - all.push({ - name: "ckSkew", - victimLabel: "light keys", - isVictim: (ck) => ck.startsWith("light"), - messages, - envConcurrencyLimit: 1, - holdSteps: 3, - maxSteps: 1_000, - }); - } - - // ckTrickle (starvation): bulk 120 + 2 trickle keys x 15. - { - const messages: ScenarioMessage[] = []; - for (let i = 0; i < 120; i++) messages.push({ runId: `bulk-${i}`, ck: "bulk", timestamp: t0 }); - for (let i = 0; i < 15; i++) - for (let k = 0; k < 2; k++) - messages.push({ - runId: `trickle${k}-${i}`, - ck: `trickle${k}`, - timestamp: t0 + 10_000 + i * 2 + k, - }); - all.push({ - name: "ckTrickle", - victimLabel: "trickle keys", - isVictim: (ck) => ck.startsWith("trickle"), - messages, - envConcurrencyLimit: 1, - holdSteps: 3, - maxSteps: 1_000, - }); - } - - // ckSybil (noisy-neighbor caps cannot fix): 20 attacker keys x 8 (older - // heads) + 1 light key x 10 (newer). 21 variants against a batch of 10. - { - const messages: ScenarioMessage[] = []; - for (let i = 0; i < 8; i++) - for (let k = 0; k < 20; k++) { - const ck = `att${String(k).padStart(2, "0")}`; - messages.push({ runId: `${ck}-${i}`, ck, timestamp: t0 + i * 20 + k }); - } - for (let i = 0; i < 10; i++) - messages.push({ runId: `light-${i}`, ck: "light", timestamp: t0 + 50_000 + i }); - all.push({ - name: "ckSybil", - victimLabel: "light key", - isVictim: (ck) => ck === "light", - messages, - envConcurrencyLimit: 25, - holdSteps: 3, - maxSteps: 300, - }); - } - - // ckManyKeys (cardinality ABOVE the pass-1 window): 60 attacker keys x 8 on a - // tied old head + 1 light key x 10. Probes the stated window limitation: the - // light key must still drain (no permanent starvation), even though 61 - // variants exceed the 30-wide pass-1 window. - { - const messages: ScenarioMessage[] = []; - for (let i = 0; i < 8; i++) - for (let k = 0; k < 60; k++) { - const ck = `att${String(k).padStart(2, "0")}`; - messages.push({ runId: `${ck}-${i}`, ck, timestamp: t0 }); - } - for (let i = 0; i < 10; i++) - messages.push({ runId: `light-${i}`, ck: "light", timestamp: t0 + 50_000 + i }); - all.push({ - name: "ckManyKeys", - victimLabel: "light key", - isVictim: (ck) => ck === "light", - messages, - envConcurrencyLimit: 25, - holdSteps: 3, - maxSteps: 1_000, - }); - } - - // ckBalanced (no-harm mixed multi-tenant): 4 symmetric keys x 25. - { - const cks = ["bal0", "bal1", "bal2", "bal3"]; - const messages: ScenarioMessage[] = []; - for (let i = 0; i < 25; i++) - for (let k = 0; k < cks.length; k++) - messages.push({ runId: `${cks[k]}-${i}`, ck: cks[k]!, timestamp: t0 + i * 4 + k }); - all.push({ - name: "ckBalanced", - victimLabel: "worst symmetric key", - isVictim: (ck) => ck.startsWith("bal"), - messages, - envConcurrencyLimit: 4, - holdSteps: 3, - maxSteps: 500, - }); - } - - // ckHeavyIdle (work conservation): a lone key with 60 msgs, nothing else - // contending. Drain-step ON must equal OFF exactly. - { - const messages: ScenarioMessage[] = []; - for (let i = 0; i < 60; i++) - messages.push({ runId: `solo-${i}`, ck: "solo", timestamp: t0 + i }); - all.push({ - name: "ckHeavyIdle", - victimLabel: "lone key", - isVictim: (ck) => ck === "solo", - messages, - envConcurrencyLimit: 25, - holdSteps: 3, - maxSteps: 300, - }); - } - - return SCENARIO_FILTER ? all.filter((s) => SCENARIO_FILTER.includes(s.name)) : all; -} - -// ---- one arm of one scenario ---- - -async function runArm( - scenario: Scenario, - vtimeEnabled: boolean, - trial: number -): Promise { - const keyPrefix = `ckbench:${scenario.name}:${vtimeEnabled ? "on" : "off"}:t${trial}:`; - const queue = createQueue(keyPrefix, vtimeEnabled); - const conn = redisConn(); - const admin = createRedisClient({ host: conn.host, port: conn.port }, { onError: () => {} }); - - try { - const env = { - ...authenticatedEnvDev, - maximumConcurrencyLimit: scenario.envConcurrencyLimit, - concurrencyLimitBurstFactor: new Decimal(1), - }; - await queue.updateEnvConcurrencyLimits(env); - - for (const msg of scenario.messages) { - await queue.enqueueMessage({ - env, - message: makeMessage({ - runId: msg.runId, - concurrencyKey: msg.ck, - timestamp: msg.timestamp, - }), - workerQueue: env.id, - skipDequeueProcessing: true, - }); - } - - // Count only steady-state (dequeue + ack) Redis ops, not enqueue. - await admin.call("CONFIG", "RESETSTAT"); - - const shard = keys.masterQueueShardForEnvironment(env.id, 2); - const total = scenario.messages.length; - const remaining = new Map(); - for (const m of scenario.messages) remaining.set(m.ck, (remaining.get(m.ck) ?? 0) + 1); - - const serves: ServeRecord[] = []; - const inFlight: { messageId: string; servedAtStep: number }[] = []; - const contentionByCk = new Map(); - let contentionTotal = 0; - let drainStep = -1; - const callLatenciesMs: number[] = []; - const armStart = performance.now(); - - for (let step = 0; step < scenario.maxSteps && serves.length < total; step++) { - let keysWithBacklog = 0; - for (const count of remaining.values()) if (count > 0) keysWithBacklog++; - - const before = performance.now(); - const messages = await queue.testDequeueFromMasterQueue(shard, env.id, 10); - callLatenciesMs.push(performance.now() - before); - - for (const m of messages) { - const ck = m.message.concurrencyKey ?? ""; - serves.push({ step, ck, messageId: m.messageId, wallMs: performance.now() - armStart }); - remaining.set(ck, (remaining.get(ck) ?? 0) - 1); - inFlight.push({ messageId: m.messageId, servedAtStep: step }); - if (keysWithBacklog >= 2) { - contentionTotal++; - contentionByCk.set(ck, (contentionByCk.get(ck) ?? 0) + 1); - } - if (serves.length === total) drainStep = step; - } - - for (let i = inFlight.length - 1; i >= 0; i--) { - const entry = inFlight[i]!; - if (entry.servedAtStep + scenario.holdSteps <= step) { - await queue.acknowledgeMessage(env.organization.id, entry.messageId, { - skipDequeueProcessing: true, - }); - inFlight.splice(i, 1); - } - } - } - - const stats = await admin.call("INFO", "commandstats"); - const redisCalls = sumRedisCalls(String(stats)); - - return { serves, drainStep, contentionByCk, contentionTotal, callLatenciesMs, redisCalls }; - } finally { - await admin.quit().catch(() => {}); - await queue.quit(); - // Clean slate for the next arm: this is a dedicated throwaway Redis. - const admin2 = createRedisClient(redisConn(), { onError: () => {} }); - await admin2.flushdb().catch(() => {}); - await admin2.quit().catch(() => {}); - } -} - -// ---- metrics ---- - -function sumRedisCalls(info: string): number { - // lines look like: cmdstat_zadd:calls=123,usec=...,... - let total = 0; - for (const line of info.split("\n")) { - const m = line.match(/cmdstat_[^:]+:calls=(\d+)/); - if (m) total += Number(m[1]); - } - return total; -} - -function pct(sorted: number[], p: number): number { - if (sorted.length === 0) return NaN; - const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1)); - return sorted[idx]!; -} - -function stats(xs: number[]) { - const s = [...xs].sort((a, b) => a - b); - const mean = xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : NaN; - return { mean, p50: pct(s, 50), p95: pct(s, 95), p99: pct(s, 99) }; -} - -// Jain's fairness index over per-key served counts during contention windows. -// 1.0 = perfectly fair; 1/n = one key took everything. -function jain(counts: number[]): number { - const nonzero = counts.filter((c) => c > 0); - if (nonzero.length === 0) return NaN; - const sum = nonzero.reduce((a, b) => a + b, 0); - const sumSq = nonzero.reduce((a, b) => a + b * b, 0); - return (sum * sum) / (nonzero.length * sumSq); -} - -function victimWaits(arm: ArmResult, s: Scenario): number[] { - return arm.serves.filter((r) => s.isVictim(r.ck)).map((r) => r.step); -} - -function firstServe(arm: ArmResult, s: Scenario): number { - const first = arm.serves.find((r) => s.isVictim(r.ck)); - return first ? first.step : -1; -} - -// ---- the bench ---- - -describe.runIf(!!REDIS_URL)("CK virtual-time micro-benchmark (A/B, external Redis)", () => { - it("runs OFF vs ON across scenarios and writes results", { timeout: 30 * 60_000 }, async () => { - const scenarios = buildScenarios(); - const report: any = { - generatedAtMs: Date.now(), - redisUrl: REDIS_URL, - trials: TRIALS, - knobs: { quantum: QUANTUM, scanWindowMultiplier: WINDOW_MULT }, - scenarios: [] as any[], - }; - - for (const s of scenarios) { - // Wall-clock latency and op-count are pooled/aggregated across trials. - // Step-based metrics are deterministic, so trial 0 is authoritative and - // later trials only assert determinism. - const offCalls: number[] = []; - const onCalls: number[] = []; - const offOps: number[] = []; - const onOps: number[] = []; - let off0: ArmResult | null = null; - let on0: ArmResult | null = null; - - for (let t = 0; t < TRIALS; t++) { - const off = await runArm(s, false, t); - const on = await runArm(s, true, t); - - // Correctness gate: identical load must serve every message exactly - // once in BOTH arms, else the comparison is meaningless. - expect(off.serves.length, `${s.name} OFF served != enqueued`).toBe(s.messages.length); - expect(on.serves.length, `${s.name} ON served != enqueued`).toBe(s.messages.length); - expect(new Set(off.serves.map((r) => r.messageId)).size).toBe(s.messages.length); - expect(new Set(on.serves.map((r) => r.messageId)).size).toBe(s.messages.length); - - offCalls.push(...off.callLatenciesMs); - onCalls.push(...on.callLatenciesMs); - offOps.push(off.redisCalls); - onOps.push(on.redisCalls); - - if (t === 0) { - off0 = off; - on0 = on; - } else { - // determinism of the logical schedule across trials - expect(firstServe(off, s), `${s.name} OFF first-serve not deterministic`).toBe( - firstServe(off0!, s) - ); - expect(firstServe(on, s), `${s.name} ON first-serve not deterministic`).toBe( - firstServe(on0!, s) - ); - expect(on.drainStep).toBe(on0!.drainStep); - } - } - - const offWait = stats(victimWaits(off0!, s)); - const onWait = stats(victimWaits(on0!, s)); - const median = (xs: number[]) => [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)]!; - - const scenarioReport = { - name: s.name, - victim: s.victimLabel, - config: { - envConcurrencyLimit: s.envConcurrencyLimit, - holdSteps: s.holdSteps, - variants: new Set(s.messages.map((m) => m.ck)).size, - messages: s.messages.length, - }, - off: { - victimWait: offWait, - victimFirstServe: firstServe(off0!, s), - drainStep: off0!.drainStep, - jain: jain([...off0!.contentionByCk.values()]), - callLatencyMs: stats(offCalls), - redisOpsMedian: median(offOps), - }, - on: { - victimWait: onWait, - victimFirstServe: firstServe(on0!, s), - drainStep: on0!.drainStep, - jain: jain([...on0!.contentionByCk.values()]), - callLatencyMs: stats(onCalls), - redisOpsMedian: median(onOps), - }, - }; - report.scenarios.push(scenarioReport); - // eslint-disable-next-line no-console - console.log( - `[ckbench] ${s.name}: victim p95 wait OFF=${offWait.p95} ON=${onWait.p95} | drain OFF=${off0!.drainStep} ON=${on0!.drainStep}` - ); - } - - mkdirSync(OUT_DIR, { recursive: true }); - writeFileSync(`${OUT_DIR}/ck-micro-results.json`, JSON.stringify(report, null, 2)); - writeFileSync(`${OUT_DIR}/ck-micro-results.md`, renderMarkdown(report)); - }); -}); - -function fmt(n: number): string { - if (Number.isNaN(n)) return "n/a"; - return Number.isInteger(n) ? String(n) : n.toFixed(2); -} -function delta(off: number, on: number): string { - if (Number.isNaN(off) || Number.isNaN(on)) return "n/a"; - if (off === 0) return on === 0 ? "0" : "+inf"; - const pctChange = ((on - off) / off) * 100; - return `${pctChange >= 0 ? "+" : ""}${pctChange.toFixed(0)}%`; -} - -function renderMarkdown(report: any): string { - const lines: string[] = []; - lines.push(`# CK virtual-time micro-benchmark results`); - lines.push(""); - lines.push( - `Redis \`${report.redisUrl}\`, ${report.trials} trial(s), quantum ${report.knobs.quantum}, window multiplier ${report.knobs.scanWindowMultiplier}.` - ); - lines.push(""); - lines.push( - `Numbers are RELATIVE (same box, same load, flag OFF vs ON). Wait is in logical dequeue steps. Latency is wall-clock per dequeue call on this box and is NOT prod-scale absolute throughput.` - ); - lines.push(""); - lines.push(`| scenario | metric | baseline (OFF) | vtime (ON) | delta |`); - lines.push(`| --- | --- | --- | --- | --- |`); - for (const s of report.scenarios) { - const rows: [string, number, number][] = [ - [`victim wait p50 (${s.victim})`, s.off.victimWait.p50, s.on.victimWait.p50], - [`victim wait p95`, s.off.victimWait.p95, s.on.victimWait.p95], - [`victim wait p99`, s.off.victimWait.p99, s.on.victimWait.p99], - [`victim first-serve step (starvation bound)`, s.off.victimFirstServe, s.on.victimFirstServe], - [`drain step (work conservation)`, s.off.drainStep, s.on.drainStep], - [`Jain fairness index (contention)`, s.off.jain, s.on.jain], - [`dequeue call p95 (ms)`, s.off.callLatencyMs.p95, s.on.callLatencyMs.p95], - [`redis ops (dequeue+ack)`, s.off.redisOpsMedian, s.on.redisOpsMedian], - ]; - rows.forEach(([metric, off, on], i) => { - lines.push( - `| ${i === 0 ? `**${s.name}**` : ""} | ${metric} | ${fmt(off)} | ${fmt(on)} | ${delta(off, on)} |` - ); - }); - } - lines.push(""); - return lines.join("\n"); -} diff --git a/internal-packages/run-engine/src/run-queue/bench/ckResourceBench.bench.test.ts b/internal-packages/run-engine/src/run-queue/bench/ckResourceBench.bench.test.ts deleted file mode 100644 index c0c0c3e65..000000000 --- a/internal-packages/run-engine/src/run-queue/bench/ckResourceBench.bench.test.ts +++ /dev/null @@ -1,390 +0,0 @@ -import { createRedisClient } from "@internal/redis"; -import { trace } from "@internal/tracing"; -import { Logger } from "@trigger.dev/core/logger"; -import { Decimal } from "@trigger.dev/database"; -import { mkdirSync, writeFileSync } from "node:fs"; -import { describe, expect, it } from "vitest"; -import { FairQueueSelectionStrategy } from "../fairQueueSelectionStrategy.js"; -import { RunQueue } from "../index.js"; -import { RunQueueFullKeyProducer } from "../keyProducer.js"; -import type { InputPayload } from "../types.js"; - -// CK virtual-time RESOURCE arm: Redis CPU + memory vs concurrency-key cardinality. -// -// Answers "how do these changes affect the run-queue Redis CPU/memory, and how do -// both react as cardinality grows (e.g. 10k concurrency keys on one base queue)". -// Drives a real RunQueue against an EXTERNAL dedicated Redis, flag OFF vs ON on -// identical load, and reads server-side metrics (INFO memory/cpu/commandstats, -// MEMORY USAGE, OBJECT ENCODING) that are unaffected by client<->server RTT. -// -// Inert unless CK_BENCH_REDIS_URL is set. FLUSHALLs the target between points, so -// point it ONLY at a dedicated throwaway store (a redis-bench lab store). -// -// export CK_BENCH_REDIS_URL="$(lab store url ckbench1)" -// pnpm exec vitest run src/run-queue/bench/ckResourceBench.bench.test.ts -// -// Env knobs: -// CK_RES_MEM_CARDS=100,1000,10000,50000 memory-at-rest sweep -// CK_RES_CPU_CARDS=100,1000,10000 cpu-under-load sweep -// CK_RES_LOAD_OPS=8000 load rounds per cpu point -// CK_RES_CONCURRENCY=64 client concurrency (beats RTT) -// CK_RES_CHURN_CARD=10000 churn/tombstone cardinality -// CK_RES_CHURN_ROUNDS=60 churn sample rounds -// CK_BENCH_OUT=./bench-results -// -// NOTE: the RunQueue Lua commands run via EVALSHA, so redis.call() ops inside a -// script do NOT show up as separate cmdstat_* lines; they roll up under evalsha. -// The reportable CPU signals are therefore total used_cpu over an identical -// workload and aggregate evalsha usec_per_call, not a per-Redis-command split. - -const REDIS_URL = process.env.CK_BENCH_REDIS_URL; -const MEM_CARDS = (process.env.CK_RES_MEM_CARDS ?? "100,1000,10000,50000") - .split(",") - .map((s) => +s.trim()); -const CPU_CARDS = (process.env.CK_RES_CPU_CARDS ?? "100,1000,10000") - .split(",") - .map((s) => +s.trim()); -const LOAD_OPS = +(process.env.CK_RES_LOAD_OPS ?? "8000"); -const CONCURRENCY = +(process.env.CK_RES_CONCURRENCY ?? "64"); -const CHURN_CARD = +(process.env.CK_RES_CHURN_CARD ?? "10000"); -const CHURN_ROUNDS = +(process.env.CK_RES_CHURN_ROUNDS ?? "60"); -const OUT_DIR = process.env.CK_BENCH_OUT ?? "./bench-results"; - -const keys = new RunQueueFullKeyProducer(); - -const testOptions = { - name: "rq", - tracer: trace.getTracer("rq"), - workers: 1, - defaultEnvConcurrency: 1_000_000, - logger: new Logger("RunQueue", "error"), - retryOptions: { - maxAttempts: 5, - factor: 1.1, - minTimeoutInMs: 100, - maxTimeoutInMs: 1_000, - randomize: true, - }, - keys, -}; - -const env = { - id: "e1234", - type: "PRODUCTION" as const, - maximumConcurrencyLimit: 1_000_000, - concurrencyLimitBurstFactor: new Decimal(1.0), - project: { id: "p1234" }, - organization: { id: "o1234" }, -}; - -function conn() { - const u = new URL(REDIS_URL!); - return { - host: u.hostname, - port: Number(u.port || "6379"), - password: decodeURIComponent(u.password || "") || undefined, - username: decodeURIComponent(u.username || "") || undefined, - }; -} - -function createQueue(keyPrefix: string, vtimeEnabled: boolean) { - const c = conn(); - return new RunQueue({ - ...testOptions, - masterQueueConsumersDisabled: true, - workerOptions: { disabled: true }, - ckVirtualTimeScheduling: { enabled: vtimeEnabled }, - queueSelectionStrategy: new FairQueueSelectionStrategy({ redis: { keyPrefix, ...c }, keys }), - redis: { keyPrefix, ...c }, - }); -} - -function makeMessage(o: Partial = {}): InputPayload { - return { - runId: "r1", - taskIdentifier: "task/my-task", - orgId: "o1234", - projectId: "p1234", - environmentId: "e1234", - environmentType: "PRODUCTION", - queue: "task/my-task", - timestamp: Date.now(), - attempt: 0, - ...o, - }; -} - -// bounded-concurrency runner (beats the ~3.6ms workstation->box RTT) -async function pool(n: number, count: number, fn: (i: number) => Promise) { - let i = 0; - await Promise.all( - Array.from({ length: Math.min(n, count) }, async () => { - while (i < count) { - const idx = i++; - await fn(idx); - } - }) - ); -} - -// ---- server-side metric helpers (separate no-prefix admin client) ---- -function admin() { - return createRedisClient(conn(), { onError: () => {} }); -} -function infoField(info: string, key: string): number { - const line = info.split("\n").find((l) => l.startsWith(key + ":")); - return line ? Number(line.split(":")[1]) : NaN; -} -async function usedMemory(a: any) { - return infoField(await a.info("memory"), "used_memory"); -} -async function usedCpu(a: any) { - const i = await a.info("cpu"); - return infoField(i, "used_cpu_user") + infoField(i, "used_cpu_sys"); -} -function evalsha(info: string) { - const line = info.split("\n").find((l) => l.startsWith("cmdstat_evalsha:")); - if (!line) return { calls: 0, usec: 0, usecPerCall: 0 }; - const g = (k: string) => Number(line.match(new RegExp(`${k}=([0-9.]+)`))?.[1] ?? 0); - return { calls: g("calls"), usec: g("usec"), usecPerCall: g("usec_per_call") }; -} - -// ---- build N distinct concurrency keys (one queued message each) ---- -async function buildCardinality(queue: RunQueue, n: number) { - const t0 = Date.now() - 500_000; - await pool(CONCURRENCY, n, async (i) => { - await queue.enqueueMessage({ - env, - message: makeMessage({ runId: `r-${i}`, concurrencyKey: `k${i}`, timestamp: t0 + i }), - workerQueue: env.id, - skipDequeueProcessing: true, - }); - }); -} - -async function findKey(a: any, prefix: string, suffix: string): Promise { - const found = await a.keys(`${prefix}*:${suffix}`); - return found[0] ?? null; -} - -// ---- the bench ---- -describe.runIf(!!REDIS_URL)("CK virtual-time resource + cardinality benchmark", () => { - it( - "measures Redis memory and CPU vs cardinality, flag OFF vs ON", - { timeout: 60 * 60_000 }, - async () => { - const a = admin(); - const report: any = { generatedAtMs: Date.now(), memory: [], cpu: [], churn: null }; - - // ---------- memory at rest ---------- - for (const n of MEM_CARDS) { - const row: any = { cardinality: n }; - for (const on of [false, true]) { - await a.flushall(); - await a.call("CONFIG", "RESETSTAT"); - const prefix = `ckres:mem:${n}:${on ? "on" : "off"}:`; - const q = createQueue(prefix, on); - try { - await q.updateEnvConcurrencyLimits(env); - await buildCardinality(q, n); - const arm = on ? "on" : "off"; - row[`used_memory_${arm}`] = await usedMemory(a); - const ckIndexKey = await findKey(a, prefix, "ckIndex"); - const ckVtimeKey = await findKey(a, prefix, "ckVtime"); - row[`ckIndex_bytes_${arm}`] = ckIndexKey - ? await a.call("MEMORY", "USAGE", ckIndexKey) - : null; - row[`ckIndex_card_${arm}`] = ckIndexKey ? await a.zcard(ckIndexKey) : 0; - if (on) { - row.ckVtime_bytes = ckVtimeKey ? await a.call("MEMORY", "USAGE", ckVtimeKey) : null; - row.ckVtime_card = ckVtimeKey ? await a.zcard(ckVtimeKey) : 0; - row.ckVtime_encoding = ckVtimeKey - ? await a.call("OBJECT", "ENCODING", ckVtimeKey) - : null; - // ckVtime must mirror ckIndex membership when built via the slow path - expect(row.ckVtime_card).toBe(row.ckIndex_card_on); - } - } finally { - await q.quit(); - } - } - row.used_memory_delta = row.used_memory_on - row.used_memory_off; - row.ckVtime_over_ckIndex = - row.ckIndex_bytes_on && row.ckVtime_bytes - ? +(row.ckVtime_bytes / row.ckIndex_bytes_on).toFixed(2) - : null; - report.memory.push(row); - // eslint-disable-next-line no-console - console.log( - `[ckres] mem N=${n}: used_memory delta=${row.used_memory_delta}B ckVtime=${row.ckVtime_bytes}B (${row.ckVtime_encoding})` - ); - } - - // ---------- CPU under identical load ---------- - for (const n of CPU_CARDS) { - const row: any = { cardinality: n }; - for (const on of [false, true]) { - await a.flushall(); - const prefix = `ckres:cpu:${n}:${on ? "on" : "off"}:`; - const q = createQueue(prefix, on); - try { - await q.updateEnvConcurrencyLimits(env); - await buildCardinality(q, n); - const shard = keys.masterQueueShardForEnvironment(env.id, 2); - - await a.call("CONFIG", "RESETSTAT"); - const cpu0 = await usedCpu(a); - const t0wall = Date.now(); - - // identical workload both arms: LOAD_OPS rounds, each round enqueues - // 10 fresh messages (rotating keys, keeps N populated) and does one - // batched dequeue (maxCount 10) + acks the served set. - let served = 0; - await pool(CONCURRENCY, LOAD_OPS, async (i) => { - for (let j = 0; j < 10; j++) { - await q.enqueueMessage({ - env, - message: makeMessage({ - runId: `L-${i}-${j}`, - concurrencyKey: `k${(i * 10 + j) % n}`, - timestamp: Date.now(), - }), - workerQueue: env.id, - skipDequeueProcessing: true, - }); - } - const msgs = await q.testDequeueFromMasterQueue(shard, env.id, 10); - served += msgs.length; - for (const m of msgs) { - await q.acknowledgeMessage(env.organization.id, m.messageId, { - skipDequeueProcessing: true, - }); - } - }); - - const cpu1 = await usedCpu(a); - const es = evalsha(await a.info("commandstats")); - const arm = on ? "on" : "off"; - row[`cpu_sec_${arm}`] = +(cpu1 - cpu0).toFixed(3); - row[`evalsha_calls_${arm}`] = es.calls; - row[`evalsha_usec_per_call_${arm}`] = +es.usecPerCall.toFixed(2); - row[`wall_ms_${arm}`] = Date.now() - t0wall; - row[`served_${arm}`] = served; - } finally { - await q.quit(); - } - } - row.cpu_sec_delta = +(row.cpu_sec_on - row.cpu_sec_off).toFixed(3); - row.cpu_overhead_pct = row.cpu_sec_off - ? Math.round(((row.cpu_sec_on - row.cpu_sec_off) / row.cpu_sec_off) * 100) - : null; - report.cpu.push(row); - // eslint-disable-next-line no-console - console.log( - `[ckres] cpu N=${n}: cpu OFF=${row.cpu_sec_off}s ON=${row.cpu_sec_on}s (${row.cpu_overhead_pct}%) evalsha usec/call OFF=${row.evalsha_usec_per_call_off} ON=${row.evalsha_usec_per_call_on}` - ); - } - - // ---------- churn: ckVtime membership stays bounded vs ckIndex ---------- - { - await a.flushall(); - const prefix = `ckres:churn:on:`; - const q = createQueue(prefix, true); - const samples: any[] = []; - try { - await q.updateEnvConcurrencyLimits(env); - await buildCardinality(q, CHURN_CARD); - const shard = keys.masterQueueShardForEnvironment(env.id, 2); - const ckIndexKey = (await findKey(a, prefix, "ckIndex"))!; - const ckVtimeKey = (await findKey(a, prefix, "ckVtime"))!; - let nextKey = CHURN_CARD; - for (let r = 0; r < CHURN_ROUNDS; r++) { - // hold cardinality: each iteration enqueues one FRESH key and drains - // one message (maxCount 1), so registration and GC churn continuously - // while total membership stays ~CHURN_CARD. - await pool(CONCURRENCY, 200, async () => { - await q.enqueueMessage({ - env, - message: makeMessage({ - runId: `C-${nextKey}`, - concurrencyKey: `k${nextKey++}`, - timestamp: Date.now(), - }), - workerQueue: env.id, - skipDequeueProcessing: true, - }); - const msgs = await q.testDequeueFromMasterQueue(shard, env.id, 1); - for (const m of msgs) { - await q.acknowledgeMessage(env.organization.id, m.messageId, { - skipDequeueProcessing: true, - }); - } - }); - if (r % 10 === 0 || r === CHURN_ROUNDS - 1) { - const ckIndexCard = await a.zcard(ckIndexKey); - const ckVtimeCard = await a.zcard(ckVtimeKey); - samples.push({ - round: r, - ckIndex: ckIndexCard, - ckVtime: ckVtimeCard, - ratio: ckIndexCard ? +(ckVtimeCard / ckIndexCard).toFixed(2) : null, - }); - } - } - report.churn = { cardinality: CHURN_CARD, rounds: CHURN_ROUNDS, samples }; - // bounded: ckVtime never wildly exceeds ckIndex (allow generous 3x for transient tombstones) - for (const s of samples) if (s.ratio !== null) expect(s.ratio).toBeLessThanOrEqual(3); - } finally { - await q.quit(); - } - } - - await a.flushall(); - await a.quit().catch(() => {}); - - mkdirSync(OUT_DIR, { recursive: true }); - writeFileSync(`${OUT_DIR}/ck-resource-results.json`, JSON.stringify(report, null, 2)); - writeFileSync(`${OUT_DIR}/ck-resource-results.md`, renderMarkdown(report)); - } - ); -}); - -function renderMarkdown(r: any): string { - const L: string[] = []; - const kb = (b: number) => (b == null ? "n/a" : (b / 1024).toFixed(1) + "KB"); - L.push(`# CK virtual-time resource + cardinality results`, ""); - L.push( - `Redis \`${(process.env.CK_BENCH_REDIS_URL || "").replace(/:[^:@/]*@/, ":***@")}\`. Relative OFF-vs-ON on one box; not prod scale.`, - "" - ); - L.push(`## Memory at rest (single base queue, one message per key)`, ""); - L.push( - `| keys (N) | used_memory OFF | used_memory ON | delta | ckIndex bytes | ckVtime bytes | ckVtime/ckIndex | ckVtime encoding |` - ); - L.push(`| --- | --- | --- | --- | --- | --- | --- | --- |`); - for (const m of r.memory) - L.push( - `| ${m.cardinality} | ${kb(m.used_memory_off)} | ${kb(m.used_memory_on)} | ${kb(m.used_memory_delta)} | ${kb(m.ckIndex_bytes_on)} | ${kb(m.ckVtime_bytes)} | ${m.ckVtime_over_ckIndex ?? "n/a"} | ${m.ckVtime_encoding ?? "n/a"} |` - ); - L.push( - "", - `## Redis CPU under identical workload (${process.env.CK_RES_LOAD_OPS ?? "8000"} rounds)`, - "" - ); - L.push( - `| keys (N) | CPU-sec OFF | CPU-sec ON | delta | overhead | evalsha usec/call OFF | evalsha usec/call ON | evalsha calls OFF/ON |` - ); - L.push(`| --- | --- | --- | --- | --- | --- | --- | --- |`); - for (const c of r.cpu) - L.push( - `| ${c.cardinality} | ${c.cpu_sec_off} | ${c.cpu_sec_on} | ${c.cpu_sec_delta} | ${c.cpu_overhead_pct}% | ${c.evalsha_usec_per_call_off} | ${c.evalsha_usec_per_call_on} | ${c.evalsha_calls_off}/${c.evalsha_calls_on} |` - ); - if (r.churn) { - L.push("", `## Tombstone / membership under churn (N=${r.churn.cardinality}, flag ON)`, ""); - L.push(`| round | ckIndex card | ckVtime card | ratio |`, `| --- | --- | --- | --- |`); - for (const s of r.churn.samples) - L.push(`| ${s.round} | ${s.ckIndex} | ${s.ckVtime} | ${s.ratio} |`); - } - L.push(""); - return L.join("\n"); -}