nicktrn
7aa871f37b
Adds an opt-in mechanism to route a configurable percentage of organizations onto the compute (MicroVM) backing of their region at trigger time, without changing their stored region settings. Routing is gated by three global feature flags - `computeMigrationEnabled`, `computeMigrationFreePercentage`, `computeMigrationPaidPercentage` - plus a per-org `computeMigrationEnabled` override that wins in both directions. A region's compute backing is resolved from a new `WorkerInstanceGroup.region` column: a container group and its MicroVM group share one geo `region`, so the migration swaps the resolved worker queue to the backing group's queue. Orgs are bucketed deterministically by id, so ramping a percentage down keeps a strict subset rather than reshuffling, and a region with no compute backing is never touched. Everything is off by default - behaviour is unchanged unless the flags are set. The flags and the worker-region groups are read on the trigger hot path from in-memory snapshots rather than the database: a small `createReloadingRegistry` helper loads each at startup and refreshes them on an interval, so no per-trigger query is added and a percentage or kill-switch change propagates within the reload interval. A cold replica whose snapshot hasn't loaded yet reads as not-migrated (the container path) and self-corrects on the next load - the same cold-start contract as the datastore / LLM-pricing registries, with a `reloading_registry_loaded` metric so a never-loaded registry is alertable. The same migration decision is consulted at deploy-time template creation so a migrated org gets a compute template built ahead of its first run. This runs in shadow mode (best-effort, never fails the deploy) by default, or - when the `computeMigrationRequireTemplate` flag is on - in required mode, built synchronously at deploy so the first run never builds on-demand and template errors surface at deploy time. So operators keep "which runs ran where" while customers only see geography: the run's actual worker queue is stored raw, and the geo region is stamped separately on `TaskRun.region` (and a new ClickHouse `region` column) at trigger time. Read surfaces - the dashboard, the API, and the Query/Logs page - show the geo region, falling back to the worker queue for runs written before the column existed. Minor follow-ups left out of scope: the percentage flags render as text inputs on the admin flags page (the catalog UI has no numeric control type yet), and `createReloadingRegistry` could later gain pub/sub for sub-second cross-replica propagation if the reload interval proves too slow.
80 lines
2.3 KiB
TypeScript
80 lines
2.3 KiB
TypeScript
import { describe, it, expect } from "vitest";
|
|
import { createReloadingRegistry } from "~/utils/reloadingRegistry.server";
|
|
|
|
describe("createReloadingRegistry", () => {
|
|
it("current() is undefined before load, snapshot after isReady", async () => {
|
|
const reg = createReloadingRegistry({
|
|
name: "test-a",
|
|
intervalMs: 10_000,
|
|
load: async () => ({ value: 42 }),
|
|
});
|
|
expect(reg.current()).toBeUndefined();
|
|
await reg.isReady;
|
|
expect(reg.isLoaded).toBe(true);
|
|
expect(reg.current()).toEqual({ value: 42 });
|
|
reg.stop();
|
|
});
|
|
|
|
it("reload() picks up a changed value", async () => {
|
|
let v = 1;
|
|
const reg = createReloadingRegistry({
|
|
name: "test-d",
|
|
intervalMs: 10_000,
|
|
load: async () => v,
|
|
});
|
|
await reg.isReady;
|
|
expect(reg.current()).toBe(1);
|
|
v = 2;
|
|
await reg.reload();
|
|
expect(reg.current()).toBe(2);
|
|
reg.stop();
|
|
});
|
|
|
|
it("newer load wins even if an older load resolves later", async () => {
|
|
// load hands the test a deferred resolver per call so completion order is controllable.
|
|
const deferred: Array<(value: number) => void> = [];
|
|
const reg = createReloadingRegistry({
|
|
name: "test-e",
|
|
intervalMs: 10_000,
|
|
load: () =>
|
|
new Promise<number>((resolve) => {
|
|
deferred.push(resolve);
|
|
}),
|
|
});
|
|
|
|
// deferred[0] is the startup load; let it complete with an initial value.
|
|
deferred[0](0);
|
|
await reg.isReady;
|
|
|
|
// start two overlapping loads; don't await yet (deferred[1] older, deferred[2] newer)
|
|
const older = reg.reload();
|
|
const newer = reg.reload();
|
|
|
|
// resolve the NEWER load first, then the OLDER load last
|
|
deferred[2](2);
|
|
deferred[1](1);
|
|
await Promise.all([older, newer]);
|
|
|
|
// the older load completing last must NOT clobber the newer snapshot
|
|
expect(reg.current()).toBe(2);
|
|
reg.stop();
|
|
});
|
|
|
|
it("autoStart:false stays inert (never loads)", async () => {
|
|
let loadCalls = 0;
|
|
const reg = createReloadingRegistry({
|
|
name: "test-inert",
|
|
intervalMs: 10_000,
|
|
autoStart: false,
|
|
load: async () => {
|
|
loadCalls++;
|
|
return 1;
|
|
},
|
|
});
|
|
expect(reg.isLoaded).toBe(false);
|
|
expect(reg.current()).toBeUndefined();
|
|
expect(loadCalls).toBe(0); // never hit the DB/load
|
|
reg.stop();
|
|
});
|
|
});
|