diff --git a/apps/webapp/app/v3/featureFlags.server.ts b/apps/webapp/app/v3/featureFlags.server.ts index e30b5df4c..e9aad8eb4 100644 --- a/apps/webapp/app/v3/featureFlags.server.ts +++ b/apps/webapp/app/v3/featureFlags.server.ts @@ -181,23 +181,21 @@ export function makeSetMultipleFlags(_prisma: PrismaClientOrTransaction = prisma }; } -// Read -> stamp -> write the global mint-kind grace metadata in one transaction. The three -// FeatureFlag rows may not exist yet, so a row FOR UPDATE can't lock them; an advisory xact lock -// serializes concurrent global flips so one can't clobber another's grace stamp (mirrors per-org). -// Every group of global flags whose value carries its own grace stamp. One transaction and one -// lock cover all of them, so a save that flips two groups can never stamp one and lose the other. +// Global flag groups whose value carries its own grace stamp. `primary` is operator-supplied; +// `derived` is computed server-side and must never be written from a request body. The pair is +// named explicitly rather than by position, so a group declared in another order stays correct. const GRACED_GLOBAL_GROUPS = [ { - keys: [ - FEATURE_FLAG.runOpsMintKind, + primary: FEATURE_FLAG.runOpsMintKind as FeatureFlagKey, + derived: [ FEATURE_FLAG.runOpsMintKindPrev, FEATURE_FLAG.runOpsMintKindFlippedAt, ] as FeatureFlagKey[], stamp: stampMintKindFlip, }, { - keys: [ - FEATURE_FLAG.runOpsMintShardSet, + primary: FEATURE_FLAG.runOpsMintShardSet as FeatureFlagKey, + derived: [ FEATURE_FLAG.runOpsMintShardSetPrev, FEATURE_FLAG.runOpsMintShardSetFlippedAt, ] as FeatureFlagKey[], @@ -205,51 +203,90 @@ const GRACED_GLOBAL_GROUPS = [ }, ] as const; -// Keys the graced path owns. They never take a bare upsert and never enter the replace sweep, -// because a server-computed stamp must not be written from a request body nor swept away. -const GRACED_GLOBAL_KEYS: FeatureFlagKey[] = GRACED_GLOBAL_GROUPS.flatMap((g) => g.keys); +const GRACED_GLOBAL_KEYS: FeatureFlagKey[] = GRACED_GLOBAL_GROUPS.flatMap((g) => [ + g.primary, + ...g.derived, +]); +function gracedGroupFor(key: FeatureFlagKey) { + return GRACED_GLOBAL_GROUPS.find((g) => g.primary === key || g.derived.includes(key)); +} + +// Strips every derived key: a grace stamp is computed here, never accepted from a caller. +function withoutDerivedKeys( + requestedFlags: Partial> +): Record { + const out: Record = { ...requestedFlags }; + for (const group of GRACED_GLOBAL_GROUPS) { + for (const derived of group.derived) { + delete out[derived]; + } + } + return out; +} + +// The rows may not exist yet, so a row FOR UPDATE cannot lock them; an advisory xact lock +// serializes concurrent global flips instead, so one cannot clobber another's stamp. +// +// Two lock ids are taken, in a fixed order. The second is this release's name; the first is the +// name an older release still takes. A deploy rolls for hours, so both versions write at once, +// and dropping the old id would leave those writers serializing against nothing. Remove the +// legacy id one release after this one ships. +async function lockGracedGroups(tx: PrismaClientOrTransaction): Promise { + await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtext('runops-global-mint-kind-flip'))`; + await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtext('runops-global-graced-flag-flip'))`; +} + +// Reads each group's current rows and returns the requested flags plus a fresh stamp for every +// group the save actually changes. A group whose primary the save omits is left untouched. +async function stampGracedGroups( + tx: PrismaClientOrTransaction, + requestedFlags: Record, + graceMs: number +): Promise> { + const existingRows = await tx.featureFlag.findMany({ + where: { key: { in: GRACED_GLOBAL_KEYS } }, + select: { key: true, value: true }, + }); + const existingGlobal: Record = {}; + for (const row of existingRows) { + existingGlobal[row.key] = row.value; + } + + // Anchor the cutover to the control-plane DB clock, not this process's wall clock. A rolling + // deploy spans hours, so every pod must date the window against one shared clock. + const [{ now }] = await tx.$queryRaw<{ now: Date }[]>`SELECT now() AS now`; + + let stamped: Record = { ...requestedFlags }; + for (const group of GRACED_GLOBAL_GROUPS) { + stamped = group.stamp(existingGlobal, stamped, now.getTime(), graceMs); + } + return stamped; +} + +// Merge-semantics write: sets what the caller asked for, stamps any graced group it changes, and +// touches nothing else. Used by the JSON admin API. export async function applyGlobalGracedFlips( client: PrismaClient, requestedFlags: Partial>, graceMs: number ): Promise<{ key: string; value: any }[]> { return client.$transaction(async (tx) => { - await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtext('runops-global-graced-flag-flip'))`; - - const existingRows = await tx.featureFlag.findMany({ - where: { key: { in: GRACED_GLOBAL_KEYS } }, - select: { key: true, value: true }, - }); - const existingGlobal: Record = {}; - for (const row of existingRows) { - existingGlobal[row.key] = row.value; - } - - // Anchor the cutover to the control-plane DB clock, not this process's wall clock. A rolling - // deploy spans hours, so every pod must date the window against one shared clock. - const [{ now }] = await tx.$queryRaw<{ now: Date }[]>`SELECT now() AS now`; - - let stamped: Record = { ...requestedFlags }; - for (const group of GRACED_GLOBAL_GROUPS) { - stamped = group.stamp(existingGlobal, stamped, now.getTime(), graceMs); - } - + await lockGracedGroups(tx); + const stamped = await stampGracedGroups(tx, withoutDerivedKeys(requestedFlags), graceMs); return makeSetMultipleFlags(tx)(stamped as Partial>); }); } -/** @deprecated Prefer applyGlobalGracedFlips, which stamps every graced group in one lock. */ -export async function applyGlobalMintKindFlip( - client: PrismaClient, - requestedFlags: Partial>, - graceMs: number -): Promise<{ key: string; value: any }[]> { - return applyGlobalGracedFlips(client, requestedFlags, graceMs); -} - -// Replace-semantics write for the global admin flags page: upsert submitted catalog flags, delete -// omitted ones unless protected, and route any graced group through the stamped path. +// Replace-semantics write for the global admin flags page: submitted flags upsert, omitted ones +// delete unless protected. One transaction covers the stamp, the upserts and the deletes, so a +// save cannot half-apply. +// +// A graced group is all-or-nothing. Submitting its primary writes the group with a fresh stamp. +// Omitting its primary deletes the primary AND its stamp together, because a stamp left behind +// without its primary keeps being served: {set: [], prevSet: [a], flippedAt: t} resolves to [a] +// for the rest of the window, which would mint into a shard the operator just removed. The +// delete ignores `isProtected` for the derived keys for the same reason. export async function replaceGlobalFeatureFlags( client: PrismaClient, params: { @@ -259,50 +296,40 @@ export async function replaceGlobalFeatureFlags( graceMs: number; } ): Promise { - // Derived stamp fields are computed server-side; never trust them from the body. - const requestedFlags: Record = { ...params.requestedFlags }; - for (const group of GRACED_GLOBAL_GROUPS) { - for (const derived of group.keys.slice(1)) { - delete requestedFlags[derived]; + const requestedFlags = withoutDerivedKeys(params.requestedFlags); + + await client.$transaction(async (tx) => { + await lockGracedGroups(tx); + const stamped = await stampGracedGroups(tx, requestedFlags, params.graceMs); + + const toWrite: Record = {}; + const keysToDelete: string[] = []; + + for (const key of params.catalogKeys) { + const group = gracedGroupFor(key); + + if (group) { + if (requestedFlags[group.primary] !== undefined) { + if (stamped[key] !== undefined) { + toWrite[key] = stamped[key]; + } + } else if (!params.isProtected(group.primary)) { + keysToDelete.push(key); + } + continue; + } + + if (key in requestedFlags) { + toWrite[key] = requestedFlags[key]; + } else if (!params.isProtected(key)) { + keysToDelete.push(key); + } } - } - const touchesGracedGroup = GRACED_GLOBAL_GROUPS.some( - (group) => requestedFlags[group.keys[0]] !== undefined - ); - if (touchesGracedGroup) { - await applyGlobalGracedFlips( - client, - requestedFlags as Partial>, - params.graceMs - ); - } + await makeSetMultipleFlags(tx)(toWrite as Partial>); - const upsertOps: ReturnType[] = []; - const keysToDelete: string[] = []; - - for (const key of params.catalogKeys) { - if (GRACED_GLOBAL_KEYS.includes(key)) { - continue; + if (keysToDelete.length > 0) { + await tx.featureFlag.deleteMany({ where: { key: { in: boundedIn(keysToDelete) } } }); } - if (key in requestedFlags) { - const value = requestedFlags[key]; - upsertOps.push( - client.featureFlag.upsert({ - where: { key }, - create: { key, value: value as any }, - update: { value: value as any }, - }) - ); - } else if (!params.isProtected(key)) { - keysToDelete.push(key); - } - } - - await client.$transaction([ - ...upsertOps, - ...(keysToDelete.length > 0 - ? [client.featureFlag.deleteMany({ where: { key: { in: boundedIn(keysToDelete) } } })] - : []), - ]); + }); } diff --git a/apps/webapp/app/v3/featureFlags.ts b/apps/webapp/app/v3/featureFlags.ts index ee4491e78..f241aaf4d 100644 --- a/apps/webapp/app/v3/featureFlags.ts +++ b/apps/webapp/app/v3/featureFlags.ts @@ -101,7 +101,7 @@ export const FeatureFlagCatalog = { [FEATURE_FLAG.runOpsMintKindFlippedAt]: z.string().datetime(), // Pins one org to a gen-2 mint shard. "new" holds the org on gen-1 run-ops ids, which is how // a canary keeps the fleet's default while one org moves. Only honored while the key is in - // the active set (RUN_OPS_MINT_SHARDS); a drained key falls through to the hash. + // the active list; a drained key falls through to the hash. [FEATURE_FLAG.runOpsMintShard]: z .string() .refine((v) => v === "new" || /^[a-z0-9]$/.test(v), 'must be a single [a-z0-9] char, or "new"'), @@ -125,8 +125,8 @@ export const FeatureFlagCatalog = { } } }), - // CSV of the shard keys eligible for root minting right now, bounded by RUN_OPS_MINT_SHARDS. - // Empty means no gen-2 minting. Reserved keys are rejected: "new" already means gen-1. + // CSV of the shard keys eligible for root minting right now. Empty means no gen-2 minting. + // Reserved keys are rejected, because "new" already means gen-1. [FEATURE_FLAG.runOpsMintShardSet]: z.string().refine( (v) => v @@ -166,6 +166,11 @@ export const GLOBAL_LOCKED_FLAGS: FeatureFlagKey[] = [ FEATURE_FLAG.taskEventRepository, FEATURE_FLAG.runOpsMintShard, FEATURE_FLAG.runOpsMintShardEnvPins, + // Grace stamps are computed server-side. An editable control here would discard what it saves. + FEATURE_FLAG.runOpsMintKindPrev, + FEATURE_FLAG.runOpsMintKindFlippedAt, + FEATURE_FLAG.runOpsMintShardSetPrev, + FEATURE_FLAG.runOpsMintShardSetFlippedAt, ]; // Flags that are read-only on the org-level dialog. diff --git a/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.test.ts b/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.test.ts index 771e52487..417ce8e26 100644 --- a/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.test.ts +++ b/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.test.ts @@ -51,6 +51,11 @@ describe("computeMintShard — the no-shards answer", () => { expect(computeMintShard({ id: "env_1" }, deps({ set: [] }))).toBe("new"); }); + it("returns new when a stale stamp is present but both lists are empty", () => { + const resolution: MintShardSetResolution = { set: [], prevSet: [], flippedAtMs: T }; + expect(computeMintShard({ id: "env_1" }, deps(resolution, { nowMs: T + 1 }))).toBe("new"); + }); + it("returns new when the grace serves an empty list", () => { const resolution: MintShardSetResolution = { set: ["a"], prevSet: [], flippedAtMs: T }; expect(computeMintShard({ id: "env_1" }, deps(resolution, { nowMs: T + 1 }))).toBe("new"); @@ -252,7 +257,7 @@ describe("computeMintShard — rendezvous properties", () => { }); }); -describe("resolveMintShardWith — cache, ceiling short-circuit and fail-safe", () => { +describe("resolveMintShardWith — cache, read failure and fail-safe", () => { function wrapperDeps( overrides: Partial = {} ): ResolveMintShardDeps & { reads: number } { @@ -374,13 +379,27 @@ describe("computeMintShard — the global override wins the complete cutover", ( deps(resolution, { globalOverride: "z", orgFeatureFlags: { runOpsMintShard: "a" }, - onPinRejected: (info) => rejected.push(info.pin), + onOverrideRejected: (info) => rejected.push(info.override), }) ); expect(shard).toBe("a"); expect(rejected).toEqual(["z"]); }); + it("reports a bad override WITHOUT the environment id, so one line covers the fleet", () => { + // Keying the report by environment would log once per environment for a fleet-wide setting. + const seen: Array<{ override: string }> = []; + for (const id of envIds(50)) { + computeMintShard( + { id }, + deps(resolution, { globalOverride: "z", onOverrideRejected: (i) => seen.push(i) }) + ); + } + expect(seen).toHaveLength(50); + expect(new Set(seen.map((i) => i.override))).toEqual(new Set(["z"])); + expect(seen.every((i) => !("environmentId" in i))).toBe(true); + }); + it("is ignored when it is not a legal value", () => { for (const bad of ["legacy", "AB", "", "a,b"]) { const shard = computeMintShard({ id: "env_1" }, deps(resolution, { globalOverride: bad })); diff --git a/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts b/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts index e7b9e2af8..02a432fb5 100644 --- a/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts @@ -3,6 +3,8 @@ import type { ShardKey } from "@trigger.dev/core/v3/isomorphic"; import { $replica } from "~/db.server"; import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; +import { BoundedTtlCache } from "~/services/realtime/boundedTtlCache"; +import { singleton } from "~/utils/singleton"; import { FEATURE_FLAG } from "~/v3/featureFlags"; import { effectiveMintShardSet, @@ -21,6 +23,7 @@ export type MintShardDeps = { graceMs: number; orgFeatureFlags: unknown; onPinRejected?: (info: { environmentId: string; pin: string; activeSet: string[] }) => void; + onOverrideRejected?: (info: { override: string; activeSet: string[] }) => void; }; function asRecord(value: unknown): Record | undefined { @@ -109,7 +112,8 @@ export function computeMintShard(environment: { id: string }, deps: MintShardDep if (activeSet.includes(override)) { return override; } - deps.onPinRejected?.({ environmentId: environment.id, pin: override, activeSet }); + // Fleet-wide, so it is reported once for the value, not once per environment. + deps.onOverrideRejected?.({ override, activeSet }); } const pin = readPin(deps.orgFeatureFlags, environment.id); @@ -138,6 +142,11 @@ type GlobalShardConfig = { resolution: MintShardSetResolution; override: unknown export type MintShardCache = { value: GlobalShardConfig; expiresAt: number } | undefined; +// A misconfiguration is reported again after this long, so a still-broken pin stays visible +// without logging on every trigger. +const REPORT_TTL_MS = 3_600_000; +const REPORT_MAX_ENTRIES = 10_000; + export type ResolveMintShardDeps = { // Reads the list rows. Injected so the cache and the fail-safe are testable without a // database, the same way computeRunIdMintKind takes its flag reader. @@ -148,12 +157,15 @@ export type ResolveMintShardDeps = { graceMs: number; orgFeatureFlags: unknown; onPinRejected?: (info: { environmentId: string; pin: string; activeSet: string[] }) => void; + onOverrideRejected?: (info: { override: string; activeSet: string[] }) => void; onReadFailed?: (error: unknown) => void; }; -// The live list is org-independent, so one process-wide entry serves every mint. One query per -// process per TTL, over one round-trip. The TTL bounds how long two processes can disagree, -// which is what the grace window is sized against. +// The live list is org-independent, so one process-wide entry serves every mint: one query per +// process per TTL, over one round-trip. Two processes can therefore disagree for the TTL PLUS the +// replica lag behind the read, which can exceed graceMs. That is tolerable here and only here, +// because a gen-2 id carries its own shard key, so disagreement cannot misroute an existing run; +// it only decides where the next root lands, and every failure direction is toward gen-1. // // A failed read falls back to gen-1 rather than guessing a list. Guessing would move every // environment's placement for the length of one blip. @@ -186,10 +198,13 @@ export async function resolveMintShardWith( graceMs: deps.graceMs, orgFeatureFlags: deps.orgFeatureFlags, onPinRejected: deps.onPinRejected, + onOverrideRejected: deps.onOverrideRejected, }); } -const liveCache: { current: MintShardCache } = { current: undefined }; +const liveCache = singleton("runOpsMintShardCache", (): { current: MintShardCache } => ({ + current: undefined, +})); async function readSetFlags(): Promise> { const rows = await $replica.featureFlag.findMany({ @@ -203,20 +218,37 @@ async function readSetFlags(): Promise> { return flags; } -// Once per environment per process: a stale pin sits on the root-trigger path and would -// otherwise log on every trigger for that environment, indefinitely. -const reportedPins = new Set(); +// A stale pin sits on the root-trigger path, so it would otherwise log on every trigger for that +// environment forever. Bounded, because the set of pinned environments is operator-controlled but +// not operator-bounded, and an unbounded Set on this path is a leak. +const reportedPins = singleton( + "runOpsMintShardReportedPins", + () => new BoundedTtlCache(REPORT_TTL_MS, REPORT_MAX_ENTRIES) +); function reportPinRejected(info: { environmentId: string; pin: string; activeSet: string[]; }): void { - if (reportedPins.has(info.environmentId)) return; - reportedPins.add(info.environmentId); + if (reportedPins.get(info.environmentId) !== undefined) return; + reportedPins.set(info.environmentId, true); logger.error("[runOpsMintShard] pinned shard is not in the active set; using the hash", info); } +// Keyed by the override value, not by environment: one bad override applies to the whole fleet, +// so one line is the correct volume. Keying by environment would log once per environment. +const reportedOverrides = singleton( + "runOpsMintShardReportedOverrides", + () => new BoundedTtlCache(REPORT_TTL_MS, REPORT_MAX_ENTRIES) +); + +function reportOverrideRejected(info: { override: string; activeSet: string[] }): void { + if (reportedOverrides.get(info.override) !== undefined) return; + reportedOverrides.set(info.override, true); + logger.error("[runOpsMintShard] override shard is not in the active set; ignoring it", info); +} + /** * Which shard an environment mints new roots into. Call only after resolveRunIdMintKind has * returned "runOpsId". Returns "new" to mean a gen-1 run-ops id, which is today's behaviour. @@ -236,6 +268,7 @@ export async function resolveMintShard(environment: { graceMs: env.RUN_OPS_MINT_FLIP_GRACE_MS, orgFeatureFlags: environment.orgFeatureFlags, onPinRejected: reportPinRejected, + onOverrideRejected: reportOverrideRejected, onReadFailed: (error) => logger.error("[runOpsMintShard] shard-set read failed; minting gen-1 (fail-safe)", { error }), }); diff --git a/apps/webapp/test/runOpsMintGlobalFlipLock.test.ts b/apps/webapp/test/runOpsMintGlobalFlipLock.test.ts index 1492fd02c..b3acd7e45 100644 --- a/apps/webapp/test/runOpsMintGlobalFlipLock.test.ts +++ b/apps/webapp/test/runOpsMintGlobalFlipLock.test.ts @@ -5,7 +5,7 @@ import type { PrismaClient } from "@trigger.dev/database"; import { postgresTest } from "@internal/testcontainers"; import { describe, expect, vi } from "vitest"; import { FEATURE_FLAG } from "~/v3/featureFlags"; -import { applyGlobalMintKindFlip, makeSetMultipleFlags } from "~/v3/featureFlags.server"; +import { applyGlobalGracedFlips, makeSetMultipleFlags } from "~/v3/featureFlags.server"; vi.setConfig({ testTimeout: 60_000 }); @@ -25,11 +25,11 @@ async function readGlobalMint(prisma: PrismaClient): Promise { +describe("applyGlobalGracedFlips — transactional stamp + serialized flips", () => { postgresTest("a genuine global flip stamps prev + flippedAt", async ({ prisma }) => { await makeSetMultipleFlags(prisma)({ [FEATURE_FLAG.runOpsMintKind]: "cuid" }); - await applyGlobalMintKindFlip(prisma, { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, 60_000); + await applyGlobalGracedFlips(prisma, { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, 60_000); const m = await readGlobalMint(prisma); expect(m[FEATURE_FLAG.runOpsMintKind]).toBe("runOpsId"); @@ -44,7 +44,7 @@ describe("applyGlobalMintKindFlip — transactional stamp + serialized flips", ( await Promise.all( Array.from({ length: 8 }, () => - applyGlobalMintKindFlip(prisma, { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, 60_000) + applyGlobalGracedFlips(prisma, { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, 60_000) ) ); diff --git a/apps/webapp/test/runOpsMintShardSetFlip.test.ts b/apps/webapp/test/runOpsMintShardSetFlip.test.ts index 55ebe43a2..1cc3c2e00 100644 --- a/apps/webapp/test/runOpsMintShardSetFlip.test.ts +++ b/apps/webapp/test/runOpsMintShardSetFlip.test.ts @@ -174,6 +174,73 @@ describe("replaceGlobalFeatureFlags — the admin page cannot bypass the stamp", expect(m[FEATURE_FLAG.runOpsMintShardSet]).toBe("a,b"); }); + postgresTest( + "omitting the list DELETES it, so unset still turns minting off", + async ({ prisma }) => { + // The admin page's unset button omits the key. If the save skipped it, unset would be a + // silent no-op and gen-2 minting would stay armed. + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { [FEATURE_FLAG.runOpsMintShardSet]: "a,b" }, + catalogKeys: CATALOG_KEYS, + isProtected: NEVER_PROTECTED, + graceMs: 60_000, + }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: {}, + catalogKeys: CATALOG_KEYS, + isProtected: NEVER_PROTECTED, + graceMs: 60_000, + }); + + const m = await readFlags(prisma, SET_KEYS); + // The stamp goes with it: a stamp without its list keeps being served for the whole window. + expect(m[FEATURE_FLAG.runOpsMintShardSet]).toBeUndefined(); + expect(m[FEATURE_FLAG.runOpsMintShardSetPrev]).toBeUndefined(); + expect(m[FEATURE_FLAG.runOpsMintShardSetFlippedAt]).toBeUndefined(); + } + ); + + postgresTest("omitting the mint kind still deletes its trio", async ({ prisma }) => { + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, + catalogKeys: CATALOG_KEYS, + isProtected: NEVER_PROTECTED, + graceMs: 60_000, + }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: {}, + catalogKeys: CATALOG_KEYS, + isProtected: NEVER_PROTECTED, + graceMs: 60_000, + }); + + const m = await readFlags(prisma, MINT_KIND_KEYS); + expect(m[FEATURE_FLAG.runOpsMintKind]).toBeUndefined(); + expect(m[FEATURE_FLAG.runOpsMintKindPrev]).toBeUndefined(); + expect(m[FEATURE_FLAG.runOpsMintKindFlippedAt]).toBeUndefined(); + }); + + postgresTest("a protected list is not deleted when omitted", async ({ prisma }) => { + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: { [FEATURE_FLAG.runOpsMintShardSet]: "a" }, + catalogKeys: CATALOG_KEYS, + isProtected: NEVER_PROTECTED, + graceMs: 60_000, + }); + + await replaceGlobalFeatureFlags(prisma, { + requestedFlags: {}, + catalogKeys: CATALOG_KEYS, + isProtected: (key) => key === FEATURE_FLAG.runOpsMintShardSet, + graceMs: 60_000, + }); + + const m = await readFlags(prisma, SET_KEYS); + expect(m[FEATURE_FLAG.runOpsMintShardSet]).toBe("a"); + }); + postgresTest("an ordinary flag keeps replace semantics", async ({ prisma }) => { await makeSetMultipleFlags(prisma)({ [FEATURE_FLAG.mollifierEnabled]: true });