fix(webapp): keep unset working on the graced global flags

Review found that this branch silently disabled the unset button for
runOpsMintKind on the global admin flags page. The graced keys were skipped
by the replace sweep so their stamp could not be bare-written, but that skip
covered the operator-supplied key as well as the server-computed ones. The
page omits a key to unset it, so the omission was read as "leave alone" and
the row survived. Before this branch the same gesture deleted it.

A graced group is now all-or-nothing. Submitting its primary writes the group
with a fresh stamp. Omitting the primary deletes the primary and its stamp
together, because a stamp left behind without its primary keeps being served:
an empty list beside a live prev list still resolves to the prev list for the
rest of the window, which would mint into a shard just removed.

Also from review:

- The whole save is one transaction again. The stamp, the upserts and the
  deletes could previously half-apply across two.
- The advisory lock takes the previous id as well as the current one, in a
  fixed order. A deploy rolls for hours, so renaming it left writers on the
  older release serializing against nothing. Drop the legacy id next release.
- A bad global override is reported once per value rather than once per
  environment. It applies to the whole fleet, so keying the report by
  environment turned one misconfiguration into a log line and a retained set
  entry per environment, on the trigger path. Both reporters are bounded now.
- The stamp keys render read-only. They were editable controls whose values
  were discarded on save.
- Groups name their primary and derived keys instead of relying on position.
- Corrected a claim in a comment: the cache TTL does not bound cross-process
  disagreement on its own, because the read goes to a replica. Stated why
  that is tolerable here specifically.
- The deprecated single-group entry point is gone; its test now covers the
  grouped one.

apps/webapp/app/v3/featureFlags.server.ts

@@ -181,23 +181,21 @@ export function makeSetMultipleFlags(_prisma: PrismaClientOrTransaction = prisma
   };
 }
 
-// Read -> stamp -> write the global mint-kind grace metadata in one transaction. The three
-// FeatureFlag rows may not exist yet, so a row FOR UPDATE can't lock them; an advisory xact lock
-// serializes concurrent global flips so one can't clobber another's grace stamp (mirrors per-org).
-// Every group of global flags whose value carries its own grace stamp. One transaction and one
-// lock cover all of them, so a save that flips two groups can never stamp one and lose the other.
+// Global flag groups whose value carries its own grace stamp. `primary` is operator-supplied;
+// `derived` is computed server-side and must never be written from a request body. The pair is
+// named explicitly rather than by position, so a group declared in another order stays correct.
 const GRACED_GLOBAL_GROUPS = [
   {
-    keys: [
-      FEATURE_FLAG.runOpsMintKind,
+    primary: FEATURE_FLAG.runOpsMintKind as FeatureFlagKey,
+    derived: [
       FEATURE_FLAG.runOpsMintKindPrev,
       FEATURE_FLAG.runOpsMintKindFlippedAt,
     ] as FeatureFlagKey[],
     stamp: stampMintKindFlip,
   },
   {
-    keys: [
-      FEATURE_FLAG.runOpsMintShardSet,
+    primary: FEATURE_FLAG.runOpsMintShardSet as FeatureFlagKey,
+    derived: [
       FEATURE_FLAG.runOpsMintShardSetPrev,
       FEATURE_FLAG.runOpsMintShardSetFlippedAt,
     ] as FeatureFlagKey[],
@@ -205,51 +203,90 @@ const GRACED_GLOBAL_GROUPS = [
   },
 ] as const;
 
-// Keys the graced path owns. They never take a bare upsert and never enter the replace sweep,
-// because a server-computed stamp must not be written from a request body nor swept away.
-const GRACED_GLOBAL_KEYS: FeatureFlagKey[] = GRACED_GLOBAL_GROUPS.flatMap((g) => g.keys);
+const GRACED_GLOBAL_KEYS: FeatureFlagKey[] = GRACED_GLOBAL_GROUPS.flatMap((g) => [
+  g.primary,
+  ...g.derived,
+]);
 
+function gracedGroupFor(key: FeatureFlagKey) {
+  return GRACED_GLOBAL_GROUPS.find((g) => g.primary === key || g.derived.includes(key));
+}
+
+// Strips every derived key: a grace stamp is computed here, never accepted from a caller.
+function withoutDerivedKeys(
+  requestedFlags: Partial<z.infer<typeof FeatureFlagCatalogSchema>>
+): Record<string, unknown> {
+  const out: Record<string, unknown> = { ...requestedFlags };
+  for (const group of GRACED_GLOBAL_GROUPS) {
+    for (const derived of group.derived) {
+      delete out[derived];
+    }
+  }
+  return out;
+}
+
+// The rows may not exist yet, so a row FOR UPDATE cannot lock them; an advisory xact lock
+// serializes concurrent global flips instead, so one cannot clobber another's stamp.
+//
+// Two lock ids are taken, in a fixed order. The second is this release's name; the first is the
+// name an older release still takes. A deploy rolls for hours, so both versions write at once,
+// and dropping the old id would leave those writers serializing against nothing. Remove the
+// legacy id one release after this one ships.
+async function lockGracedGroups(tx: PrismaClientOrTransaction): Promise<void> {
+  await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtext('runops-global-mint-kind-flip'))`;
+  await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtext('runops-global-graced-flag-flip'))`;
+}
+
+// Reads each group's current rows and returns the requested flags plus a fresh stamp for every
+// group the save actually changes. A group whose primary the save omits is left untouched.
+async function stampGracedGroups(
+  tx: PrismaClientOrTransaction,
+  requestedFlags: Record<string, unknown>,
+  graceMs: number
+): Promise<Record<string, unknown>> {
+  const existingRows = await tx.featureFlag.findMany({
+    where: { key: { in: GRACED_GLOBAL_KEYS } },
+    select: { key: true, value: true },
+  });
+  const existingGlobal: Record<string, unknown> = {};
+  for (const row of existingRows) {
+    existingGlobal[row.key] = row.value;
+  }
+
+  // Anchor the cutover to the control-plane DB clock, not this process's wall clock. A rolling
+  // deploy spans hours, so every pod must date the window against one shared clock.
+  const [{ now }] = await tx.$queryRaw<{ now: Date }[]>`SELECT now() AS now`;
+
+  let stamped: Record<string, unknown> = { ...requestedFlags };
+  for (const group of GRACED_GLOBAL_GROUPS) {
+    stamped = group.stamp(existingGlobal, stamped, now.getTime(), graceMs);
+  }
+  return stamped;
+}
+
+// Merge-semantics write: sets what the caller asked for, stamps any graced group it changes, and
+// touches nothing else. Used by the JSON admin API.
 export async function applyGlobalGracedFlips(
   client: PrismaClient,
   requestedFlags: Partial<z.infer<typeof FeatureFlagCatalogSchema>>,
   graceMs: number
 ): Promise<{ key: string; value: any }[]> {
   return client.$transaction(async (tx) => {
-    await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtext('runops-global-graced-flag-flip'))`;
-
-    const existingRows = await tx.featureFlag.findMany({
-      where: { key: { in: GRACED_GLOBAL_KEYS } },
-      select: { key: true, value: true },
-    });
-    const existingGlobal: Record<string, unknown> = {};
-    for (const row of existingRows) {
-      existingGlobal[row.key] = row.value;
-    }
-
-    // Anchor the cutover to the control-plane DB clock, not this process's wall clock. A rolling
-    // deploy spans hours, so every pod must date the window against one shared clock.
-    const [{ now }] = await tx.$queryRaw<{ now: Date }[]>`SELECT now() AS now`;
-
-    let stamped: Record<string, unknown> = { ...requestedFlags };
-    for (const group of GRACED_GLOBAL_GROUPS) {
-      stamped = group.stamp(existingGlobal, stamped, now.getTime(), graceMs);
-    }
-
+    await lockGracedGroups(tx);
+    const stamped = await stampGracedGroups(tx, withoutDerivedKeys(requestedFlags), graceMs);
     return makeSetMultipleFlags(tx)(stamped as Partial<z.infer<typeof FeatureFlagCatalogSchema>>);
   });
 }
 
-/** @deprecated Prefer applyGlobalGracedFlips, which stamps every graced group in one lock. */
-export async function applyGlobalMintKindFlip(
-  client: PrismaClient,
-  requestedFlags: Partial<z.infer<typeof FeatureFlagCatalogSchema>>,
-  graceMs: number
-): Promise<{ key: string; value: any }[]> {
-  return applyGlobalGracedFlips(client, requestedFlags, graceMs);
-}
-
-// Replace-semantics write for the global admin flags page: upsert submitted catalog flags, delete
-// omitted ones unless protected, and route any graced group through the stamped path.
+// Replace-semantics write for the global admin flags page: submitted flags upsert, omitted ones
+// delete unless protected. One transaction covers the stamp, the upserts and the deletes, so a
+// save cannot half-apply.
+//
+// A graced group is all-or-nothing. Submitting its primary writes the group with a fresh stamp.
+// Omitting its primary deletes the primary AND its stamp together, because a stamp left behind
+// without its primary keeps being served: {set: [], prevSet: [a], flippedAt: t} resolves to [a]
+// for the rest of the window, which would mint into a shard the operator just removed. The
+// delete ignores `isProtected` for the derived keys for the same reason.
 export async function replaceGlobalFeatureFlags(
   client: PrismaClient,
   params: {
@@ -259,50 +296,40 @@ export async function replaceGlobalFeatureFlags(
     graceMs: number;
   }
 ): Promise<void> {
-  // Derived stamp fields are computed server-side; never trust them from the body.
-  const requestedFlags: Record<string, unknown> = { ...params.requestedFlags };
-  for (const group of GRACED_GLOBAL_GROUPS) {
-    for (const derived of group.keys.slice(1)) {
-      delete requestedFlags[derived];
+  const requestedFlags = withoutDerivedKeys(params.requestedFlags);
+
+  await client.$transaction(async (tx) => {
+    await lockGracedGroups(tx);
+    const stamped = await stampGracedGroups(tx, requestedFlags, params.graceMs);
+
+    const toWrite: Record<string, unknown> = {};
+    const keysToDelete: string[] = [];
+
+    for (const key of params.catalogKeys) {
+      const group = gracedGroupFor(key);
+
+      if (group) {
+        if (requestedFlags[group.primary] !== undefined) {
+          if (stamped[key] !== undefined) {
+            toWrite[key] = stamped[key];
+          }
+        } else if (!params.isProtected(group.primary)) {
+          keysToDelete.push(key);
+        }
+        continue;
+      }
+
+      if (key in requestedFlags) {
+        toWrite[key] = requestedFlags[key];
+      } else if (!params.isProtected(key)) {
+        keysToDelete.push(key);
+      }
     }
-  }
 
-  const touchesGracedGroup = GRACED_GLOBAL_GROUPS.some(
-    (group) => requestedFlags[group.keys[0]] !== undefined
-  );
-  if (touchesGracedGroup) {
-    await applyGlobalGracedFlips(
-      client,
-      requestedFlags as Partial<z.infer<typeof FeatureFlagCatalogSchema>>,
-      params.graceMs
-    );
-  }
+    await makeSetMultipleFlags(tx)(toWrite as Partial<z.infer<typeof FeatureFlagCatalogSchema>>);
 
-  const upsertOps: ReturnType<typeof client.featureFlag.upsert>[] = [];
-  const keysToDelete: string[] = [];
-
-  for (const key of params.catalogKeys) {
-    if (GRACED_GLOBAL_KEYS.includes(key)) {
-      continue;
+    if (keysToDelete.length > 0) {
+      await tx.featureFlag.deleteMany({ where: { key: { in: boundedIn(keysToDelete) } } });
     }
-    if (key in requestedFlags) {
-      const value = requestedFlags[key];
-      upsertOps.push(
-        client.featureFlag.upsert({
-          where: { key },
-          create: { key, value: value as any },
-          update: { value: value as any },
-        })
-      );
-    } else if (!params.isProtected(key)) {
-      keysToDelete.push(key);
-    }
-  }
-
-  await client.$transaction([
-    ...upsertOps,
-    ...(keysToDelete.length > 0
-      ? [client.featureFlag.deleteMany({ where: { key: { in: boundedIn(keysToDelete) } } })]
-      : []),
-  ]);
+  });
 }

apps/webapp/app/v3/featureFlags.ts

@@ -101,7 +101,7 @@ export const FeatureFlagCatalog = {
   [FEATURE_FLAG.runOpsMintKindFlippedAt]: z.string().datetime(),
   // Pins one org to a gen-2 mint shard. "new" holds the org on gen-1 run-ops ids, which is how
   // a canary keeps the fleet's default while one org moves. Only honored while the key is in
-  // the active set (RUN_OPS_MINT_SHARDS); a drained key falls through to the hash.
+  // the active list; a drained key falls through to the hash.
   [FEATURE_FLAG.runOpsMintShard]: z
     .string()
     .refine((v) => v === "new" || /^[a-z0-9]$/.test(v), 'must be a single [a-z0-9] char, or "new"'),
@@ -125,8 +125,8 @@ export const FeatureFlagCatalog = {
       }
     }
   }),
-  // CSV of the shard keys eligible for root minting right now, bounded by RUN_OPS_MINT_SHARDS.
-  // Empty means no gen-2 minting. Reserved keys are rejected: "new" already means gen-1.
+  // CSV of the shard keys eligible for root minting right now. Empty means no gen-2 minting.
+  // Reserved keys are rejected, because "new" already means gen-1.
   [FEATURE_FLAG.runOpsMintShardSet]: z.string().refine(
     (v) =>
       v
@@ -166,6 +166,11 @@ export const GLOBAL_LOCKED_FLAGS: FeatureFlagKey[] = [
   FEATURE_FLAG.taskEventRepository,
   FEATURE_FLAG.runOpsMintShard,
   FEATURE_FLAG.runOpsMintShardEnvPins,
+  // Grace stamps are computed server-side. An editable control here would discard what it saves.
+  FEATURE_FLAG.runOpsMintKindPrev,
+  FEATURE_FLAG.runOpsMintKindFlippedAt,
+  FEATURE_FLAG.runOpsMintShardSetPrev,
+  FEATURE_FLAG.runOpsMintShardSetFlippedAt,
 ];
 
 // Flags that are read-only on the org-level dialog.

apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.test.ts

@@ -51,6 +51,11 @@ describe("computeMintShard — the no-shards answer", () => {
     expect(computeMintShard({ id: "env_1" }, deps({ set: [] }))).toBe("new");
   });
 
+  it("returns new when a stale stamp is present but both lists are empty", () => {
+    const resolution: MintShardSetResolution = { set: [], prevSet: [], flippedAtMs: T };
+    expect(computeMintShard({ id: "env_1" }, deps(resolution, { nowMs: T + 1 }))).toBe("new");
+  });
+
   it("returns new when the grace serves an empty list", () => {
     const resolution: MintShardSetResolution = { set: ["a"], prevSet: [], flippedAtMs: T };
     expect(computeMintShard({ id: "env_1" }, deps(resolution, { nowMs: T + 1 }))).toBe("new");
@@ -252,7 +257,7 @@ describe("computeMintShard — rendezvous properties", () => {
   });
 });
 
-describe("resolveMintShardWith — cache, ceiling short-circuit and fail-safe", () => {
+describe("resolveMintShardWith — cache, read failure and fail-safe", () => {
   function wrapperDeps(
     overrides: Partial<ResolveMintShardDeps> = {}
   ): ResolveMintShardDeps & { reads: number } {
@@ -374,13 +379,27 @@ describe("computeMintShard — the global override wins the complete cutover", (
       deps(resolution, {
         globalOverride: "z",
         orgFeatureFlags: { runOpsMintShard: "a" },
-        onPinRejected: (info) => rejected.push(info.pin),
+        onOverrideRejected: (info) => rejected.push(info.override),
       })
     );
     expect(shard).toBe("a");
     expect(rejected).toEqual(["z"]);
   });
 
+  it("reports a bad override WITHOUT the environment id, so one line covers the fleet", () => {
+    // Keying the report by environment would log once per environment for a fleet-wide setting.
+    const seen: Array<{ override: string }> = [];
+    for (const id of envIds(50)) {
+      computeMintShard(
+        { id },
+        deps(resolution, { globalOverride: "z", onOverrideRejected: (i) => seen.push(i) })
+      );
+    }
+    expect(seen).toHaveLength(50);
+    expect(new Set(seen.map((i) => i.override))).toEqual(new Set(["z"]));
+    expect(seen.every((i) => !("environmentId" in i))).toBe(true);
+  });
+
   it("is ignored when it is not a legal value", () => {
     for (const bad of ["legacy", "AB", "", "a,b"]) {
       const shard = computeMintShard({ id: "env_1" }, deps(resolution, { globalOverride: bad }));

apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts

@@ -3,6 +3,8 @@ import type { ShardKey } from "@trigger.dev/core/v3/isomorphic";
 import { $replica } from "~/db.server";
 import { env } from "~/env.server";
 import { logger } from "~/services/logger.server";
+import { BoundedTtlCache } from "~/services/realtime/boundedTtlCache";
+import { singleton } from "~/utils/singleton";
 import { FEATURE_FLAG } from "~/v3/featureFlags";
 import {
   effectiveMintShardSet,
@@ -21,6 +23,7 @@ export type MintShardDeps = {
   graceMs: number;
   orgFeatureFlags: unknown;
   onPinRejected?: (info: { environmentId: string; pin: string; activeSet: string[] }) => void;
+  onOverrideRejected?: (info: { override: string; activeSet: string[] }) => void;
 };
 
 function asRecord(value: unknown): Record<string, unknown> | undefined {
@@ -109,7 +112,8 @@ export function computeMintShard(environment: { id: string }, deps: MintShardDep
     if (activeSet.includes(override)) {
       return override;
     }
-    deps.onPinRejected?.({ environmentId: environment.id, pin: override, activeSet });
+    // Fleet-wide, so it is reported once for the value, not once per environment.
+    deps.onOverrideRejected?.({ override, activeSet });
   }
 
   const pin = readPin(deps.orgFeatureFlags, environment.id);
@@ -138,6 +142,11 @@ type GlobalShardConfig = { resolution: MintShardSetResolution; override: unknown
 
 export type MintShardCache = { value: GlobalShardConfig; expiresAt: number } | undefined;
 
+// A misconfiguration is reported again after this long, so a still-broken pin stays visible
+// without logging on every trigger.
+const REPORT_TTL_MS = 3_600_000;
+const REPORT_MAX_ENTRIES = 10_000;
+
 export type ResolveMintShardDeps = {
   // Reads the list rows. Injected so the cache and the fail-safe are testable without a
   // database, the same way computeRunIdMintKind takes its flag reader.
@@ -148,12 +157,15 @@ export type ResolveMintShardDeps = {
   graceMs: number;
   orgFeatureFlags: unknown;
   onPinRejected?: (info: { environmentId: string; pin: string; activeSet: string[] }) => void;
+  onOverrideRejected?: (info: { override: string; activeSet: string[] }) => void;
   onReadFailed?: (error: unknown) => void;
 };
 
-// The live list is org-independent, so one process-wide entry serves every mint. One query per
-// process per TTL, over one round-trip. The TTL bounds how long two processes can disagree,
-// which is what the grace window is sized against.
+// The live list is org-independent, so one process-wide entry serves every mint: one query per
+// process per TTL, over one round-trip. Two processes can therefore disagree for the TTL PLUS the
+// replica lag behind the read, which can exceed graceMs. That is tolerable here and only here,
+// because a gen-2 id carries its own shard key, so disagreement cannot misroute an existing run;
+// it only decides where the next root lands, and every failure direction is toward gen-1.
 //
 // A failed read falls back to gen-1 rather than guessing a list. Guessing would move every
 // environment's placement for the length of one blip.
@@ -186,10 +198,13 @@ export async function resolveMintShardWith(
     graceMs: deps.graceMs,
     orgFeatureFlags: deps.orgFeatureFlags,
     onPinRejected: deps.onPinRejected,
+    onOverrideRejected: deps.onOverrideRejected,
   });
 }
 
-const liveCache: { current: MintShardCache } = { current: undefined };
+const liveCache = singleton("runOpsMintShardCache", (): { current: MintShardCache } => ({
+  current: undefined,
+}));
 
 async function readSetFlags(): Promise<Record<string, unknown>> {
   const rows = await $replica.featureFlag.findMany({
@@ -203,20 +218,37 @@ async function readSetFlags(): Promise<Record<string, unknown>> {
   return flags;
 }
 
-// Once per environment per process: a stale pin sits on the root-trigger path and would
-// otherwise log on every trigger for that environment, indefinitely.
-const reportedPins = new Set<string>();
+// A stale pin sits on the root-trigger path, so it would otherwise log on every trigger for that
+// environment forever. Bounded, because the set of pinned environments is operator-controlled but
+// not operator-bounded, and an unbounded Set on this path is a leak.
+const reportedPins = singleton(
+  "runOpsMintShardReportedPins",
+  () => new BoundedTtlCache<true>(REPORT_TTL_MS, REPORT_MAX_ENTRIES)
+);
 
 function reportPinRejected(info: {
   environmentId: string;
   pin: string;
   activeSet: string[];
 }): void {
-  if (reportedPins.has(info.environmentId)) return;
-  reportedPins.add(info.environmentId);
+  if (reportedPins.get(info.environmentId) !== undefined) return;
+  reportedPins.set(info.environmentId, true);
   logger.error("[runOpsMintShard] pinned shard is not in the active set; using the hash", info);
 }
 
+// Keyed by the override value, not by environment: one bad override applies to the whole fleet,
+// so one line is the correct volume. Keying by environment would log once per environment.
+const reportedOverrides = singleton(
+  "runOpsMintShardReportedOverrides",
+  () => new BoundedTtlCache<true>(REPORT_TTL_MS, REPORT_MAX_ENTRIES)
+);
+
+function reportOverrideRejected(info: { override: string; activeSet: string[] }): void {
+  if (reportedOverrides.get(info.override) !== undefined) return;
+  reportedOverrides.set(info.override, true);
+  logger.error("[runOpsMintShard] override shard is not in the active set; ignoring it", info);
+}
+
 /**
  * Which shard an environment mints new roots into. Call only after resolveRunIdMintKind has
  * returned "runOpsId". Returns "new" to mean a gen-1 run-ops id, which is today's behaviour.
@@ -236,6 +268,7 @@ export async function resolveMintShard(environment: {
     graceMs: env.RUN_OPS_MINT_FLIP_GRACE_MS,
     orgFeatureFlags: environment.orgFeatureFlags,
     onPinRejected: reportPinRejected,
+    onOverrideRejected: reportOverrideRejected,
     onReadFailed: (error) =>
       logger.error("[runOpsMintShard] shard-set read failed; minting gen-1 (fail-safe)", { error }),
   });

apps/webapp/test/runOpsMintGlobalFlipLock.test.ts

@@ -5,7 +5,7 @@ import type { PrismaClient } from "@trigger.dev/database";
 import { postgresTest } from "@internal/testcontainers";
 import { describe, expect, vi } from "vitest";
 import { FEATURE_FLAG } from "~/v3/featureFlags";
-import { applyGlobalMintKindFlip, makeSetMultipleFlags } from "~/v3/featureFlags.server";
+import { applyGlobalGracedFlips, makeSetMultipleFlags } from "~/v3/featureFlags.server";
 
 vi.setConfig({ testTimeout: 60_000 });
 
@@ -25,11 +25,11 @@ async function readGlobalMint(prisma: PrismaClient): Promise<Record<string, unkn
   return m;
 }
 
-describe("applyGlobalMintKindFlip — transactional stamp + serialized flips", () => {
+describe("applyGlobalGracedFlips — transactional stamp + serialized flips", () => {
   postgresTest("a genuine global flip stamps prev + flippedAt", async ({ prisma }) => {
     await makeSetMultipleFlags(prisma)({ [FEATURE_FLAG.runOpsMintKind]: "cuid" });
 
-    await applyGlobalMintKindFlip(prisma, { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, 60_000);
+    await applyGlobalGracedFlips(prisma, { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, 60_000);
 
     const m = await readGlobalMint(prisma);
     expect(m[FEATURE_FLAG.runOpsMintKind]).toBe("runOpsId");
@@ -44,7 +44,7 @@ describe("applyGlobalMintKindFlip — transactional stamp + serialized flips", (
 
       await Promise.all(
         Array.from({ length: 8 }, () =>
-          applyGlobalMintKindFlip(prisma, { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, 60_000)
+          applyGlobalGracedFlips(prisma, { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, 60_000)
         )
       );
 

apps/webapp/test/runOpsMintShardSetFlip.test.ts

@@ -174,6 +174,73 @@ describe("replaceGlobalFeatureFlags — the admin page cannot bypass the stamp",
     expect(m[FEATURE_FLAG.runOpsMintShardSet]).toBe("a,b");
   });
 
+  postgresTest(
+    "omitting the list DELETES it, so unset still turns minting off",
+    async ({ prisma }) => {
+      // The admin page's unset button omits the key. If the save skipped it, unset would be a
+      // silent no-op and gen-2 minting would stay armed.
+      await replaceGlobalFeatureFlags(prisma, {
+        requestedFlags: { [FEATURE_FLAG.runOpsMintShardSet]: "a,b" },
+        catalogKeys: CATALOG_KEYS,
+        isProtected: NEVER_PROTECTED,
+        graceMs: 60_000,
+      });
+
+      await replaceGlobalFeatureFlags(prisma, {
+        requestedFlags: {},
+        catalogKeys: CATALOG_KEYS,
+        isProtected: NEVER_PROTECTED,
+        graceMs: 60_000,
+      });
+
+      const m = await readFlags(prisma, SET_KEYS);
+      // The stamp goes with it: a stamp without its list keeps being served for the whole window.
+      expect(m[FEATURE_FLAG.runOpsMintShardSet]).toBeUndefined();
+      expect(m[FEATURE_FLAG.runOpsMintShardSetPrev]).toBeUndefined();
+      expect(m[FEATURE_FLAG.runOpsMintShardSetFlippedAt]).toBeUndefined();
+    }
+  );
+
+  postgresTest("omitting the mint kind still deletes its trio", async ({ prisma }) => {
+    await replaceGlobalFeatureFlags(prisma, {
+      requestedFlags: { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" },
+      catalogKeys: CATALOG_KEYS,
+      isProtected: NEVER_PROTECTED,
+      graceMs: 60_000,
+    });
+
+    await replaceGlobalFeatureFlags(prisma, {
+      requestedFlags: {},
+      catalogKeys: CATALOG_KEYS,
+      isProtected: NEVER_PROTECTED,
+      graceMs: 60_000,
+    });
+
+    const m = await readFlags(prisma, MINT_KIND_KEYS);
+    expect(m[FEATURE_FLAG.runOpsMintKind]).toBeUndefined();
+    expect(m[FEATURE_FLAG.runOpsMintKindPrev]).toBeUndefined();
+    expect(m[FEATURE_FLAG.runOpsMintKindFlippedAt]).toBeUndefined();
+  });
+
+  postgresTest("a protected list is not deleted when omitted", async ({ prisma }) => {
+    await replaceGlobalFeatureFlags(prisma, {
+      requestedFlags: { [FEATURE_FLAG.runOpsMintShardSet]: "a" },
+      catalogKeys: CATALOG_KEYS,
+      isProtected: NEVER_PROTECTED,
+      graceMs: 60_000,
+    });
+
+    await replaceGlobalFeatureFlags(prisma, {
+      requestedFlags: {},
+      catalogKeys: CATALOG_KEYS,
+      isProtected: (key) => key === FEATURE_FLAG.runOpsMintShardSet,
+      graceMs: 60_000,
+    });
+
+    const m = await readFlags(prisma, SET_KEYS);
+    expect(m[FEATURE_FLAG.runOpsMintShardSet]).toBe("a");
+  });
+
   postgresTest("an ordinary flag keeps replace semantics", async ({ prisma }) => {
     await makeSetMultipleFlags(prisma)({ [FEATURE_FLAG.mollifierEnabled]: true });