github-featured/triggerdotdev--trigger.dev
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

fix(run-store,webapp): correct split-database read routing, write residency, and batches list ordering (#4272)

Browse Source 
## Summary

Correctness and performance fixes for deployments that split run data
across more than one database. Single-database / self-hosted deployments
are unaffected (they collapse to a single read/write path).

- **Batches list (dashboard):** for some organizations the Batches list
could hide older batches or show them out of order. It now orders and
paginates by creation time (with the id as a stable tiebreak), so every
batch appears exactly once, newest first. The pagination cursor format
changes; older in-flight cursors simply restart from the first page.
- **Reads:** waitpoint and snapshot lookups that are keyed by a single
run now read only the database that holds that run instead of querying
both, removing redundant queries on hot paths (unblock, snapshot reads).
- **Writes:** environment-scoped writes with no owning run (standalone
wait tokens, waitpoint tags, idempotency-key resets) now land in the
same database as that environment's runs, rather than defaulting to the
other one. An idempotency-key reset also falls back to the other
database when it matches nothing, so a reset still clears the key
wherever the run actually lives.

## Notes

Verified end-to-end against multi-database setups: run-keyed reads and
env-scoped writes land on the correct database with no cross-database
writes, and the batches list surfaces every batch in creation order. New
tests cover the batches ordering/reachability and the write-residency
routing.
This commit is contained in:
Daniel Sutton
2026-07-17 16:26:56 +01:00
committed by GitHub
parent 0ff0abd776
commit 821972176d
16 changed files with 1271 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/webapp/app/models/waitpointTag.server.ts
+13 -5
View File
@@ -8,10 +8,14 @@ export async function createWaitpointTag({
tag,
environmentId,
projectId,
residency,
}: {
tag: string;
environmentId: string;
projectId: string;
// Residency from the env mint kind: a tag has no owning run, so a minted-new env pins it to NEW
// instead of defaulting to the draining legacy DB.
residency?: "NEW" | "LEGACY";
}) {
if (tag.trim().length === 0) return;
@@ -19,11 +23,15 @@ export async function createWaitpointTag({
while (attempts < MAX_RETRIES) {
try {
return await runStore.upsertWaitpointTag({
environmentId,
name: tag,
projectId,
});
return await runStore.upsertWaitpointTag(
{
environmentId,
name: tag,
projectId,
},
undefined,
residency
);
} catch (error) {
if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") {
// Handle unique constraint violation (conflict)
apps/webapp/app/presenters/v3/BatchListPresenter.server.ts
+69 -21
View File
@@ -43,6 +43,28 @@ type BatchRow = {
batchVersion: string;
};
// Composite keyset cursor "<createdAt-epoch-ms>_<id>". Ordering is by createdAt then id: a batch id is
// a cuid (legacy) OR a run-ops id (new), and the two schemes occupy different lexical ranges, so `id`
// alone is not a valid chronological order across the residency split. `id` is the stable tiebreak.
// Old plain-id cursors (no "_") decode to undefined and restart from page 1 (self-healing).
type BatchCursor = { createdAt: Date; id: string };
function encodeBatchCursor(row: BatchCursor): string {
return `${row.createdAt.getTime()}_${row.id}`;
}
function decodeBatchCursor(cursor: string | undefined): BatchCursor | undefined {
if (!cursor) return undefined;
const sep = cursor.indexOf("_");
if (sep === -1) return undefined;
const ms = Number(cursor.slice(0, sep));
const id = cursor.slice(sep + 1);
// Number.isFinite accepts e.g. 1e20, but new Date(1e20) is Invalid Date — reject it so a malformed
// URL cursor self-heals to page 1 instead of reaching Prisma with an invalid date.
const createdAt = new Date(ms);
if (!Number.isFinite(ms) || Number.isNaN(createdAt.getTime()) || id.length === 0)
return undefined;
return { createdAt, id };
}
export class BatchListPresenter extends BasePresenter {
// Optional run-ops read-routing. Omitted (single-DB / self-host) => everything
// reads from `_replica` exactly as today (passthrough). Field names are local to
@@ -86,17 +108,16 @@ export class BatchListPresenter extends BasePresenter {
return scan(passthrough);
}
const newRows = await scan(this.readRoute.runOpsNew ?? passthrough);
// Always read BOTH stores and merge. The old "skip legacy when new fills the page" shortcut is
// unsound across the residency split: legacy cuid ids ("c…") sort ABOVE new run-ops ids ("0…")
// under id order, so a new-only page can hide pre-flip legacy batches that belong ahead of it.
// Ordering is by createdAt (id tiebreak), which is chronologically correct across both schemes.
const [newRows, legacyRows] = await Promise.all([
scan(this.readRoute.runOpsNew ?? passthrough),
scan(this.readRoute.runOpsLegacyReplica ?? passthrough),
]);
// New DB filled the page — skip the legacy read entirely; older rows fall on a later page.
if (newRows.length >= pageSize + 1) {
return newRows;
}
const legacyRows = await scan(this.readRoute.runOpsLegacyReplica ?? passthrough);
// De-dupe by id (new wins), re-sort under the page's keyset order, re-apply the over-fetch
// LIMIT — reproduces the pageSize+1 window a single union scan would return.
// De-dupe by id (new wins), re-sort under the page's keyset order, re-apply the over-fetch LIMIT.
const byId = new Map<string, BatchRow>();
for (const row of newRows) {
byId.set(row.id, row);
@@ -107,10 +128,16 @@ export class BatchListPresenter extends BasePresenter {
}
}
// codepoint comparator (NEVER localeCompare): BatchTaskRun.id is ASCII (cuid or run-ops id).
const sign = direction === "forward" ? 1 : -1; // forward => DESC; backward => ASC
// forward => newest-first (createdAt DESC), backward => oldest-first (ASC); id is the stable
// tiebreak (ASCII codepoint, NEVER localeCompare).
const sign = direction === "forward" ? 1 : -1;
return Array.from(byId.values())
.sort((a, b) => (a.id < b.id ? sign : a.id > b.id ? -sign : 0))
.sort((a, b) => {
const at = a.createdAt.getTime();
const bt = b.createdAt.getTime();
if (at !== bt) return at < bt ? sign : -sign;
return a.id < b.id ? sign : a.id > b.id ? -sign : 0;
})
.slice(0, pageSize + 1);
}
@@ -212,11 +239,28 @@ export class BatchListPresenter extends BasePresenter {
}
const createdAtLte: Date | undefined = time.to;
// Composite (createdAt, id) keyset — see encodeBatchCursor. An old plain-id cursor decodes to
// undefined and restarts from page 1.
const keyCursor = decodeBatchCursor(cursor);
const batches = await this.#scanBatchTaskRun(pageSize, direction, (client) =>
client.batchTaskRun.findMany({
where: {
runtimeEnvironmentId: environmentId,
...(cursor ? { id: direction === "forward" ? { lt: cursor } : { gt: cursor } } : {}),
...(keyCursor
? {
OR:
direction === "forward"
? [
{ createdAt: { lt: keyCursor.createdAt } },
{ createdAt: keyCursor.createdAt, id: { lt: keyCursor.id } },
]
: [
{ createdAt: { gt: keyCursor.createdAt } },
{ createdAt: keyCursor.createdAt, id: { gt: keyCursor.id } },
],
}
: {}),
...(friendlyId ? { friendlyId } : {}),
...(statuses && statuses.length > 0
? { status: { in: statuses }, batchVersion: { not: "v1" } }
@@ -230,7 +274,10 @@ export class BatchListPresenter extends BasePresenter {
}
: {}),
},
orderBy: { id: direction === "forward" ? "desc" : "asc" },
orderBy: [
{ createdAt: direction === "forward" ? "desc" : "asc" },
{ id: direction === "forward" ? "desc" : "asc" },
],
take: pageSize + 1,
select: {
id: true,
@@ -248,23 +295,24 @@ export class BatchListPresenter extends BasePresenter {
const hasMore = batches.length > pageSize;
//get cursors for next and previous pages
//get cursors for next and previous pages (composite (createdAt, id) keyset)
const cur = (row?: BatchRow) => (row ? encodeBatchCursor(row) : undefined);
let next: string | undefined;
let previous: string | undefined;
switch (direction) {
case "forward":
previous = cursor ? batches.at(0)?.id : undefined;
previous = cursor ? cur(batches.at(0)) : undefined;
if (hasMore) {
next = batches[pageSize - 1]?.id;
next = cur(batches[pageSize - 1]);
}
break;
case "backward":
batches.reverse();
if (hasMore) {
previous = batches[1]?.id;
next = batches[pageSize]?.id;
previous = cur(batches[1]);
next = cur(batches[pageSize]);
} else {
next = batches[pageSize - 1]?.id;
next = cur(batches[pageSize - 1]);
}
break;
}
apps/webapp/app/routes/api.v1.waitpoints.tokens.ts
+13
View File
@@ -16,6 +16,7 @@ import {
type PrismaClientOrTransaction,
} from "~/db.server";
import { type AuthenticatedEnvironment } from "~/services/apiAuth.server";
import { resolveRunIdMintKind } from "~/v3/engineVersion.server";
import { logger } from "~/services/logger.server";
import { generateHttpCallbackUrl } from "~/services/httpCallback.server";
import {
@@ -58,6 +59,16 @@ const { action } = createActionApiRoute(
const timeout = await parseDelay(body.timeout);
// A token (and its tags) has no owning run, so it can't co-locate. Resolve the env mint kind so a
// minted-new env creates them on the run-ops DB (NEW) instead of defaulting to the draining LEGACY
// DB by their cuid id-shape.
const mintKind = await resolveRunIdMintKind({
organizationId: authentication.environment.organizationId,
id: authentication.environment.id,
orgFeatureFlags: authentication.environment.organization.featureFlags,
});
const residency = mintKind === "runOpsId" ? "NEW" : "LEGACY";
//upsert tags
let tags: { id: string; name: string }[] = [];
const bodyTags = typeof body.tags === "string" ? [body.tags] : body.tags;
@@ -74,6 +85,7 @@ const { action } = createActionApiRoute(
tag,
environmentId: authentication.environment.id,
projectId: authentication.environment.projectId,
residency,
});
if (tagRecord) {
tags.push(tagRecord);
@@ -88,6 +100,7 @@ const { action } = createActionApiRoute(
idempotencyKeyExpiresAt,
timeout,
tags: bodyTags,
standaloneResidency: residency,
});
const $responseHeaders = await responseHeaders(authentication.environment);
apps/webapp/app/v3/services/resetIdempotencyKey.server.ts
+17
View File
@@ -2,6 +2,7 @@ import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
import { BaseService, ServiceValidationError } from "./baseService.server";
import { logger } from "~/services/logger.server";
import { getMollifierBuffer } from "~/v3/mollifier/mollifierBuffer.server";
import { resolveRunIdMintKind } from "~/v3/engineVersion.server";
export class ResetIdempotencyKeyService extends BaseService {
public async call(
@@ -9,12 +10,27 @@ export class ResetIdempotencyKeyService extends BaseService {
taskIdentifier: string,
authenticatedEnv: AuthenticatedEnvironment
): Promise<{ id: string }> {
// The predicate has no run id to route by. When the env mints run-ops ids its runs live on NEW,
// so pin the reset to NEW and skip the wrong-DB (0-row) write to the draining legacy DB. Resolve
// this only when the org (and its flags) is loaded on the env — which the authenticated API path
// always provides; otherwise fall back to the two-store reset (correct, just not optimized).
let residency: "NEW" | "LEGACY" = "LEGACY";
if (authenticatedEnv.organization) {
const mintKind = await resolveRunIdMintKind({
organizationId: authenticatedEnv.organizationId,
id: authenticatedEnv.id,
orgFeatureFlags: authenticatedEnv.organization.featureFlags,
});
residency = mintKind === "runOpsId" ? "NEW" : "LEGACY";
}
const { count: pgCount } = await this.runStore.clearIdempotencyKey(
{
byPredicate: {
idempotencyKey,
taskIdentifier,
runtimeEnvironmentId: authenticatedEnv.id,
residency,
},
},
this._prisma
@@ -80,6 +96,7 @@ export class ResetIdempotencyKeyService extends BaseService {
idempotencyKey,
taskIdentifier,
runtimeEnvironmentId: authenticatedEnv.id,
residency,
},
},
this._prisma
apps/webapp/test/batchListPresenter.readroute.test.ts
+148 -9
View File
@@ -309,9 +309,9 @@ describe("BatchListPresenter run-ops read routing (PG14 control-plane/legacy + P
}
);
// Split scan merge serves new + legacy in one keyset-ordered page.
// Split scan merge serves new + legacy in one createdAt-ordered page; legacy is always read.
heteroPostgresTest(
"split scan merges new (PG17) + legacy (PG14) rows under the keyset order; legacy read only when new does not fill the page",
"split scan merges new (PG17) + legacy (PG14) rows under the createdAt keyset order; legacy always read",
async ({ prisma14, prisma17 }) => {
const ctx14 = await seedParents(prisma14, "merge");
await mirrorEnvParents(prisma17, ctx14, "merge");
@@ -323,7 +323,8 @@ describe("BatchListPresenter run-ops read routing (PG14 control-plane/legacy + P
await createBatch(prisma14, ctx14, { id: "batch_d", friendlyId: "fr_d", runCount: 4 });
await createBatch(prisma17, ctx14, { id: "batch_e", friendlyId: "fr_e", runCount: 5 });
// Case A: small page fully served by new alone => legacy NOT read.
// Case A: always-merge — legacy is read even when new could fill the page (the old skip was
// unsound across the residency split). Page is the createdAt-ordered union of both DBs.
const legacySpyA = spyClient(prisma14);
const presenterA = new BatchListPresenter(prisma17, prisma17, {
runOpsNew: prisma17,
@@ -332,9 +333,9 @@ describe("BatchListPresenter run-ops read routing (PG14 control-plane/legacy + P
splitEnabled: true,
});
const pageA = await presenterA.call(baseCall(ctx14, { pageSize: 2 }));
// new ids are e, c, a -> DESC: e, c (pageSize 2). pageSize+1 = 3 rows from new fills the page.
expect(pageA.batches.map((b) => b.id)).toEqual(["batch_e", "batch_c"]);
expect(legacySpyA.counts.findMany).toBe(0);
// union newest-first (createdAt, insertion order a<b<c<d<e): e, d, c, b, a -> page of 2 = e, d.
expect(pageA.batches.map((b) => b.id)).toEqual(["batch_e", "batch_d"]);
expect(legacySpyA.counts.findMany).toBeGreaterThan(0);
// Case B: page needs legacy rows => legacy IS read and the merge is keyset-ordered union.
const legacySpyB = spyClient(prisma14);
@@ -348,8 +349,10 @@ describe("BatchListPresenter run-ops read routing (PG14 control-plane/legacy + P
// union DESC of all 5: e, d, c, b, a -> first 4.
expect(pageB.batches.map((b) => b.id)).toEqual(["batch_e", "batch_d", "batch_c", "batch_b"]);
expect(legacySpyB.counts.findMany).toBeGreaterThan(0);
// cursor parity: next is the 4th id (pageSize-th), previous undefined (no input cursor).
expect(pageB.pagination.next).toBe("batch_b");
// cursor parity: next is the FULL composite (createdAt, id) cursor of the 4th row (batch_b),
// previous undefined. Assert the complete value so a bad timestamp prefix can't pass.
const bRow = pageB.batches.find((b) => b.id === "batch_b")!;
expect(pageB.pagination.next).toBe(`${new Date(bRow.createdAt).getTime()}_batch_b`);
expect(pageB.pagination.previous).toBeUndefined();
expect(pageB.hasAnyBatches).toBe(true);
}
@@ -436,7 +439,11 @@ describe("BatchListPresenter run-ops read routing (PG14 control-plane/legacy + P
const hasMore = direct.length > 2;
const expectedPage = direct.slice(0, 2);
expect(page.batches.map((b) => b.id)).toEqual(expectedPage.map((r) => r.id));
expect(page.pagination.next).toBe(hasMore ? expectedPage[1].id : undefined);
expect(
hasMore
? page.pagination.next === `${expectedPage[1].createdAt.getTime()}_${expectedPage[1].id}`
: !page.pagination.next
).toBe(true);
expect(page.pagination.previous).toBeUndefined();
expect(page.hasAnyBatches).toBe(true);
@@ -453,6 +460,138 @@ describe("BatchListPresenter run-ops read routing (PG14 control-plane/legacy + P
}
);
// REGRESSION: a flipped org's real id mix. A cuid ("c"=0x63) sorts ABOVE a run-ops id ("0"=0x30)
// under `id DESC`, so pre-flip legacy batches belong at the top — but #scanBatchTaskRun reads new
// first, skips legacy once the page is full, and `id < cursor` can never reach a "c…" from a "0…"
// cursor. Net: pre-flip legacy batches become unreachable.
heteroPostgresTest(
"flipped org: pre-flip legacy (cuid) batches remain reachable alongside post-flip run-ops batches",
async ({ prisma14, prisma17 }) => {
const ctx = await seedParents(prisma14, "flip");
await mirrorEnvParents(prisma17, ctx, "flip");
// Pre-flip cuid batch on legacy (sorts highest); post-flip run-ops batches on new (sort below).
const LEGACY_CUID = "cm0preflipbatch0000000001";
await createBatch(prisma14, ctx, { id: LEGACY_CUID, friendlyId: "fr_preflip", runCount: 9 });
const NEW_RUNOPS = [
"06fnewbatch00000000000000a",
"06fnewbatch00000000000000b",
"06fnewbatch00000000000000c",
];
for (const id of NEW_RUNOPS) {
await createBatch(prisma17, ctx, { id, friendlyId: `fr_${id.slice(-1)}`, runCount: 1 });
}
const presenter = new BatchListPresenter(prisma17, prisma17, {
runOpsNew: prisma17,
runOpsLegacyReplica: prisma14,
controlPlaneReplica: prisma14,
splitEnabled: true,
});
// The pre-flip cuid batch is the oldest, so under newest-first it lands on a later page — but it
// must be REACHABLE by paging forward, not stranded behind the run-ops ids (the skip + id-order
// bug dropped it entirely: `id < <run-ops cursor>` never matches a "c…" id).
const seen = new Set<string>();
let cursor: string | undefined = undefined;
for (let i = 0; i < 10; i++) {
const page = await presenter.call(
baseCall(ctx, { pageSize: 2, cursor, direction: "forward" })
);
page.batches.forEach((b) => seen.add(b.id));
if (!page.pagination.next) break;
cursor = page.pagination.next;
}
expect([...seen]).toContain(LEGACY_CUID);
}
);
// REGRESSION (ordering): even with an always-merge fix, keyset-by-id is chronologically wrong across
// the flip — a cuid ("c") sorts above a run-ops id ("0"), so an OLDER pre-flip batch outranks a NEWER
// post-flip batch. The list is "newest first", so the later-created run-ops batch must come first.
heteroPostgresTest(
"flipped org: batches list is newest-first across the flip boundary (by createdAt, not id)",
async ({ prisma14, prisma17 }) => {
const ctx = await seedParents(prisma14, "order");
await mirrorEnvParents(prisma17, ctx, "order");
const OLD_LEGACY = "cm0oldbatch00000000000001"; // cuid, created EARLIER
const NEW_RUNOPS = "06fnewbatch000000000000001"; // run-ops, created LATER
await createBatch(prisma14, ctx, {
id: OLD_LEGACY,
friendlyId: "fr_old",
createdAt: new Date(Date.now() - 3_600_000),
});
await createBatch(prisma17, ctx, {
id: NEW_RUNOPS,
friendlyId: "fr_new",
createdAt: new Date(),
});
const presenter = new BatchListPresenter(prisma17, prisma17, {
runOpsNew: prisma17,
runOpsLegacyReplica: prisma14,
controlPlaneReplica: prisma14,
splitEnabled: true,
});
const page = await presenter.call(baseCall(ctx, { pageSize: 10 }));
// Newest-first: the later-created run-ops batch outranks the older legacy one.
expect(page.batches.map((b) => b.id)).toEqual([NEW_RUNOPS, OLD_LEGACY]);
}
);
// Overlap regression: batches duplicated on BOTH stores (mid-migration copies) must de-dupe to one
// each without dropping rows or underfilling pages. Proves the merge needs no post-dedup refill:
// the union of each store's top-(pageSize+1) always contains the global top, and the next page
// re-queries both stores from the cursor.
heteroPostgresTest(
"flipped org: batches duplicated across both stores de-dupe without dropping rows across pagination",
async ({ prisma14, prisma17 }) => {
const ctx = await seedParents(prisma14, "ovl");
await mirrorEnvParents(prisma17, ctx, "ovl");
const at = (secondsAgo: number) => new Date(Date.now() - secondsAgo * 1000);
// a..e newest->oldest. a,b,c live on BOTH DBs (dup); d,e only on legacy.
const rows = [
{ id: "batch_ov_a", both: true, s: 1 },
{ id: "batch_ov_b", both: true, s: 2 },
{ id: "batch_ov_c", both: true, s: 3 },
{ id: "batch_ov_d", both: false, s: 4 },
{ id: "batch_ov_e", both: false, s: 5 },
];
for (const r of rows) {
// A dup is a row COPY: identical (createdAt, id) on both DBs. Compute createdAt ONCE so both
// copies match exactly (two at(r.s) calls would drift by ms and the older copy would re-surface
// on the next page under the createdAt keyset).
const createdAt = at(r.s);
await createBatch(prisma14, ctx, { id: r.id, friendlyId: `fr_${r.id}`, createdAt });
if (r.both) {
await createBatch(prisma17, ctx, { id: r.id, friendlyId: `fr_${r.id}`, createdAt });
}
}
const seen: string[] = [];
let cursor: string | undefined = undefined;
for (let i = 0; i < 10; i++) {
const presenter = new BatchListPresenter(prisma17, prisma17, {
runOpsNew: prisma17,
runOpsLegacyReplica: prisma14,
controlPlaneReplica: prisma14,
splitEnabled: true,
});
const page = await presenter.call(
baseCall(ctx, { pageSize: 2, cursor, direction: "forward" })
);
seen.push(...page.batches.map((b) => b.id));
if (!page.pagination.next) break;
cursor = page.pagination.next;
}
// Every batch exactly once, newest-first; the three dups collapsed to one each.
expect(seen).toEqual(["batch_ov_a", "batch_ov_b", "batch_ov_c", "batch_ov_d", "batch_ov_e"]);
}
);
heteroRunOpsPostgresTest(
"scan against dedicated RunOpsPrismaClient (splitEnabled): returns batches from new DB",
async ({ prisma14, prisma17 }) => {