perf(webapp,run-store): grouped run-ops reads + mint-kind flip grace (#4227)

## Summary

Two threads on the run-ops split path.

Read path: per-item run reads are batched into grouped queries, a
waitpoint's connected-run reads are bounded, and the dedicated-schema
relation hydrators fetch only the requested columns instead of whole
rows. Retrieve also falls back to the other database when a routed read
misses, so a run whose physical residency diverges from its id shape is
still found rather than returning a spurious not-found. Fewer and
lighter queries on the run read path, with no change to results.

Mint-kind flip safety: flipping which database new runs mint to is now a
deterministic wall-clock cutover, for both per-org and global flips. For
a grace window every process resolves the same database, so a flip
cannot route two concurrent triggers that share an idempotency key to
different databases (which would bypass the per-database unique
constraint and create a duplicate run).

Supersedes the earlier #4205 and #4208.

Draft: validation in progress.
This commit is contained in:
Daniel Sutton
2026-07-13 10:17:12 +01:00
committed by GitHub
parent 5f2541d94f
commit c601739d35
38 changed files with 5596 additions and 351 deletions
+4
View File
@@ -1755,6 +1755,10 @@ const EnvironmentSchema = z
RUN_OPS_MINT_ENABLED: BoolEnv.default(false),
RUN_OPS_MINT_FLAG_CACHE_TTL_MS: z.coerce.number().int().default(30_000),
RUN_OPS_MINT_FLAG_CACHE_MAX_ENTRIES: z.coerce.number().int().default(10_000),
// Deterministic wall-clock cutover after a runOpsMintKind flip. Must exceed the sum
// of RUN_OPS_MINT_FLAG_CACHE_TTL_MS and the control-plane cache TTL so every process
// (stale or fresh) resolves to the same kind for the whole window. See mintFlipGrace.ts.
RUN_OPS_MINT_FLIP_GRACE_MS: z.coerce.number().int().default(90_000),
// Session replication (Postgres → ClickHouse sessions_v1). Shares Redis
// with the runs replicator for leader locking but has its own slot and
@@ -1,4 +1,5 @@
import type { BatchTaskRunExecutionResult } from "@trigger.dev/core/v3";
import { ownerEngine } from "@trigger.dev/core/v3/isomorphic";
import {
$replica,
type PrismaClientOrTransaction,
@@ -8,7 +9,6 @@ import {
import type { TaskRunWithAttempts } from "~/models/taskRun.server";
import { executionResultForTaskRun } from "~/models/taskRun.server";
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
import { readThroughRun } from "~/v3/runOpsMigration/readThrough.server";
import { runStore as defaultRunStore } from "~/v3/runStore.server";
import { BasePresenter } from "./basePresenter.server";
@@ -43,11 +43,14 @@ const memberRunSelect = {
} as const;
/**
* Split on: the batch row + its item rows resolve new-run-ops first, then the LEGACY RUN-OPS
* READ REPLICA ONLY (never the legacy primary — there is no such handle); each member run is
* hydrated independently via readThroughRun keyed on the member runId, so a batch whose members
* span migrated + abandoned runs returns the complete reachable set (the batch-spanning-the-line
* read; the dangling-reference termination gate is a separate, adjacent unit).
* Split on: the batch row + its members resolve new-run-ops first, then the LEGACY RUN-OPS READ
* REPLICA ONLY (never the legacy primary — there is no such handle). Members hydrate via ONE
* grouped read against `newClient` for the whole id set, then ONE grouped read against
* `legacyReplica` for just the misses that could still be legacy-resident — the same
* residency-partitioned shape as the old per-member read-through, but batched instead of fanned
* out one query per member. A batch whose members span migrated + abandoned runs returns the
* complete reachable set (the batch-spanning-the-line read; the dangling-reference termination
* gate is a separate, adjacent unit).
*
* Split off (single-DB / self-host): one passthrough read for the batch row + a single store
* id-set hydrate for the members — no legacy read, no known-migrated probe, no second connection.
@@ -133,7 +136,7 @@ export class ApiBatchResultsPresenter extends BasePresenter {
// Split: resolve the batch row new-first then off the legacy READ REPLICA only (a batch id may
// be cuid or run-ops id, and a cuid-shaped id can still have been backfilled onto NEW, so id-shape
// residency is not authoritative for the row — the new-first-then-legacy probe is), then
// hydrate every member run independently via the per-run read-through primitive.
// hydrate every member run in ONE grouped new-then-legacy read.
async #callSplit(
friendlyId: string,
env: AuthenticatedEnvironment
@@ -175,41 +178,35 @@ export class ApiBatchResultsPresenter extends BasePresenter {
};
}
const readMemberRun = (client: PrismaClientOrTransaction, taskRunId: string) =>
client.taskRun.findFirst({
where: { id: taskRunId },
select: memberRunSelect,
}) as Promise<TaskRunWithAttempts | null>;
const taskRunIds = batchRun.items.map((item) => item.taskRunId);
// Per-member fan-out: each member may live on a different DB, so a single nested include cannot
// cross the seam. Promise.all preserves batchRun.items order, unchanged from today.
const memberResults = await Promise.all(
batchRun.items.map(async (item) => {
const result = await readThroughRun<TaskRunWithAttempts>({
runId: item.taskRunId,
environmentId: env.id,
readNew: (client) => readMemberRun(client, item.taskRunId),
readLegacy: (replica) => readMemberRun(replica, item.taskRunId),
deps: {
splitEnabled: true,
// Pass the SAME resolved handles the batch row used, so the batch row and its members
// never resolve against different DBs. (Letting these fall through to readThroughRun's
// own module-level defaults would diverge from the batch read's `?? this._replica`.)
newClient,
legacyReplica,
isPastRetention: this.readThrough?.isPastRetention,
},
});
const newRows = (await newClient.taskRun.findMany({
where: { id: { in: taskRunIds } },
select: memberRunSelect,
})) as TaskRunWithAttempts[];
const runsById = new Map(newRows.map((run) => [run.id, run]));
// not-found / past-retention members are omitted (matches today's drop-undefined behavior);
// the dangling-reference termination gate (separate unit) governs whether that's permitted.
if (result.source === "not-found" || result.source === "past-retention") {
return undefined;
}
return executionResultForTaskRun(result.value);
})
// A run-ops id can only live on NEW, so only misses that AREN'T run-ops-shaped are candidates
// for the legacy probe — mirrors readThroughRun's per-id "NEW residency skips legacy" rule.
const legacyCandidateIds = taskRunIds.filter(
(id) => !runsById.has(id) && ownerEngine(id) !== "NEW"
);
if (legacyCandidateIds.length > 0) {
const legacyRows = (await legacyReplica.taskRun.findMany({
where: { id: { in: legacyCandidateIds } },
select: memberRunSelect,
})) as TaskRunWithAttempts[];
for (const run of legacyRows) {
runsById.set(run.id, run);
}
}
// not-found members are omitted (matches today's drop-undefined behavior); the
// dangling-reference termination gate (separate unit) governs whether that's permitted.
const memberResults = batchRun.items.map((item) => {
const run = runsById.get(item.taskRunId);
return run ? executionResultForTaskRun(run) : undefined;
});
return {
id: batchRun.friendlyId,
@@ -164,17 +164,13 @@ export class ApiRetrieveRunPresenter {
collect(child.lockedToVersionId);
}
const lockedWorkersByVersionId =
await controlPlaneResolver.resolveRunLockedWorkersByVersionIds([...distinctVersionIds]);
const versionById = new Map<string, string | null>(
await Promise.all(
[...distinctVersionIds].map(
async (id) =>
[
id,
(await controlPlaneResolver.resolveRunLockedWorker({ lockedToVersionId: id }))
?.lockedToVersion?.version ?? null,
] as const
)
)
[...distinctVersionIds].map((id) => [
id,
lockedWorkersByVersionId.get(id)?.lockedToVersion?.version ?? null,
])
);
const resolveVersion = (id: string | null): { version: string } | null => {
@@ -11,6 +11,12 @@ import { waitpointStatusToApiStatus } from "./WaitpointListPresenter.server";
export type WaitpointDetail = NonNullable<Awaited<ReturnType<WaitpointPresenter["call"]>>>;
// Single-sourced display bound for a waitpoint's connected run friendlyIds.
export const CONNECTED_RUNS_DISPLAY_LIMIT = 5;
// Over-read connection rows on the FK-free dedicated join so danglers don't cost a display slot.
export const CONNECTED_RUNS_CONNECTION_SCAN_LIMIT = CONNECTED_RUNS_DISPLAY_LIMIT * 5;
export class WaitpointPresenter extends BasePresenter {
constructor(
prisma?: PrismaClientOrTransaction,
@@ -76,8 +82,7 @@ export class WaitpointPresenter extends BasePresenter {
// Connected-run friendlyIds gathered across BOTH stores. The run<->waitpoint join co-locates with
// the RUN (written on the run's DB), so the waitpoint's own store misses a cross-DB connection; we
// read the join on each client and resolve the run's friendlyId on that same client, then union.
// We never relation-select `connectedRuns`: it is not a field on the dedicated subset `Waitpoint`.
// read the join on each client, resolve the run's friendlyId on that same client, and union.
async #connectedRunFriendlyIds(waitpointId: string): Promise<string[]> {
const replica = this._replica as unknown as PrismaReplicaClient;
const rawClients: PrismaReplicaClient[] =
@@ -91,47 +96,66 @@ export class WaitpointPresenter extends BasePresenter {
const friendlyIds = new Set<string>();
for (const client of clients) {
const runIds = await this.#connectedRunIdsOn(client, waitpointId);
if (runIds.length === 0) {
continue;
for (const friendlyId of await this.#connectedRunFriendlyIdsOn(client, waitpointId)) {
friendlyIds.add(friendlyId);
}
const runs = await client.taskRun.findMany({
where: { id: { in: runIds } },
select: { friendlyId: true },
take: 5,
});
for (const run of runs) {
friendlyIds.add(run.friendlyId);
}
if (friendlyIds.size >= 5) {
if (friendlyIds.size >= CONNECTED_RUNS_DISPLAY_LIMIT) {
break;
}
}
return Array.from(friendlyIds).slice(0, 5);
return Array.from(friendlyIds).slice(0, CONNECTED_RUNS_DISPLAY_LIMIT);
}
// Schema-aware read of the run ids linked to a waitpoint: the dedicated subset uses the explicit
// `WaitpointRunConnection` model, the control-plane full schema the implicit `_WaitpointRunConnections`
// M2M (A = TaskRun.id, B = Waitpoint.id). The dedicated join delegate is absent on the full client.
async #connectedRunIdsOn(client: PrismaReplicaClient, waitpointId: string): Promise<string[]> {
const joinDelegate = (
// Connected-run friendlyIds for one store, via the ORM. Two indexed reads joined in memory instead
// of a SQL JOIN onto the (very large) TaskRun table: `id IN (...)` can only plan as a PK lookup, so
// the planner can never scan TaskRun.
//
// Dedicated subset: the explicit `WaitpointRunConnection` is scalar (`taskRunId`, no FK), so a
// connection can outlive a deleted run. We over-read connection ids, resolve runs by id (a missing
// id just drops out -- danglers cost no display slot), and cap at the display limit.
//
// Control-plane full schema: no queryable join delegate (implicit M2M), so we traverse the
// `connectedRuns` relation; it cascade-deletes with the run, so no dangler can exist.
async #connectedRunFriendlyIdsOn(
client: PrismaReplicaClient,
waitpointId: string
): Promise<string[]> {
const dedicated = (
client as unknown as {
waitpointRunConnection?: {
findMany: (args: unknown) => Promise<{ taskRunId: string }[]>;
};
}
).waitpointRunConnection;
if (joinDelegate && typeof joinDelegate.findMany === "function") {
const links = await joinDelegate.findMany({
if (dedicated) {
const connections = await dedicated.findMany({
where: { waitpointId },
select: { taskRunId: true },
take: CONNECTED_RUNS_CONNECTION_SCAN_LIMIT,
});
return links.map((link) => link.taskRunId);
if (connections.length === 0) {
return [];
}
const runs = await client.taskRun.findMany({
where: { id: { in: connections.map((connection) => connection.taskRunId) } },
select: { friendlyId: true },
take: CONNECTED_RUNS_DISPLAY_LIMIT,
});
return runs.map((run) => run.friendlyId);
}
const rows = await client.$queryRaw<{ A: string }[]>`
SELECT "A" FROM "_WaitpointRunConnections" WHERE "B" = ${waitpointId}
`;
return rows.map((row) => row.A);
const waitpoint = (await (
client.waitpoint.findFirst as (
args: unknown
) => Promise<{ connectedRuns: { friendlyId: string }[] } | null>
)({
where: { id: waitpointId },
select: {
connectedRuns: { select: { friendlyId: true }, take: CONNECTED_RUNS_DISPLAY_LIMIT },
},
})) as { connectedRuns: { friendlyId: string }[] } | null;
return (waitpoint?.connectedRuns ?? []).map((run) => run.friendlyId);
}
public async call({
@@ -1,8 +1,9 @@
import type { ActionFunctionArgs } from "@remix-run/server-runtime";
import { json } from "@remix-run/server-runtime";
import { prisma } from "~/db.server";
import { env } from "~/env.server";
import { requireAdminApiRequest } from "~/services/personalAccessToken.server";
import { makeSetMultipleFlags } from "~/v3/featureFlags.server";
import { applyGlobalMintKindFlip, makeSetMultipleFlags } from "~/v3/featureFlags.server";
import { validatePartialFeatureFlags } from "~/v3/featureFlags";
export async function action({ request }: ActionFunctionArgs) {
@@ -24,9 +25,19 @@ export async function action({ request }: ActionFunctionArgs) {
);
}
const featureFlags = validationResult.data;
const setMultipleFlags = makeSetMultipleFlags(prisma);
const updatedFlags = await setMultipleFlags(featureFlags);
// Derived grace-stamp fields are computed server-side; never trust them from the body.
const {
runOpsMintKindPrev: _ignoredPrev,
runOpsMintKindFlippedAt: _ignoredFlippedAt,
...requestedFlags
} = validationResult.data;
// A global mint-kind flip stamps its grace window under a lock (applyGlobalMintKindFlip);
// any other flag save writes directly.
const updatedFlags =
requestedFlags.runOpsMintKind !== undefined
? await applyGlobalMintKindFlip(prisma, requestedFlags, env.RUN_OPS_MINT_FLIP_GRACE_MS)
: await makeSetMultipleFlags(prisma)(requestedFlags);
return json({
success: true,
@@ -1,10 +1,14 @@
import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime";
import { json } from "@remix-run/server-runtime";
import type { Prisma } from "@trigger.dev/database";
import { z } from "zod";
import { env } from "~/env.server";
import { prisma } from "~/db.server";
import { requireAdminApiRequest } from "~/services/personalAccessToken.server";
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
import { selectMintBaselineSource, stampMintKindFlip } from "~/v3/runOpsMigration/mintFlipGrace";
import { validatePartialFeatureFlags } from "~/v3/featureFlags";
import { flags as getGlobalFlags } from "~/v3/featureFlags.server";
const ParamsSchema = z.object({
organizationId: z.string(),
@@ -48,20 +52,6 @@ export async function action({ request, params }: ActionFunctionArgs) {
const { organizationId } = ParamsSchema.parse(params);
const organization = await prisma.organization.findFirst({
where: {
id: organizationId,
},
select: {
id: true,
featureFlags: true,
},
});
if (!organization) {
return json({ error: "Organization not found" }, { status: 404 });
}
try {
const body = await request.json();
@@ -77,31 +67,66 @@ export async function action({ request, params }: ActionFunctionArgs) {
);
}
// Merge new flags with existing flags
const existingFlags = organization.featureFlags
? validatePartialFeatureFlags(organization.featureFlags as Record<string, unknown>)
: { success: false as const };
// Derived grace-stamp fields are computed server-side; never trust them from the body.
const {
runOpsMintKindPrev: _ignoredPrev,
runOpsMintKindFlippedAt: _ignoredFlippedAt,
...requestedFlags
} = validationResult.data;
const mergedFlags = {
...(existingFlags.success ? existingFlags.data : {}),
...validationResult.data,
};
// Seed the flip baseline from the current GLOBAL mint flags so an org's FIRST per-org override
// is graced from the currently-effective global kind, not the hardcoded default "cuid".
const globalFlags = (await getGlobalFlags()) as Record<string, unknown>;
// Update the organization's feature flags
const updatedOrganization = await prisma.organization.update({
where: {
id: organizationId,
},
data: {
featureFlags: mergedFlags,
},
select: {
id: true,
slug: true,
featureFlags: true,
},
// Lock the org row for the whole read -> merge -> stamp -> write so a concurrent flag save
// can't clobber the grace metadata (read-then-write race). PK lookup, one row, held to commit.
const updatedOrganization = await prisma.$transaction(async (tx) => {
const rows = await tx.$queryRaw<{ featureFlags: unknown }[]>`
SELECT "featureFlags" FROM "Organization" WHERE "id" = ${organizationId} FOR UPDATE`;
if (rows.length === 0) {
return null;
}
const existingRaw = rows[0].featureFlags as Record<string, unknown> | null;
const existingResult = existingRaw
? validatePartialFeatureFlags(existingRaw)
: ({ success: false } as const);
const existingData = existingResult.success ? existingResult.data : {};
// Stamp the flip from the control-plane DB clock so the grace-window cutover is anchored to
// one authoritative time source, not whichever webapp process handled this request.
const [{ now: controlPlaneNow }] = await tx.$queryRaw<{ now: Date }[]>`SELECT now() AS now`;
const mergedFlags = stampMintKindFlip(
selectMintBaselineSource(existingRaw, globalFlags),
{
...existingData,
...requestedFlags,
},
controlPlaneNow.getTime(),
env.RUN_OPS_MINT_FLIP_GRACE_MS
);
return tx.organization.update({
where: {
id: organizationId,
},
data: {
featureFlags: mergedFlags as Prisma.InputJsonValue,
},
select: {
id: true,
slug: true,
featureFlags: true,
},
});
});
if (!updatedOrganization) {
return json({ error: "Organization not found" }, { status: 404 });
}
// Org feature flags are embedded in every env of the org; drop all its cached env rows.
controlPlaneResolver.invalidateOrganization(organizationId);
@@ -2,9 +2,11 @@ import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-r
import { json } from "@remix-run/server-runtime";
import { Prisma } from "@trigger.dev/database";
import { z } from "zod";
import { env } from "~/env.server";
import { prisma } from "~/db.server";
import { requireUser } from "~/services/session.server";
import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
import { selectMintBaselineSource, stampMintKindFlip } from "~/v3/runOpsMigration/mintFlipGrace";
import { flags as getGlobalFlags } from "~/v3/featureFlags.server";
import {
FEATURE_FLAG,
@@ -103,34 +105,78 @@ export async function action({ request, params }: ActionFunctionArgs) {
return json({ error: "Invalid JSON body" }, { status: 400 });
}
let featureFlags: typeof Prisma.JsonNull | Record<string, unknown>;
if (
body === null ||
(typeof body === "object" && !Array.isArray(body) && Object.keys(body).length === 0)
) {
featureFlags = Prisma.JsonNull;
} else {
const validationResult = validatePartialFeatureFlags(body as Record<string, unknown>);
if (!validationResult.success) {
return json(
{ error: "Invalid feature flags", details: validationResult.error.issues },
{ status: 400 }
);
// Clear all flags. No grace stamp (nothing to flip) and no read-then-write race.
try {
await prisma.organization.update({
where: { id: organizationId },
data: { featureFlags: Prisma.JsonNull },
});
} catch (e) {
if (e instanceof Prisma.PrismaClientKnownRequestError && e.code === "P2025") {
throw new Response("Organization not found", { status: 404 });
}
throw e;
}
featureFlags = validationResult.data;
controlPlaneResolver.invalidateOrganization(organizationId);
return json({ success: true });
}
try {
await prisma.organization.update({
where: { id: organizationId },
data: { featureFlags: featureFlags as Prisma.InputJsonValue },
});
} catch (e) {
if (e instanceof Prisma.PrismaClientKnownRequestError && e.code === "P2025") {
throw new Response("Organization not found", { status: 404 });
const validationResult = validatePartialFeatureFlags(body as Record<string, unknown>);
if (!validationResult.success) {
return json(
{ error: "Invalid feature flags", details: validationResult.error.issues },
{ status: 400 }
);
}
// Derived grace-stamp fields are computed server-side; never trust them from the body.
const {
runOpsMintKindPrev: _ignoredPrev,
runOpsMintKindFlippedAt: _ignoredFlippedAt,
...requestedFlags
} = validationResult.data;
// Seed the flip baseline from the current GLOBAL mint flags so an org's FIRST per-org override
// is graced from the currently-effective global kind, not the hardcoded default "cuid".
const globalFlags = (await getGlobalFlags()) as Record<string, unknown>;
// Lock the org row for the whole read -> stamp -> write so a concurrent flag save can't clobber
// the grace metadata (read-then-write race). PK lookup, one row, held to commit.
const updated = await prisma.$transaction(async (tx) => {
const rows = await tx.$queryRaw<{ featureFlags: unknown }[]>`
SELECT "featureFlags" FROM "Organization" WHERE "id" = ${organizationId} FOR UPDATE`;
if (rows.length === 0) {
return false;
}
throw e;
const existingRaw = rows[0].featureFlags as Record<string, unknown> | null;
// Anchor the flip stamp to the control-plane DB clock (see the v1 route), not this process's.
const [{ now: controlPlaneNow }] = await tx.$queryRaw<{ now: Date }[]>`SELECT now() AS now`;
const stamped = stampMintKindFlip(
selectMintBaselineSource(existingRaw, globalFlags),
requestedFlags,
controlPlaneNow.getTime(),
env.RUN_OPS_MINT_FLIP_GRACE_MS
);
await tx.organization.update({
where: { id: organizationId },
data: { featureFlags: stamped as Prisma.InputJsonValue },
});
return true;
});
if (!updated) {
throw new Response("Organization not found", { status: 404 });
}
// Org feature flags are embedded in every env of the org; drop all its cached env rows.
+45
View File
@@ -1,10 +1,13 @@
import { type z } from "zod";
import type { PrismaClient } from "@trigger.dev/database";
import { prisma, type PrismaClientOrTransaction } from "~/db.server";
import {
FEATURE_FLAG,
type FeatureFlagCatalogSchema,
type FeatureFlagKey,
FeatureFlagCatalog,
} from "~/v3/featureFlags";
import { stampMintKindFlip } from "~/v3/runOpsMigration/mintFlipGrace";
export type FlagsOptions<T extends FeatureFlagKey> = {
key: T;
@@ -146,3 +149,45 @@ export function makeSetMultipleFlags(_prisma: PrismaClientOrTransaction = prisma
return updatedFlags;
};
}
// Read -> stamp -> write the global mint-kind grace metadata in one transaction. The three
// FeatureFlag rows may not exist yet, so a row FOR UPDATE can't lock them; an advisory xact lock
// serializes concurrent global flips so one can't clobber another's grace stamp (mirrors per-org).
export async function applyGlobalMintKindFlip(
client: PrismaClient,
requestedFlags: Partial<z.infer<typeof FeatureFlagCatalogSchema>>,
graceMs: number
): Promise<{ key: string; value: any }[]> {
return client.$transaction(async (tx) => {
await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtext('runops-global-mint-kind-flip'))`;
const existingRows = await tx.featureFlag.findMany({
where: {
key: {
in: [
FEATURE_FLAG.runOpsMintKind,
FEATURE_FLAG.runOpsMintKindPrev,
FEATURE_FLAG.runOpsMintKindFlippedAt,
],
},
},
select: { key: true, value: true },
});
const existingGlobal: Record<string, unknown> = {};
for (const row of existingRows) {
existingGlobal[row.key] = row.value;
}
// Anchor the cutover to the control-plane DB clock, not this process's wall clock.
const [{ now }] = await tx.$queryRaw<{ now: Date }[]>`SELECT now() AS now`;
const stamped = stampMintKindFlip(
existingGlobal,
{ ...requestedFlags },
now.getTime(),
graceMs
) as Partial<z.infer<typeof FeatureFlagCatalogSchema>>;
return makeSetMultipleFlags(tx)(stamped);
});
}
+9
View File
@@ -19,6 +19,9 @@ export const FEATURE_FLAG = {
computeMigrationRequireTemplate: "computeMigrationRequireTemplate",
devBranchesEnabled: "devBranchesEnabled",
runOpsMintKind: "runOpsMintKind",
// Grace-linger stamp carried alongside runOpsMintKind on flip. See mintFlipGrace.ts.
runOpsMintKindPrev: "runOpsMintKindPrev",
runOpsMintKindFlippedAt: "runOpsMintKindFlippedAt",
} as const;
export const FeatureFlagCatalog = {
@@ -54,6 +57,10 @@ export const FeatureFlagCatalog = {
// Per-org run-ops-id mint cutover. Defaults to "cuid"; only honored when
// RUN_OPS_MINT_ENABLED is on AND isSplitEnabled() is true.
[FEATURE_FLAG.runOpsMintKind]: z.enum(["cuid", "runOpsId"]),
// Grace-linger stamp: the previously-effective kind and the flip timestamp, written
// by stampMintKindFlip on a genuine flip. Display-only (see ORG_LOCKED_FLAGS).
[FEATURE_FLAG.runOpsMintKindPrev]: z.enum(["cuid", "runOpsId"]),
[FEATURE_FLAG.runOpsMintKindFlippedAt]: z.string().datetime(),
};
export type FeatureFlagKey = keyof typeof FeatureFlagCatalog;
@@ -70,6 +77,8 @@ export const GLOBAL_LOCKED_FLAGS: FeatureFlagKey[] = [
export const ORG_LOCKED_FLAGS: FeatureFlagKey[] = [
FEATURE_FLAG.defaultWorkerInstanceGroupId,
FEATURE_FLAG.taskEventRepository,
FEATURE_FLAG.runOpsMintKindPrev,
FEATURE_FLAG.runOpsMintKindFlippedAt,
];
// Create a Zod schema from the existing catalog
@@ -0,0 +1,168 @@
// Real control-plane (single Postgres) proof that resolving the locked-worker version for many
// runs' distinct `lockedToVersionId`s is a GROUPED query, not one `backgroundWorker.findFirst`
// per id. The DB is never mocked: the call-counting proxy below delegates every call to the
// real Prisma client (the real query still runs against the real container) - it only tallies
// how many times each model.method pair was invoked.
import { postgresTest } from "@internal/testcontainers";
import { describe, expect } from "vitest";
import type { PrismaClient, PrismaReplicaClient } from "@trigger.dev/database";
import { ControlPlaneCache } from "./controlPlaneCache.server";
import { ControlPlaneResolver } from "./controlPlaneResolver.server";
// Wraps a real Prisma client so every `client.<model>.<method>(...)` call is tallied by
// `"<model>.<method>"` before being forwarded, unmodified, to the real delegate. No behavior is
// faked or short-circuited - this is instrumentation, not a mock.
function createCallCountingProxy<T extends object>(
client: T
): { client: T; counts: Map<string, number> } {
const counts = new Map<string, number>();
const bump = (key: string) => counts.set(key, (counts.get(key) ?? 0) + 1);
const wrappedModels = new Map<string, unknown>();
const wrapModel = (model: string, delegate: object) =>
new Proxy(delegate, {
get(target, prop, receiver) {
const value = Reflect.get(target, prop, receiver);
if (typeof prop === "string" && typeof value === "function") {
return (...args: unknown[]) => {
bump(`${model}.${prop}`);
return (value as (...a: unknown[]) => unknown).apply(target, args);
};
}
return value;
},
});
const proxy = new Proxy(client, {
get(target, prop, receiver) {
const value = Reflect.get(target, prop, receiver);
if (
typeof prop === "string" &&
value &&
typeof value === "object" &&
typeof (value as { findMany?: unknown }).findMany === "function"
) {
if (!wrappedModels.has(prop)) {
wrappedModels.set(prop, wrapModel(prop, value as object));
}
return wrappedModels.get(prop);
}
return value;
},
});
return { client: proxy as T, counts };
}
let n = 0;
async function seedEnv(prisma: PrismaClient) {
const s = n++;
const organization = await prisma.organization.create({
data: { title: `Org ${s}`, slug: `org-${s}` },
});
const project = await prisma.project.create({
data: {
name: `P ${s}`,
slug: `p-${s}`,
externalRef: `proj_${s}`,
organizationId: organization.id,
},
});
const environment = await prisma.runtimeEnvironment.create({
data: {
type: "PRODUCTION",
slug: `env-${s}`,
projectId: project.id,
organizationId: organization.id,
apiKey: `tr_${s}`,
pkApiKey: `pk_${s}`,
shortcode: `sc_${s}`,
},
});
return { organization, project, environment };
}
async function seedBackgroundWorker(
prisma: PrismaClient,
ctx: { projectId: string; runtimeEnvironmentId: string },
version: string
) {
const s = n++;
return prisma.backgroundWorker.create({
data: {
friendlyId: `worker_${s}`,
version,
contentHash: `hash_${s}`,
projectId: ctx.projectId,
runtimeEnvironmentId: ctx.runtimeEnvironmentId,
metadata: {},
},
});
}
describe("ControlPlaneResolver.resolveRunLockedWorkersByVersionIds", () => {
postgresTest(
"issues ONE grouped findMany and ZERO per-id findFirst for N distinct ids",
async ({ prisma }) => {
const { project, environment } = await seedEnv(prisma);
const workers = await Promise.all(
[0, 1, 2].map((i) =>
seedBackgroundWorker(
prisma,
{ projectId: project.id, runtimeEnvironmentId: environment.id },
`2024010${i}.0`
)
)
);
const { client: countedReplica, counts } = createCallCountingProxy(prisma);
const resolver = new ControlPlaneResolver({
controlPlanePrimary: prisma,
controlPlaneReplica: countedReplica as unknown as PrismaReplicaClient,
cache: new ControlPlaneCache(),
splitEnabled: () => true,
});
const ids = workers.map((w) => w.id);
const result = await resolver.resolveRunLockedWorkersByVersionIds(ids);
expect(counts.get("backgroundWorker.findMany") ?? 0).toBe(1);
expect(counts.get("backgroundWorker.findFirst") ?? 0).toBe(0);
for (const worker of workers) {
expect(result.get(worker.id)?.lockedToVersion?.version).toBe(worker.version);
}
}
);
postgresTest("cache-hit ids issue no query at all", async ({ prisma }) => {
const { project, environment } = await seedEnv(prisma);
const worker = await seedBackgroundWorker(
prisma,
{ projectId: project.id, runtimeEnvironmentId: environment.id },
"20240101.0"
);
const cache = new ControlPlaneCache();
const { client: countedReplica, counts } = createCallCountingProxy(prisma);
const resolver = new ControlPlaneResolver({
controlPlanePrimary: prisma,
controlPlaneReplica: countedReplica as unknown as PrismaReplicaClient,
cache,
splitEnabled: () => true,
});
// Warm the cache.
await resolver.resolveRunLockedWorkersByVersionIds([worker.id]);
expect(counts.get("backgroundWorker.findMany") ?? 0).toBe(1);
// Second call for the same id is served entirely from cache - no query at all.
const result = await resolver.resolveRunLockedWorkersByVersionIds([worker.id]);
expect(counts.get("backgroundWorker.findMany") ?? 0).toBe(1);
expect(counts.get("backgroundWorker.findFirst") ?? 0).toBe(0);
expect(result.get(worker.id)?.lockedToVersion?.version).toBe(worker.version);
});
});
@@ -70,6 +70,15 @@ function lockedWorkerKey(lockedById?: string | null, lockedToVersionId?: string
return `${lockedById ?? "_"}:${lockedToVersionId ?? "_"}`;
}
type LockedToVersionRow = {
id: string;
version: string;
sdkVersion: string;
runtime: string | null;
runtimeVersion: string | null;
supportsLazyAttempts: boolean;
};
export class ControlPlaneResolver {
private readonly controlPlanePrimary: PrismaClient;
private readonly controlPlaneReplica: PrismaReplicaClient;
@@ -243,6 +252,87 @@ export class ControlPlaneResolver {
};
}
/**
* Grouped counterpart to `resolveRunLockedWorker({ lockedToVersionId })`: resolves the
* `lockedToVersion` for many distinct BackgroundWorker ids with ONE `findMany` for the
* cache misses (BackgroundWorker is control-plane only, no run-ops residency split), instead
* of one `findFirst` per id. Cache keys/shape match `resolveRunLockedWorker`'s
* `lockedById=undefined` slot, so entries populated here are reused by that method and
* vice versa.
*/
async resolveRunLockedWorkersByVersionIds(
ids: string[]
): Promise<Map<string, ResolvedRunLockedWorker | null>> {
const result = new Map<string, ResolvedRunLockedWorker | null>();
const uniqueIds = [...new Set(ids)];
if (uniqueIds.length === 0) {
return result;
}
if (!this.splitEnabled()) {
const rows = await this.#queryLockedToVersionRows(this.controlPlanePrimary, uniqueIds);
for (const id of uniqueIds) {
result.set(id, this.#toResolvedRunLockedWorker(rows.get(id)));
}
return result;
}
const misses: string[] = [];
for (const id of uniqueIds) {
const cached = this.cache.getLockedWorker(lockedWorkerKey(undefined, id));
if (cached !== undefined) {
result.set(id, cached);
} else {
misses.push(id);
}
}
if (misses.length > 0) {
const rows = await this.#queryLockedToVersionRows(this.controlPlaneReplica, misses);
for (const id of misses) {
const value = this.#toResolvedRunLockedWorker(rows.get(id));
this.cache.setLockedWorker(lockedWorkerKey(undefined, id), value);
result.set(id, value);
}
}
return result;
}
async #queryLockedToVersionRows(
client: CpClient,
ids: string[]
): Promise<Map<string, LockedToVersionRow>> {
const rows = await client.backgroundWorker.findMany({
where: { id: { in: ids } },
select: {
id: true,
version: true,
sdkVersion: true,
runtime: true,
runtimeVersion: true,
supportsLazyAttempts: true,
},
});
return new Map(rows.map((row) => [row.id, row]));
}
#toResolvedRunLockedWorker(row: LockedToVersionRow | undefined): ResolvedRunLockedWorker | null {
if (!row) {
return null;
}
return {
lockedBy: null,
lockedToVersion: {
version: row.version,
sdkVersion: row.sdkVersion,
runtime: row.runtime,
runtimeVersion: row.runtimeVersion,
supportsLazyAttempts: row.supportsLazyAttempts,
},
};
}
async resolveWorkerVersion(args: {
environmentId: string;
backgroundWorkerId?: string;
@@ -0,0 +1,322 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
effectiveMintKind,
readMintResolution,
resolveMintFlag,
selectMintBaselineSource,
stampMintKindFlip,
type MintFlagResolution,
} from "./mintFlipGrace";
// GRACE-LINGER: during [flippedAt, flippedAt + GRACE) every process — stale or fresh —
// must resolve to the SAME (old) kind; at/after the cutover every process resolves to
// the SAME (new) kind. This collapses the cross-process divergence window.
const GRACE_MS = 90_000;
const T = 1_000_000;
describe("effectiveMintKind", () => {
beforeEach(() => vi.useFakeTimers());
afterEach(() => vi.useRealTimers());
it("returns r.kind directly when prev is missing", () => {
const r: MintFlagResolution = { kind: "cuid" };
expect(effectiveMintKind(r, T, GRACE_MS)).toBe("cuid");
});
it("returns r.kind directly when flippedAtMs is missing", () => {
const r: MintFlagResolution = { kind: "runOpsId", prev: "cuid" };
expect(effectiveMintKind(r, T, GRACE_MS)).toBe("runOpsId");
});
it("CORE: stale and fresh resolutions agree at every instant during grace, then both flip to the new kind at cutover", () => {
const stale: MintFlagResolution = { kind: "cuid" };
const fresh: MintFlagResolution = { kind: "runOpsId", prev: "cuid", flippedAtMs: T };
for (const now of [T, T + 1, T + 1_000, T + 45_000, T + GRACE_MS - 1]) {
const staleResolved = effectiveMintKind(stale, now, GRACE_MS);
const freshResolved = effectiveMintKind(fresh, now, GRACE_MS);
expect(staleResolved).toBe("cuid");
expect(freshResolved).toBe("cuid");
}
for (const now of [T + GRACE_MS, T + GRACE_MS + 1, T + GRACE_MS + 60_000]) {
expect(effectiveMintKind(fresh, now, GRACE_MS)).toBe("runOpsId");
}
});
it("boundary: exactly at flippedAt + GRACE resolves to the NEW kind", () => {
const fresh: MintFlagResolution = { kind: "runOpsId", prev: "cuid", flippedAtMs: T };
expect(effectiveMintKind(fresh, T + GRACE_MS - 1, GRACE_MS)).toBe("cuid");
expect(effectiveMintKind(fresh, T + GRACE_MS, GRACE_MS)).toBe("runOpsId");
});
});
describe("stampMintKindFlip", () => {
beforeEach(() => vi.useFakeTimers());
afterEach(() => vi.useRealTimers());
it("a genuine flip (cuid -> runOpsId) stamps prev and flippedAt", () => {
const existing = { runOpsMintKind: "cuid" };
const outgoing = { runOpsMintKind: "runOpsId" };
const result = stampMintKindFlip(existing, outgoing, T, GRACE_MS);
expect(result.runOpsMintKind).toBe("runOpsId");
expect(result.runOpsMintKindPrev).toBe("cuid");
expect(result.runOpsMintKindFlippedAt).toBe(new Date(T).toISOString());
});
it("defaults the existing effective kind to cuid when existing flags are null", () => {
const outgoing = { runOpsMintKind: "runOpsId" };
const result = stampMintKindFlip(null, outgoing, T, GRACE_MS);
expect(result.runOpsMintKind).toBe("runOpsId");
expect(result.runOpsMintKindPrev).toBe("cuid");
expect(result.runOpsMintKindFlippedAt).toBe(new Date(T).toISOString());
});
it("resubmitting the same target kind mid-grace carries the stamp forward untouched (does not reset the cutover clock)", () => {
const existing = {
runOpsMintKind: "runOpsId",
runOpsMintKindPrev: "cuid",
runOpsMintKindFlippedAt: new Date(T).toISOString(),
};
const now = T + 10_000;
const outgoing = { runOpsMintKind: "runOpsId", someOtherFlag: true };
const result = stampMintKindFlip(existing, outgoing, now, GRACE_MS);
expect(result.runOpsMintKind).toBe("runOpsId");
expect(result.runOpsMintKindPrev).toBe("cuid");
// Unrelated re-save: the cutover time must NOT slide forward.
expect(result.runOpsMintKindFlippedAt).toBe(new Date(T).toISOString());
expect(result.someOtherFlag).toBe(true);
});
it("an unchanged save after grace has elapsed carries the settled stamp forward and preserves unrelated flags", () => {
const existing = {
runOpsMintKind: "runOpsId",
runOpsMintKindPrev: "cuid",
runOpsMintKindFlippedAt: new Date(T).toISOString(),
};
const outgoing = { runOpsMintKind: "runOpsId", someOtherFlag: true };
const result = stampMintKindFlip(existing, outgoing, T + GRACE_MS + 5_000, GRACE_MS);
expect(result.runOpsMintKind).toBe("runOpsId");
expect(result.someOtherFlag).toBe(true);
expect(result.runOpsMintKindPrev).toBe("cuid");
expect(result.runOpsMintKindFlippedAt).toBe(new Date(T).toISOString());
});
it("a flip-back requested after the original grace has elapsed stamps prev := the new settled (now-effective) kind, timestamped now", () => {
const existing = {
runOpsMintKind: "runOpsId",
runOpsMintKindPrev: "cuid",
runOpsMintKindFlippedAt: new Date(T).toISOString(),
};
const now = T + GRACE_MS + 1_000;
const outgoing = { runOpsMintKind: "cuid" };
const result = stampMintKindFlip(existing, outgoing, now, GRACE_MS);
expect(result.runOpsMintKind).toBe("cuid");
expect(result.runOpsMintKindPrev).toBe("runOpsId");
expect(result.runOpsMintKindFlippedAt).toBe(new Date(now).toISOString());
});
it("a flip-back mid-grace re-stamps prev to the still-effective old kind, so it keeps serving that kind (no divergence)", () => {
const existing = {
runOpsMintKind: "runOpsId",
runOpsMintKindPrev: "cuid",
runOpsMintKindFlippedAt: new Date(T).toISOString(),
};
const now = T + 20_000;
const outgoing = { runOpsMintKind: "cuid" };
const result = stampMintKindFlip(existing, outgoing, now, GRACE_MS);
expect(result.runOpsMintKind).toBe("cuid");
expect(result.runOpsMintKindPrev).toBe("cuid");
expect(result.runOpsMintKindFlippedAt).toBe(new Date(now).toISOString());
});
it("leaves runOpsMintKind untouched when the save omits it (unrelated flag change: no inject, no spurious flip)", () => {
const existing = { runOpsMintKind: "runOpsId" };
const outgoing: Record<string, unknown> = { someOtherFlag: true };
const result = stampMintKindFlip(existing, outgoing, T, GRACE_MS);
// Must not inject a default kind or stamp a flip: doing so would pin the org to an explicit
// per-org override and make a later global flip silently skip it.
expect(result.runOpsMintKind).toBeUndefined();
expect(result.runOpsMintKindPrev).toBeUndefined();
expect(result.runOpsMintKindFlippedAt).toBeUndefined();
expect(result.someOtherFlag).toBe(true);
});
it("treats a malformed existing flippedAt as no stamp", () => {
const existing = {
runOpsMintKind: "runOpsId",
runOpsMintKindPrev: "cuid",
runOpsMintKindFlippedAt: "not-a-date",
};
const outgoing = { runOpsMintKind: "runOpsId" };
const result = stampMintKindFlip(existing, outgoing, T, GRACE_MS);
expect(result.runOpsMintKind).toBe("runOpsId");
// The malformed flippedAt is carried forward verbatim, but an unparseable timestamp is
// inert when resolved (Date.parse -> NaN -> effectiveMintKind returns the target kind).
expect(result.runOpsMintKindFlippedAt).toBe("not-a-date");
expect(
effectiveMintKind({ kind: "runOpsId", prev: "cuid", flippedAtMs: NaN }, T, GRACE_MS)
).toBe("runOpsId");
});
});
// SOURCE-CONSISTENCY: the kind and its grace stamp must come from the SAME source. A per-org
// runOpsMintKind override wins both the kind and the stamp; with no per-org override, BOTH the
// kind and the stamp come from the global FeatureFlag rows. Never mix (e.g. a per-org kind with
// the global stamp), which would date a grace window against the wrong flip.
describe("resolveMintFlag", () => {
it("a per-org override wins the kind AND owns the stamp, ignoring the global stamp entirely", () => {
const perOrg = {
runOpsMintKind: "runOpsId",
runOpsMintKindPrev: "cuid",
runOpsMintKindFlippedAt: new Date(T).toISOString(),
};
const global = {
runOpsMintKind: "cuid",
runOpsMintKindPrev: "runOpsId",
runOpsMintKindFlippedAt: new Date(T + 500_000).toISOString(),
};
expect(resolveMintFlag(perOrg, global)).toEqual({
kind: "runOpsId",
prev: "cuid",
flippedAtMs: T,
});
});
it("with NO per-org override, the kind AND the stamp come from the global rows (global flip is graced)", () => {
const global = {
runOpsMintKind: "runOpsId",
runOpsMintKindPrev: "cuid",
runOpsMintKindFlippedAt: new Date(T).toISOString(),
};
const resolution = resolveMintFlag({}, global);
expect(resolution).toEqual({ kind: "runOpsId", prev: "cuid", flippedAtMs: T });
// Mid-grace: a global flip resolves to the OLD kind for the whole window.
expect(effectiveMintKind(resolution, T + GRACE_MS - 1, GRACE_MS)).toBe("cuid");
expect(effectiveMintKind(resolution, T + GRACE_MS, GRACE_MS)).toBe("runOpsId");
});
it("a per-org override with NO per-org stamp does NOT borrow the global stamp (kind stays ungraced)", () => {
const perOrg = { runOpsMintKind: "runOpsId" };
const global = {
runOpsMintKind: "cuid",
runOpsMintKindPrev: "cuid",
runOpsMintKindFlippedAt: new Date(T).toISOString(),
};
expect(resolveMintFlag(perOrg, global)).toEqual({
kind: "runOpsId",
prev: undefined,
flippedAtMs: undefined,
});
});
it("defaults to cuid with no stamp when neither source has a kind", () => {
expect(resolveMintFlag({}, {})).toEqual({
kind: "cuid",
prev: undefined,
flippedAtMs: undefined,
});
expect(resolveMintFlag(null, null)).toEqual({
kind: "cuid",
prev: undefined,
flippedAtMs: undefined,
});
});
});
// #3b: an org's FIRST per-org runOpsMintKind override must be stamped against the currently
// EFFECTIVE kind — the global FeatureFlag resolution when the org has no override yet — not the
// hardcoded default "cuid". selectMintBaselineSource picks that same source (per-org blob if it
// sets runOpsMintKind, else the global rows) so stampMintKindFlip's baseline (storedKind +
// prev + carry-forward) is correct.
describe("selectMintBaselineSource", () => {
it("returns the per-org blob when it sets runOpsMintKind (override owns the baseline)", () => {
const perOrg = { runOpsMintKind: "cuid" };
const global = { runOpsMintKind: "runOpsId" };
expect(selectMintBaselineSource(perOrg, global)).toBe(perOrg);
});
it("falls back to the global rows when the org has no runOpsMintKind override", () => {
const perOrg = { someOtherFlag: true };
const global = { runOpsMintKind: "runOpsId" };
expect(selectMintBaselineSource(perOrg, global)).toBe(global);
});
it("returns an empty record when neither source sets a kind", () => {
expect(selectMintBaselineSource(null, null)).toEqual({});
expect(selectMintBaselineSource({ someOtherFlag: true }, null)).toEqual({});
});
});
describe("first per-org override stamps prev against the effective GLOBAL kind (#3b)", () => {
beforeEach(() => vi.useFakeTimers());
afterEach(() => vi.useRealTimers());
it("global=runOpsId, org's FIRST override -> cuid: genuine flip, prev=runOpsId, graced", () => {
const globalFlags = { runOpsMintKind: "runOpsId" };
const orgExisting = {}; // no per-org override yet
const outgoing = { runOpsMintKind: "cuid" };
const result = stampMintKindFlip(
selectMintBaselineSource(orgExisting, globalFlags),
outgoing,
T,
GRACE_MS
);
expect(result.runOpsMintKind).toBe("cuid");
// Previously stamped prev="cuid" (the hardcoded default) OR skipped the flip entirely; must
// now be "runOpsId" (the effective global kind) so the org serves it through the grace window.
expect(result.runOpsMintKindPrev).toBe("runOpsId");
expect(result.runOpsMintKindFlippedAt).toBe(new Date(T).toISOString());
const resolution = readMintResolution(result);
expect(effectiveMintKind(resolution, T + GRACE_MS - 1, GRACE_MS)).toBe("runOpsId");
expect(effectiveMintKind(resolution, T + GRACE_MS, GRACE_MS)).toBe("cuid");
});
it("global=runOpsId, org's FIRST override -> runOpsId (redundant): NOT a spurious flip, no phantom regression to cuid", () => {
const globalFlags = { runOpsMintKind: "runOpsId" };
const orgExisting = {};
const outgoing = { runOpsMintKind: "runOpsId" };
const result = stampMintKindFlip(
selectMintBaselineSource(orgExisting, globalFlags),
outgoing,
T,
GRACE_MS
);
expect(result.runOpsMintKind).toBe("runOpsId");
// Previously: storedKind defaulted to "cuid", so this looked like a genuine flip and stamped
// prev="cuid" — making the org serve cuid during a phantom grace window (a regression).
expect(result.runOpsMintKindPrev).toBeUndefined();
expect(result.runOpsMintKindFlippedAt).toBeUndefined();
});
it("mid-grace global flip carried into a redundant org override keeps the global stamp", () => {
const globalFlags = {
runOpsMintKind: "runOpsId",
runOpsMintKindPrev: "cuid",
runOpsMintKindFlippedAt: new Date(T).toISOString(),
};
const orgExisting = {};
const outgoing = { runOpsMintKind: "runOpsId" };
const result = stampMintKindFlip(
selectMintBaselineSource(orgExisting, globalFlags),
outgoing,
T + 10_000,
GRACE_MS
);
expect(result.runOpsMintKind).toBe("runOpsId");
expect(result.runOpsMintKindPrev).toBe("cuid");
expect(result.runOpsMintKindFlippedAt).toBe(new Date(T).toISOString());
});
});
@@ -0,0 +1,118 @@
import type { RunIdMintKind } from "./runOpsMintKind.server";
export type { RunIdMintKind };
export type MintFlagResolution = {
kind: RunIdMintKind;
prev?: RunIdMintKind;
flippedAtMs?: number;
};
const DEFAULT_MINT_KIND: RunIdMintKind = "cuid";
// Cutover boundary. `nowMs` is the reader's wall clock but `flippedAtMs` (in `r`) is DB-clock
// (admin routes) — so this assumes NTP-synced hosts with skew << graceMs, letting every process
// cross [flippedAtMs, flippedAtMs + graceMs) together (OLD then NEW). Accepted residual: a badly
// mis-synced host can cross early/late and briefly reopen a skew-wide cross-DB duplicate window.
export function effectiveMintKind(
r: MintFlagResolution,
nowMs: number,
graceMs: number
): RunIdMintKind {
if (r.prev === undefined || r.flippedAtMs === undefined) {
return r.kind;
}
return nowMs < r.flippedAtMs + graceMs ? r.prev : r.kind;
}
function readMintKind(flags: Record<string, unknown>, key: string): RunIdMintKind | undefined {
const value = flags[key];
return value === "cuid" || value === "runOpsId" ? value : undefined;
}
// Reads the { kind, prev, flippedAtMs } trio out of one flag record — either an org's
// featureFlags override blob or the global FeatureFlag rows projected into a record. Pure.
export function readMintResolution(
flags: Record<string, unknown> | null | undefined
): MintFlagResolution {
const source = flags ?? {};
const kind = readMintKind(source, "runOpsMintKind") ?? DEFAULT_MINT_KIND;
const prev = readMintKind(source, "runOpsMintKindPrev");
const flippedAtRaw = source.runOpsMintKindFlippedAt;
const parsed = typeof flippedAtRaw === "string" ? Date.parse(flippedAtRaw) : NaN;
const flippedAtMs = Number.isNaN(parsed) ? undefined : parsed;
return { kind, prev, flippedAtMs };
}
// SOURCE-CONSISTENT resolution: a per-org runOpsMintKind override wins the kind AND owns the
// grace stamp; with no per-org override, the kind AND the stamp both come from the global rows.
// The stamp is never read from a different source than the kind, which would date a grace
// window against the wrong flip.
export function resolveMintFlag(
perOrgOverrides: Record<string, unknown> | null | undefined,
globalFlags: Record<string, unknown> | null | undefined
): MintFlagResolution {
if (readMintKind(perOrgOverrides ?? {}, "runOpsMintKind") !== undefined) {
return readMintResolution(perOrgOverrides);
}
return readMintResolution(globalFlags);
}
// Picks the flag record that currently determines an org's effective mint kind: the per-org
// override blob when it sets runOpsMintKind, otherwise the global FeatureFlag rows. Same source
// resolveMintFlag() reads, but returned as a record so it can seed stampMintKindFlip's baseline
// (storedKind for flip-detection, prev on a genuine flip, and stamp carry-forward). This makes an
// org's first per-org override stamp against the effective GLOBAL kind, not the default "cuid".
export function selectMintBaselineSource(
perOrgOverrides: Record<string, unknown> | null | undefined,
globalFlags: Record<string, unknown> | null | undefined
): Record<string, unknown> {
if (readMintKind(perOrgOverrides ?? {}, "runOpsMintKind") !== undefined) {
return perOrgOverrides ?? {};
}
return globalFlags ?? {};
}
function resolveEffectiveFromFlags(
flags: Record<string, unknown> | null | undefined,
nowMs: number,
graceMs: number
): RunIdMintKind {
return effectiveMintKind(readMintResolution(flags), nowMs, graceMs);
}
// Stamps a grace window only when the outgoing TARGET kind differs from the stored one (a
// genuine flip); prev := the currently-effective kind. A save that leaves the target kind
// unchanged carries any in-flight stamp forward, so it can't reset the cutover clock.
export function stampMintKindFlip(
existingFlags: Record<string, unknown> | null | undefined,
outgoingFlags: Record<string, unknown>,
nowMs: number,
graceMs: number
): Record<string, unknown> {
// Only act when the save actually SETS runOpsMintKind. Omitting it (an unrelated flag change)
// must not inject the default kind, which would pin the org and make a later global flip skip it.
const outgoingKind = readMintKind(outgoingFlags, "runOpsMintKind");
if (outgoingKind === undefined) {
return outgoingFlags;
}
const storedKind = readMintKind(existingFlags ?? {}, "runOpsMintKind") ?? DEFAULT_MINT_KIND;
if (outgoingKind !== storedKind) {
// Genuine target change: serve the currently-effective kind through the new grace window.
outgoingFlags.runOpsMintKindPrev = resolveEffectiveFromFlags(existingFlags, nowMs, graceMs);
outgoingFlags.runOpsMintKindFlippedAt = new Date(nowMs).toISOString();
return outgoingFlags;
}
const existing = existingFlags ?? {};
const existingPrev = existing.runOpsMintKindPrev;
const existingFlippedAt = existing.runOpsMintKindFlippedAt;
if (existingPrev !== undefined) {
outgoingFlags.runOpsMintKindPrev = existingPrev;
}
if (existingFlippedAt !== undefined) {
outgoingFlags.runOpsMintKindFlippedAt = existingFlippedAt;
}
return outgoingFlags;
}
@@ -2,15 +2,15 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { BoundedTtlCache } from "~/services/realtime/boundedTtlCache";
import { computeRunIdMintKind, type RunIdMintKind } from "./runOpsMintKind.server";
// LOCK of the CURRENT (intentional) flip-latency behavior, NOT a change request.
// resolveRunIdMintKind caches the per-org mint kind in a process-singleton
// BoundedTtlCache (TTL RUN_OPS_MINT_FLAG_CACHE_TTL_MS, 30000ms default) with get/set
// and NO invalidation hook (runOpsMintKind.server.ts:38-45,56-81). So after a flag
// flip a process keeps minting the stale kind until its cached entry expires; in
// multi-instance prod each process expires independently. This suite reconstructs the
// same flag fn over a real cache and pins both edges of that window.
// LOCK of the raw per-process cache's flip-latency behavior in isolation, NOT a change
// request. Production resolveRunIdMintKind now wraps this same staleness in a deterministic
// wall-clock grace window (mintFlipGrace.ts) so every process resolves to the SAME effective
// kind for the whole window, then all cross together. This raw staleness is now an
// intentional, safe input to that resolution. computeRunIdMintKind is unaffected, so this
// suite's assertions stand as-is.
// Mirror of resolveRunIdMintKind's flag fn (runOpsMintKind.server.ts:56-81).
// Bare cached-flag closure — deliberately NOT the production flag fn, which now layers
// grace resolution on top (see runOpsMintKind.server.ts).
function makeCachedFlag(
cache: BoundedTtlCache<RunIdMintKind>,
liveFlag: () => RunIdMintKind
@@ -4,7 +4,8 @@ import { logger } from "~/services/logger.server";
import { BoundedTtlCache } from "~/services/realtime/boundedTtlCache";
import { singleton } from "~/utils/singleton";
import { FEATURE_FLAG } from "~/v3/featureFlags";
import { makeFlag } from "~/v3/featureFlags.server";
import { DEFAULT_CP_CACHE_TTL_MS } from "./controlPlaneCache.server";
import { effectiveMintKind, resolveMintFlag, type MintFlagResolution } from "./mintFlipGrace";
import { isSplitEnabled } from "./splitMode.server";
export type RunIdMintKind = "cuid" | "runOpsId";
@@ -34,16 +35,33 @@ export async function computeRunIdMintKind(
}
// ENV-BOUND wrapper — the only place env/$replica/isSplitEnabled are read.
const flagFn = singleton("runOpsMintFlag", () => makeFlag($replica));
const mintCache = singleton(
"runOpsMintCache",
() =>
new BoundedTtlCache<RunIdMintKind>(
new BoundedTtlCache<MintFlagResolution>(
env.RUN_OPS_MINT_FLAG_CACHE_TTL_MS,
env.RUN_OPS_MINT_FLAG_CACHE_MAX_ENTRIES
)
);
// BOOT-TIME SAFETY CHECK (warning only, never throws): the grace window only collapses
// the cross-process divergence window if it outlasts BOTH caches a flag flip has to drain
// through — this process's own mint-flag cache AND the org-flags control-plane cache a
// stale process might still be reading through. If it doesn't, warn loudly but keep booting.
const controlPlaneCacheTtlMs = env.CONTROL_PLANE_CACHE_TTL_MS ?? DEFAULT_CP_CACHE_TTL_MS;
if (env.RUN_OPS_MINT_FLIP_GRACE_MS <= env.RUN_OPS_MINT_FLAG_CACHE_TTL_MS + controlPlaneCacheTtlMs) {
logger.warn(
"[runOpsMintKind] RUN_OPS_MINT_FLIP_GRACE_MS does not exceed the sum of " +
"RUN_OPS_MINT_FLAG_CACHE_TTL_MS and the control-plane cache TTL; a flag flip can still " +
"cross-DB-duplicate a concurrent root trigger during the divergence window",
{
RUN_OPS_MINT_FLIP_GRACE_MS: env.RUN_OPS_MINT_FLIP_GRACE_MS,
RUN_OPS_MINT_FLAG_CACHE_TTL_MS: env.RUN_OPS_MINT_FLAG_CACHE_TTL_MS,
controlPlaneCacheTtlMs,
}
);
}
export async function resolveRunIdMintKind(environment: {
organizationId: string;
id: string;
@@ -54,10 +72,14 @@ export async function resolveRunIdMintKind(environment: {
masterEnabled: env.RUN_OPS_MINT_ENABLED,
splitEnabled: isSplitEnabled,
flag: async (orgId, orgFeatureFlags) => {
// The cache stores only "cuid"|"runOpsId" (never undefined), so the cache's
// "stored-undefined == miss" caveat never applies here.
// The cache stores the full { kind, prev, flippedAtMs } trio (never undefined), so the
// cache's "stored-undefined == miss" caveat never applies here. A cache HIT still passes
// back through effectiveMintKind so a cached-but-stale entry crosses the grace boundary
// on schedule, without needing an invalidation hook.
const cached = mintCache.get(orgId);
if (cached !== undefined) return cached;
if (cached !== undefined) {
return effectiveMintKind(cached, Date.now(), env.RUN_OPS_MINT_FLIP_GRACE_MS);
}
// Hot-path pass-through: use the org flags the authenticated environment already
// carries; only fall back to a DB read when the caller did NOT pass them (non-trigger
@@ -72,13 +94,34 @@ export async function resolveRunIdMintKind(environment: {
})
)?.featureFlags;
const kind = await flagFn({
key: FEATURE_FLAG.runOpsMintKind,
defaultValue: "cuid",
overrides: (overrides as Record<string, unknown>) ?? {},
const overridesRecord = (overrides as Record<string, unknown>) ?? {};
// One global read over the three mint-flag keys (kind + grace stamp), folded into the
// single cache-miss round-trip. This replaces the former single-key flag read, so a
// GLOBAL flip is now grace-stamped WITHOUT adding any new per-mint/per-resolve query.
// (The cache-hit branch above never touches the DB.)
const globalRows = await $replica.featureFlag.findMany({
where: {
key: {
in: [
FEATURE_FLAG.runOpsMintKind,
FEATURE_FLAG.runOpsMintKindPrev,
FEATURE_FLAG.runOpsMintKindFlippedAt,
],
},
},
select: { key: true, value: true },
});
mintCache.set(orgId, kind);
return kind;
const globalFlags: Record<string, unknown> = {};
for (const row of globalRows) {
globalFlags[row.key] = row.value;
}
// Source-consistent: a per-org override wins the kind AND its stamp; otherwise the
// global row wins the kind AND its stamp.
const resolution: MintFlagResolution = resolveMintFlag(overridesRecord, globalFlags);
mintCache.set(orgId, resolution);
return effectiveMintKind(resolution, Date.now(), env.RUN_OPS_MINT_FLIP_GRACE_MS);
},
});
}
@@ -0,0 +1,337 @@
// Flow-level seam proof for ApiBatchResultsPresenter, ABOVE the run-store-level coverage in
// internal-packages/run-store (batchItemMisroute, batchCompletionResidency). Proves the batch
// RESULTS READ assembles correctly when one batch's members are genuinely split across the real
// dedicated run-ops subset schema (prisma17 / RunOpsPrismaClient) and the full control-plane
// schema (prisma14) — not a mirrored full schema on both sides. No mocks.
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
import type { PrismaClient } from "@trigger.dev/database";
import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic";
import { describe, expect, vi } from "vitest";
import type { PrismaReplicaClient } from "~/db.server";
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
import { ApiBatchResultsPresenter } from "~/presenters/v3/ApiBatchResultsPresenter.server";
vi.setConfig({ testTimeout: 60_000 });
// A prisma handle that throws on any access — proves the passthrough constructor args are never
// touched on the split path.
const throwingPrisma = new Proxy(
{},
{
get(_t, prop) {
throw new Error(
`passthrough handle must not be touched on the split path (got .${String(prop)})`
);
},
}
) as unknown as PrismaReplicaClient;
// 25-char cuid-shaped id, no v1 version marker at index 25 -> classifies LEGACY.
function generateLegacyCuid(): string {
const alphabet = "0123456789abcdefghijklmnopqrstuvwxyz";
const suffix = Array.from({ length: 24 }, () => alphabet[Math.floor(Math.random() * 36)]).join(
""
);
return `c${suffix}`;
}
let seedCounter = 0;
// TaskRun on the full control-plane schema has real FKs into RuntimeEnvironment/Project.
async function seedLegacyEnv(prisma14: PrismaClient, slug: string) {
const n = seedCounter++;
const organization = await prisma14.organization.create({
data: { title: `Org ${slug}`, slug: `org-${slug}-${n}` },
});
const project = await prisma14.project.create({
data: {
name: `Proj ${slug}`,
slug: `proj-${slug}-${n}`,
organizationId: organization.id,
externalRef: `ext-${slug}-${n}`,
},
});
const environment = await prisma14.runtimeEnvironment.create({
data: {
slug: `env-${slug}-${n}`,
type: "PRODUCTION",
projectId: project.id,
organizationId: organization.id,
apiKey: `api-${slug}-${n}`,
pkApiKey: `pk-${slug}-${n}`,
shortcode: `sc-${slug}-${n}`,
},
});
return { organization, project, environment };
}
type SeedCtx = Awaited<ReturnType<typeof seedLegacyEnv>>;
type MemberSeed = {
id: string;
friendlyId: string;
status: "COMPLETED_SUCCESSFULLY" | "COMPLETED_WITH_ERRORS";
output?: string;
error?: unknown;
};
// Drop the TaskRunAttempt worker/queue FKs so attempts can be seeded without standing up
// BackgroundWorker/TaskQueue parents — incidental to this read path.
async function relaxLegacyAttemptFk(prisma14: PrismaClient) {
for (const sql of [
`ALTER TABLE "TaskRunAttempt" DROP CONSTRAINT IF EXISTS "TaskRunAttempt_backgroundWorkerId_fkey"`,
`ALTER TABLE "TaskRunAttempt" DROP CONSTRAINT IF EXISTS "TaskRunAttempt_backgroundWorkerTaskId_fkey"`,
`ALTER TABLE "TaskRunAttempt" DROP CONSTRAINT IF EXISTS "TaskRunAttempt_queueId_fkey"`,
]) {
await prisma14.$executeRawUnsafe(sql);
}
}
async function seedLegacyMember(prisma14: PrismaClient, ctx: SeedCtx, m: MemberSeed) {
const run = await prisma14.taskRun.create({
data: {
id: m.id,
friendlyId: m.friendlyId,
taskIdentifier: "my-task",
status: m.status,
payload: JSON.stringify({}),
payloadType: "application/json",
traceId: m.id,
spanId: m.id,
queue: "main",
runtimeEnvironmentId: ctx.environment.id,
projectId: ctx.project.id,
organizationId: ctx.organization.id,
environmentType: "PRODUCTION",
engine: "V2",
},
});
await prisma14.taskRunAttempt.create({
data: {
friendlyId: `attempt_${m.id}`,
number: 1,
taskRunId: run.id,
backgroundWorkerId: "bw",
backgroundWorkerTaskId: "bwt",
runtimeEnvironmentId: ctx.environment.id,
queueId: "q",
status: m.status === "COMPLETED_SUCCESSFULLY" ? "COMPLETED" : "FAILED",
output: m.output,
outputType: "application/json",
error: m.error as any,
},
});
return run;
}
// TaskRun/TaskRunAttempt on the DEDICATED (run-ops subset) schema: runtimeEnvironmentId,
// organizationId, projectId, backgroundWorkerId/backgroundWorkerTaskId/queueId are all
// scalar-only there (no real relation) — no parent rows to seed, unlike the legacy side.
async function seedNewMember(
prisma17: RunOpsPrismaClient,
ctx: { envId: string; orgId: string; projectId: string },
m: MemberSeed
) {
await prisma17.taskRun.create({
data: {
id: m.id,
engine: "V2",
status: m.status,
friendlyId: m.friendlyId,
runtimeEnvironmentId: ctx.envId,
environmentType: "PRODUCTION",
organizationId: ctx.orgId,
projectId: ctx.projectId,
taskIdentifier: "my-task",
payload: JSON.stringify({}),
payloadType: "application/json",
traceContext: {},
traceId: m.id,
spanId: m.id,
queue: "main",
isTest: false,
taskEventStore: "taskEvent",
depth: 0,
},
});
await prisma17.taskRunAttempt.create({
data: {
friendlyId: `attempt_${m.id}`,
number: 1,
taskRunId: m.id,
backgroundWorkerId: "bw",
backgroundWorkerTaskId: "bwt",
runtimeEnvironmentId: ctx.envId,
queueId: "q",
status: m.status === "COMPLETED_SUCCESSFULLY" ? "COMPLETED" : "FAILED",
output: m.output,
outputType: "application/json",
error: m.error as any,
},
});
}
// BatchTaskRunItem.taskRunId is a real FK to TaskRun on the dedicated schema too. A batch seeded
// on the dedicated DB whose items reference a LEGACY-resident (or wholly missing) member has no
// local row for that member — drop the FK so the cross-seam item row can exist, matching the
// physical reality of a split batch.
async function relaxNewBatchItemFk(prisma17: RunOpsPrismaClient) {
await prisma17.$executeRawUnsafe(
`ALTER TABLE "BatchTaskRunItem" DROP CONSTRAINT IF EXISTS "BatchTaskRunItem_taskRunId_fkey"`
);
}
async function seedBatchOnNew(
prisma17: RunOpsPrismaClient,
envId: string,
friendlyId: string,
memberIds: string[]
) {
const batch = await prisma17.batchTaskRun.create({
data: {
friendlyId,
runtimeEnvironmentId: envId,
runCount: memberIds.length,
runIds: [],
batchVersion: "runengine:v2",
},
});
// Items in a deterministic order so the result `items` order is assertable.
for (const taskRunId of memberIds) {
await prisma17.batchTaskRunItem.create({
data: { batchTaskRunId: batch.id, taskRunId, status: "COMPLETED" },
});
}
return batch;
}
const env = (ctx: SeedCtx) =>
({
id: ctx.environment.id,
type: ctx.environment.type,
slug: ctx.environment.slug,
organizationId: ctx.organization.id,
organization: { slug: ctx.organization.slug, title: ctx.organization.title },
projectId: ctx.project.id,
project: { name: ctx.project.name },
}) as unknown as AuthenticatedEnvironment;
describe("ApiBatchResultsPresenter split mode — real run-ops dedicated schema seam", () => {
// The core assembly proof: one batch, members genuinely split across the two physically
// distinct, differently-shaped DBs (dedicated subset vs full control-plane).
heteroRunOpsPostgresTest(
"a batch with members split across the dedicated NEW schema and the legacy schema returns the complete union",
async ({ prisma14, prisma17 }: { prisma14: PrismaClient; prisma17: RunOpsPrismaClient }) => {
const ctx = await seedLegacyEnv(prisma14, "split-new-legacy");
await relaxLegacyAttemptFk(prisma14);
await relaxNewBatchItemFk(prisma17);
const newMemberId = generateRunOpsId();
expect(newMemberId.length).toBe(26);
const legacyMemberId = generateLegacyCuid();
expect(legacyMemberId.length).toBe(25);
// NEW-resident member: lives ONLY on the dedicated run-ops schema (prisma17).
await seedNewMember(
prisma17,
{ envId: ctx.environment.id, orgId: ctx.organization.id, projectId: ctx.project.id },
{
id: newMemberId,
friendlyId: "run_new_member",
status: "COMPLETED_SUCCESSFULLY",
output: JSON.stringify({ from: "new" }),
}
);
// LEGACY-resident member: lives ONLY on the full control-plane schema (prisma14).
await seedLegacyMember(prisma14, ctx, {
id: legacyMemberId,
friendlyId: "run_legacy_member",
status: "COMPLETED_WITH_ERRORS",
error: { type: "BUILT_IN_ERROR", name: "Err", message: "boom", stackTrace: "" },
});
// The batch row + items live on the NEW dedicated DB; items reference both members.
const batchFriendlyId = "batch_split_seam";
await seedBatchOnNew(prisma17, ctx.environment.id, batchFriendlyId, [
newMemberId,
legacyMemberId,
]);
const presenter = new ApiBatchResultsPresenter(throwingPrisma, throwingPrisma, {
splitEnabled: true,
newClient: prisma17 as unknown as PrismaReplicaClient,
legacyReplica: prisma14 as unknown as PrismaReplicaClient,
});
const result = await presenter.call(batchFriendlyId, env(ctx));
expect(result).toBeDefined();
expect(result!.id).toBe(batchFriendlyId);
expect(result!.items).toHaveLength(2);
// Order follows item order: the NEW-resident member first, the legacy-resident second.
const [first, second] = result!.items;
expect(first).toEqual({
ok: true,
id: "run_new_member",
taskIdentifier: "my-task",
output: JSON.stringify({ from: "new" }),
outputType: "application/json",
});
expect(second).toMatchObject({
ok: false,
id: "run_legacy_member",
taskIdentifier: "my-task",
});
}
);
// Dangling member: an item id that resolves on NEITHER the dedicated NEW schema nor the legacy
// schema must be dropped from the result, not thrown — the presenter degrades to the reachable
// set. Give the missing id a legacy shape so it also exercises the "not NEW-shaped -> legacy
// candidate" branch, proving the legacy probe finding nothing doesn't throw either.
heteroRunOpsPostgresTest(
"a member id that resolves on neither the dedicated NEW schema nor the legacy schema is dropped, not thrown",
async ({ prisma14, prisma17 }: { prisma14: PrismaClient; prisma17: RunOpsPrismaClient }) => {
const ctx = await seedLegacyEnv(prisma14, "dangling");
await relaxNewBatchItemFk(prisma17);
const presentId = generateRunOpsId();
const missingId = generateLegacyCuid(); // referenced by an item but seeded on NEITHER DB
await seedNewMember(
prisma17,
{ envId: ctx.environment.id, orgId: ctx.organization.id, projectId: ctx.project.id },
{
id: presentId,
friendlyId: "run_present",
status: "COMPLETED_SUCCESSFULLY",
output: JSON.stringify({ present: true }),
}
);
const batchFriendlyId = "batch_dangling_seam";
await seedBatchOnNew(prisma17, ctx.environment.id, batchFriendlyId, [presentId, missingId]);
const presenter = new ApiBatchResultsPresenter(throwingPrisma, throwingPrisma, {
splitEnabled: true,
newClient: prisma17 as unknown as PrismaReplicaClient,
legacyReplica: prisma14 as unknown as PrismaReplicaClient,
});
const result = await presenter.call(batchFriendlyId, env(ctx));
expect(result).toBeDefined();
expect(result!.id).toBe(batchFriendlyId);
// The dangling member is silently dropped; the reachable member still returns — the call
// must not throw despite the item referencing a run absent from both physical DBs.
expect(result!.items).toHaveLength(1);
expect(result!.items[0]).toMatchObject({ ok: true, id: "run_present" });
}
);
});
@@ -0,0 +1,251 @@
// RED→GREEN: kill the #callSplit per-member N+1 in ApiBatchResultsPresenter.
//
// Today, #callSplit hydrates every batch member independently via `readThroughRun` inside
// `Promise.all(batchRun.items.map(...))` — that is one (up to two, new-then-legacy) `taskRun`
// query PER member. `#callPassthrough` already does the grouped one-query form via
// `this.runStore.findRuns({ where: { id: { in: taskRunIds } } })`.
//
// The fix replaces the per-member fan-out with ONE grouped call to the RunStore's
// `findRunsByIds` method, mirroring `#callPassthrough`. This test proves the query-count
// reduction with a call-counting Proxy over a REAL testcontainer Postgres client: every call is
// delegated unchanged to the real client (the DB still runs the query) — this is instrumentation,
// not a mock.
import { PostgresRunStore } from "@internal/run-store";
import { postgresTest } from "@internal/testcontainers";
import type { PrismaClient } from "@trigger.dev/database";
import { describe, expect } from "vitest";
import type { PrismaReplicaClient } from "~/db.server";
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
import { ApiBatchResultsPresenter } from "~/presenters/v3/ApiBatchResultsPresenter.server";
// 26-char run-ops v1 body: 24-char base32hex core + region char + version "1" at index 25.
// ownerEngine classifies residency by the VERSION CHAR AT INDEX 25, not by length — a naive id
// generator would misclassify this as LEGACY unless the last two chars are a valid region+version.
function newRunId(c: string) {
return c.repeat(24) + "01";
}
type CallCounts = { findMany: number; findFirst: number };
// Wrap a REAL client's `taskRun` delegate to tally findMany/findFirst calls, delegating every
// call unchanged to the real client (the DB still runs the query — pure instrumentation).
function countingClient(real: PrismaClient): { client: PrismaClient; counts: CallCounts } {
const counts: CallCounts = { findMany: 0, findFirst: 0 };
const countingTaskRun = new Proxy((real as any).taskRun, {
get(target, prop) {
if (prop === "findMany" || prop === "findFirst") {
counts[prop as "findMany" | "findFirst"]++;
}
return (target as any)[prop];
},
});
const client = new Proxy(real, {
get(target, prop) {
if (prop === "taskRun") {
return countingTaskRun;
}
return (target as any)[prop];
},
}) as PrismaClient;
return { client, counts };
}
let seedCounter = 0;
async function seedEnv(prisma: PrismaClient, slug: string) {
const n = seedCounter++;
const organization = await prisma.organization.create({
data: { title: `Org ${slug}`, slug: `org-${slug}-${n}` },
});
const project = await prisma.project.create({
data: {
name: `Proj ${slug}`,
slug: `proj-${slug}-${n}`,
organizationId: organization.id,
externalRef: `ext-${slug}-${n}`,
},
});
const environment = await prisma.runtimeEnvironment.create({
data: {
slug: `env-${slug}-${n}`,
type: "PRODUCTION",
projectId: project.id,
organizationId: organization.id,
apiKey: `api-${slug}-${n}`,
pkApiKey: `pk-${slug}-${n}`,
shortcode: `sc-${slug}-${n}`,
},
});
return { organization, project, environment };
}
type SeedCtx = Awaited<ReturnType<typeof seedEnv>>;
type MemberSeed = {
id: string;
friendlyId: string;
status: "COMPLETED_SUCCESSFULLY" | "COMPLETED_WITH_ERRORS";
output?: string;
error?: unknown;
};
async function seedMember(prisma: PrismaClient, ctx: SeedCtx, m: MemberSeed) {
const run = await prisma.taskRun.create({
data: {
id: m.id,
friendlyId: m.friendlyId,
taskIdentifier: "my-task",
status: m.status,
payload: JSON.stringify({}),
payloadType: "application/json",
traceId: m.id,
spanId: m.id,
queue: "main",
runtimeEnvironmentId: ctx.environment.id,
projectId: ctx.project.id,
organizationId: ctx.organization.id,
environmentType: "PRODUCTION",
engine: "V2",
},
});
await prisma.taskRunAttempt.create({
data: {
friendlyId: `attempt_${m.id}`,
number: 1,
taskRunId: run.id,
backgroundWorkerId: "bw",
backgroundWorkerTaskId: "bwt",
runtimeEnvironmentId: ctx.environment.id,
queueId: "q",
status: m.status === "COMPLETED_SUCCESSFULLY" ? "COMPLETED" : "FAILED",
output: m.output,
outputType: "application/json",
error: m.error as any,
},
});
return run;
}
// Drop the TaskRunAttempt worker/queue FKs so attempts can be seeded (their output/error is what's
// under test) without standing up BackgroundWorker/TaskQueue parents — incidental to this read path.
async function relaxFks(prisma: PrismaClient) {
for (const sql of [
`ALTER TABLE "TaskRunAttempt" DROP CONSTRAINT IF EXISTS "TaskRunAttempt_backgroundWorkerId_fkey"`,
`ALTER TABLE "TaskRunAttempt" DROP CONSTRAINT IF EXISTS "TaskRunAttempt_backgroundWorkerTaskId_fkey"`,
`ALTER TABLE "TaskRunAttempt" DROP CONSTRAINT IF EXISTS "TaskRunAttempt_queueId_fkey"`,
]) {
await prisma.$executeRawUnsafe(sql);
}
}
async function seedBatch(
prisma: PrismaClient,
ctx: SeedCtx,
friendlyId: string,
memberIds: string[]
) {
const batch = await prisma.batchTaskRun.create({
data: {
friendlyId,
runtimeEnvironmentId: ctx.environment.id,
runCount: memberIds.length,
runIds: [],
batchVersion: "runengine:v2",
},
});
// Items in a deterministic order so the result `items` order is assertable.
for (const taskRunId of memberIds) {
await prisma.batchTaskRunItem.create({
data: { batchTaskRunId: batch.id, taskRunId, status: "COMPLETED" },
});
}
return batch;
}
const env = (ctx: SeedCtx) =>
({
id: ctx.environment.id,
type: ctx.environment.type,
slug: ctx.environment.slug,
organizationId: ctx.organization.id,
organization: { slug: ctx.organization.slug, title: ctx.organization.title },
projectId: ctx.project.id,
project: { name: ctx.project.name },
}) as unknown as AuthenticatedEnvironment;
describe("ApiBatchResultsPresenter split mode — member hydration is grouped, not per-member", () => {
postgresTest(
"a batch of N members costs ONE findMany (never N findFirst) and returns the same result",
async ({ prisma }) => {
const ctx = await seedEnv(prisma, "n1");
await relaxFks(prisma);
const { client: countingPrisma, counts } = countingClient(prisma);
const memberIds = [newRunId("a"), newRunId("b"), newRunId("c")];
await seedMember(prisma, ctx, {
id: memberIds[0],
friendlyId: "run_a",
status: "COMPLETED_SUCCESSFULLY",
output: JSON.stringify({ from: "a" }),
});
await seedMember(prisma, ctx, {
id: memberIds[1],
friendlyId: "run_b",
status: "COMPLETED_WITH_ERRORS",
error: { type: "BUILT_IN_ERROR", name: "Err", message: "boom", stackTrace: "" },
});
await seedMember(prisma, ctx, {
id: memberIds[2],
friendlyId: "run_c",
status: "COMPLETED_SUCCESSFULLY",
output: JSON.stringify({ from: "c" }),
});
await seedBatch(prisma, ctx, "batch_n1", memberIds);
const runStore = new PostgresRunStore({
prisma: countingPrisma,
readOnlyPrisma: countingPrisma,
});
const presenter = new ApiBatchResultsPresenter(
countingPrisma,
countingPrisma,
{
splitEnabled: true,
newClient: countingPrisma as unknown as PrismaReplicaClient,
legacyReplica: countingPrisma as unknown as PrismaReplicaClient,
},
runStore
);
const result = await presenter.call("batch_n1", env(ctx));
expect(result).toBeDefined();
expect(result!.id).toBe("batch_n1");
expect(result!.items).toHaveLength(3);
expect(result!.items[0]).toEqual({
ok: true,
id: "run_a",
taskIdentifier: "my-task",
output: JSON.stringify({ from: "a" }),
outputType: "application/json",
});
expect(result!.items[1]).toMatchObject({ ok: false, id: "run_b" });
expect(result!.items[2]).toEqual({
ok: true,
id: "run_c",
taskIdentifier: "my-task",
output: JSON.stringify({ from: "c" }),
outputType: "application/json",
});
// The grouped-read proof: ONE findMany for the whole member set, never a findFirst per member.
expect(counts.findMany).toBe(1);
expect(counts.findFirst).toBe(0);
}
);
});
@@ -0,0 +1,266 @@
import { containerTest } from "@internal/testcontainers";
import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic";
import type { PrismaClient } from "@trigger.dev/database";
import { beforeEach, describe, expect, vi } from "vitest";
// `findRun` reads the module-level `prisma`/`$replica` (control-plane handles) both directly
// and, transitively, via the `runStore`/`controlPlaneResolver` singletons that also import them.
// A lazy Proxy - not a plain value - is captured by those singletons at their own (one-time)
// construction, so pointing it at a real container client after the fact still routes every
// call there. Not a DB mock: every call forwards to a real Postgres container.
const dbHolder = vi.hoisted(() => ({ client: undefined as PrismaClient | undefined }));
vi.mock("~/db.server", () => {
const proxy = new Proxy(
{},
{
get(_t, prop) {
if (!dbHolder.client) throw new Error("dbHolder.client not set for this test");
return (dbHolder.client as any)[prop];
},
}
);
return { prisma: proxy, $replica: proxy };
});
vi.mock("~/v3/objectStore.server", () => ({
generatePresignedUrl: vi.fn(async () => ({ success: false, error: "not-used" })),
}));
import { ApiRetrieveRunPresenter } from "~/presenters/v3/ApiRetrieveRunPresenter.server";
import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
vi.setConfig({ testTimeout: 60_000 });
// Wraps a real Prisma client so every `client.<model>.<method>(...)` call is tallied by
// `"<model>.<method>"` before being forwarded, unmodified, to the real delegate.
function createCallCountingProxy(client: PrismaClient): {
client: PrismaClient;
counts: Map<string, number>;
} {
const counts = new Map<string, number>();
const wrappedModels = new Map<string, unknown>();
const wrapModel = (model: string, delegate: object) =>
new Proxy(delegate, {
get(target, prop, receiver) {
const value = Reflect.get(target, prop, receiver);
if (typeof prop === "string" && typeof value === "function") {
return (...args: unknown[]) => {
counts.set(`${model}.${prop}`, (counts.get(`${model}.${prop}`) ?? 0) + 1);
return (value as (...a: unknown[]) => unknown).apply(target, args);
};
}
return value;
},
});
const proxy = new Proxy(client, {
get(target, prop, receiver) {
const value = Reflect.get(target, prop, receiver);
if (
typeof prop === "string" &&
value &&
typeof value === "object" &&
typeof (value as { findMany?: unknown }).findMany === "function"
) {
if (!wrappedModels.has(prop)) {
wrappedModels.set(prop, wrapModel(prop, value as object));
}
return wrappedModels.get(prop);
}
return value;
},
});
return { client: proxy as PrismaClient, counts };
}
async function seedOrgProjectEnv(prisma: PrismaClient, suffix: string) {
const organization = await prisma.organization.create({
data: { title: `test-${suffix}`, slug: `test-${suffix}` },
});
const project = await prisma.project.create({
data: {
name: `test-${suffix}`,
slug: `test-${suffix}`,
organizationId: organization.id,
externalRef: `test-${suffix}`,
},
});
const runtimeEnvironment = await prisma.runtimeEnvironment.create({
data: {
slug: `test-${suffix}`,
type: "DEVELOPMENT",
projectId: project.id,
organizationId: organization.id,
apiKey: `tr_dev_${suffix}`,
pkApiKey: `pk_dev_${suffix}`,
shortcode: `short-${suffix}`,
},
});
return { organization, project, runtimeEnvironment };
}
function authEnv(
organization: { id: string },
project: { id: string; externalRef: string },
runtimeEnvironment: { id: string; slug: string }
): AuthenticatedEnvironment {
return {
id: runtimeEnvironment.id,
slug: runtimeEnvironment.slug,
organizationId: organization.id,
organization: { id: organization.id },
project: { id: project.id, externalRef: project.externalRef },
} as unknown as AuthenticatedEnvironment;
}
async function seedBackgroundWorker(
prisma: PrismaClient,
ctx: { projectId: string; runtimeEnvironmentId: string },
version: string
) {
return prisma.backgroundWorker.create({
data: {
friendlyId: `worker_${generateRunOpsId()}`,
version,
contentHash: `hash_${generateRunOpsId()}`,
projectId: ctx.projectId,
runtimeEnvironmentId: ctx.runtimeEnvironmentId,
metadata: {},
},
});
}
interface SeedRunOpts {
id: string;
friendlyId: string;
runtimeEnvironmentId: string;
projectId: string;
organizationId: string;
lockedToVersionId?: string;
parentTaskRunId?: string;
rootTaskRunId?: string;
}
async function seedRun(prisma: PrismaClient, opts: SeedRunOpts) {
return prisma.taskRun.create({
data: {
id: opts.id,
friendlyId: opts.friendlyId,
taskIdentifier: "my-task",
payload: JSON.stringify({ hello: "world" }),
payloadType: "application/json",
traceId: `trace_${opts.id}`,
spanId: `span_${opts.id}`,
queue: "task/my-task",
runtimeEnvironmentId: opts.runtimeEnvironmentId,
projectId: opts.projectId,
organizationId: opts.organizationId,
environmentType: "DEVELOPMENT",
engine: "V2",
lockedToVersionId: opts.lockedToVersionId,
parentTaskRunId: opts.parentTaskRunId,
rootTaskRunId: opts.rootTaskRunId,
},
});
}
beforeEach(() => {
dbHolder.client = undefined;
});
describe("ApiRetrieveRunPresenter.findRun locked-worker version resolution", () => {
containerTest(
"resolves run+parent+root+children lockedToVersion with ONE grouped query, not one per id",
async ({ prisma }) => {
const proxied = createCallCountingProxy(prisma);
dbHolder.client = proxied.client;
const { organization, project, runtimeEnvironment } = await seedOrgProjectEnv(
prisma,
"grouped"
);
const workerCtx = { projectId: project.id, runtimeEnvironmentId: runtimeEnvironment.id };
// Two distinct versions - `workerA` is deliberately reused across run/root/one child to
// prove the pre-existing dedup-by-Set still collapses to the same distinct-id count.
const workerA = await seedBackgroundWorker(prisma, workerCtx, "2024.1.0");
const workerB = await seedBackgroundWorker(prisma, workerCtx, "2024.2.0");
const workerC = await seedBackgroundWorker(prisma, workerCtx, "2024.3.0");
const rootId = generateRunOpsId();
const parentId = generateRunOpsId();
const runId = generateRunOpsId();
const childId1 = generateRunOpsId();
const childId2 = generateRunOpsId();
await seedRun(prisma, {
id: rootId,
friendlyId: `run_${rootId}`,
runtimeEnvironmentId: runtimeEnvironment.id,
projectId: project.id,
organizationId: organization.id,
lockedToVersionId: workerA.id,
});
await seedRun(prisma, {
id: parentId,
friendlyId: `run_${parentId}`,
runtimeEnvironmentId: runtimeEnvironment.id,
projectId: project.id,
organizationId: organization.id,
rootTaskRunId: rootId,
lockedToVersionId: workerB.id,
});
await seedRun(prisma, {
id: runId,
friendlyId: `run_${runId}`,
runtimeEnvironmentId: runtimeEnvironment.id,
projectId: project.id,
organizationId: organization.id,
parentTaskRunId: parentId,
rootTaskRunId: rootId,
lockedToVersionId: workerA.id,
});
await seedRun(prisma, {
id: childId1,
friendlyId: `run_${childId1}`,
runtimeEnvironmentId: runtimeEnvironment.id,
projectId: project.id,
organizationId: organization.id,
parentTaskRunId: runId,
rootTaskRunId: rootId,
lockedToVersionId: workerC.id,
});
await seedRun(prisma, {
id: childId2,
friendlyId: `run_${childId2}`,
runtimeEnvironmentId: runtimeEnvironment.id,
projectId: project.id,
organizationId: organization.id,
parentTaskRunId: runId,
rootTaskRunId: rootId,
lockedToVersionId: workerA.id,
});
const env = authEnv(organization, project, runtimeEnvironment);
const found = await ApiRetrieveRunPresenter.findRun(`run_${runId}`, env);
expect(found).not.toBeNull();
expect(found!.lockedToVersion?.version).toBe(workerA.version);
expect(found!.parentTaskRun?.lockedToVersion?.version).toBe(workerB.version);
expect(found!.rootTaskRun?.lockedToVersion?.version).toBe(workerA.version);
const versionByChildFriendlyId = new Map(
found!.childRuns.map((c) => [c.friendlyId, c.lockedToVersion?.version])
);
expect(versionByChildFriendlyId.get(`run_${childId1}`)).toBe(workerC.version);
expect(versionByChildFriendlyId.get(`run_${childId2}`)).toBe(workerA.version);
// 3 distinct lockedToVersionIds (workerA, workerB, workerC) across 5 rows -> ONE grouped
// findMany, ZERO per-id findFirst.
expect(proxied.counts.get("backgroundWorker.findMany") ?? 0).toBe(1);
expect(proxied.counts.get("backgroundWorker.findFirst") ?? 0).toBe(0);
}
);
});
@@ -0,0 +1,60 @@
// The GLOBAL mint-kind flip must read -> stamp -> write the grace metadata in ONE transaction under
// an advisory lock, so concurrent global flips can't clobber each other's grace stamp (the per-org
// routes serialize the same way). NEVER mocks the DB: real testcontainers Postgres FeatureFlag rows.
import type { PrismaClient } from "@trigger.dev/database";
import { postgresTest } from "@internal/testcontainers";
import { describe, expect, vi } from "vitest";
import { FEATURE_FLAG } from "~/v3/featureFlags";
import { applyGlobalMintKindFlip, makeSetMultipleFlags } from "~/v3/featureFlags.server";
vi.setConfig({ testTimeout: 60_000 });
const MINT_KEYS = [
FEATURE_FLAG.runOpsMintKind,
FEATURE_FLAG.runOpsMintKindPrev,
FEATURE_FLAG.runOpsMintKindFlippedAt,
];
async function readGlobalMint(prisma: PrismaClient): Promise<Record<string, unknown>> {
const rows = await prisma.featureFlag.findMany({
where: { key: { in: MINT_KEYS } },
select: { key: true, value: true },
});
const m: Record<string, unknown> = {};
for (const row of rows) m[row.key] = row.value;
return m;
}
describe("applyGlobalMintKindFlip — transactional stamp + serialized flips", () => {
postgresTest("a genuine global flip stamps prev + flippedAt", async ({ prisma }) => {
await makeSetMultipleFlags(prisma)({ [FEATURE_FLAG.runOpsMintKind]: "cuid" });
await applyGlobalMintKindFlip(prisma, { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, 60_000);
const m = await readGlobalMint(prisma);
expect(m[FEATURE_FLAG.runOpsMintKind]).toBe("runOpsId");
expect(m[FEATURE_FLAG.runOpsMintKindPrev]).toBe("cuid");
expect(typeof m[FEATURE_FLAG.runOpsMintKindFlippedAt]).toBe("string");
});
postgresTest(
"concurrent flips serialize and keep one coherent grace stamp",
async ({ prisma }) => {
await makeSetMultipleFlags(prisma)({ [FEATURE_FLAG.runOpsMintKind]: "cuid" });
await Promise.all(
Array.from({ length: 8 }, () =>
applyGlobalMintKindFlip(prisma, { [FEATURE_FLAG.runOpsMintKind]: "runOpsId" }, 60_000)
)
);
// One genuine flip stamps prev=cuid + a flippedAt; same-target saves carry it forward. Under the
// advisory lock the reads/writes serialize, so the stamp is a single coherent trio rather than an
// interleaved partial write that drops prev/flippedAt.
const m = await readGlobalMint(prisma);
expect(m[FEATURE_FLAG.runOpsMintKind]).toBe("runOpsId");
expect(m[FEATURE_FLAG.runOpsMintKindPrev]).toBe("cuid");
expect(typeof m[FEATURE_FLAG.runOpsMintKindFlippedAt]).toBe("string");
}
);
});
@@ -0,0 +1,236 @@
// RED->GREEN guard: WaitpointPresenter connected-run gather must BOUND the fetch of a waitpoint's
// connected-run ids, not just the number displayed. A "displayed count <= 5" assertion would
// false-green (the take:5 on the run resolve already bounds the DISPLAY). Instead this captures the
// real `taskRun.findMany` call args via a Proxy over a REAL testcontainer Postgres client (no mocks)
// and asserts the IN-list is capped at the scan limit (danglers over-read), never unbounded.
//
// Exercises the dedicated-schema (Prisma `waitpointRunConnection`) branch: waitpoint + more than the
// scan-limit connected runs seeded on the NEW dedicated run-ops client (RunOpsPrismaClient, prisma17).
import { describe, expect, vi } from "vitest";
const legacyReplicaHolder = vi.hoisted(() => ({ client: undefined as any }));
const newClientHolder = vi.hoisted(() => ({ client: undefined as any }));
vi.mock("~/db.server", async () => {
const { Prisma } = await import("@trigger.dev/database");
const lazyProxy = (holder: { client: any }, label: string) =>
new Proxy(
{},
{
get(_t, prop) {
if (!holder.client) {
throw new Error(`${label} not set for this test`);
}
return holder.client[prop];
},
}
);
const replicaProxy = lazyProxy(legacyReplicaHolder, "legacyReplicaHolder.client");
return {
prisma: replicaProxy,
$replica: replicaProxy,
runOpsNewPrisma: lazyProxy(newClientHolder, "newClientHolder.client"),
runOpsNewReplica: lazyProxy(newClientHolder, "newClientHolder.client"),
runOpsLegacyPrisma: replicaProxy,
runOpsLegacyReplica: replicaProxy,
sqlDatabaseSchema: Prisma.sql([`public`]),
DATABASE_SCHEMA: "public",
};
});
vi.mock("~/services/clickhouse/clickhouseFactoryInstance.server", () => ({
clickhouseFactory: {
getClickhouseForOrganization: async () => ({}),
},
}));
// Echo the runId set back as runs so the presenter's final CH hydrate never runs for real -- the
// thing under test is the connected-run-id GATHER (the taskRun.findMany args), not this step.
vi.mock("~/presenters/v3/NextRunListPresenter.server", () => ({
NextRunListPresenter: class {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
constructor(..._args: unknown[]) {}
async call(_organizationId: string, _environmentId: string, opts: { runId?: string[] }) {
return {
runs: (opts.runId ?? []).map((friendlyId) => ({
friendlyId,
taskIdentifier: "echoed",
})),
};
}
},
}));
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
import type { PrismaClient } from "@trigger.dev/database";
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
import {
CONNECTED_RUNS_CONNECTION_SCAN_LIMIT,
WaitpointPresenter,
} from "~/presenters/v3/WaitpointPresenter.server";
vi.setConfig({ testTimeout: 90_000 });
type SeedContext = {
organizationId: string;
projectId: string;
environmentId: string;
};
async function seedParents(prisma: PrismaClient, slug: string): Promise<SeedContext> {
const organization = await prisma.organization.create({
data: { title: `org-${slug}`, slug: `org-${slug}` },
});
const project = await prisma.project.create({
data: {
name: `proj-${slug}`,
slug: `proj-${slug}`,
organizationId: organization.id,
externalRef: `proj-${slug}`,
},
});
const runtimeEnvironment = await prisma.runtimeEnvironment.create({
data: {
slug: `env-${slug}`,
type: "DEVELOPMENT",
projectId: project.id,
organizationId: organization.id,
apiKey: `tr_dev_${slug}`,
pkApiKey: `pk_dev_${slug}`,
shortcode: `sc-${slug}`,
},
});
return {
organizationId: organization.id,
projectId: project.id,
environmentId: runtimeEnvironment.id,
};
}
async function seedWaitpoint(
prisma: PrismaClient | RunOpsPrismaClient,
ctx: SeedContext,
friendlyId: string
) {
return (prisma as PrismaClient).waitpoint.create({
data: {
friendlyId,
type: "MANUAL",
status: "COMPLETED",
idempotencyKey: `idem-${friendlyId}`,
userProvidedIdempotencyKey: false,
outputType: "application/json",
outputIsError: false,
completedAt: new Date(),
tags: [],
projectId: ctx.projectId,
environmentId: ctx.environmentId,
},
});
}
async function seedRun(
prisma: PrismaClient | RunOpsPrismaClient,
ctx: SeedContext,
friendlyId: string
) {
return (prisma as PrismaClient).taskRun.create({
data: {
friendlyId,
taskIdentifier: "my-task",
status: "PENDING",
payload: JSON.stringify({ foo: friendlyId }),
payloadType: "application/json",
traceId: friendlyId,
spanId: friendlyId,
queue: "test",
runtimeEnvironmentId: ctx.environmentId,
projectId: ctx.projectId,
organizationId: ctx.organizationId,
environmentType: "DEVELOPMENT",
engine: "V2",
},
});
}
// Wrap the REAL dedicated run-ops client's `taskRun.findMany` to capture the args of every call,
// delegating unchanged to the real client (the DB still runs the query -- pure instrumentation,
// never a mock).
function capturingTaskRunFindMany(real: RunOpsPrismaClient): {
client: RunOpsPrismaClient;
calls: { where?: { id?: { in?: string[] } } }[];
} {
const calls: { where?: { id?: { in?: string[] } } }[] = [];
const wrappedTaskRun = new Proxy((real as any).taskRun, {
get(target, prop) {
if (prop === "findMany") {
return (...args: any[]) => {
calls.push(args[0]);
return (target as any)[prop](...args);
};
}
return (target as any)[prop];
},
});
const client = new Proxy(real as object, {
get(target, prop) {
if (prop === "taskRun") {
return wrappedTaskRun;
}
return (target as any)[prop];
},
}) as RunOpsPrismaClient;
return { client, calls };
}
// Seed MORE than the scan limit so the assertion bites: an unbounded gather IN-lists all of them,
// a correctly bounded one caps at CONNECTED_RUNS_CONNECTION_SCAN_LIMIT.
const CONNECTED_RUN_COUNT = CONNECTED_RUNS_CONNECTION_SCAN_LIMIT + 5;
describe("WaitpointPresenter bounds the connected-run-id FETCH", () => {
heteroRunOpsPostgresTest(
"a waitpoint with more connections than the scan limit caps the IN-list at the scan limit",
async ({ prisma14, prisma17 }) => {
const ctx = await seedParents(prisma14, "bounded");
const waitpoint = await seedWaitpoint(prisma17, ctx, "waitpoint_bounded");
const runs = await Promise.all(
Array.from({ length: CONNECTED_RUN_COUNT }, (_, i) => seedRun(prisma17, ctx, `run_b${i}`))
);
for (const run of runs) {
await prisma17.waitpointRunConnection.create({
data: { taskRunId: run.id, waitpointId: waitpoint.id },
});
}
legacyReplicaHolder.client = prisma14;
newClientHolder.client = prisma17;
const { client: countingPrisma17, calls } = capturingTaskRunFindMany(prisma17);
const presenter = new WaitpointPresenter(undefined, undefined, {
splitEnabled: true,
newClient: countingPrisma17 as unknown as PrismaClient,
legacyReplica: prisma14,
});
await presenter.call({
friendlyId: waitpoint.friendlyId,
environmentId: ctx.environmentId,
projectId: ctx.projectId,
});
// The guard: assert the FETCH (the IN-list built from the connected-run-id gather) is
// bounded, not just the eventual displayed count. An unbounded gather would IN-list every
// connection row; the dedicated branch over-reads up to CONNECTED_RUNS_CONNECTION_SCAN_LIMIT
// (25) so a display slot is never lost to a dangler, so the IN-list must be capped at the
// scan limit -- above the display limit (5), but never unbounded.
expect(calls.length).toBeGreaterThan(0);
for (const call of calls) {
expect(call.where?.id?.in?.length ?? 0).toBeLessThanOrEqual(
CONNECTED_RUNS_CONNECTION_SCAN_LIMIT
);
}
}
);
});
@@ -32,6 +32,7 @@ vi.mock("~/db.server", async () => {
$replica: proxy,
runOpsNewPrisma: proxy,
sqlDatabaseSchema: Prisma.sql([`public`]),
DATABASE_SCHEMA: "public",
};
});
@@ -0,0 +1,199 @@
// RED->GREEN guard: #connectedRunIdsOn used to cap the connection-id fetch at
// CONNECTED_RUNS_DISPLAY_LIMIT BEFORE checking the runs exist. The dedicated store's
// WaitpointRunConnection is FK-free, so dangling rows can starve out real ones within the cap.
// The fix existence-filters AT THE QUERY (JOIN to TaskRun) so the LIMIT only ever lands on rows
// whose run exists.
import { describe, expect, vi } from "vitest";
const legacyReplicaHolder = vi.hoisted(() => ({ client: undefined as any }));
const newClientHolder = vi.hoisted(() => ({ client: undefined as any }));
vi.mock("~/db.server", async () => {
const { Prisma } = await import("@trigger.dev/database");
const lazyProxy = (holder: { client: any }, label: string) =>
new Proxy(
{},
{
get(_t, prop) {
if (!holder.client) {
throw new Error(`${label} not set for this test`);
}
return holder.client[prop];
},
}
);
const replicaProxy = lazyProxy(legacyReplicaHolder, "legacyReplicaHolder.client");
return {
prisma: replicaProxy,
$replica: replicaProxy,
runOpsNewPrisma: lazyProxy(newClientHolder, "newClientHolder.client"),
runOpsNewReplica: lazyProxy(newClientHolder, "newClientHolder.client"),
runOpsLegacyPrisma: replicaProxy,
runOpsLegacyReplica: replicaProxy,
sqlDatabaseSchema: Prisma.sql([`public`]),
DATABASE_SCHEMA: "public",
};
});
vi.mock("~/services/clickhouse/clickhouseFactoryInstance.server", () => ({
clickhouseFactory: {
getClickhouseForOrganization: async () => ({}),
},
}));
vi.mock("~/presenters/v3/NextRunListPresenter.server", () => ({
NextRunListPresenter: class {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
constructor(..._args: unknown[]) {}
async call(_organizationId: string, _environmentId: string, opts: { runId?: string[] }) {
return {
runs: (opts.runId ?? []).map((friendlyId) => ({
friendlyId,
taskIdentifier: "echoed",
})),
};
}
},
}));
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
import type { PrismaClient } from "@trigger.dev/database";
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
import {
CONNECTED_RUNS_DISPLAY_LIMIT,
WaitpointPresenter,
} from "~/presenters/v3/WaitpointPresenter.server";
vi.setConfig({ testTimeout: 90_000 });
type SeedContext = {
organizationId: string;
projectId: string;
environmentId: string;
};
async function seedParents(prisma: PrismaClient, slug: string): Promise<SeedContext> {
const organization = await prisma.organization.create({
data: { title: `org-${slug}`, slug: `org-${slug}` },
});
const project = await prisma.project.create({
data: {
name: `proj-${slug}`,
slug: `proj-${slug}`,
organizationId: organization.id,
externalRef: `proj-${slug}`,
},
});
const runtimeEnvironment = await prisma.runtimeEnvironment.create({
data: {
slug: `env-${slug}`,
type: "DEVELOPMENT",
projectId: project.id,
organizationId: organization.id,
apiKey: `tr_dev_${slug}`,
pkApiKey: `pk_dev_${slug}`,
shortcode: `sc-${slug}`,
},
});
return {
organizationId: organization.id,
projectId: project.id,
environmentId: runtimeEnvironment.id,
};
}
async function seedWaitpoint(prisma: RunOpsPrismaClient, ctx: SeedContext, friendlyId: string) {
return (prisma as unknown as PrismaClient).waitpoint.create({
data: {
friendlyId,
type: "MANUAL",
status: "COMPLETED",
idempotencyKey: `idem-${friendlyId}`,
userProvidedIdempotencyKey: false,
outputType: "application/json",
outputIsError: false,
completedAt: new Date(),
tags: [],
projectId: ctx.projectId,
environmentId: ctx.environmentId,
},
});
}
async function seedRun(prisma: RunOpsPrismaClient, ctx: SeedContext, friendlyId: string) {
return (prisma as unknown as PrismaClient).taskRun.create({
data: {
friendlyId,
taskIdentifier: "my-task",
status: "PENDING",
payload: JSON.stringify({ foo: friendlyId }),
payloadType: "application/json",
traceId: friendlyId,
spanId: friendlyId,
queue: "test",
runtimeEnvironmentId: ctx.environmentId,
projectId: ctx.projectId,
organizationId: ctx.organizationId,
environmentType: "DEVELOPMENT",
engine: "V2",
},
});
}
// Inserted before the real rows so the pre-fix cap-then-check code is starved of real ids.
const DANGLING_CONNECTION_COUNT = 10;
const REAL_CONNECTED_RUN_COUNT = 3; // <= CONNECTED_RUNS_DISPLAY_LIMIT (5): all must show up
describe("WaitpointPresenter#connectedRunIdsOn is robust to dangling (FK-free) connection rows", () => {
heteroRunOpsPostgresTest(
"returns the existing connected runs even when dangling connections precede them",
async ({ prisma14, prisma17 }) => {
const ctx = await seedParents(prisma14, "dangling");
const waitpoint = await seedWaitpoint(prisma17, ctx, "waitpoint_dangling");
// Dangling connections: taskRunId points at a run that was NEVER created. The dedicated
// `WaitpointRunConnection` model is scalar/FK-free, so this insert is legal.
for (let i = 0; i < DANGLING_CONNECTION_COUNT; i++) {
await prisma17.waitpointRunConnection.create({
data: { taskRunId: `nonexistent_run_${i}`, waitpointId: waitpoint.id },
});
}
// Real, existing connected runs -- created (and connected) AFTER the dangling rows.
const realRuns = await Promise.all(
Array.from({ length: REAL_CONNECTED_RUN_COUNT }, (_, i) =>
seedRun(prisma17, ctx, `run_dangling_real_${i}`)
)
);
for (const run of realRuns) {
await prisma17.waitpointRunConnection.create({
data: { taskRunId: run.id, waitpointId: waitpoint.id },
});
}
legacyReplicaHolder.client = prisma14;
newClientHolder.client = prisma17;
const presenter = new WaitpointPresenter(undefined, undefined, {
splitEnabled: true,
newClient: prisma17 as unknown as PrismaClient,
legacyReplica: prisma14,
});
const result = await presenter.call({
friendlyId: waitpoint.friendlyId,
environmentId: ctx.environmentId,
projectId: ctx.projectId,
});
expect(result).not.toBeNull();
const friendlyIds = (result?.connectedRuns ?? []).map((r) => r.friendlyId).sort();
expect(friendlyIds.length).toBeLessThanOrEqual(CONNECTED_RUNS_DISPLAY_LIMIT);
// The bug: cap-before-existence-check lets dangling rows starve out the real ones, so this
// fails (returns fewer than REAL_CONNECTED_RUN_COUNT, often zero) on unfixed code.
expect(friendlyIds).toEqual(
realRuns.map((r) => r.friendlyId).sort((a, b) => a.localeCompare(b))
);
}
);
});
@@ -31,6 +31,7 @@ vi.mock("~/db.server", async () => {
runOpsLegacyPrisma: replicaProxy,
runOpsLegacyReplica: replicaProxy,
sqlDatabaseSchema: Prisma.sql([`public`]),
DATABASE_SCHEMA: "public",
};
});
@@ -39,6 +39,7 @@ vi.mock("~/db.server", async () => {
runOpsLegacyPrisma: replicaProxy,
runOpsLegacyReplica: replicaProxy,
sqlDatabaseSchema: Prisma.sql([`public`]),
DATABASE_SCHEMA: "public",
};
});
@@ -67,11 +68,17 @@ import { setupClickhouseReplication } from "./utils/replicationUtils";
vi.setConfig({ testTimeout: 90_000 });
// A read client whose waitpoint.findFirst is recorded; throws if used after being marked
// forbidden, so we can prove a store was NEVER read. Every other access forwards to the real
// client (so the inlined `environment` join + connectedRuns relation still resolve).
// A read client whose waitpoint.findFirst calls are recorded, split by kind. The presenter issues
// two shapes of waitpoint.findFirst: a HYDRATE (loads the waitpoint scalar row -- no `connectedRuns`
// in its select) and a connectedRuns-RELATION read (control-plane branch of the connected-runs
// gather -- select is exactly `{ connectedRuns }`). The `forbidden` guard throws ONLY on a HYDRATE,
// so a store can be proven to never be HYDRATED while still allowing the connectedRuns cross-DB
// union to legitimately read it (that union always touched both stores; it was invisible when the
// old code did it via raw SQL). Every other access forwards to the real client.
function recording(client: PrismaClient, opts: { forbidden?: boolean } = {}) {
const calls: unknown[] = [];
const hydrateCalls: unknown[] = [];
const connectedRunsCalls: unknown[] = [];
return {
handle: new Proxy(client, {
get(target, prop) {
@@ -81,8 +88,16 @@ function recording(client: PrismaClient, opts: { forbidden?: boolean } = {}) {
if (wpProp === "findFirst") {
return (args: unknown) => {
calls.push(args);
if (opts.forbidden) {
throw new Error("this store must never be read");
const select = (args as { select?: Record<string, unknown> })?.select ?? {};
const keys = Object.keys(select);
const isConnectedRunsRead = keys.length === 1 && keys[0] === "connectedRuns";
if (isConnectedRunsRead) {
connectedRunsCalls.push(args);
} else {
hydrateCalls.push(args);
if (opts.forbidden) {
throw new Error("this store must never be hydrated");
}
}
return (wpTarget as any).findFirst(args);
};
@@ -95,6 +110,8 @@ function recording(client: PrismaClient, opts: { forbidden?: boolean } = {}) {
},
}) as unknown as PrismaReplicaClient,
calls,
hydrateCalls,
connectedRunsCalls,
};
}
@@ -254,9 +271,12 @@ describe("WaitpointPresenter dual-DB read-through (hetero PG14 + PG17, no connec
const result = await presenter.call(callArgs(ctx, seeded.friendlyId));
expect(result?.id).toBe(seeded.friendlyId);
// New-first short-circuit: legacy never probed (the throwing handle proves it).
expect(newClient.calls.length).toBe(1);
expect(legacy.calls.length).toBe(0);
// New-first short-circuit: the HYDRATE answered from new and never fell through to a legacy
// hydrate (the throwing handle proves it). The connectedRuns cross-DB union still reads legacy
// legitimately -- that read is allowed and must NOT count as a hydrate.
expect(newClient.hydrateCalls.length).toBe(1);
expect(legacy.hydrateCalls.length).toBe(0);
expect(legacy.connectedRunsCalls.length).toBeGreaterThan(0);
}
);
@@ -285,8 +305,11 @@ describe("WaitpointPresenter dual-DB read-through (hetero PG14 + PG17, no connec
expect(result?.id).toBe(seeded.friendlyId);
expect(result?.tags).toEqual(["one"]);
// Exactly one read on the single client; the second handle is never touched.
expect(single.calls.length).toBe(1);
// Two reads on the single client, both on the one handle: the first findFirst hydrates the
// waitpoint, the second loads the `connectedRuns` relation (the implicit M2M has no queryable
// join delegate, so the ORM must traverse the relation with a second findFirst). The second
// handle is never touched -- passthrough is structural, the split branch never fires.
expect(single.calls.length).toBe(2);
expect(second.calls.length).toBe(0);
}
);
@@ -0,0 +1,208 @@
// FLOW-level gap: a waitpoint's connected runs SPLIT across BOTH physical DBs at once (some legacy,
// some new-resident) rather than the existing single-direction cross-DB cases in
// waitpointPresenter.dedicatedConnectedRuns.readthrough.test.ts. Asserts #connectedRunFriendlyIds
// unions friendlyIds gathered from BOTH stores in one read AND stays bounded to
// CONNECTED_RUNS_DISPLAY_LIMIT even when the combined total exceeds it. Real two-physical-DB
// topology (heteroRunOpsPostgresTest); never mocked beyond the same db.server/clickhouse/
// NextRunListPresenter echo seams the sibling presenter tests use.
import { describe, expect, vi } from "vitest";
const legacyReplicaHolder = vi.hoisted(() => ({ client: undefined as any }));
const newClientHolder = vi.hoisted(() => ({ client: undefined as any }));
vi.mock("~/db.server", async () => {
const { Prisma } = await import("@trigger.dev/database");
const lazyProxy = (holder: { client: any }, label: string) =>
new Proxy(
{},
{
get(_t, prop) {
if (!holder.client) {
throw new Error(`${label} not set for this test`);
}
return holder.client[prop];
},
}
);
const replicaProxy = lazyProxy(legacyReplicaHolder, "legacyReplicaHolder.client");
return {
prisma: replicaProxy,
$replica: replicaProxy,
runOpsNewPrisma: lazyProxy(newClientHolder, "newClientHolder.client"),
runOpsNewReplica: lazyProxy(newClientHolder, "newClientHolder.client"),
runOpsLegacyPrisma: replicaProxy,
runOpsLegacyReplica: replicaProxy,
sqlDatabaseSchema: Prisma.sql([`public`]),
DATABASE_SCHEMA: "public",
};
});
vi.mock("~/services/clickhouse/clickhouseFactoryInstance.server", () => ({
clickhouseFactory: {
getClickhouseForOrganization: async () => ({}),
},
}));
// Echo the runId set back as runs so `result.connectedRuns` == the friendlyIds the presenter
// gathered cross-DB (isolates the gather from the CH hydrate, as the sibling tests do).
vi.mock("~/presenters/v3/NextRunListPresenter.server", () => ({
NextRunListPresenter: class {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
constructor(..._args: unknown[]) {}
async call(_organizationId: string, _environmentId: string, opts: { runId?: string[] }) {
return {
runs: (opts.runId ?? []).map((friendlyId) => ({
friendlyId,
taskIdentifier: "echoed",
})),
};
}
},
}));
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
import type { PrismaClient } from "@trigger.dev/database";
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
import {
CONNECTED_RUNS_DISPLAY_LIMIT,
WaitpointPresenter,
} from "~/presenters/v3/WaitpointPresenter.server";
vi.setConfig({ testTimeout: 90_000 });
type SeedContext = {
organizationId: string;
projectId: string;
environmentId: string;
};
async function seedParents(prisma: PrismaClient, slug: string): Promise<SeedContext> {
const organization = await prisma.organization.create({
data: { title: `org-${slug}`, slug: `org-${slug}` },
});
const project = await prisma.project.create({
data: {
name: `proj-${slug}`,
slug: `proj-${slug}`,
organizationId: organization.id,
externalRef: `proj-${slug}`,
},
});
const runtimeEnvironment = await prisma.runtimeEnvironment.create({
data: {
slug: `env-${slug}`,
type: "DEVELOPMENT",
projectId: project.id,
organizationId: organization.id,
apiKey: `tr_dev_${slug}`,
pkApiKey: `pk_dev_${slug}`,
shortcode: `sc-${slug}`,
},
});
return {
organizationId: organization.id,
projectId: project.id,
environmentId: runtimeEnvironment.id,
};
}
async function seedRun(
prisma: PrismaClient | RunOpsPrismaClient,
ctx: SeedContext,
friendlyId: string
) {
return (prisma as PrismaClient).taskRun.create({
data: {
friendlyId,
taskIdentifier: "my-task",
status: "PENDING",
payload: JSON.stringify({ foo: friendlyId }),
payloadType: "application/json",
traceId: friendlyId,
spanId: friendlyId,
queue: "test",
runtimeEnvironmentId: ctx.environmentId,
projectId: ctx.projectId,
organizationId: ctx.organizationId,
environmentType: "DEVELOPMENT",
engine: "V2",
},
});
}
describe("WaitpointPresenter — connected runs SPLIT across both physical DBs", () => {
heteroRunOpsPostgresTest(
"unions connected-run friendlyIds from both stores and stays bounded to the display limit",
async ({ prisma14, prisma17 }) => {
const ctx = await seedParents(prisma14, "split");
// Waitpoint resident on LEGACY.
const waitpoint = await prisma14.waitpoint.create({
data: {
friendlyId: "waitpoint_split",
type: "MANUAL",
status: "COMPLETED",
idempotencyKey: "idem-waitpoint_split",
userProvidedIdempotencyKey: false,
outputType: "application/json",
outputIsError: false,
completedAt: new Date(),
tags: [],
projectId: ctx.projectId,
environmentId: ctx.environmentId,
},
});
// 2 connected runs resident + joined on NEW (below the limit on its own).
const NEW_RUN_FRIENDLY_IDS = ["run_split_new0", "run_split_new1"];
for (const friendlyId of NEW_RUN_FRIENDLY_IDS) {
const run = await seedRun(prisma17, ctx, friendlyId);
await prisma17.waitpointRunConnection.create({
data: { taskRunId: run.id, waitpointId: waitpoint.id },
});
}
// 6 connected runs resident + joined on LEGACY via the implicit M2M — more than enough, on its
// own, to push the combined total past CONNECTED_RUNS_DISPLAY_LIMIT (5).
const LEGACY_RUN_FRIENDLY_IDS = Array.from({ length: 6 }, (_, i) => `run_split_leg${i}`);
for (const friendlyId of LEGACY_RUN_FRIENDLY_IDS) {
const run = await seedRun(prisma14, ctx, friendlyId);
await prisma14.waitpoint.update({
where: { id: waitpoint.id },
data: { connectedRuns: { connect: [{ id: run.id }] } },
});
}
legacyReplicaHolder.client = prisma14;
newClientHolder.client = prisma17;
const presenter = new WaitpointPresenter(undefined, undefined, {
splitEnabled: true,
newClient: prisma17 as unknown as PrismaClient,
legacyReplica: prisma14,
});
const result = await presenter.call({
friendlyId: waitpoint.friendlyId,
environmentId: ctx.environmentId,
projectId: ctx.projectId,
});
const returnedIds = result?.connectedRuns.map((r) => r.friendlyId) ?? [];
// Bounded: the combined total across both DBs (8) must never surface more than the display
// limit, proving the fetch is capped globally, not just per-DB.
expect(returnedIds.length).toBeLessThanOrEqual(CONNECTED_RUNS_DISPLAY_LIMIT);
expect(returnedIds.length).toBeGreaterThan(0);
// BOTH stores contributed: at least one NEW-resident and one LEGACY-resident connected run
// friendlyId made it into the result. A single-store gather (the bug this guards against)
// would surface only one side.
const fromNew = returnedIds.filter((id) => NEW_RUN_FRIENDLY_IDS.includes(id));
const fromLegacy = returnedIds.filter((id) => LEGACY_RUN_FRIENDLY_IDS.includes(id));
expect(fromNew.length).toBeGreaterThan(0);
expect(fromLegacy.length).toBeGreaterThan(0);
expect(fromNew.length + fromLegacy.length).toBe(returnedIds.length);
}
);
});
@@ -0,0 +1,302 @@
// Completion -> resume FLOW test for the direction crossDbTokenBlock.test.ts's writer fix enables but
// never itself exercises end-to-end: a LEGACY (cuid) run blocks on a standalone MANUAL token that is
// NEW-resident (run-ops id) — e.g. the token was minted co-located with a run-ops sibling. This test
// drives the real engine path (blockRunWithWaitpoint -> completeWaitpoint -> continueRunIfUnblocked)
// across the two PHYSICAL DBs to prove the blocking edge is discovered and the run actually resumes,
// not just that the edge write succeeds (that unit-level assertion already lives in
// runOpsStore.crossDbTokenBlock.test.ts). Mirrors completeWaitpointReadResidency.test.ts's own
// cross-DB case, but with the run/token residencies swapped. Real two-physical-DB topology; never
// mocked.
import {
heteroRunOpsPostgresTest,
network,
redisContainer,
redisOptions,
} from "@internal/testcontainers";
import { trace } from "@internal/tracing";
import { PostgresRunStore, RoutingRunStore, type CreateRunInput } from "@internal/run-store";
import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic";
import type { PrismaClient } from "@trigger.dev/database";
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
import { expect, vi } from "vitest";
import { RunEngine } from "../index.js";
const twoDbEngineTest = heteroRunOpsPostgresTest.extend<{
redisContainer: any;
redisOptions: any;
}>({
network,
redisContainer,
redisOptions,
});
// 25-char cuid body (no v1 version marker at index 25) -> classifies LEGACY -> #legacy (prisma14).
const CUID_25 = "c".repeat(25);
function baseEngineOptions(redisOptions: any, prisma: any) {
return {
prisma,
worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 },
queue: {
redis: redisOptions,
masterQueueConsumersDisabled: true,
processWorkerQueueDebounceMs: 50,
},
runLock: { redis: redisOptions },
machines: {
defaultMachine: "small-1x" as const,
machines: {
"small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 },
},
baseCostInCents: 0.0001,
},
tracer: trace.getTracer("test", "0.0.0"),
};
}
async function seedControlPlaneEnv(prisma: PrismaClient, suffix: string) {
const organization = await prisma.organization.create({
data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
});
const project = await prisma.project.create({
data: {
name: `Project ${suffix}`,
slug: `project-${suffix}`,
externalRef: `proj_${suffix}`,
organizationId: organization.id,
},
});
const environment = await prisma.runtimeEnvironment.create({
data: {
type: "PRODUCTION",
slug: `prod-${suffix}`,
projectId: project.id,
organizationId: organization.id,
apiKey: `tr_prod_${suffix}`,
pkApiKey: `pk_prod_${suffix}`,
shortcode: `short_${suffix}`,
maximumConcurrencyLimit: 10,
},
});
return { organization, project, environment };
}
function buildCreateRunInput(p: {
runId: string;
friendlyId: string;
organizationId: string;
projectId: string;
runtimeEnvironmentId: string;
}): CreateRunInput {
return {
data: {
id: p.runId,
engine: "V2",
status: "EXECUTING",
friendlyId: p.friendlyId,
runtimeEnvironmentId: p.runtimeEnvironmentId,
environmentType: "PRODUCTION",
organizationId: p.organizationId,
projectId: p.projectId,
taskIdentifier: "parent-task",
payload: "{}",
payloadType: "application/json",
context: {},
traceContext: {},
traceId: `trace_${p.runId}`,
spanId: `span_${p.runId}`,
runTags: [],
queue: "task/parent-task",
isTest: false,
taskEventStore: "taskEvent",
depth: 0,
createdAt: new Date("2024-01-01T00:00:00.000Z"),
},
snapshot: {
engine: "V2",
executionStatus: "RUN_CREATED",
description: "Run was created",
runStatus: "PENDING",
environmentId: p.runtimeEnvironmentId,
environmentType: "PRODUCTION",
projectId: p.projectId,
organizationId: p.organizationId,
},
};
}
// Seed an EXECUTING LEGACY (cuid) parent run on #legacy (prisma14), plus a standalone MANUAL token
// that is NEW-resident (run-ops id, minted co-located with some other run-ops sibling) on #new
// (prisma17) ONLY — the token is never created on #legacy.
async function seedExecutingLegacyParentAndNewToken(
prisma14: PrismaClient,
prisma17: RunOpsPrismaClient,
router: RoutingRunStore,
parentRunId: string,
waitpointId: string,
suffix: string
) {
const env = await seedControlPlaneEnv(prisma14, suffix);
await router.createRun(
buildCreateRunInput({
runId: parentRunId,
friendlyId: `run_${suffix}_parent`,
organizationId: env.organization.id,
projectId: env.project.id,
runtimeEnvironmentId: env.environment.id,
})
);
const created = await router.findLatestExecutionSnapshot(parentRunId);
await router.createExecutionSnapshot(
{
run: { id: parentRunId, status: "EXECUTING", attemptNumber: 1 },
snapshot: { executionStatus: "EXECUTING", description: "parent executing" },
previousSnapshotId: created!.id,
environmentId: env.environment.id,
environmentType: "PRODUCTION",
projectId: env.project.id,
organizationId: env.organization.id,
},
prisma14
);
// Standalone MANUAL token lives on #new ONLY (run-ops id) — NOT on #legacy.
await prisma17.waitpoint.create({
data: {
id: waitpointId,
friendlyId: `wp_${suffix}`,
type: "MANUAL",
status: "PENDING",
idempotencyKey: `idem_${waitpointId}`,
userProvidedIdempotencyKey: false,
projectId: env.project.id,
environmentId: env.environment.id,
},
});
return env;
}
function makeRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) {
const newStore = new PostgresRunStore({
prisma: prisma17 as never,
readOnlyPrisma: prisma17 as never,
schemaVariant: "dedicated",
});
const legacyStore = new PostgresRunStore({
prisma: prisma14,
readOnlyPrisma: prisma14,
schemaVariant: "legacy",
});
return new RoutingRunStore({ new: newStore, legacy: legacyStore });
}
describe("RunEngine completion->resume flow — a LEGACY run blocked on a NEW-resident token resumes across the seam", () => {
twoDbEngineTest(
"completeWaitpoint on the NEW token finds the #legacy edge and unblocks the LEGACY run",
async ({ prisma14, prisma17, redisOptions }) => {
const router = makeRouter(prisma14 as unknown as PrismaClient, prisma17);
// The harness builds the schema with `prisma db push`, which re-creates the FKs that the
// run-ops split migrations drop in prod (see runOpsStore.crossDbTokenBlock.test.ts). Mirror
// those drops on this clone so the cross-DB edge write exercises real prod state.
await (prisma14 as unknown as PrismaClient).$executeRawUnsafe(
`ALTER TABLE "TaskRunWaitpoint" DROP CONSTRAINT IF EXISTS "TaskRunWaitpoint_waitpointId_fkey"`
);
await (prisma14 as unknown as PrismaClient).$executeRawUnsafe(
`ALTER TABLE "_WaitpointRunConnections" DROP CONSTRAINT IF EXISTS "_WaitpointRunConnections_B_fkey"`
);
// continueRunIfUnblocked's resume snapshot connects the now-COMPLETED (NEW-resident) token via
// _completedWaitpoints (migration 20260705230000 drops this FK in prod; see
// runOpsStore.crossDbCompletedWaitpoint.test.ts).
await (prisma14 as unknown as PrismaClient).$executeRawUnsafe(
`ALTER TABLE "_completedWaitpoints" DROP CONSTRAINT IF EXISTS "_completedWaitpoints_B_fkey"`
);
const engine = new RunEngine({
store: router,
...baseEngineOptions(redisOptions, prisma14),
});
try {
const parentRunId = `run_${CUID_25}`; // LEGACY run -> #legacy
const tokenId = `waitpoint_${generateRunOpsId()}`; // NEW-resident standalone token -> #new
const env = await seedExecutingLegacyParentAndNewToken(
prisma14 as unknown as PrismaClient,
prisma17,
router,
parentRunId,
tokenId,
"xdbresume"
);
// Block the LEGACY run on the NEW token. The edge must land on #legacy (the run's own DB,
// FK-free now that the migration-dropped constraints are mirrored above).
await engine.blockRunWithWaitpoint({
runId: parentRunId,
waitpoints: tokenId,
projectId: env.project.id,
organizationId: env.organization.id,
tx: prisma14 as unknown as PrismaClient,
});
expect(
await (prisma14 as unknown as PrismaClient).taskRunWaitpoint.count({
where: { taskRunId: parentRunId },
})
).toBe(1);
expect(await prisma17.taskRunWaitpoint.count({ where: { taskRunId: parentRunId } })).toBe(
0
);
const blocked = await engine.getRunExecutionData({ runId: parentRunId });
expect(blocked?.snapshot.executionStatus).toBe("EXECUTING_WITH_WAITPOINTS");
const enqueueSpy = vi.spyOn((engine as any).worker, "enqueue");
// Complete the NEW token via the engine path. forWaitpointCompletion resolves #new (run-ops
// id, no pins); completeWaitpoint marks it COMPLETED there, then fans the waitpointId edge
// read across BOTH DBs (findManyTaskRunWaitpoints) to discover the #legacy-resident edge.
const completed = await engine.completeWaitpoint({
id: tokenId,
output: { value: '{"resumed":"legacy-run-new-token"}', isError: false },
});
expect(completed.status).toBe("COMPLETED");
// Token COMPLETED on #new only.
expect((await prisma17.waitpoint.findFirst({ where: { id: tokenId } }))?.status).toBe(
"COMPLETED"
);
expect(
await (prisma14 as unknown as PrismaClient).waitpoint.findFirst({
where: { id: tokenId },
})
).toBeNull();
// The fan-out found the #legacy edge and enqueued the unblock for the LEGACY run.
const continueEnqueued = enqueueSpy.mock.calls.some(
([arg]) =>
(arg as any)?.job === "continueRunIfUnblocked" &&
(arg as any)?.payload?.runId === parentRunId
);
expect(continueEnqueued).toBe(true);
// Driving the unblock body re-resolves the NEW token's COMPLETED status across the seam
// (continueRunIfUnblocked's blockingWaitpoints read rehydrates `waitpoint` cross-DB) and
// actually resumes the LEGACY run — the higher-level outcome, not just the edge write.
const result = await (engine as any).waitpointSystem.continueRunIfUnblocked({
runId: parentRunId,
});
expect(result.status).toBe("unblocked");
const after = await engine.getRunExecutionData({ runId: parentRunId });
expect(after?.snapshot.executionStatus).not.toBe("EXECUTING_WITH_WAITPOINTS");
} finally {
await engine.quit();
}
}
);
});
@@ -0,0 +1,286 @@
// RED→GREEN for bounding the connected-runs read path so it can never emit an unbounded id list.
//
// Mirrors the fix already shipped in `apps/webapp/app/presenters/v3/WaitpointPresenter.server.ts`
// `#connectedRunIdsOn`: existence-JOIN to TaskRun (so a dangling connection row can never occupy a
// LIMIT slot ahead of a real run), then LIMIT to `CONNECTED_RUNS_LIMIT` on BOTH schema branches.
//
// `PostgresRunStore.findWaitpointConnectedRunIds` currently has neither: the dedicated branch does
// a bare `waitpointRunConnection.findMany` (no LIMIT, no existence check) and the legacy branch does
// a bare `SELECT "A" FROM "_WaitpointRunConnections"` (same). `RoutingRunStore.findWaitpointConnectedRunIds`
// unions the two stores' results with no final cap, so even if each store were individually bounded
// the cross-DB union would not be.
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
import { describe, expect } from "vitest";
import { CONNECTED_RUNS_LIMIT, PostgresRunStore } from "./PostgresRunStore.js";
import { RoutingRunStore } from "./runOpsStore.js";
function seedEnvironmentDedicated(suffix: string) {
return {
organization: { id: `org_${suffix}` },
project: { id: `proj_${suffix}` },
environment: { id: `env_${suffix}` },
};
}
async function seedEnvironmentLegacy(prisma: any, suffix: string) {
const organization = await prisma.organization.create({
data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
});
const project = await prisma.project.create({
data: {
name: `Project ${suffix}`,
slug: `project-${suffix}`,
externalRef: `proj_${suffix}`,
organizationId: organization.id,
},
});
const environment = await prisma.runtimeEnvironment.create({
data: {
type: "DEVELOPMENT",
slug: "dev",
projectId: project.id,
organizationId: organization.id,
apiKey: `tr_dev_${suffix}`,
pkApiKey: `pk_dev_${suffix}`,
shortcode: `short_${suffix}`,
},
});
return { organization, project, environment };
}
function taskRunData(opts: {
id: string;
friendlyId: string;
organizationId: string;
projectId: string;
runtimeEnvironmentId: string;
}) {
return {
id: opts.id,
engine: "V2" as const,
status: "PENDING" as const,
friendlyId: opts.friendlyId,
runtimeEnvironmentId: opts.runtimeEnvironmentId,
environmentType: "DEVELOPMENT" as const,
organizationId: opts.organizationId,
projectId: opts.projectId,
taskIdentifier: "my-task",
payload: "{}",
payloadType: "application/json",
traceContext: {},
traceId: `trace_${opts.id}`,
spanId: `span_${opts.id}`,
queue: "task/my-task",
isTest: false,
taskEventStore: "taskEvent",
depth: 0,
};
}
describe("PostgresRunStore.findWaitpointConnectedRunIds — bounded via existence-JOIN + LIMIT", () => {
heteroRunOpsPostgresTest(
"dedicated schema: caps at CONNECTED_RUNS_LIMIT and never returns a dangling (run-less) connection",
async ({ prisma17 }) => {
const store = new PostgresRunStore({
prisma: prisma17 as never,
readOnlyPrisma: prisma17 as never,
schemaVariant: "dedicated",
});
const newEnv = seedEnvironmentDedicated("bound_new");
const waitpointId = "waitpoint_bound_new";
await prisma17.waitpoint.create({
data: {
id: waitpointId,
friendlyId: "wp_bound_new",
type: "MANUAL",
status: "PENDING",
idempotencyKey: `idem_${waitpointId}`,
userProvidedIdempotencyKey: false,
projectId: newEnv.project.id,
environmentId: newEnv.environment.id,
},
});
// Dangling connections FIRST: taskRunId points at runs that were NEVER created. Legal on the
// dedicated schema — `WaitpointRunConnection.taskRunId` is a scalar, no FK (see schema.prisma).
const danglingRunIds = Array.from({ length: 20 }, (_, i) => `run_dangling_${i}`);
for (const runId of danglingRunIds) {
await store.blockRunWithWaitpointEdges({
runId,
waitpointIds: [waitpointId],
projectId: newEnv.project.id,
});
}
// A few REAL connected runs, inserted AFTER the dangling ones.
const realRunIds = ["run_real_0", "run_real_1", "run_real_2"];
for (const [i, id] of realRunIds.entries()) {
await prisma17.taskRun.create({
data: taskRunData({
id,
friendlyId: `run_bound_new_${i}`,
organizationId: newEnv.organization.id,
projectId: newEnv.project.id,
runtimeEnvironmentId: newEnv.environment.id,
}),
});
await store.blockRunWithWaitpointEdges({
runId: id,
waitpointIds: [waitpointId],
projectId: newEnv.project.id,
});
}
const result = await store.findWaitpointConnectedRunIds(waitpointId);
// Bounded.
expect(result.length).toBeLessThanOrEqual(CONNECTED_RUNS_LIMIT);
// Never a dangling id — every returned id must be one of the REAL runs.
for (const id of result) {
expect(realRunIds).toContain(id);
}
// Not starved out: all 3 real runs (well under the limit) must be present.
for (const id of realRunIds) {
expect(result).toContain(id);
}
}
);
heteroRunOpsPostgresTest(
"legacy schema: caps at CONNECTED_RUNS_LIMIT instead of returning every connected run",
async ({ prisma14 }) => {
const store = new PostgresRunStore({
prisma: prisma14 as never,
readOnlyPrisma: prisma14 as never,
schemaVariant: "legacy",
});
const legEnv = await seedEnvironmentLegacy(prisma14, "bound_leg");
const waitpointId = "waitpoint_bound_leg";
await prisma14.waitpoint.create({
data: {
id: waitpointId,
friendlyId: "wp_bound_leg",
type: "MANUAL",
status: "PENDING",
idempotencyKey: `idem_${waitpointId}`,
userProvidedIdempotencyKey: false,
projectId: legEnv.project.id,
environmentId: legEnv.environment.id,
},
});
// More real connected runs than CONNECTED_RUNS_LIMIT — the legacy `_WaitpointRunConnections`
// table still enforces an FK from "A" to TaskRun, so every row here is a real run.
const realRunIds = Array.from({ length: CONNECTED_RUNS_LIMIT + 3 }, (_, i) => `run_leg_${i}`);
for (const [i, id] of realRunIds.entries()) {
await prisma14.taskRun.create({
data: taskRunData({
id,
friendlyId: `run_bound_leg_${i}`,
organizationId: legEnv.organization.id,
projectId: legEnv.project.id,
runtimeEnvironmentId: legEnv.environment.id,
}),
});
await store.blockRunWithWaitpointEdges({
runId: id,
waitpointIds: [waitpointId],
projectId: legEnv.project.id,
});
}
const result = await store.findWaitpointConnectedRunIds(waitpointId);
expect(result.length).toBeLessThanOrEqual(CONNECTED_RUNS_LIMIT);
}
);
});
describe("RoutingRunStore.findWaitpointConnectedRunIds — the cross-DB union is also bounded", () => {
heteroRunOpsPostgresTest(
"slices the merged NEW+LEGACY result to CONNECTED_RUNS_LIMIT",
async ({ prisma14, prisma17 }) => {
const legacyStore = new PostgresRunStore({
prisma: prisma14 as never,
readOnlyPrisma: prisma14 as never,
schemaVariant: "legacy",
});
const newStore = new PostgresRunStore({
prisma: prisma17 as never,
readOnlyPrisma: prisma17 as never,
schemaVariant: "dedicated",
});
const router = new RoutingRunStore({ new: newStore, legacy: legacyStore });
const legEnv = await seedEnvironmentLegacy(prisma14, "bound_union_leg");
const newEnv = seedEnvironmentDedicated("bound_union_new");
const waitpointId = "waitpoint_bound_union";
// The legacy `TaskRunWaitpoint.waitpointId` FK is still live in this test DB (built via
// `prisma db push` from schema.prisma, which still declares the relation even though a later
// migration drops the constraint in real deployments) — the legacy leg needs a real row here.
await prisma14.waitpoint.create({
data: {
id: waitpointId,
friendlyId: "wp_bound_union",
type: "MANUAL",
status: "PENDING",
idempotencyKey: `idem_${waitpointId}`,
userProvidedIdempotencyKey: false,
projectId: legEnv.project.id,
environmentId: legEnv.environment.id,
},
});
// Each store individually caps at CONNECTED_RUNS_LIMIT, but a distinct set of runs on EACH side
// means the union of the two capped results can still exceed the limit without a final slice.
const legacyRunIds = Array.from(
{ length: CONNECTED_RUNS_LIMIT },
(_, i) => `run_union_leg_${i}`
);
for (const [i, id] of legacyRunIds.entries()) {
await prisma14.taskRun.create({
data: taskRunData({
id,
friendlyId: `run_union_leg_${i}`,
organizationId: legEnv.organization.id,
projectId: legEnv.project.id,
runtimeEnvironmentId: legEnv.environment.id,
}),
});
await legacyStore.blockRunWithWaitpointEdges({
runId: id,
waitpointIds: [waitpointId],
projectId: legEnv.project.id,
});
}
const newRunIds = Array.from(
{ length: CONNECTED_RUNS_LIMIT },
(_, i) => `run_union_new_${i}`
);
for (const [i, id] of newRunIds.entries()) {
await prisma17.taskRun.create({
data: taskRunData({
id,
friendlyId: `run_union_new_${i}`,
organizationId: newEnv.organization.id,
projectId: newEnv.project.id,
runtimeEnvironmentId: newEnv.environment.id,
}),
});
await newStore.blockRunWithWaitpointEdges({
runId: id,
waitpointIds: [waitpointId],
projectId: newEnv.project.id,
});
}
const result = await router.findWaitpointConnectedRunIds(waitpointId);
expect(result.length).toBeLessThanOrEqual(CONNECTED_RUNS_LIMIT);
}
);
});
@@ -0,0 +1,130 @@
// RED→GREEN for bounding the DISPLAY `connectedRuns` RELATION hydration (findWaitpoint /
// findManyWaitpoints with `include`/`select: { connectedRuns }`), NOT the id-list helper
// `findWaitpointConnectedRunIds` (covered separately in connectedRunsBounded.test.ts).
//
// The dedicated-schema hydrator `hydrateConnectedRuns` goes through the generic
// `batchHydrateJoinRelation`, which fetched EVERY WaitpointRunConnection link with no per-parent
// cap — so a heavily-fanned-in waitpoint hydrated an unbounded connectedRuns list, bypassing the
// CONNECTED_RUNS_LIMIT the presenter/`findWaitpointConnectedRunIds` already enforce.
//
// The fix bounds it per parent via a window function + existence-JOIN to TaskRun, mirroring the
// bounded helper: a dangling connection row never occupies a LIMIT slot, and the returned relation
// is capped at CONNECTED_RUNS_LIMIT.
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
import { describe, expect } from "vitest";
import { CONNECTED_RUNS_LIMIT, PostgresRunStore } from "./PostgresRunStore.js";
function seedEnvironmentDedicated(suffix: string) {
return {
organization: { id: `org_${suffix}` },
project: { id: `proj_${suffix}` },
environment: { id: `env_${suffix}` },
};
}
function taskRunData(opts: {
id: string;
friendlyId: string;
organizationId: string;
projectId: string;
runtimeEnvironmentId: string;
}) {
return {
id: opts.id,
engine: "V2" as const,
status: "PENDING" as const,
friendlyId: opts.friendlyId,
runtimeEnvironmentId: opts.runtimeEnvironmentId,
environmentType: "DEVELOPMENT" as const,
organizationId: opts.organizationId,
projectId: opts.projectId,
taskIdentifier: "my-task",
payload: "{}",
payloadType: "application/json",
traceContext: {},
traceId: `trace_${opts.id}`,
spanId: `span_${opts.id}`,
queue: "task/my-task",
isTest: false,
taskEventStore: "taskEvent",
depth: 0,
};
}
describe("PostgresRunStore.findWaitpoint — connectedRuns relation is bounded", () => {
heteroRunOpsPostgresTest(
"dedicated schema: include connectedRuns caps at CONNECTED_RUNS_LIMIT and skips dangling connections",
async ({ prisma17 }) => {
const store = new PostgresRunStore({
prisma: prisma17 as never,
readOnlyPrisma: prisma17 as never,
schemaVariant: "dedicated",
});
const env = seedEnvironmentDedicated("cr_incl_new");
const waitpointId = "waitpoint_cr_incl_new";
await prisma17.waitpoint.create({
data: {
id: waitpointId,
friendlyId: "wp_cr_incl_new",
type: "MANUAL",
status: "PENDING",
idempotencyKey: `idem_${waitpointId}`,
userProvidedIdempotencyKey: false,
projectId: env.project.id,
environmentId: env.environment.id,
},
});
// Dangling connections (taskRunId points at runs that were never created). Legal on the
// dedicated schema — WaitpointRunConnection.taskRunId is a scalar with no FK.
const danglingRunIds = Array.from({ length: 20 }, (_, i) => `run_cr_dangling_${i}`);
for (const runId of danglingRunIds) {
await store.blockRunWithWaitpointEdges({
runId,
waitpointIds: [waitpointId],
projectId: env.project.id,
});
}
// More REAL connected runs than the limit.
const realRunIds = Array.from(
{ length: CONNECTED_RUNS_LIMIT + 3 },
(_, i) => `run_cr_real_${i}`
);
for (const [i, id] of realRunIds.entries()) {
await prisma17.taskRun.create({
data: taskRunData({
id,
friendlyId: `run_cr_incl_new_${i}`,
organizationId: env.organization.id,
projectId: env.project.id,
runtimeEnvironmentId: env.environment.id,
}),
});
await store.blockRunWithWaitpointEdges({
runId: id,
waitpointIds: [waitpointId],
projectId: env.project.id,
});
}
const waitpoint = (await store.findWaitpoint({
where: { id: waitpointId },
include: { connectedRuns: true },
})) as unknown as { connectedRuns: { id: string }[] } | null;
expect(waitpoint).not.toBeNull();
const connectedRuns = waitpoint!.connectedRuns;
// Bounded to exactly the cap (fixture connects more real runs than the limit).
expect(connectedRuns).toHaveLength(CONNECTED_RUNS_LIMIT);
// Never a dangling id — every returned run must be a REAL run.
for (const run of connectedRuns) {
expect(realRunIds).toContain(run.id);
expect(danglingRunIds).not.toContain(run.id);
}
}
);
});
@@ -0,0 +1,175 @@
// RED→GREEN: `findRunsByIds` is the grouped replacement for `Promise.all(ids.map(id =>
// findRun(id)))` — it must issue ONE `findMany` for the whole id batch, never a `findFirst`
// per id. `postgresTest` is a REAL PrismaClient over the standard (non-dedicated) schema. The
// counting proxy wraps its `taskRun` delegate to tally `findFirst`/`findMany` calls while
// delegating every call to the real client — the DB still runs the query; this is
// instrumentation, not a mock.
import { postgresTest } from "@internal/testcontainers";
import type { PrismaClient } from "@trigger.dev/database";
import { describe, expect } from "vitest";
import { PostgresRunStore } from "./PostgresRunStore.js";
import type { CreateRunInput } from "./types.js";
type CallCounts = { findMany: number; findFirst: number };
// Wraps the named delegates on a REAL client to tally `findMany`/`findFirst` calls PER delegate,
// delegating every call unchanged to the real client (the DB still runs the query).
function countingClient(
real: PrismaClient,
delegateNames: string[]
): { client: PrismaClient; counts: Record<string, CallCounts> } {
const counts: Record<string, CallCounts> = {};
for (const name of delegateNames) {
counts[name] = { findMany: 0, findFirst: 0 };
}
const wrapped = new Map(
delegateNames.map((name) => [
name,
new Proxy((real as any)[name], {
get(target, prop) {
if (prop === "findMany" || prop === "findFirst") {
counts[name][prop as "findMany" | "findFirst"]++;
}
return (target as any)[prop];
},
}),
])
);
const client = new Proxy(real, {
get(target, prop) {
if (typeof prop === "string" && wrapped.has(prop)) {
return wrapped.get(prop);
}
return (target as any)[prop];
},
}) as PrismaClient;
return { client, counts };
}
async function seedEnvironment(prisma: PrismaClient) {
const organization = await prisma.organization.create({
data: {
title: "Test Organization",
slug: "test-organization-findrunsbyids",
},
});
const project = await prisma.project.create({
data: {
name: "Test Project",
slug: "test-project-findrunsbyids",
externalRef: "proj_findrunsbyids",
organizationId: organization.id,
},
});
const environment = await prisma.runtimeEnvironment.create({
data: {
type: "DEVELOPMENT",
slug: "dev",
projectId: project.id,
organizationId: organization.id,
apiKey: "tr_dev_apikey_findrunsbyids",
pkApiKey: "pk_dev_apikey_findrunsbyids",
shortcode: "short_code_findrunsbyids",
},
});
return { organization, project, environment };
}
function buildCreateRunInput(params: {
runId: string;
friendlyId: string;
organizationId: string;
projectId: string;
runtimeEnvironmentId: string;
}): CreateRunInput {
return {
data: {
id: params.runId,
engine: "V2",
status: "PENDING",
friendlyId: params.friendlyId,
runtimeEnvironmentId: params.runtimeEnvironmentId,
environmentType: "DEVELOPMENT",
organizationId: params.organizationId,
projectId: params.projectId,
taskIdentifier: "my-task",
payload: "{}",
payloadType: "application/json",
traceContext: {},
traceId: `trace_${params.runId}`,
spanId: `span_${params.runId}`,
queue: "task/my-task",
isTest: false,
taskEventStore: "taskEvent",
depth: 0,
},
snapshot: {
engine: "V2",
executionStatus: "RUN_CREATED",
description: "Run was created",
runStatus: "PENDING",
environmentId: params.runtimeEnvironmentId,
environmentType: "DEVELOPMENT",
projectId: params.projectId,
organizationId: params.organizationId,
},
};
}
describe("PostgresRunStore.findRunsByIds", () => {
postgresTest(
"resolves N ids with ONE grouped findMany, never a findFirst per id",
async ({ prisma }) => {
const { organization, project, environment } = await seedEnvironment(prisma);
const seedStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma });
const runId1 = "run_findbyids_1";
const runId2 = "run_findbyids_2";
await seedStore.createRun(
buildCreateRunInput({
runId: runId1,
friendlyId: "run_findbyids_1_friendly",
organizationId: organization.id,
projectId: project.id,
runtimeEnvironmentId: environment.id,
})
);
await seedStore.createRun(
buildCreateRunInput({
runId: runId2,
friendlyId: "run_findbyids_2_friendly",
organizationId: organization.id,
projectId: project.id,
runtimeEnvironmentId: environment.id,
})
);
const counting = countingClient(prisma, ["taskRun"]);
const readStore = new PostgresRunStore({ prisma, readOnlyPrisma: counting.client });
const byId = await readStore.findRunsByIds([runId1, runId2], {
select: { friendlyId: true, status: true },
});
expect(byId.size).toBe(2);
expect(byId.get(runId1)?.friendlyId).toBe("run_findbyids_1_friendly");
expect(byId.get(runId1)?.status).toBe("PENDING");
expect(byId.get(runId2)?.friendlyId).toBe("run_findbyids_2_friendly");
expect(byId.get(runId2)?.status).toBe("PENDING");
// The select omitted `id`; the id we force-inject for keying must not leak into the value
// (it would otherwise violate the declared payload type and expose an unrequested id).
expect("id" in (byId.get(runId1) as object)).toBe(false);
expect("id" in (byId.get(runId2) as object)).toBe(false);
// GROUPED: one findMany for the WHOLE batch, never one findFirst per id.
expect(counting.counts.taskRun.findMany).toBe(1);
expect(counting.counts.taskRun.findFirst).toBe(0);
}
);
});
@@ -0,0 +1,239 @@
// RED→GREEN: the dedicated-schema relation hydrators (`hydrateBlockingTaskRuns` /
// `#hydrateDedicatedRelations`) resolve a relation PER PARENT ROW, so for N parents each requesting
// the same relation, the store issues N round trips (a `findFirst`/`findMany` per row) instead of one
// grouped query for the whole batch. `heteroRunOpsPostgresTest.prisma17` is a REAL RunOpsPrismaClient
// over the dedicated subset schema. The counting proxy wraps its `taskRun` delegate to tally
// `findFirst`/`findMany` calls while delegating every call to the real client — the DB still runs the
// query; this is instrumentation, not a mock.
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
import { describe, expect } from "vitest";
import { PostgresRunStore } from "./PostgresRunStore.js";
import type { CreateRunInput } from "./types.js";
type CallCounts = { findMany: number; findFirst: number };
// Wraps the named delegates on a REAL client to tally `findMany`/`findFirst` calls PER delegate,
// delegating every call unchanged to the real client (the DB still runs the query).
function countingClient(
real: RunOpsPrismaClient,
delegateNames: string[]
): { client: RunOpsPrismaClient; counts: Record<string, CallCounts>; queryRaw: { count: number } } {
const counts: Record<string, CallCounts> = {};
for (const name of delegateNames) {
counts[name] = { findMany: 0, findFirst: 0 };
}
const queryRaw = { count: 0 };
const wrapped = new Map(
delegateNames.map((name) => [
name,
new Proxy((real as any)[name], {
get(target, prop) {
if (prop === "findMany" || prop === "findFirst") {
counts[name][prop as "findMany" | "findFirst"]++;
}
return (target as any)[prop];
},
}),
])
);
const client = new Proxy(real, {
get(target, prop) {
if (typeof prop === "string" && wrapped.has(prop)) {
return wrapped.get(prop);
}
// Tally the grouped raw query the bounded connectedRuns hydrator issues, delegating unchanged.
if (prop === "$queryRaw") {
return (...args: unknown[]) => {
queryRaw.count++;
return (target as any).$queryRaw(...args);
};
}
return (target as any)[prop];
},
}) as RunOpsPrismaClient;
return { client, counts, queryRaw };
}
function seedEnvironmentDedicated(suffix: string) {
return {
organization: { id: `org_${suffix}` },
project: { id: `proj_${suffix}` },
environment: { id: `env_${suffix}` },
};
}
function buildCreateRunInput(params: {
runId: string;
friendlyId: string;
organizationId: string;
projectId: string;
runtimeEnvironmentId: string;
}): CreateRunInput {
return {
data: {
id: params.runId,
engine: "V2",
status: "PENDING",
friendlyId: params.friendlyId,
runtimeEnvironmentId: params.runtimeEnvironmentId,
environmentType: "DEVELOPMENT",
organizationId: params.organizationId,
projectId: params.projectId,
taskIdentifier: "my-task",
payload: '{"hello":"world"}',
payloadType: "application/json",
traceContext: { trace: "ctx" },
traceId: `trace_${params.runId}`,
spanId: `span_${params.runId}`,
runTags: [],
queue: "task/my-task",
isTest: false,
taskEventStore: "taskEvent",
depth: 0,
createdAt: new Date("2024-01-01T00:00:00.000Z"),
},
snapshot: {
engine: "V2",
executionStatus: "RUN_CREATED",
description: "Run was created",
runStatus: "PENDING",
environmentId: params.runtimeEnvironmentId,
environmentType: "DEVELOPMENT",
projectId: params.projectId,
organizationId: params.organizationId,
},
};
}
function makeStore(prisma: RunOpsPrismaClient) {
return new PostgresRunStore({
prisma: prisma as never,
readOnlyPrisma: prisma as never,
schemaVariant: "dedicated",
});
}
describe("PostgresRunStore dedicated relation hydrators — grouped batch reads", () => {
heteroRunOpsPostgresTest(
"hydrateBlockingTaskRuns resolves N edges' `taskRun` with ONE grouped findMany, not N findFirst",
async ({ prisma17 }) => {
const seedStore = makeStore(prisma17);
const env = seedEnvironmentDedicated("hbt");
const waitpointId = "wp_hbt_shared";
await prisma17.waitpoint.create({
data: {
id: waitpointId,
friendlyId: "wp_hbt_friendly",
type: "MANUAL",
status: "PENDING",
idempotencyKey: `idem_${waitpointId}`,
userProvidedIdempotencyKey: false,
projectId: env.project.id,
environmentId: env.environment.id,
},
});
const runIds = ["run_hbt_1", "run_hbt_2", "run_hbt_3"];
for (const runId of runIds) {
await seedStore.createRun(
buildCreateRunInput({
runId,
friendlyId: `${runId}_friendly`,
organizationId: env.organization.id,
projectId: env.project.id,
runtimeEnvironmentId: env.environment.id,
})
);
await seedStore.blockRunWithWaitpointEdges({
runId,
waitpointIds: [waitpointId],
projectId: env.project.id,
});
}
const counting = countingClient(prisma17, ["taskRun"]);
const readStore = makeStore(counting.client);
const waitpoint = (await readStore.findWaitpoint({
where: { id: waitpointId },
include: {
blockingTaskRuns: { select: { taskRun: { select: { id: true, friendlyId: true } } } },
},
})) as { blockingTaskRuns: { taskRun: { id: string; friendlyId: string } | null }[] } | null;
const blocking = waitpoint?.blockingTaskRuns ?? [];
expect(blocking).toHaveLength(3);
expect(blocking.map((b) => b.taskRun?.id).sort()).toEqual([...runIds].sort());
// GROUPED: one findMany for the WHOLE batch, never one findFirst per edge.
expect(counting.counts.taskRun.findMany).toBe(1);
expect(counting.counts.taskRun.findFirst).toBe(0);
}
);
heteroRunOpsPostgresTest(
"findManyWaitpoints hydrates N parents' `connectedRuns` with ONE grouped join query + ONE grouped target query",
async ({ prisma17 }) => {
const seedStore = makeStore(prisma17);
const env = seedEnvironmentDedicated("cr");
const waitpointIds = ["wp_cr_1", "wp_cr_2", "wp_cr_3"];
const runIds = ["run_cr_1", "run_cr_2", "run_cr_3"];
for (const [i, waitpointId] of waitpointIds.entries()) {
await prisma17.waitpoint.create({
data: {
id: waitpointId,
friendlyId: `${waitpointId}_friendly`,
type: "MANUAL",
status: "PENDING",
idempotencyKey: `idem_${waitpointId}`,
userProvidedIdempotencyKey: false,
projectId: env.project.id,
environmentId: env.environment.id,
},
});
await seedStore.createRun(
buildCreateRunInput({
runId: runIds[i],
friendlyId: `${runIds[i]}_friendly`,
organizationId: env.organization.id,
projectId: env.project.id,
runtimeEnvironmentId: env.environment.id,
})
);
// One connection per waitpoint, to its own distinct run.
await seedStore.blockRunWithWaitpointEdges({
runId: runIds[i],
waitpointIds: [waitpointId],
projectId: env.project.id,
});
}
const counting = countingClient(prisma17, ["waitpointRunConnection", "taskRun"]);
const readStore = makeStore(counting.client);
const rows = (await readStore.findManyWaitpoints({
where: { id: { in: waitpointIds } },
include: { connectedRuns: { select: { id: true, friendlyId: true } } },
})) as { id: string; connectedRuns: { id: string; friendlyId: string }[] }[];
expect(rows).toHaveLength(3);
const byWaitpointId = new Map(rows.map((r) => [r.id, r.connectedRuns]));
for (const [i, waitpointId] of waitpointIds.entries()) {
const connected = byWaitpointId.get(waitpointId) ?? [];
expect(connected).toHaveLength(1);
expect(connected[0].id).toBe(runIds[i]);
}
// GROUPED: one bounded join query (raw, ROW_NUMBER-per-parent) + one target query for the
// WHOLE batch, never one pair per parent. The bounded hydrator replaces the delegate
// `waitpointRunConnection.findMany` with a single `$queryRaw`.
expect(counting.queryRaw.count).toBe(1);
expect(counting.counts.waitpointRunConnection.findMany).toBe(0);
expect(counting.counts.taskRun.findMany).toBe(1);
expect(counting.counts.taskRun.findFirst).toBe(0);
}
);
});
@@ -0,0 +1,226 @@
// RED→GREEN: the dedicated-schema relation hydrators (`batchHydrateJoinRelation` /
// `batchHydrateEdgeTarget`) fetch the target row's FULL columns (no `select`) and only strip fields
// in JS afterwards via `applyProjection`. At prod scale (a 6.68B-row `TaskRun`) that over-fetches
// every wide TOASTed column (`payload`/`output`/`context`) even when the caller only asked for
// `friendlyId`. The RESULT is identical with/without the fix (`applyProjection` trims either way),
// so this asserts the QUERY SHAPE Prisma actually receives, via a `$extends` query hook on a REAL
// `heteroRunOpsPostgresTest.prisma17` client — the DB still runs the query, this only observes it.
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
import { describe, expect } from "vitest";
import { PostgresRunStore } from "./PostgresRunStore.js";
import type { CreateRunInput } from "./types.js";
function seedEnvironmentDedicated(suffix: string) {
return {
organization: { id: `org_${suffix}` },
project: { id: `proj_${suffix}` },
environment: { id: `env_${suffix}` },
};
}
function buildCreateRunInput(params: {
runId: string;
friendlyId: string;
organizationId: string;
projectId: string;
runtimeEnvironmentId: string;
}): CreateRunInput {
return {
data: {
id: params.runId,
engine: "V2",
status: "PENDING",
friendlyId: params.friendlyId,
runtimeEnvironmentId: params.runtimeEnvironmentId,
environmentType: "DEVELOPMENT",
organizationId: params.organizationId,
projectId: params.projectId,
taskIdentifier: "my-task",
payload: '{"hello":"world"}',
payloadType: "application/json",
traceContext: { trace: "ctx" },
traceId: `trace_${params.runId}`,
spanId: `span_${params.runId}`,
runTags: [],
queue: "task/my-task",
isTest: false,
taskEventStore: "taskEvent",
depth: 0,
createdAt: new Date("2024-01-01T00:00:00.000Z"),
},
snapshot: {
engine: "V2",
executionStatus: "RUN_CREATED",
description: "Run was created",
runStatus: "PENDING",
environmentId: params.runtimeEnvironmentId,
environmentType: "DEVELOPMENT",
projectId: params.projectId,
organizationId: params.organizationId,
},
};
}
function makeStore(prisma: RunOpsPrismaClient) {
return new PostgresRunStore({
prisma: prisma as never,
readOnlyPrisma: prisma as never,
schemaVariant: "dedicated",
});
}
// Wraps `taskRun.findMany` with a REAL Prisma Client Extension query hook: every call's `args` is
// captured, then delegated unchanged to `query(args)` so the DB still executes it. This is
// observation, not a mock — the extension can't fabricate a result, only see what's sent.
function withCapturedTaskRunFindManyArgs(real: RunOpsPrismaClient) {
const capturedArgs: Record<string, unknown>[] = [];
const extended = (real as unknown as { $extends: (config: unknown) => unknown }).$extends({
query: {
taskRun: {
findMany({
args,
query,
}: {
args: Record<string, unknown>;
query: (args: unknown) => Promise<unknown>;
}) {
capturedArgs.push(args);
return query(args);
},
},
},
});
return { client: extended as RunOpsPrismaClient, capturedArgs };
}
describe("PostgresRunStore dedicated relation hydrators — target findMany select pushdown", () => {
heteroRunOpsPostgresTest(
"hydrateConnectedRuns narrows the target taskRun.findMany to the caller's select instead of fetching the full row",
async ({ prisma17 }) => {
const seedStore = makeStore(prisma17);
const env = seedEnvironmentDedicated("select_pushdown");
const runId = "run_select_pushdown";
const waitpointId = "wp_select_pushdown";
await prisma17.waitpoint.create({
data: {
id: waitpointId,
friendlyId: `${waitpointId}_friendly`,
type: "MANUAL",
status: "PENDING",
idempotencyKey: `idem_${waitpointId}`,
userProvidedIdempotencyKey: false,
projectId: env.project.id,
environmentId: env.environment.id,
},
});
await seedStore.createRun(
buildCreateRunInput({
runId,
friendlyId: `${runId}_friendly`,
organizationId: env.organization.id,
projectId: env.project.id,
runtimeEnvironmentId: env.environment.id,
})
);
// Seeds BOTH the TaskRunWaitpoint edge and the WaitpointRunConnection join row that
// `connectedRuns` reads from.
await seedStore.blockRunWithWaitpointEdges({
runId,
waitpointIds: [waitpointId],
projectId: env.project.id,
});
const { client: capturingClient, capturedArgs } = withCapturedTaskRunFindManyArgs(prisma17);
const readStore = makeStore(capturingClient);
const waitpoint = (await readStore.findWaitpoint({
where: { id: waitpointId },
include: { connectedRuns: { select: { friendlyId: true } } },
})) as { connectedRuns: { friendlyId: string }[] } | null;
expect(capturedArgs).toHaveLength(1);
const targetArgs = capturedArgs[0] as { where: unknown; select?: Record<string, unknown> };
// THE FIX: the target `findMany` carries a `select` narrowed to the caller's projection (plus
// the `id` the hydrator keys off), never a bare where-only (= full row) query.
expect(targetArgs.select).toBeDefined();
expect(targetArgs.select).toMatchObject({ friendlyId: true, id: true });
expect(targetArgs.select?.payload).toBeUndefined();
expect(targetArgs.select?.output).toBeUndefined();
expect(targetArgs.select?.context).toBeUndefined();
// `applyProjection` still trims correctly: exactly `{ friendlyId }`, no `id` leak, no wide
// columns — proving the pushdown didn't change the caller-visible result.
expect(waitpoint?.connectedRuns).toHaveLength(1);
expect(waitpoint?.connectedRuns[0]).toEqual({ friendlyId: `${runId}_friendly` });
}
);
heteroRunOpsPostgresTest(
"hydrateEdgeTaskRun (batchHydrateEdgeTarget) narrows the target taskRun.findMany to the caller's select instead of fetching the full row",
async ({ prisma17 }) => {
const seedStore = makeStore(prisma17);
const env = seedEnvironmentDedicated("edge_pushdown");
const runId = "run_edge_pushdown";
const waitpointId = "wp_edge_pushdown";
await prisma17.waitpoint.create({
data: {
id: waitpointId,
friendlyId: `${waitpointId}_friendly`,
type: "MANUAL",
status: "PENDING",
idempotencyKey: `idem_${waitpointId}`,
userProvidedIdempotencyKey: false,
projectId: env.project.id,
environmentId: env.environment.id,
},
});
await seedStore.createRun(
buildCreateRunInput({
runId,
friendlyId: `${runId}_friendly`,
organizationId: env.organization.id,
projectId: env.project.id,
runtimeEnvironmentId: env.environment.id,
})
);
await seedStore.blockRunWithWaitpointEdges({
runId,
waitpointIds: [waitpointId],
projectId: env.project.id,
});
const { client: capturingClient, capturedArgs } = withCapturedTaskRunFindManyArgs(prisma17);
const readStore = makeStore(capturingClient);
// TaskRunWaitpoint's `taskRun` relation is the dedicated `hydrateEdgeTaskRun` ->
// `batchHydrateEdgeTarget` path (the scalar-FK-on-the-parent variant).
const edges = (await readStore.findManyTaskRunWaitpoints({
where: { waitpointId },
select: { taskRun: { select: { friendlyId: true } } },
})) as { taskRun: { friendlyId: string } | null }[];
expect(capturedArgs).toHaveLength(1);
const targetArgs = capturedArgs[0] as { where: unknown; select?: Record<string, unknown> };
expect(targetArgs.select).toBeDefined();
expect(targetArgs.select).toMatchObject({ friendlyId: true, id: true });
expect(targetArgs.select?.payload).toBeUndefined();
expect(targetArgs.select?.output).toBeUndefined();
expect(targetArgs.select?.context).toBeUndefined();
expect(edges).toHaveLength(1);
expect(edges[0].taskRun).toEqual({ friendlyId: `${runId}_friendly` });
}
);
});
@@ -0,0 +1,139 @@
// RED→GREEN: bare-projection dedicated-relation hydration (e.g. `connectedRuns: true`) shares ONE
// target object reference across every parent bucket that links it, so an in-place mutation on one
// parent's hydrated target aliases into another's. Two waitpoints connecting to the SAME run should
// come back with DISTINCT `TaskRun` objects, not the same instance.
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
import { describe, expect } from "vitest";
import { PostgresRunStore } from "./PostgresRunStore.js";
import type { CreateRunInput } from "./types.js";
function seedEnvironmentDedicated(suffix: string) {
return {
organization: { id: `org_${suffix}` },
project: { id: `proj_${suffix}` },
environment: { id: `env_${suffix}` },
};
}
function buildCreateRunInput(params: {
runId: string;
friendlyId: string;
organizationId: string;
projectId: string;
runtimeEnvironmentId: string;
}): CreateRunInput {
return {
data: {
id: params.runId,
engine: "V2",
status: "PENDING",
friendlyId: params.friendlyId,
runtimeEnvironmentId: params.runtimeEnvironmentId,
environmentType: "DEVELOPMENT",
organizationId: params.organizationId,
projectId: params.projectId,
taskIdentifier: "my-task",
payload: '{"hello":"world"}',
payloadType: "application/json",
traceContext: { trace: "ctx" },
traceId: `trace_${params.runId}`,
spanId: `span_${params.runId}`,
runTags: [],
queue: "task/my-task",
isTest: false,
taskEventStore: "taskEvent",
depth: 0,
createdAt: new Date("2024-01-01T00:00:00.000Z"),
},
snapshot: {
engine: "V2",
executionStatus: "RUN_CREATED",
description: "Run was created",
runStatus: "PENDING",
environmentId: params.runtimeEnvironmentId,
environmentType: "DEVELOPMENT",
projectId: params.projectId,
organizationId: params.organizationId,
},
};
}
function makeStore(prisma: RunOpsPrismaClient) {
return new PostgresRunStore({
prisma: prisma as never,
readOnlyPrisma: prisma as never,
schemaVariant: "dedicated",
});
}
describe("PostgresRunStore dedicated relation hydrators — shared target identity (bare projection)", () => {
heteroRunOpsPostgresTest(
"findManyWaitpoints hydrates bare `connectedRuns: true` for two waitpoints sharing one run as DISTINCT objects",
async ({ prisma17 }) => {
const store = makeStore(prisma17);
const env = seedEnvironmentDedicated("shared_target");
const runId = "run_shared_target";
const waitpointIds = ["wp_shared_1", "wp_shared_2"];
await store.createRun(
buildCreateRunInput({
runId,
friendlyId: "run_shared_target_friendly",
organizationId: env.organization.id,
projectId: env.project.id,
runtimeEnvironmentId: env.environment.id,
})
);
for (const waitpointId of waitpointIds) {
await prisma17.waitpoint.create({
data: {
id: waitpointId,
friendlyId: `${waitpointId}_friendly`,
type: "MANUAL",
status: "PENDING",
idempotencyKey: `idem_${waitpointId}`,
userProvidedIdempotencyKey: false,
projectId: env.project.id,
environmentId: env.environment.id,
},
});
}
// Both waitpoints connect to the SAME run: the target-row `findMany` in
// `batchHydrateJoinRelation` returns ONE row for the run, shared across both parents' buckets.
await store.blockRunWithWaitpointEdges({
runId,
waitpointIds,
projectId: env.project.id,
});
// Bare projection (`connectedRuns: true`, no sub-select) — the buggy path: `applyProjection`
// returns the row UNCHANGED instead of a fresh object.
const rows = (await store.findManyWaitpoints({
where: { id: { in: waitpointIds } },
include: { connectedRuns: true },
})) as { id: string; connectedRuns: { id: string; friendlyId: string }[] }[];
expect(rows).toHaveLength(2);
const byWaitpointId = new Map(rows.map((r) => [r.id, r.connectedRuns]));
const target1 = byWaitpointId.get("wp_shared_1")?.[0];
const target2 = byWaitpointId.get("wp_shared_2")?.[0];
expect(target1).toBeDefined();
expect(target2).toBeDefined();
expect(target1!.id).toBe(runId);
expect(target2!.id).toBe(runId);
// THE BUG: both hydrated targets are the SAME object reference.
expect(target1).not.toBe(target2);
// Mutating a top-level field on one parent's hydrated target must NOT change the other's.
(target1 as { friendlyId: string }).friendlyId = "mutated";
expect(target2!.friendlyId).not.toBe("mutated");
}
);
});
@@ -89,6 +89,10 @@ export interface RunOpsTransactionalClient extends RunOpsCapableClient {
*/
export type RunStoreSchemaVariant = "legacy" | "dedicated";
// Mirrors the webapp's `CONNECTED_RUNS_DISPLAY_LIMIT`
// (apps/webapp/app/presenters/v3/WaitpointPresenter.server.ts) — keep the values in sync.
export const CONNECTED_RUNS_LIMIT = 5;
export type PostgresRunStoreOptions = {
prisma: RunOpsCapableClient;
readOnlyPrisma: RunOpsCapableClient;
@@ -99,13 +103,15 @@ export type PostgresRunStoreOptions = {
// A caller sub-select for a relation: `{ select?, include? }` or `true` for a bare `key: true`.
type SubProjection = { select?: any; include?: any } | true | undefined;
// Hydrates one dedicated-schema relation for a single parent row, honoring the caller's sub-projection.
// Hydrates one dedicated-schema relation for a WHOLE batch of parent rows in one grouped pass:
// one query for the join/target rows spanning every parent id, never one per parent. Returns a
// Map keyed by parent `id` to the already-defaulted (null / []) hydrated value.
type DedicatedRelationHydrator = (
client: RunOpsCapableClient,
parent: Record<string, unknown>,
parents: Record<string, unknown>[],
projection: { select?: any; include?: any } | undefined,
store: PostgresRunStore
) => Promise<unknown>;
) => Promise<Map<string, unknown>>;
// The dedicated-schema relation keys (with hydrators) for a single Prisma model.
type DedicatedRelationSpec = Record<string, DedicatedRelationHydrator>;
@@ -119,13 +125,23 @@ function projectionOf(sub: SubProjection): { select?: any; include?: any } | und
}
// Apply a caller sub-projection to a hydrated row (or array) so only requested fields remain.
//
// Bare-projection path (no `select`): return a SHALLOW CLONE, not the row itself, so every parent
// bucket that links the same target gets a distinct top-level object — two parents sharing one
// target (e.g. two waitpoints connected to the same run) must not alias through a shared reference.
// This only protects top-level mutation; a deep in-place mutation of a nested field would still
// alias, which matches the realistic redaction/patch cases and avoids a costly deep clone on hot
// reads.
function applyProjection<T extends Record<string, unknown> | null>(
row: T,
projection: { select?: any; include?: any } | undefined
): T {
if (!row || !projection?.select) {
if (!row) {
return row;
}
if (!projection?.select) {
return { ...row } as T;
}
const keys = Object.keys(projection.select).filter((k) => projection.select[k]);
const out: Record<string, unknown> = {};
for (const k of keys) {
@@ -175,153 +191,247 @@ function stripDedicatedRelations(
return { stripped: args, requested };
}
// --- per-model dedicated-schema relation hydrators ---
// --- per-model dedicated-schema relation hydrators (batched across the WHOLE parent array) ---
// Narrows a hydrator's target `findMany` to the caller's `select` (avoids fetching the wide
// TOASTed columns just to strip them in `applyProjection`); a bare/`include` projection stays a
// full-row fetch. `keepKeys` are the column(s) the hydrator's Map is keyed on.
function targetFindManyArgs(
where: unknown,
projection: { select?: any; include?: any } | undefined,
keepKeys: string[]
): { where: unknown; select?: Record<string, unknown> } {
if (projection?.select) {
const select: Record<string, unknown> = { ...projection.select };
for (const key of keepKeys) {
select[key] = true;
}
return { where, select };
}
return { where };
}
// Generic to-many relation reached via an explicit join model: one grouped query for the join rows
// spanning every parent id, then one grouped query for the distinct target rows, then an in-memory
// (DB-free) assembly per parent. `joinParentField`/`joinTargetField` name the join row's two FK
// columns; `targetDelegate` is the model the join points at.
async function batchHydrateJoinRelation(
join: RunOpsDelegate<"findMany"> | undefined,
targetDelegate: RunOpsDelegate<"findMany">,
parentIds: string[],
joinParentField: string,
joinTargetField: string,
projection: { select?: any; include?: any } | undefined
): Promise<Map<string, unknown[]>> {
const byParent = new Map<string, unknown[]>(parentIds.map((id) => [id, []]));
if (!join || parentIds.length === 0) {
return byParent;
}
const links = (await join.findMany({
where: { [joinParentField]: { in: parentIds } },
select: { [joinParentField]: true, [joinTargetField]: true },
})) as Record<string, string>[];
if (links.length === 0) {
return byParent;
}
const targetIds = [...new Set(links.map((l) => l[joinTargetField]))];
const rows = (await targetDelegate.findMany(
targetFindManyArgs({ id: { in: targetIds } }, projection, ["id"])
)) as Record<string, unknown>[];
const byTargetId = new Map(rows.map((r) => [r.id as string, r]));
for (const link of links) {
const target = byTargetId.get(link[joinTargetField]);
const bucket = byParent.get(link[joinParentField]);
if (target && bucket) {
bucket.push(applyProjection(target, projection));
}
}
return byParent;
}
// Waitpoint where completedByTaskRunId = run.id (the @unique scalar back-pointer); at most one.
const hydrateAssociatedWaitpoint: DedicatedRelationHydrator = async (
client,
parent,
parents,
projection
) => {
const wp = (await client.waitpoint.findFirst({
where: { completedByTaskRunId: parent.id as string },
})) as Record<string, unknown> | null;
return applyProjection(wp, projection);
const parentIds = parents.map((p) => p.id as string);
const byParent = new Map<string, unknown>(parentIds.map((id) => [id, null]));
if (parentIds.length === 0) {
return byParent;
}
const rows = (await client.waitpoint.findMany(
targetFindManyArgs({ completedByTaskRunId: { in: parentIds } }, projection, [
"completedByTaskRunId",
])
)) as Record<string, unknown>[];
for (const row of rows) {
const runId = row.completedByTaskRunId as string | undefined;
if (runId && byParent.has(runId)) {
byParent.set(runId, applyProjection(row, projection));
}
}
return byParent;
};
// Display connections for a run: WaitpointRunConnection → Waitpoint rows.
const hydrateConnectedWaitpoints: DedicatedRelationHydrator = async (
client,
parent,
projection
) => {
const join = client.waitpointRunConnection;
if (!join) {
return [];
}
const links = (await join.findMany({
where: { taskRunId: parent.id as string },
select: { waitpointId: true },
})) as { waitpointId: string }[];
if (links.length === 0) {
return [];
}
const rows = (await client.waitpoint.findMany({
where: { id: { in: links.map((l) => l.waitpointId) } },
})) as Record<string, unknown>[];
return rows.map((r) => applyProjection(r, projection));
};
const hydrateConnectedWaitpoints: DedicatedRelationHydrator = async (client, parents, projection) =>
batchHydrateJoinRelation(
client.waitpointRunConnection,
client.waitpoint,
parents.map((p) => p.id as string),
"taskRunId",
"waitpointId",
projection
);
// Completed waitpoints for a snapshot: CompletedWaitpoint join → Waitpoint rows.
const hydrateCompletedWaitpoints: DedicatedRelationHydrator = async (
client,
parent,
projection
) => {
const join = client.completedWaitpoint;
if (!join) {
return [];
}
const links = (await join.findMany({
where: { snapshotId: parent.id as string },
select: { waitpointId: true },
})) as { waitpointId: string }[];
if (links.length === 0) {
return [];
}
const rows = (await client.waitpoint.findMany({
where: { id: { in: links.map((l) => l.waitpointId) } },
})) as Record<string, unknown>[];
return rows.map((r) => applyProjection(r, projection));
};
const hydrateCompletedWaitpoints: DedicatedRelationHydrator = async (client, parents, projection) =>
batchHydrateJoinRelation(
client.completedWaitpoint,
client.waitpoint,
parents.map((p) => p.id as string),
"snapshotId",
"waitpointId",
projection
);
// Runs a waitpoint is blocking: TaskRunWaitpoint rows keyed by waitpointId. A nested `taskRun`
// select (the run-engine's getWaitpoint shape) is resolved from the scalar TaskRunWaitpoint.taskRunId.
const hydrateBlockingTaskRuns: DedicatedRelationHydrator = async (client, parent, projection) => {
const hydrateBlockingTaskRuns: DedicatedRelationHydrator = async (client, parents, projection) => {
const parentIds = parents.map((p) => p.id as string);
const byParent = new Map<string, unknown[]>(parentIds.map((id) => [id, []]));
if (parentIds.length === 0) {
return byParent;
}
const edges = (await client.taskRunWaitpoint.findMany({
where: { waitpointId: parent.id as string },
where: { waitpointId: { in: parentIds } },
})) as Record<string, unknown>[];
const nestedTaskRun = projection?.select?.taskRun;
if (!nestedTaskRun) {
return edges;
const runProjection = nestedTaskRun ? projectionOf(nestedTaskRun as SubProjection) : undefined;
let byRunId = new Map<string, Record<string, unknown>>();
if (nestedTaskRun) {
const runIds = [...new Set(edges.map((e) => e.taskRunId as string))];
const runs = (
runIds.length > 0
? await client.taskRun.findMany(
targetFindManyArgs({ id: { in: runIds } }, runProjection, ["id"])
)
: []
) as Record<string, unknown>[];
byRunId = new Map(runs.map((r) => [r.id as string, r]));
}
const runProjection = projectionOf(nestedTaskRun as SubProjection);
return Promise.all(
edges.map(async (edge) => {
const run = (await client.taskRun.findFirst({
where: { id: edge.taskRunId as string },
})) as Record<string, unknown> | null;
return { ...edge, taskRun: applyProjection(run, runProjection) };
})
);
for (const edge of edges) {
const bucket = byParent.get(edge.waitpointId as string);
if (!bucket) continue;
bucket.push(
nestedTaskRun
? {
...edge,
taskRun: applyProjection(byRunId.get(edge.taskRunId as string) ?? null, runProjection),
}
: edge
);
}
return byParent;
};
// Display connections for a waitpoint: WaitpointRunConnection → TaskRun rows.
const hydrateConnectedRuns: DedicatedRelationHydrator = async (client, parent, projection) => {
const join = client.waitpointRunConnection;
if (!join) {
return [];
// Display connections for a waitpoint: WaitpointRunConnection → TaskRun rows. Bounded per parent to
// CONNECTED_RUNS_LIMIT via a window function + existence-JOIN to TaskRun, mirroring the id-list
// helper findWaitpointConnectedRunIds: a dangling (run-less) connection row never occupies a LIMIT
// slot, and a heavily-fanned-in waitpoint never hydrates an unbounded connectedRuns list. This is
// the DISPLAY relation only — functional blocking reads (hydrateBlockingTaskRuns) stay uncapped.
const hydrateConnectedRuns: DedicatedRelationHydrator = async (client, parents, projection) => {
const parentIds = parents.map((p) => p.id as string);
const byParent = new Map<string, unknown[]>(parentIds.map((id) => [id, []]));
if (parentIds.length === 0) {
return byParent;
}
const links = (await join.findMany({
where: { waitpointId: parent.id as string },
select: { taskRunId: true },
})) as { taskRunId: string }[];
// One grouped query for the bounded edges across every parent: ROW_NUMBER partitioned per
// waitpoint keeps at most CONNECTED_RUNS_LIMIT rows per parent (uses @@index([waitpointId])).
const links = (await client.$queryRaw`
SELECT ranked."waitpointId" AS "waitpointId", ranked."taskRunId" AS "taskRunId"
FROM (
SELECT c."waitpointId", c."taskRunId",
ROW_NUMBER() OVER (PARTITION BY c."waitpointId" ORDER BY c."id") AS rn
FROM "WaitpointRunConnection" c
JOIN "TaskRun" t ON t."id" = c."taskRunId"
WHERE c."waitpointId" = ANY(${parentIds}::text[])
) ranked
WHERE ranked.rn <= ${CONNECTED_RUNS_LIMIT}
`) as { waitpointId: string; taskRunId: string }[];
if (links.length === 0) {
return [];
return byParent;
}
const rows = (await client.taskRun.findMany({
where: { id: { in: links.map((l) => l.taskRunId) } },
})) as Record<string, unknown>[];
return rows.map((r) => applyProjection(r, projection));
const targetIds = [...new Set(links.map((l) => l.taskRunId))];
const rows = (await client.taskRun.findMany(
targetFindManyArgs({ id: { in: targetIds } }, projection, ["id"])
)) as Record<string, unknown>[];
const byTargetId = new Map(rows.map((r) => [r.id as string, r]));
for (const link of links) {
const target = byTargetId.get(link.taskRunId);
const bucket = byParent.get(link.waitpointId);
if (target && bucket) {
bucket.push(applyProjection(target, projection));
}
}
return byParent;
};
// Snapshots that completed a waitpoint: CompletedWaitpoint join → TaskRunExecutionSnapshot rows.
const hydrateCompletedExecutionSnapshots: DedicatedRelationHydrator = async (
client,
parent,
parents,
projection
) => {
const join = client.completedWaitpoint;
if (!join) {
return [];
}
const links = (await join.findMany({
where: { waitpointId: parent.id as string },
select: { snapshotId: true },
})) as { snapshotId: string }[];
if (links.length === 0) {
return [];
}
const rows = (await client.taskRunExecutionSnapshot.findMany({
where: { id: { in: links.map((l) => l.snapshotId) } },
})) as Record<string, unknown>[];
return rows.map((r) => applyProjection(r, projection));
};
) =>
batchHydrateJoinRelation(
client.completedWaitpoint,
client.taskRunExecutionSnapshot,
parents.map((p) => p.id as string),
"waitpointId",
"snapshotId",
projection
);
// The waitpoint a block edge points at, resolved from the edge's scalar `waitpointId`. The edge's
// own client only finds a co-resident token; the router re-resolves cross-DB.
const hydrateEdgeWaitpoint: DedicatedRelationHydrator = async (client, parent, projection) => {
const waitpointId = parent.waitpointId as string | undefined;
if (!waitpointId) {
return null;
}
const wp = (await client.waitpoint.findFirst({
where: { id: waitpointId },
})) as Record<string, unknown> | null;
return applyProjection(wp, projection);
};
// The waitpoint each block edge points at, resolved from its scalar `waitpointId`. The edge's own
// client only finds a co-resident token; the router re-resolves cross-DB.
const hydrateEdgeWaitpoint: DedicatedRelationHydrator = async (client, parents, projection) =>
batchHydrateEdgeTarget(client.waitpoint, parents, "waitpointId", projection);
// The run a block edge belongs to, resolved from the edge's scalar `taskRunId`.
const hydrateEdgeTaskRun: DedicatedRelationHydrator = async (client, parent, projection) => {
const taskRunId = parent.taskRunId as string | undefined;
if (!taskRunId) {
return null;
// The run each block edge belongs to, resolved from its scalar `taskRunId`.
const hydrateEdgeTaskRun: DedicatedRelationHydrator = async (client, parents, projection) =>
batchHydrateEdgeTarget(client.taskRun, parents, "taskRunId", projection);
// Generic to-one relation reached via a scalar FK ON the parent itself (not a join model): one
// grouped target query for every distinct FK value across the batch.
async function batchHydrateEdgeTarget(
targetDelegate: RunOpsDelegate<"findMany">,
parents: Record<string, unknown>[],
fkField: string,
projection: { select?: any; include?: any } | undefined
): Promise<Map<string, unknown>> {
const byParent = new Map<string, unknown>();
const targetIds: string[] = [];
for (const p of parents) {
byParent.set(p.id as string, null);
const fk = p[fkField] as string | undefined;
if (fk) targetIds.push(fk);
}
const run = (await client.taskRun.findFirst({
where: { id: taskRunId },
})) as Record<string, unknown> | null;
return applyProjection(run, projection);
};
if (targetIds.length === 0) {
return byParent;
}
const rows = (await targetDelegate.findMany(
targetFindManyArgs({ id: { in: [...new Set(targetIds)] } }, projection, ["id"])
)) as Record<string, unknown>[];
const byTargetId = new Map(rows.map((r) => [r.id as string, r]));
for (const p of parents) {
const fk = p[fkField] as string | undefined;
if (fk) {
byParent.set(p.id as string, applyProjection(byTargetId.get(fk) ?? null, projection));
}
}
return byParent;
}
const TASK_RUN_DEDICATED: DedicatedRelationSpec = {
associatedWaitpoint: hydrateAssociatedWaitpoint,
@@ -1444,17 +1554,74 @@ export class PostgresRunStore implements RunStore {
cursor,
...stripped,
})) as Record<string, unknown>[];
for (const row of rows) {
await this.#hydrateDedicatedRelations(
prisma as RunOpsCapableClient,
row,
requested,
TASK_RUN_DEDICATED
);
}
await this.#hydrateDedicatedRelations(
prisma as RunOpsCapableClient,
rows,
requested,
TASK_RUN_DEDICATED
);
return rows;
}
// Grouped replacement for `Promise.all(ids.map(id => findRun(id)))`: a thin wrapper over
// `findRuns` (so it inherits dedicated-relation hydration + read-your-writes client routing),
// bounding the whole id batch into one round trip instead of one per id.
findRunsByIds<S extends Prisma.TaskRunSelect>(
ids: string[],
args: { select: S },
client?: ReadClient
): Promise<Map<string, Prisma.TaskRunGetPayload<{ select: S }>>>;
findRunsByIds<I extends Prisma.TaskRunInclude>(
ids: string[],
args: { include: I },
client?: ReadClient
): Promise<Map<string, Prisma.TaskRunGetPayload<{ include: I }>>>;
findRunsByIds(ids: string[], client?: ReadClient): Promise<Map<string, TaskRun>>;
async findRunsByIds(
ids: string[],
argsOrClient?: { select?: Prisma.TaskRunSelect; include?: Prisma.TaskRunInclude } | ReadClient,
_client?: ReadClient
): Promise<Map<string, unknown>> {
if (ids.length === 0) {
return new Map();
}
const hasSelectOrInclude =
argsOrClient != null &&
typeof argsOrClient === "object" &&
("select" in argsOrClient || "include" in argsOrClient);
const args = hasSelectOrInclude
? (argsOrClient as { select?: Prisma.TaskRunSelect; include?: Prisma.TaskRunInclude })
: undefined;
// Slot recovery mirrors `findRuns`'s overloads: when `argsOrClient` isn't a
// `{ select | include }` object it may itself BE the client (2-arg call) or be undefined
// with the client in the 3rd slot (an explicit `(ids, undefined, client)` call).
const client =
args === undefined ? ((argsOrClient as ReadClient | undefined) ?? _client) : _client;
// Force `id` into the projection so the map can key off it, even when the caller's select
// omits it — `findRuns` would otherwise strip it back out as an added-for-merge-only field.
const projected = args?.select
? { select: { ...args.select, id: true } }
: args?.include
? { include: args.include }
: {};
const rows = (await this.findRuns(
{ where: { id: { in: ids } }, ...projected } as Parameters<PostgresRunStore["findRuns"]>[0],
client
)) as Record<string, unknown>[];
const byId = new Map<string, unknown>();
// Strip the id we force-injected for map keying when the caller's select did not ask for it,
// so returned values match the declared payload type and never leak an unrequested id.
const stripInjectedId = args?.select != null && !("id" in args.select);
for (const row of rows) {
const key = row.id as string;
if (stripInjectedId) {
delete row.id;
}
byId.set(key, row);
}
return byId;
}
// --- run-ops persistence ---
async findLatestExecutionSnapshot(
@@ -1563,14 +1730,12 @@ export class PostgresRunStore implements RunStore {
cursor,
...stripped,
})) as Record<string, unknown>[];
for (const row of rows) {
await this.#hydrateDedicatedRelations(
prisma as RunOpsCapableClient,
row,
requested,
SNAPSHOT_DEDICATED
);
}
await this.#hydrateDedicatedRelations(
prisma as RunOpsCapableClient,
rows,
requested,
SNAPSHOT_DEDICATED
);
return rows as Prisma.TaskRunExecutionSnapshotGetPayload<T>[];
}
@@ -1710,23 +1875,34 @@ export class PostgresRunStore implements RunStore {
// Reverse of `connectedRuns`: the run ids linked to a waitpoint. Co-resident with the RUN (the join
// is written on the run's DB in blockRunWithWaitpointEdges), so the waitpoint's own store can MISS a
// cross-DB run — the router fans this across BOTH DBs.
// Bounded to CONNECTED_RUNS_LIMIT via an existence-JOIN to TaskRun, mirroring the webapp's
// `#connectedRunIdsOn`: a dangling connection row (dedicated schema: FK-free `taskRunId`) can
// never occupy a LIMIT slot ahead of a real run, and a heavily-fanned-in waitpoint can never emit
// an unbounded id list.
async findWaitpointConnectedRunIds(waitpointId: string, client?: ReadClient): Promise<string[]> {
const prisma = client ?? this.readOnlyPrisma;
const joinDelegate = (prisma as RunOpsCapableClient).waitpointRunConnection;
if (this.schemaVariant === "dedicated" && joinDelegate) {
const links = (await joinDelegate.findMany({
where: { waitpointId },
select: { taskRunId: true },
})) as { taskRunId: string }[];
return links.map((l) => l.taskRunId);
const rows = await prisma.$queryRaw<{ taskRunId: string }[]>`
SELECT c."taskRunId" AS "taskRunId"
FROM "WaitpointRunConnection" c
JOIN "TaskRun" t ON t."id" = c."taskRunId"
WHERE c."waitpointId" = ${waitpointId}
LIMIT ${CONNECTED_RUNS_LIMIT}
`;
return rows.map((row) => row.taskRunId);
}
// Legacy implicit M2M `_WaitpointRunConnections`: A = TaskRun.id, B = Waitpoint.id (alphabetical).
const result = await prisma.$queryRaw<{ A: string }[]>`
SELECT "A" FROM "_WaitpointRunConnections" WHERE "B" = ${waitpointId}
const rows = await prisma.$queryRaw<{ A: string }[]>`
SELECT c."A" AS "A"
FROM "_WaitpointRunConnections" c
JOIN "TaskRun" t ON t."id" = c."A"
WHERE c."B" = ${waitpointId}
LIMIT ${CONNECTED_RUNS_LIMIT}
`;
return result.map((r) => r.A);
return rows.map((row) => row.A);
}
// Reverse of `completedExecutionSnapshots`: the snapshot ids that completed a waitpoint. The join is
@@ -1949,14 +2125,12 @@ export class PostgresRunStore implements RunStore {
cursor,
...stripped,
})) as Record<string, unknown>[];
for (const row of rows) {
await this.#hydrateDedicatedRelations(
prisma as RunOpsCapableClient,
row,
requested,
WAITPOINT_DEDICATED
);
}
await this.#hydrateDedicatedRelations(
prisma as RunOpsCapableClient,
rows,
requested,
WAITPOINT_DEDICATED
);
return rows as Prisma.WaitpointGetPayload<T>[];
}
@@ -2019,14 +2193,12 @@ export class PostgresRunStore implements RunStore {
cursor,
...stripped,
})) as Record<string, unknown>[];
for (const row of rows) {
await this.#hydrateDedicatedRelations(
prisma as RunOpsCapableClient,
row,
requested,
TASK_RUN_WAITPOINT_DEDICATED
);
}
await this.#hydrateDedicatedRelations(
prisma as RunOpsCapableClient,
rows,
requested,
TASK_RUN_WAITPOINT_DEDICATED
);
return rows as Prisma.TaskRunWaitpointGetPayload<T>[];
}
@@ -2267,14 +2439,15 @@ export class PostgresRunStore implements RunStore {
if (!row) {
return row;
}
await this.#hydrateDedicatedRelations(client, row, requested, spec);
await this.#hydrateDedicatedRelations(client, [row], requested, spec);
return row;
}
// Hydrate each requested dedicated-schema relation key onto `row` in place, honoring the caller's sub-select.
// Hydrate each requested dedicated-schema relation key onto EVERY row in `rows` in ONE grouped
// pass per key (never one query per row), honoring the caller's sub-select.
async #hydrateDedicatedRelations(
client: RunOpsCapableClient,
row: Record<string, unknown>,
rows: Record<string, unknown>[],
requested: Record<string, SubProjection>,
spec: DedicatedRelationSpec
): Promise<void> {
@@ -2284,7 +2457,10 @@ export class PostgresRunStore implements RunStore {
continue;
}
const subArgs = requested[key];
row[key] = await hydrator(client, row, projectionOf(subArgs), this);
const byParentId = await hydrator(client, rows, projectionOf(subArgs), this);
for (const row of rows) {
row[key] = byParentId.get(row.id as string);
}
}
}
@@ -0,0 +1,392 @@
// RED→GREEN for the grouped-read primitive `RoutingRunStore.findRunsByIds`.
//
// Today, callers that need N runs by id do `Promise.all(ids.map(id => readThroughRun(id)))`, which
// fans out to `findRun` PER ID — each one either a direct `taskRun.findFirst` on the owning store, or
// (for an unclassifiable id) a NEW-then-LEGACY probe. For N ids that is O(N) round trips.
//
// `findRunsByIds` is the grouped replacement: it reuses the router's existing bounded id-set
// machinery (`#findRunsByIdSet` via `findRuns`), which queries NEW for the WHOLE id set in one
// `findMany`, then probes LEGACY in one more `findMany` only for the ids NEW missed. So for any N
// (spread across both stores), the call count is CONSTANT (one grouped `findMany` per store queried),
// never N `findFirst` calls.
//
// `heteroRunOpsPostgresTest` gives the REAL split topology: prisma17 = dedicated subset schema
// (#new), prisma14 = full legacy schema on a SEPARATE physical PG container (#legacy). The counting
// proxy wraps each REAL client's `taskRun` delegate to tally `findFirst`/`findMany` calls while
// delegating every call to the real client — the DB still runs the query; this is instrumentation,
// not a mock.
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
import type { PrismaClient } from "@trigger.dev/database";
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
import { describe, expect } from "vitest";
import { PostgresRunStore } from "./PostgresRunStore.js";
import { RoutingRunStore } from "./runOpsStore.js";
type AnyClient = PrismaClient | RunOpsPrismaClient;
// ownerEngine classifies a 26-char body with version "1" at index 25 (and a valid base32hex/region
// char elsewhere) as NEW; anything else (including any 25-char id, regardless of content) as LEGACY.
// The distinguishing prefix varies by `n` so multiple ids in one test stay unique.
const CUID_25 = (n: number) => `c${String(n).padStart(2, "0")}`.padEnd(25, "c");
const NEW_ID_26 = (n: number) => `${String(n).padStart(2, "0")}${"k".repeat(22)}01`;
type CallCounts = { findMany: number; findFirst: number };
// Wrap a REAL client's `taskRun` delegate to tally `findFirst`/`findMany` invocations, delegating
// every call unchanged to the real client (the DB still runs the query — pure instrumentation).
function countingClient<C extends AnyClient>(real: C): { client: C; counts: CallCounts } {
const counts: CallCounts = { findMany: 0, findFirst: 0 };
const countingTaskRun = new Proxy((real as any).taskRun, {
get(target, prop) {
if (prop === "findMany" || prop === "findFirst") {
counts[prop as "findMany" | "findFirst"]++;
}
return (target as any)[prop];
},
});
const client = new Proxy(real, {
get(target, prop) {
if (prop === "taskRun") {
return countingTaskRun;
}
return (target as any)[prop];
},
}) as C;
return { client, counts };
}
async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) {
const organization = await prisma.organization.create({
data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
});
const project = await prisma.project.create({
data: {
name: `Project ${suffix}`,
slug: `project-${suffix}`,
externalRef: `proj_${suffix}`,
organizationId: organization.id,
},
});
const environment = await prisma.runtimeEnvironment.create({
data: {
type: "DEVELOPMENT",
slug: "dev",
projectId: project.id,
organizationId: organization.id,
apiKey: `tr_dev_${suffix}`,
pkApiKey: `pk_dev_${suffix}`,
shortcode: `short_${suffix}`,
},
});
return { organization, project, environment };
}
function seedEnvironmentDedicated(suffix: string) {
return {
organization: { id: `org_${suffix}` },
project: { id: `proj_${suffix}` },
environment: { id: `env_${suffix}` },
};
}
function taskRunData(opts: {
id: string;
friendlyId: string;
organizationId: string;
projectId: string;
runtimeEnvironmentId: string;
}) {
return {
id: opts.id,
engine: "V2" as const,
status: "PENDING" as const,
friendlyId: opts.friendlyId,
runtimeEnvironmentId: opts.runtimeEnvironmentId,
environmentType: "DEVELOPMENT" as const,
organizationId: opts.organizationId,
projectId: opts.projectId,
taskIdentifier: "my-task",
payload: "{}",
payloadType: "application/json",
traceContext: {},
traceId: `trace_${opts.id}`,
spanId: `span_${opts.id}`,
queue: "task/my-task",
isTest: false,
taskEventStore: "taskEvent",
depth: 0,
};
}
describe("RoutingRunStore.findRunsByIds — grouped, residency-partitioned read of a run id set", () => {
heteroRunOpsPostgresTest(
"returns all N rows with a CONSTANT number of grouped findMany calls (never per-id findFirst)",
async ({ prisma14, prisma17 }) => {
const legacyCounting = countingClient(prisma14);
const newCounting = countingClient(prisma17);
const legacyStore = new PostgresRunStore({
prisma: legacyCounting.client,
readOnlyPrisma: legacyCounting.client,
schemaVariant: "legacy",
});
const newStore = new PostgresRunStore({
prisma: newCounting.client as never,
readOnlyPrisma: newCounting.client as never,
schemaVariant: "dedicated",
});
const router = new RoutingRunStore({ new: newStore, legacy: legacyStore });
// Seed 2 LEGACY-resident (cuid) runs + 2 NEW-resident (run-ops id) runs — ids spread across
// both stores.
const legEnv = await seedEnvironmentLegacy(prisma14, "grp_leg");
const newEnv = seedEnvironmentDedicated("grp_new");
const legacyIds = [`run_${CUID_25(1)}`, `run_${CUID_25(2)}`];
const newIds = [`run_${NEW_ID_26(1)}`, `run_${NEW_ID_26(2)}`];
for (const [i, id] of legacyIds.entries()) {
await prisma14.taskRun.create({
data: taskRunData({
id,
friendlyId: `run_grp_leg_${i}`,
organizationId: legEnv.organization.id,
projectId: legEnv.project.id,
runtimeEnvironmentId: legEnv.environment.id,
}),
});
}
for (const [i, id] of newIds.entries()) {
await prisma17.taskRun.create({
data: taskRunData({
id,
friendlyId: `run_grp_new_${i}`,
organizationId: newEnv.organization.id,
projectId: newEnv.project.id,
runtimeEnvironmentId: newEnv.environment.id,
}),
});
}
const allIds = [...legacyIds, ...newIds];
// Reset counters after seeding (seeding uses `prisma.taskRun.create`, not find*).
legacyCounting.counts.findMany = 0;
legacyCounting.counts.findFirst = 0;
newCounting.counts.findMany = 0;
newCounting.counts.findFirst = 0;
const result = await router.findRunsByIds(allIds, {
select: { friendlyId: true },
});
// All N rows returned, keyed by id.
expect(result.size).toBe(4);
for (const id of allIds) {
expect(result.has(id)).toBe(true);
}
expect(result.get(legacyIds[0])?.friendlyId).toBe("run_grp_leg_0");
expect(result.get(newIds[0])?.friendlyId).toBe("run_grp_new_0");
// The select omitted `id`; the id we force-inject for keying must not leak into the value.
expect("id" in (result.get(legacyIds[0]) as object)).toBe(false);
expect("id" in (result.get(newIds[0]) as object)).toBe(false);
// GROUPED: exactly one findMany call per store queried (NEW always queried for the whole
// set; LEGACY queried once more for the misses) — never N per-id findFirst calls.
expect(newCounting.counts.findMany).toBe(1);
expect(legacyCounting.counts.findMany).toBe(1);
expect(newCounting.counts.findFirst).toBe(0);
expect(legacyCounting.counts.findFirst).toBe(0);
}
);
});
describe("RoutingRunStore.findManyTaskRunWaitpoints — grouped taskRun hydration", () => {
heteroRunOpsPostgresTest(
"hydrates N edges' `taskRun` with a CONSTANT number of grouped findMany calls (never per-edge findFirst)",
async ({ prisma14, prisma17 }) => {
const legacyCounting = countingClient(prisma14);
const newCounting = countingClient(prisma17);
const legacyStore = new PostgresRunStore({
prisma: legacyCounting.client,
readOnlyPrisma: legacyCounting.client,
schemaVariant: "legacy",
});
const newStore = new PostgresRunStore({
prisma: newCounting.client as never,
readOnlyPrisma: newCounting.client as never,
schemaVariant: "dedicated",
});
const router = new RoutingRunStore({ new: newStore, legacy: legacyStore });
const legEnv = await seedEnvironmentLegacy(prisma14, "edge_leg");
const newEnv = seedEnvironmentDedicated("edge_new");
const legacyRunIds = [`run_${CUID_25(11)}`, `run_${CUID_25(12)}`];
const newRunIds = [`run_${NEW_ID_26(11)}`, `run_${NEW_ID_26(12)}`];
for (const [i, id] of legacyRunIds.entries()) {
await prisma14.taskRun.create({
data: taskRunData({
id,
friendlyId: `run_edge_leg_${i}`,
organizationId: legEnv.organization.id,
projectId: legEnv.project.id,
runtimeEnvironmentId: legEnv.environment.id,
}),
});
}
for (const [i, id] of newRunIds.entries()) {
await prisma17.taskRun.create({
data: taskRunData({
id,
friendlyId: `run_edge_new_${i}`,
organizationId: newEnv.organization.id,
projectId: newEnv.project.id,
runtimeEnvironmentId: newEnv.environment.id,
}),
});
}
// Block edges are FK-free on the dedicated (#new) subset schema, so they can point at a run on
// EITHER DB — create one edge per run, all on #new.
const allRunIds = [...legacyRunIds, ...newRunIds];
const edgeIds: string[] = [];
for (const runId of allRunIds) {
const edge = await prisma17.taskRunWaitpoint.create({
data: {
taskRunId: runId,
waitpointId: `wp_${runId}`,
projectId: newEnv.project.id,
},
});
edgeIds.push(edge.id);
}
legacyCounting.counts.findMany = 0;
legacyCounting.counts.findFirst = 0;
newCounting.counts.findMany = 0;
newCounting.counts.findFirst = 0;
const rows = await router.findManyTaskRunWaitpoints({
where: { id: { in: edgeIds } },
select: {
id: true,
taskRunId: true,
taskRun: { select: { id: true, friendlyId: true } },
},
});
expect(rows).toHaveLength(4);
const byRunId = new Map(rows.map((r) => [r.taskRunId, r.taskRun]));
expect((byRunId.get(legacyRunIds[0]) as any)?.friendlyId).toBe("run_edge_leg_0");
expect((byRunId.get(newRunIds[0]) as any)?.friendlyId).toBe("run_edge_new_0");
// GROUPED: one findMany per store queried for the WHOLE edge set, never one findFirst per edge.
expect(newCounting.counts.findMany).toBe(1);
expect(legacyCounting.counts.findMany).toBe(1);
expect(newCounting.counts.findFirst).toBe(0);
expect(legacyCounting.counts.findFirst).toBe(0);
}
);
});
describe("RoutingRunStore.findWaitpoint connectedRuns — grouped, order-preserving hydration", () => {
heteroRunOpsPostgresTest(
"hydrates N connectedRuns with a CONSTANT number of grouped findMany calls, in the join's own order",
async ({ prisma14, prisma17 }) => {
const legacyCounting = countingClient(prisma14);
const newCounting = countingClient(prisma17);
const legacyStore = new PostgresRunStore({
prisma: legacyCounting.client,
readOnlyPrisma: legacyCounting.client,
schemaVariant: "legacy",
});
const newStore = new PostgresRunStore({
prisma: newCounting.client as never,
readOnlyPrisma: newCounting.client as never,
schemaVariant: "dedicated",
});
const router = new RoutingRunStore({ new: newStore, legacy: legacyStore });
const legEnv = await seedEnvironmentLegacy(prisma14, "conn_leg");
const newEnv = seedEnvironmentDedicated("conn_new");
const waitpointId = `waitpoint_${CUID_25(20)}`;
await prisma14.waitpoint.create({
data: {
id: waitpointId,
friendlyId: "wp_conn",
type: "MANUAL",
status: "PENDING",
idempotencyKey: `idem_${waitpointId}`,
userProvidedIdempotencyKey: false,
projectId: legEnv.project.id,
environmentId: legEnv.environment.id,
},
});
const legacyRunIds = [`run_${CUID_25(21)}`, `run_${CUID_25(22)}`];
const newRunIds = [`run_${NEW_ID_26(21)}`, `run_${NEW_ID_26(22)}`];
for (const [i, id] of legacyRunIds.entries()) {
await prisma14.taskRun.create({
data: taskRunData({
id,
friendlyId: `run_conn_leg_${i}`,
organizationId: legEnv.organization.id,
projectId: legEnv.project.id,
runtimeEnvironmentId: legEnv.environment.id,
}),
});
await router.blockRunWithWaitpointEdges({
runId: id,
waitpointIds: [waitpointId],
projectId: legEnv.project.id,
});
}
for (const [i, id] of newRunIds.entries()) {
await prisma17.taskRun.create({
data: taskRunData({
id,
friendlyId: `run_conn_new_${i}`,
organizationId: newEnv.organization.id,
projectId: newEnv.project.id,
runtimeEnvironmentId: newEnv.environment.id,
}),
});
await router.blockRunWithWaitpointEdges({
runId: id,
waitpointIds: [waitpointId],
projectId: newEnv.project.id,
});
}
// The order the connection join itself returns, independent of any assumption about Postgres's
// internal row order — the fix must preserve exactly THIS order, not resort by residency/leg.
const expectedOrder = await router.findWaitpointConnectedRunIds(waitpointId);
expect(expectedOrder).toHaveLength(4);
legacyCounting.counts.findMany = 0;
legacyCounting.counts.findFirst = 0;
newCounting.counts.findMany = 0;
newCounting.counts.findFirst = 0;
const waitpoint = (await router.findWaitpoint({
where: { id: waitpointId },
include: { connectedRuns: { select: { id: true, friendlyId: true } } },
})) as { connectedRuns: { id: string; friendlyId: string }[] } | null;
const connected = waitpoint?.connectedRuns ?? [];
expect(connected).toHaveLength(4);
expect(connected.map((r) => r.id)).toEqual(expectedOrder);
// GROUPED: one findMany per store for the WHOLE connected-run set, never one findFirst per run.
expect(newCounting.counts.findMany).toBe(1);
expect(legacyCounting.counts.findMany).toBe(1);
expect(newCounting.counts.findFirst).toBe(0);
expect(legacyCounting.counts.findFirst).toBe(0);
}
);
});
@@ -0,0 +1,296 @@
// RED→GREEN lock for RoutingRunStore.findRun's ON-MISS FAN-OUT for a CLASSIFIABLE id.
//
// THE BUG: findRun for a classifiable id routed to the SINGLE owning store (by id-shape
// classification) and returned whatever it gave — no on-miss fallback. When a run's PHYSICAL
// residency does not match its id-shape classification (e.g. a pre-#4154 27-char base62 run that
// lives on the NEW store but now classifies LEGACY), findRun routed to the wrong store, missed,
// and returned null → a spurious 404 — even though the run is physically present on the OTHER DB
// (and runs.list surfaces it). The unclassifiable path already fanned out NEW→LEGACY; this makes
// the classifiable path equally robust.
//
// Uses the REAL two-physical-DB split (heteroRunOpsPostgresTest: prisma14 = full/legacy on PG14,
// prisma17 = dedicated run-ops subset on PG17). NEVER mocked. The residency/classification MISMATCH
// is simulated deterministically by injecting a custom `classify` fn into the RoutingRunStore
// constructor — the physical row is written to the NEW store while `classify` reports its id LEGACY.
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
import { ownerEngine, type Residency } from "@trigger.dev/core/v3/isomorphic";
import type { PrismaClient } from "@trigger.dev/database";
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
import { describe, expect } from "vitest";
import { PostgresRunStore } from "./PostgresRunStore.js";
import { RoutingRunStore } from "./runOpsStore.js";
import type { CreateRunInput, RunStore } from "./types.js";
// ownerEngine classifies by internal-id LENGTH/version char: 25 chars → cuid → LEGACY,
// a v1 body (version "1" at index 25) → run-ops id → NEW.
function cuidLegacy(seed: string): string {
return (seed + "c".repeat(25)).slice(0, 25);
}
function runOpsNew(seed: string): string {
return (seed.replace(/[^0-9a-v]/g, "0") + "k".repeat(24)).slice(0, 24) + "01";
}
function makeDedicatedStore(prisma17: RunOpsPrismaClient) {
return new PostgresRunStore({
prisma: prisma17 as never,
readOnlyPrisma: prisma17 as never,
schemaVariant: "dedicated",
});
}
function makeLegacyStore(prisma14: PrismaClient) {
return new PostgresRunStore({
prisma: prisma14,
readOnlyPrisma: prisma14,
schemaVariant: "legacy",
});
}
// Wrap a real store so findRun/findRunOnPrimary calls are COUNTED while every method still delegates
// to the REAL PostgresRunStore (this is instrumentation, not a behavior mock — the underlying reads,
// writes, getters all run for real). Lets us assert the FAST PATH does not touch the other store.
function countingReads(
inner: RunStore,
counts: { findRun: number; findRunOnPrimary: number }
): RunStore {
return new Proxy(inner, {
get(target, prop) {
// Read via target[prop] so getters (e.g. primaryReadClient) run with `this` = the real store.
const value = (target as unknown as Record<string | symbol, unknown>)[prop];
if (typeof value !== "function") return value;
if (prop === "findRun" || prop === "findRunOnPrimary") {
return (...args: unknown[]) => {
counts[prop as "findRun" | "findRunOnPrimary"] += 1;
return (value as (...a: unknown[]) => unknown).apply(target, args);
};
}
return (value as (...a: unknown[]) => unknown).bind(target);
},
}) as unknown as RunStore;
}
async function seedLegacyEnvironment(prisma14: PrismaClient, suffix: string) {
const organization = await prisma14.organization.create({
data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
});
const project = await prisma14.project.create({
data: {
name: `Project ${suffix}`,
slug: `project-${suffix}`,
externalRef: `proj_${suffix}`,
organizationId: organization.id,
},
});
const environment = await prisma14.runtimeEnvironment.create({
data: {
type: "DEVELOPMENT",
slug: "dev",
projectId: project.id,
organizationId: organization.id,
apiKey: `tr_dev_${suffix}`,
pkApiKey: `pk_dev_${suffix}`,
shortcode: `short_${suffix}`,
},
});
return {
organizationId: organization.id,
projectId: project.id,
runtimeEnvironmentId: environment.id,
environmentId: environment.id,
};
}
function buildCreateRunInput(params: {
runId: string;
friendlyId: string;
organizationId: string;
projectId: string;
runtimeEnvironmentId: string;
}): CreateRunInput {
return {
data: {
id: params.runId,
engine: "V2",
status: "PENDING",
friendlyId: params.friendlyId,
runtimeEnvironmentId: params.runtimeEnvironmentId,
environmentType: "DEVELOPMENT",
organizationId: params.organizationId,
projectId: params.projectId,
taskIdentifier: "my-task",
payload: "{}",
payloadType: "application/json",
traceContext: {},
traceId: `trace_${params.runId}`,
spanId: `span_${params.runId}`,
queue: "task/my-task",
isTest: false,
taskEventStore: "taskEvent",
depth: 0,
},
snapshot: {
engine: "V2",
executionStatus: "RUN_CREATED",
description: "Run was created",
runStatus: "PENDING",
environmentId: params.runtimeEnvironmentId,
environmentType: "DEVELOPMENT",
projectId: params.projectId,
organizationId: params.organizationId,
},
};
}
// Insert a TaskRun row DIRECTLY onto the NEW (dedicated) store, bypassing routing, so we can force a
// residency/classification MISMATCH: the row is physically on #new while `classify` calls its id LEGACY.
async function insertRunOnNewStore(
prisma17: RunOpsPrismaClient,
params: {
runId: string;
friendlyId: string;
environmentId: string;
organizationId: string;
projectId: string;
}
) {
await prisma17.taskRun.create({
data: {
id: params.runId,
engine: "V2",
status: "PENDING",
friendlyId: params.friendlyId,
runtimeEnvironmentId: params.environmentId,
environmentType: "DEVELOPMENT",
organizationId: params.organizationId,
projectId: params.projectId,
taskIdentifier: "my-task",
payload: "{}",
payloadType: "application/json",
traceContext: {},
traceId: `trace_${params.runId}`,
spanId: `span_${params.runId}`,
queue: "task/my-task",
isTest: false,
taskEventStore: "taskEvent",
depth: 0,
},
});
}
describe("RoutingRunStore.findRun — on-miss fan-out for a classifiable id (residency ≠ classification)", () => {
// ── THE BUG: a run physically on #new whose id classifies LEGACY must still be found ──
// Without the on-miss fallback, findRun routes to #legacy (per classification), misses, returns null.
heteroRunOpsPostgresTest(
"returns a #new-resident run whose id classifies LEGACY (owning-store miss → other-store fallback)",
async ({ prisma14, prisma17 }) => {
const env = await seedLegacyEnvironment(prisma14, "mm1");
const newStore = makeDedicatedStore(prisma17);
const legacyStore = makeLegacyStore(prisma14);
// A run-ops-shaped id (so ownerEngine would say NEW), but we FORCE classify → LEGACY to model
// a residency/classification mismatch: physically on #new, classified LEGACY.
const mismatchId = runOpsNew("mm1");
const classify = (id: string): Residency => (id === mismatchId ? "LEGACY" : ownerEngine(id));
const router = new RoutingRunStore({ new: newStore, legacy: legacyStore, classify });
await insertRunOnNewStore(prisma17, {
runId: mismatchId,
friendlyId: "run_mm1",
environmentId: env.environmentId,
organizationId: env.organizationId,
projectId: env.projectId,
});
// Physical residency sanity: on #new only.
expect(await prisma17.taskRun.findUnique({ where: { id: mismatchId } })).not.toBeNull();
expect(await prisma14.taskRun.findUnique({ where: { id: mismatchId } })).toBeNull();
// classify → LEGACY routes to #legacy (miss); the fix falls back to #new and finds the run.
const byId = (await router.findRun({ id: mismatchId }, { select: { id: true } })) as Record<
string,
unknown
> | null;
expect(byId?.id).toBe(mismatchId);
// Same on the read-your-writes primary variant (a caller-passed writer → findRunOnPrimary).
const byIdPrimary = (await router.findRun(
{ id: mismatchId },
{ select: { id: true } },
prisma14
)) as Record<string, unknown> | null;
expect(byIdPrimary?.id).toBe(mismatchId);
}
);
// ── FAST PATH: a run found in its CLASSIFIED store is a SINGLE read (no second-store probe) ──
heteroRunOpsPostgresTest(
"does NOT read the other store when the classified (owning) store hits",
async ({ prisma14, prisma17 }) => {
const env = await seedLegacyEnvironment(prisma14, "mm2");
// NEW-resident run-ops-id run: owning store = #new. Wrap #legacy to catch any stray probe.
const newCounts = { findRun: 0, findRunOnPrimary: 0 };
const legacyCounts = { findRun: 0, findRunOnPrimary: 0 };
const newStore = countingReads(makeDedicatedStore(prisma17), newCounts);
const legacyStore = countingReads(makeLegacyStore(prisma14), legacyCounts);
const router = new RoutingRunStore({ new: newStore, legacy: legacyStore });
const newId = runOpsNew("mm2n"); // classifies NEW
await router.createRun(
buildCreateRunInput({
runId: newId,
friendlyId: "run_mm2_new",
organizationId: env.organizationId,
projectId: env.projectId,
runtimeEnvironmentId: env.runtimeEnvironmentId,
})
);
const hit = (await router.findRun({ id: newId }, { select: { id: true } })) as Record<
string,
unknown
> | null;
expect(hit?.id).toBe(newId);
// Owning store read exactly once; the other store NOT touched (fast path preserved).
expect(newCounts.findRun).toBe(1);
expect(legacyCounts.findRun).toBe(0);
expect(legacyCounts.findRunOnPrimary).toBe(0);
// Symmetric: a cuid run whose owning store is #legacy must not probe #new on a hit.
const legacyId = cuidLegacy("mm2l"); // classifies LEGACY
await router.createRun(
buildCreateRunInput({
runId: legacyId,
friendlyId: "run_mm2_legacy",
organizationId: env.organizationId,
projectId: env.projectId,
runtimeEnvironmentId: env.runtimeEnvironmentId,
})
);
newCounts.findRun = 0;
legacyCounts.findRun = 0;
const hitLegacy = (await router.findRun(
{ id: legacyId },
{ select: { id: true } }
)) as Record<string, unknown> | null;
expect(hitLegacy?.id).toBe(legacyId);
expect(legacyCounts.findRun).toBe(1);
expect(newCounts.findRun).toBe(0);
}
);
// ── A genuine miss on BOTH stores still returns null (fan-out exhausted) ──
heteroRunOpsPostgresTest(
"returns null when the run is on neither store",
async ({ prisma14, prisma17 }) => {
const newStore = makeDedicatedStore(prisma17);
const legacyStore = makeLegacyStore(prisma14);
const router = new RoutingRunStore({ new: newStore, legacy: legacyStore });
expect(
await router.findRun({ id: cuidLegacy("ghost") }, { select: { id: true } })
).toBeNull();
}
);
});
+106 -23
View File
@@ -27,6 +27,7 @@ import type {
WaitpointColocationOptions,
} from "./types.js";
import { isReadReplicaClient } from "./readReplicaClient.js";
import { CONNECTED_RUNS_LIMIT } from "./PostgresRunStore.js";
/**
* Run-ops routing substrate for the TaskRun-core method group. Implements {@link RunStore}
@@ -238,10 +239,9 @@ export class RoutingRunStore implements RunStore {
const onPrimary = readYourWrites(argsOrClient, _client);
const id = idFromWhere(where);
if (id !== undefined) {
// Residency-classifiable (id/friendlyId): route to the owning store.
const store = this.#routeOrNew(id);
const method = onPrimary ? "findRunOnPrimary" : "findRun";
return (store[method] as (...rest: unknown[]) => Promise<unknown>)(where, args);
// Residency-classifiable (id/friendlyId): route to the owning store, then fall back to the
// OTHER store on a miss.
return this.#findRunRouted(id, where, args, onPrimary);
}
// Unclassifiable where (e.g. spanId, idempotencyKey): the run may live on either DB,
// so fan out NEW-first then LEGACY rather than defaulting to NEW — defaulting silently
@@ -249,6 +249,31 @@ export class RoutingRunStore implements RunStore {
return this.#findRunUnrouted(where, args, onPrimary);
}
// A classifiable id names its OWNING store by id-shape, but physical residency can diverge from
// classification (a pre-#4154 base62 run lives on #new yet classifies LEGACY). Read the owning
// store first — a hit is a SINGLE read (the fast path) — then, ONLY on a miss, probe the OTHER
// store before returning null, so a run whose residency ≠ its id-shape is found rather than 404'd.
// Both legs run the SAME index-covered TaskRun lookup; the fan-out cost is paid only on the (rare)
// miss. Mirrors #findRunUnrouted's shape but keyed on the classified owner.
async #findRunRouted(
id: string,
where: Prisma.TaskRunWhereInput,
args: unknown,
onPrimary: boolean
): Promise<unknown> {
const method = onPrimary ? "findRunOnPrimary" : "findRun";
const owning = this.#routeOrNew(id);
const fromOwning = await (owning[method] as (...rest: unknown[]) => Promise<unknown>)(
where,
args
);
if (fromOwning != null) {
return fromOwning;
}
const other = owning === this.#new ? this.#legacy : this.#new;
return (other[method] as (...rest: unknown[]) => Promise<unknown>)(where, args);
}
async #findRunUnrouted(
where: Prisma.TaskRunWhereInput,
args: unknown,
@@ -376,6 +401,62 @@ export class RoutingRunStore implements RunStore {
return finalizeRows([...byId.values()], args, addedFields);
}
// Canonical grouped replacement for `Promise.all(ids.map(id => readThroughRun(id)))`: reuses
// `findRuns`'s bounded id-set path (`#findRunsByIdSet`), so NEW is queried once for the whole set
// and LEGACY once more only for the misses — never one round trip per id. Returns an id-keyed Map;
// missing/duplicate ids are simply absent/collapsed.
findRunsByIds<S extends Prisma.TaskRunSelect>(
ids: string[],
args: { select: S },
client?: ReadClient
): Promise<Map<string, Prisma.TaskRunGetPayload<{ select: S }>>>;
findRunsByIds<I extends Prisma.TaskRunInclude>(
ids: string[],
args: { include: I },
client?: ReadClient
): Promise<Map<string, Prisma.TaskRunGetPayload<{ include: I }>>>;
findRunsByIds(ids: string[], client?: ReadClient): Promise<Map<string, TaskRun>>;
async findRunsByIds(
ids: string[],
argsOrClient?:
| { select?: Record<string, unknown>; include?: Record<string, unknown> }
| ReadClient,
_client?: ReadClient
): Promise<Map<string, unknown>> {
if (ids.length === 0) {
return new Map();
}
const args = selectOrIncludeArgs(argsOrClient);
// Mirrors `readYourWrites`'s slot recovery: when `argsOrClient` isn't a `{select|include}`
// object it may itself BE the client (2-arg call) or be undefined with the client in the
// 3rd slot (an explicit `(ids, undefined, client)` call, e.g. from a relation hydrator).
const client =
args === undefined ? ((argsOrClient as ReadClient | undefined) ?? _client) : _client;
// Force `id` into the projection so the map can key off it, even when the caller's select
// omits it — `findRuns` would otherwise strip it back out as an added-for-merge-only field.
const projected = args?.select
? { select: { ...args.select, id: true } }
: args?.include
? { include: args.include }
: {};
const rows = (await this.findRuns(
{ where: { id: { in: ids } }, ...projected } as FindRunsArgs,
client
)) as Record<string, unknown>[];
const byId = new Map<string, unknown>();
// Strip the id we force-injected for map keying when the caller's select did not ask for it,
// so returned values match the declared payload type and never leak an unrequested id.
const stripInjectedId = args?.select != null && !("id" in (args.select as object));
for (const row of rows) {
const key = row.id as string;
if (stripInjectedId) {
delete row.id;
}
byId.set(key, row);
}
return byId;
}
// ---------------------------------------------------------------------------
// TaskRun-core: update-family — route by run id in params
// ---------------------------------------------------------------------------
@@ -880,7 +961,8 @@ export class RoutingRunStore implements RunStore {
// Keyed by waitpointId, but the WaitpointRunConnection / CompletedWaitpoint join co-locates with the
// RUN/snapshot — which can be on the OTHER DB from a cross-DB token — so fan out to BOTH stores and
// merge. Dedup by value: a token mirrored onto both DBs during drain can carry the same join
// row on each leg.
// row on each leg. Each sub-store already caps at CONNECTED_RUNS_LIMIT, but a disjoint run set on
// each side can still make the union exceed it, so slice again after the merge.
async findWaitpointConnectedRunIds(waitpointId: string, client?: ReadClient): Promise<string[]> {
const [fromNew, fromLegacy] = await Promise.all([
this.#new.findWaitpointConnectedRunIds(
@@ -892,7 +974,7 @@ export class RoutingRunStore implements RunStore {
RoutingRunStore.#ownPrimary(this.#legacy, client)
),
]);
return uniqueStrings([...fromNew, ...fromLegacy]);
return uniqueStrings([...fromNew, ...fromLegacy]).slice(0, CONNECTED_RUNS_LIMIT);
}
async findWaitpointCompletedSnapshotIds(
@@ -1123,23 +1205,20 @@ export class RoutingRunStore implements RunStore {
}
// connectedRuns: the WaitpointRunConnection join co-locates with the run, so read the connected run
// ids from EACH store, then resolve the TaskRun rows across BOTH DBs (findRun routes by id).
// ids from EACH store, then resolve the TaskRun rows in ONE grouped, residency-partitioned read
// (`findRunsByIds`) and reorder to match `runIds` (the join's own order).
async #reresolveConnectedRunsCrossDb(
waitpointId: string,
projection: SubProjection,
client?: ReadClient
): Promise<unknown[]> {
const runIds = await this.findWaitpointConnectedRunIds(waitpointId, client);
const findRun = (this.findRun as (...rest: unknown[]) => Promise<unknown>).bind(this);
const args = projectionAsArgs(projection);
const runs: unknown[] = [];
for (const runId of runIds) {
const run = await findRun({ id: runId }, args, client);
if (run != null) {
runs.push(run);
}
}
return runs;
const findRunsByIds = (
this.findRunsByIds as (...rest: unknown[]) => Promise<Map<string, unknown>>
).bind(this);
const byId = runIds.length > 0 ? await findRunsByIds(runIds, args, client) : new Map();
return runIds.map((id) => byId.get(id)).filter((run) => run != null);
}
// completedExecutionSnapshots: the CompletedWaitpoint join co-locates with the snapshot/run, so read
@@ -1300,21 +1379,25 @@ export class RoutingRunStore implements RunStore {
}
}
// Resolve each edge's `taskRun` from its scalar `taskRunId` across BOTH stores (findRun routes by
// id and falls back NEW→LEGACY). A missing run is left null (display-only callers tolerate it; the
// blocked-run resume path keys off `waitpoint`).
// Resolve every edge's `taskRun` from its scalar `taskRunId` in ONE grouped, residency-partitioned
// read (`findRunsByIds`) rather than one `findRun` per edge. A missing run is left null
// (display-only callers tolerate it; the blocked-run resume path keys off `waitpoint`).
async #hydrateEdgeTaskRunsCrossDb(
edges: Record<string, unknown>[],
projection: SubProjection,
client?: ReadClient
): Promise<void> {
// Bind to `this`: findRun reaches the private #routeOrNew/#findRunUnrouted members, so an unbound
// reference loses `this` and throws on the first private access.
const findRun = (this.findRun as (...rest: unknown[]) => Promise<unknown>).bind(this);
const ids = uniqueStrings(edges.map((e) => e.taskRunId));
const args = projectionAsArgs(projection);
// Bind to `this`: findRunsByIds reaches private members, so an unbound reference throws.
const findRunsByIds = (
this.findRunsByIds as (...rest: unknown[]) => Promise<Map<string, unknown>>
).bind(this);
const byId =
ids.length > 0 ? await findRunsByIds(ids, args, client) : new Map<string, unknown>();
for (const edge of edges) {
const id = edge.taskRunId as string | undefined;
const run = id ? await findRun({ id }, args, client) : null;
const run = id ? byId.get(id) : undefined;
edge.taskRun = applyEdgeProjection((run as Record<string, unknown>) ?? null, projection);
}
}
@@ -0,0 +1,306 @@
// SCHEDULES over the run-ops cutover SEAM.
//
// schedule-engine (internal-packages/schedule-engine) has NO residency logic of its own: a cron
// schedule fires and calls back into `onTriggerScheduledTask` (apps/webapp/app/v3/scheduleEngine.server.ts),
// which mints the run via the ORDINARY TriggerTaskService -> RunEngine.trigger -> RoutingRunStore.createRun
// path — the same mint path already exercised by internal-packages/run-engine/src/engine/tests/triggerCreateRouting.test.ts
// and the `case 1` / `case 1b` findRuns-by-id-set tests in runOpsStore.mixedResidency.test.ts. There is no
// schedule-specific residency branch to test at the mint step; schedules simply inherit whatever the trigger
// path already does. Case A below adds exactly ONE small confirming test (using the previously-untested
// `scheduleId`/`scheduleInstanceId` scalar fields specifically) rather than re-deriving that generic coverage.
//
// The one piece of schedule-adjacent logic that genuinely spans the seam and had NO hetero coverage is
// `rescheduleRun` (RoutingRunStore.rescheduleRun, runOpsStore.ts:617) — the write delegate behind the
// delayed-run "reschedule" API (apps/webapp/app/v3/services/rescheduleTaskRun.server.ts) and
// `DelayedRunSystem.rescheduleDelayedRun`. It is a pure `#routeForWrite(runId)` mechanical delegate with
// no dedicated mixed-residency test anywhere in the existing matrix (runOpsStore.mixedResidency.test.ts
// covers batch/waitpoint/find methods but not rescheduleRun). Case B below closes that gap.
//
// Real two-physical-DB fixture, NO MOCKS: heteroRunOpsPostgresTest (prisma14 = full control-plane/LEGACY
// schema, prisma17 = RunOpsPrismaClient dedicated NEW-subset schema).
import { heteroRunOpsPostgresTest } from "@internal/testcontainers";
import type { PrismaClient } from "@trigger.dev/database";
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
import { describe, expect } from "vitest";
import { PostgresRunStore } from "./PostgresRunStore.js";
import { RoutingRunStore } from "./runOpsStore.js";
import type { CreateRunInput, RunStoreSchemaVariant } from "./types.js";
type AnyClient = PrismaClient | RunOpsPrismaClient;
// ownerEngine classifies by internal-id LENGTH after stripping a leading `<prefix>_`
// (runOpsResidency.ts): 25 chars → cuid → LEGACY, a v1 body (version "1" at index 25 of a 26-char
// body) → run-ops id → NEW. Mirrors the helpers in runOpsStore.mixedResidency.test.ts so ids classify
// the same way here (the documented ID pitfall: a naive generator misclassifies NEW ids as LEGACY).
function cuidLegacy(seed: string): string {
return (seed + "c".repeat(25)).slice(0, 25); // 25 chars → LEGACY (#legacy / prisma14)
}
function runOpsNew(seed: string): string {
return (seed.replace(/[^0-9a-v]/g, "0") + "k".repeat(24)).slice(0, 24) + "01"; // 26 chars, version "1" at index 25 → NEW (#new / prisma17)
}
async function seedEnvironment(
prisma: AnyClient,
schemaVariant: RunStoreSchemaVariant,
suffix: string
) {
if (schemaVariant === "dedicated") {
return {
organization: { id: `org_${suffix}` },
project: { id: `proj_${suffix}` },
environment: { id: `env_${suffix}` },
};
}
const organization = await (prisma as PrismaClient).organization.create({
data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
});
const project = await (prisma as PrismaClient).project.create({
data: {
name: `Project ${suffix}`,
slug: `project-${suffix}`,
externalRef: `proj_${suffix}`,
organizationId: organization.id,
},
});
const environment = await (prisma as PrismaClient).runtimeEnvironment.create({
data: {
type: "DEVELOPMENT",
slug: "dev",
projectId: project.id,
organizationId: organization.id,
apiKey: `tr_dev_${suffix}`,
pkApiKey: `pk_dev_${suffix}`,
shortcode: `short_${suffix}`,
},
});
return { organization, project, environment };
}
async function seedSharedEnv(prisma14: PrismaClient, suffix: string) {
const legacy = await seedEnvironment(prisma14, "legacy", suffix);
return {
organizationId: legacy.organization.id,
projectId: legacy.project.id,
runtimeEnvironmentId: legacy.environment.id,
environmentId: legacy.environment.id,
};
}
function buildCreateRunInput(params: {
runId: string;
friendlyId: string;
organizationId: string;
projectId: string;
runtimeEnvironmentId: string;
status?: "PENDING" | "DELAYED";
scheduleId?: string;
scheduleInstanceId?: string;
}): CreateRunInput {
return {
data: {
id: params.runId,
engine: "V2",
status: params.status ?? "PENDING",
friendlyId: params.friendlyId,
runtimeEnvironmentId: params.runtimeEnvironmentId,
environmentType: "DEVELOPMENT",
organizationId: params.organizationId,
projectId: params.projectId,
taskIdentifier: "my-scheduled-task",
payload: '{"hello":"world"}',
payloadType: "application/json",
traceContext: { trace: "ctx" },
traceId: `trace_${params.runId}`,
spanId: `span_${params.runId}`,
runTags: [],
queue: "task/my-scheduled-task",
isTest: false,
taskEventStore: "taskEvent",
depth: 0,
createdAt: new Date("2024-01-01T00:00:00.000Z"),
...(params.scheduleId && { scheduleId: params.scheduleId }),
...(params.scheduleInstanceId && { scheduleInstanceId: params.scheduleInstanceId }),
},
snapshot: {
engine: "V2",
executionStatus: "RUN_CREATED",
description: "Run was created",
runStatus: params.status ?? "PENDING",
environmentId: params.runtimeEnvironmentId,
environmentType: "DEVELOPMENT",
projectId: params.projectId,
organizationId: params.organizationId,
},
};
}
function makeDedicatedStore(prisma17: RunOpsPrismaClient) {
return new PostgresRunStore({
prisma: prisma17 as never,
readOnlyPrisma: prisma17 as never,
schemaVariant: "dedicated",
});
}
function makeLegacyStore(prisma14: PrismaClient) {
return new PostgresRunStore({
prisma: prisma14,
readOnlyPrisma: prisma14,
schemaVariant: "legacy",
});
}
// The REAL production split topology: #new = dedicated subset on prisma17, #legacy = full schema on
// prisma14.
function makeSplitRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) {
const legacyStore = makeLegacyStore(prisma14);
const newStore = makeDedicatedStore(prisma17);
return {
router: new RoutingRunStore({ new: newStore, legacy: legacyStore }),
legacyStore,
newStore,
};
}
describe("Schedules over the cutover seam", () => {
// ── Case A: a schedule-minted run (scheduleId/scheduleInstanceId set) lands on the correct
// physical store under mixed residency, and is found by plain id lookup (the ONLY lookup that
// exists — there is no scheduleInstanceId-keyed TaskRun query anywhere in the codebase; the field
// is write-only metadata stamped at creation). This confirms schedules have no distinct mint-routing
// seam beyond the already-covered generic createRun routing. ──
heteroRunOpsPostgresTest(
"case A: a schedule-minted run routes to the owning store and resolves by id, for both residencies",
async ({ prisma14, prisma17 }) => {
const { router } = makeSplitRouter(prisma14, prisma17);
const env = await seedSharedEnv(prisma14, "schedA");
const legacyRun = cuidLegacy("schAl"); // pre-cutover mint shape → #legacy
const newRun = runOpsNew("schAn"); // run-ops mint shape → #new
await router.createRun(
buildCreateRunInput({
runId: legacyRun,
friendlyId: "run_schedA_legacy",
scheduleId: "sched_A",
scheduleInstanceId: "schedinst_A_legacy",
...env,
})
);
await router.createRun(
buildCreateRunInput({
runId: newRun,
friendlyId: "run_schedA_new",
scheduleId: "sched_A",
scheduleInstanceId: "schedinst_A_new",
...env,
})
);
// Physical residency: each landed on its OWN DB only, with the scheduleInstanceId intact.
const onLegacy = await prisma14.taskRun.findUnique({ where: { id: legacyRun } });
expect(onLegacy?.scheduleInstanceId).toBe("schedinst_A_legacy");
expect(await prisma17.taskRun.findUnique({ where: { id: legacyRun } })).toBeNull();
const onNew = await prisma17.taskRun.findUnique({ where: { id: newRun } });
expect(onNew?.scheduleInstanceId).toBe("schedinst_A_new");
expect(await prisma14.taskRun.findUnique({ where: { id: newRun } })).toBeNull();
// The only lookup a schedule-minted run ever needs (by id) resolves through the router on
// BOTH residencies.
const foundLegacy = (await router.findRun(
{ id: legacyRun },
{ select: { id: true, scheduleInstanceId: true } }
)) as Record<string, any> | null;
expect(foundLegacy?.id).toBe(legacyRun);
expect(foundLegacy?.scheduleInstanceId).toBe("schedinst_A_legacy");
const foundNew = (await router.findRun(
{ id: newRun },
{ select: { id: true, scheduleInstanceId: true } }
)) as Record<string, any> | null;
expect(foundNew?.id).toBe(newRun);
expect(foundNew?.scheduleInstanceId).toBe("schedinst_A_new");
}
);
// ── Case B: rescheduleRun (the write delegate behind the delayed-run "reschedule" API and
// DelayedRunSystem.rescheduleDelayedRun) routes to the OWNING store for a mixed-residency
// population, and does NOT touch the other physical DB. No existing hetero test covers this
// write path. ──
heteroRunOpsPostgresTest(
"case B: rescheduleRun routes the write to the owning store only, for a mixed-residency population",
async ({ prisma14, prisma17 }) => {
const { router } = makeSplitRouter(prisma14, prisma17);
const env = await seedSharedEnv(prisma14, "schedB");
const legacyRun = cuidLegacy("schBl");
const newRun = runOpsNew("schBn");
await router.createRun(
buildCreateRunInput({
runId: legacyRun,
friendlyId: "run_schedB_legacy",
status: "DELAYED",
...env,
})
);
await router.createRun(
buildCreateRunInput({
runId: newRun,
friendlyId: "run_schedB_new",
status: "DELAYED",
...env,
})
);
const legacyDelayUntil = new Date("2027-05-01T00:00:00.000Z");
const newDelayUntil = new Date("2027-06-01T00:00:00.000Z");
const updatedLegacy = await router.rescheduleRun(legacyRun, {
delayUntil: legacyDelayUntil,
snapshot: {
environmentId: env.environmentId,
environmentType: "DEVELOPMENT",
projectId: env.projectId,
organizationId: env.organizationId,
},
});
const updatedNew = await router.rescheduleRun(newRun, {
delayUntil: newDelayUntil,
snapshot: {
environmentId: env.environmentId,
environmentType: "DEVELOPMENT",
projectId: env.projectId,
organizationId: env.organizationId,
},
});
expect(updatedLegacy.id).toBe(legacyRun);
expect(updatedLegacy.delayUntil).toEqual(legacyDelayUntil);
expect(updatedNew.id).toBe(newRun);
expect(updatedNew.delayUntil).toEqual(newDelayUntil);
// The write landed on the OWNING physical DB only.
const legacyRow = await prisma14.taskRun.findUnique({ where: { id: legacyRun } });
expect(legacyRow?.delayUntil).toEqual(legacyDelayUntil);
const legacySnapshots = await prisma14.taskRunExecutionSnapshot.findMany({
where: { runId: legacyRun, executionStatus: "DELAYED" },
});
expect(legacySnapshots).toHaveLength(1);
const newRow = await prisma17.taskRun.findUnique({ where: { id: newRun } });
expect(newRow?.delayUntil).toEqual(newDelayUntil);
const newSnapshots = await prisma17.taskRunExecutionSnapshot.findMany({
where: { runId: newRun, executionStatus: "DELAYED" },
});
expect(newSnapshots).toHaveLength(1);
// Cross-DB isolation: the OTHER physical DB's row for each run is untouched by the other
// reschedule call — neither run exists on the non-owning DB at all, so there is nothing there
// to have been (mis)updated.
expect(await prisma17.taskRun.findUnique({ where: { id: legacyRun } })).toBeNull();
expect(await prisma14.taskRun.findUnique({ where: { id: newRun } })).toBeNull();
}
);
});
+15
View File
@@ -573,6 +573,21 @@ export interface RunStore {
client?: ReadClient
): Promise<TaskRun[]>;
// Grouped replacement for `Promise.all(ids.map(id => findRun(id)))`: one round trip for the
// whole id batch instead of one per id. Returns an id-keyed Map; missing/duplicate ids are
// simply absent/collapsed.
findRunsByIds<S extends Prisma.TaskRunSelect>(
ids: string[],
args: { select: S },
client?: ReadClient
): Promise<Map<string, Prisma.TaskRunGetPayload<{ select: S }>>>;
findRunsByIds<I extends Prisma.TaskRunInclude>(
ids: string[],
args: { include: I },
client?: ReadClient
): Promise<Map<string, Prisma.TaskRunGetPayload<{ include: I }>>>;
findRunsByIds(ids: string[], client?: ReadClient): Promise<Map<string, TaskRun>>;
// --- run-ops persistence ---
// Snapshots, waitpoints, implicit M:N joins, dependents, attempts and checkpoints. The
// generic model wrappers are thin generics over the Prisma `*Args` types so include/select