From cecdfd94be97cadbe041f513aabe9c8d711e5b1e Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Fri, 17 Jul 2026 16:54:25 +0100 Subject: [PATCH 001/300] fix: only count preview branches toward the preview branch limit (#4283) --- .server-changes/fix-preview-branch-limit-count.md | 6 ++++++ apps/webapp/app/presenters/v3/LimitsPresenter.server.ts | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .server-changes/fix-preview-branch-limit-count.md diff --git a/.server-changes/fix-preview-branch-limit-count.md b/.server-changes/fix-preview-branch-limit-count.md new file mode 100644 index 000000000..d5f5b26d7 --- /dev/null +++ b/.server-changes/fix-preview-branch-limit-count.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +The "Preview branches" usage on the Limits page now counts only preview branches. diff --git a/apps/webapp/app/presenters/v3/LimitsPresenter.server.ts b/apps/webapp/app/presenters/v3/LimitsPresenter.server.ts index cbf3abc5c..e468efd92 100644 --- a/apps/webapp/app/presenters/v3/LimitsPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/LimitsPresenter.server.ts @@ -155,7 +155,8 @@ export class LimitsPresenter extends BasePresenter { const activeBranchCount = await this._replica.runtimeEnvironment.count({ where: { projectId, - branchName: { + type: "PREVIEW", + parentEnvironmentId: { not: null, }, archivedAt: null, From ae96b6c1755500ec158b0cbde3435585e9c426aa Mon Sep 17 00:00:00 2001 From: Daniel Sutton <45313566+d-cs@users.noreply.github.com> Date: Sun, 19 Jul 2026 17:57:41 +0100 Subject: [PATCH 002/300] fix: read-your-writes + global-scope idempotency correctness under the run-ops split (#4284) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What & why Two related correctness fixes for the run-ops DB split. Under the split, run-store reads can route to a **lagging read replica**; a just-written run/waitpoint/batch can then be missed, causing a wrong decision. **1. Read-your-writes → owning primary.** Surfaced first as an intermittent `wait.until({ idempotencyKey })` re-wait on retry. Auditing the run-store read surface found the same class at sibling sites (some gating mutations or returning spurious 404s, others tolerable/self-healing). Reads that must observe their own writes now route to the owning **primary** (`findRun`/`findWaitpoint`/`findBatchTaskRunByFriendlyId` → `*OnPrimary`, a primary re-read on a miss, or a retryable 404 where the SDK polls). Read-view reads stay on the replica. All additive — the happy path is unchanged. **2. Global-scope idempotency across the split.** A `global`-scope key carries no per-run salt, so the same `(env, task, key)` triggered concurrently from parents resident on **different** run-ops DBs could dedup-miss on each DB and create a duplicate (the per-DB unique index can't enforce cross-DB uniqueness). Such triggers (global scope, or scope-absent, while split is active) are serialized through the existing Redis idempotency claim, the loser resolves the winner by id across both DBs, and the claim is reacquired on the expired/failed clear-and-recreate path. `run`/`attempt` scope embed the run id and never contend. ## Stacked for review This is the **base** of a 2-PR stack, split so review is easier: - **This PR** — production code only (34 files). - **Stacked tests PR → https://github.com/triggerdotdev/trigger.dev/pull/4285** — the caller-driven guards (55 test files) on top of this branch. ## Validation Local run-ops split, **both 2-DB and 3-DB**, fresh boot on this branch: SDK canary 64/71 (only the known concurrency/input-streams/s3 failures), quarantine sweep **0 unexpected** (340 pass / 16 known / 4 local) in each topology, dashboard e2e 0 failed. No product regressions. --- apps/webapp/app/env.server.ts | 3 + .../app/models/runtimeEnvironment.server.ts | 24 +- .../v3/ApiWaitpointPresenter.server.ts | 53 +- .../presenters/v3/BatchPresenter.server.ts | 14 +- .../app/routes/api.v1.batches.$batchId.ts | 5 + ....runs.$runFriendlyId.input-streams.wait.ts | 30 +- ...uns.$runFriendlyId.session-streams.wait.ts | 28 +- .../app/routes/api.v1.runs.$runId.metadata.ts | 13 + ...i.v1.sessions.$session.end-and-continue.ts | 15 +- ...ens.$waitpointFriendlyId.callback.$hash.ts | 11 +- .../app/routes/api.v2.batches.$batchId.ts | 5 + .../routes/realtime.v1.batches.$batchId.ts | 12 +- .../app/routes/realtime.v1.runs.$runId.ts | 28 +- .../realtime.v1.streams.$runId.$streamId.ts | 40 +- ...streams.$runId.$target.$streamId.append.ts | 40 +- ...ime.v1.streams.$runId.$target.$streamId.ts | 91 ++-- ...ltime.v1.streams.$runId.input.$streamId.ts | 59 +-- ...am.runs.$runParam.idempotencyKey.reset.tsx | 27 +- ...ram.realtime.v1.sessions.$sessionId.$io.ts | 41 +- ...am.realtime.v1.streams.$runId.$streamId.ts | 30 +- ...ltime.v1.streams.$runId.input.$streamId.ts | 30 +- .../route.tsx | 28 +- .../route.tsx | 10 +- .../resources.taskruns.$runParam.cancel.ts | 35 +- .../resources.taskruns.$runParam.replay.ts | 7 +- .../app/routes/sync.traces.runs.$traceId.ts | 11 +- .../concerns/idempotencyKeys.server.ts | 471 +++++++++++++----- .../runEngine/services/triggerTask.server.ts | 13 +- .../app/services/dashboardAgent.server.ts | 4 +- .../realtime/sessionRunManager.server.ts | 6 +- apps/webapp/app/v3/mollifier/claimTtl.ts | 12 + .../v3/mollifier/idempotencyClaim.server.ts | 43 +- .../resolveBatchForRealtime.server.ts | 24 + .../app/v3/services/batchTriggerV3.server.ts | 23 +- .../test/mollifierIdempotencyClaim.test.ts | 8 +- .../src/engine/systems/runAttemptSystem.ts | 21 +- .../src/engine/systems/ttlSystem.ts | 34 +- ...OpsStore.residencyMismatchFallback.test.ts | 296 ----------- .../run-store/src/runOpsStore.ts | 62 +-- packages/redis-worker/src/mollifier/buffer.ts | 17 + 40 files changed, 935 insertions(+), 789 deletions(-) create mode 100644 apps/webapp/app/v3/mollifier/claimTtl.ts create mode 100644 apps/webapp/app/v3/realtime/resolveBatchForRealtime.server.ts delete mode 100644 internal-packages/run-store/src/runOpsStore.residencyMismatchFallback.test.ts diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 4e2d4c584..973560ca0 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -1311,6 +1311,9 @@ const EnvironmentSchema = z // claim TTL), how long a waiter blocks before timing out, and the // waiter poll interval. TRIGGER_MOLLIFIER_CLAIM_TTL_SECONDS: z.coerce.number().int().positive().default(30), + // Pipeline floor: the claim never shrinks below this even for a short customer key TTL, so it + // can't expire mid-pipeline and let a loser re-claim (cross-DB duplicate under the split). + TRIGGER_MOLLIFIER_CLAIM_MIN_TTL_SECONDS: z.coerce.number().int().positive().default(5), TRIGGER_MOLLIFIER_CLAIM_WAIT_MS: z.coerce.number().int().positive().default(5_000), TRIGGER_MOLLIFIER_CLAIM_POLL_MS: z.coerce.number().int().positive().default(25), diff --git a/apps/webapp/app/models/runtimeEnvironment.server.ts b/apps/webapp/app/models/runtimeEnvironment.server.ts index 987394ea4..939cd55e9 100644 --- a/apps/webapp/app/models/runtimeEnvironment.server.ts +++ b/apps/webapp/app/models/runtimeEnvironment.server.ts @@ -274,19 +274,17 @@ export async function findEnvironmentFromRun( ): Promise { // Run-ops scalars (runTags/batchId/runtimeEnvironmentId) from the run store; the env half is // resolved via the control-plane resolver so the run-ops DB can split without a cross-DB join. - const taskRun = await runStore.findRun( - { - id: runId, - }, - { - select: { - runTags: true, - batchId: true, - runtimeEnvironmentId: true, - }, - }, - tx ?? $replica - ); + const select = { + runTags: true, + batchId: true, + runtimeEnvironmentId: true, + } as const; + let taskRun = await runStore.findRun({ id: runId }, { select }, tx ?? $replica); + if (!taskRun) { + // Read-your-writes: a just-created run may not have replicated. Re-read the owning primary before + // treating it as absent, so runMetadataUpdated doesn't drop a live run's final metadata + publish. + taskRun = await runStore.findRun({ id: runId }, { select }, prisma); + } if (!taskRun) { return null; } diff --git a/apps/webapp/app/presenters/v3/ApiWaitpointPresenter.server.ts b/apps/webapp/app/presenters/v3/ApiWaitpointPresenter.server.ts index c4477efef..eb1d0dd0c 100644 --- a/apps/webapp/app/presenters/v3/ApiWaitpointPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/ApiWaitpointPresenter.server.ts @@ -42,29 +42,36 @@ export class ApiWaitpointPresenter extends BasePresenter { return this.trace("call", async (span) => { // The store routes by the waitpointId's residency (id shape) and reads the owning // store's replica. waitpointId is pre-decoded from the friendlyId via WaitpointId.toId. - const waitpoint = await this.runStore.findWaitpoint({ - where: { - id: waitpointId, - environmentId: environment.id, - }, - select: { - id: true, - friendlyId: true, - type: true, - status: true, - idempotencyKey: true, - userProvidedIdempotencyKey: true, - idempotencyKeyExpiresAt: true, - inactiveIdempotencyKey: true, - output: true, - outputType: true, - outputIsError: true, - completedAfter: true, - completedAt: true, - createdAt: true, - tags: true, - }, - }); + const where = { + id: waitpointId, + environmentId: environment.id, + }; + const select = { + id: true, + friendlyId: true, + type: true, + status: true, + idempotencyKey: true, + userProvidedIdempotencyKey: true, + idempotencyKeyExpiresAt: true, + inactiveIdempotencyKey: true, + output: true, + outputType: true, + outputIsError: true, + completedAfter: true, + completedAt: true, + createdAt: true, + tags: true, + } as const; + + let waitpoint = await this.runStore.findWaitpoint({ where, select }); + + // Read-your-writes on a public GET: a just-minted token may not be on the owning store's + // replica yet, so a replica miss would 404 a live token. Re-read the owning primary before + // concluding it doesn't exist (mirrors the metadata GET loader + the complete/callback paths). + if (!waitpoint) { + waitpoint = await this.runStore.findWaitpointOnPrimary({ where, select }); + } if (!waitpoint) { logger.error(`WaitpointPresenter: Waitpoint not found`, { diff --git a/apps/webapp/app/presenters/v3/BatchPresenter.server.ts b/apps/webapp/app/presenters/v3/BatchPresenter.server.ts index b9190a6f6..3e9ef0be8 100644 --- a/apps/webapp/app/presenters/v3/BatchPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/BatchPresenter.server.ts @@ -47,13 +47,25 @@ export class BatchPresenter extends BasePresenter { // The BatchTaskRun (run-ops) is read through the run store, which routes by residency. The // runtimeEnvironment (control-plane) is resolved separately because the cross-seam FK is // dropped, so the batch row cannot single-SQL join to control-plane RuntimeEnvironment. - const batch = await this.runStore.findBatchTaskRunByFriendlyId( + let batch = await this.runStore.findBatchTaskRunByFriendlyId( batchId, environmentId, { include: BATCH_INCLUDE }, this._replica ); + // Read-your-writes: findBatchTaskRunByFriendlyId defaults to (and here reads) the replica, so a + // batch created within the replica's apply window returns null under lag. Re-read from the owning + // primary on a miss so a live batch's detail page never spuriously 404s ("Batch not found"). + if (!batch) { + batch = await this.runStore.findBatchTaskRunByFriendlyId( + batchId, + environmentId, + { include: BATCH_INCLUDE }, + this._prisma + ); + } + if (!batch) { throw new Error("Batch not found"); } diff --git a/apps/webapp/app/routes/api.v1.batches.$batchId.ts b/apps/webapp/app/routes/api.v1.batches.$batchId.ts index 01acd3d14..713716f20 100644 --- a/apps/webapp/app/routes/api.v1.batches.$batchId.ts +++ b/apps/webapp/app/routes/api.v1.batches.$batchId.ts @@ -12,6 +12,11 @@ export const loader = createLoaderApiRoute( params: ParamsSchema, allowJWT: true, corsStrategy: "all", + // A just-created batch may not yet have replicated to the read replica this client-less + // findBatchTaskRunByFriendlyId lookup routes to; return a retryable 404 so the SDK retries through + // replica lag rather than stranding a live batch on a permanent 404 (mirrors the run-get routes, + // e.g. api.v3.runs.$runId). + shouldRetryNotFound: true, findResource: (params, auth) => { return runStore.findBatchTaskRunByFriendlyId(params.batchId, auth.environment.id, { include: { errors: true }, diff --git a/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.input-streams.wait.ts b/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.input-streams.wait.ts index 3c081b228..024779ac6 100644 --- a/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.input-streams.wait.ts +++ b/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.input-streams.wait.ts @@ -33,21 +33,23 @@ const { action, loader } = createActionApiRoute( }, async ({ authentication, body, params }) => { try { - const run = await runStore.findRun( - { - friendlyId: params.runFriendlyId, - runtimeEnvironmentId: authentication.environment.id, + const where = { + friendlyId: params.runFriendlyId, + runtimeEnvironmentId: authentication.environment.id, + }; + const args = { + select: { + id: true, + friendlyId: true, + realtimeStreamsVersion: true, + streamBasinName: true, }, - { - select: { - id: true, - friendlyId: true, - realtimeStreamsVersion: true, - streamBasinName: true, - }, - }, - $replica - ); + }; + // Replica lag can null out a live run; a spurious 404 fails the .wait() registration on a run + // that exists. Re-read the owning primary on a replica miss. + const run = + (await runStore.findRun(where, args, $replica)) ?? + (await runStore.findRunOnPrimary(where, args)); if (!run) { return json({ error: "Run not found" }, { status: 404 }); diff --git a/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.session-streams.wait.ts b/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.session-streams.wait.ts index f6c58ab74..c00ff51b3 100644 --- a/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.session-streams.wait.ts +++ b/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.session-streams.wait.ts @@ -39,20 +39,22 @@ const { action, loader } = createActionApiRoute( }, async ({ authentication, body, params }) => { try { - const run = await runStore.findRun( - { - friendlyId: params.runFriendlyId, - runtimeEnvironmentId: authentication.environment.id, + const where = { + friendlyId: params.runFriendlyId, + runtimeEnvironmentId: authentication.environment.id, + }; + const args = { + select: { + id: true, + friendlyId: true, + realtimeStreamsVersion: true, }, - { - select: { - id: true, - friendlyId: true, - realtimeStreamsVersion: true, - }, - }, - $replica - ); + }; + // Replica lag can null out a live run; a spurious 404 fails the .wait() registration on a run + // that exists. Re-read the owning primary on a replica miss. + const run = + (await runStore.findRun(where, args, $replica)) ?? + (await runStore.findRunOnPrimary(where, args)); if (!run) { return json({ error: "Run not found" }, { status: 404 }); diff --git a/apps/webapp/app/routes/api.v1.runs.$runId.metadata.ts b/apps/webapp/app/routes/api.v1.runs.$runId.metadata.ts index 58cf572f4..ac26302f8 100644 --- a/apps/webapp/app/routes/api.v1.runs.$runId.metadata.ts +++ b/apps/webapp/app/routes/api.v1.runs.$runId.metadata.ts @@ -64,6 +64,19 @@ export async function loader({ request, params }: LoaderFunctionArgs) { ); } + // Read-your-writes: a run drained from the buffer to the primary but not yet replicated misses + // both the replica read and the buffer. Re-read the owning primary before 404ing. + const primaryRun = await runStore.findRunOnPrimary( + { friendlyId: parsed.data.runId, runtimeEnvironmentId: env.id }, + { select: { metadata: true, metadataType: true } } + ); + if (primaryRun) { + return json( + { metadata: primaryRun.metadata, metadataType: primaryRun.metadataType }, + { status: 200 } + ); + } + return json({ error: "Run not found" }, { status: 404 }); } diff --git a/apps/webapp/app/routes/api.v1.sessions.$session.end-and-continue.ts b/apps/webapp/app/routes/api.v1.sessions.$session.end-and-continue.ts index 35770928c..c6e5a90f6 100644 --- a/apps/webapp/app/routes/api.v1.sessions.$session.end-and-continue.ts +++ b/apps/webapp/app/routes/api.v1.sessions.$session.end-and-continue.ts @@ -75,7 +75,7 @@ const { action, loader } = createActionApiRoute( // SDK exposes via `ctx.run.id`). Internally `Session.currentRunId` // stores the TaskRun.id cuid, so resolve before handing to the // optimistic-claim service. - const callingRun = await runStore.findRun( + let callingRun = await runStore.findRun( { friendlyId: body.callingRunId, runtimeEnvironmentId: authentication.environment.id, @@ -83,6 +83,19 @@ const { action, loader } = createActionApiRoute( { select: { id: true } }, $replica ); + if (!callingRun) { + // Replica lag: `callingRunId` is the agent's own live run (it is executing this request), so it + // exists on the owning primary even when the read replica has not caught up. Re-read the primary + // before 404ing — otherwise a lagging replica turns a legitimate handoff into a spurious + // "callingRunId not found in this environment". + callingRun = await runStore.findRunOnPrimary( + { + friendlyId: body.callingRunId, + runtimeEnvironmentId: authentication.environment.id, + }, + { select: { id: true } } + ); + } if (!callingRun) { return json({ error: "callingRunId not found in this environment" }, { status: 404 }); } diff --git a/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts b/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts index 502db2aca..431a7eb25 100644 --- a/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts +++ b/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts @@ -36,13 +36,22 @@ export async function action({ request, params }: ActionFunctionArgs) { // Resolve wherever the waitpoint resides. The store routes by the waitpoint id's residency // (id-shape) and probes both run-ops DBs, so a token on either store resolves; the env is // resolved below from the row via the control-plane resolver. - const waitpoint = await runStore.findWaitpoint({ + let waitpoint = await runStore.findWaitpoint({ where: { id: waitpointId, }, select: { id: true, status: true, environmentId: true }, }); + if (!waitpoint) { + // Read-your-writes: a token whose callback fires right after mint may not have replicated + // yet. Re-read the owning primary before 404ing (mirrors complete.ts's primary fallback). + waitpoint = await runStore.findWaitpointOnPrimary({ + where: { id: waitpointId }, + select: { id: true, status: true, environmentId: true }, + }); + } + if (!waitpoint) { return json({ error: "Waitpoint not found" }, { status: 404 }); } diff --git a/apps/webapp/app/routes/api.v2.batches.$batchId.ts b/apps/webapp/app/routes/api.v2.batches.$batchId.ts index b6ef4136b..4f23f14bc 100644 --- a/apps/webapp/app/routes/api.v2.batches.$batchId.ts +++ b/apps/webapp/app/routes/api.v2.batches.$batchId.ts @@ -12,6 +12,11 @@ export const loader = createLoaderApiRoute( params: ParamsSchema, allowJWT: true, corsStrategy: "all", + // A just-created batch may not yet have replicated to the read replica this client-less + // findBatchTaskRunByFriendlyId lookup routes to; return a retryable 404 so the SDK retries through + // replica lag rather than stranding a live batch on a permanent 404 (mirrors the run-get routes, + // e.g. api.v3.runs.$runId). + shouldRetryNotFound: true, findResource: (params, auth) => { return runStore.findBatchTaskRunByFriendlyId(params.batchId, auth.environment.id, { include: { errors: true }, diff --git a/apps/webapp/app/routes/realtime.v1.batches.$batchId.ts b/apps/webapp/app/routes/realtime.v1.batches.$batchId.ts index 29ad61d66..8a761b32f 100644 --- a/apps/webapp/app/routes/realtime.v1.batches.$batchId.ts +++ b/apps/webapp/app/routes/realtime.v1.batches.$batchId.ts @@ -2,7 +2,7 @@ import { z } from "zod"; import { getRequestAbortSignal } from "~/services/httpAsyncStorage.server"; import { resolveRealtimeStreamClient } from "~/services/realtime/resolveRealtimeStreamClient.server"; import { anyResource, createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; -import { runStore } from "~/v3/runStore.server"; +import { resolveBatchTaskRunForRealtime } from "~/v3/realtime/resolveBatchForRealtime.server"; const ParamsSchema = z.object({ batchId: z.string(), @@ -13,9 +13,13 @@ export const loader = createLoaderApiRoute( params: ParamsSchema, allowJWT: true, corsStrategy: "all", - findResource: (params, auth) => { - return runStore.findBatchTaskRunByFriendlyId(params.batchId, auth.environment.id); - }, + // A just-created batch may not yet have replicated to the read replica the client-less lookup uses. + // shouldRetryNotFound stamps a retryable 404 for the zodfetch GET; the realtime resolver ALSO + // re-reads the owning primary on a replica miss, so the Electric ShapeStream consumer (which ignores + // x-should-retry) doesn't strand a live batch on a permanent 404. Mirrors the run-get routes. + shouldRetryNotFound: true, + findResource: (params, auth) => + resolveBatchTaskRunForRealtime(params.batchId, auth.environment.id), authorization: { action: "read", // See sibling note in api.v1.batches.$batchId.ts — `{type: "runs"}` diff --git a/apps/webapp/app/routes/realtime.v1.runs.$runId.ts b/apps/webapp/app/routes/realtime.v1.runs.$runId.ts index 51b47b22a..c65bf0ecd 100644 --- a/apps/webapp/app/routes/realtime.v1.runs.$runId.ts +++ b/apps/webapp/app/routes/realtime.v1.runs.$runId.ts @@ -15,22 +15,24 @@ export const loader = createLoaderApiRoute( allowJWT: true, corsStrategy: "all", findResource: async (params, authentication) => { - return runStore.findRun( - { - friendlyId: params.runId, - runtimeEnvironmentId: authentication.environment.id, - }, - { - include: { - batch: { - select: { - friendlyId: true, - }, + const where = { + friendlyId: params.runId, + runtimeEnvironmentId: authentication.environment.id, + }; + const args = { + include: { + batch: { + select: { + friendlyId: true, }, }, }, - $replica - ); + }; + // Replica lag can null out a run that already exists on the owning primary. A spurious 404 + // here permanently fails the client's realtime subscription (the SSE client treats 404 as + // "stream gone" — nonRetryableStatuses). Re-read the primary on a replica miss. + const run = await runStore.findRun(where, args, $replica); + return run ?? runStore.findRunOnPrimary(where, args); }, authorization: { action: "read", diff --git a/apps/webapp/app/routes/realtime.v1.streams.$runId.$streamId.ts b/apps/webapp/app/routes/realtime.v1.streams.$runId.$streamId.ts index 895e336fd..ab941fd05 100644 --- a/apps/webapp/app/routes/realtime.v1.streams.$runId.$streamId.ts +++ b/apps/webapp/app/routes/realtime.v1.streams.$runId.$streamId.ts @@ -16,29 +16,29 @@ export const loader = createLoaderApiRoute( allowJWT: true, corsStrategy: "all", findResource: async (params, auth) => { - const run = await runStore.findRun( - { - friendlyId: params.runId, - runtimeEnvironmentId: auth.environment.id, - }, - { - select: { - id: true, - friendlyId: true, - taskIdentifier: true, - runTags: true, - realtimeStreamsVersion: true, - streamBasinName: true, - batch: { - select: { - friendlyId: true, - }, + const where = { + friendlyId: params.runId, + runtimeEnvironmentId: auth.environment.id, + }; + const args = { + select: { + id: true, + friendlyId: true, + taskIdentifier: true, + runTags: true, + realtimeStreamsVersion: true, + streamBasinName: true, + batch: { + select: { + friendlyId: true, }, }, }, - $replica - ); - return run; + }; + // Replica lag can null out a live run; a spurious 404 permanently fails the SSE subscription + // (the client treats 404 as "stream gone"). Re-read the owning primary on a replica miss. + const run = await runStore.findRun(where, args, $replica); + return run ?? runStore.findRunOnPrimary(where, args); }, authorization: { action: "read", diff --git a/apps/webapp/app/routes/realtime.v1.streams.$runId.$target.$streamId.append.ts b/apps/webapp/app/routes/realtime.v1.streams.$runId.$target.$streamId.append.ts index 12400e8d0..7195a4ae9 100644 --- a/apps/webapp/app/routes/realtime.v1.streams.$runId.$target.$streamId.append.ts +++ b/apps/webapp/app/routes/realtime.v1.streams.$runId.$target.$streamId.append.ts @@ -27,29 +27,31 @@ const { action } = createActionApiRoute( maxContentLength: MAX_APPEND_BODY_BYTES, }, async ({ request, params, authentication }) => { - const run = await runStore.findRun( - { - friendlyId: params.runId, - runtimeEnvironmentId: authentication.environment.id, - }, - { - select: { - id: true, - friendlyId: true, - parentTaskRun: { - select: { - friendlyId: true, - }, + const where = { + friendlyId: params.runId, + runtimeEnvironmentId: authentication.environment.id, + }; + const args = { + select: { + id: true, + friendlyId: true, + parentTaskRun: { + select: { + friendlyId: true, }, - rootTaskRun: { - select: { - friendlyId: true, - }, + }, + rootTaskRun: { + select: { + friendlyId: true, }, }, }, - $replica - ); + }; + // Replica lag can null out a live run; a spurious 404 permanently fails the append. Re-read the + // owning primary on a replica miss. + const run = + (await runStore.findRun(where, args, $replica)) ?? + (await runStore.findRunOnPrimary(where, args)); if (!run) { return new Response("Run not found", { status: 404 }); diff --git a/apps/webapp/app/routes/realtime.v1.streams.$runId.$target.$streamId.ts b/apps/webapp/app/routes/realtime.v1.streams.$runId.$target.$streamId.ts index 01a2b550d..31c556457 100644 --- a/apps/webapp/app/routes/realtime.v1.streams.$runId.$target.$streamId.ts +++ b/apps/webapp/app/routes/realtime.v1.streams.$runId.$target.$streamId.ts @@ -19,32 +19,34 @@ const { action } = createActionApiRoute( params: ParamsSchema, }, async ({ request, params, authentication }) => { - const run = await runStore.findRun( - { - friendlyId: params.runId, - runtimeEnvironmentId: authentication.environment.id, - }, - { - select: { - id: true, - friendlyId: true, - streamBasinName: true, - parentTaskRun: { - select: { - friendlyId: true, - streamBasinName: true, - }, + const where = { + friendlyId: params.runId, + runtimeEnvironmentId: authentication.environment.id, + }; + const args = { + select: { + id: true, + friendlyId: true, + streamBasinName: true, + parentTaskRun: { + select: { + friendlyId: true, + streamBasinName: true, }, - rootTaskRun: { - select: { - friendlyId: true, - streamBasinName: true, - }, + }, + rootTaskRun: { + select: { + friendlyId: true, + streamBasinName: true, }, }, }, - $replica - ); + }; + // Replica lag can null out a live run; a spurious 404 permanently fails the ingest client. + // Re-read the owning primary on a replica miss. + const run = + (await runStore.findRun(where, args, $replica)) ?? + (await runStore.findRunOnPrimary(where, args)); if (!run) { return new Response("Run not found", { status: 404 }); @@ -154,32 +156,33 @@ const loader = createLoaderApiRoute( allowJWT: false, corsStrategy: "none", findResource: async (params, authentication) => { - return runStore.findRun( - { - friendlyId: params.runId, - runtimeEnvironmentId: authentication.environment.id, - }, - { - select: { - id: true, - friendlyId: true, - streamBasinName: true, - parentTaskRun: { - select: { - friendlyId: true, - streamBasinName: true, - }, + const where = { + friendlyId: params.runId, + runtimeEnvironmentId: authentication.environment.id, + }; + const args = { + select: { + id: true, + friendlyId: true, + streamBasinName: true, + parentTaskRun: { + select: { + friendlyId: true, + streamBasinName: true, }, - rootTaskRun: { - select: { - friendlyId: true, - streamBasinName: true, - }, + }, + rootTaskRun: { + select: { + friendlyId: true, + streamBasinName: true, }, }, }, - $replica - ); + }; + // Replica lag can null out a live run; a spurious 404 permanently fails the HEAD probe. + // Re-read the owning primary on a replica miss. + const run = await runStore.findRun(where, args, $replica); + return run ?? runStore.findRunOnPrimary(where, args); }, }, async ({ request, params, resource: run, authentication }) => { diff --git a/apps/webapp/app/routes/realtime.v1.streams.$runId.input.$streamId.ts b/apps/webapp/app/routes/realtime.v1.streams.$runId.input.$streamId.ts index 78fe332b8..3c67fdb94 100644 --- a/apps/webapp/app/routes/realtime.v1.streams.$runId.input.$streamId.ts +++ b/apps/webapp/app/routes/realtime.v1.streams.$runId.input.$streamId.ts @@ -39,22 +39,24 @@ const { action } = createActionApiRoute( }, }, async ({ request, params, authentication }) => { - const run = await runStore.findRun( - { - friendlyId: params.runId, - runtimeEnvironmentId: authentication.environment.id, + const where = { + friendlyId: params.runId, + runtimeEnvironmentId: authentication.environment.id, + }; + const args = { + select: { + id: true, + friendlyId: true, + completedAt: true, + realtimeStreamsVersion: true, + streamBasinName: true, }, - { - select: { - id: true, - friendlyId: true, - completedAt: true, - realtimeStreamsVersion: true, - streamBasinName: true, - }, - }, - $replica - ); + }; + // Replica lag can null out a live run; a spurious 404 permanently fails the input send. Re-read + // the owning primary on a replica miss. + const run = + (await runStore.findRun(where, args, $replica)) ?? + (await runStore.findRunOnPrimary(where, args)); if (!run) { return json({ ok: false, error: "Run not found" }, { status: 404 }); @@ -133,22 +135,23 @@ const loader = createLoaderApiRoute( allowJWT: true, corsStrategy: "all", findResource: async (params, auth) => { - return runStore.findRun( - { - friendlyId: params.runId, - runtimeEnvironmentId: auth.environment.id, - }, - { - include: { - batch: { - select: { - friendlyId: true, - }, + const where = { + friendlyId: params.runId, + runtimeEnvironmentId: auth.environment.id, + }; + const args = { + include: { + batch: { + select: { + friendlyId: true, }, }, }, - $replica - ); + }; + // Replica lag can null out a live run; a spurious 404 permanently fails the SSE tail (the + // client treats 404 as "stream gone"). Re-read the owning primary on a replica miss. + const run = await runStore.findRun(where, args, $replica); + return run ?? runStore.findRunOnPrimary(where, args); }, authorization: { action: "read", diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx index 9567ce084..dea9ed18e 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx @@ -12,20 +12,19 @@ export const action: ActionFunction = async ({ request, params }) => { const { projectParam, organizationSlug, envParam, runParam } = v3RunParamsSchema.parse(params); try { - const taskRun = await runStore.findRun( - { - friendlyId: runParam, - }, - { - select: { - id: true, - idempotencyKey: true, - taskIdentifier: true, - projectId: true, - runtimeEnvironmentId: true, - }, - } - ); + const resetSelect = { + id: true, + idempotencyKey: true, + taskIdentifier: true, + projectId: true, + runtimeEnvironmentId: true, + }; + let taskRun = await runStore.findRun({ friendlyId: runParam }, { select: resetSelect }); + if (!taskRun) { + // Read-your-writes: a just-created run may not have replicated. Re-read the owning primary + // before 404ing — this null gates the reset mutation below (mirrors cancel/replay). + taskRun = await runStore.findRunOnPrimary({ friendlyId: runParam }, { select: resetSelect }); + } if (!taskRun) { return jsonWithErrorMessage({}, request, "Run not found"); diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.sessions.$sessionId.$io.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.sessions.$sessionId.$io.ts index c001d66d5..062b75503 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.sessions.$sessionId.$io.ts +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.sessions.$sessionId.$io.ts @@ -1,6 +1,6 @@ import { type LoaderFunctionArgs } from "@remix-run/server-runtime"; import { z } from "zod"; -import { $replica } from "~/db.server"; +import { $replica, prisma } from "~/db.server"; import { runStore } from "~/v3/runStore.server"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; @@ -8,7 +8,7 @@ import { getRequestAbortSignal } from "~/services/httpAsyncStorage.server"; import { S2RealtimeStreams } from "~/services/realtime/s2realtimeStreams.server"; import { canonicalSessionAddressingKey, - resolveSessionByIdOrExternalId, + resolveSessionWithWriterFallback, } from "~/services/realtime/sessions.server"; import { getRealtimeStreamInstance } from "~/services/realtime/v1StreamsGlobal.server"; import { requireUserId } from "~/services/session.server"; @@ -51,22 +51,27 @@ export async function loader({ request, params }: LoaderFunctionArgs) { // Verify the run lives in this environment — keeps callers from // subscribing to arbitrary sessions via `/runs/$runParam/...`. - const run = await runStore.findRun( - { - friendlyId: runParam, - runtimeEnvironmentId: environment.id, - }, - { - select: { id: true, friendlyId: true }, - }, - $replica - ); + const runWhere = { + friendlyId: runParam, + runtimeEnvironmentId: environment.id, + }; + const runArgs = { + select: { id: true, friendlyId: true }, + }; + // Replica lag can null out a live run; a spurious 404 breaks the dashboard Agent tab subscription + // (useRealtimeStream surfaces the error and does not auto-retry). Re-read the primary on a miss. + const run = + (await runStore.findRun(runWhere, runArgs, $replica)) ?? + (await runStore.findRunOnPrimary(runWhere, runArgs)); if (!run) { return new Response("Run not found", { status: 404 }); } - const session = await resolveSessionByIdOrExternalId($replica, environment.id, sessionId); + // Replica lag can null out a just-created session; a spurious 404 breaks the dashboard Agent tab + // subscription (useRealtimeStream surfaces the error and does not auto-retry). Resolve replica-first + // with a writer fallback — the same helper the sibling `.in/append` route uses. + const session = await resolveSessionWithWriterFallback(environment.id, sessionId); if (!session) { return new Response("Session not found", { status: 404 }); @@ -76,10 +81,12 @@ export async function loader({ request, params }: LoaderFunctionArgs) { // this environment is enough to subscribe to any session in the same // environment — defeats the point of scoping subscriptions through the // run route. SessionRun.runId is indexed (@unique), so this is cheap. - const linkedSessionRun = await $replica.sessionRun.findFirst({ - where: { runId: run.id, sessionId: session.id }, - select: { id: true }, - }); + // Replica lag can null out the just-created run↔session linkage row; a spurious 404 breaks the + // dashboard Agent tab subscription (client does not auto-retry). Re-read the primary on a miss. + const linkWhere = { runId: run.id, sessionId: session.id }; + const linkedSessionRun = + (await $replica.sessionRun.findFirst({ where: linkWhere, select: { id: true } })) ?? + (await prisma.sessionRun.findFirst({ where: linkWhere, select: { id: true } })); if (!linkedSessionRun) { return new Response("Session not found for run", { status: 404 }); diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.$streamId.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.$streamId.ts index 6c50b0f14..4fbb309e2 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.$streamId.ts +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.$streamId.ts @@ -45,21 +45,23 @@ export async function loader({ request, params }: LoaderFunctionArgs) { return new Response("Environment not found", { status: 404 }); } - const run = await runStore.findRun( - { - friendlyId: runId, - runtimeEnvironmentId: environment.id, + const runWhere = { + friendlyId: runId, + runtimeEnvironmentId: environment.id, + }; + const runArgs = { + select: { + id: true, + friendlyId: true, + realtimeStreamsVersion: true, + streamBasinName: true, }, - { - select: { - id: true, - friendlyId: true, - realtimeStreamsVersion: true, - streamBasinName: true, - }, - }, - $replica - ); + }; + // Replica lag can null out a live run; a spurious 404 breaks the dashboard Agent tab subscription + // (useRealtimeStream surfaces the error and does not auto-retry). Re-read the primary on a miss. + const run = + (await runStore.findRun(runWhere, runArgs, $replica)) ?? + (await runStore.findRunOnPrimary(runWhere, runArgs)); if (!run) { return new Response("Run not found", { status: 404 }); diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.input.$streamId.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.input.$streamId.ts index 1ecc7819c..7eae8f6fd 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.input.$streamId.ts +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.input.$streamId.ts @@ -47,21 +47,23 @@ export async function loader({ request, params }: LoaderFunctionArgs) { return new Response("Environment not found", { status: 404 }); } - const run = await runStore.findRun( - { - friendlyId: runId, - runtimeEnvironmentId: environment.id, + const runWhere = { + friendlyId: runId, + runtimeEnvironmentId: environment.id, + }; + const runArgs = { + select: { + id: true, + friendlyId: true, + realtimeStreamsVersion: true, + streamBasinName: true, }, - { - select: { - id: true, - friendlyId: true, - realtimeStreamsVersion: true, - streamBasinName: true, - }, - }, - $replica - ); + }; + // Replica lag can null out a live run; a spurious 404 breaks the dashboard Agent tab input-stream + // subscription (useRealtimeStream surfaces the error, no auto-retry). Re-read the primary on a miss. + const run = + (await runStore.findRun(runWhere, runArgs, $replica)) ?? + (await runStore.findRunOnPrimary(runWhere, runArgs)); if (!run) { return new Response("Run not found", { status: 404 }); diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.streams.$streamKey/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.streams.$streamKey/route.tsx index 3d8cec515..433816134 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.streams.$streamKey/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.streams.$streamKey/route.tsx @@ -60,18 +60,22 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { throw new Response("Not Found", { status: 404 }); } - const run = await runStore.findRun( - { friendlyId: runParam, projectId: project.id }, - { - select: { - id: true, - friendlyId: true, - realtimeStreamsVersion: true, - streamBasinName: true, - runtimeEnvironmentId: true, - }, - } - ); + const runWhere = { friendlyId: runParam, projectId: project.id }; + const runArgs = { + select: { + id: true, + friendlyId: true, + realtimeStreamsVersion: true, + streamBasinName: true, + runtimeEnvironmentId: true, + }, + }; + // Client-less findRun defaults to the read replica; replica lag can null out a live run and 404 a + // valid stream-viewer request (useRealtimeStream surfaces the error, no auto-retry). Re-read the + // owning primary on a replica miss. + const run = + (await runStore.findRun(runWhere, runArgs)) ?? + (await runStore.findRunOnPrimary(runWhere, runArgs)); if (!run) { throw new Response("Not Found", { status: 404 }); diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.$waitpointFriendlyId.complete/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.$waitpointFriendlyId.complete/route.tsx index 16592ba33..e1ed1b2cc 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.$waitpointFriendlyId.complete/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.$waitpointFriendlyId.complete/route.tsx @@ -81,7 +81,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const waitpointId = WaitpointId.toId(waitpointFriendlyId); - const waitpoint = await runStore.findWaitpoint({ + let waitpoint = await runStore.findWaitpoint({ select: { projectId: true, environmentId: true, @@ -90,6 +90,14 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { id: waitpointId, }, }); + if (!waitpoint) { + // Read-your-writes: a just-minted token may not have replicated. Re-read the owning primary + // before the auth guard / "No waitpoint found" (mirrors the token complete/callback routes). + waitpoint = await runStore.findWaitpointOnPrimary({ + select: { projectId: true, environmentId: true }, + where: { id: waitpointId }, + }); + } if (waitpoint?.projectId !== project.id) { return redirectWithErrorMessage( diff --git a/apps/webapp/app/routes/resources.taskruns.$runParam.cancel.ts b/apps/webapp/app/routes/resources.taskruns.$runParam.cancel.ts index ce74becc7..cf6b03177 100644 --- a/apps/webapp/app/routes/resources.taskruns.$runParam.cancel.ts +++ b/apps/webapp/app/routes/resources.taskruns.$runParam.cancel.ts @@ -80,21 +80,26 @@ export const action = dashboardAction( // The project-scope + membership auth is a control-plane concern resolved // separately below; joining project/organization here is a cross-DB join // that returns nothing once the run lives in the run-ops DB. - const taskRun = await runStore.findRun( - { friendlyId: runParam }, - { - select: { - id: true, - engine: true, - status: true, - friendlyId: true, - taskEventStore: true, - createdAt: true, - completedAt: true, - projectId: true, - }, - } - ); + const cancelRunSelect = { + id: true, + engine: true, + status: true, + friendlyId: true, + taskEventStore: true, + createdAt: true, + completedAt: true, + projectId: true, + }; + let taskRun = await runStore.findRun({ friendlyId: runParam }, { select: cancelRunSelect }); + if (!taskRun) { + // Read-your-writes: a just-created run may not have replicated. Re-read the owning primary + // before treating it as absent (mirrors resolveRunOrganizationId's primary fallback above). + taskRun = await runStore.findRun( + { friendlyId: runParam }, + { select: cancelRunSelect }, + prisma + ); + } // Project-scope + membership auth is control-plane only — keyed by the // run's projectId. A miss is treated as not-found (mirrors the old where). diff --git a/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts b/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts index 8074a3d3b..d6fe04be9 100644 --- a/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts +++ b/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts @@ -323,7 +323,12 @@ export const action = dashboardAction( try { // Run-ops read keyed by friendlyId only; membership auth is re-checked on the // control plane below, keyed off the resolved run's projectId. - const pgRun = await runStore.findRun({ friendlyId: runParam }); + let pgRun = await runStore.findRun({ friendlyId: runParam }); + if (!pgRun) { + // Read-your-writes: a just-created run may not have replicated. Re-read the owning primary + // before falling back to the mollifier buffer (mirrors resolveRunOrganizationId above). + pgRun = await runStore.findRun({ friendlyId: runParam }, prisma); + } // Mollifier read-fallback: if the original isn't in PG yet, synthesise a // TaskRun from the buffered snapshot. Needs project/org/env slugs for the diff --git a/apps/webapp/app/routes/sync.traces.runs.$traceId.ts b/apps/webapp/app/routes/sync.traces.runs.$traceId.ts index e89772a10..5fd1f7c65 100644 --- a/apps/webapp/app/routes/sync.traces.runs.$traceId.ts +++ b/apps/webapp/app/routes/sync.traces.runs.$traceId.ts @@ -23,7 +23,7 @@ export async function loader({ params, request }: LoaderFunctionArgs) { return new Response("No user found in cookie", { status: 401 }); } - const run = await runStore.findRun( + let run = await runStore.findRun( { traceId, }, @@ -35,6 +35,15 @@ export async function loader({ params, request }: LoaderFunctionArgs) { $replica ); + if (!run) { + // Read-your-writes: a just-created run may not have replicated yet. Re-read the owning + // primary before 404ing so a live run's realtime trace feed isn't spuriously not-found. + run = await runStore.findRunOnPrimary( + { traceId }, + { select: { runtimeEnvironmentId: true } } + ); + } + if (!run) { return new Response("No run found", { status: 404 }); } diff --git a/apps/webapp/app/runEngine/concerns/idempotencyKeys.server.ts b/apps/webapp/app/runEngine/concerns/idempotencyKeys.server.ts index ea50261a5..f6696865e 100644 --- a/apps/webapp/app/runEngine/concerns/idempotencyKeys.server.ts +++ b/apps/webapp/app/runEngine/concerns/idempotencyKeys.server.ts @@ -1,5 +1,5 @@ -import { RunId } from "@trigger.dev/core/v3/isomorphic"; -import type { PrismaClientOrTransaction, TaskRun } from "@trigger.dev/database"; +import { ownerEngine, RunId } from "@trigger.dev/core/v3/isomorphic"; +import type { PrismaClientOrTransaction, TaskRun, Waitpoint } from "@trigger.dev/database"; import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; import { resolveIdempotencyKeyTTL } from "~/utils/idempotencyKeys.server"; @@ -8,7 +8,8 @@ import type { RunEngine } from "~/v3/runEngine.server"; import { shouldIdempotencyKeyBeCleared } from "~/v3/taskStatus"; import { getMollifierBuffer } from "~/v3/mollifier/mollifierBuffer.server"; import { findRunByIdWithMollifierFallback } from "~/v3/mollifier/readFallback.server"; -import { claimOrAwait } from "~/v3/mollifier/idempotencyClaim.server"; +import { claimOrAwait, resetResolvedClaim } from "~/v3/mollifier/idempotencyClaim.server"; +import { computeClaimTtlSeconds } from "~/v3/mollifier/claimTtl"; import { makeResolveMollifierFlag } from "~/v3/mollifier/mollifierGate.server"; import { runStore } from "~/v3/runStore.server"; import { runOpsLegacyPrisma, runOpsNewPrisma } from "~/db.server"; @@ -24,6 +25,13 @@ import type { TraceEventConcern, TriggerTaskRequest } from "../types"; // handleTriggerRequest. const resolveOrgMollifierFlag = makeResolveMollifierFlag(); +// Cap on the claim-loser recreate re-acquisition loop (see +// reacquireClearedGlobalWinner). Each pass reopens a stale resolved slot and +// re-enters the claim; bounded so a pathological stream of expired/failed +// winners can't spin forever. On exhaustion we fall open to the create with +// PG's unique index as the backstop. +const MAX_CLEARED_WINNER_REACQUIRES = 5; + // Claim ownership context returned to the caller when the // IdempotencyKeyConcern won a pre-gate claim. Caller MUST publish the // winning runId on pipeline success (`publishClaim`) or release the @@ -173,6 +181,14 @@ export class IdempotencyKeyConcern { } ); + // `global`-scope (or scope-absent) keys under the split have no per-run salt, so the Redis claim is + // their only cross-DB dedup mutex. Computed here (not just in the claim block below) because the + // expired/failed clear-and-recreate path must serialise through it too. + const idempotencyKeyScope = request.body.options?.idempotencyKeyOptions?.scope; + const globalUnderSplit = + (idempotencyKeyScope === "global" || idempotencyKeyScope === undefined) && + (await isSplitEnabled()); + const existingRun = idempotencyKey ? await runStore.findRun( { @@ -209,107 +225,37 @@ export class IdempotencyKeyConcern { } if (existingRun) { - // The idempotency key has expired - if (existingRun.idempotencyKeyExpiresAt && existingRun.idempotencyKeyExpiresAt < new Date()) { - logger.debug("[TriggerTaskService][call] Idempotency key has expired", { - idempotencyKey: request.options?.idempotencyKey, - run: existingRun, + const handled = await this.handleExistingRun(request, parentStore, existingRun, { + idempotencyKey, + idempotencyKeyExpiresAt, + dedupClient, + }); + // A LIVE cached hit (or andWait waitpoint wiring) is terminal. + if (handled.isCached) { + return handled; + } + // isCached === false → the existing run was EXPIRED/FAILED, so handleExistingRun cleared its key + // and we must recreate. For a global-scope key under split that recreate has to be claim- + // serialised too — otherwise two concurrent cross-residency recreates each create a run the + // per-DB unique index can't dedup (the same hole reacquireClearedGlobalWinner closes on the + // claim-loser path). Non-split / non-global: the plain unserialised recreate is safe. + if (globalUnderSplit) { + return await this.reacquireClearedGlobalWinner(request, parentStore, { + idempotencyKey, + idempotencyKeyExpiresAt, + dedupClient, + ttlSeconds: computeClaimTtlSeconds({ + keyExpiresAt: idempotencyKeyExpiresAt, + now: Date.now(), + minTtlSeconds: env.TRIGGER_MOLLIFIER_CLAIM_MIN_TTL_SECONDS, + maxTtlSeconds: env.TRIGGER_MOLLIFIER_CLAIM_TTL_SECONDS, + }), + clearedRunId: existingRun.friendlyId, + safetyNetMs: env.TRIGGER_MOLLIFIER_CLAIM_WAIT_MS, + pollStepMs: env.TRIGGER_MOLLIFIER_CLAIM_POLL_MS, }); - - // Update the existing run to remove the idempotency key - await runStore.clearIdempotencyKey( - { byId: { runId: existingRun.id, idempotencyKey } }, - dedupClient - ); - - return { isCached: false, idempotencyKey, idempotencyKeyExpiresAt }; } - - // If the existing run failed or was expired, we clear the key and do a new run - if (shouldIdempotencyKeyBeCleared(existingRun.status)) { - logger.debug("[TriggerTaskService][call] Idempotency key should be cleared", { - idempotencyKey: request.options?.idempotencyKey, - runStatus: existingRun.status, - runId: existingRun.id, - }); - - // Update the existing run to remove the idempotency key - await runStore.clearIdempotencyKey( - { byId: { runId: existingRun.id, idempotencyKey } }, - dedupClient - ); - - return { isCached: false, idempotencyKey, idempotencyKeyExpiresAt }; - } - - // We have an idempotent run, so we return it - const parentRunId = request.body.options?.parentRunId; - const resumeParentOnCompletion = request.body.options?.resumeParentOnCompletion; - - //We're using `andWait` so we need to block the parent run with a waitpoint - if (resumeParentOnCompletion && parentRunId) { - // `parentRunId` comes from the request body and isn't re-validated - // here, so confirm the parent run is in the caller's environment - // before wiring a waitpoint against it. - const parentRunInternalId = RunId.fromFriendlyId(parentRunId); - const parentRunInCallerEnv = await runStore.findRun( - { - id: parentRunInternalId, - runtimeEnvironmentId: request.environment.id, - }, - { select: { id: true } }, - this.prisma - ); - if (!parentRunInCallerEnv) { - throw new ServiceValidationError("Parent run not found in the calling environment", 404); - } - - // Get or create waitpoint lazily (existing run may not have one if it was standalone) - let associatedWaitpoint = existingRun.associatedWaitpoint; - if (!associatedWaitpoint) { - associatedWaitpoint = await this.engine.getOrCreateRunWaitpoint({ - runId: existingRun.id, - projectId: request.environment.projectId, - environmentId: request.environment.id, - }); - } - - await this.traceEventConcern.traceIdempotentRun( - request, - parentStore, - { - existingRun, - idempotencyKey, - incomplete: associatedWaitpoint.status === "PENDING", - isError: associatedWaitpoint.outputIsError, - }, - async (event) => { - const spanId = - request.options?.parentAsLinkType === "replay" - ? event.spanId - : event.traceparent?.spanId - ? `${event.traceparent.spanId}:${event.spanId}` - : event.spanId; - - await this.engine.blockRunWithWaitpoint({ - runId: parentRunInternalId, - waitpoints: associatedWaitpoint!.id, - spanIdToComplete: spanId, - batch: request.options?.batchId - ? { - id: request.options.batchId, - index: request.options.batchIndex ?? 0, - } - : undefined, - projectId: request.environment.projectId, - organizationId: request.environment.organizationId, - tx: dedupClient, - }); - } - ); - } - - return { isCached: true, run: existingRun }; + return handled; } // Pre-gate claim — closes the PG+buffer race during gate transition. @@ -325,28 +271,41 @@ export class IdempotencyKeyConcern { // claim's Redis SETNX keeps its RTT off the hot path for those requests // during staged rollout. The org-flag check is a pure in-memory read of // `Organization.featureFlags`, no DB query. + // + // Under the run-ops split the claim ALSO acts as the only cross-DB mutex a + // `global`-scope key has: that key is per (environment, task), so it can be + // triggered concurrently from two parents on DIFFERENT physical DBs where + // each probe misses and the per-DB unique index can't enforce uniqueness. + // For that case the claim is eligible regardless of the per-org flag AND of + // resumeParentOnCompletion (the loser wires its parent waitpoint against the + // winner in the resolved branch below). An absent scope (pre-hashed key / + // older SDK) is treated conservatively as possibly-global — harmless for a + // real run/attempt key, whose hash already embeds the parent id so two + // parents mint DISTINCT keys that never share a claim slot. + // (idempotencyKeyScope / globalUnderSplit are computed above — they also gate the expired/failed + // recreate serialisation.) const claimEligible = - !request.body.options?.resumeParentOnCompletion && !request.body.options?.debounce && !request.options?.oneTimeUseToken && - (await resolveOrgMollifierFlag({ - envId: request.environment.id, - orgId: request.environment.organizationId, - taskId: request.taskId, - orgFeatureFlags: - (request.environment.organization?.featureFlags as - | Record - | null - | undefined) ?? null, - })); + (globalUnderSplit || + (!request.body.options?.resumeParentOnCompletion && + (await resolveOrgMollifierFlag({ + envId: request.environment.id, + orgId: request.environment.organizationId, + taskId: request.taskId, + orgFeatureFlags: + (request.environment.organization?.featureFlags as + | Record + | null + | undefined) ?? null, + })))); if (claimEligible) { - const ttlSeconds = Math.max( - 1, - Math.min( - env.TRIGGER_MOLLIFIER_CLAIM_TTL_SECONDS, - Math.ceil((idempotencyKeyExpiresAt.getTime() - Date.now()) / 1000) - ) - ); + const ttlSeconds = computeClaimTtlSeconds({ + keyExpiresAt: idempotencyKeyExpiresAt, + now: Date.now(), + minTtlSeconds: env.TRIGGER_MOLLIFIER_CLAIM_MIN_TTL_SECONDS, + maxTtlSeconds: env.TRIGGER_MOLLIFIER_CLAIM_TTL_SECONDS, + }); const outcome = await claimOrAwait({ envId: request.environment.id, taskIdentifier: request.taskId, @@ -356,6 +315,44 @@ export class IdempotencyKeyConcern { pollStepMs: env.TRIGGER_MOLLIFIER_CLAIM_POLL_MS, }); if (outcome.kind === "resolved") { + // Global-under-split loser: the winner lives on ITS parent's DB, which + // may differ from this loser's parent DB. Resolve the winner by id + // across both DBs (classify the winner friendlyId → NEW/LEGACY client, + // then let the router route+fall-back by id-shape) and feed it through + // the same existing-run handling the PG-hit path uses — so expiry-clear, + // status-clear, and the resumeParentOnCompletion waitpoint wiring all + // apply to the loser exactly as they would to a plain cached hit. + if (globalUnderSplit) { + const winner = await this.resolveWinnerAcrossDbs(outcome.runId, request.environment.id); + if (winner) { + const resolved = await this.handleExistingRun(request, parentStore, winner, { + idempotencyKey, + idempotencyKeyExpiresAt, + dedupClient, + }); + // A LIVE winner (cached hit, or andWait waitpoint wired) is terminal. + if (resolved.isCached) { + return resolved; + } + // CRITICAL (CodeRabbit): the resolved winner was EXPIRED or FAILED, so + // handleExistingRun cleared its key and would have us CREATE a new run. + // The initial create is serialised by the claim, but this clear-and- + // recreate is not — and under the split the per-DB unique index can't + // dedup a cross-residency recreate, so two concurrent losers clearing + // the SAME winner would each create → duplicate run. Re-serialise the + // recreate through the claim so exactly one caller recreates (and + // publishes) while the rest resolve to the fresh run. + return await this.reacquireClearedGlobalWinner(request, parentStore, { + idempotencyKey, + idempotencyKeyExpiresAt, + dedupClient, + ttlSeconds, + clearedRunId: outcome.runId, + safetyNetMs: env.TRIGGER_MOLLIFIER_CLAIM_WAIT_MS, + pollStepMs: env.TRIGGER_MOLLIFIER_CLAIM_POLL_MS, + }); + } + } // Another concurrent trigger committed first. Re-resolve via the // existing checks: writer-side PG findFirst first (defeats // replica lag), then buffer fallback for the buffered case. @@ -421,4 +418,238 @@ export class IdempotencyKeyConcern { return { isCached: false, idempotencyKey, idempotencyKeyExpiresAt }; } + + // Resolve an already-existing idempotent run: honour key expiry / status + // clearing, and for `andWait` (resumeParentOnCompletion) block the calling + // parent on the run's waitpoint. Extracted so both the PG-hit path and the + // cross-DB claim-loser path resolve an existing run identically. + private async handleExistingRun( + request: TriggerTaskRequest, + parentStore: string | undefined, + existingRun: TaskRun & { associatedWaitpoint?: Waitpoint | null }, + ctx: { + idempotencyKey: string; + idempotencyKeyExpiresAt: Date; + dedupClient: PrismaClientOrTransaction; + } + ): Promise { + const { idempotencyKey, idempotencyKeyExpiresAt, dedupClient } = ctx; + + // The idempotency key has expired + if (existingRun.idempotencyKeyExpiresAt && existingRun.idempotencyKeyExpiresAt < new Date()) { + logger.debug("[TriggerTaskService][call] Idempotency key has expired", { + idempotencyKey: request.options?.idempotencyKey, + run: existingRun, + }); + + // Update the existing run to remove the idempotency key + await runStore.clearIdempotencyKey( + { byId: { runId: existingRun.id, idempotencyKey } }, + dedupClient + ); + + return { isCached: false, idempotencyKey, idempotencyKeyExpiresAt }; + } + + // If the existing run failed or was expired, we clear the key and do a new run + if (shouldIdempotencyKeyBeCleared(existingRun.status)) { + logger.debug("[TriggerTaskService][call] Idempotency key should be cleared", { + idempotencyKey: request.options?.idempotencyKey, + runStatus: existingRun.status, + runId: existingRun.id, + }); + + // Update the existing run to remove the idempotency key + await runStore.clearIdempotencyKey( + { byId: { runId: existingRun.id, idempotencyKey } }, + dedupClient + ); + + return { isCached: false, idempotencyKey, idempotencyKeyExpiresAt }; + } + + // We have an idempotent run, so we return it + const parentRunId = request.body.options?.parentRunId; + const resumeParentOnCompletion = request.body.options?.resumeParentOnCompletion; + + //We're using `andWait` so we need to block the parent run with a waitpoint + if (resumeParentOnCompletion && parentRunId) { + // `parentRunId` comes from the request body and isn't re-validated + // here, so confirm the parent run is in the caller's environment + // before wiring a waitpoint against it. + const parentRunInternalId = RunId.fromFriendlyId(parentRunId); + const parentRunInCallerEnv = await runStore.findRun( + { + id: parentRunInternalId, + runtimeEnvironmentId: request.environment.id, + }, + { select: { id: true } }, + this.prisma + ); + if (!parentRunInCallerEnv) { + throw new ServiceValidationError("Parent run not found in the calling environment", 404); + } + + // Get or create waitpoint lazily (existing run may not have one if it was standalone) + let associatedWaitpoint = existingRun.associatedWaitpoint; + if (!associatedWaitpoint) { + associatedWaitpoint = await this.engine.getOrCreateRunWaitpoint({ + runId: existingRun.id, + projectId: request.environment.projectId, + environmentId: request.environment.id, + }); + } + + await this.traceEventConcern.traceIdempotentRun( + request, + parentStore, + { + existingRun, + idempotencyKey, + incomplete: associatedWaitpoint.status === "PENDING", + isError: associatedWaitpoint.outputIsError, + }, + async (event) => { + const spanId = + request.options?.parentAsLinkType === "replay" + ? event.spanId + : event.traceparent?.spanId + ? `${event.traceparent.spanId}:${event.spanId}` + : event.spanId; + + await this.engine.blockRunWithWaitpoint({ + runId: parentRunInternalId, + waitpoints: associatedWaitpoint!.id, + spanIdToComplete: spanId, + batch: request.options?.batchId + ? { + id: request.options.batchId, + index: request.options.batchIndex ?? 0, + } + : undefined, + projectId: request.environment.projectId, + organizationId: request.environment.organizationId, + tx: dedupClient, + }); + } + ); + } + + return { isCached: true, run: existingRun }; + } + + // Re-serialise a cross-DB recreate through the claim after a claim-loser's + // resolved winner turned out to be EXPIRED / FAILED (its key was cleared by + // handleExistingRun). Without this, concurrent losers clearing the same + // winner each create a new run on their own DB — the per-DB unique index + // can't dedup a cross-residency pair, so the initial-create's serialisation + // is lost on the recreate. Each pass: compare-and-delete the stale resolved + // slot (keyed on the cleared runId — never an unconditional DEL, so a + // reacquirer that already re-published a NEW winner is not wiped), then + // re-enter claimOrAwait. Exactly one caller wins the re-claim and recreates + // (returning its claim to publish); the rest resolve to the fresh run. If a + // re-claim resolves to ANOTHER cleared winner we advance and loop, bounded + // by MAX_CLEARED_WINNER_REACQUIRES; on exhaustion (or an unfindable + // resolution) we fail CLOSED with a retryable 503 so the SDK retry re-serialises, + // rather than fall open to an unserialised cross-DB create. + private async reacquireClearedGlobalWinner( + request: TriggerTaskRequest, + parentStore: string | undefined, + ctx: { + idempotencyKey: string; + idempotencyKeyExpiresAt: Date; + dedupClient: PrismaClientOrTransaction; + ttlSeconds: number; + clearedRunId: string; + safetyNetMs: number; + pollStepMs: number; + } + ): Promise { + const { idempotencyKey, idempotencyKeyExpiresAt, dedupClient, ttlSeconds } = ctx; + const claimInput = { + envId: request.environment.id, + taskIdentifier: request.taskId, + idempotencyKey, + }; + let staleRunId = ctx.clearedRunId; + for (let attempt = 0; attempt < MAX_CLEARED_WINNER_REACQUIRES; attempt++) { + await resetResolvedClaim({ ...claimInput, runId: staleRunId }); + + const outcome = await claimOrAwait({ + ...claimInput, + ttlSeconds, + safetyNetMs: ctx.safetyNetMs, + pollStepMs: ctx.pollStepMs, + }); + + if (outcome.kind === "timed_out") { + throw new ServiceValidationError("Idempotency claim resolution timed out", 503); + } + if (outcome.kind === "claimed") { + // We own the recreate. Caller MUST publish the new runId / release on error. + return { + isCached: false, + idempotencyKey, + idempotencyKeyExpiresAt, + claim: { ...claimInput, token: outcome.token }, + }; + } + // resolved: another caller won the recreate. Honour it like any cached + // hit (incl. andWait wiring). If it too was cleared, advance and loop. + const winner = await this.resolveWinnerAcrossDbs(outcome.runId, request.environment.id); + if (!winner) { + logger.warn("idempotency reacquire resolved but runId not findable", { + envId: request.environment.id, + taskIdentifier: request.taskId, + claimedRunId: outcome.runId, + }); + break; + } + const resolved = await this.handleExistingRun(request, parentStore, winner, { + idempotencyKey, + idempotencyKeyExpiresAt, + dedupClient, + }); + if (resolved.isCached) { + return resolved; + } + staleRunId = outcome.runId; + } + // Exhausted the bounded reacquires (or the winner was unfindable). Rather than fall through to an + // UNSERIALISED create — which under global-scope-split can dual-create across DBs (the per-DB unique + // index can't dedup cross-residency) — fail closed with a retryable 503 so the SDK retry re-serialises + // through a fresh claim (mirrors the timed_out branch above). + throw new ServiceValidationError( + "Idempotency claim could not be re-serialised after repeated cleared winners", + 503 + ); + } + + // Resolve a claim winner (a run friendlyId) across both split DBs. Classify + // the id-shape to pick the writer client for read-your-writes, then read by + // id — the routing store routes to the owning store and falls back to the + // other, so a winner on either DB is found. Returns null when the id can't be + // classified or the row genuinely isn't there (caller falls through). + private async resolveWinnerAcrossDbs( + winnerFriendlyId: string, + environmentId: string + ): Promise<(TaskRun & { associatedWaitpoint?: Waitpoint | null }) | null> { + let internalId: string; + try { + internalId = RunId.fromFriendlyId(winnerFriendlyId); + } catch { + return null; + } + let client: PrismaClientOrTransaction; + try { + client = ownerEngine(internalId) === "NEW" ? runOpsNewPrisma : runOpsLegacyPrisma; + } catch { + client = this.prisma; + } + return runStore.findRun( + { id: internalId, runtimeEnvironmentId: environmentId }, + { include: { associatedWaitpoint: true } }, + client + ); + } } diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.ts b/apps/webapp/app/runEngine/services/triggerTask.server.ts index f4981fece..ff22a479a 100644 --- a/apps/webapp/app/runEngine/services/triggerTask.server.ts +++ b/apps/webapp/app/runEngine/services/triggerTask.server.ts @@ -752,7 +752,7 @@ export class RunEngineTriggerTaskService { // Pipeline returned successfully — publish the claim if we held // one. Waiters polling for our key resolve to this runId. if (idempotencyClaim && result?.run?.friendlyId) { - await publishMollifierClaim({ + const published = await publishMollifierClaim({ envId: idempotencyClaim.envId, taskIdentifier: idempotencyClaim.taskIdentifier, idempotencyKey: idempotencyClaim.idempotencyKey, @@ -760,6 +760,17 @@ export class RunEngineTriggerTaskService { runId: result.run.friendlyId, ttlSeconds: env.TRIGGER_MOLLIFIER_CLAIM_TTL_SECONDS, }); + if (!published) { + // Our claim expired mid-pipeline and another claimant took it, so this publish no-op'd: a + // different run is now canonical for this key while we return ours (a cross-DB dup under the + // split). Rare now the claim TTL is floored (C1); surfaced for monitoring pending auto- + // convergence (re-resolve the current winner + cancel this orphan). + logger.warn("mollifier claim publish no-op'd; winner lost the claim mid-pipeline", { + envId: idempotencyClaim.envId, + taskIdentifier: idempotencyClaim.taskIdentifier, + runId: result.run.friendlyId, + }); + } } return result; } catch (err) { diff --git a/apps/webapp/app/services/dashboardAgent.server.ts b/apps/webapp/app/services/dashboardAgent.server.ts index 14eb51fcb..039ee3c3a 100644 --- a/apps/webapp/app/services/dashboardAgent.server.ts +++ b/apps/webapp/app/services/dashboardAgent.server.ts @@ -212,7 +212,9 @@ export async function resolveRunCommit( environmentId: string, runFriendlyId: string ): Promise<{ sha: string; version: string; dirty: boolean } | null> { - const run = await runStore.findRun( + // Read-your-writes: a just-locked run's lockedToVersionId may not have replicated. Read the owning + // primary so a live, pinned run resolves its commit instead of silently falling back to branch head. + const run = await runStore.findRunOnPrimary( { friendlyId: runFriendlyId, runtimeEnvironmentId: environmentId }, { select: { lockedToVersionId: true } } ); diff --git a/apps/webapp/app/services/realtime/sessionRunManager.server.ts b/apps/webapp/app/services/realtime/sessionRunManager.server.ts index 5ff462154..10cd966d9 100644 --- a/apps/webapp/app/services/realtime/sessionRunManager.server.ts +++ b/apps/webapp/app/services/realtime/sessionRunManager.server.ts @@ -478,8 +478,10 @@ export async function swapSessionRun(params: SwapSessionRunParams): Promise { - // Use the read replica — this is a hot-path probe and stale-by-ms is - // fine. The append handler re-checks if it ends up reusing the runId. + // Use the read replica — hot-path probe, stale-by-ms is fine. The dangerous + // stale shape (looks-vanished → double-trigger) is closed by the writer re-probe + // in ensureRunForSession; a stale-non-final reuse self-heals via the durable S2 + // stream + next-append re-probe (there is no append-handler re-check). // `friendlyId` is fetched alongside `status` so the dead-run-detection // branch in `ensureRunForSession` can forward the public-form id as // `payload.previousRunId` without a second read. `Session.currentRunId` diff --git a/apps/webapp/app/v3/mollifier/claimTtl.ts b/apps/webapp/app/v3/mollifier/claimTtl.ts new file mode 100644 index 000000000..7fc8687ce --- /dev/null +++ b/apps/webapp/app/v3/mollifier/claimTtl.ts @@ -0,0 +1,12 @@ +// The claim is a serialization lock that must outlive the winner's create-and-publish pipeline. A short +// customer key TTL must NOT shrink it below a pipeline floor (else the claim expires mid-pipeline and a +// polling loser re-claims → cross-DB duplicate). Floor at `minTtlSeconds` independent of key TTL; cap at max. +export function computeClaimTtlSeconds(input: { + keyExpiresAt: Date; + now: number; + minTtlSeconds: number; + maxTtlSeconds: number; +}): number { + const keyTtlSeconds = Math.ceil((input.keyExpiresAt.getTime() - input.now) / 1000); + return Math.min(input.maxTtlSeconds, Math.max(input.minTtlSeconds, keyTtlSeconds)); +} diff --git a/apps/webapp/app/v3/mollifier/idempotencyClaim.server.ts b/apps/webapp/app/v3/mollifier/idempotencyClaim.server.ts index 47c9733c9..191ff6205 100644 --- a/apps/webapp/app/v3/mollifier/idempotencyClaim.server.ts +++ b/apps/webapp/app/v3/mollifier/idempotencyClaim.server.ts @@ -163,12 +163,14 @@ export async function publishClaim(input: { runId: string; ttlSeconds?: number; buffer?: MollifierBuffer | null; -}): Promise { +}): Promise { const buffer = input.buffer === undefined ? getMollifierBuffer() : input.buffer; - if (!buffer) return; + if (!buffer) return true; const ttlSeconds = input.ttlSeconds ?? DEFAULT_CLAIM_TTL_SECONDS; try { - await buffer.publishClaim({ + // false = compare-and-set no-op: a stale claimant (our TTL expired) already moved in, so our + // publish did NOT set the winner. The caller decides how to converge. + return await buffer.publishClaim({ envId: input.envId, taskIdentifier: input.taskIdentifier, idempotencyKey: input.idempotencyKey, @@ -182,6 +184,8 @@ export async function publishClaim(input: { taskIdentifier: input.taskIdentifier, err: err instanceof Error ? err.message : String(err), }); + // Unknown publish state (transient error) — the claim TTL is the safety net; don't signal a no-op. + return true; } } @@ -213,6 +217,39 @@ export async function releaseClaim(input: { } } +// Reopen a stale RESOLVED claim slot whose winner was cleared (expired / +// failed), so the claim-loser reacquire path can re-serialise a cross-DB +// recreate through a fresh claim. Compare-and-delete on the observed winner +// runId (buffer-side) so a concurrent reacquirer's freshly published winner +// is never wiped. Best-effort: on buffer-null / error we no-op and let the +// reacquire's claimOrAwait proceed (fail-open — the caller caps attempts and +// ultimately falls through to the create, PG unique index as backstop). +export async function resetResolvedClaim(input: { + envId: string; + taskIdentifier: string; + idempotencyKey: string; + runId: string; + buffer?: MollifierBuffer | null; +}): Promise { + const buffer = input.buffer === undefined ? getMollifierBuffer() : input.buffer; + if (!buffer) return false; + try { + return await buffer.resetResolvedClaim({ + envId: input.envId, + taskIdentifier: input.taskIdentifier, + idempotencyKey: input.idempotencyKey, + expectedRunId: input.runId, + }); + } catch (err) { + logger.warn("idempotency resolved-claim reset failed", { + envId: input.envId, + taskIdentifier: input.taskIdentifier, + err: err instanceof Error ? err.message : String(err), + }); + return false; + } +} + function defaultSleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } diff --git a/apps/webapp/app/v3/realtime/resolveBatchForRealtime.server.ts b/apps/webapp/app/v3/realtime/resolveBatchForRealtime.server.ts new file mode 100644 index 000000000..edd318f92 --- /dev/null +++ b/apps/webapp/app/v3/realtime/resolveBatchForRealtime.server.ts @@ -0,0 +1,24 @@ +import { prisma } from "~/db.server"; +import { runStore } from "~/v3/runStore.server"; + +type BatchStore = Pick; + +// The realtime batch route reads the batch client-less (replica), which can miss a just-created batch +// under replica lag. `shouldRetryNotFound` covers the zodfetch GET, but the Electric ShapeStream +// consumer (self-hosters) ignores `x-should-retry`, so re-read the owning primary on a miss — passing a +// non-replica writer flips each store leg to its own primary — to avoid a permanent 404. +export function resolveBatchTaskRunForRealtime( + friendlyId: string, + environmentId: string, + deps?: { store?: BatchStore; writer?: unknown } +) { + const store = deps?.store ?? runStore; + const writer = deps?.writer ?? prisma; + return store + .findBatchTaskRunByFriendlyId(friendlyId, environmentId) + .then( + (onReplica) => + onReplica ?? + store.findBatchTaskRunByFriendlyId(friendlyId, environmentId, undefined, writer as never) + ); +} diff --git a/apps/webapp/app/v3/services/batchTriggerV3.server.ts b/apps/webapp/app/v3/services/batchTriggerV3.server.ts index a50ba6374..fdb87d2d2 100644 --- a/apps/webapp/app/v3/services/batchTriggerV3.server.ts +++ b/apps/webapp/app/v3/services/batchTriggerV3.server.ts @@ -195,18 +195,23 @@ export class BatchTriggerV3Service extends BaseService { span.setAttribute("batchId", batchId); const dependentAttempt = body?.dependentAttempt - ? await this.runStore.findTaskRunAttempt({ - // Scope to the caller's environment (see dependentAttemptWhere). - where: dependentAttemptWhere(body.dependentAttempt, environment.id), - include: { - taskRun: { - select: { - id: true, - status: true, + ? await this.runStore.findTaskRunAttempt( + { + // Scope to the caller's environment (see dependentAttemptWhere). + where: dependentAttemptWhere(body.dependentAttempt, environment.id), + include: { + taskRun: { + select: { + id: true, + status: true, + }, }, }, }, - }) + // Read-your-writes: the dependent parent attempt's terminal state must be seen even + // when the read replica lags, so route this read to the owning primary. + this._prisma + ) : undefined; if ( diff --git a/apps/webapp/test/mollifierIdempotencyClaim.test.ts b/apps/webapp/test/mollifierIdempotencyClaim.test.ts index 1b093250f..84083087a 100644 --- a/apps/webapp/test/mollifierIdempotencyClaim.test.ts +++ b/apps/webapp/test/mollifierIdempotencyClaim.test.ts @@ -181,13 +181,13 @@ describe("publishClaim", () => { expect(buffer.publishClaim).toHaveBeenCalledOnce(); }); - it("no-op when buffer is null", async () => { + it("no buffer → true (nothing to converge, not a no-op'd publish)", async () => { await expect( publishClaim({ ...baseInput, token: "owner-token", runId: "run_X", buffer: null }) - ).resolves.toBeUndefined(); + ).resolves.toBe(true); }); - it("swallows errors so trigger pipeline isn't broken by Redis hiccups", async () => { + it("swallows errors so trigger pipeline isn't broken by Redis hiccups (returns true, unknown state)", async () => { const buffer = { publishClaim: vi.fn(async () => { throw new Error("ECONNREFUSED"); @@ -195,7 +195,7 @@ describe("publishClaim", () => { } as unknown as MollifierBuffer; await expect( publishClaim({ ...baseInput, token: "owner-token", runId: "run_X", buffer }) - ).resolves.toBeUndefined(); + ).resolves.toBe(true); }); }); diff --git a/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts b/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts index 7f53c7648..922b41cd2 100644 --- a/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts @@ -170,7 +170,9 @@ export class RunAttemptSystem { } public async resolveTaskRunContext(runId: string): Promise { - const run = await this.$.runStore.findRun( + // read-your-writes: a just-created/locked run may not be on a replica yet; read the owning primary + // so a live run's execution context resolves instead of throwing a spurious 404. + const run = await this.$.runStore.findRunOnPrimary( { id: runId, }, @@ -711,8 +713,8 @@ export class RunAttemptSystem { const completedAt = new Date(); - // Read current usage values to calculate new totals (safe under runLock) - const currentRun = await this.$.runStore.findRun( + // Read current usage totals on the owning primary (read-your-writes; a replica lags) + const currentRun = await this.$.runStore.findRunOnPrimary( { id: runId }, { select: { @@ -904,7 +906,8 @@ export class RunAttemptSystem { const failedAt = new Date(); const retryResult = await retryOutcomeFromCompletion( - this.$.readOnlyPrisma, + // read-your-writes: lock-time maxAttempts/lockedRetryConfig may not be on a replica yet + this.$.prisma, this.$.runStore, { runId, @@ -917,7 +920,9 @@ export class RunAttemptSystem { // Force requeue means it was crashed so the attempt span needs to be closed if (forceRequeue) { - const minimalRun = await this.$.runStore.findRun( + // read-your-writes: the run was just written in this flow; read the owning primary so the + // requeue event re-read cannot false-miss on a lagging replica (mirrors the :906 read). + const minimalRun = await this.$.runStore.findRunOnPrimary( { id: runId, }, @@ -1375,7 +1380,7 @@ export class RunAttemptSystem { // Calculate updated usage if we have attempt duration data let usageUpdate: { usageDurationMs: number; costInCents: number } | undefined; if (attemptDurationMs !== undefined) { - const currentRun = await this.$.runStore.findRun( + const currentRun = await this.$.runStore.findRunOnPrimary( { id: runId }, { select: { @@ -1589,8 +1594,8 @@ export class RunAttemptSystem { const truncatedError = this.#truncateTaskRunError(error); - // Read current usage values to calculate new totals - const currentRun = await this.$.runStore.findRun( + // Read current usage totals on the owning primary (read-your-writes; a replica lags) + const currentRun = await this.$.runStore.findRunOnPrimary( { id: runId }, { select: { diff --git a/internal-packages/run-engine/src/engine/systems/ttlSystem.ts b/internal-packages/run-engine/src/engine/systems/ttlSystem.ts index af6f41eac..0fb3b8387 100644 --- a/internal-packages/run-engine/src/engine/systems/ttlSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/ttlSystem.ts @@ -158,22 +158,26 @@ export class TtlSystem { const skipped: { runId: string; reason: string }[] = []; // Fetch all runs in a single query (no snapshot data needed) - const runs = await this.$.runStore.findRuns({ - where: { id: { in: runIds } }, - select: { - id: true, - spanId: true, - status: true, - lockedAt: true, - ttl: true, - taskEventStore: true, - createdAt: true, - associatedWaitpoint: { select: { id: true } }, - organizationId: true, - projectId: true, - runtimeEnvironmentId: true, + const runs = await this.$.runStore.findRuns( + { + where: { id: { in: runIds } }, + select: { + id: true, + spanId: true, + status: true, + lockedAt: true, + ttl: true, + taskEventStore: true, + createdAt: true, + associatedWaitpoint: { select: { id: true } }, + organizationId: true, + projectId: true, + runtimeEnvironmentId: true, + }, + // read-your-writes: the queue slot is already claimed; a lagging replica would orphan the run }, - }); + this.$.prisma + ); // Filter runs that can be expired const runsToExpire: typeof runs = []; diff --git a/internal-packages/run-store/src/runOpsStore.residencyMismatchFallback.test.ts b/internal-packages/run-store/src/runOpsStore.residencyMismatchFallback.test.ts deleted file mode 100644 index 8897c2956..000000000 --- a/internal-packages/run-store/src/runOpsStore.residencyMismatchFallback.test.ts +++ /dev/null @@ -1,296 +0,0 @@ -// RED→GREEN lock for RoutingRunStore.findRun's ON-MISS FAN-OUT for a CLASSIFIABLE id. -// -// THE BUG: findRun for a classifiable id routed to the SINGLE owning store (by id-shape -// classification) and returned whatever it gave — no on-miss fallback. When a run's PHYSICAL -// residency does not match its id-shape classification (e.g. a pre-#4154 27-char base62 run that -// lives on the NEW store but now classifies LEGACY), findRun routed to the wrong store, missed, -// and returned null → a spurious 404 — even though the run is physically present on the OTHER DB -// (and runs.list surfaces it). The unclassifiable path already fanned out NEW→LEGACY; this makes -// the classifiable path equally robust. -// -// Uses the REAL two-physical-DB split (heteroRunOpsPostgresTest: prisma14 = full/legacy on PG14, -// prisma17 = dedicated run-ops subset on PG17). NEVER mocked. The residency/classification MISMATCH -// is simulated deterministically by injecting a custom `classify` fn into the RoutingRunStore -// constructor — the physical row is written to the NEW store while `classify` reports its id LEGACY. - -import { heteroRunOpsPostgresTest } from "@internal/testcontainers"; -import { ownerEngine, type Residency } from "@trigger.dev/core/v3/isomorphic"; -import type { PrismaClient } from "@trigger.dev/database"; -import type { RunOpsPrismaClient } from "@internal/run-ops-database"; -import { describe, expect } from "vitest"; -import { PostgresRunStore } from "./PostgresRunStore.js"; -import { RoutingRunStore } from "./runOpsStore.js"; -import type { CreateRunInput, RunStore } from "./types.js"; - -// ownerEngine classifies by internal-id LENGTH/version char: 25 chars → cuid → LEGACY, -// a v1 body (version "1" at index 25) → run-ops id → NEW. -function cuidLegacy(seed: string): string { - return (seed + "c".repeat(25)).slice(0, 25); -} -function runOpsNew(seed: string): string { - return (seed.replace(/[^0-9a-v]/g, "0") + "k".repeat(24)).slice(0, 24) + "01"; -} - -function makeDedicatedStore(prisma17: RunOpsPrismaClient) { - return new PostgresRunStore({ - prisma: prisma17 as never, - readOnlyPrisma: prisma17 as never, - schemaVariant: "dedicated", - }); -} - -function makeLegacyStore(prisma14: PrismaClient) { - return new PostgresRunStore({ - prisma: prisma14, - readOnlyPrisma: prisma14, - schemaVariant: "legacy", - }); -} - -// Wrap a real store so findRun/findRunOnPrimary calls are COUNTED while every method still delegates -// to the REAL PostgresRunStore (this is instrumentation, not a behavior mock — the underlying reads, -// writes, getters all run for real). Lets us assert the FAST PATH does not touch the other store. -function countingReads( - inner: RunStore, - counts: { findRun: number; findRunOnPrimary: number } -): RunStore { - return new Proxy(inner, { - get(target, prop) { - // Read via target[prop] so getters (e.g. primaryReadClient) run with `this` = the real store. - const value = (target as unknown as Record)[prop]; - if (typeof value !== "function") return value; - if (prop === "findRun" || prop === "findRunOnPrimary") { - return (...args: unknown[]) => { - counts[prop as "findRun" | "findRunOnPrimary"] += 1; - return (value as (...a: unknown[]) => unknown).apply(target, args); - }; - } - return (value as (...a: unknown[]) => unknown).bind(target); - }, - }) as unknown as RunStore; -} - -async function seedLegacyEnvironment(prisma14: PrismaClient, suffix: string) { - const organization = await prisma14.organization.create({ - data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, - }); - const project = await prisma14.project.create({ - data: { - name: `Project ${suffix}`, - slug: `project-${suffix}`, - externalRef: `proj_${suffix}`, - organizationId: organization.id, - }, - }); - const environment = await prisma14.runtimeEnvironment.create({ - data: { - type: "DEVELOPMENT", - slug: "dev", - projectId: project.id, - organizationId: organization.id, - apiKey: `tr_dev_${suffix}`, - pkApiKey: `pk_dev_${suffix}`, - shortcode: `short_${suffix}`, - }, - }); - return { - organizationId: organization.id, - projectId: project.id, - runtimeEnvironmentId: environment.id, - environmentId: environment.id, - }; -} - -function buildCreateRunInput(params: { - runId: string; - friendlyId: string; - organizationId: string; - projectId: string; - runtimeEnvironmentId: string; -}): CreateRunInput { - return { - data: { - id: params.runId, - engine: "V2", - status: "PENDING", - friendlyId: params.friendlyId, - runtimeEnvironmentId: params.runtimeEnvironmentId, - environmentType: "DEVELOPMENT", - organizationId: params.organizationId, - projectId: params.projectId, - taskIdentifier: "my-task", - payload: "{}", - payloadType: "application/json", - traceContext: {}, - traceId: `trace_${params.runId}`, - spanId: `span_${params.runId}`, - queue: "task/my-task", - isTest: false, - taskEventStore: "taskEvent", - depth: 0, - }, - snapshot: { - engine: "V2", - executionStatus: "RUN_CREATED", - description: "Run was created", - runStatus: "PENDING", - environmentId: params.runtimeEnvironmentId, - environmentType: "DEVELOPMENT", - projectId: params.projectId, - organizationId: params.organizationId, - }, - }; -} - -// Insert a TaskRun row DIRECTLY onto the NEW (dedicated) store, bypassing routing, so we can force a -// residency/classification MISMATCH: the row is physically on #new while `classify` calls its id LEGACY. -async function insertRunOnNewStore( - prisma17: RunOpsPrismaClient, - params: { - runId: string; - friendlyId: string; - environmentId: string; - organizationId: string; - projectId: string; - } -) { - await prisma17.taskRun.create({ - data: { - id: params.runId, - engine: "V2", - status: "PENDING", - friendlyId: params.friendlyId, - runtimeEnvironmentId: params.environmentId, - environmentType: "DEVELOPMENT", - organizationId: params.organizationId, - projectId: params.projectId, - taskIdentifier: "my-task", - payload: "{}", - payloadType: "application/json", - traceContext: {}, - traceId: `trace_${params.runId}`, - spanId: `span_${params.runId}`, - queue: "task/my-task", - isTest: false, - taskEventStore: "taskEvent", - depth: 0, - }, - }); -} - -describe("RoutingRunStore.findRun — on-miss fan-out for a classifiable id (residency ≠ classification)", () => { - // ── THE BUG: a run physically on #new whose id classifies LEGACY must still be found ── - // Without the on-miss fallback, findRun routes to #legacy (per classification), misses, returns null. - heteroRunOpsPostgresTest( - "returns a #new-resident run whose id classifies LEGACY (owning-store miss → other-store fallback)", - async ({ prisma14, prisma17 }) => { - const env = await seedLegacyEnvironment(prisma14, "mm1"); - const newStore = makeDedicatedStore(prisma17); - const legacyStore = makeLegacyStore(prisma14); - - // A run-ops-shaped id (so ownerEngine would say NEW), but we FORCE classify → LEGACY to model - // a residency/classification mismatch: physically on #new, classified LEGACY. - const mismatchId = runOpsNew("mm1"); - const classify = (id: string): Residency => (id === mismatchId ? "LEGACY" : ownerEngine(id)); - const router = new RoutingRunStore({ new: newStore, legacy: legacyStore, classify }); - - await insertRunOnNewStore(prisma17, { - runId: mismatchId, - friendlyId: "run_mm1", - environmentId: env.environmentId, - organizationId: env.organizationId, - projectId: env.projectId, - }); - - // Physical residency sanity: on #new only. - expect(await prisma17.taskRun.findUnique({ where: { id: mismatchId } })).not.toBeNull(); - expect(await prisma14.taskRun.findUnique({ where: { id: mismatchId } })).toBeNull(); - - // classify → LEGACY routes to #legacy (miss); the fix falls back to #new and finds the run. - const byId = (await router.findRun({ id: mismatchId }, { select: { id: true } })) as Record< - string, - unknown - > | null; - expect(byId?.id).toBe(mismatchId); - - // Same on the read-your-writes primary variant (a caller-passed writer → findRunOnPrimary). - const byIdPrimary = (await router.findRun( - { id: mismatchId }, - { select: { id: true } }, - prisma14 - )) as Record | null; - expect(byIdPrimary?.id).toBe(mismatchId); - } - ); - - // ── FAST PATH: a run found in its CLASSIFIED store is a SINGLE read (no second-store probe) ── - heteroRunOpsPostgresTest( - "does NOT read the other store when the classified (owning) store hits", - async ({ prisma14, prisma17 }) => { - const env = await seedLegacyEnvironment(prisma14, "mm2"); - - // NEW-resident run-ops-id run: owning store = #new. Wrap #legacy to catch any stray probe. - const newCounts = { findRun: 0, findRunOnPrimary: 0 }; - const legacyCounts = { findRun: 0, findRunOnPrimary: 0 }; - const newStore = countingReads(makeDedicatedStore(prisma17), newCounts); - const legacyStore = countingReads(makeLegacyStore(prisma14), legacyCounts); - const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); - - const newId = runOpsNew("mm2n"); // classifies NEW - await router.createRun( - buildCreateRunInput({ - runId: newId, - friendlyId: "run_mm2_new", - organizationId: env.organizationId, - projectId: env.projectId, - runtimeEnvironmentId: env.runtimeEnvironmentId, - }) - ); - - const hit = (await router.findRun({ id: newId }, { select: { id: true } })) as Record< - string, - unknown - > | null; - expect(hit?.id).toBe(newId); - // Owning store read exactly once; the other store NOT touched (fast path preserved). - expect(newCounts.findRun).toBe(1); - expect(legacyCounts.findRun).toBe(0); - expect(legacyCounts.findRunOnPrimary).toBe(0); - - // Symmetric: a cuid run whose owning store is #legacy must not probe #new on a hit. - const legacyId = cuidLegacy("mm2l"); // classifies LEGACY - await router.createRun( - buildCreateRunInput({ - runId: legacyId, - friendlyId: "run_mm2_legacy", - organizationId: env.organizationId, - projectId: env.projectId, - runtimeEnvironmentId: env.runtimeEnvironmentId, - }) - ); - newCounts.findRun = 0; - legacyCounts.findRun = 0; - - const hitLegacy = (await router.findRun( - { id: legacyId }, - { select: { id: true } } - )) as Record | null; - expect(hitLegacy?.id).toBe(legacyId); - expect(legacyCounts.findRun).toBe(1); - expect(newCounts.findRun).toBe(0); - } - ); - - // ── A genuine miss on BOTH stores still returns null (fan-out exhausted) ── - heteroRunOpsPostgresTest( - "returns null when the run is on neither store", - async ({ prisma14, prisma17 }) => { - const newStore = makeDedicatedStore(prisma17); - const legacyStore = makeLegacyStore(prisma14); - const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); - expect( - await router.findRun({ id: cuidLegacy("ghost") }, { select: { id: true } }) - ).toBeNull(); - } - ); -}); diff --git a/internal-packages/run-store/src/runOpsStore.ts b/internal-packages/run-store/src/runOpsStore.ts index f6b613a39..d14ae5252 100644 --- a/internal-packages/run-store/src/runOpsStore.ts +++ b/internal-packages/run-store/src/runOpsStore.ts @@ -138,20 +138,6 @@ export class RoutingRunStore implements RunStore { return this.#routeOrNew(runId).runInTransaction(runId, fn); } - // A waitpoint WRITE co-locates with its run by id-shape (cuid → LEGACY, run-ops id → NEW, - // unclassifiable → LEGACY), mirroring how `blockRunWithWaitpointEdges` routes the edge by - // run id. The caller's `tx` is never forwarded: a routed write must run on the OWNING store's - // OWN client so the row lands in that store's database (same-store atomicity comes from the - // owning store opening its own transaction, never from a caller-supplied one). - #routeWaitpointWrite( - id: string | undefined, - _tx?: PrismaClientOrTransaction - ): { store: RunStore; tx?: PrismaClientOrTransaction } { - const store = - typeof id === "string" && this.#classifySafe(id) === "NEW" ? this.#new : this.#legacy; - return { store, tx: undefined }; - } - // Resolve which store ACTUALLY holds a waitpoint id: drain-on-read can relocate a cuid // waitpoint onto NEW while keeping its id, so probe the id-shape's home then the other. // `onPrimary` probes each store's own primary (read-your-writes callers; a fresh row may not @@ -243,9 +229,11 @@ export class RoutingRunStore implements RunStore { const onPrimary = readYourWrites(argsOrClient, _client); const id = idFromWhere(where); if (id !== undefined) { - // Residency-classifiable (id/friendlyId): route to the owning store, then fall back to the - // OTHER store on a miss. - return this.#findRunRouted(id, where, args, onPrimary); + // Residency-classifiable (id/friendlyId): id-shape is destiny for a run's whole life — a run-ops + // id lives on NEW, a cuid on LEGACY — so read the one owning store (no cross-store fan-out). + const store = this.#routeOrNew(id); + const method = onPrimary ? "findRunOnPrimary" : "findRun"; + return (store[method] as (...rest: unknown[]) => Promise)(where, args); } // Unclassifiable where (e.g. spanId, idempotencyKey): the run may live on either DB, // so fan out NEW-first then LEGACY rather than defaulting to NEW — defaulting silently @@ -253,31 +241,6 @@ export class RoutingRunStore implements RunStore { return this.#findRunUnrouted(where, args, onPrimary); } - // A classifiable id names its OWNING store by id-shape, but physical residency can diverge from - // classification (a pre-#4154 base62 run lives on #new yet classifies LEGACY). Read the owning - // store first — a hit is a SINGLE read (the fast path) — then, ONLY on a miss, probe the OTHER - // store before returning null, so a run whose residency ≠ its id-shape is found rather than 404'd. - // Both legs run the SAME index-covered TaskRun lookup; the fan-out cost is paid only on the (rare) - // miss. Mirrors #findRunUnrouted's shape but keyed on the classified owner. - async #findRunRouted( - id: string, - where: Prisma.TaskRunWhereInput, - args: unknown, - onPrimary: boolean - ): Promise { - const method = onPrimary ? "findRunOnPrimary" : "findRun"; - const owning = this.#routeOrNew(id); - const fromOwning = await (owning[method] as (...rest: unknown[]) => Promise)( - where, - args - ); - if (fromOwning != null) { - return fromOwning; - } - const other = owning === this.#new ? this.#legacy : this.#new; - return (other[method] as (...rest: unknown[]) => Promise)(where, args); - } - async #findRunUnrouted( where: Prisma.TaskRunWhereInput, args: unknown, @@ -1187,7 +1150,7 @@ export class RoutingRunStore implements RunStore { opts?.residency, RoutingRunStore.#waitpointId(data) ); - // Never forward the caller's tx into a routed write (matches #routeWaitpointWrite). + // Never forward the caller's tx into a routed write (it runs on the owning store's own client). return store.createWaitpoint(args, undefined); } @@ -1244,6 +1207,11 @@ export class RoutingRunStore implements RunStore { args as Record ); const id = RoutingRunStore.#waitpointId((args as { where?: unknown }).where); + // A colocated lookup (no id, resolved via `coLocateWithRunId`) is the (env,idempotencyKey) dedup + // probe of createDateTimeWaitpoint/createManualWaitpoint: read-your-writes within the owning + // run/tree. On a retry it must observe attempt 1's just-written waitpoint to short-circuit, so it + // reads the owning store's PRIMARY — the replica can lag and miss it, re-arming/re-blocking the run. + const coLocatedDedup = id === undefined && opts?.coLocateWithRunId !== undefined; const store = id !== undefined ? await this.#resolveWaitpointStore(id, client !== undefined) @@ -1254,7 +1222,7 @@ export class RoutingRunStore implements RunStore { store !== undefined ? ((await store.findWaitpoint( scalarArgs as typeof args, - RoutingRunStore.#ownPrimary(store, client) + coLocatedDedup ? store.primaryReadClient : RoutingRunStore.#ownPrimary(store, client) )) as Record | null) : (((await this.#new.findWaitpoint( scalarArgs as typeof args, @@ -1701,7 +1669,7 @@ export class RoutingRunStore implements RunStore { ): Promise { // Route by the batch's classifiable internal id: run-ops id→NEW, cuid→LEGACY. The caller's // `tx` is never forwarded — the create runs on the owning store's own client so the batch and - // its co-resident child runs/items land on the same DB. Mirrors #routeWaitpointWrite / + // its co-resident child runs/items land on the same DB. Mirrors the by-id waitpoint-write routing / // updateBatchTaskRun. const store = await this.#routeOrNewForWrite(data.id); return store.createBatchTaskRun(data, undefined); @@ -1718,7 +1686,7 @@ export class RoutingRunStore implements RunStore { const id = typeof args.where.id === "string" ? args.where.id : (args.where.friendlyId ?? undefined); // The caller's `tx` is never forwarded — the update runs on the owning store's own client so - // it targets the DB the batch actually lives on. Mirrors #routeWaitpointWrite. + // it targets the DB the batch actually lives on. Mirrors the by-id waitpoint-write routing. const store = this.#routeOrNew(id); return store.updateBatchTaskRun(args, undefined); } @@ -1880,7 +1848,7 @@ export class RoutingRunStore implements RunStore { // WaitpointTag — a standalone entity (no run/waitpoint FK) keyed by (environmentId, name). // --------------------------------------------------------------------------- - // Callers never mint a tag id (defaults to cuid), so #routeWaitpointWrite always resolves LEGACY + // Callers never mint a tag id (defaults to cuid), so a tag write always resolves LEGACY // today — deliberately single-homed, like standalone waitpoint tokens. If tag-id minting is ever made // residency-aware, findManyWaitpointTags must de-dupe by (environmentId, name) or names will duplicate. upsertWaitpointTag( diff --git a/packages/redis-worker/src/mollifier/buffer.ts b/packages/redis-worker/src/mollifier/buffer.ts index d2281cd43..179de94ec 100644 --- a/packages/redis-worker/src/mollifier/buffer.ts +++ b/packages/redis-worker/src/mollifier/buffer.ts @@ -462,6 +462,23 @@ export class MollifierBuffer { await this.redis.releaseMollifierClaim(claimKey, `${PENDING_PREFIX}${input.token}`); } + // Reopen a RESOLVED claim slot whose winner was cleared, so the claim-loser + // reacquire path can re-serialise a cross-DB recreate through a fresh claim. + // Compare-and-delete on the observed `expectedRunId` (never an unconditional + // DEL) so a concurrent reacquirer that already re-published a NEW winner is + // never wiped. Reuses the lookup self-heal's compare-and-delete Lua. Returns + // true if the stale slot was cleared. + async resetResolvedClaim( + input: IdempotencyLookupInput & { expectedRunId: string } + ): Promise { + const claimKey = makeIdempotencyClaimKey(input); + const deleted = (await this.redis.delMollifierKeyIfEquals( + claimKey, + input.expectedRunId + )) as number; + return deleted === 1; + } + // Read the current claim value, used by the wait/poll loop on losers // to detect "pending" → "resolved" transitions and timeouts. async readClaim(input: IdempotencyLookupInput): Promise { From a7c734c2235d456b93b8f9d1759bfe2ef59b6bd9 Mon Sep 17 00:00:00 2001 From: Daniel Sutton <45313566+d-cs@users.noreply.github.com> Date: Sun, 19 Jul 2026 18:06:45 +0100 Subject: [PATCH 003/300] test: caller-driven replica-lag + idempotency guards (stacked on #4284) (#4285) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Stacked on #4284 — tests only This PR contains **only the tests** that guard the production fixes in #4284 (its base). Review #4284 first; this branch adds no production code. ## What Caller-driven replica-lag and idempotency guards for every fixed site: - Each guard **drives the real exported caller** (route loader/action, presenter `.call()`, service, or engine method) against a **real Postgres** with the owning replica frozen via the shared `laggingReplica` testcontainer primitive — never a store-seam reimplementation. - For a **fixed** site the guard goes **RED when the production change is reverted**; for a **tolerated read-view** site it's a caller-driven **GREEN** proof the miss self-heals (returns null/empty, no mutation, row live on primary). - The **global-scope idempotency** guard drives the real dedup + claim path through a **real `MollifierBuffer` over a Redis testcontainer** (real SETNX/poll/publish), and covers the cross-DB **andWait** waitpoint wiring and the **expired/failed clear-and-recreate** reacquire cases. Run with `vitest --no-file-parallelism` (testcontainers). Verified GREEN, and revert→RED verified per fixed site. --- .../test/batchServices.replicaLag.test.ts | 679 +++++++++++ .../test/bulkActionV2.replicaLag.test.ts | 205 ++++ .../test/cancelRouteReplicaLag.guard.test.ts | 266 +++++ apps/webapp/test/claimTtl.test.ts | 55 + ...ngineReplicaReads.replicaLag.guard.test.ts | 289 +++++ ...EnvironmentFromRunReplicaLag.guard.test.ts | 196 ++++ ...mpotencyExpiredRecreateReserialize.test.ts | 93 ++ ...otencyGlobalScopeCrossDbConcurrent.test.ts | 930 +++++++++++++++ ...mpotencyResetRouteReplicaLag.guard.test.ts | 276 +++++ .../metadataRouteReplicaLag.guard.test.ts | 261 +++++ .../test/mollifierClaimResolution.test.ts | 26 + ...entersSessionBatchReplicaLag.guard.test.ts | 358 ++++++ apps/webapp/test/publishClaimResult.test.ts | 33 + .../test/reacquireClearedGlobalWinner.test.ts | 44 + .../test/readRunForEvent.replicaLag.test.ts | 198 ++++ .../test/realtimeServices.replicaLag.test.ts | 555 +++++++++ ...meSessionsIoRoute.replicaLag.guard.test.ts | 263 +++++ .../realtimeStreamRoutes.replicaLag.test.ts | 707 ++++++++++++ .../test/replayRouteReplicaLag.guard.test.ts | 261 +++++ .../test/resolveBatchForRealtime.test.ts | 47 + .../routesBatchGetReplicaLag.guard.test.ts | 400 +++++++ .../runGetRoutes.replicaLag.guard.test.ts | 1021 +++++++++++++++++ .../test/runPresenters.replicaLag.test.ts | 533 +++++++++ .../runsRepositoryConvert.replicaLag.test.ts | 152 +++ ...onWaitpointRoutes.replicaLag.guard.test.ts | 351 ++++++ .../test/spanTraceRoutes.replicaLag.test.ts | 545 +++++++++ ...pointCallbackRouteReplicaLag.guard.test.ts | 193 ++++ ...pointCompleteRouteReplicaLag.guard.test.ts | 251 ++++ ...itpointPresenters.replicaLag.guard.test.ts | 568 +++++++++ ...yDecisionReadAfterWrite.replicaLag.test.ts | 185 +++ .../ttlSystemExpireReplicaLag.guard.test.ts | 154 +++ ...ncySweeperCallbackReplicaLag.guard.test.ts | 122 ++ .../runAttemptSystemReplicaLag.guard.test.ts | 516 +++++++++ ...ependentAttemptReadView.replicaLag.test.ts | 251 ++++ ...atchIdempotencyDedupReadAfterWrite.test.ts | 161 +++ ...tore.bulkActionReadView.replicaLag.test.ts | 253 ++++ ...cancelRunReadAfterWrite.replicaLag.test.ts | 223 ++++ ....dashboardAgentReadView.replicaLag.test.ts | 224 ++++ ...psStore.engineReadViews.replicaLag.test.ts | 359 ++++++ ...tore.idempotencyGlobalScopeCrossDb.test.ts | 328 ++++++ ...dempotencyResetReadView.replicaLag.test.ts | 227 ++++ ...modelRuntimeEnvReadView.replicaLag.test.ts | 183 +++ ...e.presentersRunReadView.replicaLag.test.ts | 435 +++++++ ...ersSessionBatchReadView.replicaLag.test.ts | 344 ++++++ ...entersWaitpointReadView.replicaLag.test.ts | 516 +++++++++ ...ealtimeServicesReadView.replicaLag.test.ts | 356 ++++++ ...re.replayReadAfterWrite.replicaLag.test.ts | 211 ++++ ...re.resolveRunForMutationReplicaLag.test.ts | 200 ++++ ....routesBatchGetReadView.replicaLag.test.ts | 312 +++++ ...sRealtimeStreamReadView.replicaLag.test.ts | 590 ++++++++++ ...re.routesRunGetReadView.replicaLag.test.ts | 664 +++++++++++ ...routesSpanTraceReadView.replicaLag.test.ts | 322 ++++++ ...viceBatchFamilyReadView.replicaLag.test.ts | 346 ++++++ ...onMetadataRouteReadView.replicaLag.test.ts | 231 ++++ ...tore.sessionRunProbeReadAfterWrite.test.ts | 211 ++++ ...e.storeReceiverReadView.replicaLag.test.ts | 288 +++++ .../run-store/src/runOpsStore.test.ts | 27 +- ...OpsStore.ttlExpireOrphanReplicaLag.test.ts | 222 ++++ ....usageCostUndercountReadAfterWrite.test.ts | 253 ++++ ...AndReplayLoaderReadView.replicaLag.test.ts | 246 ++++ ...itpointCompleteTokenReadAfterWrite.test.ts | 175 +++ ...Store.waitpointDedupReadAfterWrite.test.ts | 288 +++++ 62 files changed, 19168 insertions(+), 11 deletions(-) create mode 100644 apps/webapp/test/batchServices.replicaLag.test.ts create mode 100644 apps/webapp/test/bulkActionV2.replicaLag.test.ts create mode 100644 apps/webapp/test/cancelRouteReplicaLag.guard.test.ts create mode 100644 apps/webapp/test/claimTtl.test.ts create mode 100644 apps/webapp/test/engineReplicaReads.replicaLag.guard.test.ts create mode 100644 apps/webapp/test/findEnvironmentFromRunReplicaLag.guard.test.ts create mode 100644 apps/webapp/test/idempotencyExpiredRecreateReserialize.test.ts create mode 100644 apps/webapp/test/idempotencyGlobalScopeCrossDbConcurrent.test.ts create mode 100644 apps/webapp/test/idempotencyResetRouteReplicaLag.guard.test.ts create mode 100644 apps/webapp/test/metadataRouteReplicaLag.guard.test.ts create mode 100644 apps/webapp/test/presentersSessionBatchReplicaLag.guard.test.ts create mode 100644 apps/webapp/test/publishClaimResult.test.ts create mode 100644 apps/webapp/test/reacquireClearedGlobalWinner.test.ts create mode 100644 apps/webapp/test/readRunForEvent.replicaLag.test.ts create mode 100644 apps/webapp/test/realtimeServices.replicaLag.test.ts create mode 100644 apps/webapp/test/realtimeSessionsIoRoute.replicaLag.guard.test.ts create mode 100644 apps/webapp/test/realtimeStreamRoutes.replicaLag.test.ts create mode 100644 apps/webapp/test/replayRouteReplicaLag.guard.test.ts create mode 100644 apps/webapp/test/resolveBatchForRealtime.test.ts create mode 100644 apps/webapp/test/routesBatchGetReplicaLag.guard.test.ts create mode 100644 apps/webapp/test/runGetRoutes.replicaLag.guard.test.ts create mode 100644 apps/webapp/test/runPresenters.replicaLag.test.ts create mode 100644 apps/webapp/test/runsRepositoryConvert.replicaLag.test.ts create mode 100644 apps/webapp/test/sessionWaitpointRoutes.replicaLag.guard.test.ts create mode 100644 apps/webapp/test/spanTraceRoutes.replicaLag.test.ts create mode 100644 apps/webapp/test/waitpointCallbackRouteReplicaLag.guard.test.ts create mode 100644 apps/webapp/test/waitpointCompleteRouteReplicaLag.guard.test.ts create mode 100644 apps/webapp/test/waitpointPresenters.replicaLag.guard.test.ts create mode 100644 internal-packages/run-engine/src/engine/retryDecisionReadAfterWrite.replicaLag.test.ts create mode 100644 internal-packages/run-engine/src/engine/systems/ttlSystemExpireReplicaLag.guard.test.ts create mode 100644 internal-packages/run-engine/src/engine/tests/concurrencySweeperCallbackReplicaLag.guard.test.ts create mode 100644 internal-packages/run-engine/src/engine/tests/runAttemptSystemReplicaLag.guard.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.batchDependentAttemptReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.batchIdempotencyDedupReadAfterWrite.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.bulkActionReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.cancelRunReadAfterWrite.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.dashboardAgentReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.engineReadViews.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.idempotencyGlobalScopeCrossDb.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.idempotencyResetReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.modelRuntimeEnvReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.presentersRunReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.presentersSessionBatchReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.presentersWaitpointReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.realtimeServicesReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.replayReadAfterWrite.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.resolveRunForMutationReplicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.routesBatchGetReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.routesRealtimeStreamReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.routesRunGetReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.routesSpanTraceReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.serviceBatchFamilyReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.sessionMetadataRouteReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.sessionRunProbeReadAfterWrite.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.storeReceiverReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.ttlExpireOrphanReplicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.usageCostUndercountReadAfterWrite.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.waitpointCompleteRouteAndReplayLoaderReadView.replicaLag.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.waitpointCompleteTokenReadAfterWrite.test.ts create mode 100644 internal-packages/run-store/src/runOpsStore.waitpointDedupReadAfterWrite.test.ts diff --git a/apps/webapp/test/batchServices.replicaLag.test.ts b/apps/webapp/test/batchServices.replicaLag.test.ts new file mode 100644 index 000000000..cae6ab053 --- /dev/null +++ b/apps/webapp/test/batchServices.replicaLag.test.ts @@ -0,0 +1,679 @@ +// Replica-lag guards for the batch-service reads. +// +// Each case drives the REAL exported caller (a webapp service method that contains the run-store +// read) against a real Postgres testcontainer, with the store's read replica FROZEN via the shared +// `laggingReplica` primitive so a replica-routed read misses the just-written row while the primary +// still holds it. Only deps orthogonal to the read path are mocked (engine enqueue/count, env +// resolution, object-store download, id minting, control-plane env assertion, worker enqueue). +// +// Reads under guard (PostgresRunStore client-less defaults): +// findBatchTaskRunById / findBatchTaskRunByIdempotencyKey / countBatchTaskRunItems → PRIMARY +// findTaskRunAttempt → REPLICA (client ?? this.readOnlyPrisma) +// +// Property: the PRIMARY-routed reads observe the live row under lag (asserted on output) and +// wasHit() === false proves they never touched the frozen replica; the dependent-attempt +// read is threaded the primary client so it still sees a live terminal parent and the "parent +// already in a terminal state" guard fires. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import { PostgresRunStore } from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import { BatchId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// ---- Module mocks orthogonal to the run-store read path. ----------------------------------------- +const dbHolder = vi.hoisted(() => ({ prisma: undefined as any })); +vi.mock("~/db.server", () => ({ + get prisma() { + return dbHolder.prisma; + }, + get $replica() { + return dbHolder.prisma; + }, +})); +// Prevent the run engine + run-store singletons from constructing at import; every caller here is +// driven with an explicitly-injected store/engine, so these defaults are never used. +vi.mock("~/v3/runEngine.server", () => ({ engine: {} })); +vi.mock("~/v3/runStore.server", () => ({ runStore: {} })); +vi.mock("~/v3/batchTriggerWorker.server", () => ({ + batchTriggerWorker: { enqueue: vi.fn(async () => {}) }, +})); +vi.mock("~/services/platform.v3.server", async (importOriginal) => ({ + ...((await importOriginal()) as Record), + getEntitlement: vi.fn(async () => ({ hasAccess: true })), +})); +// Env resolution is downstream of the batch read in both processBatchTaskRun callers. +vi.mock("~/models/runtimeEnvironment.server", () => ({ + findEnvironmentById: vi.fn(async () => null), +})); +// Control-plane env existence assertion (batchTriggerV3.call preamble) — orthogonal to the reads. +vi.mock("~/v3/runOpsMigration/controlPlaneResolver.server", () => ({ + controlPlaneResolver: { assertEnvExists: vi.fn(async () => {}) }, +})); +// Batch friendly-id minting — deterministic id so the created batch is predictable. +const mintHolder = vi.hoisted(() => ({ id: "", friendlyId: "" })); +vi.mock("~/v3/runOpsMigration/mintBatchFriendlyId.server", () => ({ + mintBatchFriendlyId: vi.fn(async () => ({ + id: mintHolder.id, + friendlyId: mintHolder.friendlyId, + })), +})); +// Object-store payload download for the existing-batch response (idempotency dedup path). +const objHolder = vi.hoisted(() => ({ + packet: { data: "[]", dataType: "application/json" } as { data?: string; dataType: string }, +})); +vi.mock("~/v3/objectStore.server", () => ({ + downloadPacketFromObjectStore: vi.fn(async () => objHolder.packet), + uploadPacketToObjectStore: vi.fn(async () => "uploaded"), +})); + +import { findEnvironmentById } from "~/models/runtimeEnvironment.server"; +// REAL callers under guard. +import { + BatchTriggerV3Service, + tryCompleteBatchV3, +} from "../app/v3/services/batchTriggerV3.server"; +import { StreamBatchItemsService } from "../app/runEngine/services/streamBatchItems.server"; +import { RunEngineBatchTriggerService } from "../app/runEngine/services/batchTrigger.server"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +// A PostgresRunStore whose PRIMARY is the live container and whose read replica is frozen for the +// given models (missing mode → replica-routed reads return null/[]/0). `wasHit(model)` proves whether +// a read actually consulted the frozen replica. +function storeWithFrozenReplica(prisma: PrismaClient, models: string[]) { + const replica = laggingReplica( + prisma, + models.map((model) => ({ model, mode: "missing" as const })) + ); + const store = new PostgresRunStore({ + prisma, + readOnlyPrisma: replica.client, + schemaVariant: "legacy", + }); + return { store, replica }; +} + +async function createBatchOnPrimary( + prisma: PrismaClient, + params: { + id: string; + friendlyId: string; + runtimeEnvironmentId: string; + runCount: number; + status?: "PENDING" | "PROCESSING" | "COMPLETED" | "PARTIAL_FAILED" | "ABORTED"; + sealed?: boolean; + expectedCount?: number; + runIds?: string[]; + idempotencyKey?: string; + idempotencyKeyExpiresAt?: Date | null; + payload?: string; + payloadType?: string; + processingCompletedAt?: Date | null; + } +) { + return prisma.batchTaskRun.create({ + data: { + id: params.id, + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + runCount: params.runCount, + expectedCount: params.expectedCount ?? params.runCount, + status: params.status ?? "PENDING", + sealed: params.sealed ?? false, + runIds: params.runIds ?? [], + batchVersion: "v3", + idempotencyKey: params.idempotencyKey ?? null, + idempotencyKeyExpiresAt: params.idempotencyKeyExpiresAt ?? null, + payload: params.payload, + payloadType: params.payloadType ?? "application/json", + processingCompletedAt: params.processingCompletedAt ?? null, + }, + }); +} + +// Seed a real TaskRun + a COMPLETED TaskRunAttempt (with its required FK graph) on the PRIMARY, whose +// parent run is in a FINAL status. Used to prove the dependent-attempt guard fires on a live row. +async function seedTerminalAttempt( + prisma: PrismaClient, + seed: { organization: { id: string }; project: { id: string }; environment: { id: string } }, + suffix: string +): Promise<{ attemptFriendlyId: string }> { + const envId = seed.environment.id; + const projectId = seed.project.id; + + const run = await prisma.taskRun.create({ + data: { + friendlyId: `run_parent_${suffix}`, + taskIdentifier: "parent-task", + payload: "{}", + payloadType: "application/json", + traceId: `t_${suffix}`, + spanId: `s_${suffix}`, + runtimeEnvironmentId: envId, + projectId, + organizationId: seed.organization.id, + environmentType: "DEVELOPMENT", + queue: "task/parent-task", + status: "COMPLETED_SUCCESSFULLY", + }, + }); + + const worker = await prisma.backgroundWorker.create({ + data: { + friendlyId: `worker_${suffix}`, + contentHash: `hash_${suffix}`, + projectId, + runtimeEnvironmentId: envId, + version: "20240101.1", + metadata: {}, + }, + }); + const queue = await prisma.taskQueue.create({ + data: { + friendlyId: `queue_${suffix}`, + name: "task/parent-task", + projectId, + runtimeEnvironmentId: envId, + }, + }); + const workerTask = await prisma.backgroundWorkerTask.create({ + data: { + friendlyId: `wt_${suffix}`, + slug: "parent-task", + filePath: "src/trigger/parent.ts", + workerId: worker.id, + projectId, + runtimeEnvironmentId: envId, + queueId: queue.id, + }, + }); + const attemptFriendlyId = `attempt_${suffix}`; + await prisma.taskRunAttempt.create({ + data: { + friendlyId: attemptFriendlyId, + number: 1, + taskRunId: run.id, + backgroundWorkerId: worker.id, + backgroundWorkerTaskId: workerTask.id, + runtimeEnvironmentId: envId, + queueId: queue.id, + status: "COMPLETED", + }, + }); + return { attemptFriendlyId }; +} + +async function* emptyItems() { + // no items +} + +// Minimal AuthenticatedEnvironment shape the traced callers read (attributesFromAuthenticatedEnv + +// the env-ownership check). Not the run-store read path — purely the tracing/ownership periphery. +function authEnv(seed: { + organization: { id: string; slug: string; title: string }; + project: { id: string; name: string; slug: string }; + environment: { id: string }; +}) { + return { + id: seed.environment.id, + type: "DEVELOPMENT", + slug: "dev", + organizationId: seed.organization.id, + projectId: seed.project.id, + organization: { + id: seed.organization.id, + slug: seed.organization.slug, + title: seed.organization.title, + featureFlags: {}, + }, + project: { id: seed.project.id, name: seed.project.name, slug: seed.project.slug }, + }; +} + +describe("batch-svc — run-ops replica-lag guards", () => { + // ================================================================================================ + // tryCompleteBatchV3 — findBatchTaskRunById + countBatchTaskRunItems (both PRIMARY-routed) + // With the batch + its items frozen on the replica, the completion function reads both on the + // primary, sees the sealed+fully-completed batch, and transitions it to COMPLETED. + // ================================================================================================ + heteroPostgresTest( + "tryCompleteBatchV3 completes a sealed batch whose row+items have not replicated (reads primary)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + dbHolder.prisma = prisma; + const suffix = `trycomplete_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const { store, replica } = storeWithFrozenReplica(prisma, [ + "batchTaskRun", + "batchTaskRunItem", + ]); + + const { id: batchId, friendlyId } = BatchId.generate(); + await createBatchOnPrimary(prisma, { + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 2, + expectedCount: 2, + status: "PENDING", + sealed: true, + }); + + // Two COMPLETED items on the primary (each needs a real TaskRun FK). + for (const n of [1, 2]) { + const run = await prisma.taskRun.create({ + data: { + friendlyId: `run_${suffix}_${n}`, + taskIdentifier: "child-task", + payload: "{}", + payloadType: "application/json", + traceId: `t_${suffix}_${n}`, + spanId: `s_${suffix}_${n}`, + runtimeEnvironmentId: seed.environment.id, + projectId: seed.project.id, + organizationId: seed.organization.id, + environmentType: "DEVELOPMENT", + queue: "task/child-task", + status: "COMPLETED_SUCCESSFULLY", + }, + }); + await prisma.batchTaskRunItem.create({ + data: { batchTaskRunId: batchId, taskRunId: run.id, status: "COMPLETED" }, + }); + } + + await tryCompleteBatchV3(batchId, prisma as never, false, store); + + // The frozen replica was NEVER consulted for either read — both routed to the primary. + expect(replica.wasHit("batchTaskRun")).toBe(false); + expect(replica.wasHit("batchTaskRunItem")).toBe(false); + + // Observable outcome: the batch transitioned to COMPLETED with the full completed count. Had + // either read hit the frozen replica, findBatchTaskRunById→null (early return, no transition) or + // countBatchTaskRunItems→0 (< expectedCount, no transition). + const finalBatch = await prisma.batchTaskRun.findFirstOrThrow({ where: { id: batchId } }); + expect(finalBatch.status).toBe("COMPLETED"); + expect(finalBatch.completedCount).toBe(2); + } + ); + + // ================================================================================================ + // StreamBatchItemsService.call — initial findBatchTaskRunById (PRIMARY-routed) + // The entry batch lookup reads the primary, finds the live PENDING batch (frozen on the replica), + // and seals it — a replica-routed read would miss it. + // ================================================================================================ + heteroPostgresTest( + "streamBatchItems finds+seals a live batch whose row has not replicated (entry read → primary)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + dbHolder.prisma = prisma; + const suffix = `stream134_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const { store, replica } = storeWithFrozenReplica(prisma, ["batchTaskRun"]); + const { id: batchId, friendlyId } = BatchId.generate(); + await createBatchOnPrimary(prisma, { + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 0, + status: "PENDING", + sealed: false, + }); + + const fakeEngine = { + runStore: store, + getBatchEnqueuedCount: vi.fn(async () => 0), + enqueueBatchItem: vi.fn(async () => ({ enqueued: true })), + }; + const service = new StreamBatchItemsService({ engine: fakeEngine as never }); + + const result = await service.call(authEnv(seed) as never, friendlyId, emptyItems(), { + maxItemBytes: 1024, + concurrency: 1, + }); + + expect(replica.wasHit("batchTaskRun")).toBe(false); + expect(result.sealed).toBe(true); + expect(result.id).toBe(friendlyId); + // The primary write actually sealed the row. + const sealed = await prisma.batchTaskRun.findFirstOrThrow({ where: { id: batchId } }); + expect(sealed.sealed).toBe(true); + expect(sealed.status).toBe("PROCESSING"); + } + ); + + // ================================================================================================ + // StreamBatchItemsService.call — count-mismatch re-query findBatchTaskRunById (PRIMARY-routed) + // A concurrent fast-completion marks the batch COMPLETED on the primary between the entry read and + // the count check; the re-query reads the primary, sees COMPLETED, and returns sealed:true. + // ================================================================================================ + heteroPostgresTest( + "streamBatchItems count-mismatch re-query reads the primary (fast-completed batch → sealed:true)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + dbHolder.prisma = prisma; + const suffix = `stream206_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const { store, replica } = storeWithFrozenReplica(prisma, ["batchTaskRun"]); + const { id: batchId, friendlyId } = BatchId.generate(); + await createBatchOnPrimary(prisma, { + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 1, + status: "PENDING", + sealed: false, + }); + + const fakeEngine = { + runStore: store, + // Simulate the BatchQueue completing the batch on the PRIMARY before the count check, and + // return a mismatched enqueued count so the caller takes the re-query branch. + getBatchEnqueuedCount: vi.fn(async () => { + await prisma.batchTaskRun.update({ + where: { id: batchId }, + data: { status: "COMPLETED", processingCompletedAt: new Date() }, + }); + return 0; + }), + enqueueBatchItem: vi.fn(async () => ({ enqueued: true })), + }; + const service = new StreamBatchItemsService({ engine: fakeEngine as never }); + + const result = await service.call(authEnv(seed) as never, friendlyId, emptyItems(), { + maxItemBytes: 1024, + concurrency: 1, + }); + + expect(replica.wasHit("batchTaskRun")).toBe(false); + // Re-query found the COMPLETED batch on the primary → idempotent-retry success. + expect(result.sealed).toBe(true); + expect(result.id).toBe(friendlyId); + } + ); + + // ================================================================================================ + // StreamBatchItemsService.call — seal-race re-query findBatchTaskRunById (PRIMARY-routed) + // A concurrent request seals the batch (PROCESSING) on the primary before this request's + // conditional seal, so the conditional update matches 0 rows; the re-query reads the primary, sees + // PROCESSING, and returns sealed:true. + // ================================================================================================ + heteroPostgresTest( + "streamBatchItems seal-race re-query reads the primary (concurrently-sealed batch → sealed:true)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + dbHolder.prisma = prisma; + const suffix = `stream294_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const { store, replica } = storeWithFrozenReplica(prisma, ["batchTaskRun"]); + const { id: batchId, friendlyId } = BatchId.generate(); + await createBatchOnPrimary(prisma, { + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 0, + status: "PENDING", + sealed: false, + }); + + const fakeEngine = { + runStore: store, + // A concurrent request seals the batch on the PRIMARY before this request's conditional seal, + // so the conditional update matches nothing and the caller takes the re-query branch. + getBatchEnqueuedCount: vi.fn(async () => { + await prisma.batchTaskRun.update({ + where: { id: batchId }, + data: { status: "PROCESSING", sealed: true, sealedAt: new Date() }, + }); + return 0; + }), + enqueueBatchItem: vi.fn(async () => ({ enqueued: true })), + }; + const service = new StreamBatchItemsService({ engine: fakeEngine as never }); + + const result = await service.call(authEnv(seed) as never, friendlyId, emptyItems(), { + maxItemBytes: 1024, + concurrency: 1, + }); + + expect(replica.wasHit("batchTaskRun")).toBe(false); + // Re-query found the concurrently-sealed batch on the primary → sealed:true (no spurious throw). + expect(result.sealed).toBe(true); + expect(result.id).toBe(friendlyId); + } + ); + + // ================================================================================================ + // RunEngineBatchTriggerService.processBatchTaskRun — findBatchTaskRunById (PRIMARY-routed) + // The worker reads the just-written batch on the primary and proceeds to resolve its environment. + // ================================================================================================ + heteroPostgresTest( + "runEngine processBatchTaskRun reads the batch on the primary when the replica lags", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + dbHolder.prisma = prisma; + const suffix = `retrigger_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const { store, replica } = storeWithFrozenReplica(prisma, ["batchTaskRun"]); + const { id: batchId, friendlyId } = BatchId.generate(); + await createBatchOnPrimary(prisma, { + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 1, + status: "PENDING", + }); + + vi.mocked(findEnvironmentById).mockClear(); + vi.mocked(findEnvironmentById).mockResolvedValue(null as never); + + const fakeEngine = { runStore: store }; + const service = new RunEngineBatchTriggerService( + "sequential", + prisma as never, + fakeEngine as never + ); + + await service.processBatchTaskRun({ + batchId, + processingId: "0", + range: { start: 0, count: 50 }, + attemptCount: 0, + strategy: "sequential", + }); + + expect(replica.wasHit("batchTaskRun")).toBe(false); + // The batch was found on the primary → env resolution was reached with the batch's env id. Had + // the read hit the frozen replica the batch would be null and findEnvironmentById never called. + expect(vi.mocked(findEnvironmentById)).toHaveBeenCalledWith(seed.environment.id); + } + ); + + // ================================================================================================ + // BatchTriggerV3Service.processBatchTaskRun — findBatchTaskRunById (PRIMARY-routed) + // Same property on the v3 worker path (injected runStore instead of an engine). + // ================================================================================================ + heteroPostgresTest( + "batchTriggerV3 processBatchTaskRun reads the batch on the primary when the replica lags", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + dbHolder.prisma = prisma; + const suffix = `v3process_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const { store, replica } = storeWithFrozenReplica(prisma, ["batchTaskRun"]); + const { id: batchId, friendlyId } = BatchId.generate(); + await createBatchOnPrimary(prisma, { + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 1, + status: "PENDING", + }); + + vi.mocked(findEnvironmentById).mockClear(); + vi.mocked(findEnvironmentById).mockResolvedValue(null as never); + + const service = new BatchTriggerV3Service(undefined, undefined, prisma as never, store); + + await service.processBatchTaskRun({ + batchId, + processingId: "0", + range: { start: 0, count: 50 }, + attemptCount: 0, + strategy: "sequential", + }); + + expect(replica.wasHit("batchTaskRun")).toBe(false); + expect(vi.mocked(findEnvironmentById)).toHaveBeenCalledWith(seed.environment.id); + } + ); + + // ================================================================================================ + // BatchTriggerV3Service.call — findBatchTaskRunByIdempotencyKey (PRIMARY-routed) + // The idempotency probe reads the primary, finds the just-written batch, and returns the cached + // response — so no duplicate batch is triggered. + // ================================================================================================ + heteroPostgresTest( + "batchTriggerV3.call dedups on the primary when the just-written batch has not replicated", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + dbHolder.prisma = prisma; + const suffix = `v3idem_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const { store, replica } = storeWithFrozenReplica(prisma, ["batchTaskRun"]); + const { id: batchId, friendlyId } = BatchId.generate(); + const idempotencyKey = `idem_${suffix}`; + await createBatchOnPrimary(prisma, { + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 1, + status: "COMPLETED", + runIds: ["run_cached_1"], + idempotencyKey, + idempotencyKeyExpiresAt: null, + payload: '[{"task":"my-task"}]', + payloadType: "application/json", + }); + objHolder.packet = { data: '[{"task":"my-task"}]', dataType: "application/json" }; + + const environment = { + id: seed.environment.id, + organizationId: seed.organization.id, + type: "DEVELOPMENT", + organization: { id: seed.organization.id, featureFlags: {} }, + project: { id: seed.project.id }, + }; + const service = new BatchTriggerV3Service( + undefined, + undefined, + prisma as never, + store, + (async () => "cuid") as never + ); + + const result = await service.call( + environment as never, + { items: [{ task: "my-task", payload: "{}", options: {} }] } as never, + { idempotencyKey } + ); + + expect(replica.wasHit("batchTaskRun")).toBe(false); + // The existing batch was found on the primary → returned as cached (no duplicate). + expect(result.isCached).toBe(true); + expect(result.id).toBe(friendlyId); + expect(result.idempotencyKey).toBe(idempotencyKey); + } + ); + + // ================================================================================================ + // BatchTriggerV3Service.call — findTaskRunAttempt (dependent-attempt read) + // The dependent-attempt read is threaded the primary client, so under replica lag it still finds a + // live TERMINAL parent attempt and the caller rejects the batch with a ServiceValidationError + // ("parent already in a terminal state") rather than building a batch for a dead parent. + // wasHit("taskRunAttempt") === false confirms the read hit the primary, not the frozen replica. + // ================================================================================================ + heteroPostgresTest( + "batchTriggerV3.call rejects a batch whose live terminal dependent-attempt has not replicated", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + dbHolder.prisma = prisma; + const suffix = `v3dep_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + // The dependent (parent) attempt + its FINAL run live on the PRIMARY only. + const { attemptFriendlyId } = await seedTerminalAttempt(prisma, seed, suffix); + + const { store, replica } = storeWithFrozenReplica(prisma, ["taskRunAttempt"]); + mintHolder.id = `batch_${suffix}`; + mintHolder.friendlyId = `batch_friendly_${suffix}`; + + const environment = { + id: seed.environment.id, + organizationId: seed.organization.id, + type: "DEVELOPMENT", + organization: { id: seed.organization.id, featureFlags: {} }, + project: { id: seed.project.id }, + }; + const service = new BatchTriggerV3Service( + undefined, + undefined, + prisma as never, + store, + (async () => "cuid") as never + ); + + // The dependent-attempt read hits the primary, finds the live terminal attempt, and the caller + // rejects — the frozen replica is never consulted for it. + await expect( + service.call( + environment as never, + { + items: [{ task: "child-task", payload: "{}", options: {} }], + dependentAttempt: attemptFriendlyId, + } as never, + {} + ) + ).rejects.toThrow(/already in a terminal state/); + + // The dependent-attempt read hit the PRIMARY, never the frozen replica. + expect(replica.wasHit("taskRunAttempt")).toBe(false); + } + ); +}); diff --git a/apps/webapp/test/bulkActionV2.replicaLag.test.ts b/apps/webapp/test/bulkActionV2.replicaLag.test.ts new file mode 100644 index 000000000..a38530e60 --- /dev/null +++ b/apps/webapp/test/bulkActionV2.replicaLag.test.ts @@ -0,0 +1,205 @@ +// Under replica lag, BulkActionService.process (CANCEL and REPLAY) tolerates a member whose row has +// not replicated: the per-batch member-hydration findRuns misses, the member is skipped this pass (no +// throw, no mutation — not cancelled, not replayed), and the run stays live on the primary. The id set +// comes from ClickHouse, fed downstream of Postgres and lagging more than the PG replica, so an id +// cannot reach this batch before its PG row is on the replica — the skip window is unreachable. +// +// Drives the REAL exported BulkActionService.process against a real Postgres testcontainer. Tenant + +// group are seeded on the primary; only orthogonal peripherals are mocked (ClickHouse-sourced id list, +// commonWorker enqueue, db/engine/run-store singletons so the module imports don't construct at load). +// The member-hydration store is a real PostgresRunStore whose read replica is frozen via the shared +// laggingReplica primitive. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import { PostgresRunStore } from "@internal/run-store"; +import { BulkActionStatus, BulkActionType, type PrismaClient } from "@trigger.dev/database"; +import { BulkActionId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// Prevent the db / engine / run-store singletons from constructing at import (the service is driven with +// an explicitly-injected prisma + store). +vi.mock("~/db.server", () => ({ prisma: {}, $replica: {} })); +vi.mock("~/v3/runEngine.server", () => ({ engine: {} })); +vi.mock("~/v3/runStore.server", () => ({ runStore: {} })); +// commonWorker.enqueue is the "next batch" scheduling side effect — orthogonal to the hydration read. +vi.mock("~/v3/commonWorker.server", () => ({ commonWorker: { enqueue: vi.fn(async () => {}) } })); +// ClickHouse client factory — the id list is injected via the RunsRepository mock below. +vi.mock("~/services/clickhouse/clickhouseFactoryInstance.server", () => ({ + clickhouseFactory: { getClickhouseForOrganization: vi.fn(async () => ({})) }, +})); +// RunsRepository.listRunIds is the ClickHouse-sourced id page. Keep parseRunListInputOptions REAL; swap +// only the repository so listRunIds returns our controlled member id set + a terminal (null) cursor. +const listHolder = vi.hoisted(() => ({ runIds: [] as string[] })); +vi.mock("~/services/runsRepository/runsRepository.server", async (importOriginal) => ({ + ...((await importOriginal()) as Record), + RunsRepository: class { + constructor(_opts: any) {} + async listRunIds() { + return { runIds: listHolder.runIds, pagination: { nextCursor: null, previousCursor: null } }; + } + async countRuns() { + return listHolder.runIds.length; + } + }, +})); + +import { BulkActionService } from "~/v3/services/bulk/BulkActionV2.server"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +async function seedRun( + prisma: PrismaClient, + seed: { organization: { id: string }; project: { id: string }; environment: { id: string } }, + suffix: string, + status: "EXECUTING" | "COMPLETED_SUCCESSFULLY" +) { + const runId = `run_${suffix}`; + await prisma.taskRun.create({ + data: { + id: runId, + engine: "V2", + status, + friendlyId: `run_fr_${suffix}`, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceId: `trace_${suffix}`, + spanId: `span_${suffix}`, + queue: "task/my-task", + runtimeEnvironmentId: seed.environment.id, + projectId: seed.project.id, + organizationId: seed.organization.id, + environmentType: "DEVELOPMENT", + }, + }); + return runId; +} + +async function seedGroup( + prisma: PrismaClient, + seed: { project: { id: string }; environment: { id: string } }, + type: BulkActionType +) { + const { id, friendlyId } = BulkActionId.generate(); + await prisma.bulkActionGroup.create({ + data: { + id, + friendlyId, + projectId: seed.project.id, + environmentId: seed.environment.id, + type, + params: {}, + queryName: "bulk_action_v1", + status: BulkActionStatus.PENDING, + totalCount: 1, + }, + }); + return id; +} + +describe("BulkActionV2.process tolerates members whose rows have not replicated", () => { + // CANCEL. + heteroPostgresTest( + "CANCEL skips a member whose row has not replicated, leaving the run live on the primary", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `bulk_cancel_${seq++}`; + const seed = await seedTenant(prisma, suffix); + const runId = await seedRun(prisma, seed, suffix, "EXECUTING"); + const groupId = await seedGroup(prisma, seed, BulkActionType.CANCEL); + + listHolder.runIds = [runId]; + + // Member-hydration store: primary live, taskRun frozen-missing on the replica. + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + const store = new PostgresRunStore({ + prisma, + readOnlyPrisma: replica.client as never, + schemaVariant: "legacy", + }); + + const service = new BulkActionService(prisma as never, prisma as never, store); + await service.process(groupId, { continueInline: true }); + + // The hydration read really hit the (lagging) replica. + expect(replica.wasHit("taskRun")).toBe(true); + + // OBSERVABLE OUTPUT: the member was SKIPPED — the run was NOT cancelled (still EXECUTING). + const onPrimary = await prisma.taskRun.findFirstOrThrow({ where: { id: runId } }); + expect(onPrimary.status).toBe("EXECUTING"); + + // No progress was counted for the skipped member (success=0, failure=0). + const group = await prisma.bulkActionGroup.findFirstOrThrow({ where: { id: groupId } }); + expect(group.successCount).toBe(0); + expect(group.failureCount).toBe(0); + } + ); + + // REPLAY. + heteroPostgresTest( + "REPLAY skips a member whose row has not replicated, leaving the run live on the primary", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `bulk_replay_${seq++}`; + const seed = await seedTenant(prisma, suffix); + const runId = await seedRun(prisma, seed, suffix, "COMPLETED_SUCCESSFULLY"); + const groupId = await seedGroup(prisma, seed, BulkActionType.REPLAY); + + listHolder.runIds = [runId]; + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + const store = new PostgresRunStore({ + prisma, + readOnlyPrisma: replica.client as never, + schemaVariant: "legacy", + }); + + const service = new BulkActionService(prisma as never, prisma as never, store); + await service.process(groupId, { continueInline: true }); + + expect(replica.wasHit("taskRun")).toBe(true); + + // OBSERVABLE OUTPUT: the member was SKIPPED — no child replay run was created for this member. + const childRuns = await prisma.taskRun.findMany({ + where: { rootTaskRunId: runId }, + }); + expect(childRuns).toHaveLength(0); + + const group = await prisma.bulkActionGroup.findFirstOrThrow({ where: { id: groupId } }); + expect(group.successCount).toBe(0); + expect(group.failureCount).toBe(0); + + // The skip is pure lag: the member run is genuinely live on the primary. + const onPrimary = await prisma.taskRun.findFirstOrThrow({ where: { id: runId } }); + expect(onPrimary.friendlyId).toBe(`run_fr_${suffix}`); + } + ); +}); diff --git a/apps/webapp/test/cancelRouteReplicaLag.guard.test.ts b/apps/webapp/test/cancelRouteReplicaLag.guard.test.ts new file mode 100644 index 000000000..f85717411 --- /dev/null +++ b/apps/webapp/test/cancelRouteReplicaLag.guard.test.ts @@ -0,0 +1,266 @@ +// Property: the CANCEL route resolves a just-created run under replica lag. The real exported `action` +// finds a run present only on the primary (replica frozen "missing", buffer disabled) via the primary +// re-read and cancels it, returning the success redirect rather than "Run not found". +// +// Drives the REAL route `action` (not a direct store call) against a REAL Postgres testcontainer with a +// REAL lagging replica (shared `laggingReplica`, `taskRun` frozen "missing"). Only peripherals are +// mocked: the dashboard auth gate (rbac/session), the downstream CancelTaskRunService, the mollifier +// buffer (disabled), and the toast/redirect formatting. The run read and the found/not-found decision +// run for real. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// ---- Holders wired into the mocked module singletons before each action() call. ------------------ +// `primaryHolder.client` -> the real container (the writer / owning primary). +// `replicaHolder.client` -> a lagging replica over the SAME container: `taskRun` reads come back empty +// (row "not replicated yet"), every other model + all writes forward to the real container. +const primaryHolder = vi.hoisted(() => ({ client: undefined as any })); +const replicaHolder = vi.hoisted(() => ({ client: undefined as any })); + +// Records every CancelTaskRunService.call(run) so the test can assert the cancel was reached with the +// right run. +const cancelCalls = vi.hoisted(() => ({ runs: [] as any[] })); + +// The user the (mocked) dashboard auth resolves to — must match the seeded OrgMember so the route's +// real control-plane membership check (`prisma.project.findFirst({ ... members: { some: { userId }}}`) +// authorizes the cancel. +const AUTH = vi.hoisted(() => ({ userId: "user_cancel_guard" })); + +// ~/db.server: point the two proxies the run-store / control-plane singletons read at our holders. +// Never mocks the DB itself — the proxies forward to real testcontainer clients. Run-ops split +// handles are left undefined so runStore.server falls back to the single control-plane store +// (buildRunStore split-off): writer = `prisma`, replica = `$replica`. That is the exact webapp +// single-DB topology this read-your-writes property lives in. +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + const lazyProxy = (holder: { client: any }, label: string) => + new Proxy( + {}, + { + get(_t, prop) { + if (!holder.client) throw new Error(`${label} not set for this test`); + const value = holder.client[prop]; + // The run-store singleton memoizes each Prisma delegate on first access; re-resolve through + // the holder so it always routes to the current test's client (mirrors the sibling + // waitpointCallback.controlPlane test's proxy). + if (value !== null && typeof value === "object") { + return new Proxy(value, { get: (_d, method) => holder.client[prop][method] }); + } + return value; + }, + } + ); + return { + prisma: lazyProxy(primaryHolder, "primaryHolder.client"), + $replica: lazyProxy(replicaHolder, "replicaHolder.client"), + // Split-off: leaving these undefined makes runStore.server build the single-DB passthrough store. + runOpsNewPrismaClient: undefined, + runOpsNewReplicaClient: undefined, + runOpsLegacyPrisma: undefined, + runOpsLegacyReplica: undefined, + sqlDatabaseSchema: Prisma.sql([`public`]), + }; +}); + +// Dashboard auth gate (peripheral): a passing ability + constant user id. The route's REAL +// control-plane membership check still runs against the seeded OrgMember. +vi.mock("~/services/rbac.server", () => ({ + rbac: { + authenticateSession: async () => ({ + ok: true, + user: { id: AUTH.userId, email: "guard@example.com", admin: false }, + ability: { can: () => true, canSuper: () => true }, + }), + }, +})); + +// Prevent the remix-auth strategy chain (auth.server validates secrets at module load) from loading +// via dashboardBuilder.server's static `getUserId` import. Auth outcome is owned by the rbac mock. +vi.mock("~/services/session.server", () => ({ + getUserId: async () => undefined, + requireUserId: async () => AUTH.userId, +})); + +// Buffer disabled: getEntry never returns an entry, so the buffer fallback is a clean miss and the +// only thing that can find the run is the run-store read (replica, then the primary re-read). +vi.mock("~/v3/mollifier/mollifierBuffer.server", () => ({ + getMollifierBuffer: () => null, +})); + +// Downstream cancel is engine work, not the read under test: record the call and no-op. +vi.mock("~/v3/services/cancelTaskRun.server", () => ({ + CancelTaskRunService: class { + async call(run: any) { + cancelCalls.runs.push(run); + } + }, +})); + +// Toast/redirect formatting is not under test; return marker Responses so assertions don't depend on +// SESSION_SECRET / cookie machinery. The success path returns a 302 here; the "Run not found" path +// returns a remix `json(...)` (200) from the real route code, which is how a failure to cancel shows up. +vi.mock("~/models/message.server", () => ({ + redirectWithSuccessMessage: async (path: string, _req: Request, message: string) => + new Response(null, { status: 302, headers: { "x-redirect": path, "x-toast": message } }), + redirectWithErrorMessage: async (path: string, _req: Request, message: string) => + new Response(null, { status: 302, headers: { "x-redirect": path, "x-error": message } }), +})); + +import { PostgresRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; +// The REAL route action under test. +import { action } from "~/routes/resources.taskruns.$runParam.cancel"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + // The user the auth mock resolves to, joined to the org so the route's real membership check passes. + await prisma.user.create({ + data: { + id: AUTH.userId, + email: `guard-${suffix}@example.com`, + authenticationMethod: "MAGIC_LINK", + }, + }); + await prisma.orgMember.create({ + data: { userId: AUTH.userId, organizationId: organization.id, role: "ADMIN" }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(p: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: p.runId, + engine: "V2", + status: "PENDING", + friendlyId: p.friendlyId, + runtimeEnvironmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: p.organizationId, + projectId: p.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: p.projectId, + organizationId: p.organizationId, + }, + }; +} + +function cancelRequest(redirectUrl: string) { + const body = new URLSearchParams({ redirectUrl }).toString(); + return new Request("http://localhost/resources/taskruns/x/cancel", { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body, + }); +} + +describe("cancel route resolves a just-created run under replica lag", () => { + heteroPostgresTest( + "cancels a live run whose row has not yet replicated", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `cancel_guard_${seq++}`; + + const seed = await seedTenant(prisma, suffix); + + // Seed the run on the PRIMARY (writer) only. The lagging replica will not see it. + const runId = `run_${"c".repeat(21)}`; // cuid-shaped -> legacy single store + const friendlyId = `run_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + // A REAL lagging replica over the same container: taskRun reads miss; everything else forwards. + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + + primaryHolder.client = prisma; + replicaHolder.client = replica.client; + + cancelCalls.runs.length = 0; + + const redirectUrl = "/orgs/x/projects/y/runs"; + const res = await action({ + request: cancelRequest(redirectUrl), + params: { runParam: friendlyId }, + context: {} as never, + }); + + // The replica WAS consulted (the read-your-writes hazard was really exercised)... + expect(replica.wasHit("taskRun")).toBe(true); + + // ...and the primary re-read found the run, so the cancel was reached with the right run. + expect(cancelCalls.runs).toHaveLength(1); + expect(cancelCalls.runs[0].friendlyId).toBe(friendlyId); + expect(cancelCalls.runs[0].id).toBe(runId); + + // The action returned the success redirect, NOT the "Run not found" json(200). + expect(res.status).toBe(302); + expect(res.headers.get("x-redirect")).toBe(redirectUrl); + expect(res.headers.get("x-toast")).toBe("Canceled run"); + + // Belt-and-braces: the body is not the "Run not found" field-error json. + const bodyText = await res.clone().text(); + expect(bodyText).not.toContain("Run not found"); + } + ); +}); diff --git a/apps/webapp/test/claimTtl.test.ts b/apps/webapp/test/claimTtl.test.ts new file mode 100644 index 000000000..36da2bbfa --- /dev/null +++ b/apps/webapp/test/claimTtl.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { computeClaimTtlSeconds } from "~/v3/mollifier/claimTtl"; + +// The mollifier idempotency claim is a serialization lock that must outlive the winner's +// create-and-publish pipeline. If its TTL is derived from the customer's key TTL alone, a short key +// TTL expires the claim mid-pipeline and a polling loser re-claims → a second creator → cross-DB dup. +describe("computeClaimTtlSeconds", () => { + const now = Date.parse("2026-01-01T00:00:00.000Z"); + const minTtlSeconds = 5; + const maxTtlSeconds = 30; + + it("floors a short key TTL at the pipeline minimum (claim can't expire mid-pipeline)", () => { + expect( + computeClaimTtlSeconds({ + keyExpiresAt: new Date(now + 2_000), + now, + minTtlSeconds, + maxTtlSeconds, + }) + ).toBe(5); + }); + + it("caps a long key TTL at the maximum", () => { + expect( + computeClaimTtlSeconds({ + keyExpiresAt: new Date(now + 3_600_000), + now, + minTtlSeconds, + maxTtlSeconds, + }) + ).toBe(30); + }); + + it("uses the key TTL when it sits between the floor and the cap", () => { + expect( + computeClaimTtlSeconds({ + keyExpiresAt: new Date(now + 12_000), + now, + minTtlSeconds, + maxTtlSeconds, + }) + ).toBe(12); + }); + + it("floors an already-expired key at the minimum", () => { + expect( + computeClaimTtlSeconds({ + keyExpiresAt: new Date(now - 1_000), + now, + minTtlSeconds, + maxTtlSeconds, + }) + ).toBe(5); + }); +}); diff --git a/apps/webapp/test/engineReplicaReads.replicaLag.guard.test.ts b/apps/webapp/test/engineReplicaReads.replicaLag.guard.test.ts new file mode 100644 index 000000000..9520d0b6e --- /dev/null +++ b/apps/webapp/test/engineReplicaReads.replicaLag.guard.test.ts @@ -0,0 +1,289 @@ +// Replica-lag properties for engine-adjacent webapp reads. Each case drives the REAL exported caller +// (resolveRunForMutation, resolveRunCommit) through a real PostgresRunStore whose owning replica is +// FROZEN via the shared `laggingReplica` primitive; only orthogonal collaborators are mocked. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import { PostgresRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import { afterEach, describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// --------------------------------------------------------------------------------------------------- +// Module singletons the webapp-service callers close over. Filled per-test; a stable Proxy keeps the +// named import binding constant while forwarding to the per-test client/store. +// --------------------------------------------------------------------------------------------------- +const H = vi.hoisted(() => { + const proxyTo = (get: () => any, label: string) => + new Proxy( + {}, + { + get(_t, prop) { + const target = get(); + if (!target) throw new Error(`${label} not set for this test`); + const value = target[prop]; + return typeof value === "function" ? value.bind(target) : value; + }, + } + ); + return { + dbHolder: { primary: undefined as any, replica: undefined as any }, + storeHolder: { store: undefined as any }, + proxyTo, + }; +}); +const { dbHolder, storeHolder } = H; + +vi.mock("~/db.server", () => ({ + prisma: H.proxyTo(() => H.dbHolder.primary, "dbHolder.primary"), + $replica: H.proxyTo(() => H.dbHolder.replica, "dbHolder.replica"), +})); + +vi.mock("~/v3/runStore.server", () => ({ + runStore: H.proxyTo(() => H.storeHolder.store, "storeHolder.store"), +})); + +// The buffer is orthogonal to resolveRunForMutation; force a clean miss so the only recovery path is +// the writer probe under test. +vi.mock("~/v3/mollifier/mollifierBuffer.server", () => ({ + getMollifierBuffer: () => null, +})); + +// dashboardAgent.server pulls a heavy import graph (SDK, GitHub app, rbac) at module load that is +// entirely orthogonal to the findRun under test — stub those leaves so the module imports cleanly. +vi.mock("~/env.server", () => ({ + env: { API_ORIGIN: "https://api.local", APP_ORIGIN: "https://app.local" }, +})); +vi.mock("~/services/gitHub.server", () => ({ githubApp: {} })); +vi.mock("~/services/logger.server", () => ({ + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); +vi.mock("@trigger.dev/rbac", () => ({ signUserActorToken: vi.fn() })); +vi.mock("@trigger.dev/sdk", () => ({ TriggerClient: class {} })); +vi.mock("@trigger.dev/sdk/ai", () => ({ chat: {} })); + +// The REAL callers under guard. +import { resolveRunForMutation } from "~/v3/mollifier/resolveRunForMutation.server"; +import { resolveRunCommit } from "~/services/dashboardAgent.server"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(p: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + lockedToVersionId?: string; +}): CreateRunInput { + return { + data: { + id: p.runId, + engine: "V2", + status: "PENDING", + friendlyId: p.friendlyId, + runtimeEnvironmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: p.organizationId, + projectId: p.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + lockedToVersionId: p.lockedToVersionId, + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: p.projectId, + organizationId: p.organizationId, + }, + }; +} + +afterEach(() => { + dbHolder.primary = undefined; + dbHolder.replica = undefined; + storeHolder.store = undefined; +}); + +// =================================================================================================== +// resolveRunForMutation findRun — a replica miss is recovered by the writer probe. Driving the REAL +// exported resolveRunForMutation under a frozen replica returns {source:"pg"} via the writer probe, +// with the replica genuinely consulted first. +// =================================================================================================== +describe("resolveRunForMutation — replica lag", () => { + heteroPostgresTest( + "a live run absent on the replica is recovered via the writer probe (source: pg)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `rrfm_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const runId = `run_${"a".repeat(21)}`; + const friendlyId = `run_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + // The runStore singleton the caller reads through: writer=primary, replica used only via the + // client the caller threads. We route findRun's client explicitly through db.server handles. + storeHolder.store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + dbHolder.primary = prisma; + dbHolder.replica = replica.client; + + const result = await resolveRunForMutation({ + runParam: friendlyId, + environmentId: seed.environment.id, + organizationId: seed.organization.id, + }); + + // The replica was consulted first (read-your-writes hazard genuinely exercised)... + expect(replica.wasHit("taskRun")).toBe(true); + // ...and the writer probe recovered the live run. + expect(result).not.toBeNull(); + expect(result?.source).toBe("pg"); + expect(result?.friendlyId).toBe(friendlyId); + } + ); + + heteroPostgresTest( + "a genuinely absent run still resolves to null (writer probe misses too)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `rrfm_absent_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + storeHolder.store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + dbHolder.primary = prisma; + dbHolder.replica = replica.client; + + const result = await resolveRunForMutation({ + runParam: "run_does_not_exist", + environmentId: seed.environment.id, + organizationId: seed.organization.id, + }); + expect(result).toBeNull(); + } + ); +}); + +// =================================================================================================== +// resolveRunCommit findRun — a live pinned run resolves its commit via the owning PRIMARY +// (findRunOnPrimary), so the frozen replica is never consulted. Routing this read to the replica would +// miss the row and return null, silently falling back to the branch head on a LIVE deployed+pinned run. +// =================================================================================================== +describe("resolveRunCommit — replica lag", () => { + heteroPostgresTest( + "a live pinned run resolves its commit via the primary", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `rrc_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + // A deployed BackgroundWorker + WorkerDeployment carrying the commit the run is pinned to. + const worker = await prisma.backgroundWorker.create({ + data: { + friendlyId: `worker_${suffix}`, + contentHash: `hash_${suffix}`, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + version: "20240101.1", + metadata: {}, + }, + }); + await prisma.workerDeployment.create({ + data: { + friendlyId: `deployment_${suffix}`, + contentHash: `hash_${suffix}`, + version: "20240101.1", + shortCode: `sc_${suffix}`, + projectId: seed.project.id, + environmentId: seed.environment.id, + workerId: worker.id, + commitSHA: "deadbeefcafe", + git: { dirty: false }, + }, + }); + + const runId = `run_${"b".repeat(21)}`; + const friendlyId = `run_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + lockedToVersionId: worker.id, + }) + ); + + // The store the caller reads through: its REPLICA is frozen (row not visible), primary is live. + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + storeHolder.store = new PostgresRunStore({ prisma, readOnlyPrisma: replica.client }); + dbHolder.primary = prisma; + dbHolder.replica = replica.client; + + const result = await resolveRunCommit(seed.environment.id, friendlyId); + + // The read routes to the owning PRIMARY (findRunOnPrimary), so the frozen replica is never + // consulted, and the live pinned run + its deployment commit resolve. + expect(replica.wasHit("taskRun")).toBe(false); + expect(result).not.toBeNull(); + expect(result?.sha).toBe("deadbeefcafe"); + expect(result?.version).toBe("20240101.1"); + expect(result?.dirty).toBe(false); + } + ); +}); diff --git a/apps/webapp/test/findEnvironmentFromRunReplicaLag.guard.test.ts b/apps/webapp/test/findEnvironmentFromRunReplicaLag.guard.test.ts new file mode 100644 index 000000000..e233931ce --- /dev/null +++ b/apps/webapp/test/findEnvironmentFromRunReplicaLag.guard.test.ts @@ -0,0 +1,196 @@ +// Verifies findEnvironmentFromRun resolves a live, primary-resident run whose row has not yet +// replicated, so the runMetadataUpdated handler keeps the metadata write + realtime publish. Drives the +// REAL exported caller as the handler does (via $replica, no tx) over a real Postgres testcontainer with +// taskRun frozen "missing" on the replica; only the control-plane env resolver + db.server proxies are mocked. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// Holders wired into the mocked db.server singletons before each call. +// primaryHolder.client -> the real container (writer / owning primary). +// replicaHolder.client -> a lagging replica over the SAME container: taskRun reads come back empty. +const primaryHolder = vi.hoisted(() => ({ client: undefined as any })); +const replicaHolder = vi.hoisted(() => ({ client: undefined as any })); + +// The env the (mocked) control-plane resolver returns, plus a record of the ids it was asked for, so +// the test can prove the caller reached the env-resolve with the run's real runtimeEnvironmentId. +const resolver = vi.hoisted(() => ({ + env: undefined as any, + calls: [] as string[], +})); + +// ~/db.server: point the two proxies the run-store singleton reads at our holders. Never mocks the DB +// itself — the proxies forward to real testcontainer clients. Run-ops split handles left undefined so +// runStore.server builds the single-DB passthrough store (writer = prisma, replica = $replica) — the +// exact webapp single-DB topology these reads run in. +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + const lazyProxy = (holder: { client: any }, label: string) => + new Proxy( + {}, + { + get(_t, prop) { + if (!holder.client) throw new Error(`${label} not set for this test`); + const value = holder.client[prop]; + // The run-store singleton memoizes each Prisma delegate on first access; re-resolve through + // the holder so it always routes to the current test's client. + if (value !== null && typeof value === "object") { + return new Proxy(value, { get: (_d, method) => holder.client[prop][method] }); + } + return value; + }, + } + ); + return { + prisma: lazyProxy(primaryHolder, "primaryHolder.client"), + $replica: lazyProxy(replicaHolder, "replicaHolder.client"), + runOpsNewPrismaClient: undefined, + runOpsNewReplicaClient: undefined, + runOpsLegacyPrisma: undefined, + runOpsLegacyReplica: undefined, + sqlDatabaseSchema: Prisma.sql([`public`]), + }; +}); + +// Downstream env hydration is orthogonal to the run read under test: return a fixed env and record +// the environmentId it was asked to resolve. This is only reached when the run read succeeds, so it +// doubles as a witness that the caller got past the null-guard. +vi.mock("~/v3/runOpsMigration/controlPlaneResolver.server", () => ({ + controlPlaneResolver: { + resolveAuthenticatedEnv: async (environmentId: string) => { + resolver.calls.push(environmentId); + return resolver.env; + }, + }, +})); + +import { PostgresRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; +// The REAL exported caller under test. +import { findEnvironmentFromRun } from "~/models/runtimeEnvironment.server"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(p: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: p.runId, + engine: "V2", + status: "PENDING", + friendlyId: p.friendlyId, + runtimeEnvironmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: p.organizationId, + projectId: p.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: ["alpha", "beta"], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: p.projectId, + organizationId: p.organizationId, + }, + }; +} + +describe("findEnvironmentFromRun resolves a primary-resident run under replica lag", () => { + heteroPostgresTest( + "a run not yet on the replica resolves via the primary re-read (non-null, keeping the metadata write)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `envfromrun_${seq++}`; + + const seed = await seedTenant(prisma, suffix); + + // Seed the run on the PRIMARY (writer) only. The lagging replica will not see it. + const runId = `run_${"e".repeat(21)}`; + const friendlyId = `run_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + // A REAL lagging replica over the same container: taskRun reads miss; everything else forwards. + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + + primaryHolder.client = prisma; + replicaHolder.client = replica.client; + + // The env the downstream resolver returns once the run is found. + resolver.env = { id: seed.environment.id, __marker: "resolved-env" }; + resolver.calls.length = 0; + + // Drive the REAL exported caller exactly as the runMetadataUpdated handler does — no tx, so the + // read routes to $replica (the lagging replica). + const result = await findEnvironmentFromRun(runId); + + // The replica WAS consulted (the lag was really exercised on taskRun), yet the caller resolves + // this live, primary-resident run to a non-null result. + expect(replica.wasHit("taskRun")).toBe(true); + expect(result).not.toBeNull(); + expect(result!.runTags).toEqual(["alpha", "beta"]); + expect(result!.batchId).toBeNull(); + expect(result!.environment).toEqual(resolver.env); + + // The env-resolve was reached with the run's real runtimeEnvironmentId (past the null-guard). + expect(resolver.calls).toEqual([seed.environment.id]); + } + ); +}); diff --git a/apps/webapp/test/idempotencyExpiredRecreateReserialize.test.ts b/apps/webapp/test/idempotencyExpiredRecreateReserialize.test.ts new file mode 100644 index 000000000..364ce4685 --- /dev/null +++ b/apps/webapp/test/idempotencyExpiredRecreateReserialize.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it, vi } from "vitest"; + +// Devin #2: a global-scope (or scope-absent) key whose existing run is EXPIRED/FAILED gets its key +// cleared and recreated. Under the run-ops split that recreate must serialise through the claim (same +// cross-DB dup risk as the claim-loser cleared path) rather than fall through to an unserialised create. +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: {}, + runOpsNewPrisma: {}, + runOpsLegacyPrisma: {}, + runOpsNewReplica: {}, + runOpsLegacyReplica: {}, +})); + +const h = vi.hoisted(() => ({ existingRun: null as unknown, splitEnabled: true })); + +vi.mock("~/v3/runStore.server", () => ({ + runStore: { findRun: vi.fn(async () => h.existingRun) }, +})); +vi.mock("~/v3/mollifier/mollifierBuffer.server", () => ({ getMollifierBuffer: () => null })); +vi.mock("~/v3/mollifier/mollifierGate.server", () => ({ + makeResolveMollifierFlag: () => async () => false, +})); +vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ + isSplitEnabled: async () => h.splitEnabled, +})); +vi.mock("~/runEngine/concerns/idempotencyResidency.server", () => ({ + resolveIdempotencyDedupClient: async () => ({}), +})); + +import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server"; +import type { TriggerTaskRequest } from "~/runEngine/types"; + +function makeRequest(): TriggerTaskRequest { + return { + taskId: "my-task", + environment: { id: "env_a", organizationId: "org_1", organization: { featureFlags: {} } }, + options: {}, + body: { options: { idempotencyKey: "k-1" } }, // scope absent → treated as global + } as unknown as TriggerTaskRequest; +} + +// handleExistingRun's documented contract for an expired/failed run: clear the key, return isCached:false. +const CLEARED = { + isCached: false as const, + idempotencyKey: "k-1", + idempotencyKeyExpiresAt: new Date(Date.now() + 60_000), +}; + +describe("IdempotencyKeyConcern · expired/failed recreate re-serialisation", () => { + it("routes the cleared recreate through the claim under global-scope split (no unserialised create)", async () => { + h.existingRun = { id: "run_internal", friendlyId: "run_friendly" }; + h.splitEnabled = true; + const concern = new IdempotencyKeyConcern({} as never, {} as never, {} as never); + vi.spyOn( + concern as never as { handleExistingRun: () => unknown }, + "handleExistingRun" + ).mockResolvedValue(CLEARED); + const SENTINEL = { ...CLEARED, claim: { token: "t" } }; + const reacquire = vi + .spyOn( + concern as never as { reacquireClearedGlobalWinner: () => unknown }, + "reacquireClearedGlobalWinner" + ) + .mockResolvedValue(SENTINEL); + + const result = await concern.handleTriggerRequest(makeRequest(), undefined); + + expect(reacquire).toHaveBeenCalledOnce(); + expect(result).toBe(SENTINEL); + }); + + it("does NOT re-serialise when the split is off — plain recreate", async () => { + h.existingRun = { id: "run_internal", friendlyId: "run_friendly" }; + h.splitEnabled = false; + const concern = new IdempotencyKeyConcern({} as never, {} as never, {} as never); + vi.spyOn( + concern as never as { handleExistingRun: () => unknown }, + "handleExistingRun" + ).mockResolvedValue(CLEARED); + const reacquire = vi + .spyOn( + concern as never as { reacquireClearedGlobalWinner: () => unknown }, + "reacquireClearedGlobalWinner" + ) + .mockResolvedValue({} as never); + + const result = await concern.handleTriggerRequest(makeRequest(), undefined); + + expect(reacquire).not.toHaveBeenCalled(); + expect(result).toBe(CLEARED); + }); +}); diff --git a/apps/webapp/test/idempotencyGlobalScopeCrossDbConcurrent.test.ts b/apps/webapp/test/idempotencyGlobalScopeCrossDbConcurrent.test.ts new file mode 100644 index 000000000..b501cb553 --- /dev/null +++ b/apps/webapp/test/idempotencyGlobalScopeCrossDbConcurrent.test.ts @@ -0,0 +1,930 @@ +// CONCURRENT cross-DB GLOBAL-scope idempotency dedup for the run-ops split: one global key triggered +// concurrently from two parents on DIFFERENT physical DBs still yields exactly ONE child. +// +// The hard case dedup fan-out alone cannot cover: both triggers' dedup probes run BEFORE either child +// is created, so each probe misses and the per-DB unique index on (runtimeEnvironmentId, +// taskIdentifier, idempotencyKey) cannot enforce cross-DB uniqueness. The Redis idempotency-claim +// primitive (claimOrAwait/publishClaim) is the cross-DB mutual-exclusion gate for global-scope keys +// while the split is active, regardless of the per-org mollifier flag: the claim loser resolves the +// winner by id across both DBs and returns it as a cached hit → one child. +// +// Setup: drives the REAL IdempotencyKeyConcern.handleTriggerRequest against a REAL two-physical-DB +// RoutingRunStore (heteroPostgresTest, never mocked on the read/dedup path) AND a REAL MollifierBuffer +// over a Redis testcontainer. SETNX arbitration runs for real — one claimant wins, the other pends +// and polls until the winner publishes; only the caller's create + publishClaim glue is played here. +// +// Determinism: the loser's contended claim genuinely returns "pending" (it has probed PG, missed, and +// is now blocked on the winner). A barrier opens on that real "pending" outcome so the winner holds +// its create+publish until both claimants contend, then publishes well inside the loser's poll +// deadline — so the loser resolves on every run, not on wall-clock luck. + +import { + heteroPostgresTest, + network, + redisContainer, + redisOptions, + type StartedNetwork, + type StartedRedisContainer, +} from "@internal/testcontainers"; +import { PostgresRunStore, RoutingRunStore } from "@internal/run-store"; +import { generateRunOpsId, RunId } from "@trigger.dev/core/v3/isomorphic"; +import type { PrismaClient, TaskRunStatus } from "@trigger.dev/database"; +import { MollifierBuffer } from "@trigger.dev/redis-worker"; +import type { IdempotencyClaimResult } from "@trigger.dev/redis-worker"; +import type { RedisOptions } from "ioredis"; +import { describe, expect, vi } from "vitest"; + +// hookTimeout is bumped because the per-test bufferHarness fixture closes a real Redis client +// (MollifierBuffer.close → redis.quit) during teardown, which can outrun the 10s default. +vi.setConfig({ testTimeout: 60_000, hookTimeout: 30_000 }); + +// --- Module wiring ----------------------------------------------------------------------------- +// Hoisted holders so the (import-time) mocks below can defer to per-test values. +const h = vi.hoisted(() => ({ + router: null as unknown, + buffer: null as unknown, + splitOn: true, +})); + +// The concern reads `runStore` (module singleton) for both the id-less dedup probe and the +// by-id winner resolution. Point it at the per-test RoutingRunStore built over the two real +// containers. +vi.mock("~/v3/runStore.server", () => ({ + runStore: new Proxy( + {}, + { + get(_t, prop) { + const target = h.router as Record; + const value = target[prop]; + return typeof value === "function" ? value.bind(target) : value; + }, + } + ), +})); + +// Real claim algorithm (claimOrAwait/publishClaim) backed by a REAL MollifierBuffer over a Redis +// testcontainer (see buildRealBuffer). The read/dedup path (PG findRun) stays real too. +vi.mock("~/v3/mollifier/mollifierBuffer.server", () => ({ + getMollifierBuffer: () => h.buffer, +})); + +// Split ON so resolveIdempotencyDedupClient routes the dedup client by the parent's residency and +// runStore behaves as the RoutingRunStore. +vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ + isSplitEnabled: async () => h.splitOn, +})); + +// The run-ops db.server handles are only ever used as the dedup-client SENTINEL under routing +// (their identity is never forwarded to a routed store — only their presence signals +// read-your-writes). Truthy placeholders suffice; the real containers are reached via the router. +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: {}, + runOpsNewPrisma: { __sentinel: "new" }, + runOpsLegacyPrisma: { __sentinel: "legacy" }, + runOpsNewReplica: {}, + runOpsLegacyReplica: {}, +})); + +import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server"; +import { publishClaim } from "~/v3/mollifier/idempotencyClaim.server"; +import type { TraceEventConcern, TriggerTaskRequest } from "~/runEngine/types"; + +// --- Real MollifierBuffer over Redis, instrumented for the barrier ----------------------------- +// Builds the SAME MollifierBuffer the webapp uses (constructed with { redisOptions }), pointed at +// the per-test Redis container. The instrumentation is a thin observability wrapper only: it counts +// claim attempts and opens a barrier when a claim genuinely returns "pending" (a real SETNX loser). +// The SETNX itself, publishClaim and readClaim are the buffer's real Redis ops — nothing is faked. +type BufferHarness = { + buffer: MollifierBuffer; + secondClaimPending: Promise; + // Resolves once TWO claimants have genuinely returned "pending" — i.e. both + // losers of a winner + two-loser trio have probed PG (missed) and are + // blocked on the winner. Used by the EXPIRED/FAILED reacquire cases so the + // winner holds its create+publish until BOTH losers contend, guaranteeing + // both take the claim-loser path (not a PG hit) on the SAME resolved winner. + bothLosersPending: Promise; + readonly claimCalls: number; + close: () => Promise; +}; + +function buildRealBuffer(options: RedisOptions): BufferHarness { + const real = new MollifierBuffer({ redisOptions: options }); + let claimCalls = 0; + let pendingCount = 0; + let resolveSecondPending!: () => void; + const secondClaimPending = new Promise((r) => (resolveSecondPending = r)); + let resolveBothLosersPending!: () => void; + const bothLosersPending = new Promise((r) => (resolveBothLosersPending = r)); + + const buffer = new Proxy(real, { + get(target, prop, receiver) { + if (prop === "claimIdempotency") { + return async ( + input: Parameters[0] + ): Promise => { + claimCalls += 1; + const result = await target.claimIdempotency(input); + // A real "pending" means the SETNX already had a winner and THIS caller is the loser: it + // has probed PG (missed) and is now blocked on the winner. Open the barrier so the winner + // can proceed to create + publish, guaranteeing both claimants contend first. + if (result.kind === "pending") { + pendingCount += 1; + resolveSecondPending(); + if (pendingCount >= 2) resolveBothLosersPending(); + } + return result; + }; + } + const value = Reflect.get(target, prop, receiver); + return typeof value === "function" + ? (value as (...a: unknown[]) => unknown).bind(target) + : value; + }, + }) as unknown as MollifierBuffer; + + return { + buffer, + secondClaimPending, + bothLosersPending, + get claimCalls() { + return claimCalls; + }, + close: () => real.close(), + }; +} + +// heteroPostgresTest gives us the two physical Postgres DBs (PG14 legacy + PG17 new). Compose a +// per-test Redis container alongside (the isolatedRedisTest / postgresAndRedisTest precedent: +// network + redisContainer + redisOptions), then hand each test a real MollifierBuffer over it. The +// harness is a fixture so its Redis client is closed on teardown WHILE the container is still up +// (fixtures tear down in reverse: harness before redisContainer), never leaking a client. +const heteroPgWithRedisTest = heteroPostgresTest.extend<{ + network: StartedNetwork; + redisContainer: StartedRedisContainer; + redisOptions: RedisOptions; + bufferHarness: BufferHarness; +}>({ + network, + redisContainer, + redisOptions, + bufferHarness: async ({ redisOptions }, use) => { + const harness = buildRealBuffer(redisOptions); + try { + await use(harness); + } finally { + await harness.close(); + } + }, +}); + +// --- Real split RoutingRunStore over the two containers ---------------------------------------- +function makeSplitRouter(prisma14: PrismaClient, prisma17: PrismaClient) { + const legacyStore = new PostgresRunStore({ prisma: prisma14, readOnlyPrisma: prisma14 }); + const newStore = new PostgresRunStore({ prisma: prisma17, readOnlyPrisma: prisma17 }); + return new RoutingRunStore({ new: newStore, legacy: legacyStore }); +} + +type SharedEnv = { + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}; + +// Seed the SAME logical env (identical scalar ids) on BOTH physical DBs so a child's FK-bearing +// create resolves whichever DB its id-shape routes it to. +async function seedSharedEnv( + prisma14: PrismaClient, + prisma17: PrismaClient, + suffix: string +): Promise { + const organizationId = `org_${suffix}`; + const projectId = `proj_${suffix}`; + const runtimeEnvironmentId = `env_${suffix}`; + for (const prisma of [prisma14, prisma17]) { + await prisma.organization.create({ + data: { id: organizationId, title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + await prisma.project.create({ + data: { + id: projectId, + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId, + }, + }); + await prisma.runtimeEnvironment.create({ + data: { + id: runtimeEnvironmentId, + type: "DEVELOPMENT", + slug: "dev", + projectId, + organizationId, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + } + return { organizationId, projectId, runtimeEnvironmentId }; +} + +function makeCreateRunInput(params: { + runId: string; + friendlyId: string; + env: SharedEnv; + idempotencyKey: string; + taskIdentifier: string; + // Winner-seed overrides: an EXPIRED-key winner (expiry in the past) or a + // FAILED-status winner, so the resolved-winner clear-and-recreate branch of + // handleExistingRun fires. Default is a live PENDING run with a 24h key. + status?: TaskRunStatus; + idempotencyKeyExpiresAt?: Date; +}) { + return { + data: { + id: params.runId, + engine: "V2" as const, + status: params.status ?? ("PENDING" as const), + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.env.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: params.env.organizationId, + projectId: params.env.projectId, + idempotencyKey: params.idempotencyKey, + idempotencyKeyExpiresAt: + params.idempotencyKeyExpiresAt ?? new Date(Date.now() + 24 * 60 * 60 * 1000), + taskIdentifier: params.taskIdentifier, + payload: '{"hello":"world"}', + payloadType: "application/json", + traceId: `trace_${params.runId}`, + spanId: `span_${params.runId}`, + queue: "task/my-task", + isTest: false, + depth: 1, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: params.env.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + projectId: params.env.projectId, + organizationId: params.env.organizationId, + }, + }; +} + +type Residency = "NEW" | "LEGACY"; + +// A NEW-resident id (run-ops id) → routes to #new; a LEGACY-resident id (cuid) → routes to #legacy. +function mintChildId(residency: Residency): string { + return residency === "NEW" ? generateRunOpsId() : RunId.generate().id; +} +function mintParentFriendlyId(residency: Residency): string { + return `run_${residency === "NEW" ? generateRunOpsId() : RunId.generate().id}`; +} + +type TriggerSpec = { + parentResidency: Residency; + idempotencyKey: string; + scope?: "run" | "attempt" | "global"; + taskIdentifier: string; + // andWait: block this parent on the resolved run's waitpoint. + resumeParentOnCompletion?: boolean; + // Explicit parent friendlyId (a seeded parent) — overrides the random mint. Required for the + // andWait path so the parent-in-caller-env validation can find a real row. + parentRunFriendlyId?: string; +}; + +function makeRequest(env: SharedEnv, spec: TriggerSpec): TriggerTaskRequest { + return { + taskId: spec.taskIdentifier, + environment: { + id: env.runtimeEnvironmentId, + organizationId: env.organizationId, + projectId: env.projectId, + // No mollifierEnabled override → the per-org mollifier flag resolves false, so the ONLY thing + // that can make the claim eligible is the global-under-split rule. + organization: { featureFlags: {} }, + }, + options: {}, + body: { + options: { + idempotencyKey: spec.idempotencyKey, + parentRunId: spec.parentRunFriendlyId ?? mintParentFriendlyId(spec.parentResidency), + ...(spec.resumeParentOnCompletion ? { resumeParentOnCompletion: true } : {}), + ...(spec.scope ? { idempotencyKeyOptions: { key: "user-key", scope: spec.scope } } : {}), + }, + }, + } as unknown as TriggerTaskRequest; +} + +async function countChildren( + prisma14: PrismaClient, + prisma17: PrismaClient, + env: SharedEnv, + idempotencyKey: string, + taskIdentifier: string +) { + const where = { + runtimeEnvironmentId: env.runtimeEnvironmentId, + idempotencyKey, + taskIdentifier, + }; + const legacy = await prisma14.taskRun.count({ where }); + const nw = await prisma17.taskRun.count({ where }); + return { legacy, new: nw, total: legacy + nw }; +} + +// Drive two triggers whose dedup PROBES are both forced to run before either child is created. +// `A` is the first (winning) trigger; `B` is the second (losing) trigger. Returns each outcome. +async function driveConcurrentPair(opts: { + concern: IdempotencyKeyConcern; + router: RoutingRunStore; + buffer: { secondClaimPending: Promise }; + env: SharedEnv; + a: { spec: TriggerSpec; childResidency: Residency }; + b: { spec: TriggerSpec; childResidency: Residency }; +}) { + const { concern, router, env } = opts; + + const childA = mintChildId(opts.a.childResidency); + const childAFriendly = `run_${childA}`; + const childB = mintChildId(opts.b.childResidency); + const childBFriendly = `run_${childB}`; + + const reqA = makeRequest(env, opts.a.spec); + const reqB = makeRequest(env, opts.b.spec); + + // Phase 1: A probes + (maybe) claims. Runs to completion but creates NOTHING — the create is the + // caller's job, played below. A is the first claimant, so it wins the SETNX and returns immediately. + const aRes = await concern.handleTriggerRequest(reqA, undefined); + + let resolveBReturned!: () => void; + const bReturned = new Promise((r) => (resolveBReturned = r)); + + const createChild = (childId: string, friendlyId: string, spec: TriggerSpec) => + router.createRun( + makeCreateRunInput({ + runId: childId, + friendlyId, + env, + idempotencyKey: spec.idempotencyKey, + taskIdentifier: spec.taskIdentifier, + }) as never + ); + + // B: probe + (maybe) claim-wait. This blocks on the claim until A publishes. + const bFlow = (async () => { + const res = await concern.handleTriggerRequest(reqB, undefined); + resolveBReturned(); + if (!res.isCached) { + await createChild(childB, childBFriendly, opts.b.spec); + if (res.claim) { + await publishClaim({ ...res.claim, runId: childBFriendly }); + } + } + return res; + })(); + + // A: create + publish, but hold the create open until B has entered the claim wait (via the real + // "pending" barrier) or fully returned without a claim (the negative control) — so B's probe + // always precedes A's create. + const aFlow = (async () => { + if (!aRes.isCached) { + await Promise.race([opts.buffer.secondClaimPending, bReturned]); + await createChild(childA, childAFriendly, opts.a.spec); + if (aRes.claim) { + await publishClaim({ ...aRes.claim, runId: childAFriendly }); + } + } + })(); + + const [, bRes] = await Promise.all([aFlow, bFlow]); + return { aRes, bRes, childAFriendly, childBFriendly }; +} + +// Drive ONE winner + TWO losers against a single global key. The winner (A) wins the claim and +// creates an EXPIRED- or FAILED-status child, then publishes it — but HOLDS the create until BOTH +// losers (B, C) have probed PG (missed) and are blocked on the claim (`bothLosersPending`). Both +// losers then resolve to the winner's cleared child. A single loser recreating a cleared winner is +// correct; the DUPLICATE only appears when TWO losers clear the same winner and each recreates on a +// different DB (no cross-DB unique backstop) — hence a trio, not a pair. The winner-seed overrides +// (status / idempotencyKeyExpiresAt) pick which clear branch of handleExistingRun fires. +async function driveWinnerPlusTwoLosers(opts: { + concern: IdempotencyKeyConcern; + router: RoutingRunStore; + buffer: { bothLosersPending: Promise }; + env: SharedEnv; + idempotencyKey: string; + taskIdentifier: string; + winnerSeed: { status?: TaskRunStatus; idempotencyKeyExpiresAt?: Date }; + a: { spec: TriggerSpec; childResidency: Residency }; + b: { spec: TriggerSpec; childResidency: Residency }; + c: { spec: TriggerSpec; childResidency: Residency }; +}) { + const { concern, router, env } = opts; + + const childW = mintChildId(opts.a.childResidency); + const childWFriendly = `run_${childW}`; + const childB = mintChildId(opts.b.childResidency); + const childBFriendly = `run_${childB}`; + const childC = mintChildId(opts.c.childResidency); + const childCFriendly = `run_${childC}`; + + const reqA = makeRequest(env, opts.a.spec); + const reqB = makeRequest(env, opts.b.spec); + const reqC = makeRequest(env, opts.c.spec); + + const createChild = ( + childId: string, + friendlyId: string, + spec: TriggerSpec, + seed?: { status?: TaskRunStatus; idempotencyKeyExpiresAt?: Date } + ) => + router.createRun( + makeCreateRunInput({ + runId: childId, + friendlyId, + env, + idempotencyKey: spec.idempotencyKey, + taskIdentifier: spec.taskIdentifier, + ...seed, + }) as never + ); + + // Phase 1: A probes + wins the claim (no create yet — that's the caller's job below). + const aRes = await concern.handleTriggerRequest(reqA, undefined); + + // Loser flow: probe + claim-wait; on a NON-cached return, create its child and (if it holds a + // reacquired claim) publish it. Exactly one loser reacquires the claim and creates; the other + // resolves to that fresh run as a cached hit and creates nothing. + const loserFlow = ( + req: TriggerTaskRequest, + childId: string, + friendlyId: string, + spec: TriggerSpec + ) => + (async () => { + const res = await concern.handleTriggerRequest(req, undefined); + if (!res.isCached) { + await createChild(childId, friendlyId, spec); + if (res.claim) { + await publishClaim({ ...res.claim, runId: friendlyId }); + } + } + return res; + })(); + + const bFlow = loserFlow(reqB, childB, childBFriendly, opts.b.spec); + const cFlow = loserFlow(reqC, childC, childCFriendly, opts.c.spec); + + // A: hold the winner create+publish until BOTH losers are blocked on the claim, so both probe + // BEFORE the winner child exists and both take the claim-loser path on the SAME resolved winner. + const aFlow = (async () => { + if (!aRes.isCached) { + await opts.buffer.bothLosersPending; + await createChild(childW, childWFriendly, opts.a.spec, opts.winnerSeed); + if (aRes.claim) { + await publishClaim({ ...aRes.claim, runId: childWFriendly }); + } + } + })(); + + const [, bRes, cRes] = await Promise.all([aFlow, bFlow, cFlow]); + return { aRes, bRes, cRes, childWFriendly, childBFriendly, childCFriendly }; +} + +describe("run-ops split — CONCURRENT global-scope idempotency dedup across two different-residency parents", () => { + heteroPgWithRedisTest( + "global scope, NEW-parent winner + LEGACY-parent loser: exactly one child, loser resolves to winner", + async ({ prisma14, prisma17, bufferHarness }) => { + const env = await seedSharedEnv(prisma14, prisma17, "gcc_a"); + const router = makeSplitRouter(prisma14, prisma17); + h.router = router; + h.buffer = bufferHarness.buffer; + h.splitOn = true; + + const idempotencyKey = "global-key-concurrent-a"; + const taskIdentifier = "child-task"; + const concern = new IdempotencyKeyConcern(prisma14 as never, {} as never, {} as never); + + const { bRes, childAFriendly } = await driveConcurrentPair({ + concern, + router, + buffer: bufferHarness, + env, + a: { + spec: { parentResidency: "NEW", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "NEW", + }, + b: { + spec: { parentResidency: "LEGACY", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "LEGACY", + }, + }); + + const counts = await countChildren(prisma14, prisma17, env, idempotencyKey, taskIdentifier); + + // Load-bearing assertion: one global key ⇒ exactly ONE child, across two physical DBs. + expect(counts.total).toBe(1); + // The loser must be a cached hit resolving to the winner's run. + expect(bRes.isCached).toBe(true); + if (bRes.isCached === true) { + expect(bRes.run.friendlyId).toBe(childAFriendly); + } + // The claim was actually contended (both triggers hit the real SETNX) — proves the mutex was + // exercised for real. + expect(bufferHarness.claimCalls).toBeGreaterThanOrEqual(2); + } + ); + + heteroPgWithRedisTest( + "global scope, LEGACY-parent winner + NEW-parent loser: exactly one child (symmetric)", + async ({ prisma14, prisma17, bufferHarness }) => { + const env = await seedSharedEnv(prisma14, prisma17, "gcc_b"); + const router = makeSplitRouter(prisma14, prisma17); + h.router = router; + h.buffer = bufferHarness.buffer; + h.splitOn = true; + + const idempotencyKey = "global-key-concurrent-b"; + const taskIdentifier = "child-task"; + const concern = new IdempotencyKeyConcern(prisma14 as never, {} as never, {} as never); + + const { bRes, childAFriendly } = await driveConcurrentPair({ + concern, + router, + buffer: bufferHarness, + env, + a: { + spec: { parentResidency: "LEGACY", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "LEGACY", + }, + b: { + spec: { parentResidency: "NEW", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "NEW", + }, + }); + + const counts = await countChildren(prisma14, prisma17, env, idempotencyKey, taskIdentifier); + expect(counts.total).toBe(1); + expect(bRes.isCached).toBe(true); + if (bRes.isCached === true) { + expect(bRes.run.friendlyId).toBe(childAFriendly); + } + } + ); + + heteroPgWithRedisTest( + "global scope andWait loser: exactly one child AND the loser's parent (other DB) is blocked on the WINNER's run waitpoint", + async ({ prisma14, prisma17, bufferHarness }) => { + // The by-id cross-DB resolution (resolveWinnerAcrossDbs) the plain cases don't fully assert: + // the LOSER is an andWait trigger whose parent lives on the LEGACY DB, while the WINNER's child + // lives on the NEW DB. When the loser resolves the claim it must (1) dedup to the single winner + // child (across DBs), and (2) wire ITS parent's waitpoint against the WINNER's run — i.e. find + // the winner on the other DB, get/create the winner's run waitpoint, and block the loser's + // parent on that waitpoint. A real engine can't span two physical DBs, so we stub the two + // waitpoint methods to record their args; the resolution + routing that feeds them is real. + const env = await seedSharedEnv(prisma14, prisma17, "gcc_andwait"); + const router = makeSplitRouter(prisma14, prisma17); + h.router = router; + h.buffer = bufferHarness.buffer; + h.splitOn = true; + + const idempotencyKey = "global-key-concurrent-andwait"; + const taskIdentifier = "child-task"; + const parentTaskIdentifier = "parent-task"; + + // The loser's parent lives on the LEGACY (PG14) DB. Its friendlyId is a cuid-shaped run id so + // RunId.fromFriendlyId → routes the parent-in-caller-env validation to the legacy store. + const loserParent = RunId.generate(); + await prisma14.taskRun.create({ + data: { + id: loserParent.id, + friendlyId: loserParent.friendlyId, + engine: "V2", + status: "EXECUTING", + runtimeEnvironmentId: env.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: env.organizationId, + projectId: env.projectId, + taskIdentifier: parentTaskIdentifier, + payload: "{}", + payloadType: "application/json", + traceId: `trace_${loserParent.id}`, + spanId: `span_${loserParent.id}`, + queue: "task/parent-task", + isTest: false, + depth: 0, + }, + }); + + // Stub engine: record the winner waitpoint get/create + the parent block. + const getOrCreateCalls: Array<{ runId: string; projectId: string; environmentId: string }> = + []; + const blockCalls: Array<{ runId: string; waitpoints: string | string[] }> = []; + const winnerWaitpoint = { + id: "waitpoint_winner_run", + status: "PENDING" as const, + outputIsError: false, + }; + const stubEngine = { + async getOrCreateRunWaitpoint(input: { + runId: string; + projectId: string; + environmentId: string; + }) { + getOrCreateCalls.push(input); + return winnerWaitpoint; + }, + async blockRunWithWaitpoint(input: { runId: string; waitpoints: string | string[] }) { + blockCalls.push({ runId: input.runId, waitpoints: input.waitpoints }); + return {}; + }, + }; + + // Stub traceEventConcern: just run the callback with a minimal span (the andWait path reads + // event.spanId / event.traceparent). + const stubTrace: Partial = { + async traceIdempotentRun(_request, _parentStore, _options, callback) { + return callback( + { + spanId: "span_trace", + traceparent: undefined, + traceId: "trace_x", + traceContext: {}, + setAttribute: () => {}, + failWithError: () => {}, + stop: () => {}, + } as never, + "test" + ); + }, + }; + + const concern = new IdempotencyKeyConcern( + prisma14 as never, + stubEngine as never, + stubTrace as never + ); + + const { bRes, childAFriendly } = await driveConcurrentPair({ + concern, + router, + buffer: bufferHarness, + env, + a: { + // Winner: NEW parent, plain global trigger, NEW-resident child (lands on the NEW DB). + spec: { parentResidency: "NEW", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "NEW", + }, + b: { + // Loser: andWait, LEGACY parent (seeded above, on the OTHER DB), same global key. + spec: { + parentResidency: "LEGACY", + idempotencyKey, + scope: "global", + taskIdentifier, + resumeParentOnCompletion: true, + parentRunFriendlyId: loserParent.friendlyId, + }, + childResidency: "LEGACY", + }, + }); + + const counts = await countChildren(prisma14, prisma17, env, idempotencyKey, taskIdentifier); + const winnerInternalId = RunId.fromFriendlyId(childAFriendly); + + // (1) Dedup held across DBs: exactly ONE child, and the loser is a cached hit to the winner. + expect(counts.total).toBe(1); + expect(counts.new).toBe(1); // the winner child lives on the NEW DB + expect(counts.legacy).toBe(0); // the loser NEVER created a legacy-DB child + expect(bRes.isCached).toBe(true); + if (bRes.isCached === true) { + expect(bRes.run.friendlyId).toBe(childAFriendly); + } + + // (2) Cross-DB waitpoint linkage: resolveWinnerAcrossDbs found the winner on the NEW DB, and + // the loser got/created the waitpoint against the WINNER's run id (not its own parent's). + expect(getOrCreateCalls).toHaveLength(1); + expect(getOrCreateCalls[0]!.runId).toBe(winnerInternalId); + expect(getOrCreateCalls[0]!.environmentId).toBe(env.runtimeEnvironmentId); + + // (3) …and the loser's parent (which lives on the LEGACY DB) is blocked on the WINNER's run + // waitpoint — the by-id cross-DB resolution wired one DB's parent to the other DB's run. + expect(blockCalls).toHaveLength(1); + expect(blockCalls[0]!.runId).toBe(loserParent.id); + expect(blockCalls[0]!.waitpoints).toBe(winnerWaitpoint.id); + + // The claim was contended for real (both triggers hit the SETNX). + expect(bufferHarness.claimCalls).toBeGreaterThanOrEqual(2); + } + ); + + heteroPgWithRedisTest( + "options-absent under split (pre-hashed key / older SDK): treated conservatively → one child", + async ({ prisma14, prisma17, bufferHarness }) => { + const env = await seedSharedEnv(prisma14, prisma17, "gcc_c"); + const router = makeSplitRouter(prisma14, prisma17); + h.router = router; + h.buffer = bufferHarness.buffer; + h.splitOn = true; + + const idempotencyKey = "prehashed-key-concurrent-c"; + const taskIdentifier = "child-task"; + const concern = new IdempotencyKeyConcern(prisma14 as never, {} as never, {} as never); + + const { bRes } = await driveConcurrentPair({ + concern, + router, + buffer: bufferHarness, + env, + a: { + // No `scope` → no idempotencyKeyOptions on the wire (pre-hashed key / older SDK). + spec: { parentResidency: "NEW", idempotencyKey, taskIdentifier }, + childResidency: "NEW", + }, + b: { + spec: { parentResidency: "LEGACY", idempotencyKey, taskIdentifier }, + childResidency: "LEGACY", + }, + }); + + const counts = await countChildren(prisma14, prisma17, env, idempotencyKey, taskIdentifier); + expect(counts.total).toBe(1); + expect(bRes.isCached).toBe(true); + } + ); + + heteroPgWithRedisTest( + "global scope, EXPIRED-key winner cleared by TWO concurrent losers: exactly one NEW child (reacquired)", + async ({ prisma14, prisma17, bufferHarness }) => { + // The critical residual this covers: the resolved winner's key is EXPIRED, so each loser's + // handleExistingRun clears it and would recreate a NEW run. Two losers on different-residency + // parents recreate on different DBs where the per-DB unique index can't dedup them, so the + // recreate must re-serialise through the claim — one loser reacquires + creates, the other + // resolves to it → exactly ONE new child. + const env = await seedSharedEnv(prisma14, prisma17, "gcc_expired"); + const router = makeSplitRouter(prisma14, prisma17); + h.router = router; + h.buffer = bufferHarness.buffer; + h.splitOn = true; + + const idempotencyKey = "global-key-expired-winner"; + const taskIdentifier = "child-task"; + const concern = new IdempotencyKeyConcern(prisma14 as never, {} as never, {} as never); + + const { bRes, cRes, childWFriendly, childBFriendly, childCFriendly } = + await driveWinnerPlusTwoLosers({ + concern, + router, + buffer: bufferHarness, + env, + idempotencyKey, + taskIdentifier, + // Winner child is a genuinely-expired run: an EXPIRED status AND an already-expired key, so + // handleExistingRun's expiry branch clears it. Both attributes matter for determinism: the + // expiry drives the clear; the EXPIRED status keeps the SECOND loser on the clear path even + // after the first loser's clear has NULLed the key + expiry (a plain PENDING winner would + // then look "live" to the second reader and dedup by accident, masking the duplicate). + winnerSeed: { status: "EXPIRED", idempotencyKeyExpiresAt: new Date(Date.now() - 60_000) }, + a: { + spec: { parentResidency: "NEW", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "NEW", + }, + b: { + spec: { parentResidency: "LEGACY", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "LEGACY", + }, + c: { + spec: { parentResidency: "NEW", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "NEW", + }, + }); + + // The winner's key was cleared, so only the recreated child(ren) still carry the key. Exactly + // ONE new child ⇒ the reacquire serialised the two losers' recreate across the split. + const counts = await countChildren(prisma14, prisma17, env, idempotencyKey, taskIdentifier); + expect(counts.total).toBe(1); + // Exactly one loser recreated (claim reacquired); the other resolved to it as a cached hit. + const cachedCount = [bRes, cRes].filter((r) => r.isCached).length; + expect(cachedCount).toBe(1); + // The cached loser must resolve to the RECREATED run (the other loser's fresh child), NOT the + // cleared winner — a stale isCached still pointing at childWFriendly would otherwise slip past. + const recreaterFriendly = bRes.isCached ? childCFriendly : childBFriendly; + const cachedLoser = bRes.isCached ? bRes : cRes; + if (cachedLoser.isCached === true) { + expect(cachedLoser.run.friendlyId).toBe(recreaterFriendly); + expect(cachedLoser.run.friendlyId).not.toBe(childWFriendly); + } + // Both losers genuinely contended on the claim (winner + two losers ⇒ ≥3 attempts). + expect(bufferHarness.claimCalls).toBeGreaterThanOrEqual(3); + } + ); + + heteroPgWithRedisTest( + "global scope, FAILED-status winner cleared by TWO concurrent losers: exactly one NEW child (reacquired)", + async ({ prisma14, prisma17, bufferHarness }) => { + // Same critical residual via the OTHER clear branch: the resolved winner is in a failed status + // (shouldIdempotencyKeyBeCleared === true), so handleExistingRun clears its key and each loser + // would recreate → the recreate re-serialises through the claim to yield exactly one child. + const env = await seedSharedEnv(prisma14, prisma17, "gcc_failed"); + const router = makeSplitRouter(prisma14, prisma17); + h.router = router; + h.buffer = bufferHarness.buffer; + h.splitOn = true; + + const idempotencyKey = "global-key-failed-winner"; + const taskIdentifier = "child-task"; + const concern = new IdempotencyKeyConcern(prisma14 as never, {} as never, {} as never); + + const { bRes, cRes, childWFriendly, childBFriendly, childCFriendly } = + await driveWinnerPlusTwoLosers({ + concern, + router, + buffer: bufferHarness, + env, + idempotencyKey, + taskIdentifier, + // Winner child carries a LIVE (future) key but a failed status → status-clear branch fires. + winnerSeed: { status: "COMPLETED_WITH_ERRORS" }, + a: { + spec: { parentResidency: "NEW", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "NEW", + }, + b: { + spec: { parentResidency: "LEGACY", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "LEGACY", + }, + c: { + spec: { parentResidency: "NEW", idempotencyKey, scope: "global", taskIdentifier }, + childResidency: "NEW", + }, + }); + + const counts = await countChildren(prisma14, prisma17, env, idempotencyKey, taskIdentifier); + expect(counts.total).toBe(1); + const cachedCount = [bRes, cRes].filter((r) => r.isCached).length; + expect(cachedCount).toBe(1); + // The cached loser must resolve to the RECREATED run (the other loser's fresh child), NOT the + // cleared winner — a stale isCached still pointing at childWFriendly would otherwise slip past. + const recreaterFriendly = bRes.isCached ? childCFriendly : childBFriendly; + const cachedLoser = bRes.isCached ? bRes : cRes; + if (cachedLoser.isCached === true) { + expect(cachedLoser.run.friendlyId).toBe(recreaterFriendly); + expect(cachedLoser.run.friendlyId).not.toBe(childWFriendly); + } + expect(bufferHarness.claimCalls).toBeGreaterThanOrEqual(3); + } + ); + + heteroPgWithRedisTest( + "NEGATIVE CONTROL — run scope, distinct per-parent keys: TWO children, no claim contention", + async ({ prisma14, prisma17, bufferHarness }) => { + const env = await seedSharedEnv(prisma14, prisma17, "gcc_d"); + const router = makeSplitRouter(prisma14, prisma17); + h.router = router; + h.buffer = bufferHarness.buffer; + h.splitOn = true; + + const taskIdentifier = "child-task"; + // Run-scope keys embed the parent run id in their hash, so two parents produce DIFFERENT + // hashed keys. These are genuinely distinct runs and must NOT be deduped. + const keyA = "run-scope-hash-A"; + const keyB = "run-scope-hash-B"; + const concern = new IdempotencyKeyConcern(prisma14 as never, {} as never, {} as never); + + const { aRes, bRes } = await driveConcurrentPair({ + concern, + router, + buffer: bufferHarness, + env, + a: { + spec: { parentResidency: "NEW", idempotencyKey: keyA, scope: "run", taskIdentifier }, + childResidency: "NEW", + }, + b: { + spec: { parentResidency: "LEGACY", idempotencyKey: keyB, scope: "run", taskIdentifier }, + childResidency: "LEGACY", + }, + }); + + const a = await countChildren(prisma14, prisma17, env, keyA, taskIdentifier); + const b = await countChildren(prisma14, prisma17, env, keyB, taskIdentifier); + + // Two distinct keys ⇒ two distinct children, one per DB. Neither is a cached hit. + expect(a.total).toBe(1); + expect(b.total).toBe(1); + expect(aRes.isCached).toBe(false); + expect(bRes.isCached).toBe(false); + // Run-scope under split is NOT global-under-split and the org isn't mollifier-enabled, so the + // claim mutex is never touched — the negative control never contends on the claim. + expect(bufferHarness.claimCalls).toBe(0); + } + ); +}); diff --git a/apps/webapp/test/idempotencyResetRouteReplicaLag.guard.test.ts b/apps/webapp/test/idempotencyResetRouteReplicaLag.guard.test.ts new file mode 100644 index 000000000..2d2b99f7e --- /dev/null +++ b/apps/webapp/test/idempotencyResetRouteReplicaLag.guard.test.ts @@ -0,0 +1,276 @@ +// Property: the idempotency-key RESET route action resolves a live run under replica lag. When the +// replica-first findRun misses a just-created run, the action re-reads the owning primary via +// findRunOnPrimary, reaches ResetIdempotencyKeyService, and returns the "Idempotency key reset +// successfully" toast — never a "Run not found" short-circuit for a run the user can see. +// +// Drives the REAL exported route action against a real split store (heteroRunOpsPostgresTest, two +// Postgres containers) whose owning replica is frozen with the shared lagging-replica primitive. Only +// module seams that inject the store / assert the outcome are mocked (dependency injection at the +// module boundary, not reimplementation — the store classes and findRun/findRunOnPrimary are real): +// runStore (the RoutingRunStore built here), db (the live legacy container the tenant is seeded on), +// session (a fixed userId), and resetIdempotencyKey (a recording stub of the reached service). + +import type * as RemixNode from "@remix-run/node"; +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { PostgresRunStore, RoutingRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; + +vi.setConfig({ testTimeout: 60_000 }); + +// Mutable holders shared with the hoisted vi.mock factories below. The route module is imported +// dynamically inside each test AFTER these are populated, and the mock factories read them lazily via +// getters, so each test injects its own live store + prisma client. +const h = vi.hoisted(() => ({ + runStore: undefined as unknown, + prisma: undefined as unknown, + userId: "usr_reset_guard", + resetCalls: [] as Array<{ idempotencyKey: string; taskIdentifier: string; envId: string }>, +})); + +vi.mock("~/services/session.server", () => ({ + requireUserId: vi.fn(async () => h.userId), +})); + +vi.mock("~/v3/runStore.server", () => ({ + get runStore() { + return h.runStore; + }, +})); + +vi.mock("~/db.server", () => ({ + get prisma() { + return h.prisma; + }, +})); + +vi.mock("~/v3/services/resetIdempotencyKey.server", () => ({ + ResetIdempotencyKeyService: class { + async call(idempotencyKey: string, taskIdentifier: string, authenticatedEnv: { id: string }) { + h.resetCalls.push({ idempotencyKey, taskIdentifier, envId: authenticatedEnv?.id }); + return { id: idempotencyKey }; + } + }, +})); + +// Test-safe cookie session (fixed secret, real @remix-run/node storage) so the route's toast +// round-trip is exercised end-to-end without loading app env config (`env.server`/SESSION_SECRET). +vi.mock("~/models/message.server", async () => { + const { createCookieSessionStorage, json } = (await vi.importActual( + "@remix-run/node" + )) as typeof RemixNode; + const { getSession, commitSession } = createCookieSessionStorage({ + cookie: { + name: "__message", + path: "/", + httpOnly: true, + sameSite: "lax", + secrets: ["test-secret"], + }, + }); + const jsonWith = async ( + data: unknown, + request: Request, + message: string, + type: "success" | "error" + ) => { + const session = await getSession(request.headers.get("cookie")); + session.flash("toastMessage", { message, type, options: { ephemeral: true } }); + return json(data, { headers: { "Set-Cookie": await commitSession(session) } }); + }; + return { + getSession, + commitSession, + jsonWithSuccessMessage: (data: unknown, request: Request, message: string) => + jsonWith(data, request, message, "success"), + jsonWithErrorMessage: (data: unknown, request: Request, message: string) => + jsonWith(data, request, message, "error"), + }; +}); + +const RESET_SELECT = { + id: true, + idempotencyKey: true, + taskIdentifier: true, + projectId: true, + runtimeEnvironmentId: true, +} as const; + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + taskIdentifier: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + idempotencyKey: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "COMPLETED_SUCCESSFULLY", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: params.taskIdentifier, + idempotencyKey: params.idempotencyKey, + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + } as CreateRunInput; +} + +// Decode the flashed toast from the action's Set-Cookie so we assert the user-visible outcome, not an +// internal. Uses the mocked test-safe message.server session above (no env.server / SESSION_SECRET). +async function readToast( + response: Response +): Promise<{ message: string; type: string } | undefined> { + const { getSession } = await import("~/models/message.server"); + const setCookie = response.headers.get("Set-Cookie"); + const session = await getSession(setCookie); + return session.get("toastMessage") as { message: string; type: string } | undefined; +} + +describe("idempotency-key reset route action reads-your-writes under a lagging split replica", () => { + heteroRunOpsPostgresTest( + "returns the success toast for a run not yet replicated on the owning replica", + async ({ prisma14, prisma17 }) => { + h.resetCalls = []; + + // Seed the org/project/env + an authorized user on the LEGACY control-plane container. + const suffix = "reset_route_guard"; + const organization = await prisma14.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma14.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma14.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + const user = await prisma14.user.create({ + data: { + id: h.userId, + email: `${suffix}@example.com`, + authenticationMethod: "MAGIC_LINK", + }, + }); + await prisma14.orgMember.create({ + data: { organizationId: organization.id, userId: user.id, role: "ADMIN" }, + }); + + // Build the REAL split store exactly as runStore.server holds it: the LEGACY store's replica lags + // (frozen), its writer is live; the NEW store is live but empty. friendlyId reads fan out across + // both stores' replicas (miss under lag) then, on the primary re-read, both writers (legacy hits). + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const runId = `run_${"c".repeat(25)}`; // cuid body -> LEGACY-resident + const friendlyId = "run_reset_route_guard"; + const idempotencyKey = "user-supplied-key-guard"; + const taskIdentifier = "my-task"; + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + taskIdentifier, + organizationId: organization.id, + projectId: project.id, + runtimeEnvironmentId: environment.id, + idempotencyKey, + }) + ); + + // Sanity: the run exists on the owning primary but NOT on the (lagging) replica — the exact + // read-your-writes window under test. + expect(await router.findRun({ friendlyId }, { select: RESET_SELECT })).toBeNull(); + expect( + await router.findRunOnPrimary({ friendlyId }, { select: RESET_SELECT }) + ).not.toBeNull(); + + const primarySpy = vi.spyOn(router, "findRunOnPrimary"); + + // Inject the live store + prisma the route will hold, then drive the REAL exported action. + h.runStore = router; + h.prisma = prisma14; + + const { action } = + await import("~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset"); + + const request = new Request("http://localhost/reset", { method: "POST" }); + const response: Response = await action({ + request, + params: { + organizationSlug: organization.slug, + projectParam: project.slug, + envParam: environment.slug, + runParam: friendlyId, + }, + context: {}, + } as never); + + const toast = await readToast(response); + + // Property: the action re-reads the owning primary on the replica miss, finds the run, and + // reaches the reset service — rather than returning not-found for a run the user can see. + expect(legacyReplica.wasHit()).toBe(true); // the lagging replica path really executed in the action + expect(primarySpy).toHaveBeenCalled(); // the owning-primary re-read was taken + expect(h.resetCalls).toHaveLength(1); // ResetIdempotencyKeyService.call was reached + expect(h.resetCalls[0]).toMatchObject({ + idempotencyKey, + taskIdentifier, + envId: environment.id, + }); + expect(toast?.type).toBe("success"); + expect(toast?.message).toBe("Idempotency key reset successfully"); + // And it is NOT the not-found short-circuit. + expect(toast?.message).not.toBe("Run not found"); + } + ); +}); diff --git a/apps/webapp/test/metadataRouteReplicaLag.guard.test.ts b/apps/webapp/test/metadataRouteReplicaLag.guard.test.ts new file mode 100644 index 000000000..9a27055c3 --- /dev/null +++ b/apps/webapp/test/metadataRouteReplicaLag.guard.test.ts @@ -0,0 +1,261 @@ +// Property: the metadata GET loader resolves a live run on a replica+buffer double-miss via its primary +// fallback, returning 200 + the run's metadata. Drives the REAL exported `loader` end-to-end: +// 1. runStore.findRun(..., $replica) → REPLICA (lagging) → null +// 2. findRunByIdWithMollifierFallback(...) → buffer MISS (null) +// 3. runStore.findRunOnPrimary(...) → owning PRIMARY → HIT +// Store is REAL: a split RoutingRunStore over two testcontainer Postgres DBs, the owning +// (legacy/control-plane) REPLICA frozen behind the shared laggingReplica. Only the loader's webapp +// singletons (auth, $replica brand, mollifier buffer, route builder, logging) are stubbed. + +import { describe, expect, vi } from "vitest"; +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { PostgresRunStore, RoutingRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; + +// vi.mock factories are hoisted above the imports, so anything they reference must be created inside +// vi.hoisted. `holder.store` is filled in per-test with the REAL split router; the mocked +// `~/v3/runStore.server` export is a stable Proxy that forwards every property access to it. The +// branded `$replica` marker uses the global-registry symbol the run-store brands replicas with +// (readReplicaClient.ts) — a branded client makes the routing store keep the read on the owning +// REPLICA (no primary escalation), which under lag is exactly the miss the primary fallback recovers. +const { holder } = vi.hoisted(() => { + const REPLICA_BRAND = Symbol.for("trigger.dev/run-store/read-replica"); + return { + holder: { + store: undefined as unknown, + brandedReplica: { [REPLICA_BRAND]: true } as object, + environment: undefined as unknown, + bufferResult: null as unknown, + }, + }; +}); + +vi.mock("~/db.server", () => ({ prisma: {}, $replica: holder.brandedReplica })); +vi.mock("~/env.server", () => ({ + env: { + TASK_RUN_METADATA_MAXIMUM_SIZE: 256 * 1024, + TRIGGER_MOLLIFIER_METADATA_MAX_RETRIES: 3, + TRIGGER_MOLLIFIER_METADATA_BACKOFF_BASE_MS: 10, + TRIGGER_MOLLIFIER_METADATA_BACKOFF_STEP_MS: 10, + }, +})); +// The route module builds its action at import time via createActionApiRoute; stub the builder so the +// heavy platform/auth middleware graph never evaluates. We drive the exported `loader` directly. +vi.mock("~/services/routeBuilders/apiBuilder.server", () => ({ + createActionApiRoute: () => ({ action: vi.fn() }), +})); +vi.mock("~/services/apiAuth.server", () => ({ + authenticateApiRequest: vi.fn(async () => ({ environment: holder.environment })), +})); +// Inject the REAL split router. A stable Proxy keeps the named import binding constant while +// forwarding every method to the per-test router set in holder.store. +vi.mock("~/v3/runStore.server", () => ({ + runStore: new Proxy( + {}, + { + get(_target, prop) { + const store = holder.store as Record; + if (!store) throw new Error("test bug: holder.store not initialised before loader ran"); + const value = store[prop]; + return typeof value === "function" + ? (value as (...a: unknown[]) => unknown).bind(store) + : value; + }, + } + ), +})); +// Buffer fallback returns whatever the test set (null = buffer miss, the double-miss scenario). +vi.mock("~/v3/mollifier/readFallback.server", () => ({ + findRunByIdWithMollifierFallback: vi.fn(async () => holder.bufferResult), +})); +// Action-only leaf; stub to keep the import graph light. +vi.mock("~/v3/mollifier/applyMetadataMutation.server", () => ({ + applyMetadataMutationToBufferedRun: vi.fn(), +})); +vi.mock("~/services/metadata/updateMetadataInstance.server", () => ({ + updateMetadataService: { call: vi.fn(async () => undefined) }, +})); +vi.mock("~/v3/services/common.server", () => ({ + ServiceValidationError: class extends Error {}, +})); +vi.mock("~/services/logger.server", () => ({ + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +import { loader } from "~/routes/api.v1.runs.$runId.metadata"; + +// A cuid (25 chars after `run_`) classifies LEGACY, so both the create and the friendlyId-keyed reads +// route to the legacy (control-plane) store — the store that owns this run. +const CUID_25 = "c".repeat(25); + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + metadata?: string; + metadataType?: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "PENDING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + metadata: params.metadata, + metadataType: params.metadataType, + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: ["alpha"], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +function buildRouterWithLaggingLegacyReplica(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +async function callLoader(friendlyId: string) { + const response = await loader({ + request: new Request("https://api.trigger.dev/api/v1/runs/" + friendlyId + "/metadata", { + headers: { Authorization: "Bearer tr_dev_meta" }, + }), + params: { runId: friendlyId }, + context: {} as never, + }); + return response as Response; +} + +describe("metadata GET loader under replica lag", () => { + heteroRunOpsPostgresTest( + "metadata GET loader resolves a live run on a replica and buffer double-miss", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( + prisma14, + prisma17 + ); + + const seed = await seedEnvironmentLegacy(prisma14, "meta"); + const runId = `run_${CUID_25}`; // cuid → LEGACY-owned + const friendlyId = "run_meta_live"; + const metadata = '{"phase":"one"}'; + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + metadata, + metadataType: "application/json", + }) + ); + + // Wire the loader's module singletons: the real split router, the authenticated env, and a + // buffer that MISSES (the run has drained from the buffer to the primary but the replica has + // not caught up — the exact double-miss under test). + holder.store = router; + holder.environment = { id: seed.environment.id, organizationId: seed.organization.id }; + holder.bufferResult = null; + + const response = await callLoader(friendlyId); + const body = (await response.json()) as { + metadata?: unknown; + metadataType?: unknown; + error?: string; + }; + + // The replica WAS consulted (and, being frozen, missed) — proving the read genuinely went + // through the lagging replica and the recovery is the primary fallback, not a lucky replica hit. + expect(legacyReplica.wasHit()).toBe(true); + + // The property: the loader re-reads the owning primary and returns the live run's metadata. + expect(response.status).toBe(200); + expect(body.metadata).toBe(metadata); + expect(body.metadataType).toBe("application/json"); + } + ); + + // Negative control: when the run truly does not exist anywhere (replica miss + buffer miss + + // primary miss), the loader must still 404. This pins the behavior to "recover a LIVE run" rather + // than "never 404". + heteroRunOpsPostgresTest( + "metadata GET loader 404s when the run is absent on the primary too", + async ({ prisma14, prisma17 }) => { + const { router } = buildRouterWithLaggingLegacyReplica(prisma14, prisma17); + const seed = await seedEnvironmentLegacy(prisma14, "meta_absent"); + + holder.store = router; + holder.environment = { id: seed.environment.id, organizationId: seed.organization.id }; + holder.bufferResult = null; + + const response = await callLoader("run_does_not_exist"); + expect(response.status).toBe(404); + } + ); +}); diff --git a/apps/webapp/test/mollifierClaimResolution.test.ts b/apps/webapp/test/mollifierClaimResolution.test.ts index c35c24c1c..597f264e2 100644 --- a/apps/webapp/test/mollifierClaimResolution.test.ts +++ b/apps/webapp/test/mollifierClaimResolution.test.ts @@ -156,4 +156,30 @@ describe("IdempotencyKeyConcern · claim resolution", () => { h.orgFlag = true; // restore for any later tests in this file } }); + + it("floors the claim TTL for a short idempotency-key TTL so the claim can't expire mid-pipeline", async () => { + // A 2s customer key TTL must NOT shrink the claim below the pipeline floor — else the claim expires + // while the winner is still creating and a polling loser re-claims (cross-DB dup under the split). + // Capture the ttlSeconds the concern hands the buffer's claim. + let capturedTtl: number | undefined; + h.buffer = { + claimIdempotency: vi.fn(async (input: { ttlSeconds: number }) => { + capturedTtl = input.ttlSeconds; + return { kind: "claimed" as const }; + }), + lookupIdempotency: vi.fn(async () => null), + } as unknown as MollifierBuffer; + + const findFirst = vi.fn(async () => null); + const concern = makeConcern({ findFirst }); + + const request = makeRequest(); + (request.body.options as { idempotencyKeyTTL?: string }).idempotencyKeyTTL = "2s"; + + const result = await concern.handleTriggerRequest(request, undefined); + + expect(result.isCached).toBe(false); + // Floored at TRIGGER_MOLLIFIER_CLAIM_MIN_TTL_SECONDS (default 5), NOT the ~2s key TTL. + expect(capturedTtl).toBeGreaterThanOrEqual(5); + }); }); diff --git a/apps/webapp/test/presentersSessionBatchReplicaLag.guard.test.ts b/apps/webapp/test/presentersSessionBatchReplicaLag.guard.test.ts new file mode 100644 index 000000000..a9d0fe141 --- /dev/null +++ b/apps/webapp/test/presentersSessionBatchReplicaLag.guard.test.ts @@ -0,0 +1,358 @@ +// Replica-lag properties for the AI-session + batch DASHBOARD presenters. Drives the REAL exported +// presenter classes (SessionListPresenter, SessionPresenter, BatchPresenter) end-to-end via `.call()` +// against a real Postgres (heteroPostgresTest) whose READ replica is FROZEN via the shared +// `laggingReplica` primitive (run/batch row on the primary, not yet replicated); only orthogonal +// peripherals are mocked (ClickHouse session list, displayable-env, worker-deployment probe, presign). + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// ~/db.server: the two proxies the run-store / control-plane singletons read from point at our +// per-test holders (never mocks the DB itself — the proxies forward to a real testcontainer client). +// Run-ops split handles left undefined => runStore.server builds the single-DB passthrough store +// (writer = `prisma`, replica = `$replica`), the exact webapp single-DB topology these reads live in. +const primaryHolder = vi.hoisted(() => ({ client: undefined as any })); +const replicaHolder = vi.hoisted(() => ({ client: undefined as any })); + +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + const lazyProxy = (holder: { client: any }, label: string) => + new Proxy( + {}, + { + get(_t, prop) { + if (!holder.client) throw new Error(`${label} not set for this test`); + const value = holder.client[prop]; + if (value !== null && typeof value === "object") { + return new Proxy(value, { get: (_d, method) => holder.client[prop][method] }); + } + return value; + }, + } + ); + return { + prisma: lazyProxy(primaryHolder, "primaryHolder.client"), + $replica: lazyProxy(replicaHolder, "replicaHolder.client"), + runOpsNewPrismaClient: undefined, + runOpsNewReplicaClient: undefined, + runOpsLegacyPrisma: undefined, + runOpsLegacyReplica: undefined, + sqlDatabaseSchema: Prisma.sql([`public`]), + }; +}); + +// ---- Orthogonal peripherals --------------------------------------------------------------------- + +// A stub displayable environment (control-plane resolve, orthogonal to the run-ops run/batch read). +const STUB_ENV = { + id: "env_stub", + type: "DEVELOPMENT" as const, + slug: "dev", + organizationId: "org_stub", + projectId: "proj_stub", + userId: undefined, + branchName: null, + git: null, +}; + +vi.mock("~/models/runtimeEnvironment.server", () => ({ + findDisplayableEnvironment: async () => STUB_ENV, +})); + +// SessionListPresenter's possibleTasks probe — orthogonal; no worker seeded -> empty task list. +vi.mock("~/v3/models/workerDeployment.server", () => ({ + findCurrentWorkerFromEnvironment: async () => null, +})); + +// SessionListPresenter's session list comes from ClickHouse via SessionsRepository — orthogonal to +// its run-store run read. The mock returns a controlled session whose currentRunId points at the run +// we seed on the primary only, so that findRuns read is exercised for real against the lag. +const sessionListHolder = vi.hoisted(() => ({ sessions: [] as any[] })); +vi.mock("~/services/sessionsRepository/sessionsRepository.server", () => ({ + LEGACY_PLAYGROUND_TAG: "__playground__", + SessionsRepository: class { + constructor(_deps: any) {} + async listSessions() { + return { + sessions: sessionListHolder.sessions, + pagination: { nextCursor: null, previousCursor: null }, + }; + } + }, +})); + +// SessionPresenter snapshot presign — orthogonal object-store call; a miss is handled gracefully. +vi.mock("~/v3/objectStore.server", () => ({ + generatePresignedUrl: async () => ({ success: false as const, error: "stub" }), +})); + +import { PostgresRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; +// The REAL callers under guard. +import { SessionListPresenter } from "~/presenters/v3/SessionListPresenter.server"; +import { SessionPresenter } from "~/presenters/v3/SessionPresenter.server"; +import { BatchPresenter } from "~/presenters/v3/BatchPresenter.server"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(p: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: p.runId, + engine: "V2", + status: "EXECUTING", + friendlyId: p.friendlyId, + runtimeEnvironmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: p.organizationId, + projectId: p.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: `trace_${p.runId}`, + spanId: `span_${p.runId}`, + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "EXECUTING", + description: "Run is executing", + runStatus: "EXECUTING", + environmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: p.projectId, + organizationId: p.organizationId, + }, + }; +} + +describe("session/batch dashboard presenters under replica lag", () => { + // Read 1 — SessionListPresenter findRuns. + heteroPostgresTest( + "SessionListPresenter.call renders the list row with currentRunFriendlyId undefined under lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `sesslist_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + // Seed the current run on the PRIMARY only. + const runId = `run_sesslist_${suffix}`; + const runFriendlyId = `run_sl_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId: runFriendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + // ClickHouse session list (mocked) yields ONE session pointing at that run. + sessionListHolder.sessions = [ + { + id: `sess_${suffix}`, + friendlyId: `session_${suffix}`, + externalId: null, + type: "chat", + taskIdentifier: "my-agent", + isTest: false, + tags: [], + closedAt: null, + closedReason: null, + expiresAt: null, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + updatedAt: new Date("2024-01-01T00:00:00.000Z"), + currentRunId: runId, + }, + ]; + + // A REAL lagging replica: taskRun reads miss; everything else forwards to the real container. + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + primaryHolder.client = prisma; + replicaHolder.client = replica.client; + + const presenter = new SessionListPresenter(replica.client as any, {} as any); + const result = await presenter.call(seed.organization.id, seed.environment.id, { + projectId: seed.project.id, + }); + + // The run read really hit the (lagging) replica. + expect(replica.wasHit("taskRun")).toBe(true); + + // OBSERVABLE OUTPUT: the session still renders; the run-link is simply absent under lag. + expect(result.sessions).toHaveLength(1); + expect(result.sessions[0].id).toBe(`sess_${suffix}`); + expect(result.sessions[0].currentRunFriendlyId).toBeUndefined(); + + // The omission is pure replica lag: the run exists on the PRIMARY. + const onPrimary = await prisma.taskRun.findFirst({ where: { id: runId } }); + expect(onPrimary?.friendlyId).toBe(runFriendlyId); + } + ); + + // Read 2 — SessionPresenter findRuns + findRun (currentRun fallback). + heteroPostgresTest( + "SessionPresenter.call returns the detail with currentRun and history run null under lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `sessdetail_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + // Seed the run on the PRIMARY only. + const runId = `run_sessdetail_${suffix}`; + const runFriendlyId = `run_sd_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId: runFriendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + // Seed the Session (currentRunId -> the run) + a SessionRun history row (runId -> the run). + const sessionFriendlyId = `session_${suffix}`; + const session = await prisma.session.create({ + data: { + friendlyId: sessionFriendlyId, + type: "chat", + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + environmentType: "DEVELOPMENT", + organizationId: seed.organization.id, + taskIdentifier: "my-agent", + triggerConfig: { basePayload: {} }, + currentRunId: runId, + }, + }); + await prisma.sessionRun.create({ + data: { sessionId: session.id, runId, reason: "initial" }, + }); + + // taskRun frozen-missing on the replica; Session + SessionRun forward to the primary. + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + primaryHolder.client = prisma; + replicaHolder.client = replica.client; + + const presenter = new SessionPresenter(replica.client as any); + const result = await presenter.call({ + userId: "user_x", + environmentId: seed.environment.id, + sessionParam: sessionFriendlyId, + projectExternalRef: seed.project.externalRef, + environmentSlug: "dev", + }); + + // The run reads really hit the (lagging) replica. + expect(replica.wasHit("taskRun")).toBe(true); + + // OBSERVABLE OUTPUT: the detail resolves (session found via the forwarding replica)... + expect(result).not.toBeNull(); + expect(result!.friendlyId).toBe(sessionFriendlyId); + // ...currentRun link self-heals to null under lag (both the map read AND the fallback miss)... + expect(result!.currentRun).toBeNull(); + // ...and the history row renders with a null run summary. + expect(result!.runs).toHaveLength(1); + expect(result!.runs[0].run).toBeNull(); + + // The nulls are pure replica lag: the run exists on the PRIMARY. + const onPrimary = await prisma.taskRun.findFirst({ where: { id: runId } }); + expect(onPrimary?.friendlyId).toBe(runFriendlyId); + } + ); + + // Read 3 — BatchPresenter findBatchTaskRunByFriendlyId (recovered via primary re-read). + heteroPostgresTest( + "BatchPresenter.call returns a live batch under lag via the primary re-read", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `batch_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + // Seed the batch on the PRIMARY only. + const batchId = `batch_${suffix}`; + const batchFriendlyId = `batch_fr_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createBatchTaskRun({ + id: batchId, + friendlyId: batchFriendlyId, + runtimeEnvironmentId: seed.environment.id, + }); + + // batchTaskRun frozen-missing on the replica. + const replica = laggingReplica(prisma, [{ model: "batchTaskRun", mode: "missing" }]); + + // BatchPresenter reads via an INJECTED real store: writer = primary, read-replica = lagging. + const store = new PostgresRunStore({ prisma, readOnlyPrisma: replica.client as any }); + const presenter = new BatchPresenter( + prisma as any, // _prisma -> the owning primary (the re-read target) + replica.client as any, // _replica -> the lagging replica (the read under test) + { resolveDisplayableEnvironment: async () => STUB_ENV as any }, + store + ); + + const result = await presenter.call({ + environmentId: seed.environment.id, + batchId: batchFriendlyId, + userId: "user_x", + }); + + // The read really hit the (lagging) replica (the read-your-writes hazard was exercised)... + expect(replica.wasHit("batchTaskRun")).toBe(true); + + // ...and the presenter returned the LIVE batch via the primary re-read, not "Batch not found". + expect(result.id).toBe(batchId); + expect(result.friendlyId).toBe(batchFriendlyId); + + // The row is genuinely on the PRIMARY (so the recovery is a real primary read, not the replica). + const onPrimary = await prisma.batchTaskRun.findFirst({ where: { id: batchId } }); + expect(onPrimary?.friendlyId).toBe(batchFriendlyId); + } + ); +}); diff --git a/apps/webapp/test/publishClaimResult.test.ts b/apps/webapp/test/publishClaimResult.test.ts new file mode 100644 index 000000000..a07258550 --- /dev/null +++ b/apps/webapp/test/publishClaimResult.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it, vi } from "vitest"; +import { publishClaim } from "~/v3/mollifier/idempotencyClaim.server"; +import type { MollifierBuffer } from "@trigger.dev/redis-worker"; + +// publishClaim compare-and-sets on the caller's token; a `false` result means a stale claimant already +// moved in (the winner's claim expired mid-pipeline), so the publish no-op'd. The caller must be able to +// SEE that rather than silently assuming its run is canonical. +function fakeBuffer(publishResult: boolean): MollifierBuffer { + return { publishClaim: vi.fn(async () => publishResult) } as unknown as MollifierBuffer; +} + +const base = { + envId: "env_1", + taskIdentifier: "task", + idempotencyKey: "k", + token: "tok", + runId: "run_1", + ttlSeconds: 30, +}; + +describe("publishClaim result", () => { + it("returns false when the buffer compare-and-set no-ops (a stale claimant already moved in)", async () => { + expect(await publishClaim({ ...base, buffer: fakeBuffer(false) })).toBe(false); + }); + + it("returns true when the publish sets the winner", async () => { + expect(await publishClaim({ ...base, buffer: fakeBuffer(true) })).toBe(true); + }); + + it("returns true when the mollifier buffer is unavailable (nothing to converge)", async () => { + expect(await publishClaim({ ...base, buffer: null })).toBe(true); + }); +}); diff --git a/apps/webapp/test/reacquireClearedGlobalWinner.test.ts b/apps/webapp/test/reacquireClearedGlobalWinner.test.ts new file mode 100644 index 000000000..9bf7492c0 --- /dev/null +++ b/apps/webapp/test/reacquireClearedGlobalWinner.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it, vi } from "vitest"; + +// Force every reacquire attempt to RESOLVE to a winner (never `claimed`/`timed_out`); the spied +// handleExistingRun keeps reporting that winner as cleared, so the bounded loop advances to exhaustion. +vi.mock("~/v3/mollifier/idempotencyClaim.server", () => ({ + resetResolvedClaim: vi.fn(async () => {}), + claimOrAwait: vi.fn(async () => ({ kind: "resolved", runId: "run_cleared" })), +})); + +import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server"; + +describe("reacquireClearedGlobalWinner — fail closed on exhaustion", () => { + it("throws a retryable 503 after exhausting reacquires (never falls through to an unserialised create)", async () => { + const concern = new IdempotencyKeyConcern({} as never, {} as never, {} as never); + // Winner is findable but perpetually cleared → each pass advances staleRunId, exhausting the bound. + vi.spyOn( + concern as unknown as { resolveWinnerAcrossDbs: () => unknown }, + "resolveWinnerAcrossDbs" + ).mockResolvedValue({ friendlyId: "run_cleared" }); + vi.spyOn( + concern as unknown as { handleExistingRun: () => unknown }, + "handleExistingRun" + ).mockResolvedValue({ isCached: false }); + + const request = { environment: { id: "env_1" }, taskId: "task" } as never; + const ctx = { + idempotencyKey: "k", + idempotencyKeyExpiresAt: new Date(Date.now() + 60_000), + dedupClient: {} as never, + ttlSeconds: 30, + clearedRunId: "run_cleared", + safetyNetMs: 5_000, + pollStepMs: 25, + }; + + await expect( + ( + concern as unknown as { + reacquireClearedGlobalWinner: (...a: unknown[]) => Promise; + } + ).reacquireClearedGlobalWinner(request, undefined, ctx) + ).rejects.toMatchObject({ name: "ServiceValidationError", status: 503 }); + }); +}); diff --git a/apps/webapp/test/readRunForEvent.replicaLag.test.ts b/apps/webapp/test/readRunForEvent.replicaLag.test.ts new file mode 100644 index 000000000..877f920e4 --- /dev/null +++ b/apps/webapp/test/readRunForEvent.replicaLag.test.ts @@ -0,0 +1,198 @@ +// Verifies readRunForEvent tolerates replica lag on its event-enrichment read (the store.findRun +// closures inside readThroughRun, which in single-DB passthrough route to the replica). Drives the REAL +// exported function over a real Postgres testcontainer with the replica FROZEN via laggingReplica. +// Two properties: (a) PRESENT-BUT-STALE — a completed run whose UPDATE has not replicated returns the row +// with correct immutable identity and stale status, no throw; (b) MISSING — an unreplicated INSERT returns +// null (no throw), the event fires without enrichment. Both self-heal; the row is live on the primary. + +import { containerTest, laggingReplica } from "@internal/testcontainers"; +import { PostgresRunStore } from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; +import { readRunForEvent, type EventReadDeps } from "~/v3/runEngineHandlersShared.server"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// Exactly the immutable+mutable mix a real event-bus enrichment read selects. +const EVENT_SELECT = { + id: true, + friendlyId: true, + traceId: true, + spanId: true, + createdAt: true, + completedAt: true, + taskIdentifier: true, + status: true, + organizationId: true, +} as const; + +async function seedEnvironment(prisma: PrismaClient, slugSuffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +describe("readRunForEvent tolerates replica lag on its event-enrichment read", () => { + // (a) PRESENT-BUT-STALE. + containerTest( + "readRunForEvent returns a present-but-stale run under lag — correct immutable fields, stale status, no throw", + async ({ prisma }) => { + const { organization, project, environment } = await seedEnvironment(prisma, "rrfe_stale"); + + const runId = "c".repeat(25); // cuid-shaped → LEGACY; passthrough reads it as-is + const friendlyId = "run_rrfe_stale"; + const createdAt = new Date("2024-01-01T00:00:00.000Z"); + const completedAt = new Date("2024-01-01T00:05:00.000Z"); + + // The run is COMPLETED on the PRIMARY (completion is an UPDATE applied on the primary). + await prisma.taskRun.create({ + data: { + id: runId, + engine: "V2", + status: "COMPLETED_SUCCESSFULLY", + friendlyId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceId: "trace_rrfe", + spanId: "span_rrfe", + queue: "task/my-task", + runtimeEnvironmentId: environment.id, + projectId: project.id, + organizationId: organization.id, + environmentType: "DEVELOPMENT", + isTest: false, + taskEventStore: "taskEvent", + createdAt, + completedAt, + }, + }); + + // A FROZEN replica: the taskRun row is present but PRE-completion (status EXECUTING, completedAt + // null) — the UPDATE has not replicated. Immutable fields match the live row (they were set at + // INSERT, before the lag window). Everything else forwards to the real container. + const replica = laggingReplica(prisma, [ + { + model: "taskRun", + mode: "frozen", + rows: [ + { + id: runId, + friendlyId, + traceId: "trace_rrfe", + spanId: "span_rrfe", + createdAt, + taskIdentifier: "my-task", + organizationId: organization.id, + // STALE mutable fields (pre-completion snapshot): + status: "EXECUTING", + completedAt: null, + }, + ], + }, + ]); + + const store = new PostgresRunStore({ prisma, readOnlyPrisma: replica.client as never }); + const deps: EventReadDeps = { + store, + newReplica: replica.client as never, + legacyReplica: replica.client as never, + splitEnabled: false, + }; + + const run = await readRunForEvent(runId, environment.id, EVENT_SELECT, deps); + + // The enrichment read really hit the (lagging) replica. + expect(replica.wasHit("taskRun")).toBe(true); + + // OBSERVABLE OUTPUT: the run resolves (present-but-stale) — no throw, no null. + expect(run).not.toBeNull(); + // Immutable identity is CORRECT even off the stale replica: + expect(run!.id).toBe(runId); + expect(run!.friendlyId).toBe(friendlyId); + expect(run!.traceId).toBe("trace_rrfe"); + expect(run!.spanId).toBe("span_rrfe"); + expect(run!.taskIdentifier).toBe("my-task"); + expect(run!.createdAt).toEqual(createdAt); + // Mutable completion fields are STALE off the replica (this is the tolerated staleness): + expect(run!.status).toBe("EXECUTING"); + expect(run!.completedAt).toBeNull(); + + // The staleness is pure lag: on the PRIMARY the completion is applied. + const onPrimary = await prisma.taskRun.findFirstOrThrow({ where: { id: runId } }); + expect(onPrimary.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(onPrimary.completedAt).toEqual(completedAt); + // Immutable fields are identical on primary and the stale replica read. + expect(onPrimary.friendlyId).toBe(run!.friendlyId); + } + ); + + // (b) MISSING (INSERT not yet replicated). + containerTest( + "readRunForEvent returns null (no throw) when a live run's INSERT has not replicated", + async ({ prisma }) => { + const { organization, project, environment } = await seedEnvironment(prisma, "rrfe_missing"); + + const runId = "d".repeat(25); + await prisma.taskRun.create({ + data: { + id: runId, + engine: "V2", + status: "EXECUTING", + friendlyId: "run_rrfe_missing", + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceId: "trace_m", + spanId: "span_m", + queue: "task/my-task", + runtimeEnvironmentId: environment.id, + projectId: project.id, + organizationId: organization.id, + environmentType: "DEVELOPMENT", + isTest: false, + taskEventStore: "taskEvent", + }, + }); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + const store = new PostgresRunStore({ prisma, readOnlyPrisma: replica.client as never }); + const deps: EventReadDeps = { + store, + newReplica: replica.client as never, + legacyReplica: replica.client as never, + splitEnabled: false, + }; + + const run = await readRunForEvent(runId, environment.id, EVENT_SELECT, deps); + + expect(replica.wasHit("taskRun")).toBe(true); + // OBSERVABLE OUTPUT: not-found degrades to null (no throw); the event fires without enrichment. + expect(run).toBeNull(); + + // The null is pure lag: the run is live on the PRIMARY. + const onPrimary = await prisma.taskRun.findFirstOrThrow({ where: { id: runId } }); + expect(onPrimary.friendlyId).toBe("run_rrfe_missing"); + } + ); +}); diff --git a/apps/webapp/test/realtimeServices.replicaLag.test.ts b/apps/webapp/test/realtimeServices.replicaLag.test.ts new file mode 100644 index 000000000..7df81778d --- /dev/null +++ b/apps/webapp/test/realtimeServices.replicaLag.test.ts @@ -0,0 +1,555 @@ +// Replica-lag guards for the realtime-services reads. +// +// Drives the REAL exported callers end-to-end against a real Postgres (heteroPostgresTest) whose +// replica is FROZEN via the shared `laggingReplica` primitive (taskRun mode:"missing"), asserting a +// concrete observable on each caller's output. Every read here is a DISPLAY / probe resolve with a +// real tolerance the caller owns — none is a read-your-writes gate a stale read corrupts: +// +// 1. RunHydrator.hydrateByIds (findRuns on $replica) — under lag the un-replicated row is OMITTED +// from the frame ([] returned, no throw); the feed self-heals on the next hydrate tick. +// 2. RunHydrator.getRunById / #fetch (findRun on $replica) — returns null ("not yet visible"), no +// throw; the short-TTL cache re-fetches. A read-only display resolve, never a decision. +// 3. ensureRunForSession → getRunStatusAndFriendlyId (findRun on $replica) — the replica probe +// misses the live currentRun, the caller's WRITER re-probe recovers it, and the run is reused +// (triggered:false) without a second trigger. +// 4. swapSessionRun → resolveRunFriendlyId (findRun on $replica) — the replica misses the calling +// run's friendlyId and the caller falls back to the cuid (?? runId); the swap still COMPLETES +// (swapped:true). +// 5. serializeSessionWithFriendlyRunId (client-less findRun → replica) — a pre-existing +// currentRunId pointer resolves to null on the wire (safe degraded direction); GET/PATCH only +// serialize pre-existing pointers, so this display staleness self-heals on the next GET. +// 6. serializeSessionsWithFriendlyRunIds (client-less findRuns → replica) — the un-replicated run +// drops out of the id→friendlyId map so that session's currentRunId serializes null. Same +// self-healing display resolve, batched. +// +// Only webapp singletons orthogonal to the read (db.server handles, the runStore singleton, logger, +// the downstream Trigger/Cancel services) are mocked; the read path and the found/not-found + +// fallback decisions are the genuine article. Reads 1/2 and 5/6 take their store/replica by +// injection, so they drive the real caller with NO module mocking. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient, Session } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// --- Holders wired per-test into the mocked sessionRunManager singletons (reads 3 & 4 only). -------- +const primaryHolder = vi.hoisted(() => ({ client: undefined as any })); +const replicaHolder = vi.hoisted(() => ({ client: undefined as any })); +const storeHolder = vi.hoisted(() => ({ store: undefined as any })); +// Records every TriggerTaskService.call so read 3 can assert NO double-trigger and read 4 can assert +// which previousRunId the resolveRunFriendlyId fallback forwarded. +const triggerState = vi.hoisted(() => ({ + calls: [] as Array<{ taskIdentifier: string; body: any; options: any }>, + result: { run: { id: "", friendlyId: "" } } as { run: { id: string; friendlyId: string } }, +})); + +// db.server: two lazy proxies forwarding to the per-test holders. Never mocks the DB — the proxies +// forward to real testcontainer clients (primary = writer, replica = the frozen lagging client). +vi.mock("~/db.server", () => { + const lazyProxy = (holder: { client: any }, label: string) => + new Proxy( + {}, + { + get(_t, prop) { + if (!holder.client) throw new Error(`${label} not set for this test`); + const value = holder.client[prop]; + if (value !== null && typeof value === "object") { + return new Proxy(value, { get: (_d, method) => holder.client[prop][method] }); + } + return value; + }, + } + ); + return { + prisma: lazyProxy(primaryHolder, "primaryHolder.client"), + $replica: lazyProxy(replicaHolder, "replicaHolder.client"), + }; +}); + +// runStore singleton: a stable Proxy forwarding every method to the per-test real PostgresRunStore. +vi.mock("~/v3/runStore.server", () => ({ + runStore: new Proxy( + {}, + { + get(_t, prop) { + const store = storeHolder.store as Record; + if (!store) throw new Error("test bug: storeHolder.store not set before caller ran"); + const value = store[prop]; + return typeof value === "function" + ? (value as (...a: unknown[]) => unknown).bind(store) + : value; + }, + } + ), +})); + +vi.mock("~/services/logger.server", () => ({ + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +// Downstream trigger is engine work, not the read under test: record the call, return the seeded run. +vi.mock("~/v3/services/triggerTask.server", () => ({ + TriggerTaskService: class { + async call(taskIdentifier: string, _environment: any, body: any, options: any) { + triggerState.calls.push({ taskIdentifier, body, options }); + return triggerState.result; + } + }, +})); + +vi.mock("~/v3/services/cancelTaskRun.server", () => ({ + CancelTaskRunService: class { + async call() {} + }, +})); + +import { PostgresRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; +// The REAL exported callers under guard. +import { RunHydrator } from "~/services/realtime/runReader.server"; +import { ensureRunForSession, swapSessionRun } from "~/services/realtime/sessionRunManager.server"; +import { + serializeSessionWithFriendlyRunId, + serializeSessionsWithFriendlyRunIds, +} from "~/services/realtime/sessions.server"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(p: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + status?: CreateRunInput["data"]["status"]; +}): CreateRunInput { + return { + data: { + id: p.runId, + engine: "V2", + status: p.status ?? "PENDING", + friendlyId: p.friendlyId, + runtimeEnvironmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: p.organizationId, + projectId: p.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: p.status ?? "PENDING", + environmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: p.projectId, + organizationId: p.organizationId, + }, + }; +} + +// A cuid-shaped run id (Session.currentRunId stores the internal cuid). +const cuidRunId = (suffix: string) => `run_${suffix.padEnd(24, "x").slice(0, 24)}`; + +describe("realtime-svc — replica-lag guards", () => { + // RunHydrator.hydrateByIds + heteroPostgresTest( + "hydrateByIds omits an un-replicated run from the frame ([]), never throws", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `rr_hydrate_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const runId = cuidRunId(`h${seq}`); + const friendlyId = `run_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + // The hydrator holds the real store; hydrateByIds passes `options.replica` as the read client. + const hydrator = new RunHydrator({ + replica: replica.client as PrismaClient, + runStore: writerStore, + cacheTtlMs: 0, + }); + + const rows = await hydrator.hydrateByIds(seed.environment.id, [runId]); + + // Observable: the frame is empty (row not visible on the replica), no throw. + expect(rows).toEqual([]); + expect(replica.wasHit("taskRun")).toBe(true); + + // The row IS on the primary — the next hydrate tick (writer/primary) recovers it. + const onPrimary = await writerStore.findRuns( + { + where: { runtimeEnvironmentId: seed.environment.id, id: { in: [runId] } }, + select: { id: true, friendlyId: true }, + }, + prisma + ); + expect(onPrimary).toHaveLength(1); + expect(onPrimary[0]!.friendlyId).toBe(friendlyId); + } + ); + + // RunHydrator.getRunById / #fetch + heteroPostgresTest( + "getRunById returns null for an un-replicated run (absent frame), never throws", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `rr_fetch_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const runId = cuidRunId(`f${seq}`); + const friendlyId = `run_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + const hydrator = new RunHydrator({ + replica: replica.client as PrismaClient, + runStore: writerStore, + cacheTtlMs: 0, + }); + + const row = await hydrator.getRunById(seed.environment.id, runId); + + // Observable: null (not yet visible), no throw. + expect(row).toBeNull(); + expect(replica.wasHit("taskRun")).toBe(true); + + const onPrimary = await writerStore.findRunOnPrimary( + { id: runId, runtimeEnvironmentId: seed.environment.id }, + { select: { friendlyId: true } } + ); + expect(onPrimary?.friendlyId).toBe(friendlyId); + } + ); + + // ensureRunForSession → getRunStatusAndFriendlyId + heteroPostgresTest( + "ensureRunForSession reuses a live run whose row missed the replica (writer re-probe) — NO double-trigger", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `srm_ensure_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + // The live current run — PENDING (non-final) — present on the PRIMARY only. + const runId = cuidRunId(`e${seq}`); + const friendlyId = `run_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "PENDING", + }) + ); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + primaryHolder.client = prisma; // the mocked `prisma` (writer re-probe target) + replicaHolder.client = replica.client; // the mocked `$replica` (probe target — lags) + storeHolder.store = writerStore; + triggerState.calls.length = 0; + triggerState.result = { run: { id: cuidRunId(`e2${seq}`), friendlyId: `run_${suffix}_2` } }; + + const session = { + id: `session_${suffix}`, + friendlyId: `session_${suffix}`, + taskIdentifier: "my-task", + triggerConfig: { basePayload: {} }, + currentRunId: runId, + currentRunVersion: 0, + } as unknown as Pick< + Session, + | "id" + | "friendlyId" + | "taskIdentifier" + | "triggerConfig" + | "currentRunId" + | "currentRunVersion" + >; + + const result = await ensureRunForSession({ + session, + environment: { id: seed.environment.id } as unknown as AuthenticatedEnvironment, + reason: "manual", + }); + + // Observable: the writer re-probe recovered the live run → reuse it, do NOT trigger a second run. + expect(result).toEqual({ runId, triggered: false }); + expect(triggerState.calls).toHaveLength(0); + // The replica WAS consulted first (and, frozen, missed) — proving the recovery is the writer + // re-probe, not a lucky replica hit. + expect(replica.wasHit("taskRun")).toBe(true); + + // Proof the run is a live row on the primary (writer read returns it non-final). + const onPrimary = await writerStore.findRunOnPrimary( + { id: runId }, + { select: { status: true, friendlyId: true } } + ); + expect(onPrimary?.status).toBe("PENDING"); + } + ); + + // swapSessionRun → resolveRunFriendlyId + heteroPostgresTest( + "swapSessionRun completes under lag; resolveRunFriendlyId falls back to the cuid for previousRunId", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `srm_swap_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + // The calling run — present on the PRIMARY only; its friendlyId will NOT be visible on the replica. + const callingRunId = cuidRunId(`s${seq}`); + const callingFriendlyId = `run_${suffix}_calling`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId: callingRunId, + friendlyId: callingFriendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + // A real Session row whose optimistic claim (currentRunId === callingRunId, version 0) can succeed. + const sessionRow = await prisma.session.create({ + data: { + friendlyId: `session_${suffix}`, + type: "chat.agent", + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + environmentType: "DEVELOPMENT", + organizationId: seed.organization.id, + taskIdentifier: "my-task", + triggerConfig: { basePayload: {} }, + currentRunId: callingRunId, + currentRunVersion: 0, + }, + }); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + primaryHolder.client = prisma; + replicaHolder.client = replica.client; + storeHolder.store = writerStore; + triggerState.calls.length = 0; + const newRunId = cuidRunId(`sn${seq}`); + const newFriendlyId = `run_${suffix}_new`; + triggerState.result = { run: { id: newRunId, friendlyId: newFriendlyId } }; + + const result = await swapSessionRun({ + session: sessionRow, + callingRunId, + environment: { id: seed.environment.id } as unknown as AuthenticatedEnvironment, + reason: "upgrade", + }); + + // Observable 1: the swap COMPLETED — the replica miss did not fail it. + expect(result).toEqual({ runId: newRunId, swapped: true }); + + // Observable 2: resolveRunFriendlyId missed on the replica and degraded to the cuid, so the + // previousRunId forwarded to the triggered run is the calling run's cuid (documented fallback). + expect(triggerState.calls).toHaveLength(1); + expect(triggerState.calls[0]!.body.payload.previousRunId).toBe(callingRunId); + expect(replica.wasHit("taskRun")).toBe(true); + + // Proof the null was lag-induced: the primary holds the resolvable friendlyId (≠ the cuid). + const onPrimary = await writerStore.findRunOnPrimary( + { id: callingRunId }, + { select: { friendlyId: true } } + ); + expect(onPrimary?.friendlyId).toBe(callingFriendlyId); + expect(callingFriendlyId).not.toBe(callingRunId); + + // Wait for the fire-and-forget SessionRun audit write (keyed by the new runId) to land before + // teardown, so it can't race a closing pool. Poll for the row rather than sleep a fixed interval. + await vi.waitFor(async () => { + expect(await prisma.sessionRun.findFirst({ where: { runId: newRunId } })).not.toBeNull(); + }); + } + ); + + // serializeSessionWithFriendlyRunId + heteroPostgresTest( + "serializeSessionWithFriendlyRunId serializes currentRunId=null when the run row lags the replica", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `sess_one_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const runId = cuidRunId(`o${seq}`); + const friendlyId = `run_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const sessionRow = await prisma.session.create({ + data: { + friendlyId: `session_${suffix}`, + type: "chat.agent", + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + environmentType: "DEVELOPMENT", + organizationId: seed.organization.id, + taskIdentifier: "my-task", + triggerConfig: { basePayload: {} }, + currentRunId: runId, + currentRunVersion: 0, + }, + }); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + // Inject a store whose REPLICA lags; the serializer's client-less findRun reads it. + const laggingStore = new PostgresRunStore({ prisma, readOnlyPrisma: replica.client }); + + const item = await serializeSessionWithFriendlyRunId(sessionRow, laggingStore); + + // Observable: the pre-existing currentRunId pointer resolves to null (safe degraded direction). + expect(item.currentRunId).toBeNull(); + expect(replica.wasHit("taskRun")).toBe(true); + expect(item.id).toBe(sessionRow.friendlyId); + + // Proof the row exists on the primary — the client's next GET (replica caught up) resolves it. + const onPrimary = await writerStore.findRunOnPrimary( + { id: runId, projectId: seed.project.id, runtimeEnvironmentId: seed.environment.id }, + { select: { friendlyId: true } } + ); + expect(onPrimary?.friendlyId).toBe(friendlyId); + } + ); + + // serializeSessionsWithFriendlyRunIds + heteroPostgresTest( + "serializeSessionsWithFriendlyRunIds serializes currentRunId=null for a session whose run lags the replica", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `sess_list_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const runId = cuidRunId(`l${seq}`); + const friendlyId = `run_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const sessionRow = await prisma.session.create({ + data: { + friendlyId: `session_${suffix}`, + type: "chat.agent", + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + environmentType: "DEVELOPMENT", + organizationId: seed.organization.id, + taskIdentifier: "my-task", + triggerConfig: { basePayload: {} }, + currentRunId: runId, + currentRunVersion: 0, + }, + }); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + const laggingStore = new PostgresRunStore({ prisma, readOnlyPrisma: replica.client }); + + const items = await serializeSessionsWithFriendlyRunIds( + [sessionRow], + { projectId: seed.project.id, runtimeEnvironmentId: seed.environment.id }, + laggingStore + ); + + // Observable: the un-replicated run drops out of the id→friendlyId map → currentRunId null. + expect(items).toHaveLength(1); + expect(items[0]!.currentRunId).toBeNull(); + expect(replica.wasHit("taskRun")).toBe(true); + + // Proof the row exists on the primary — the next list fetch resolves it. + const onPrimary = await writerStore.findRuns( + { + where: { + id: { in: [runId] }, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }, + select: { id: true, friendlyId: true }, + }, + prisma + ); + expect(onPrimary).toHaveLength(1); + expect(onPrimary[0]!.friendlyId).toBe(friendlyId); + } + ); +}); diff --git a/apps/webapp/test/realtimeSessionsIoRoute.replicaLag.guard.test.ts b/apps/webapp/test/realtimeSessionsIoRoute.replicaLag.guard.test.ts new file mode 100644 index 000000000..11e0bc24e --- /dev/null +++ b/apps/webapp/test/realtimeSessionsIoRoute.replicaLag.guard.test.ts @@ -0,0 +1,263 @@ +// Property: under a lagging control-plane replica the dashboard Agent-tab SSE subscribe loader still +// resolves a just-created Session / SessionRun and establishes the stream (200), never surfacing a +// "Session not found" / "Session not found for run" 404 for a live subscription. The Session read uses +// a replica-first writer fallback and the run<->session linkage read re-reads the primary on a replica +// miss. +// +// Drives the REAL exported route loader against a real Postgres testcontainer whose control-plane read +// replica is a real lagging replica (the shared laggingReplica primitive); the DB is never mocked. Only +// orthogonal deps are mocked (dashboard auth/session, project/environment slug resolution, the realtime +// stream instance, the request abort signal). Case A freezes Session on the replica; Case B freezes +// SessionRun; wasHit proves the frozen replica was really consulted, so no green is a lucky primary hit. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// ---- Holders wired into the mocked module singletons before each loader() call. ------------------ +// `primaryHolder.client` -> the real container (the writer / owning primary). +// `replicaHolder.client` -> a lagging replica over the SAME container: the frozen model's reads come +// back empty (row "not replicated yet"); every other model + all writes forward to the real container. +const primaryHolder = vi.hoisted(() => ({ client: undefined as any })); +const replicaHolder = vi.hoisted(() => ({ client: undefined as any })); + +// The user the (mocked) dashboard auth resolves to. +const AUTH = vi.hoisted(() => ({ userId: "user_sessions_io_guard" })); + +// Staged fix-orthogonal control-plane resolutions (project + environment slug lookups). +const cpLookups = vi.hoisted(() => ({ project: undefined as any, environment: undefined as any })); + +// ~/db.server: point the two proxies the run-store / control-plane singletons read at our holders. +// Never mocks the DB itself — the proxies forward to real testcontainer clients. Run-ops split handles +// are left undefined so runStore.server falls back to the single control-plane store (writer = `prisma`, +// replica = `$replica`) — the exact webapp single-DB topology the read-your-writes hazard lives in. +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + const lazyProxy = (holder: { client: any }, label: string) => + new Proxy( + {}, + { + get(_t, prop) { + if (!holder.client) throw new Error(`${label} not set for this test`); + const value = holder.client[prop]; + // The run-store singleton memoizes each Prisma delegate on first access; re-resolve through + // the holder so it always routes to the current test's client (mirrors the sibling guards). + if (value !== null && typeof value === "object") { + return new Proxy(value, { get: (_d, method) => holder.client[prop][method] }); + } + return value; + }, + } + ); + return { + prisma: lazyProxy(primaryHolder, "primaryHolder.client"), + $replica: lazyProxy(replicaHolder, "replicaHolder.client"), + // Split-off: leaving these undefined makes runStore.server build the single-DB passthrough store. + runOpsNewPrismaClient: undefined, + runOpsNewReplicaClient: undefined, + runOpsLegacyPrisma: undefined, + runOpsLegacyReplica: undefined, + sqlDatabaseSchema: Prisma.sql([`public`]), + }; +}); + +// Dashboard auth (orthogonal): a fixed user id. +vi.mock("~/services/session.server", () => ({ + getUserId: async () => AUTH.userId, + requireUserId: async () => AUTH.userId, +})); + +// Project / environment slug resolution (orthogonal control-plane auth reads): return what's staged. +vi.mock("~/models/project.server", () => ({ + findProjectBySlug: async () => cpLookups.project, +})); +vi.mock("~/models/runtimeEnvironment.server", () => ({ + findEnvironmentBySlug: async () => cpLookups.environment, +})); + +// Realtime stream backend (orthogonal): make `getRealtimeStreamInstance` return a REAL instance of the +// (mocked) S2RealtimeStreams class so the loader's `instanceof S2RealtimeStreams` gate passes, and its +// `streamResponseFromSessionStream` returns a marker 200 Response — proof the loader reached the stream +// rather than 404'ing on a session/linkage read. +vi.mock("~/services/realtime/s2realtimeStreams.server", () => { + class S2RealtimeStreams { + streamResponseFromSessionStream() { + return new Response("stream-established", { + status: 200, + headers: { "x-stream": "established" }, + }); + } + } + return { S2RealtimeStreams }; +}); +vi.mock("~/services/realtime/v1StreamsGlobal.server", async () => { + const { S2RealtimeStreams } = await import("~/services/realtime/s2realtimeStreams.server"); + return { getRealtimeStreamInstance: () => new (S2RealtimeStreams as any)() }; +}); + +// Request abort signal is sourced from AsyncLocalStorage in prod; not under test. +vi.mock("~/services/httpAsyncStorage.server", () => ({ + getRequestAbortSignal: () => new AbortController().signal, +})); + +// The REAL loader under test. +import { loader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.sessions.$sessionId.$io"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +async function seedSessionRunLinkage( + prisma: PrismaClient, + seed: { organization: any; project: any; environment: any }, + suffix: string +) { + const session = await prisma.session.create({ + data: { + friendlyId: `session_${suffix}`, + type: "chat", + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + environmentType: "DEVELOPMENT", + organizationId: seed.organization.id, + taskIdentifier: "agent-task", + triggerConfig: { basePayload: {} }, + }, + }); + const run = await prisma.taskRun.create({ + data: { + friendlyId: `run_${suffix}`, + engine: "V2", + taskIdentifier: "agent-task", + payload: "{}", + payloadType: "application/json", + traceId: `trace_${suffix}`, + spanId: `span_${suffix}`, + queue: "task/agent-task", + projectId: seed.project.id, + organizationId: seed.organization.id, + runtimeEnvironmentId: seed.environment.id, + runTags: [], + }, + }); + const sessionRun = await prisma.sessionRun.create({ + data: { sessionId: session.id, runId: run.id, reason: "initial" }, + }); + return { session, run, sessionRun }; +} + +function subscribeRequest() { + return new Request( + "http://localhost/resources/orgs/o/projects/p/env/dev/runs/r/realtime/v1/sessions/s/out" + ); +} + +function loaderParams( + seed: { organization: any; project: any }, + runFriendlyId: string, + sessionFriendlyId: string +) { + return { + organizationSlug: seed.organization.slug, + projectParam: seed.project.slug, + envParam: "dev", + runParam: runFriendlyId, + sessionId: sessionFriendlyId, + io: "out", + }; +} + +function stageCpLookups(seed: { organization: any; project: any; environment: any }) { + cpLookups.project = { id: seed.project.id, organizationId: seed.organization.id }; + cpLookups.environment = { id: seed.environment.id, type: "DEVELOPMENT", slug: "dev" }; +} + +describe("sessions.$sessionId.$io SSE subscribe loader under control-plane replica lag", () => { + heteroPostgresTest( + "establishes the stream for a Session not yet replicated via the writer fallback", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `sessguard_a_${seq++}`; + + const seed = await seedTenant(prisma, suffix); + const { session, run } = await seedSessionRunLinkage(prisma, seed, suffix); + stageCpLookups(seed); + + // Freeze `Session` on the replica: the just-created row is not visible there, only on the writer. + const replica = laggingReplica(prisma, [{ model: "session", mode: "missing" }]); + primaryHolder.client = prisma; + replicaHolder.client = replica.client; + + const res = (await loader({ + request: subscribeRequest(), + params: loaderParams(seed, run.friendlyId, session.friendlyId), + context: {} as never, + })) as Response; + + // Frozen replica really consulted (not a lucky primary hit); writer fallback then resolved it. + expect(replica.wasHit("session")).toBe(true); + expect(res.status).not.toBe(404); + expect(res.status).toBe(200); + expect(res.headers.get("x-stream")).toBe("established"); + const body = await res.clone().text(); + expect(body).not.toContain("Session not found"); + } + ); + + heteroPostgresTest( + "establishes the stream for a SessionRun linkage not yet replicated via the primary re-read", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `sessguard_b_${seq++}`; + + const seed = await seedTenant(prisma, suffix); + const { session, run } = await seedSessionRunLinkage(prisma, seed, suffix); + stageCpLookups(seed); + + // Freeze `SessionRun` on the replica: the linkage row is missing there, but the Session is present. + const replica = laggingReplica(prisma, [{ model: "sessionRun", mode: "missing" }]); + primaryHolder.client = prisma; + replicaHolder.client = replica.client; + + const res = (await loader({ + request: subscribeRequest(), + params: loaderParams(seed, run.friendlyId, session.friendlyId), + context: {} as never, + })) as Response; + + // Frozen replica really consulted; primary re-read then resolved the linkage. + expect(replica.wasHit("sessionRun")).toBe(true); + expect(res.status).not.toBe(404); + expect(res.status).toBe(200); + expect(res.headers.get("x-stream")).toBe("established"); + const body = await res.clone().text(); + expect(body).not.toContain("Session not found for run"); + } + ); +}); diff --git a/apps/webapp/test/realtimeStreamRoutes.replicaLag.test.ts b/apps/webapp/test/realtimeStreamRoutes.replicaLag.test.ts new file mode 100644 index 000000000..fb8cf5caf --- /dev/null +++ b/apps/webapp/test/realtimeStreamRoutes.replicaLag.test.ts @@ -0,0 +1,707 @@ +// Property: every realtime-stream route resolves a live run under replica lag. Each case drives the +// REAL exported loader/action from a webapp route module against a REAL Postgres testcontainer whose +// replica is FROZEN via the shared `laggingReplica` (taskRun "missing"). The REAL webapp `runStore` +// singleton is built over the mocked `~/db.server` handles as the single-DB webapp holds it +// (prisma = writer/primary, $replica = frozen replica; split handles undefined → single passthrough +// store). Only peripherals are mocked: bearer/rbac + dashboard-session auth, project/env slug +// resolvers, the downstream realtime backends (S2 / native), the run engine + waitpoint caches, the +// control-plane env resolver. The run READ and the found/not-found decision run for real. +// +// Each route reads the run from the REPLICA (`runStore.findRun(..., $replica)` or a client-less +// findRun served from readOnlyPrisma). Under lag a run present on the owning PRIMARY reads back null; +// the primary re-read (`run ?? runStore.findRunOnPrimary(where, args)`) resolves it, so every case +// gets PAST the run 404 (asserted on the caller's real Response). + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// --- Holders wired per-test into the mocked module singletons ------------------------------------ +// `primaryHolder.client` -> the real container (writer / owning primary). +// `replicaHolder.client` -> the frozen replica over the SAME container (taskRun reads come back null). +const primaryHolder = vi.hoisted(() => ({ client: undefined as any })); +const replicaHolder = vi.hoisted(() => ({ client: undefined as any })); + +// The authenticated environment the (mocked) auth resolves to + the dashboard project/session it +// resolves. Filled per-test from the seeded tenant so the routes' run reads scope correctly. +const ctx = vi.hoisted(() => ({ + environment: undefined as any, + project: undefined as any, + session: undefined as any, + userId: "user_realtime_guard", +})); + +// A single marker class used for BOTH the S2 backend mock and the getRealtimeStreamInstance mock, so +// the routes' `instanceof S2RealtimeStreams` checks pass. Every method returns a marker Response / +// benign value — reaching any of them proves the run read was tolerated (got past the 404). +const streamMarker = vi.hoisted(() => { + class S2Marker { + streamResponse() { + return new Response("marker:streamResponse", { status: 200 }); + } + streamResponseFromSessionStream() { + return new Response("marker:sessionStream", { status: 200 }); + } + ingestData() { + return new Response("marker:ingestData", { status: 200 }); + } + async initializeStream() { + return { responseHeaders: {} as Record }; + } + async appendPart() { + return undefined; + } + async getLastChunkIndex() { + return 0; + } + async readRecords() { + return [] as any[]; + } + async readSessionStreamRecords() { + return [] as any[]; + } + } + return { S2Marker }; +}); + +// ~/db.server: stable lazy proxies (captured by the runStore singleton at first import) that forward +// every access to the current test's client. Run-ops split handles undefined → single passthrough +// store (writer = prisma, replica = $replica) — the exact single-DB webapp topology this property +// lives in. Never mocks the DB itself. +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + const lazyProxy = (holder: { client: any }, label: string) => + new Proxy( + {}, + { + get(_t, prop) { + if (!holder.client) throw new Error(`${label} not set for this test`); + const value = holder.client[prop]; + if (value !== null && typeof value === "object") { + return new Proxy(value, { get: (_d, method) => holder.client[prop][method] }); + } + return value; + }, + } + ); + return { + prisma: lazyProxy(primaryHolder, "primaryHolder.client"), + $replica: lazyProxy(replicaHolder, "replicaHolder.client"), + runOpsNewPrismaClient: undefined, + runOpsNewReplicaClient: undefined, + runOpsLegacyPrisma: undefined, + runOpsLegacyReplica: undefined, + sqlDatabaseSchema: Prisma.sql([`public`]), + }; +}); + +// Bearer/JWT auth for the API-builder routes (createLoaderApiRoute / createActionApiRoute call +// rbac.authenticateBearer). Passing auth + permissive ability; the run read is unaffected. +vi.mock("~/services/rbac.server", () => ({ + rbac: { + authenticateBearer: async () => ({ + ok: true, + environment: ctx.environment, + subject: { type: "private" }, + ability: { can: () => true, canSuper: () => true }, + jwt: undefined, + }), + }, +})); + +// Dashboard-session auth + slug resolvers for the resources.* routes (peripheral). +vi.mock("~/services/session.server", () => ({ + requireUserId: async () => ctx.userId, + getUserId: async () => ctx.userId, +})); +vi.mock("~/models/project.server", () => ({ + findProjectBySlug: async () => ctx.project, +})); +vi.mock("~/models/runtimeEnvironment.server", () => ({ + findEnvironmentBySlug: async () => ctx.environment, +})); + +// Downstream realtime backends: markers. Reaching them proves the run read was tolerated. +vi.mock("~/services/realtime/s2realtimeStreams.server", () => ({ + S2RealtimeStreams: streamMarker.S2Marker, +})); +vi.mock("~/services/realtime/v1StreamsGlobal.server", () => ({ + getRealtimeStreamInstance: () => new streamMarker.S2Marker(), +})); +vi.mock("~/services/realtime/resolveRealtimeStreamClient.server", () => ({ + resolveRealtimeStreamClient: async () => ({ + streamRun: async () => new Response("marker:streamRun", { status: 200 }), + }), +})); + +// Session resolution (peripheral): return a marker session so the run↔session linkage check is +// what runs next (and, absent a seeded SessionRun row, yields a DISTINCT 404 body we assert on). +vi.mock("~/services/realtime/sessions.server", () => ({ + resolveSessionByIdOrExternalId: async () => ctx.session, + resolveSessionWithWriterFallback: async () => ctx.session, + canonicalSessionAddressingKey: () => "addr_marker", + isSessionFriendlyIdForm: () => false, + serializeSession: (s: any) => s, +})); + +// Control-plane env resolver used by the plain-action stream route + the streamKey dashboard route. +vi.mock("~/v3/runOpsMigration/controlPlaneResolver.server", () => ({ + controlPlaneResolver: { + resolveAuthenticatedEnv: async () => ctx.environment, + }, +})); + +// Run engine + waitpoint caches for the .wait() routes (peripheral). +vi.mock("~/v3/runEngine.server", () => ({ + engine: { + createManualWaitpoint: async () => ({ waitpoint: { id: "waitpoint_marker" }, isCached: false }), + completeWaitpoint: async () => ({}), + }, +})); +vi.mock("~/services/inputStreamWaitpointCache.server", () => ({ + getInputStreamWaitpoint: async () => null, + setInputStreamWaitpoint: async () => undefined, + deleteInputStreamWaitpoint: async () => undefined, +})); +vi.mock("~/services/sessionStreamWaitpointCache.server", () => ({ + addSessionStreamWaitpoint: async () => undefined, + removeSessionStreamWaitpoint: async () => undefined, +})); +vi.mock("~/models/waitpointTag.server", () => ({ + createWaitpointTag: async () => ({}), + MAX_TAGS_PER_WAITPOINT: 10, +})); + +vi.mock("~/services/httpAsyncStorage.server", () => ({ + getRequestAbortSignal: () => undefined, +})); +vi.mock("~/services/logger.server", () => ({ + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +import { PostgresRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; + +// The REAL exported callers under guard. +import { loader as realtimeRunLoader } from "~/routes/realtime.v1.runs.$runId"; +import { loader as streamStreamIdLoader } from "~/routes/realtime.v1.streams.$runId.$streamId"; +import { + action as streamTargetAction, + loader as streamTargetLoader, +} from "~/routes/realtime.v1.streams.$runId.$target.$streamId"; +import { action as streamTargetAppendAction } from "~/routes/realtime.v1.streams.$runId.$target.$streamId.append"; +import { + action as streamInputAction, + loader as streamInputLoader, +} from "~/routes/realtime.v1.streams.$runId.input.$streamId"; +import { action as inputStreamsWaitAction } from "~/routes/api.v1.runs.$runFriendlyId.input-streams.wait"; +import { action as sessionStreamsWaitAction } from "~/routes/api.v1.runs.$runFriendlyId.session-streams.wait"; +import { loader as dashSessionIoLoader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.sessions.$sessionId.$io"; +import { loader as dashStreamLoader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.$streamId"; +import { loader as dashInputStreamLoader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.input.$streamId"; +import { loader as dashStreamKeyLoader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.streams.$streamKey/route"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + // The streamKey dashboard route queries `$replica.project` with a member filter directly (not the + // mocked findProjectBySlug), so seed a real user + org membership the resolved userId matches. + const userId = `user_${suffix}`; + await prisma.user.create({ + data: { id: userId, email: `guard-${suffix}@example.com`, authenticationMethod: "MAGIC_LINK" }, + }); + await prisma.orgMember.create({ + data: { userId, organizationId: organization.id, role: "ADMIN" }, + }); + return { organization, project, environment, userId }; +} + +function buildCreateRunInput(p: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: p.runId, + engine: "V2", + status: "EXECUTING", + friendlyId: p.friendlyId, + runtimeEnvironmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: p.organizationId, + projectId: p.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "EXECUTING", + description: "Run is executing", + runStatus: "EXECUTING", + environmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: p.projectId, + organizationId: p.organizationId, + }, + }; +} + +// Seed a live run on the PRIMARY only, freeze the replica, and wire the module singletons. +async function setupLiveRunWithFrozenReplica(prisma: PrismaClient) { + const suffix = `rt_${seq++}`; + const seed = await seedTenant(prisma, suffix); + const runId = `run_${suffix}_${"a".repeat(16)}`; + const friendlyId = `run_${suffix}`; + + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + primaryHolder.client = prisma; + replicaHolder.client = replica.client; + + ctx.environment = { + id: seed.environment.id, + type: "DEVELOPMENT", + slug: "dev", + apiKey: seed.environment.apiKey, + organizationId: seed.organization.id, + projectId: seed.project.id, + organization: { id: seed.organization.id, slug: seed.organization.slug, featureFlags: {} }, + project: { + id: seed.project.id, + slug: seed.project.slug, + externalRef: seed.project.externalRef, + }, + }; + ctx.project = { id: seed.project.id, slug: seed.project.slug }; + ctx.session = { id: "session_marker", friendlyId: "session_marker", externalId: null }; + ctx.userId = seed.userId; + + return { seed, runId, friendlyId, replica }; +} + +// Normalize: some dashboard loaders THROW a Response on 404, others RETURN it. +async function invoke(fn: () => Promise): Promise { + try { + return await fn(); + } catch (e) { + if (e instanceof Response) return e; + throw e; + } +} + +// The API-builder enforces `maxContentLength` by REQUIRING a content-length header (missing → 413 +// before the handler). Set it so the run read under test is actually reached. +function jsonPost(url: string, body: unknown) { + const payload = JSON.stringify(body); + return new Request(url, { + method: "POST", + headers: { + "content-type": "application/json", + "content-length": String(Buffer.byteLength(payload)), + }, + body: payload, + }); +} + +function rawPost(url: string, body: string) { + return new Request(url, { + method: "POST", + headers: { "content-length": String(Buffer.byteLength(body)) }, + body, + }); +} + +describe("realtime-stream routes under replica lag", () => { + // 1. realtime.v1.runs.$runId loader (createLoaderApiRoute findResource) — SDK subscribeToRun. + heteroPostgresTest( + "realtime run loader resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + realtimeRunLoader({ + request: new Request(`http://localhost/realtime/v1/runs/${friendlyId}`), + params: { runId: friendlyId }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + // findResource re-reads the primary → run found → handler reaches the stream client marker. + expect(res.status).toBe(200); + expect(await res.clone().text()).toBe("marker:streamRun"); + } + ); + + // 2. realtime.v1.streams.$streamId loader (createLoaderApiRoute findResource) — SSE subscribe. + heteroPostgresTest( + "realtime stream loader resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + streamStreamIdLoader({ + request: new Request(`http://localhost/realtime/v1/streams/${friendlyId}/s1`), + params: { runId: friendlyId, streamId: "s1" }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(200); + expect(await res.clone().text()).toBe("marker:streamResponse"); + } + ); + + // 4. realtime.v1.streams.$target.$streamId action (createActionApiRoute). + heteroPostgresTest( + "realtime stream-target action resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + streamTargetAction({ + request: new Request(`http://localhost/realtime/v1/streams/${friendlyId}/self/s1`, { + method: "POST", + body: "chunk-data", + }), + params: { runId: friendlyId, target: "self", streamId: "s1" }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(200); + expect(await res.clone().text()).toBe("marker:ingestData"); + } + ); + + // 5. realtime.v1.streams.$target.$streamId loader (createLoaderApiRoute findResource, HEAD). + heteroPostgresTest( + "realtime stream-target loader resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + streamTargetLoader({ + request: new Request(`http://localhost/realtime/v1/streams/${friendlyId}/self/s1`, { + method: "HEAD", + }), + params: { runId: friendlyId, target: "self", streamId: "s1" }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + // Run found → handler reaches the HEAD getLastChunkIndex path (200 + X-Last-Chunk-Index). + expect(res.status).toBe(200); + expect(res.headers.get("X-Last-Chunk-Index")).toBe("0"); + } + ); + + // 6. realtime.v1.streams.$target.$streamId.append action (createActionApiRoute). + heteroPostgresTest( + "realtime stream-target append action resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + streamTargetAppendAction({ + request: rawPost( + `http://localhost/realtime/v1/streams/${friendlyId}/self/s1/append`, + "part-data" + ), + params: { runId: friendlyId, target: "self", streamId: "s1" }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + // The first (replica) run read is recovered via the primary re-read → the second read (already on + // primary) + appendPart marker → {ok:true} 200. + expect(res.status).toBe(200); + expect(await res.clone().json()).toEqual({ ok: true }); + } + ); + + // 7. realtime.v1.streams.input.$streamId action (createActionApiRoute, .send()). + heteroPostgresTest( + "realtime input-stream action resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + streamInputAction({ + request: jsonPost(`http://localhost/realtime/v1/streams/${friendlyId}/input/s1`, { + data: { hello: "world" }, + }), + params: { runId: friendlyId, streamId: "s1" }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + // Run found → appendPart marker → {ok:true}. + expect(res.status).toBe(200); + expect(await res.clone().json()).toEqual({ ok: true }); + } + ); + + // 8. realtime.v1.streams.input.$streamId loader (createLoaderApiRoute findResource, SSE tail). + heteroPostgresTest( + "realtime input-stream loader resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + streamInputLoader({ + request: new Request(`http://localhost/realtime/v1/streams/${friendlyId}/input/s1`), + params: { runId: friendlyId, streamId: "s1" }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(200); + expect(await res.clone().text()).toBe("marker:streamResponse"); + } + ); + + // 9. api.v1 input-streams.wait action (createActionApiRoute, .wait()). + heteroPostgresTest( + "input-streams wait action resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + inputStreamsWaitAction({ + request: jsonPost(`http://localhost/api/v1/runs/${friendlyId}/input-streams/wait`, { + streamId: "s1", + }), + params: { runFriendlyId: friendlyId }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + // Run found → createManualWaitpoint marker → waitpoint json. + expect(res.status).toBe(200); + expect(await res.clone().json()).toMatchObject({ isCached: false }); + } + ); + + // 10. api.v1 session-streams.wait action (createActionApiRoute, .wait()). + heteroPostgresTest( + "session-streams wait action resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + sessionStreamsWaitAction({ + request: jsonPost(`http://localhost/api/v1/runs/${friendlyId}/session-streams/wait`, { + session: "session_marker", + io: "out", + }), + params: { runFriendlyId: friendlyId }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + // Run found → createManualWaitpoint marker → waitpoint json. + expect(res.status).toBe(200); + expect(await res.clone().json()).toMatchObject({ isCached: false }); + } + ); + + // 11. dashboard sessions.$sessionId.$io loader. + heteroPostgresTest( + "dashboard session-io loader resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + dashSessionIoLoader({ + request: new Request( + `http://localhost/resources/.../runs/${friendlyId}/realtime/v1/sessions/session_marker/out` + ), + params: { + organizationSlug: ctx.environment.organization.slug, + projectParam: ctx.project.slug, + envParam: "dev", + runParam: friendlyId, + sessionId: "session_marker", + io: "out", + }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + const body = await res.clone().text(); + // Run read recovered via the primary re-read → past the run 404 → the next gate (run↔session + // linkage, no seeded SessionRun) returns "Session not found for run". Asserting on body text pins + // this to the run-read decision specifically. + expect(body).not.toContain("Run not found"); + expect(body).toContain("Session not found for run"); + } + ); + + // 12. dashboard streams.$streamId loader. + heteroPostgresTest( + "dashboard stream loader resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + dashStreamLoader({ + request: new Request(`http://localhost/resources/.../streams/${friendlyId}/s1`), + params: { + organizationSlug: ctx.environment.organization.slug, + projectParam: ctx.project.slug, + envParam: "dev", + runParam: friendlyId, + runId: friendlyId, + streamId: "s1", + }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(200); + expect(await res.clone().text()).toBe("marker:streamResponse"); + } + ); + + // 13. dashboard streams.input.$streamId loader. + heteroPostgresTest( + "dashboard input-stream loader resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + dashInputStreamLoader({ + request: new Request(`http://localhost/resources/.../streams/${friendlyId}/input/s1`), + params: { + organizationSlug: ctx.environment.organization.slug, + projectParam: ctx.project.slug, + envParam: "dev", + runParam: friendlyId, + runId: friendlyId, + streamId: "s1", + }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(200); + expect(await res.clone().text()).toBe("marker:streamResponse"); + } + ); + + // 14. dashboard streams.$streamKey loader (CLIENT-LESS findRun → replica). + heteroPostgresTest( + "dashboard stream-key loader resolves a live run under replica lag", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const { friendlyId, replica } = await setupLiveRunWithFrozenReplica(prisma); + + const res = await invoke( + () => + dashStreamKeyLoader({ + request: new Request( + `http://localhost/resources/.../streams/${friendlyId}/streams/my-stream` + ), + params: { + organizationSlug: ctx.environment.organization.slug, + projectParam: ctx.project.slug, + envParam: "dev", + runParam: friendlyId, + streamKey: "my-stream", + }, + context: {} as never, + }) as Promise + ); + + expect(replica.wasHit("taskRun")).toBe(true); + // The client-less findRun (replica) is recovered via findRunOnPrimary → env resolves (slug + // matches) → streamResponse marker. + expect(res.status).toBe(200); + expect(await res.clone().text()).toBe("marker:streamResponse"); + } + ); +}); diff --git a/apps/webapp/test/replayRouteReplicaLag.guard.test.ts b/apps/webapp/test/replayRouteReplicaLag.guard.test.ts new file mode 100644 index 000000000..e9bba9d66 --- /dev/null +++ b/apps/webapp/test/replayRouteReplicaLag.guard.test.ts @@ -0,0 +1,261 @@ +// Property: the replay ACTION resolves a source run that exists only on the owning primary and REPLAYS +// it (findRun by friendlyId, re-read on `prisma` when the replica misses) rather than reporting "Run +// not found". Drives the REAL exported route `action` on a real split topology +// (heteroRunOpsPostgresTest) with the owning legacy replica FROZEN and an empty mollifier buffer, so +// the primary re-read is the only path that resolves the run. Only orthogonal deps are stubbed (the +// auth/RBAC wrapper, downstream ReplayTaskRunService.call, the mollifier buffer, redirect helpers). + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect, vi } from "vitest"; + +// ---- hoisted holders (vi.mock factories run before imports) --------------------------------- +const routerHolder = vi.hoisted(() => ({ router: undefined as any })); +const primaryHolder = vi.hoisted(() => ({ client: undefined as any })); +const replayCallSpy = vi.hoisted(() => vi.fn()); + +// The route reads the source run through the `runStore` singleton. Point it at the split +// RoutingRunStore we build per-test (frozen legacy replica + real primary). +vi.mock("~/v3/runStore.server", () => ({ + runStore: new Proxy( + {}, + { + get(_t, prop) { + const router = routerHolder.router; + if (!router) throw new Error("routerHolder.router not set for this test"); + const value = (router as any)[prop]; + return typeof value === "function" ? value.bind(router) : value; + }, + } + ), +})); + +// The `prisma` singleton is the client the action passes as the primary re-read override, and the +// one controlPlaneResolver + the org-membership auth query read. Point every db.server handle at +// the real control-plane (PG14) container. The DB is never mocked — the proxy forwards to a real +// testcontainer client. (Re-resolving delegates per access mirrors waitpointCallback.controlPlane.) +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + const lazyProxy = (holder: { client: any }, label: string) => + new Proxy( + {}, + { + get(_t, prop) { + if (!holder.client) throw new Error(`${label} not set for this test`); + const value = holder.client[prop]; + if (value !== null && typeof value === "object") { + return new Proxy(value, { get: (_d, method) => holder.client[prop][method] }); + } + return value; + }, + } + ); + const cp = lazyProxy(primaryHolder, "primaryHolder.client"); + return { + prisma: cp, + $replica: cp, + runOpsNewPrisma: cp, + runOpsNewReplica: cp, + runOpsLegacyPrisma: cp, + runOpsLegacyReplica: cp, + runOpsNewPrismaClient: cp, + runOpsNewReplicaClient: cp, + runOpsLegacyPrismaClient: cp, + runOpsLegacyReplicaClient: cp, + runOpsSplitReadEnabled: false, + sqlDatabaseSchema: Prisma.sql([`public`]), + Prisma, + }; +}); + +// Bypass ONLY the auth/RBAC wrapper — return the handler verbatim so the REAL action body runs. +vi.mock("~/services/routeBuilders/dashboardBuilder", () => ({ + dashboardAction: (_options: unknown, handler: unknown) => handler, + dashboardLoader: (_options: unknown, handler: unknown) => handler, +})); + +// Downstream collaborator: observe whether the action reached the replay, without triggering one. +vi.mock("~/v3/services/replayTaskRun.server", () => ({ + ReplayTaskRunService: class { + call = replayCallSpy; + }, +})); + +// Force the mollifier buffer empty so the ONLY way pgRun resolves is the primary re-read. +vi.mock("~/v3/mollifier/mollifierBuffer.server", () => ({ + getMollifierBuffer: () => null, +})); + +// Make the redirect helpers session-free so the returned Response is deterministic regardless of +// SESSION_SECRET; keep every other export real. +vi.mock("~/models/message.server", async (importOriginal) => { + const actual = (await importOriginal()) as Record; + return { + ...actual, + redirectWithSuccessMessage: (path: string) => + new Response(null, { status: 302, headers: { location: path } }), + redirectWithErrorMessage: (path: string) => + new Response(null, { status: 302, headers: { location: path } }), + }; +}); + +import { PostgresRunStore, RoutingRunStore } from "@internal/run-store"; +import { action } from "~/routes/resources.taskruns.$runParam.replay"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +const CUID_25 = "c".repeat(25); // classifies LEGACY + +let seedN = 0; +async function seedTenant(prisma: PrismaClient) { + const suffix = `replay_guard_${seedN++}`; + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + const user = await prisma.user.create({ + data: { + email: `${suffix}@test.local`, + authenticationMethod: "MAGIC_LINK", + admin: false, + }, + }); + // Org membership: the action's authorization query is + // project.findFirst({ where: { id, organization: { members: { some: { userId } } } } }) + await prisma.orgMember.create({ + data: { userId: user.id, organizationId: organization.id, role: "ADMIN" }, + }); + return { organization, project, environment, user }; +} + +function sourceRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: "COMPLETED_SUCCESSFULLY" as const, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: JSON.stringify({ hello: "world" }), + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +// LEGACY-owning router whose legacy replica is frozen; the NEW store is a real (empty) store so the +// on-miss fan-out to the other store's replica also misses. Mirrors the split topology db.server +// builds in production (buildRunStore split path). +function buildRouterWithFrozenLegacyReplica(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyReplica }; +} + +const FAILED_REDIRECT = "/failed-redirect-sentinel"; + +function replayRequest() { + const body = new URLSearchParams({ failedRedirect: FAILED_REDIRECT }); + return new Request("http://localhost/replay", { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body, + }); +} + +describe("replay action resolves a source run under owning-replica lag via the primary re-read", () => { + heteroRunOpsPostgresTest( + "resolves a source run that exists only on the owning primary and replays it", + async ({ prisma14, prisma17 }) => { + const seed = await seedTenant(prisma14 as unknown as PrismaClient); + const runId = `run_${CUID_25}`; + const friendlyId = "run_replay_guard"; + await (prisma14 as unknown as PrismaClient).taskRun.create({ + data: sourceRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + // Wire the singletons the mocked modules read. + primaryHolder.client = prisma14; + const { router, legacyReplica } = buildRouterWithFrozenLegacyReplica( + prisma14 as unknown as PrismaClient, + prisma17 + ); + routerHolder.router = router; + replayCallSpy.mockReset(); + replayCallSpy.mockResolvedValue({ + id: "run_new_internal", + friendlyId: "run_new_friendly", + spanId: "span_new", + }); + + // Drive the REAL action handler (auth wrapper stubbed to a passthrough). + const response = (await action({ + request: replayRequest(), + params: { runParam: friendlyId }, + user: { id: seed.user.id }, + } as never)) as Response; + + // The frozen replica was consulted (proves we exercised the lag path, not a warm replica). + expect(legacyReplica.wasHit()).toBe(true); + + // The primary re-read resolved the source run, so the action reached the replay with the correct + // source run (a replica-only read would leave pgRun null and report "Run not found" instead). + expect(replayCallSpy).toHaveBeenCalledTimes(1); + expect(replayCallSpy.mock.calls[0][0]).toMatchObject({ friendlyId }); + + // User-facing: a success redirect to the new run's path, NOT the failedRedirect sentinel. + expect(response.status).toBe(302); + expect(response.headers.get("location")).not.toBe(FAILED_REDIRECT); + expect(response.headers.get("location")).toContain("run_new_friendly"); + } + ); +}); diff --git a/apps/webapp/test/resolveBatchForRealtime.test.ts b/apps/webapp/test/resolveBatchForRealtime.test.ts new file mode 100644 index 000000000..40bfab816 --- /dev/null +++ b/apps/webapp/test/resolveBatchForRealtime.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it, vi } from "vitest"; +import { resolveBatchTaskRunForRealtime } from "~/v3/realtime/resolveBatchForRealtime.server"; + +// The realtime batch route reads the batch client-less (replica). Under replica lag a just-created batch +// misses; `shouldRetryNotFound` covers the zodfetch GET, but the Electric ShapeStream consumer +// (self-hosters) ignores `x-should-retry`, so the route must re-read the owning PRIMARY on a miss. +// Passing a (non-replica) writer client flips each store leg to its own primary. +function laggingStore(batch: { id: string; friendlyId: string }) { + return { + findBatchTaskRunByFriendlyId: vi.fn( + async (_friendlyId: string, _envId: string, _args: unknown, client?: unknown) => + client ? batch : null + ), + }; +} + +describe("resolveBatchTaskRunForRealtime", () => { + it("re-reads the primary when the replica misses a fresh batch", async () => { + const store = laggingStore({ id: "b_1", friendlyId: "batch_1" }); + const found = await resolveBatchTaskRunForRealtime("batch_1", "env_1", { + store: store as never, + writer: {} as never, + }); + expect(found).toEqual({ id: "b_1", friendlyId: "batch_1" }); + expect(store.findBatchTaskRunByFriendlyId).toHaveBeenCalledTimes(2); + }); + + it("returns null when the batch is genuinely absent on both replica and primary", async () => { + const store = { findBatchTaskRunByFriendlyId: vi.fn(async () => null) }; + const found = await resolveBatchTaskRunForRealtime("nope", "env_1", { + store: store as never, + writer: {} as never, + }); + expect(found).toBeNull(); + }); + + it("does not re-read the primary when the replica already has the batch", async () => { + const store = { + findBatchTaskRunByFriendlyId: vi.fn(async () => ({ id: "b_2", friendlyId: "batch_2" })), + }; + await resolveBatchTaskRunForRealtime("batch_2", "env_1", { + store: store as never, + writer: {} as never, + }); + expect(store.findBatchTaskRunByFriendlyId).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/webapp/test/routesBatchGetReplicaLag.guard.test.ts b/apps/webapp/test/routesBatchGetReplicaLag.guard.test.ts new file mode 100644 index 000000000..6fe7a6467 --- /dev/null +++ b/apps/webapp/test/routesBatchGetReplicaLag.guard.test.ts @@ -0,0 +1,400 @@ +// Verifies the routes-batch-get read paths under replica lag — five external-API call sites across two +// store methods with different default routing. Each case drives the REAL exported route caller +// (loader/action) end-to-end against a real split Postgres whose owning LEGACY replica is FROZEN via +// laggingReplica; the run-store read path is real, never mocked. Only orthogonal deps are stubbed +// (bearer auth, request-idempotency cache, realtime stream, abort signal). +// +// findBatchTaskRunByFriendlyId (GET/subscribe) routes to the REPLICA: under lag a live batch reads as +// null and the resource gate returns a 404 carrying x-should-retry, which the SDK obeys — so the property +// is that the header is "true" (retryable, self-heals through lag), matching the sibling run-get routes. +// findBatchTaskRunById (dedup) routes to the PRIMARY: under owning-replica lag the just-written batch is +// still found (frozen replica never consulted), so the retried request dedups to the cached batch (200) +// instead of minting a duplicate — proven by wasHit("batchTaskRun") === false and the 200 dedup body. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { PostgresRunStore, RoutingRunStore } from "@internal/run-store"; +import type { CreateBatchTaskRunData } from "@internal/run-store"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// ---- Hoisted singleton holder + a stable Proxy that forwards every run-store method to the per-test +// REAL RoutingRunStore set in holder.store. vi.mock factories (hoisted above the imports) reference +// these, so they must be created inside vi.hoisted. +const { holder, storeProxy } = vi.hoisted(() => { + const holder = { + store: undefined as unknown, + environment: undefined as unknown, + cachedRequest: null as { id: string } | null, + }; + const storeProxy = new Proxy( + {}, + { + get(_target, prop) { + const store = holder.store as Record | undefined; + if (!store) throw new Error("test bug: holder.store not initialised before caller ran"); + const value = store[prop]; + return typeof value === "function" + ? (value as (...a: unknown[]) => unknown).bind(store) + : value; + }, + } + ); + return { holder, storeProxy }; +}); + +// db.server is imported transitively by the api-builder graph; provide inert stubs so importing the +// real routes never constructs a live Prisma client. The routes never read from these — the run-store +// read path is the injected RoutingRunStore below. +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + return { prisma: {}, $replica: {}, sqlDatabaseSchema: Prisma.sql([`public`]) }; +}); + +// Bearer auth (orthogonal): return ok with the seeded environment + a permissive ability. The +// friendlyId loaders 404 at the resource gate BEFORE authorization, so the ability only matters for the +// two PRIMARY action sites (authorization runs before their handler). +vi.mock("~/services/rbac.server", () => ({ + rbac: { + authenticateBearer: async () => ({ + ok: true, + environment: holder.environment, + subject: { type: "private" }, + ability: { can: () => true, canSuper: () => true }, + jwt: undefined, + }), + }, +})); + +// Inject the REAL split router as the app-level run store (used by v1/v2 batch loaders, the realtime +// loader, and the v2.tasks.batch action). +vi.mock("~/v3/runStore.server", () => ({ runStore: storeProxy })); + +// api.v3.batches reads through `engine.runStore`. Override test/setup.ts's no-op engine so `runStore` +// resolves to the injected router while any other engine access stays a no-op (dedup path touches none). +vi.mock("~/v3/runEngine.server", () => ({ + engine: new Proxy( + {}, + { + get(_target, prop) { + if (prop === "runStore") return storeProxy; + return () => Promise.resolve(undefined); + }, + } + ), +})); + +// Request-idempotency cache lookup (orthogonal): return whatever the test seeded. The REAL +// findCachedEntity callback then drives runStore.findBatchTaskRunById against the injected router. +vi.mock("~/services/requestIdempotencyInstance.server", () => ({ + requestIdempotency: { + checkRequest: async () => holder.cachedRequest, + saveRequest: async () => {}, + }, +})); + +// Realtime stream client (orthogonal). Under lag the realtime loader 404s at the resource gate +// before reaching it; stubbing it just keeps the import light. `getRequestAbortSignal` is likewise only +// reached on the found path, so httpAsyncStorage.server is left real (the logger needs getHttpContext). +vi.mock("~/services/realtime/resolveRealtimeStreamClient.server", () => ({ + resolveRealtimeStreamClient: async () => ({ + streamBatch: async () => new Response("stream", { status: 200 }), + }), +})); + +// The REAL callers under test. +import { loader as loaderV1 } from "~/routes/api.v1.batches.$batchId"; +import { loader as loaderV2 } from "~/routes/api.v2.batches.$batchId"; +import { loader as loaderRealtime } from "~/routes/realtime.v1.batches.$batchId"; +import { action as actionTasksBatch } from "~/routes/api.v2.tasks.batch"; +import { action as actionCreateBatch } from "~/routes/api.v3.batches"; + +// A cuid (25 chars after the `batch_` prefix) classifies LEGACY, so the batch is owned by the legacy +// (control-plane) store; the client-less reads then land on the LEGACY leg. +const CUID_25 = "c".repeat(25); + +let seq = 0; + +async function seedEnvironment(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +// Build the RoutingRunStore exactly as the webapp holds it, with the LEGACY replica frozen in "missing" +// mode for the batch models: a REPLICA-routed batch read comes back null AND flips wasHit — so a null +// result + wasHit(true) proves REPLICA routing (the friendlyId reads above), and a correct row + +// wasHit(false) proves PRIMARY routing (the dedup reads above). +function buildLaggingRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyReplica = laggingReplica(prisma14, [ + { model: "batchTaskRun", mode: "missing" }, + { model: "batchTaskRunItem", mode: "missing" }, + ]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +function batchData(overrides: Partial): CreateBatchTaskRunData { + return { + id: `batch_${CUID_25}`, + friendlyId: "batch_route_get_f", + runtimeEnvironmentId: "PLACEHOLDER", + status: "PENDING", + runCount: 1, + runIds: ["run_child_1"], + expectedCount: 1, + batchVersion: "runengine:v2", + sealed: false, + ...overrides, + }; +} + +function getRequest(url: string, apiKey: string) { + return new Request(url, { headers: { Authorization: `Bearer ${apiKey}` } }); +} + +// The slim AuthenticatedEnvironment the routes + tenantContext read. Only orthogonal fields — the +// actual batch read is keyed off `.id`. +function authEnv(seed: Awaited>) { + return { + id: seed.environment.id, + apiKey: seed.environment.apiKey, + type: seed.environment.type, + slug: seed.environment.slug, + organization: { + id: seed.organization.id, + slug: seed.organization.slug, + title: seed.organization.title, + }, + project: { + id: seed.project.id, + slug: seed.project.slug, + externalRef: seed.project.externalRef, + }, + }; +} + +describe("routes-batch-get callers under a lagging replica", () => { + // ── api.v1.batches loader — findBatchTaskRunByFriendlyId (REPLICA) ─────────────────── + heteroRunOpsPostgresTest( + "api.v1.batches loader: a live batch stale on the replica returns a retryable 404 (x-should-retry:true)", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const suffix = `bget_v1_${seq++}`; + const seed = await seedEnvironment(prisma14, suffix); + const friendlyId = `batch_${suffix}_f`; + await legacyStore.createBatchTaskRun( + batchData({ friendlyId, runtimeEnvironmentId: seed.environment.id }) + ); + + holder.store = router; + holder.environment = authEnv(seed); + + const res = (await loaderV1({ + request: getRequest( + `http://localhost/api/v1/batches/${friendlyId}`, + seed.environment.apiKey + ), + params: { batchId: friendlyId }, + context: {} as never, + })) as Response; + + // The owning REPLICA was genuinely consulted and (frozen) missed. + expect(legacyReplica.wasHit("batchTaskRun")).toBe(true); + // Real loader output: a 404 for a LIVE batch stale on the replica... + expect(res.status).toBe(404); + // ...but retryable → the SDK retries through lag until the replica catches up. + expect(res.headers.get("x-should-retry")).toBe("true"); + } + ); + + // ── api.v2.batches loader — findBatchTaskRunByFriendlyId (REPLICA) ─────────────────── + heteroRunOpsPostgresTest( + "api.v2.batches loader: client-less replica read, stale-null returns a retryable 404", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const suffix = `bget_v2_${seq++}`; + const seed = await seedEnvironment(prisma14, suffix); + const friendlyId = `batch_${suffix}_f`; + await legacyStore.createBatchTaskRun( + batchData({ + friendlyId, + runtimeEnvironmentId: seed.environment.id, + processingCompletedAt: new Date(), + }) + ); + + holder.store = router; + holder.environment = authEnv(seed); + + const res = (await loaderV2({ + request: getRequest( + `http://localhost/api/v2/batches/${friendlyId}`, + seed.environment.apiKey + ), + params: { batchId: friendlyId }, + context: {} as never, + })) as Response; + + expect(legacyReplica.wasHit("batchTaskRun")).toBe(true); + expect(res.status).toBe(404); + expect(res.headers.get("x-should-retry")).toBe("true"); + } + ); + + // ── realtime.v1.batches loader — replica miss recovered by the owning-primary re-read ───────────── + // The realtime loader re-reads the owning primary on a replica miss (the ShapeStream consumer ignores + // x-should-retry, so a 404 here would strand), so a stale-on-replica batch reaches streamBatch. + heteroRunOpsPostgresTest( + "realtime.v1.batches loader: a batch stale on the replica is recovered via the owning-primary re-read (reaches streamBatch, no 404)", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const suffix = `bget_rt_${seq++}`; + const seed = await seedEnvironment(prisma14, suffix); + const friendlyId = `batch_${suffix}_f`; + const batchId = `batch_${CUID_25}`; + await legacyStore.createBatchTaskRun( + batchData({ id: batchId, friendlyId, runtimeEnvironmentId: seed.environment.id }) + ); + + holder.store = router; + holder.environment = authEnv(seed); + + const res = (await loaderRealtime({ + request: getRequest( + `http://localhost/realtime/v1/batches/${friendlyId}`, + seed.environment.apiKey + ), + params: { batchId: friendlyId }, + context: {} as never, + })) as Response; + + // Replica was consulted first (the miss)… + expect(legacyReplica.wasHit("batchTaskRun")).toBe(true); + // …then the owning-primary re-read found the live batch, so the subscription starts (streamBatch + // returns 200) instead of a resource-gate 404. + expect(res.status).toBe(200); + } + ); + + // ── api.v2.tasks.batch action — findBatchTaskRunById (PRIMARY) ─────────────────────────────────────── + heteroRunOpsPostgresTest( + "api.v2.tasks.batch action: request-idempotency dedup reads the owning primary, finding the cached batch under lag (200 dedup, no duplicate)", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const suffix = `bget_tb_${seq++}`; + const seed = await seedEnvironment(prisma14, suffix); + const batchId = `batch_${CUID_25}`; + const friendlyId = `batch_${suffix}_f`; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 5, + }) + ); + + holder.store = router; + holder.environment = authEnv(seed); + holder.cachedRequest = { id: batchId }; + + const body = JSON.stringify({ items: [{ task: "my-task" }] }); + const res = (await actionTasksBatch({ + request: new Request("http://localhost/api/v2/tasks/batch", { + method: "POST", + headers: { + "content-type": "application/json", + "content-length": String(Buffer.byteLength(body)), + authorization: `Bearer ${seed.environment.apiKey}`, + "x-trigger-request-idempotency-key": "req_idem_tb", + }, + body, + }), + params: {}, + context: {} as never, + })) as Response; + + // PRIMARY routing tolerance: the frozen owning replica was NEVER consulted... + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + // ...and the real action returned the cached batch (dedup), not a duplicate create. + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ id: friendlyId, runCount: 5 }); + } + ); + + // ── api.v3.batches action — findBatchTaskRunById (PRIMARY) ─────────────────────────────────────────── + heteroRunOpsPostgresTest( + "api.v3.batches action: create-batch idempotency dedup reads the primary, finding the cached batch under lag (isCached 200, no duplicate)", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const suffix = `bget_v3_${seq++}`; + const seed = await seedEnvironment(prisma14, suffix); + const batchId = `batch_${CUID_25}`; + const friendlyId = `batch_${suffix}_f`; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 3, + }) + ); + + holder.store = router; + holder.environment = authEnv(seed); + holder.cachedRequest = { id: batchId }; + + const body = JSON.stringify({ runCount: 3, idempotencyKey: "idem_v3" }); + const res = (await actionCreateBatch({ + request: new Request("http://localhost/api/v3/batches", { + method: "POST", + headers: { + "content-type": "application/json", + "content-length": String(Buffer.byteLength(body)), + authorization: `Bearer ${seed.environment.apiKey}`, + }, + body, + }), + params: {}, + context: {} as never, + })) as Response; + + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ id: friendlyId, runCount: 3, isCached: true }); + } + ); +}); diff --git a/apps/webapp/test/runGetRoutes.replicaLag.guard.test.ts b/apps/webapp/test/runGetRoutes.replicaLag.guard.test.ts new file mode 100644 index 000000000..8d844f550 --- /dev/null +++ b/apps/webapp/test/runGetRoutes.replicaLag.guard.test.ts @@ -0,0 +1,1021 @@ +// Replica-lag proof for the "routes-run-get" reads — every run-GET read that RoutingRunStore serves +// from the OWNING store's REPLICA. This file imports and drives the REAL exported loaders/actions +// from the route modules, against a REAL split RoutingRunStore over two Postgres testcontainers with +// the owning (legacy) store's REPLICA FROZEN via the shared laggingReplica primitive. Only deps +// orthogonal to the read are mocked (auth/session, control-plane env resolution, redirect/toast +// formatting, downstream engine/clickhouse/event/presenter services); the run-store read path each +// case's decision hangs on is the genuine article. +// +// For every case, the concrete proof driven through the real caller is the CORRECT observable output +// under lag: +// * frozen-STALE replica (the run WAS replicated but carries an out-of-date snapshot): the caller's +// routing/redirect/auth/queue-key DECISION is driven only by IMMUTABLE fields (projectId, +// runtimeEnvironmentId, spanId, traceId, organizationId, engine, queue, concurrencyKey, createdAt), +// so the real caller returns the CORRECT redirect / 200 payload even off the lagging replica — the +// only staleness is cosmetic (status/completedAt) which the UI self-heals on the next poll. +// * frozen-MISSING replica + a documented recovery: logs.$logId returns 200 with runStatus undefined +// (optional annotation); the finished-attempt read returns null → empty output on a run that still +// renders; the cancel/replay action context-resolvers fall back to the owning PRIMARY and resolve +// the org (proven by the action NOT spuriously denying). +// The run-store read really goes through the frozen replica in each case — asserted with +// `wasHit("taskRun")` — so no proof is a lucky primary hit. + +import { describe, expect, vi } from "vitest"; +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { PostgresRunStore, RoutingRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; + +vi.setConfig({ testTimeout: 120_000, hookTimeout: 120_000 }); + +// ── Hoisted holders wired per-test before each real caller runs ─────────────────────────────────── +// `cp.client` -> the real control-plane container (prisma14): backs the mocked ~/db.server `prisma` +// AND (branded) `$replica`, used for the orthogonal project/orgMember auth reads. +// `router.store` -> the real split RoutingRunStore whose legacy replica is frozen. +// `authUser`, `resolvedEnv`, `logDetail`, `project`, `environment` feed the mocked fix-orthogonal deps. +const cp = vi.hoisted(() => ({ client: undefined as any })); +const router = vi.hoisted(() => ({ store: undefined as any })); +const authUser = vi.hoisted(() => ({ id: "user_rrg_guard", admin: false, isImpersonating: false })); +const resolved = vi.hoisted(() => ({ + authEnv: undefined as any, + env: undefined as any, + lockedWorker: null as any, +})); +const logDetail = vi.hoisted(() => ({ result: undefined as any })); +const cpLookups = vi.hoisted(() => ({ project: undefined as any, environment: undefined as any })); +const cancelCalls = vi.hoisted(() => ({ runs: [] as any[] })); +const replayCalls = vi.hoisted(() => ({ runs: [] as any[] })); + +const READ_REPLICA_BRAND = Symbol.for("trigger.dev/run-store/read-replica"); + +// ~/db.server: `prisma` -> real writer; `$replica` -> a BRANDED proxy over the real writer. The brand +// makes RoutingRunStore treat a `$replica`-arg read as a replica read (no primary escalation) exactly +// as production does, while property access (project/orgMember/taskSchedule.findFirst) forwards to the +// real control-plane client so the orthogonal auth reads work. Run-ops split handles are left +// undefined; the router is injected directly via the ~/v3/runStore.server mock below. +vi.mock("~/db.server", () => { + const brandedReplica = new Proxy( + {}, + { + get(_t, prop) { + if (prop === READ_REPLICA_BRAND) return true; + const c = cp.client; + if (!c) throw new Error("cp.client not set for this test"); + const value = c[prop]; + return typeof value === "function" ? value.bind(c) : value; + }, + } + ); + const prismaProxy = new Proxy( + {}, + { + get(_t, prop) { + const c = cp.client; + if (!c) throw new Error("cp.client not set for this test"); + const value = c[prop]; + return typeof value === "function" ? value.bind(c) : value; + }, + } + ); + return { + prisma: prismaProxy, + $replica: brandedReplica, + runOpsNewPrismaClient: undefined, + runOpsNewReplicaClient: undefined, + runOpsLegacyPrisma: undefined, + runOpsLegacyReplica: undefined, + }; +}); + +// Inject the REAL split router. A stable Proxy keeps the named import binding constant while +// forwarding every method to the per-test router in `router.store`. +vi.mock("~/v3/runStore.server", () => ({ + runStore: new Proxy( + {}, + { + get(_t, prop) { + const store = router.store; + if (!store) throw new Error("router.store not set for this test"); + const value = store[prop]; + return typeof value === "function" ? value.bind(store) : value; + }, + } + ), +})); + +// Auth/session (orthogonal): fixed user id. +vi.mock("~/services/session.server", () => ({ + requireUserId: async () => authUser.id, + requireUser: async () => authUser, + getUserId: async () => authUser.id, +})); + +// Control-plane env resolution (a downstream cross-DB lookup, orthogonal to the run-store read): +// returns whatever the test staged. +vi.mock("~/v3/runOpsMigration/controlPlaneResolver.server", () => ({ + controlPlaneResolver: { + resolveAuthenticatedEnv: async () => resolved.authEnv, + resolveEnv: async () => resolved.env, + resolveRunLockedWorker: async () => resolved.lockedWorker, + }, +})); + +// Redirect/toast formatting (orthogonal): marker Responses so assertions don't need cookie machinery. +vi.mock("~/models/message.server", () => ({ + redirectWithSuccessMessage: async (path: string, _req: Request, message: string) => + new Response(null, { status: 302, headers: { "x-redirect": path, "x-toast": message } }), + redirectWithErrorMessage: async (path: string, _req: Request, message: string) => + new Response(null, { status: 302, headers: { "x-redirect": path, "x-error": message } }), +})); + +// Buffer disabled everywhere so a run-store miss is a clean miss (no buffer masking the read outcome). +vi.mock("~/v3/mollifier/mollifierBuffer.server", () => ({ getMollifierBuffer: () => null })); + +// debug-loader downstream: the run-queue Redis introspection is not the read under test. +vi.mock("~/v3/runEngine.server", () => ({ + engine: { + runQueue: { + getQueueConcurrencyLimit: async () => 10, + getEnvConcurrencyLimit: async () => 20, + currentConcurrencyOfQueue: async () => 1, + currentConcurrencyOfEnvironment: async () => 2, + keys: { + queueCurrentConcurrencyKey: () => "qcc", + envCurrentConcurrencyKey: () => "ecc", + queueConcurrencyLimitKey: () => "qcl", + envConcurrencyLimitKey: () => "ecl", + }, + }, + }, +})); + +// logs.$logId downstream: ClickHouse + presenter + slug lookups are orthogonal. +vi.mock("~/services/clickhouse/clickhouseFactoryInstance.server", () => ({ + clickhouseFactory: { getClickhouseForOrganization: async () => ({}) }, +})); +vi.mock("~/presenters/v3/LogDetailPresenter.server", () => ({ + LogDetailPresenter: class { + async call() { + return logDetail.result; + } + }, +})); +vi.mock("~/models/project.server", () => ({ + findProjectBySlug: async () => cpLookups.project, +})); +vi.mock("~/models/runtimeEnvironment.server", () => ({ + findEnvironmentBySlug: async () => cpLookups.environment, + displayableEnvironment: (env: any) => ({ id: env.id, type: env.type, slug: env.slug }), +})); + +// logs.download downstream: event repository + trace export are orthogonal. +vi.mock("~/v3/eventRepository/index.server", () => ({ + getEventRepositoryForStore: async () => ({ + streamTraceEvents: async function* () { + yield {} as any; + }, + }), +})); +vi.mock("~/v3/eventRepository/traceExport.server", () => ({ + getTraceExportFormat: () => ({ extension: "log" }), + streamTraceExport: async function* () { + yield "trace-line\n"; + }, +})); +vi.mock("~/v3/taskEventStore.server", () => ({ + getTaskEventStoreTableForRun: () => "taskEvent", +})); +vi.mock("~/env.server", () => ({ + env: { APP_ORIGIN: "https://app.test", TASK_RUN_METADATA_MAXIMUM_SIZE: 262144 }, +})); + +// replay-loader downstream: regions/worker/queue lookups are orthogonal. +vi.mock("~/presenters/v3/RegionsPresenter.server", () => ({ + RegionsPresenter: class { + async call() { + return { regions: [] }; + } + }, +})); +vi.mock("~/v3/models/workerDeployment.server", () => ({ + findCurrentWorkerDeployment: async () => null, +})); +vi.mock("~/runEngine/concerns/workerQueueSplit.server", () => ({ + regionForDisplay: () => undefined, +})); + +// Downstream cancel/replay services (engine work, orthogonal): record the call. +vi.mock("~/v3/services/cancelTaskRun.server", () => ({ + CancelTaskRunService: class { + async call(run: any) { + cancelCalls.runs.push(run); + } + }, +})); +vi.mock("~/v3/services/replayTaskRun.server", () => ({ + ReplayTaskRunService: class { + async call(run: any) { + replayCalls.runs.push(run); + return { id: "new_run", friendlyId: "run_replayed", spanId: "span_new" }; + } + }, +})); + +// RBAC gate (orthogonal) for the dashboardAction routes (cancel/replay). Passing ability; the route's +// own control-plane membership check still runs against seeded data. +vi.mock("~/services/rbac.server", () => ({ + rbac: { + authenticateSession: async () => ({ + ok: true, + user: { id: authUser.id, email: "guard@example.com", admin: false }, + ability: { can: () => true, canSuper: () => true }, + }), + }, +})); +vi.mock("~/services/logger.server", () => ({ + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +// ── Real callers under guard ────────────────────────────────────────────────────────────────────── +import { loader as orgsRedirectLoader } from "~/routes/orgs.$organizationSlug.projects.$projectParam.runs.$runParam"; +import { loader as shortLinkLoader } from "~/routes/runs.$runParam"; +import { loader as inspectorLoader } from "~/routes/resources.runs.$runParam"; +import { loader as debugLoader } from "~/routes/resources.taskruns.$runParam.debug"; +import { loader as logDetailLoader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId"; +import { loader as logsDownloadLoader } from "~/routes/resources.runs.$runParam.logs.download"; +import { + loader as replayLoader, + action as replayAction, +} from "~/routes/resources.taskruns.$runParam.replay"; +import { action as cancelAction } from "~/routes/resources.taskruns.$runParam.cancel"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +// A cuid-shaped id (25 chars, no run-ops marker) classifies LEGACY → friendlyId-keyed reads route to +// the legacy (control-plane / prisma14) store, whose replica we freeze. +const CUID_25 = "e".repeat(25); +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + await prisma.user.create({ + data: { + id: authUser.id, + email: `guard-${suffix}@example.com`, + authenticationMethod: "MAGIC_LINK", + }, + }); + const orgMember = await prisma.orgMember.create({ + data: { userId: authUser.id, organizationId: organization.id, role: "ADMIN" }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + // Link the dev env to the org member so the replay loader's DEVELOPMENT env-list query + // (orgMember.userId === userId) surfaces it. + orgMemberId: orgMember.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(p: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + status?: string; + spanId?: string; + traceId?: string; + createdAt?: Date; + completedAt?: Date | null; + concurrencyKey?: string | null; +}): CreateRunInput { + return { + data: { + id: p.runId, + engine: "V2", + status: (p.status ?? "PENDING") as never, + friendlyId: p.friendlyId, + runtimeEnvironmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: p.organizationId, + projectId: p.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: p.traceId ?? (p.spanId ? `trace_${p.friendlyId}` : "trace_1"), + spanId: p.spanId ?? "span_1", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + concurrencyKey: p.concurrencyKey ?? undefined, + createdAt: p.createdAt ?? new Date("2024-01-01T00:00:00.000Z"), + completedAt: p.completedAt ?? undefined, + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: (p.status ?? "PENDING") as never, + environmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: p.projectId, + organizationId: p.organizationId, + }, + }; +} + +// Build the router as runStore.server holds it: legacy store (prisma14) + dedicated store (prisma17). +// The legacy replica is a frozen laggingReplica; a friendlyId/cuid-routed read is served stale/missing. +function buildRouterWithFrozenLegacyReplica( + prisma14: PrismaClient, + prisma17: RunOpsPrismaClient, + legacyReplica: AnyClient +) { + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica as never, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + return new RoutingRunStore({ new: newStore, legacy: legacyStore }); +} + +function authEnvFor(seed: { organization: any; project: any; environment: any }) { + return { + id: seed.environment.id, + slug: seed.environment.slug, + type: seed.environment.type, + apiKey: seed.environment.apiKey, + organizationId: seed.organization.id, + organization: { + id: seed.organization.id, + slug: seed.organization.slug, + title: seed.organization.title, + }, + project: { + id: seed.project.id, + slug: seed.project.slug, + name: seed.project.name, + externalRef: seed.project.externalRef, + }, + git: null, + }; +} + +describe("routes-run-get — REAL loaders/actions vs a frozen owning replica", () => { + // orgs redirect loader — canonical run redirect + heteroRunOpsPostgresTest( + "orgs redirect loader: frozen-STALE replica → 302 to the correct v3 path (immutable projectId/runtimeEnvironmentId); frozen-MISSING → transient 404", + async ({ prisma14, prisma17 }) => { + const suffix = `rrg1_${seq++}`; + cp.client = prisma14; + const seed = await seedTenant(prisma14 as unknown as PrismaClient, suffix); + const friendlyId = `run_${suffix}`; + const runId = `run_${CUID_25}`; + const writer = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await writer.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + }) + ); + resolved.authEnv = authEnvFor(seed); + + const req = (fid: string) => + new Request( + `http://localhost/orgs/${seed.organization.slug}/projects/${seed.project.slug}/runs/${fid}` + ); + const params = (fid: string) => ({ + organizationSlug: seed.organization.slug, + projectParam: seed.project.slug, + runParam: fid, + }); + + // (a) frozen-STALE: the run IS on the replica but with an out-of-date snapshot. projectId + + // runtimeEnvironmentId are immutable, so the redirect target is correct off the lagging replica. + const stale = laggingReplica(prisma14, [ + { + model: "taskRun", + mode: "frozen", + rows: [ + { + friendlyId, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "PENDING", + }, + ], + }, + ]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + stale.client + ); + const staleRes = (await orgsRedirectLoader({ + request: req(friendlyId), + params: params(friendlyId), + context: {} as never, + })) as Response; + expect(stale.wasHit("taskRun")).toBe(true); + expect(staleRes.status).toBe(302); + expect(staleRes.headers.get("location")).toBe( + `/orgs/${seed.organization.slug}/projects/${seed.project.slug}/env/${seed.environment.slug}/runs/${friendlyId}` + ); + + // (b) frozen-MISSING: transient not-found (drives no mutation; self-heals on next load). + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + missing.client + ); + let missStatus = 0; + try { + await orgsRedirectLoader({ + request: req(friendlyId), + params: params(friendlyId), + context: {} as never, + }); + } catch (e) { + missStatus = (e as Response).status; + } + expect(missing.wasHit("taskRun")).toBe(true); + expect(missStatus).toBe(404); + } + ); + + // short-link loader — public short-link redirect + heteroRunOpsPostgresTest( + "short-link loader: frozen-STALE → 302 to correct v3 path with ?span (immutable spanId/projectId); frozen-MISSING → error-redirect (no crash)", + async ({ prisma14, prisma17 }) => { + const suffix = `rrg2_${seq++}`; + cp.client = prisma14; + const seed = await seedTenant(prisma14 as unknown as PrismaClient, suffix); + const friendlyId = `run_${suffix}`; + const runId = `run_${CUID_25}`; + const spanId = "span_short_fixed"; + const writer = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await writer.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + spanId, + }) + ); + resolved.authEnv = authEnvFor(seed); + + const stale = laggingReplica(prisma14, [ + { + model: "taskRun", + mode: "frozen", + rows: [ + { + friendlyId, + spanId, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }, + ], + }, + ]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + stale.client + ); + const staleRes = (await shortLinkLoader({ + request: new Request(`http://localhost/runs/${friendlyId}`), + params: { runParam: friendlyId }, + context: {} as never, + })) as Response; + expect(stale.wasHit("taskRun")).toBe(true); + expect(staleRes.status).toBe(302); + const loc = staleRes.headers.get("location")!; + expect(loc).toContain( + `/orgs/${seed.organization.slug}/projects/${seed.project.slug}/env/${seed.environment.slug}/runs/${friendlyId}` + ); + expect(loc).toContain(`span=${spanId}`); + + // frozen-MISSING → the loader returns the error-redirect marker (302, x-error), never throws/500. + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + missing.client + ); + const missRes = (await shortLinkLoader({ + request: new Request(`http://localhost/runs/${friendlyId}`), + params: { runParam: friendlyId }, + context: {} as never, + })) as Response; + expect(missing.wasHit("taskRun")).toBe(true); + expect(missRes.status).toBe(302); + expect(missRes.headers.get("x-error")).toContain("doesn't exist"); + } + ); + + // inspector loader — findRun + the finished-attempt read + heteroRunOpsPostgresTest( + "inspector loader: frozen-STALE run → 200 typedjson with correct immutable friendlyId; the finished-attempt read missing on the replica → empty output, run still renders", + async ({ prisma14, prisma17 }) => { + const suffix = `rrg3_${seq++}`; + cp.client = prisma14; + const seed = await seedTenant(prisma14 as unknown as PrismaClient, suffix); + const friendlyId = `run_${suffix}`; + const runId = `run_${CUID_25}`; + const createdAt = new Date("2024-02-02T00:00:00.000Z"); + const writer = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await writer.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED_SUCCESSFULLY", + createdAt, + }) + ); + resolved.authEnv = authEnvFor(seed); + resolved.lockedWorker = null; + + // Stale run row (immutable id/projectId/createdAt/friendlyId correct) AND the finished-attempt + // read misses on the replica → output undefined, but the run still renders 200. + const stale = laggingReplica(prisma14, [ + { + model: "taskRun", + mode: "frozen", + rows: [ + { + id: runId, + friendlyId, + status: "COMPLETED_SUCCESSFULLY", + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + createdAt, + queue: "task/my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + runTags: [], + baseCostInCents: 0, + costInCents: 0, + }, + ], + }, + { model: "taskRunAttempt", mode: "missing" }, + ]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + stale.client + ); + const res = (await inspectorLoader({ + request: new Request(`http://localhost/resources/runs/${friendlyId}`), + params: { runParam: friendlyId }, + context: {} as never, + })) as Response; + expect(stale.wasHit("taskRun")).toBe(true); + expect(stale.wasHit("taskRunAttempt")).toBe(true); // the finished-attempt read hit the replica + expect(res.status).toBe(200); + const body = await res.json(); + // typedjson envelope: payload is under .json in remix-typedjson serialisation + const data = body.json ?? body; + expect(data.friendlyId).toBe(friendlyId); + expect(data.isFinished).toBe(true); + // finished-attempt read missed on the replica → output absent (typedjson encodes the + // undefined as null over the wire), but the run itself rendered. + expect(data.output ?? null).toBeNull(); + } + ); + + // debug loader — admin queue-debug + heteroRunOpsPostgresTest( + "debug loader: frozen-STALE → 200 with the run + queue keys derived from immutable engine/queue/concurrencyKey", + async ({ prisma14, prisma17 }) => { + const suffix = `rrg4_${seq++}`; + cp.client = prisma14; + const seed = await seedTenant(prisma14 as unknown as PrismaClient, suffix); + const friendlyId = `run_${suffix}`; + const runId = `run_${CUID_25}`; + const writer = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await writer.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + concurrencyKey: "ck-site4", + }) + ); + resolved.authEnv = authEnvFor(seed); + + const stale = laggingReplica(prisma14, [ + { + model: "taskRun", + mode: "frozen", + rows: [ + { + id: runId, + friendlyId, + engine: "V2", + queue: "task/my-task", + concurrencyKey: "ck-site4", + queueTimestamp: null, + runtimeEnvironmentId: seed.environment.id, + projectId: seed.project.id, + }, + ], + }, + ]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + stale.client + ); + const res = (await debugLoader({ + request: new Request(`http://localhost/resources/taskruns/${friendlyId}/debug`), + params: { runParam: friendlyId }, + context: {} as never, + })) as Response; + expect(stale.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(200); + const body = await res.json(); + const data = body.json ?? body; + expect(data.engine).toBe("V2"); + expect(data.run.queue).toBe("task/my-task"); + expect(data.run.concurrencyKey).toBe("ck-site4"); + } + ); + + // log-detail loader — run-status annotation (branded $replica) + heteroRunOpsPostgresTest( + "log-detail loader: findRun($replica-branded) MISSING on the replica → 200 with runStatus undefined (optional annotation self-heals)", + async ({ prisma14, prisma17 }) => { + const suffix = `rrg5_${seq++}`; + cp.client = prisma14; + const seed = await seedTenant(prisma14 as unknown as PrismaClient, suffix); + const friendlyId = `run_${suffix}`; + const runId = `run_${CUID_25}`; + const writer = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await writer.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED_SUCCESSFULLY", + }) + ); + cpLookups.project = { id: seed.project.id, organizationId: seed.organization.id }; + cpLookups.environment = { id: seed.environment.id, type: "DEVELOPMENT", slug: "dev" }; + logDetail.result = { runId: friendlyId, message: "hello", someField: 1 }; + + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + missing.client + ); + + const logId = encodeURIComponent(`trace_x::span_x::${friendlyId}::2024-01-01T00:00:00.000Z`); + const res = (await logDetailLoader({ + request: new Request(`http://localhost/resources/orgs/o/projects/p/env/dev/logs/${logId}`), + params: { + organizationSlug: seed.organization.slug, + projectParam: seed.project.slug, + envParam: "dev", + logId, + }, + context: {} as never, + })) as Response; + expect(missing.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(200); + const body = await res.json(); + const data = body.json ?? body; + // The log detail is returned; the run-status annotation the lagging replica couldn't supply is + // simply undefined (the caller optional-chains `run?.status`). + expect(data.message).toBe("hello"); + expect(data.runStatus ?? null).toBeNull(); + } + ); + + // trace-download loader — trace-export download + heteroRunOpsPostgresTest( + "trace-download loader: frozen-STALE (completedAt null) → 200 gzip stream; immutable traceId/org/createdAt drive the export window, stale-null completedAt = open-ended (superset) not lossy", + async ({ prisma14, prisma17 }) => { + const suffix = `rrg6_${seq++}`; + cp.client = prisma14; + const seed = await seedTenant(prisma14 as unknown as PrismaClient, suffix); + const friendlyId = `run_${suffix}`; + const runId = `run_${CUID_25}`; + const traceId = "trace_dl_fixed"; + const createdAt = new Date("2024-03-03T00:00:00.000Z"); + const writer = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await writer.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED_SUCCESSFULLY", + traceId, + createdAt, + completedAt: new Date("2024-03-03T00:05:00.000Z"), + }) + ); + resolved.authEnv = authEnvFor(seed); + + const stale = laggingReplica(prisma14, [ + { + model: "taskRun", + mode: "frozen", + rows: [ + { + friendlyId, + traceId, + organizationId: seed.organization.id, + runtimeEnvironmentId: seed.environment.id, + createdAt, + completedAt: null, + taskEventStore: "taskEvent", + taskIdentifier: "my-task", + }, + ], + }, + ]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + stale.client + ); + const res = (await logsDownloadLoader({ + request: new Request(`http://localhost/resources/runs/${friendlyId}/logs/download`), + params: { runParam: friendlyId }, + context: {} as never, + })) as Response; + expect(stale.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(200); + expect(res.headers.get("Content-Encoding")).toBe("gzip"); + expect(res.headers.get("Content-Disposition")).toContain(`${friendlyId}.log`); + } + ); + + // replay loader — replay dialog + heteroRunOpsPostgresTest( + "replay loader: frozen-STALE run → 200 typedjson replay payload built from immutable seed fields (payload/tags/queue)", + async ({ prisma14, prisma17 }) => { + const suffix = `rrg8_${seq++}`; + cp.client = prisma14; + const seed = await seedTenant(prisma14 as unknown as PrismaClient, suffix); + const friendlyId = `run_${suffix}`; + const runId = `run_${CUID_25}`; + // Seed a project row the replay loader's loadProjectEnvironments ($replica.project.findFirst) finds + // with an environment matching runtimeEnvironmentId. + const writer = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await writer.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED_SUCCESSFULLY", + }) + ); + + const stale = laggingReplica(prisma14, [ + { + model: "taskRun", + mode: "frozen", + rows: [ + { + friendlyId, + payload: '{"hello":"world"}', + payloadType: "application/json", + seedMetadata: null, + seedMetadataType: null, + runtimeEnvironmentId: seed.environment.id, + projectId: seed.project.id, + concurrencyKey: null, + maxAttempts: null, + maxDurationInSeconds: null, + machinePreset: null, + workerQueue: null, + region: null, + ttl: null, + idempotencyKey: null, + runTags: [], + queue: "task/my-task", + taskIdentifier: "my-task", + }, + ], + }, + ]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + stale.client + ); + const res = (await replayLoader({ + request: new Request(`http://localhost/resources/taskruns/${friendlyId}/replay`), + params: { runParam: friendlyId }, + context: {} as never, + })) as Response; + expect(stale.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(200); + const body = await res.json(); + const data = body.json ?? body; + expect(data.queue).toBe("task/my-task"); + expect(data.runTags).toEqual([]); + } + ); + + // cancel action — resolveRunOrganizationId (dashboardAction) + // The action's `context` resolver reads the run by friendlyId (client-less → REPLICA) to resolve + // the org that scopes the RBAC check. Under a frozen-MISSING replica (+ drained buffer) the first + // read misses and the PRIMARY FALLBACK re-reads the owning primary to recover the org. This is + // load-bearing: authenticateAndAuthorize denies a scoped action when `ctx.organizationId` is absent + // (hasScope=false → fail-closed). Driving the REAL action under lag returns the 302 success + // redirect and reaches the cancel, reachable only if the fallback resolved the org. + heteroRunOpsPostgresTest( + "cancel action ctx resolver: frozen-MISSING replica → primary fallback resolves the org → action authorizes and cancels (not fail-closed 403)", + async ({ prisma14, prisma17 }) => { + const suffix = `rrg7_${seq++}`; + cp.client = prisma14; + const seed = await seedTenant(prisma14 as unknown as PrismaClient, suffix); + const friendlyId = `run_${suffix}`; + const runId = `run_${CUID_25}`; + const writer = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await writer.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + }) + ); + // The context resolver's resolveEnv (mocked control-plane) returns the org for the primary-found run. + resolved.env = { organizationId: seed.organization.id }; + cancelCalls.runs.length = 0; + + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + missing.client + ); + + const redirectUrl = `/orgs/${seed.organization.slug}/projects/${seed.project.slug}/runs`; + const res = (await cancelAction({ + request: new Request(`http://localhost/resources/taskruns/${friendlyId}/cancel`, { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ redirectUrl }).toString(), + }), + params: { runParam: friendlyId }, + context: {} as never, + })) as Response; + + expect(missing.wasHit("taskRun")).toBe(true); + // Authorized + cancelled — impossible unless the ctx resolver recovered the org via the primary + // fallback (a missing scope would have failed authorization closed with a redirect/403). + expect(res.status).toBe(302); + expect(res.headers.get("x-toast")).toBe("Canceled run"); + expect(cancelCalls.runs).toHaveLength(1); + expect(cancelCalls.runs[0].friendlyId).toBe(friendlyId); + } + ); + + // replay action — resolveRunOrganizationId (dashboardAction) + // Identical shape to the cancel action above: the ctx resolver's client-less findRun misses the + // frozen replica, the PRIMARY FALLBACK recovers the org, and the scoped RBAC check therefore + // authorizes. The REAL replay action under lag returns the 302 success redirect and reaches the + // replay service (only reachable with a resolved org scope). + heteroRunOpsPostgresTest( + "replay action ctx resolver: frozen-MISSING replica → primary fallback resolves the org → action authorizes and replays (not fail-closed 403)", + async ({ prisma14, prisma17 }) => { + const suffix = `rrg8b_${seq++}`; + cp.client = prisma14; + const seed = await seedTenant(prisma14 as unknown as PrismaClient, suffix); + const friendlyId = `run_${suffix}`; + const runId = `run_${CUID_25}`; + const writer = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await writer.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED_SUCCESSFULLY", + }) + ); + resolved.env = { organizationId: seed.organization.id }; + resolved.authEnv = authEnvFor(seed); + replayCalls.runs.length = 0; + + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + router.store = buildRouterWithFrozenLegacyReplica( + prisma14 as any, + prisma17 as any, + missing.client + ); + + const failedRedirect = `/orgs/${seed.organization.slug}/projects/${seed.project.slug}/runs/${friendlyId}`; + const res = (await replayAction({ + request: new Request(`http://localhost/resources/taskruns/${friendlyId}/replay`, { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ failedRedirect }).toString(), + }), + params: { runParam: friendlyId }, + context: {} as never, + })) as Response; + + expect(missing.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(302); + expect(res.headers.get("x-toast")).toBe("Replaying run"); + expect(replayCalls.runs).toHaveLength(1); + expect(replayCalls.runs[0].friendlyId).toBe(friendlyId); + } + ); +}); diff --git a/apps/webapp/test/runPresenters.replicaLag.test.ts b/apps/webapp/test/runPresenters.replicaLag.test.ts new file mode 100644 index 000000000..1eaaeb054 --- /dev/null +++ b/apps/webapp/test/runPresenters.replicaLag.test.ts @@ -0,0 +1,533 @@ +// Replica-lag proof for seven run-detail PRESENTER reads. Imports and drives the REAL exported presenter +// method / SSE loader that contains each read against a real Postgres (prisma14) whose run-store replica +// is frozen by the shared laggingReplica primitive (taskRun "missing"). Only orthogonal deps are stubbed +// (auth/session, presign, the mollifier buffer/read-fallback, the event repository); the run-store read +// path is the genuine article — a single PostgresRunStore wired as the webapp's single-DB runStore +// singleton (writer = prisma14, replica = frozen). Each case asserts the REAL caller's observable output +// under lag and separately confirms the run is live on the primary, so every miss is purely replica lag. +// +// Every read is a read-only dashboard/API GET whose stale/absent value is a documented fallback +// (buffer / typed-error / retryable-404) or a cosmetic omission that self-heals on the next poll once +// the replica catches up — none drives a mutation, a wrong terminal state, or a non-self-healing +// failure. Each case's per-test comment names the tolerating mechanism. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { PostgresRunStore } from "@internal/run-store"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 90_000, hookTimeout: 90_000 }); + +// ---- Holders wired per-test before each real-caller invocation. ----------------------------------- +// dbHolder.$replica -> the BRANDED frozen replica (taskRun reads miss; every other model forwards to +// the real container). dbHolder.prisma -> the real writer (prisma14). +// storeHolder.store -> the single PostgresRunStore the mocked `runStore` singleton forwards to. +// bufferHolder.result-> the mollifier read-fallback result (null = buffer miss, the realistic double-miss). +// bufferHolder.calls -> counts findRunByIdWithMollifierFallback invocations (proves the buffer fallback +// path actually ran for read 1, not merely that null happened). +// sessionHolder.userId -> the id requireUserId resolves to (read 4). +// eventRepoHolder.repo -> the event repository getEventRepositoryForStore returns (read 7). +const dbHolder = vi.hoisted(() => ({ prisma: null as any, $replica: null as any })); +const storeHolder = vi.hoisted(() => ({ store: undefined as any })); +const bufferHolder = vi.hoisted(() => ({ result: null as any, calls: 0 })); +const sessionHolder = vi.hoisted(() => ({ userId: "user_presenters_guard" })); +const eventRepoHolder = vi.hoisted(() => ({ repo: undefined as any })); + +// ~/db.server: redirect the module-level handles to the real test clients. NOT a DB mock — reads hit a +// real Postgres container; this only chooses which real client each module resolves. $replica is the +// frozen replica; the run-ops split handles are inert (single-DB webapp topology). +vi.mock("~/db.server", () => ({ + get prisma() { + return dbHolder.prisma; + }, + get $replica() { + return dbHolder.$replica; + }, + runOpsLegacyReplica: undefined, + runOpsNewReplica: undefined, + runOpsSplitReadEnabled: false, +})); + +// The ONE wiring boundary: inject the real single-DB store the presenters read through. A stable getter +// keeps the named import binding constant while returning the per-test store. +vi.mock("~/v3/runStore.server", () => ({ + get runStore() { + return storeHolder.store; + }, +})); + +// Presign is never exercised (payloads are inline JSON on the null path); inert stub keeps the import light. +vi.mock("~/v3/objectStore.server", () => ({ + generatePresignedUrl: vi.fn(async () => ({ success: false, error: "not-used" })), +})); + +// The mollifier read-fallback is a downstream service ORTHOGONAL to the run-store read under test. It +// returns bufferHolder.result (null = buffer miss) and counts calls so read 1 can prove the fallback ran. +vi.mock("~/v3/mollifier/readFallback.server", () => ({ + findRunByIdWithMollifierFallback: vi.fn(async () => { + bufferHolder.calls++; + return bufferHolder.result; + }), +})); + +// The mollifier buffer (read 4) — disabled so the stream loader's fallback is a clean miss and the only +// thing that could resolve a traceId is the run-store read (frozen -> null -> 404). +vi.mock("~/v3/mollifier/mollifierBuffer.server", () => ({ + getMollifierBuffer: () => null, +})); + +// Dashboard session auth (read 4) — resolves a fixed user id. Orthogonal to the run read. +vi.mock("~/services/session.server", () => ({ + requireUserId: vi.fn(async () => sessionHolder.userId), + getUserId: vi.fn(async () => sessionHolder.userId), + requireUser: vi.fn(async () => ({ id: sessionHolder.userId })), + getUser: vi.fn(async () => ({ id: sessionHolder.userId })), +})); + +// The event repository (read 7) — a downstream store, not the run read. getSpan returns eventRepoHolder.repo's +// span so #getSpan proceeds to the run-store findRuns({parentSpanId}) read under test. +vi.mock("~/v3/eventRepository/index.server", () => ({ + getEventRepositoryForStore: vi.fn(async () => eventRepoHolder.repo), +})); + +// The REAL exported callers under proof. +import { ApiRetrieveRunPresenter } from "~/presenters/v3/ApiRetrieveRunPresenter.server"; +import { ApiRunResultPresenter } from "~/presenters/v3/ApiRunResultPresenter.server"; +import { RunPresenter, RunNotInPgError } from "~/presenters/v3/RunPresenter.server"; +import { RunStreamPresenter } from "~/presenters/v3/RunStreamPresenter.server"; +import { PlaygroundPresenter } from "~/presenters/v3/PlaygroundPresenter.server"; +import { SpanPresenter } from "~/presenters/v3/SpanPresenter.server"; + +let seq = 0; + +type Seed = { + organizationId: string; + projectId: string; + projectSlug: string; + environmentId: string; + environmentSlug: string; +}; + +async function seedTenant(prisma: PrismaClient, suffix: string): Promise { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + await prisma.user.create({ + data: { + id: sessionHolder.userId, + email: `guard-${suffix}@example.com`, + authenticationMethod: "MAGIC_LINK", + }, + }); + await prisma.orgMember.create({ + data: { userId: sessionHolder.userId, organizationId: organization.id, role: "ADMIN" }, + }); + return { + organizationId: organization.id, + projectId: project.id, + projectSlug: project.slug, + environmentId: environment.id, + environmentSlug: environment.slug, + }; +} + +async function seedRun( + prisma: PrismaClient, + seed: Seed, + opts: { + id: string; + friendlyId: string; + status?: string; + spanId?: string; + parentSpanId?: string; + } +) { + await prisma.taskRun.create({ + data: { + id: opts.id, + engine: "V2", + status: (opts.status ?? "EXECUTING") as never, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: seed.environmentId, + environmentType: "DEVELOPMENT", + organizationId: seed.organizationId, + projectId: seed.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: opts.spanId ?? `span_${opts.id}`, + ...(opts.parentSpanId !== undefined ? { parentSpanId: opts.parentSpanId } : {}), + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }, + }); +} + +// Wire the per-test holders: the branded frozen replica over `prisma`, and the single PostgresRunStore +// (writer = prisma, replica = frozen) the mocked runStore singleton forwards to. Returns the frozen +// handle so a test can assert wasHit("taskRun"). +function wireFrozenStore(prisma: PrismaClient) { + const frozen = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + const store = new PostgresRunStore({ prisma, readOnlyPrisma: frozen.client as never }); + dbHolder.prisma = prisma; + dbHolder.$replica = frozen.client; + storeHolder.store = store; + bufferHolder.result = null; + bufferHolder.calls = 0; + return frozen; +} + +describe("run-detail presenter reads against a frozen run-store replica", () => { + // ApiRetrieveRunPresenter findRun (+$replica) — public GET /api/v3/runs/:runId. Drive the REAL static + // findRun. Under lag the replica misses and the buffer fallback (findRunByIdWithMollifierFallback) + // fires (proven by call count), buffer misses too, so the caller returns null. The retrieve route maps + // null to a 404 carrying `x-should-retry: true`, so the SDK retrieve/poll loop re-fetches and the next + // poll (replica caught up) succeeds. Self-healing; read-only. + heteroPostgresTest( + "ApiRetrieveRunPresenter.findRun returns null for a live run absent on the replica (retryable-404 contract)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `retrieve_${seq++}`; + const seed = await seedTenant(prisma, suffix); + const runId = `run_id_${suffix}`; + const friendlyId = `run_${suffix}`; + await seedRun(prisma, seed, { id: runId, friendlyId, status: "COMPLETED_SUCCESSFULLY" }); + + const frozen = wireFrozenStore(prisma); + + const env = { id: seed.environmentId, organizationId: seed.organizationId } as any; + const result = await ApiRetrieveRunPresenter.findRun(friendlyId, env); + + // The frozen replica WAS consulted (the read-your-writes window is really exercised)... + expect(frozen.wasHit("taskRun")).toBe(true); + // ...the presenter's documented buffer fallback FIRED (not merely that null happened)... + expect(bufferHolder.calls).toBe(1); + // ...and the observable caller output is null (retrieve route -> retryable 404). + expect(result).toBeNull(); + + // Primary contrast: the run is genuinely live — the miss is purely replica lag. + const onPrimary = await prisma.taskRun.findFirst({ + where: { friendlyId }, + select: { id: true }, + }); + expect(onPrimary?.id).toBe(runId); + } + ); + + // ApiRunResultPresenter findRun (no client) — GET /api/v1/runs/:runId/result poll (SDK waitForRun + // result). Drive the REAL call(); the store is injected via the presenter's own constructor seam (4th + // arg). Under lag the replica misses -> null -> the presenter returns undefined, the poll's "not + // finished yet" signal — the SDK result-poll loop retries and succeeds once the replica catches up. + // Read-only. + heteroPostgresTest( + "ApiRunResultPresenter.call returns undefined for a live run absent on the replica (poll retries)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `result_${seq++}`; + const seed = await seedTenant(prisma, suffix); + const runId = `run_id_${suffix}`; + const friendlyId = `run_${suffix}`; + await seedRun(prisma, seed, { id: runId, friendlyId, status: "COMPLETED_SUCCESSFULLY" }); + + const frozen = wireFrozenStore(prisma); + + // Construct exactly as the route does, but inject the real frozen store through the ctor seam. + const presenter = new ApiRunResultPresenter( + prisma, + frozen.client as any, + undefined, + storeHolder.store + ); + // call() wraps in traceWithEnv, which reads env slug/org/project attributes. + const env = { + id: seed.environmentId, + type: "DEVELOPMENT", + slug: seed.environmentSlug, + organizationId: seed.organizationId, + organization: { id: seed.organizationId, slug: seed.projectSlug, title: "Org" }, + projectId: seed.projectId, + project: { id: seed.projectId, name: "Project" }, + } as any; + const result = await presenter.call(friendlyId, env); + + expect(frozen.wasHit("taskRun")).toBe(true); + expect(result).toBeUndefined(); + + const onPrimary = await prisma.taskRun.findFirst({ + where: { friendlyId }, + select: { id: true }, + }); + expect(onPrimary?.id).toBe(runId); + } + ); + + // RunPresenter findRun (no client) — dashboard run-detail page loader. Drive the REAL call(). Under + // lag the replica misses -> null -> the presenter throws the typed RunNotInPgError, the route's signal + // to fall back to the synthesised mollifier-buffer view (and off the noisy Prisma error path); the + // page self-heals on the next poll. Read-only. + heteroPostgresTest( + "RunPresenter.call throws RunNotInPgError for a live run absent on the replica (route buffer view)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `detail_${seq++}`; + const seed = await seedTenant(prisma, suffix); + const runId = `run_id_${suffix}`; + const friendlyId = `run_${suffix}`; + await seedRun(prisma, seed, { id: runId, friendlyId }); + + const frozen = wireFrozenStore(prisma); + + const presenter = new RunPresenter(prisma); + const call = presenter.call({ + userId: sessionHolder.userId, + projectSlug: seed.projectSlug, + environmentSlug: seed.environmentSlug, + runFriendlyId: friendlyId, + showDeletedLogs: false, + showDebug: false, + }); + + await expect(call).rejects.toBeInstanceOf(RunNotInPgError); + expect(frozen.wasHit("taskRun")).toBe(true); + + const onPrimary = await prisma.taskRun.findFirst({ + where: { friendlyId }, + select: { id: true }, + }); + expect(onPrimary?.id).toBe(runId); + } + ); + + // RunStreamPresenter findRun (no client) — run-detail SSE trace-stream loader. Drive the REAL loader + // from createLoader(). Under lag the replica misses -> run null -> not authorized -> traceId null -> + // buffer disabled -> the handler throws Response(404), which createSSELoader rethrows. 404 is the + // SSE-reconnect contract: the dashboard reconnects and the next connection (replica caught up) + // attaches. + heteroPostgresTest( + "RunStreamPresenter loader throws Response 404 for a live run absent on the replica (SSE reconnect)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `stream_${seq++}`; + const seed = await seedTenant(prisma, suffix); + const runId = `run_id_${suffix}`; + const friendlyId = `run_${suffix}`; + await seedRun(prisma, seed, { id: runId, friendlyId }); + + const frozen = wireFrozenStore(prisma); + + const loader = new RunStreamPresenter(prisma).createLoader(); + const request = new Request(`http://localhost/resources/runs/${friendlyId}/stream`); + + let thrown: unknown; + try { + await loader({ request, params: { runParam: friendlyId } } as any); + } catch (e) { + thrown = e; + } + + expect(thrown).toBeInstanceOf(Response); + expect((thrown as Response).status).toBe(404); + expect(frozen.wasHit("taskRun")).toBe(true); + + const onPrimary = await prisma.taskRun.findFirst({ + where: { friendlyId }, + select: { id: true }, + }); + expect(onPrimary?.id).toBe(runId); + } + ); + + // PlaygroundPresenter findRuns (no client) — agent-playground conversation list. Drive the REAL + // getRecentConversations. The conversation row is read off $replica (playgroundConversation not + // frozen), then each backing run's scalars via a client-less findRuns over the id set (frozen -> []). + // The missing run is absent from runsById, so the conversation renders with runFriendlyId=null / + // runStatus=null / isActive=false — a cosmetic "status unknown" on one row that self-heals next load. + // The row still renders. + heteroPostgresTest( + "PlaygroundPresenter.getRecentConversations renders the row with null run fields when the run is absent on the replica", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `playground_${seq++}`; + const seed = await seedTenant(prisma, suffix); + const runId = `run_id_${suffix}`; + const friendlyId = `run_${suffix}`; + await seedRun(prisma, seed, { id: runId, friendlyId, status: "EXECUTING" }); + + await prisma.playgroundConversation.create({ + data: { + chatId: `chat_${suffix}`, + title: "Conversation", + agentSlug: "my-agent", + runId, + projectId: seed.projectId, + runtimeEnvironmentId: seed.environmentId, + userId: sessionHolder.userId, + }, + }); + + const frozen = wireFrozenStore(prisma); + + const presenter = new PlaygroundPresenter(); + const conversations = await presenter.getRecentConversations({ + environmentId: seed.environmentId, + agentSlug: "my-agent", + userId: sessionHolder.userId, + }); + + expect(frozen.wasHit("taskRun")).toBe(true); + // The conversation row IS returned (not dropped)... + expect(conversations).toHaveLength(1); + // ...but its backing-run fields self-heal to null because the run missed on the replica. + expect(conversations[0].chatId).toBe(`chat_${suffix}`); + expect(conversations[0].runFriendlyId).toBeNull(); + expect(conversations[0].runStatus).toBeNull(); + expect(conversations[0].isActive).toBe(false); + + const onPrimary = await prisma.taskRun.findFirst({ + where: { id: runId }, + select: { id: true }, + }); + expect(onPrimary?.id).toBe(runId); + } + ); + + // SpanPresenter findRun (+this._replica) — span-detail (run inspector) panel. Drive the REAL public + // findRun with the originalRunId branch; it passes the branded this._replica. Under lag the replica + // misses -> null. The span-detail panel renders its not-found/loading state for this poll and + // self-heals next tick. Read-only. + heteroPostgresTest( + "SpanPresenter.findRun returns null for a live run absent on the replica (span-detail not-found this poll)", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `span_${seq++}`; + const seed = await seedTenant(prisma, suffix); + const runId = `run_id_${suffix}`; + const friendlyId = `run_${suffix}`; + await seedRun(prisma, seed, { id: runId, friendlyId }); + + const frozen = wireFrozenStore(prisma); + + // _prisma = writer, _replica = branded frozen (exactly how the webapp constructs it). + const presenter = new SpanPresenter(prisma, frozen.client as any); + const run = await presenter.findRun({ + originalRunId: friendlyId, + spanId: `span_${runId}`, + environmentId: seed.environmentId, + }); + + expect(frozen.wasHit("taskRun")).toBe(true); + expect(run).toBeNull(); + + const onPrimary = await prisma.taskRun.findFirst({ + where: { friendlyId }, + select: { id: true }, + }); + expect(onPrimary?.id).toBe(runId); + } + ); + + // SpanPresenter findRuns {parentSpanId} (+this._replica) — the "triggered runs" list on the + // span-detail panel. Drive the REAL call() end-to-end: it resolves the parent run on the primary, + // gets the span from the (stubbed) event repository, then reads the child runs via + // runStore.findRuns({parentSpanId}, this._replica) — the frozen replica. Under lag the just-triggered + // child is invisible -> span.triggeredRuns === [] — a cosmetic "one fewer child shown" that self-heals + // next poll; the child still exists and executes. Read-only. + heteroPostgresTest( + "SpanPresenter.call yields empty triggeredRuns when the child run is absent on the replica", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `triggered_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const parentRunId = `run_parent_${suffix}`; + const parentFriendlyId = `run_${suffix}`; + const spanId = `span_detail_${suffix}`; + await seedRun(prisma, seed, { + id: parentRunId, + friendlyId: parentFriendlyId, + spanId, + }); + + // The child run whose parentSpanId is the inspected span — the one that should appear in the + // triggered-runs list but is missing on the frozen replica. + const childRunId = `run_child_${suffix}`; + const childFriendlyId = `run_child_fr_${suffix}`; + await seedRun(prisma, seed, { + id: childRunId, + friendlyId: childFriendlyId, + spanId: `span_child_${suffix}`, + parentSpanId: spanId, + }); + + const frozen = wireFrozenStore(prisma); + + // Stub the event repository so getSpan yields a minimal "attempt" span; this lets #getSpan proceed + // to the run-store findRuns({parentSpanId}) read under test without ClickHouse. + eventRepoHolder.repo = { + getSpan: async () => ({ + spanId, + parentId: undefined, + message: "attempt", + isError: false, + isPartial: false, + isCancelled: false, + level: "TRACE", + startTime: new Date(), + duration: 0, + events: [], + style: {}, + properties: {}, + resourceProperties: {}, + entity: { type: "attempt", id: undefined }, + metadata: {}, + }), + // Not reached on this path (getRun returns undefined before a trace summary is needed). + getTraceSummary: async () => null, + }; + + const presenter = new SpanPresenter(prisma, frozen.client as any); + const result = await presenter.call({ + userId: sessionHolder.userId, + projectSlug: seed.projectSlug, + envSlug: seed.environmentSlug, + spanId, + runFriendlyId: parentFriendlyId, + }); + + expect(frozen.wasHit("taskRun")).toBe(true); + expect(result?.type).toBe("span"); + // The child triggered by this span is omitted from the list under replica lag. + expect((result as any).span.triggeredRuns).toEqual([]); + + // Primary contrast: the child genuinely exists with this parentSpanId — the omission is pure lag. + const childOnPrimary = await prisma.taskRun.findFirst({ + where: { parentSpanId: spanId }, + select: { friendlyId: true }, + }); + expect(childOnPrimary?.friendlyId).toBe(childFriendlyId); + } + ); +}); diff --git a/apps/webapp/test/runsRepositoryConvert.replicaLag.test.ts b/apps/webapp/test/runsRepositoryConvert.replicaLag.test.ts new file mode 100644 index 000000000..7b5187c26 --- /dev/null +++ b/apps/webapp/test/runsRepositoryConvert.replicaLag.test.ts @@ -0,0 +1,152 @@ +// Property: convertRunListInputOptionsToFilterRunsOptions tolerates a batch friendlyId → id resolution +// miss under replica lag. Drives the REAL exported function (not a reimplementation) with a store whose +// replica is FROZEN via the shared `laggingReplica`, so the batch lookup misses the just-written row +// while the owning primary still holds it. +// +// The read (store.findBatchTaskRunByFriendlyId, no client → REPLICA) resolves a `batch_` friendlyId to +// an internal id for a ClickHouse list FILTER, behind an `if (batch)` guard. Under lag it returns null, +// the guard is skipped, and the returned FilterRunsOptions keeps `batchId` as the UNRESOLVED friendlyId +// (matches no ClickHouse batch_id this load; the next revalidation resolves it once replicated). The +// function does NOT throw and mutates nothing; the batch is live on the primary (internal id differs), +// proving the miss is pure lag. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import { PostgresRunStore } from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// Prevent the run-store / db singletons from constructing at import; the function under test is driven +// with an explicitly-injected store + prisma, so these module defaults are never used at runtime. +vi.mock("~/db.server", () => ({})); +vi.mock("~/v3/runStore.server", () => ({ runStore: {} })); + +import { convertRunListInputOptionsToFilterRunsOptions } from "~/services/runsRepository/runsRepository.server"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +describe("convertRunListInputOptionsToFilterRunsOptions under replica lag", () => { + heteroPostgresTest( + "leaves a not-yet-replicated batch friendlyId unresolved in the filter without throwing", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `convert_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + // Seed the batch on the PRIMARY only. Internal id differs from the friendlyId. + const batchId = `batchinternal_${suffix}`; + const batchFriendlyId = `batch_${suffix}`; + await prisma.batchTaskRun.create({ + data: { + id: batchId, + friendlyId: batchFriendlyId, + runtimeEnvironmentId: seed.environment.id, + status: "PENDING", + batchVersion: "v3", + }, + }); + + // batchTaskRun frozen-missing on the replica; the client-less read routes there. + const replica = laggingReplica(prisma, [{ model: "batchTaskRun", mode: "missing" }]); + const store = new PostgresRunStore({ + prisma, + readOnlyPrisma: replica.client as never, + schemaVariant: "legacy", + }); + + const result = await convertRunListInputOptionsToFilterRunsOptions( + { + organizationId: seed.organization.id, + projectId: seed.project.id, + environmentId: seed.environment.id, + batchId: batchFriendlyId, + }, + prisma as never, + store + ); + + // The batch resolution really hit the (lagging) replica. + expect(replica.wasHit("batchTaskRun")).toBe(true); + + // OBSERVABLE OUTPUT: the friendlyId is left UNRESOLVED (not swapped to the internal id) — the + // ClickHouse batch_id filter matches nothing this load; the function did not throw. + expect(result.batchId).toBe(batchFriendlyId); + expect(result.batchId).not.toBe(batchId); + + // The miss is pure lag: the batch is live on the PRIMARY (and would resolve on revalidation). + const onPrimary = await prisma.batchTaskRun.findFirst({ + where: { friendlyId: batchFriendlyId }, + }); + expect(onPrimary?.id).toBe(batchId); + } + ); + + // Control: with a live replica the SAME function resolves the friendlyId to the internal id — proving + // the unresolved result above is caused solely by the replica lag, not by the code path being dead. + heteroPostgresTest( + "control: resolves the batch friendlyId to the internal id with a caught-up replica", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `convert_ok_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const batchId = `batchinternal_${suffix}`; + const batchFriendlyId = `batch_${suffix}`; + await prisma.batchTaskRun.create({ + data: { + id: batchId, + friendlyId: batchFriendlyId, + runtimeEnvironmentId: seed.environment.id, + status: "PENDING", + batchVersion: "v3", + }, + }); + + const store = new PostgresRunStore({ + prisma, + readOnlyPrisma: prisma, + schemaVariant: "legacy", + }); + + const result = await convertRunListInputOptionsToFilterRunsOptions( + { + organizationId: seed.organization.id, + projectId: seed.project.id, + environmentId: seed.environment.id, + batchId: batchFriendlyId, + }, + prisma as never, + store + ); + + expect(result.batchId).toBe(batchId); + } + ); +}); diff --git a/apps/webapp/test/sessionWaitpointRoutes.replicaLag.guard.test.ts b/apps/webapp/test/sessionWaitpointRoutes.replicaLag.guard.test.ts new file mode 100644 index 000000000..9a622b6b5 --- /dev/null +++ b/apps/webapp/test/sessionWaitpointRoutes.replicaLag.guard.test.ts @@ -0,0 +1,351 @@ +// Replica-lag properties for the route-session-waitpoint reads. Each case drives the REAL route handler +// createActionApiRoute wraps + exports as `action` (captured from the builder, not reimplemented) +// against a real Postgres (heteroPostgresTest) whose read replica is FROZEN via the shared +// `laggingReplica` primitive; only orthogonal deps are mocked (auth/session resolution, route-builder +// middleware, downstream swap/engine services, logging). +// +// end-and-continue calling-run resolve: a live calling run absent on the replica is recovered via the +// owning-primary re-read (findRunOnPrimary), so the handoff swap is reached rather than a 404. +// waitpoint-token complete lookup: a client-less findWaitpoint misses the just-minted token on the +// replica, yet the owning-primary re-read (findWaitpointOnPrimary) resolves it and the waitpoint +// completes (200), reaching engine.completeWaitpoint. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect, vi } from "vitest"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// ---- Hoisted holders wired per-test before the captured handler runs. --------------------------- +const H = vi.hoisted(() => ({ + store: undefined as any, // the REAL PostgresRunStore for the current case (mocked runStore forwards here) + primary: undefined as any, // db.server `prisma` -> the real container (owning primary / writer) + replica: undefined as any, // db.server `$replica` -> the lagging replica over the same container + swap: { calls: [] as any[], result: undefined as any }, // swapSessionRun recorder + canned result + engine: { calls: [] as any[] }, // engine.completeWaitpoint recorder + handlers: [] as Array<{ config: any; handler: any }>, // captured inner handlers, one per route import +})); + +// ~/db.server: lazy proxies that always resolve through the holder so the run-store singleton's +// memoized delegates route to the current test's client. Never mocks the DB itself. +vi.mock("~/db.server", () => { + const lazyProxy = (key: "primary" | "replica", label: string) => + new Proxy( + {}, + { + get(_t, prop) { + const client = H[key]; + if (!client) throw new Error(`${label} not set for this test`); + const value = client[prop]; + if (value !== null && typeof value === "object") { + return new Proxy(value, { get: (_d, method) => H[key][prop][method] }); + } + return value; + }, + } + ); + return { + prisma: lazyProxy("primary", "H.primary"), + $replica: lazyProxy("replica", "H.replica"), + runOpsNewPrismaClient: undefined, + runOpsNewReplicaClient: undefined, + runOpsLegacyPrisma: undefined, + runOpsLegacyReplica: undefined, + }; +}); + +// The REAL split router / PostgresRunStore is injected per-test into H.store; the mocked singleton is +// a stable Proxy forwarding every method to it. This is what the routes read/write through. +vi.mock("~/v3/runStore.server", () => ({ + runStore: new Proxy( + {}, + { + get(_t, prop) { + const store = H.store; + if (!store) throw new Error("test bug: H.store not initialised before handler ran"); + const value = store[prop]; + return typeof value === "function" ? value.bind(store) : value; + }, + } + ), +})); + +// Route builder: capture the inner handler each route hands to createActionApiRoute so we can drive +// the REAL caller directly, bypassing only the auth/body middleware (orthogonal). +vi.mock("~/services/routeBuilders/apiBuilder.server", () => ({ + anyResource: (x: unknown) => x, + createActionApiRoute: (config: any, handler: any) => { + H.handlers.push({ config, handler }); + return { action: vi.fn(), loader: vi.fn() }; + }, +})); + +// Session resolution lives in `findResource` (which we bypass by passing `resource` directly); stub so +// the heavy realtime import graph never evaluates. +vi.mock("~/services/realtime/sessions.server", () => ({ + resolveSessionByIdOrExternalId: vi.fn(async () => null), +})); + +// Downstream swap is engine/realtime work, not the read under test: record the call + return a canned +// result so the success branch (and its read-after-write findRun on the primary) is exercised. +vi.mock("~/services/realtime/sessionRunManager.server", () => ({ + swapSessionRun: vi.fn(async (params: any) => { + H.swap.calls.push(params); + return H.swap.result; + }), +})); + +// Waitpoint completion downstream: orthogonal. Record the engine call; canned packet. +vi.mock("~/v3/runEngine.server", () => ({ + engine: { + completeWaitpoint: vi.fn(async (args: any) => { + H.engine.calls.push(args); + return { id: args.id }; + }), + }, +})); +vi.mock("~/runEngine/concerns/waitpointCompletionPacket.server", () => ({ + processWaitpointCompletionPacket: vi.fn(async () => ({ + data: "OK", + dataType: "application/json", + })), +})); + +vi.mock("~/env.server", () => ({ + env: { TASK_PAYLOAD_MAXIMUM_SIZE: 3 * 1024 * 1024 }, +})); +vi.mock("~/services/logger.server", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); + +import { PostgresRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; + +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(p: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: p.runId, + engine: "V2", + status: "PENDING", + friendlyId: p.friendlyId, + runtimeEnvironmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: p.organizationId, + projectId: p.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: p.projectId, + organizationId: p.organizationId, + }, + }; +} + +// Import the target route once and return the inner handler the builder captured for it. Matching on +// the zod params shape keeps this robust regardless of import order. +async function loadHandler(modulePath: string, paramKey: string) { + await import(modulePath); + const entry = H.handlers.find((h) => { + try { + return Boolean(h.config?.params?.shape?.[paramKey]); + } catch { + return false; + } + }); + if (!entry) throw new Error(`handler for ${modulePath} (param ${paramKey}) not captured`); + return entry.handler as (args: any) => Promise; +} + +describe("route-session-waitpoint reads under replica lag", () => { + // end-and-continue calling-run resolve + heteroPostgresTest( + "end-and-continue: a live calling run absent on the replica is recovered via the primary re-read", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `eac_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + // Seed the calling run on the PRIMARY only; the lagging replica will not see it. + const runId = `run_${"a".repeat(21)}${seq}`; + const friendlyId = `run_call_${suffix}`; + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + + // The run-store the mocked singleton forwards to: writer + readReplica both the real container, + // so findRunOnPrimary (this.prisma) hits and the read-after-write findRun(prisma) hits. + H.store = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + H.primary = prisma; // db.server `prisma` + H.replica = replica.client; // db.server `$replica` -> lagging (the client the route passes to findRun) + H.swap = { calls: [], result: { runId, swapped: true } }; + + const handler = await loadHandler( + "~/routes/api.v1.sessions.$session.end-and-continue", + "session" + ); + + const session = { + id: `session_${suffix}`, + friendlyId: `session_${suffix}`, + externalId: null, + closedAt: null, + expiresAt: null, + }; + + const res = await handler({ + authentication: { environment: seed.environment }, + params: { session: session.friendlyId }, + body: { callingRunId: friendlyId, reason: "upgrade" }, + resource: session, + }); + + // The replica WAS consulted (frozen -> missed): the read genuinely went through the lagging + // replica, so recovery is the primary re-read, not a lucky replica hit. + expect(replica.wasHit("taskRun")).toBe(true); + + // The primary re-read resolved the calling run, so the swap was reached with its cuid... + expect(H.swap.calls).toHaveLength(1); + expect(H.swap.calls[0].callingRunId).toBe(runId); + + // ...and the handler returned 200 with the handoff body, not a 404. + expect(res.status).toBe(200); + const body = (await res.clone().json()) as { + runId?: string; + swapped?: boolean; + error?: string; + }; + expect(body.swapped).toBe(true); + expect(body.runId).toBe(friendlyId); + expect(body.error).toBeUndefined(); + } + ); + + // waitpoint-token complete lookup + heteroPostgresTest( + "waitpoint-token complete: a token absent on the replica is resolved via the primary re-read and completes", + async ({ prisma14 }) => { + const prisma = prisma14 as unknown as PrismaClient; + const suffix = `wctok_${seq++}`; + const seed = await seedTenant(prisma, suffix); + + const waitpointFriendlyId = `waitpoint_${"b".repeat(21)}${seq}`; + const waitpointId = WaitpointId.toId(waitpointFriendlyId); + + // Seed a still-PENDING MANUAL token waitpoint on the PRIMARY only. + const writerStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + await writerStore.upsertWaitpoint({ + where: { + environmentId_idempotencyKey: { + environmentId: seed.environment.id, + idempotencyKey: waitpointId, + }, + }, + create: { + id: waitpointId, + friendlyId: waitpointFriendlyId, + type: "MANUAL", + status: "PENDING", + idempotencyKey: waitpointId, + userProvidedIdempotencyKey: false, + projectId: seed.project.id, + environmentId: seed.environment.id, + }, + update: {}, + }); + + // findWaitpoint is client-less in the route -> owning REPLICA (readOnlyPrisma). Freeze it. + const replica = laggingReplica(prisma, [{ model: "waitpoint", mode: "missing" }]); + H.store = new PostgresRunStore({ prisma, readOnlyPrisma: replica.client }); + H.primary = prisma; + H.replica = prisma; // not used by this route, but a valid client + H.engine = { calls: [] }; + + const handler = await loadHandler( + "~/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.complete", + "waitpointFriendlyId" + ); + + const res = await handler({ + authentication: { environment: seed.environment }, + params: { waitpointFriendlyId }, + body: { data: { ok: true } }, + }); + + // The replica WAS consulted (frozen -> missed) on the client-less findWaitpoint... + expect(replica.wasHit("waitpoint")).toBe(true); + + // ...yet the existing findWaitpointOnPrimary fallback resolved the token, so the real handler + // completed the waitpoint (200 success) rather than 404ing it. + expect(res.status).toBe(200); + const body = (await res.clone().json()) as { success?: boolean; error?: string }; + expect(body.success).toBe(true); + expect(body.error).toBeUndefined(); + + // The completion actually reached the engine with the resolved waitpoint id. + expect(H.engine.calls).toHaveLength(1); + expect(H.engine.calls[0].id).toBe(waitpointId); + } + ); +}); diff --git a/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts b/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts new file mode 100644 index 000000000..f3df2d2ac --- /dev/null +++ b/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts @@ -0,0 +1,545 @@ +// Replica-lag properties for the run-trace / span-detail reads, each driven through its REAL exported +// route caller (never a reimplemented read) against a real split Postgres with the owning LEGACY replica +// FROZEN via laggingReplica. The routes pass a branded $replica, so reads stay on the owning replica. +// Only orthogonal webapp singletons are mocked (bearer auth, mollifier buffer, ClickHouse repository, +// formatters, session cookie, control-plane resolver, longPollingFetch). +// +// Properties per read: the spans/trace findResource reads emit a RETRYABLE 404 (x-should-retry:true) on a +// replica+buffer miss and return 200 once the replica catches up (proven with a caught-up store); the +// triggeredRuns list simply omits a just-triggered child under lag (200, eventually consistent); the sync +// trace-runs loader recovers a live run via a primary re-read on a replica miss and still 404s a truly-absent run. + +import { describe, expect, vi } from "vitest"; +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { PostgresRunStore, RoutingRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +// ---- Hoisted holders wired into the mocked module singletons before each loader call. ------------- +// The branded `$replica` marker uses the global-registry symbol the run-store brands replicas with +// (readReplicaClient.ts) so the routing store keeps the read on the owning REPLICA. It also carries a +// live `orgMember` delegate (pointed at the real prisma14) because the sync route reads +// `$replica.orgMember.findFirst` directly — orthogonal to the run read under test. +const { holder } = vi.hoisted(() => { + const REPLICA_BRAND = Symbol.for("trigger.dev/run-store/read-replica"); + return { + holder: { + REPLICA_BRAND, + store: undefined as unknown, + replicaMarker: undefined as unknown, + environment: undefined as unknown, + bufferResult: null as unknown, + userId: undefined as unknown, + resolvedEnv: undefined as unknown, + span: undefined as unknown, + traceSummary: undefined as unknown, + }, + }; +}); + +// Run-store singleton: a stable Proxy forwarding every method to the per-test RoutingRunStore. +vi.mock("~/v3/runStore.server", () => ({ + runStore: new Proxy( + {}, + { + get(_t, prop) { + const store = holder.store as Record; + if (!store) throw new Error("test bug: holder.store not initialised before loader ran"); + const value = store[prop]; + return typeof value === "function" + ? (value as (...a: unknown[]) => unknown).bind(store) + : value; + }, + } + ), +})); + +// `$replica` brand marker (routes it to the owning replica) + a live orgMember delegate for the sync route. +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: new Proxy( + {}, + { + get(_t, prop) { + if (prop === holder.REPLICA_BRAND) return true; + const marker = holder.replicaMarker as Record | undefined; + return marker ? marker[prop] : undefined; + }, + } + ), +})); + +// Bearer auth + ability (orthogonal): resolve to the seeded environment and grant every check. +vi.mock("~/services/rbac.server", () => ({ + rbac: { + authenticateBearer: async () => ({ + ok: true, + environment: holder.environment, + subject: { type: "privateKey" }, + jwt: undefined, + ability: { can: () => true, canSuper: () => true }, + }), + }, +})); + +// Buffer fallback (mollifier): a clean MISS so the only run source is the run-store read path. +vi.mock("~/v3/mollifier/readFallback.server", () => ({ + findRunByIdWithMollifierFallback: vi.fn(async () => holder.bufferResult), +})); + +// ClickHouse event repository (downstream, orthogonal): return synthetic span / trace shapes. +vi.mock("~/v3/eventRepository/index.server", () => ({ + getEventRepositoryForStore: async () => ({ + getSpan: async () => holder.span, + getTraceDetailedSubtreeSummary: async () => holder.traceSummary, + }), +})); +vi.mock("~/v3/taskEventStore.server", () => ({ + getTaskEventStoreTableForRun: () => "taskEvent", +})); +vi.mock("~/components/runs/v3/ai", () => ({ + extractAISpanData: () => undefined, +})); +vi.mock("~/v3/mollifier/syntheticApiResponses.server", () => ({ + buildSyntheticSpanDetailBody: (r: unknown) => ({ synthetic: true, run: r }), + buildSyntheticTraceBody: (r: unknown) => ({ synthetic: true, run: r }), +})); + +// Sync-route peripherals (orthogonal to the run read under test). +vi.mock("~/services/session.server", () => ({ + getUserId: async () => holder.userId, +})); +vi.mock("~/v3/runOpsMigration/controlPlaneResolver.server", () => ({ + controlPlaneResolver: { resolveEnv: async () => holder.resolvedEnv }, +})); +vi.mock("~/utils/longPollingFetch", () => ({ + longPollingFetch: async () => + new Response("shape-stream-ok", { status: 200, headers: { "x-longpoll": "hit" } }), +})); +// Ensure ELECTRIC_ORIGIN is defined for the sync route's happy path; keep every other env value real +// so the heavy apiBuilder import graph still loads. +vi.mock("~/env.server", async (importOriginal) => { + const actual = (await importOriginal()) as { env: Record }; + return { env: { ...actual.env, ELECTRIC_ORIGIN: "http://electric.test" } }; +}); + +import { loader as spansLoader } from "~/routes/api.v1.runs.$runId.spans.$spanId"; +import { loader as traceLoader } from "~/routes/api.v1.runs.$runId.trace"; +import { loader as syncTraceRunsLoader } from "~/routes/sync.traces.runs.$traceId"; + +// A cuid (25 chars after `run_`) classifies LEGACY, so both the create and the friendlyId/traceId +// reads route to the legacy (control-plane) store — the store that owns these runs. +const CUID_25 = "c".repeat(25); +let seq = 0; + +async function seedTenant(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +// The AuthenticatedEnvironment shape the real wrapper + handlers read. +function authEnvironment(seed: Awaited>) { + return { + id: seed.environment.id, + apiKey: seed.environment.apiKey, + type: "DEVELOPMENT", + slug: "dev", + organizationId: seed.organization.id, + organization: { id: seed.organization.id, slug: seed.organization.slug }, + project: { + id: seed.project.id, + slug: seed.project.slug, + externalRef: seed.project.externalRef, + }, + }; +} + +function buildCreateRunInput(p: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + traceId: string; + spanId: string; + parentSpanId?: string; + taskIdentifier?: string; +}): CreateRunInput { + return { + data: { + id: p.runId, + engine: "V2", + status: "PENDING", + friendlyId: p.friendlyId, + runtimeEnvironmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: p.organizationId, + projectId: p.projectId, + taskIdentifier: p.taskIdentifier ?? "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: p.traceId, + spanId: p.spanId, + parentSpanId: p.parentSpanId, + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: p.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: p.projectId, + organizationId: p.organizationId, + }, + }; +} + +// Build the split router. `legacyLag` configures the legacy store's frozen replica. +function buildRouter( + prisma14: PrismaClient, + prisma17: RunOpsPrismaClient, + legacyLag: Parameters[1] +) { + const legacyReplica = laggingReplica(prisma14, legacyLag); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +function spansRequest(runId: string, spanId: string) { + return { + request: new Request(`https://api.trigger.dev/api/v1/runs/${runId}/spans/${spanId}`, { + headers: { Authorization: "Bearer tr_dev_x" }, + }), + params: { runId, spanId }, + context: {} as never, + }; +} +function traceRequest(runId: string) { + return { + request: new Request(`https://api.trigger.dev/api/v1/runs/${runId}/trace`, { + headers: { Authorization: "Bearer tr_dev_x" }, + }), + params: { runId }, + context: {} as never, + }; +} +function syncRequest(traceId: string) { + return { + request: new Request(`https://app.trigger.dev/sync/traces/runs/${traceId}?live=true`), + params: { traceId }, + context: {} as never, + } as never; +} + +describe("run-trace/span-detail route loaders under a lagging replica", () => { + // spans loader — findResource findRun ($replica) + heteroRunOpsPostgresTest( + "spans loader: replica+buffer double-miss returns a retryable 404 (x-should-retry:true), self-healing to 200 once the replica catches up", + async ({ prisma14, prisma17 }) => { + const suffix = `spans_find_${seq++}`; + const seed = await seedTenant(prisma14, suffix); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${suffix}`; + + // Seed the live run on the LEGACY primary (writer) only. + const lagged = buildRouter(prisma14, prisma17, [{ model: "taskRun", mode: "missing" }]); + await lagged.legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + traceId: `trace_${suffix}`, + spanId: `span_${suffix}`, + }) + ); + + holder.store = lagged.router; + holder.environment = authEnvironment(seed); + holder.bufferResult = null; + + const res = (await spansLoader(spansRequest(friendlyId, `span_${suffix}`))) as Response; + + // The frozen replica WAS consulted (the lag was really exercised). + expect(lagged.legacyReplica.wasHit("taskRun")).toBe(true); + // The caller emits the documented RETRYABLE not-found, not a terminal 404. + expect(res.status).toBe(404); + expect(res.headers.get("x-should-retry")).toBe("true"); + + // Self-heal proof: point the store at a NON-lagging replica (replica caught up == the SDK retry + // landing after replication) and the SAME loader now resolves the run and returns 200. + holder.span = { + spanId: `span_${suffix}`, + parentId: undefined, + message: "root", + isError: false, + isPartial: false, + isCancelled: false, + level: "TRACE", + startTime: new Date(), + duration: 1_000_000, + properties: undefined, + events: undefined, + entity: { type: "task" }, + }; + const caughtUp = buildRouter(prisma14, prisma17, []); // no models frozen + holder.store = caughtUp.router; + const res2 = (await spansLoader(spansRequest(friendlyId, `span_${suffix}`))) as Response; + expect(res2.status).toBe(200); + const body2 = (await res2.json()) as { runId?: string; spanId?: string }; + expect(body2.runId).toBe(friendlyId); + expect(body2.spanId).toBe(`span_${suffix}`); + } + ); + + // spans loader — handler findRuns triggeredRuns ($replica) + heteroRunOpsPostgresTest( + "spans loader: a just-triggered child on the primary is omitted from triggeredRuns under lag (200, list self-heals)", + async ({ prisma14, prisma17 }) => { + const suffix = `spans_children_${seq++}`; + const seed = await seedTenant(prisma14, suffix); + const parentRunId = `run_${CUID_25}`; + const parentFriendlyId = `run_${suffix}_p`; + const spanId = `span_${suffix}`; + + // Seed the parent run AND a child run (parentSpanId = spanId) on the LEGACY primary. + const writerStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await writerStore.createRun( + buildCreateRunInput({ + runId: parentRunId, + friendlyId: parentFriendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + traceId: `trace_${suffix}`, + spanId, + }) + ); + const childRunId = `run_${"d".repeat(25)}`; + const childFriendlyId = `run_${suffix}_c`; + await writerStore.createRun( + buildCreateRunInput({ + runId: childRunId, + friendlyId: childFriendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + traceId: `trace_${suffix}`, + spanId: `childspan_${suffix}`, + parentSpanId: spanId, + taskIdentifier: "child-task", + }) + ); + + // Capture the PARENT row as the frozen replica snapshot (== "parent replicated, child written + // after and not yet replicated"). The parent resolves on the replica; the child does not. + const parentSnapshot = (await prisma14.taskRun.findFirstOrThrow({ + where: { id: parentRunId }, + })) as unknown as Record; + + const lagged = buildRouter(prisma14, prisma17, [ + { model: "taskRun", mode: "frozen", rows: [parentSnapshot] }, + ]); + holder.store = lagged.router; + holder.environment = authEnvironment(seed); + holder.bufferResult = null; + holder.span = { + spanId, + parentId: undefined, + message: "root", + isError: false, + isPartial: false, + isCancelled: false, + level: "TRACE", + startTime: new Date(), + duration: 2_000_000, + properties: undefined, + events: undefined, + entity: { type: "task" }, + }; + + const res = (await spansLoader(spansRequest(parentFriendlyId, spanId))) as Response; + const body = (await res.json()) as { runId?: string; triggeredRuns?: unknown }; + + // The parent resolved via the (frozen) replica, so the read genuinely went through it. + expect(lagged.legacyReplica.wasHit("taskRun")).toBe(true); + // 200 with the span, and the lagging child is simply OMITTED from the list. + expect(res.status).toBe(200); + expect(body.runId).toBe(parentFriendlyId); + expect(body.triggeredRuns).toBeUndefined(); + + // Prove the omission is lag (the child is present on the primary right now). + const onPrimary = await prisma14.taskRun.findMany({ + where: { runtimeEnvironmentId: seed.environment.id, parentSpanId: spanId }, + select: { friendlyId: true }, + }); + expect(onPrimary.map((r) => r.friendlyId)).toContain(childFriendlyId); + } + ); + + // trace loader — findResource findRun ($replica) + heteroRunOpsPostgresTest( + "trace loader: replica+buffer double-miss returns a retryable 404 (x-should-retry:true), self-healing to 200 once the replica catches up", + async ({ prisma14, prisma17 }) => { + const suffix = `trace_find_${seq++}`; + const seed = await seedTenant(prisma14, suffix); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${suffix}`; + + const lagged = buildRouter(prisma14, prisma17, [{ model: "taskRun", mode: "missing" }]); + await lagged.legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + traceId: `trace_${suffix}`, + spanId: `span_${suffix}`, + }) + ); + + holder.store = lagged.router; + holder.environment = authEnvironment(seed); + holder.bufferResult = null; + + const res = (await traceLoader(traceRequest(friendlyId))) as Response; + expect(lagged.legacyReplica.wasHit("taskRun")).toBe(true); + expect(res.status).toBe(404); + expect(res.headers.get("x-should-retry")).toBe("true"); + + // Self-heal proof: caught-up replica -> the same loader resolves the run and returns the trace. + holder.traceSummary = { rootSpanId: `span_${suffix}`, spans: [] }; + const caughtUp = buildRouter(prisma14, prisma17, []); + holder.store = caughtUp.router; + const res2 = (await traceLoader(traceRequest(friendlyId))) as Response; + expect(res2.status).toBe(200); + const body2 = (await res2.json()) as { trace?: unknown }; + expect(body2.trace).toEqual({ rootSpanId: `span_${suffix}`, spans: [] }); + } + ); + + // sync trace-runs loader — findRun by traceId ($replica), with a primary fallback + heteroRunOpsPostgresTest( + "sync trace-runs loader: a live run lagging the replica is recovered via the primary fallback", + async ({ prisma14, prisma17 }) => { + const suffix = `sync_${seq++}`; + const seed = await seedTenant(prisma14, suffix); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${suffix}`; + const traceId = `trace_${suffix}`; + const userId = `user_${suffix}`; + + // The dashboard user, joined to the org so the route's real orgMember check passes. + await prisma14.user.create({ + data: { id: userId, email: `u-${suffix}@example.com`, authenticationMethod: "MAGIC_LINK" }, + }); + await prisma14.orgMember.create({ + data: { userId, organizationId: seed.organization.id, role: "ADMIN" }, + }); + + const lagged = buildRouter(prisma14, prisma17, [{ model: "taskRun", mode: "missing" }]); + await lagged.legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + traceId, + spanId: `span_${suffix}`, + }) + ); + + holder.store = lagged.router; + holder.environment = authEnvironment(seed); + holder.userId = userId; + holder.resolvedEnv = { organizationId: seed.organization.id }; + holder.replicaMarker = { orgMember: prisma14.orgMember }; + + const res = (await syncTraceRunsLoader(syncRequest(traceId))) as Response; + + // The frozen replica WAS consulted (the lag was really exercised). + expect(lagged.legacyReplica.wasHit("taskRun")).toBe(true); + + // The primary fallback recovers the live run and the loader proceeds to the shape stream (200). + expect(res.status).toBe(200); + expect(res.headers.get("x-longpoll")).toBe("hit"); + } + ); + + // Negative control for the sync route: a truly-absent run must still 404 (the primary fallback + // recovers a LIVE run, it does not turn every miss into a 200). + heteroRunOpsPostgresTest( + "sync trace-runs loader: 404s when the run is absent on the primary too", + async ({ prisma14, prisma17 }) => { + const suffix = `sync_absent_${seq++}`; + const seed = await seedTenant(prisma14, suffix); + const userId = `user_${suffix}`; + await prisma14.user.create({ + data: { id: userId, email: `u-${suffix}@example.com`, authenticationMethod: "MAGIC_LINK" }, + }); + await prisma14.orgMember.create({ + data: { userId, organizationId: seed.organization.id, role: "ADMIN" }, + }); + + const lagged = buildRouter(prisma14, prisma17, [{ model: "taskRun", mode: "missing" }]); + holder.store = lagged.router; + holder.environment = authEnvironment(seed); + holder.userId = userId; + holder.resolvedEnv = { organizationId: seed.organization.id }; + holder.replicaMarker = { orgMember: prisma14.orgMember }; + + const res = (await syncTraceRunsLoader(syncRequest("trace_does_not_exist"))) as Response; + expect(res.status).toBe(404); + } + ); +}); diff --git a/apps/webapp/test/waitpointCallbackRouteReplicaLag.guard.test.ts b/apps/webapp/test/waitpointCallbackRouteReplicaLag.guard.test.ts new file mode 100644 index 000000000..8ede36fcb --- /dev/null +++ b/apps/webapp/test/waitpointCallbackRouteReplicaLag.guard.test.ts @@ -0,0 +1,193 @@ +// Property: the HTTP-callback route reads-your-writes via a primary fallback. +// +// When the replica-routed `runStore.findWaitpoint({ where: { id } })` returns null (a token whose +// callback fires right after mint has not replicated yet), the route re-reads the OWNING PRIMARY via +// `runStore.findWaitpointOnPrimary(...)` before giving up, so a just-minted token still resolves. +// +// This drives the REAL exported route `action` end-to-end against a lagging split replica (rather +// than calling the store methods directly), so it verifies the route itself — not just the store — +// honours the fallback. +// +// The DB is never mocked: `~/db.server`'s `$replica`/`prisma` proxies forward to a real testcontainer +// (PG17). The replica proxy is wrapped so `waitpoint` reads come back empty (replication lag) while +// every other model — crucially `runtimeEnvironment`, which control-plane env resolution reads — +// forwards to the real row. The primary proxy is the unwrapped client, so `findWaitpointOnPrimary` +// sees the token. `~/v3/runEngine.server` is stubbed to a no-op by test/setup.ts, so the PENDING +// happy path reaches a real 200 without a live engine. +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; + +const replicaHolder = vi.hoisted(() => ({ client: undefined as any })); +const primaryHolder = vi.hoisted(() => ({ client: undefined as any })); + +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + const lazyProxy = (holder: { client: any }, label: string) => + new Proxy( + {}, + { + get(_t, prop) { + if (!holder.client) { + throw new Error(`${label} not set for this test`); + } + const value = holder.client[prop]; + // The `runStore` singleton memoizes each Prisma delegate on first access, pinning it to + // the first test's (later-dropped) DB. Re-resolve so it routes to the current client. + if (value !== null && typeof value === "object") { + return new Proxy(value, { + get: (_d, method) => holder.client[prop][method], + }); + } + return value; + }, + } + ); + return { + prisma: lazyProxy(primaryHolder, "primaryHolder.client"), + $replica: lazyProxy(replicaHolder, "replicaHolder.client"), + runOpsNewPrisma: lazyProxy(replicaHolder, "replicaHolder.client"), + runOpsNewReplica: lazyProxy(replicaHolder, "replicaHolder.client"), + runOpsLegacyReplica: lazyProxy(replicaHolder, "replicaHolder.client"), + runOpsSplitReadEnabled: true, + sqlDatabaseSchema: Prisma.sql([`public`]), + }; +}); + +import type { PrismaClient } from "@trigger.dev/database"; +import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import { action } from "~/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash"; +import { generateHttpCallbackUrl } from "~/services/httpCallback.server"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +function callbackRequest(body: unknown) { + const payload = JSON.stringify(body); + return new Request("http://localhost/callback", { + method: "POST", + headers: { "content-type": "application/json", "content-length": String(payload.length) }, + body: payload, + }); +} + +// Derives the same hash `verifyHttpCallbackHash` checks, via the production URL helper. +function hashFor(waitpointId: string, apiKey: string) { + const url = generateHttpCallbackUrl(waitpointId, apiKey); + return url.split("/").pop()!; +} + +let n = 0; +async function seedControlPlane(prisma: PrismaClient) { + const s = n++; + const organization = await prisma.organization.create({ + data: { title: `Org ${s}`, slug: `org-lag-${s}` }, + }); + const project = await prisma.project.create({ + data: { + name: `P ${s}`, + slug: `p-lag-${s}`, + externalRef: `proj_lag_${s}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "PRODUCTION", + slug: `env-lag-${s}`, + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_lag_${s}`, + pkApiKey: `pk_lag_${s}`, + shortcode: `sc_lag_${s}`, + }, + }); + return { organization, project, environment }; +} + +// A just-minted MANUAL token whose `id` matches `WaitpointId.toId(friendlyId)`, so the route's +// friendlyId->id conversion + the hash (computed over `id`) line up. +async function seedJustMintedToken( + prisma: PrismaClient, + ctx: { environmentId: string; projectId: string }, + status: "PENDING" | "COMPLETED" = "PENDING" +) { + const s = n++; + const { id, friendlyId } = WaitpointId.generate(); + await prisma.waitpoint.create({ + data: { + id, + friendlyId, + type: "MANUAL", + status, + idempotencyKey: `idem_lag_${s}`, + userProvidedIdempotencyKey: false, + environmentId: ctx.environmentId, + projectId: ctx.projectId, + }, + }); + return { id, friendlyId }; +} + +describe("HTTP-callback route: read-your-writes primary fallback under replica lag (real route action)", () => { + // Happy path: a just-minted PENDING token is invisible on the (lagging) replica; the route resolves + // it on the primary and completes it -> 200. + heteroPostgresTest( + "PENDING token invisible on replica resolves via primary and completes -> 200", + async ({ prisma17 }) => { + const cp = await seedControlPlane(prisma17 as unknown as PrismaClient); + const token = await seedJustMintedToken(prisma17 as unknown as PrismaClient, { + environmentId: cp.environment.id, + projectId: cp.project.id, + }); + + const replica = laggingReplica(prisma17 as unknown as PrismaClient, [ + { model: "waitpoint", mode: "missing" }, + ]); + primaryHolder.client = prisma17; // findWaitpointOnPrimary sees the token + replicaHolder.client = replica.client; // findWaitpoint (+ env resolution) reads here; waitpoint starved + + // No parent env -> the hash is computed over the env's own apiKey. + const hash = hashFor(token.id, cp.environment.apiKey); + + const res = await action({ + request: callbackRequest({ ok: true }), + params: { waitpointFriendlyId: token.friendlyId, hash }, + context: {} as never, + }); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ success: true }); + // The replica read genuinely fired and missed — the token was invisible there. + expect(replica.wasHit()).toBe(true); + } + ); + + // Independent-of-engine proof: a PENDING token with the WRONG hash resolves on the primary and the + // hash check runs -> 401. The 401 (not a not-found) pins that the primary fallback resolves the row, + // not that it merely masks the engine's no-op. + heteroPostgresTest( + "PENDING token invisible on replica resolves via primary then fails hash -> 401", + async ({ prisma17 }) => { + const cp = await seedControlPlane(prisma17 as unknown as PrismaClient); + const token = await seedJustMintedToken(prisma17 as unknown as PrismaClient, { + environmentId: cp.environment.id, + projectId: cp.project.id, + }); + + const replica = laggingReplica(prisma17 as unknown as PrismaClient, [ + { model: "waitpoint", mode: "missing" }, + ]); + primaryHolder.client = prisma17; + replicaHolder.client = replica.client; + + const res = await action({ + request: callbackRequest({ ok: true }), + params: { waitpointFriendlyId: token.friendlyId, hash: "not-the-right-hash" }, + context: {} as never, + }); + + expect(res.status).toBe(401); + expect(await res.json()).toEqual({ error: "Invalid URL, hash doesn't match" }); + expect(replica.wasHit()).toBe(true); + } + ); +}); diff --git a/apps/webapp/test/waitpointCompleteRouteReplicaLag.guard.test.ts b/apps/webapp/test/waitpointCompleteRouteReplicaLag.guard.test.ts new file mode 100644 index 000000000..438d1a94d --- /dev/null +++ b/apps/webapp/test/waitpointCompleteRouteReplicaLag.guard.test.ts @@ -0,0 +1,251 @@ +// Property: under split replica lag the dashboard "complete waitpoint" route action still completes a +// just-minted token. It resolves the waitpoint by id via findWaitpoint (owning REPLICA), and on a null +// re-reads via findWaitpointOnPrimary before the projectId guard, so a token invisible on the lagging +// replica passes the guard and completion proceeds instead of failing with "No waitpoint found". +// Drives the REAL exported action; only peripheral collaborators are mocked. The seam — runStore over a +// split topology whose owning replica is frozen — is a REAL RoutingRunStore over real testcontainer +// Postgres. + +import { describe, expect, vi } from "vitest"; + +const runStoreHolder = vi.hoisted(() => ({ store: undefined as any })); +// $replica.project.findUnique — return the seeded project id so the guard's only remaining variable +// is whether the waitpoint read resolved (this read hits `project`, never the lagging `waitpoint`). +const projectHolder = vi.hoisted(() => ({ id: undefined as string | undefined })); +const engineHolder = vi.hoisted(() => ({ calls: [] as any[] })); + +vi.mock("~/v3/runStore.server", () => ({ + get runStore() { + return runStoreHolder.store; + }, +})); + +vi.mock("~/db.server", () => ({ + $replica: { + project: { + findUnique: async () => (projectHolder.id ? { id: projectHolder.id } : null), + }, + }, +})); + +vi.mock("~/v3/runEngine.server", () => ({ + engine: { + completeWaitpoint: async (args: any) => { + engineHolder.calls.push(args); + return { id: args.id }; + }, + }, +})); + +vi.mock("~/services/session.server", () => ({ + requireUserId: async () => "user_test", +})); + +vi.mock("~/env.server", () => ({ + env: { TASK_PAYLOAD_MAXIMUM_SIZE: 3_000_000 }, +})); + +vi.mock("~/services/logger.server", () => ({ + logger: { error: () => {}, info: () => {}, debug: () => {}, warn: () => {} }, +})); + +// Distinguishable stand-ins for the redirect helpers so we can read the outcome + message off the +// action's return without the real session-cookie machinery. +vi.mock("~/models/message.server", () => ({ + redirectWithErrorMessage: (redirect: string, _req: unknown, message: string) => + new Response(null, { + status: 302, + headers: { location: redirect, "x-outcome": "error", "x-message": message }, + }), + redirectWithSuccessMessage: (redirect: string, _req: unknown, message: string) => + new Response(null, { + status: 302, + headers: { location: redirect, "x-outcome": "success", "x-message": message }, + }), +})); + +// MANUAL-branch collaborators — the token completion path resolves the env then completes. Return an +// env whose id matches the seeded waitpoint's environmentId so the env guard passes. +const envHolder = vi.hoisted(() => ({ id: undefined as string | undefined })); +vi.mock("~/models/runtimeEnvironment.server", () => ({ + findEnvironmentBySlug: async () => (envHolder.id ? { id: envHolder.id } : null), +})); +vi.mock("~/runEngine/concerns/waitpointCompletionPacket.server", () => ({ + processWaitpointCompletionPacket: async () => ({ data: undefined, dataType: "application/json" }), +})); + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import { PostgresRunStore, RoutingRunStore } from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import { action } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.$waitpointFriendlyId.complete/route"; + +vi.setConfig({ testTimeout: 120_000, hookTimeout: 120_000 }); + +const WAITPOINT_CROSS_SEAM_FKS = [ + "Waitpoint_environmentId_fkey", + "Waitpoint_projectId_fkey", +] as const; + +async function dropWaitpointCrossSeamFks(prisma: PrismaClient) { + for (const c of WAITPOINT_CROSS_SEAM_FKS) { + await prisma.$executeRawUnsafe(`ALTER TABLE "Waitpoint" DROP CONSTRAINT IF EXISTS "${c}"`); + } +} + +// Seed a standalone PENDING MANUAL token on the writer, exactly as minting a resume token does. +async function seedPendingTokenWaitpoint( + store: PostgresRunStore, + params: { + id: string; + friendlyId: string; + projectId: string; + environmentId: string; + type?: "MANUAL" | "DATETIME"; + } +) { + await store.upsertWaitpoint({ + where: { + environmentId_idempotencyKey: { + environmentId: params.environmentId, + idempotencyKey: params.id, + }, + }, + create: { + id: params.id, + friendlyId: params.friendlyId, + type: params.type ?? "MANUAL", + status: "PENDING", + idempotencyKey: params.id, + userProvidedIdempotencyKey: false, + projectId: params.projectId, + environmentId: params.environmentId, + }, + update: {}, + }); +} + +function completeRequest(kind: "MANUAL" | "DATETIME") { + const body = new URLSearchParams(); + body.set("type", kind); + if (kind === "MANUAL") body.set("payload", "{}"); + body.set("successRedirect", "/success"); + body.set("failureRedirect", "/failure"); + return new Request("http://localhost/complete", { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body: body.toString(), + }); +} + +const params = (friendlyId: string) => ({ + organizationSlug: "org-slug", + projectParam: "proj-slug", + envParam: "dev", + waitpointFriendlyId: friendlyId, +}); + +describe("complete-waitpoint dashboard route reads-your-writes under split replica lag", () => { + // LEGACY-resident (cuid) token minted on the control-plane writer; its replica lags. The action's + // findWaitpoint(id) misses, and the findWaitpointOnPrimary fallback must resolve it so the just- + // minted token passes the projectId guard and completes — NOT "No waitpoint found". + heteroRunOpsPostgresTest( + "MANUAL token invisible on the lagging owning replica completes via the primary fallback", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "waitpoint", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + runStoreHolder.store = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + await dropWaitpointCrossSeamFks(prisma14 as unknown as PrismaClient); + + // id = WaitpointId.toId(friendlyId): the route computes the internal id from the friendly id, + // so the DB id must be exactly that bare cuid (classifies LEGACY -> control-plane store). + const { id: waitpointId, friendlyId } = WaitpointId.generate(); + const projectId = "proj_wc_route_leg"; + const environmentId = "env_wc_route_leg"; + projectHolder.id = projectId; + envHolder.id = environmentId; + engineHolder.calls = []; + + await seedPendingTokenWaitpoint(legacyStore, { + id: waitpointId, + friendlyId, + projectId, + environmentId, + }); + + const res = (await action({ + request: completeRequest("MANUAL"), + params: params(friendlyId), + context: {} as never, + })) as Response; + + // Property: the frozen replica was consulted (so the miss is real), yet the token completed via + // the owning-primary re-read — success, engine invoked, and NOT "No waitpoint found". + expect(legacyReplica.wasHit()).toBe(true); + expect(res.headers.get("x-outcome")).toBe("success"); + expect(res.headers.get("x-message")).toBe("Waitpoint completed"); + expect(res.headers.get("x-message")).not.toBe("No waitpoint found"); + expect(engineHolder.calls).toHaveLength(1); + expect(engineHolder.calls[0].id).toBe(waitpointId); + } + ); + + // Same seam via the DATETIME "skip" branch (also gated by the shared projectId guard). Kept as a + // second, mock-light assertion of the fallback so the guard doesn't hinge on MANUAL-branch helpers. + heteroRunOpsPostgresTest( + "DATETIME skip on a lag-invisible token resolves via the primary fallback", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "waitpoint", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + runStoreHolder.store = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + await dropWaitpointCrossSeamFks(prisma14 as unknown as PrismaClient); + + const { id: waitpointId, friendlyId } = WaitpointId.generate(); + const projectId = "proj_wc_route_dt"; + const environmentId = "env_wc_route_dt"; + projectHolder.id = projectId; + envHolder.id = environmentId; + engineHolder.calls = []; + + await seedPendingTokenWaitpoint(legacyStore, { + id: waitpointId, + friendlyId, + projectId, + environmentId, + type: "DATETIME", + }); + + const res = (await action({ + request: completeRequest("DATETIME"), + params: params(friendlyId), + context: {} as never, + })) as Response; + + expect(legacyReplica.wasHit()).toBe(true); + expect(res.headers.get("x-outcome")).toBe("success"); + expect(res.headers.get("x-message")).toBe("Waitpoint skipped"); + expect(engineHolder.calls).toHaveLength(1); + expect(engineHolder.calls[0].id).toBe(waitpointId); + } + ); +}); diff --git a/apps/webapp/test/waitpointPresenters.replicaLag.guard.test.ts b/apps/webapp/test/waitpointPresenters.replicaLag.guard.test.ts new file mode 100644 index 000000000..f81ce6fa2 --- /dev/null +++ b/apps/webapp/test/waitpointPresenters.replicaLag.guard.test.ts @@ -0,0 +1,568 @@ +// Property: the waitpoint-family READ presenters behave correctly under replica lag. Drives the REAL +// exported presenter classes (ApiWaitpointPresenter, WaitpointListPresenter, WaitpointPresenter, +// WaitpointTagListPresenter) through the REAL split RoutingRunStore over two testcontainer Postgres DBs +// (heteroRunOpsPostgresTest), the owning (LEGACY / control-plane) REPLICA frozen behind the shared +// laggingReplica, injected as the presenter's `runStore` arg. Only peripheral webapp singletons +// (db.server clients, runStore.server, logger, httpCallback URL, engine-version gate, clickhouse +// factory, control-plane env resolver, NextRunListPresenter leaf, ServiceValidationError) are stubbed. +// +// What each case proves, from the real caller's observable output under lag: +// 1 ApiWaitpointPresenter findWaitpoint — recovers a LIVE token on a replica miss via a +// findWaitpointOnPrimary fallback (returns the token; absent the fallback it throws +// "Waitpoint not found"). +// 2 WaitpointListPresenter findManyWaitpoints — the token list omits the just-minted token +// (tokens: []); eventual-consistency render, no write gated. +// 3 WaitpointListPresenter findWaitpoint — #probeAnyToken → hasAnyTokens false → cosmetic +// "no tokens yet" copy; self-heals next fetch. +// 4 WaitpointPresenter findWaitpoint — dashboard DETAIL loader returns null → not-found render; +// the user reloads. No write. +// 5 WaitpointPresenter findWaitpointConnectedRunIds — connected-runs sub-list empty on the detail +// render (connectedRuns: []); display only. +// 6 WaitpointPresenter findRuns — connected-run friendlyIds unresolved (connectedRuns: []) though +// the connection id was gathered; display only. +// 7 WaitpointTagListPresenter findManyWaitpointTags — tag-filter dropdown omits a just-created tag +// (tags: []); eventual-consistency list, no write. +// +// For cases 2-7, the test additionally drives the SAME presenter over a non-lagging router to prove +// the stale value is pure lag (the row is live on the primary), not absence. + +import { heteroRunOpsPostgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect, vi } from "vitest"; + +// ── Fix-orthogonal module stubs (never the run-store read path) ──────────────────────────────────── +const { holder } = vi.hoisted(() => ({ + holder: { env: undefined as unknown }, +})); + +vi.mock("~/db.server", () => ({ prisma: {}, $replica: {} })); +vi.mock("~/v3/runStore.server", () => ({ runStore: {} })); +vi.mock("~/services/logger.server", () => ({ + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() }, +})); +vi.mock("~/services/httpCallback.server", () => ({ + generateHttpCallbackUrl: () => "https://api.trigger.dev/http-callback", +})); +// ApiWaitpointPresenter imports ServiceValidationError from baseService.server, which itself drags the +// run-engine singleton at import; stub to a light Error subclass so the import graph stays cheap. +vi.mock("~/v3/services/baseService.server", () => ({ + ServiceValidationError: class ServiceValidationError extends Error {}, +})); +// Engine-version gate for the token LIST presenter — return V2 so it proceeds to the read under test. +vi.mock("~/v3/engineVersion.server", () => ({ + determineEngineVersion: vi.fn(async () => "V2"), +})); +// Reached by WaitpointPresenter only when connectedRuns is non-empty — never under lag; stub so import +// resolves without the real clickhouse client. +vi.mock("~/services/clickhouse/clickhouseFactoryInstance.server", () => ({ + clickhouseFactory: { getClickhouseForOrganization: vi.fn(async () => ({})) }, +})); +// The detail presenter resolves the env-derived fields (apiKey/organizationId) off this control-plane +// singleton after the waitpoint read succeeds. Peripheral to the run-store read path. +vi.mock("~/v3/runOpsMigration/controlPlaneResolver.server", () => ({ + controlPlaneResolver: { resolveAuthenticatedEnv: vi.fn(async () => holder.env) }, +})); +// Connected-run hydration leaf; never instantiated under lag (connectedRuns empty), but the import +// must resolve without its heavy graph. +vi.mock("~/presenters/v3/NextRunListPresenter.server", () => ({ + NextRunListPresenter: class { + async call() { + return { runs: [] }; + } + }, +})); + +import { PostgresRunStore, RoutingRunStore } from "@internal/run-store"; +import { ApiWaitpointPresenter } from "~/presenters/v3/ApiWaitpointPresenter.server"; +import { WaitpointListPresenter } from "~/presenters/v3/WaitpointListPresenter.server"; +import { WaitpointPresenter } from "~/presenters/v3/WaitpointPresenter.server"; +import { WaitpointTagListPresenter } from "~/presenters/v3/WaitpointTagListPresenter.server"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +const CUID_25 = "e".repeat(25); // cuid id-shape → LEGACY (control-plane / prisma14) + +// A recording "replica" that has NOT caught up: reads for the SELECTED models/raw come back empty and +// flip `wasHit`, so a replica-routed read misses the just-written row. Everything else forwards to the +// real client (writes always land on the PRIMARY, so freezing the readOnly client never affects seeding). +function laggingReplica( + real: C, + freeze: { waitpoint?: boolean; taskRun?: boolean; waitpointTag?: boolean; raw?: boolean } +): { client: C; wasHit: () => boolean } { + let hit = false; + function frozenModel(target: any) { + return new Proxy(target, { + get(innerTarget, prop) { + if (prop === "findFirst" || prop === "findMany" || prop === "findUnique") { + return async () => { + hit = true; + return prop === "findMany" ? [] : null; + }; + } + if (prop === "findFirstOrThrow" || prop === "findUniqueOrThrow") { + return async () => { + hit = true; + throw new Error("lagging replica: row not visible"); + }; + } + return (innerTarget as any)[prop]; + }, + }); + } + const frozenWaitpoint = freeze.waitpoint ? frozenModel((real as any).waitpoint) : undefined; + const frozenTaskRun = freeze.taskRun ? frozenModel((real as any).taskRun) : undefined; + const frozenWaitpointTag = freeze.waitpointTag + ? frozenModel((real as any).waitpointTag) + : undefined; + const client = new Proxy(real, { + get(target, prop) { + if (prop === "waitpoint" && frozenWaitpoint) return frozenWaitpoint; + if (prop === "taskRun" && frozenTaskRun) return frozenTaskRun; + if (prop === "waitpointTag" && frozenWaitpointTag) return frozenWaitpointTag; + if (freeze.raw && (prop === "$queryRaw" || prop === "$queryRawUnsafe")) { + return async () => { + hit = true; + return []; + }; + } + return (target as any)[prop]; + }, + }) as C; + return { client, wasHit: () => hit }; +} + +// LEGACY-owning router whose legacy replica is frozen per `freeze`; the NEW store is real (non-lagging), +// so the on-miss fan-out to the other store's replica also legitimately misses. +function buildRouter( + prisma14: PrismaClient, + prisma17: RunOpsPrismaClient, + freeze: { waitpoint?: boolean; taskRun?: boolean; waitpointTag?: boolean; raw?: boolean } +) { + const legacyReplica = laggingReplica(prisma14, freeze); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +// A router with a fully non-lagging legacy replica — used to prove each case's stale value is +// pure lag (the row is live on the primary), by driving the SAME presenter against it. +function buildHealthyRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + return buildRouter(prisma14, prisma17, {}); +} + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +async function seedManualWaitpoint( + store: PostgresRunStore, + params: { + id: string; + friendlyId: string; + projectId: string; + environmentId: string; + tags?: string[]; + } +) { + await store.upsertWaitpoint({ + where: { + environmentId_idempotencyKey: { + environmentId: params.environmentId, + idempotencyKey: params.id, + }, + }, + create: { + id: params.id, + friendlyId: params.friendlyId, + type: "MANUAL", + status: "PENDING", + idempotencyKey: params.id, + userProvidedIdempotencyKey: false, + projectId: params.projectId, + environmentId: params.environmentId, + ...(params.tags ? { tags: params.tags } : {}), + }, + update: {}, + }); +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: "COMPLETED_SUCCESSFULLY" as const, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: JSON.stringify({ hello: "world" }), + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +function apiEnvironment(env: { id: string; apiKey: string; projectId: string }) { + return { + id: env.id, + type: "DEVELOPMENT" as const, + project: { id: env.projectId, engine: "V2" as const }, + apiKey: env.apiKey, + }; +} + +describe("waitpoint-family read presenters under replica lag", () => { + // ApiWaitpointPresenter findWaitpoint — recovers a live token via a primary fallback + heteroRunOpsPostgresTest( + "ApiWaitpointPresenter.call resolves a live token under replica lag", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, { + waitpoint: true, + }); + const seed = await seedEnvironmentLegacy(prisma14, "api_wp"); + const waitpointId = `waitpoint_${CUID_25}`; // cuid → LEGACY + await seedManualWaitpoint(legacyStore, { + id: waitpointId, + friendlyId: "waitpoint_api_wp", + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + + const presenter = new ApiWaitpointPresenter(prisma14, prisma14, undefined, router); + + // The presenter's replica read misses, then re-reads the owning primary and returns the LIVE + // token via the findWaitpointOnPrimary fallback. + const result = await presenter.call( + apiEnvironment({ + id: seed.environment.id, + apiKey: seed.environment.apiKey, + projectId: seed.project.id, + }), + waitpointId + ); + + // The replica WAS consulted (and, frozen, missed) — proving the recovery is the primary fallback. + expect(legacyReplica.wasHit()).toBe(true); + expect(result.id).toBe("waitpoint_api_wp"); + expect(result.type).toBe("MANUAL"); + expect(result.status).toBe("WAITING"); + } + ); + + // Negative control: a token that truly does not exist anywhere still throws (the fallback recovers a + // LIVE token, it does not blanket-swallow the not-found). + heteroRunOpsPostgresTest( + "control: ApiWaitpointPresenter.call throws when the token is absent on the primary too", + async ({ prisma14, prisma17 }) => { + const { router } = buildRouter(prisma14, prisma17, { waitpoint: true }); + const seed = await seedEnvironmentLegacy(prisma14, "api_absent"); + const presenter = new ApiWaitpointPresenter(prisma14, prisma14, undefined, router); + await expect( + presenter.call( + apiEnvironment({ + id: seed.environment.id, + apiKey: seed.environment.apiKey, + projectId: seed.project.id, + }), + `waitpoint_${"f".repeat(25)}` + ) + ).rejects.toThrow(/not found/i); + } + ); + + // WaitpointListPresenter findManyWaitpoints / findWaitpoint + heteroRunOpsPostgresTest( + "WaitpointListPresenter.call omits a just-minted token under replica lag", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, { + waitpoint: true, + }); + const seed = await seedEnvironmentLegacy(prisma14, "list"); + await seedManualWaitpoint(legacyStore, { + id: `waitpoint_${CUID_25}`, + friendlyId: "waitpoint_list", + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + + const env = apiEnvironment({ + id: seed.environment.id, + apiKey: seed.environment.apiKey, + projectId: seed.project.id, + }); + + const laggy = await new WaitpointListPresenter(prisma14, prisma14, undefined, router).call({ + environment: env, + }); + + // The list render omits the token this pass (findManyWaitpoints), and the empty-state probe + // reports hasAnyTokens=false (the #probeAnyToken findWaitpoint read, cosmetic copy). No write + // gated; self-heals next fetch. + expect(laggy.success).toBe(true); + if (!laggy.success) throw new Error("unreachable"); + expect(laggy.tokens).toEqual([]); + expect(laggy.hasAnyTokens).toBe(false); + expect(legacyReplica.wasHit()).toBe(true); + + // Prove pure lag: the SAME presenter over a healthy router surfaces the live token. + const healthy = await new WaitpointListPresenter( + prisma14, + prisma14, + undefined, + buildHealthyRouter(prisma14, prisma17).router + ).call({ environment: env }); + expect(healthy.success).toBe(true); + if (!healthy.success) throw new Error("unreachable"); + expect(healthy.tokens).toHaveLength(1); + expect(healthy.tokens[0]!.id).toBe("waitpoint_list"); + expect(healthy.hasAnyTokens).toBe(true); + } + ); + + // WaitpointPresenter findWaitpoint + heteroRunOpsPostgresTest( + "WaitpointPresenter.call returns null for a token detail under replica lag", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, { + waitpoint: true, + }); + const seed = await seedEnvironmentLegacy(prisma14, "detail"); + const friendlyId = "waitpoint_detail"; + await seedManualWaitpoint(legacyStore, { + id: `waitpoint_${CUID_25}`, + friendlyId, + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + holder.env = { + id: seed.environment.id, + organizationId: seed.organization.id, + apiKey: seed.environment.apiKey, + }; + + const laggy = await new WaitpointPresenter(prisma14, prisma14, undefined, router).call({ + friendlyId, + environmentId: seed.environment.id, + projectId: seed.project.id, + }); + + // The detail loader returns null → not-found page → user reloads. No write. + expect(laggy).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + const healthy = await new WaitpointPresenter( + prisma14, + prisma14, + undefined, + buildHealthyRouter(prisma14, prisma17).router + ).call({ friendlyId, environmentId: seed.environment.id, projectId: seed.project.id }); + expect(healthy).not.toBeNull(); + expect(healthy!.id).toBe(friendlyId); + expect(healthy!.type).toBe("MANUAL"); + } + ); + + // WaitpointPresenter findWaitpointConnectedRunIds + heteroRunOpsPostgresTest( + "WaitpointPresenter.call renders an empty connected-runs list when the connection join lags", + async ({ prisma14, prisma17 }) => { + // Freeze only the raw connection JOIN: the waitpoint read succeeds, so the caller reaches the + // connection-id gather, which comes back empty under lag. + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, { raw: true }); + const seed = await seedEnvironmentLegacy(prisma14, "conn68"); + const waitpointId = `waitpoint_${CUID_25}`; + const runId = `run_${CUID_25}`; // cuid → LEGACY, co-located with the token + const friendlyId = "waitpoint_conn68"; + await seedManualWaitpoint(legacyStore, { + id: waitpointId, + friendlyId, + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId: "run_conn68", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + await router.blockRunWithWaitpointEdges({ + runId, + waitpointIds: [waitpointId], + projectId: seed.project.id, + }); + holder.env = { + id: seed.environment.id, + organizationId: seed.organization.id, + apiKey: seed.environment.apiKey, + }; + + const laggy = await new WaitpointPresenter(prisma14, prisma14, undefined, router).call({ + friendlyId, + environmentId: seed.environment.id, + projectId: seed.project.id, + }); + + // The token detail renders (found via replica) but the connected-runs list is empty this pass — + // display only, no decision on the stale set. + expect(laggy).not.toBeNull(); + expect(laggy!.id).toBe(friendlyId); + expect(laggy!.connectedRuns).toEqual([]); + expect(legacyReplica.wasHit()).toBe(true); + + // Prove pure lag: over a healthy router the connection id is gathered. + const healthyRouter = buildHealthyRouter(prisma14, prisma17).router; + const connectedIds = await healthyRouter.findWaitpointConnectedRunIds(waitpointId); + expect(connectedIds).toEqual([runId]); + } + ); + + // WaitpointPresenter findRuns + heteroRunOpsPostgresTest( + "WaitpointPresenter.call leaves connected-runs empty when the run lookup lags", + async ({ prisma14, prisma17 }) => { + // Freeze only taskRun reads: the waitpoint read and the connection gather succeed, so the caller + // reaches the friendlyId resolution, which comes back empty. + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, { + taskRun: true, + }); + const seed = await seedEnvironmentLegacy(prisma14, "conn72"); + const waitpointId = `waitpoint_${CUID_25}`; + const runId = `run_${CUID_25}`; + const friendlyId = "waitpoint_conn72"; + await seedManualWaitpoint(legacyStore, { + id: waitpointId, + friendlyId, + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId: "run_conn72", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + await router.blockRunWithWaitpointEdges({ + runId, + waitpointIds: [waitpointId], + projectId: seed.project.id, + }); + holder.env = { + id: seed.environment.id, + organizationId: seed.organization.id, + apiKey: seed.environment.apiKey, + }; + + // Confirm this read is genuinely reached: the connection gather ran (raw NOT frozen); the taskRun + // read is the one that lags. + const connectedIds = await router.findWaitpointConnectedRunIds(waitpointId); + expect(connectedIds).toEqual([runId]); + + const laggy = await new WaitpointPresenter(prisma14, prisma14, undefined, router).call({ + friendlyId, + environmentId: seed.environment.id, + projectId: seed.project.id, + }); + + // Connected-run friendlyId resolution missed under lag → connectedRuns empty this render; + // display only. + expect(laggy).not.toBeNull(); + expect(laggy!.connectedRuns).toEqual([]); + expect(legacyReplica.wasHit()).toBe(true); + + // Prove pure lag: over a healthy router findRuns resolves the run's friendlyId. + const healthyRouter = buildHealthyRouter(prisma14, prisma17).router; + const runs = (await healthyRouter.findRuns({ + where: { id: { in: [runId] } }, + select: { friendlyId: true }, + take: 5, + })) as Array<{ friendlyId: string }>; + expect(runs).toHaveLength(1); + expect(runs[0]!.friendlyId).toBe("run_conn72"); + } + ); + + // WaitpointTagListPresenter findManyWaitpointTags + heteroRunOpsPostgresTest( + "WaitpointTagListPresenter.call omits a just-created tag under replica lag", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, { + waitpointTag: true, + }); + const seed = await seedEnvironmentLegacy(prisma14, "tags"); + await legacyStore.upsertWaitpointTag({ + environmentId: seed.environment.id, + name: "my-tag", + projectId: seed.project.id, + }); + + const laggy = await new WaitpointTagListPresenter(prisma14, prisma14, undefined, router).call( + { environmentId: seed.environment.id } + ); + + // The tag-filter dropdown omits the new tag this render; eventual-consistency list. + expect(laggy.tags).toEqual([]); + expect(legacyReplica.wasHit()).toBe(true); + + const healthy = await new WaitpointTagListPresenter( + prisma14, + prisma14, + undefined, + buildHealthyRouter(prisma14, prisma17).router + ).call({ environmentId: seed.environment.id }); + expect(healthy.tags).toEqual([{ name: "my-tag" }]); + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/retryDecisionReadAfterWrite.replicaLag.test.ts b/internal-packages/run-engine/src/engine/retryDecisionReadAfterWrite.replicaLag.test.ts new file mode 100644 index 000000000..6833f0bcd --- /dev/null +++ b/internal-packages/run-engine/src/engine/retryDecisionReadAfterWrite.replicaLag.test.ts @@ -0,0 +1,185 @@ +// Property: the retry-vs-fail decision must read maxAttempts / lockedRetryConfig from the OWNING +// PRIMARY. Those fields are written at lock time (a recent primary write); served from a lagging +// replica they read null and a run WITH retries remaining is permanently failed instead of retried. +// Freeze the replica at the pre-lock snapshot via the shared `laggingReplica` primitive and drive the +// REAL retryOutcomeFromCompletion: reading via the primary yields "retry", reading via the stale +// replica yields "fail_run" — the replica lag alone is what flips the decision. + +import { laggingReplica, postgresTest } from "@internal/testcontainers"; +import { PostgresRunStore } from "@internal/run-store"; +import type { CreateRunInput } from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { TaskRunError, TaskRunExecutionRetry } from "@trigger.dev/core/v3"; +import { describe, expect } from "vitest"; +import { retryOutcomeFromCompletion } from "./retrying.js"; + +async function seedEnvironment(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "EXECUTING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "EXECUTING", + description: "Run is executing", + runStatus: "EXECUTING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// A plain retriable error (BUILT_IN_ERROR is retriable and not OOM), so the decision falls through +// to the maxAttempts / lockedRetryConfig gates — exactly the read this test targets. +const RETRIABLE_ERROR: TaskRunError = { + type: "BUILT_IN_ERROR", + name: "Error", + message: "boom", + stackTrace: "at handler (task.ts:1:1)", +}; + +// The completion carries explicit retry settings (the normal SDK-failure path), so once the run's +// maxAttempts is read as present the outcome is deterministically "retry" — isolating the read as +// the ONLY thing that flips the decision. +const RETRY_SETTINGS: TaskRunExecutionRetry = { timestamp: Date.now() + 60_000, delay: 60_000 }; + +describe("retry-vs-fail decision under replica lag", () => { + postgresTest( + "a retriable attempt retries when lock-time maxAttempts is read from the primary, not the lagging replica", + async ({ prisma }) => { + const { organization, project, environment } = await seedEnvironment(prisma, "retry_lag"); + const runId = `run_${"a".repeat(24)}`; + + // Build the store the way the engine does; its readOnlyPrisma is a replica (set below). + // The write client is the primary. + const runId_friendly = "run_retrylag"; + const primaryStore = new PostgresRunStore({ + prisma, + readOnlyPrisma: prisma, + schemaVariant: "legacy", + }); + await primaryStore.createRun( + buildCreateRunInput({ + runId, + friendlyId: runId_friendly, + organizationId: organization.id, + projectId: project.id, + runtimeEnvironmentId: environment.id, + }) + ); + + // Snapshot the run as the replica still sees it: PRE-LOCK, so maxAttempts / lockedRetryConfig + // are still null (the run was created but not yet locked to a worker). + const staleRun = await prisma.taskRun.findFirstOrThrow({ where: { id: runId } }); + expect(staleRun.maxAttempts).toBeNull(); + expect(staleRun.lockedRetryConfig).toBeNull(); + + // The attempt is locked: the primary now records maxAttempts + lockedRetryConfig (the recent + // lock-time write). This run has 3 attempts and is on attempt 1 -> it SHOULD retry. + await prisma.taskRun.update({ + where: { id: runId }, + data: { + maxAttempts: 3, + lockedRetryConfig: { + maxAttempts: 3, + minTimeoutInMs: 1000, + maxTimeoutInMs: 10000, + factor: 2, + }, + machinePreset: "small-1x", + usageDurationMs: 100, + costInCents: 0, + }, + }); + + // ...but the replica lags, frozen at the pre-lock snapshot (maxAttempts = null). + const replica = laggingReplica(prisma, [ + { model: "taskRun", mode: "frozen", rows: [staleRun] }, + ]); + + // The store as the engine holds it: reads default to the (lagging) replica. + const store = new PostgresRunStore({ + prisma, + readOnlyPrisma: replica.client, + schemaVariant: "legacy", + }); + + // Mirrors what the engine does — it passes `this.$.readOnlyPrisma` (the REPLICA) as the first + // arg. `readClient` models what the engine threads in. + const decide = (readClient: typeof prisma | typeof replica.client) => + retryOutcomeFromCompletion(readClient, store, { + runId, + attemptNumber: 1, + error: RETRIABLE_ERROR, + retryUsingQueue: false, + retrySettings: RETRY_SETTINGS, + }); + + // Reading the decision on the owning PRIMARY (this.$.prisma) sees maxAttempts = 3, so the run + // retries — the behaviour the engine must get. + const fromPrimary = await decide(prisma); + expect(fromPrimary.outcome).toBe("retry"); + + // Reading the same decision off the lagging replica (this.$.readOnlyPrisma) nulls maxAttempts, so + // a run with retries remaining is permanently FAILED. Pinning fail_run documents the misrouting + // the engine must never do. + const fromReplica = await decide(replica.client); + expect(replica.wasHit("taskRun")).toBe(true); // the decision read hit the stale replica + expect(fromReplica.outcome).toBe("fail_run"); + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/systems/ttlSystemExpireReplicaLag.guard.test.ts b/internal-packages/run-engine/src/engine/systems/ttlSystemExpireReplicaLag.guard.test.ts new file mode 100644 index 000000000..10414f3aa --- /dev/null +++ b/internal-packages/run-engine/src/engine/systems/ttlSystemExpireReplicaLag.guard.test.ts @@ -0,0 +1,154 @@ +// Verifies TtlSystem.expireRunsBatch reads its findRuns existence check off the owning store's WRITER, +// so a PENDING run whose row has not yet replicated is still found and EXPIRED (never orphaned as +// not_found). Because the TTL Lua script has already dequeued the run, a stale replica miss would not +// self-heal. Drives the REAL engine method against a PostgresRunStore whose replica is frozen; real +// Postgres + Redis via containerTest, laggingReplica from @internal/testcontainers, never mocked. + +import { containerTest, laggingReplica } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { expect } from "vitest"; +import { PostgresRunStore } from "@internal/run-store"; +import type { RunStore, CreateRunInput } from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import { RunEngine } from "../index.js"; +import { setupAuthenticatedEnvironment } from "../tests/setup.js"; + +vi.setConfig({ testTimeout: 60_000 }); + +function createEngine(prisma: PrismaClient, redisOptions: any, store: RunStore) { + return new RunEngine({ + prisma, + store, + worker: { + redis: redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: redisOptions, + processWorkerQueueDebounceMs: 50, + masterQueueConsumersDisabled: true, + // Disable the automatic TTL sweep so the ONLY caller of expireRunsBatch is our explicit + // invocation — nothing races it or re-expires the run behind our back. + ttlSystem: { disabled: true }, + }, + runLock: { + redis: redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "PENDING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "PRODUCTION", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_ttl_lag", + spanId: "span_ttl_lag", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + ttl: "1s", + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "QUEUED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "PRODUCTION", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +describe("TtlSystem.expireRunsBatch — read-your-writes under replica lag", () => { + containerTest( + "expires a PENDING run whose row is not yet visible on the lagging replica", + async ({ prisma, redisOptions }) => { + const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + // The store's replica lags: `taskRun` reads come back empty, exactly as just after a + // very-short-TTL run is created and immediately swept. The WRITER (prisma) is up to date. + const replica = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + const store = new PostgresRunStore({ prisma, readOnlyPrisma: replica.client }); + + const engine = createEngine(prisma, redisOptions, store); + + try { + // Seed the run directly through the store's WRITER so its row exists on the primary but is NOT + // visible on the lagging replica. (Created directly rather than via engine.trigger so that + // setup performs no replica reads that could interfere with the single read under test.) + const runId = "run_ttl_lag_guard_0000001"; + await store.createRun( + buildCreateRunInput({ + runId, + friendlyId: "run_ttllag1", + organizationId: authenticatedEnvironment.organization.id, + projectId: authenticatedEnvironment.project.id, + runtimeEnvironmentId: authenticatedEnvironment.id, + }) + ); + + // Sanity: the run really IS an expirable PENDING run on the writer, so the only reason it + // could fail to expire is a misrouted read against the stale replica. + const onPrimary = await store.findRun({ id: runId }, prisma); + expect(onPrimary).not.toBeNull(); + expect(onPrimary!.status).toBe("PENDING"); + + // Drive the REAL engine method. Its findRuns read is handed the writer (this.$.prisma) and + // finds the run rather than hitting the lagging replica. + const result = await engine.ttlSystem.expireRunsBatch([runId]); + + // The run is expired and nothing is skipped. + expect(result.expired).toEqual([runId]); + expect(result.skipped).toEqual([]); + + // The run is durably EXPIRED on the writer (the persisted outcome under test). + const persisted = await prisma.taskRun.findUniqueOrThrow({ where: { id: runId } }); + expect(persisted.status).toBe("EXPIRED"); + + // The existence-check read routes to the writer, so the stale replica is NEVER consulted for the + // expiry check — the seam these assertions pin. + expect(replica.wasHit("taskRun")).toBe(false); + } finally { + await engine.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/tests/concurrencySweeperCallbackReplicaLag.guard.test.ts b/internal-packages/run-engine/src/engine/tests/concurrencySweeperCallbackReplicaLag.guard.test.ts new file mode 100644 index 000000000..4dac600ed --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/concurrencySweeperCallbackReplicaLag.guard.test.ts @@ -0,0 +1,122 @@ +// Replica-lag guard for the run-store read inside the private #concurrencySweeperCallback. The +// callback cannot be called by name (it is private), but the engine wires the REAL bound method as +// the RunQueue's concurrency-sweeper callback (`callback: this.#concurrencySweeperCallback.bind(this)`) +// and `RunQueue.options` is public-readonly, so this test invokes +// `engine.runQueue.options.concurrencySweeper.callback([...])` — the exact production callback object, +// driven end-to-end over the engine's real runStore. RunQueue's own Redis scan is the only thing +// bypassed (it merely feeds the callback the runIds it already receives from +// `processCurrentConcurrencyRunIds`). +// +// The read: findRuns for runs finished > 10 min ago with a final status and an org set (client-less → +// REPLICA); the callback releases the stale concurrency those runs still hold. +// +// Property: this lag is tolerable and self-healing. Under lag the read misses the finished run and the +// callback returns [] (no runs marked for ack), so the run keeps its concurrency slot THIS scan; the +// sweeper runs periodically, so the next scan (once replicated) releases it — a miss delays cleanup by +// one scan, never wrongly releases a live run, and never leaks permanently. A second engine over a +// caught-up replica proves the SAME wired callback returns the run. + +import { containerTest, laggingReplica } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { PostgresRunStore } from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; +import { RunEngine } from "../index.js"; +import { setupAuthenticatedEnvironment } from "./setup.js"; + +function baseEngineOptions(redisOptions: any) { + return { + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x" as const, + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }; +} + +async function seedFinishedRun(prisma: PrismaClient, environment: any, runId: string) { + // Completed 30 min ago, final status, org set — matches the callback's predicate exactly. On the PRIMARY. + await prisma.taskRun.create({ + data: { + id: runId, + engine: "V2", + status: "COMPLETED_SUCCESSFULLY", + friendlyId: `fr_${runId}`, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceId: `trace_${runId}`, + spanId: `span_${runId}`, + queue: "task/my-task", + runtimeEnvironmentId: environment.id, + projectId: environment.project.id, + organizationId: environment.organization.id, + environmentType: "PRODUCTION", + createdAt: new Date(Date.now() - 60 * 60 * 1000), + completedAt: new Date(Date.now() - 30 * 60 * 1000), + }, + }); +} + +describe("concurrency sweeper callback — replica-lag guard", () => { + containerTest( + "the wired sweeper callback misses a finished run under lag (returns []) — self-healing; a caught-up replica proves the same callback finds it", + async ({ prisma, redisOptions }) => { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const runId = "run_nnnnnnnnnnnnnnnnnnnnnn91"; + await seedFinishedRun(prisma, environment, runId); + + // Engine A — taskRun frozen-missing on the replica; the callback's client-less findRuns routes there. + const laggingReplicaHandle = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + const laggingStore = new PostgresRunStore({ + prisma, + readOnlyPrisma: laggingReplicaHandle.client as never, + }); + const laggingEngine = new RunEngine({ + prisma, + readOnlyPrisma: laggingReplicaHandle.client as never, + store: laggingStore, + ...baseEngineOptions(redisOptions), + }); + + // Engine B — caught-up replica (control), proving the wired callback path is live. + const liveStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const liveEngine = new RunEngine({ + prisma, + readOnlyPrisma: prisma, + store: liveStore, + ...baseEngineOptions(redisOptions), + }); + + try { + // Lagging: the REAL wired callback (engine's private #concurrencySweeperCallback) misses the + // finished run and returns [] — the run keeps its concurrency slot this scan (no throw). + const lagging = await laggingEngine.runQueue.options.concurrencySweeper!.callback([runId]); + expect(lagging).toEqual([]); + expect(laggingReplicaHandle.wasHit("taskRun")).toBe(true); + + // Caught-up control: the SAME wired callback finds the finished run and marks it for ack. + const caughtUp = await liveEngine.runQueue.options.concurrencySweeper!.callback([runId]); + expect(caughtUp.map((r) => r.id)).toEqual([runId]); + expect(caughtUp[0]!.orgId).toBe(environment.organization.id); + + // The miss is pure lag: the run is genuinely finished on the PRIMARY (so a later scan releases it). + const onPrimary = await prisma.taskRun.findFirstOrThrow({ where: { id: runId } }); + expect(onPrimary.status).toBe("COMPLETED_SUCCESSFULLY"); + } finally { + await laggingEngine.quit(); + await liveEngine.quit(); + } + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/tests/runAttemptSystemReplicaLag.guard.test.ts b/internal-packages/run-engine/src/engine/tests/runAttemptSystemReplicaLag.guard.test.ts new file mode 100644 index 000000000..1ebabdc82 --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/runAttemptSystemReplicaLag.guard.test.ts @@ -0,0 +1,516 @@ +// Property: the completion-path reads in runAttemptSystem hit the owning primary, not a lagging replica. +// These drive the REAL RunEngine methods end-to-end (trigger → dequeue → startRunAttempt → +// complete/cancel), not a reimplementation. The store is a real PostgresRunStore whose primary is the +// live container and whose replica is a lagging proxy serving a STALE row for exactly the read under +// test. The same proxy is the engine's readOnlyPrisma, so a read routed to the owning primary +// (this.$.prisma / findRunOnPrimary) never touches the proxy, while a replica-routed read +// (this.$.readOnlyPrisma / client-less findRun) hits the stale value. Unlike the retry test that passes +// its own client into retryOutcomeFromCompletion, this exercises WHICH client the engine threads. + +import { assertNonNullable, containerTest, laggingReplica } from "@internal/testcontainers"; +import { trace } from "@internal/tracing"; +import { PostgresRunStore } from "@internal/run-store"; +import type { PrismaClient } from "@trigger.dev/database"; +import { setTimeout } from "node:timers/promises"; +import { describe, expect } from "vitest"; +import { RunEngine } from "../index.js"; +import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js"; + +// A read-replica that lags its primary for exactly two completion-path reads, matched by the engine's +// own `select` shape so no unrelated read is disturbed. `retryStale` fabricates the pre-lock snapshot +// (maxAttempts/lockedRetryConfig still null) for the retry-decision read; `usageStale` fabricates a +// pre-accumulation snapshot (usage totals still at their earlier value) for the usage RMW read. Every +// other property/method forwards to the real client. `hits` proves whether the lagging replica was +// actually consulted (0 when the reads go to the primary; >0 when they hit the replica). +type LagState = { + retryStale: boolean; + usageStale: { usageDurationMs: number; costInCents: number; machinePreset: string | null } | null; +}; + +function laggingReadReplica( + real: C +): { client: C; state: LagState; hits: { retry: number; usage: number } } { + const state: LagState = { retryStale: false, usageStale: null }; + const hits = { retry: 0, usage: 0 }; + + const isRetryDecisionSelect = (select: any) => + !!select && select.maxAttempts === true && select.lockedRetryConfig === true; + + // The usage RMW read (attemptSucceeded / cancelRun / permanentlyFailRun) selects exactly these three + // scalars and nothing else — and crucially never maxAttempts, which distinguishes it from the retry + // read above. + const isUsageSelect = (select: any) => + !!select && + select.usageDurationMs === true && + select.costInCents === true && + select.machinePreset === true && + !("maxAttempts" in select) && + !("lockedRetryConfig" in select); + + const realTaskRun = (real as any).taskRun; + + const taskRunProxy = new Proxy(realTaskRun, { + get(target, prop) { + if (prop === "findFirst") { + return async (args?: { select?: any }) => { + const select = args?.select; + if (state.retryStale && isRetryDecisionSelect(select)) { + hits.retry++; + // Replica has not applied the lock-time write yet: no retry config visible. + return { + maxAttempts: null, + lockedRetryConfig: null, + usageDurationMs: 0, + costInCents: 0, + machinePreset: null, + }; + } + if (state.usageStale && isUsageSelect(select)) { + hits.usage++; + // Replica has not applied the accumulated-usage write yet: the earlier totals. + return { ...state.usageStale }; + } + return target.findFirst(args); + }; + } + const value = target[prop]; + return typeof value === "function" ? value.bind(target) : value; + }, + }); + + const client = new Proxy(real, { + get(target, prop) { + if (prop === "taskRun") { + return taskRunProxy; + } + const value = (target as any)[prop]; + return typeof value === "function" ? value.bind(target) : value; + }, + }) as C; + + return { client, state, hits }; +} + +function baseEngineOptions(redisOptions: any) { + return { + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x" as const, + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }; +} + +const triggerParams = (friendlyId: string, environment: any, taskIdentifier: string) => ({ + number: 1, + friendlyId, + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: "t12345", + spanId: "s12345", + workerQueue: "main", + queue: `task/${taskIdentifier}`, + isTest: false, + tags: [] as string[], +}); + +// Drive the real engine to a locked, EXECUTING first attempt over a store whose replica is the +// lagging proxy. Returns everything the individual guards need. +async function triggerAndStartAttempt( + prisma: PrismaClient, + redisOptions: any, + friendlyId: string, + retryOptions?: any +) { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const replica = laggingReadReplica(prisma); + const store = new PostgresRunStore({ + prisma, + readOnlyPrisma: replica.client as never, + }); + const engine = new RunEngine({ + prisma, + readOnlyPrisma: replica.client as never, + store, + ...baseEngineOptions(redisOptions), + }); + + try { + const taskIdentifier = "test-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier, undefined, retryOptions); + + const run = await engine.trigger( + triggerParams(friendlyId, environment, taskIdentifier), + prisma + ); + + await setTimeout(500); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "test_guard", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + const attempt = await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: dequeued[0].snapshot.id, + }); + + return { engine, environment, run, attempt, store, replica }; + } catch (e) { + // Setup threw after the engine was constructed: quit it here so a failed helper does not leak the + // engine (the callers only run `await engine.quit()` on the success path). + await engine.quit(); + throw e; + } +} + +describe("RunAttemptSystem completion-path reads must hit the owning primary, not a lagging replica", () => { + // Retry-vs-fail decision. A retriable failure with attempts remaining must RETRY. The retry config + // (maxAttempts / lockedRetryConfig) is a lock-time write; if the decision read is served by a lagging + // replica it reads null and the run is permanently FAILED instead. Reading via the owning primary + // (this.$.prisma) sees the config and the run retries; a replica-routed read fails it. + containerTest( + "attemptFailed retries a retriable run whose lock-time retry config has not replicated", + async ({ prisma, redisOptions }) => { + const { engine, run, attempt, replica } = await triggerAndStartAttempt( + prisma, + redisOptions, + "run_nnnnnnnnnnnnnnnnnnnnnnnn01" + ); + + try { + // The primary now holds maxAttempts=3 (set at lock time). Freeze the replica at the pre-lock + // snapshot: the ONLY thing that could flip the outcome is which client serves the decision read. + const primaryRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(primaryRun.maxAttempts).toBe(3); + replica.state.retryStale = true; + + const result = await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: false, + id: run.id, + error: { + type: "BUILT_IN_ERROR", + name: "UserError", + message: "boom", + stackTrace: "Error: boom\n at :1:1", + }, + retry: { timestamp: Date.now(), delay: 0 }, + }, + }); + + // Decision read hit the primary → maxAttempts=3 seen → the run retries. + expect(result.attemptStatus).toBe("RETRY_IMMEDIATELY"); + expect(result.snapshot.executionStatus).toBe("EXECUTING"); + expect(result.run.status).toBe("EXECUTING"); + // The lagging replica was never consulted for the decision (a replica-routed read would make this > 0). + expect(replica.hits.retry).toBe(0); + } finally { + await engine.quit(); + } + } + ); + + // attemptSucceeded usage/cost RMW. The cumulative usage total is a read-modify-write: read current + // total, add this attempt, persist. If the read is served by a lagging replica it misses a + // just-persisted earlier-attempt total and the final usage undercounts. Reading via findRunOnPrimary + // accumulates on the primary total; a replica-routed read undercounts. + containerTest( + "attemptSucceeded accumulates usage on top of the primary total, not a stale replica total", + async ({ prisma, redisOptions }) => { + const { engine, run, attempt, replica } = await triggerAndStartAttempt( + prisma, + redisOptions, + "run_nnnnnnnnnnnnnnnnnnnnnnnn11" + ); + + try { + // Simulate an earlier attempt's usage already persisted on the PRIMARY (1000ms) that the + // lagging replica has not yet applied (it still reports 0). + await prisma.taskRun.update({ + where: { id: run.id }, + data: { usageDurationMs: 1000, costInCents: 5, machinePreset: "small-1x" }, + }); + replica.state.usageStale = { + usageDurationMs: 0, + costInCents: 0, + machinePreset: "small-1x", + }; + + const result = await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: true, + id: run.id, + output: `{"ok":true}`, + outputType: "application/json", + usage: { durationMs: 500 }, + }, + }); + expect(result.run.status).toBe("COMPLETED_SUCCESSFULLY"); + + // Primary read: 1000 (primary) + 500 (this attempt) = 1500. A replica read would see 0 → 500 (undercount). + const finalRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(finalRun.usageDurationMs).toBe(1500); + expect(replica.hits.usage).toBe(0); + } finally { + await engine.quit(); + } + } + ); + + // #permanentlyFailRun usage/cost RMW. A non-retriable failure permanently fails the run and still + // accumulates the final attempt's usage. Reached with a non-retriable error so + // retryOutcomeFromCompletion short-circuits to fail_run BEFORE its own read — isolating this to the + // #permanentlyFailRun usage read. Reading via findRunOnPrimary accumulates on the primary total; a + // replica-routed read undercounts. + containerTest( + "permanentlyFailRun accumulates usage on top of the primary total, not a stale replica total", + async ({ prisma, redisOptions }) => { + const { engine, run, attempt, replica } = await triggerAndStartAttempt( + prisma, + redisOptions, + "run_nnnnnnnnnnnnnnnnnnnnnnnn21" + ); + + try { + await prisma.taskRun.update({ + where: { id: run.id }, + data: { usageDurationMs: 1000, costInCents: 5, machinePreset: "small-1x" }, + }); + replica.state.usageStale = { + usageDurationMs: 0, + costInCents: 0, + machinePreset: "small-1x", + }; + + const result = await engine.completeRunAttempt({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: false, + id: run.id, + // Non-retriable internal error → straight to fail_run → #permanentlyFailRun. + error: { type: "INTERNAL_ERROR", code: "DISK_SPACE_EXCEEDED" }, + usage: { durationMs: 500 }, + }, + }); + expect(result.attemptStatus).toBe("RUN_FINISHED"); + expect(result.snapshot.executionStatus).toBe("FINISHED"); + + // Primary read: 1000 + 500 = 1500. A replica read would see 0 → 500 (undercount). + const finalRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(finalRun.usageDurationMs).toBe(1500); + expect(replica.hits.usage).toBe(0); + } finally { + await engine.quit(); + } + } + ); + + // cancelRun usage/cost RMW. Cancelling a run with attempt-duration data accumulates that usage. Driven + // through the real RunAttemptSystem.cancelRun (the engine's public cancelRun does not expose + // attemptDurationMs, so the real method is invoked via engine.runAttemptSystem). Reading via + // findRunOnPrimary accumulates on the primary total; a replica-routed read undercounts. + containerTest( + "cancelRun accumulates usage on top of the primary total, not a stale replica total", + async ({ prisma, redisOptions }) => { + const { engine, run, replica } = await triggerAndStartAttempt( + prisma, + redisOptions, + "run_nnnnnnnnnnnnnnnnnnnnnnnn31" + ); + + try { + await prisma.taskRun.update({ + where: { id: run.id }, + data: { usageDurationMs: 1000, costInCents: 5, machinePreset: "small-1x" }, + }); + replica.state.usageStale = { + usageDurationMs: 0, + costInCents: 0, + machinePreset: "small-1x", + }; + + const result = await engine.runAttemptSystem.cancelRun({ + runId: run.id, + reason: "guard test cancel", + finalizeRun: true, + attemptDurationMs: 500, + }); + assertNonNullable(result); + + // Primary read: 1000 + 500 = 1500. A replica read would see 0 → 500 (undercount). + const finalRun = await prisma.taskRun.findFirstOrThrow({ where: { id: run.id } }); + expect(finalRun.usageDurationMs).toBe(1500); + expect(replica.hits.usage).toBe(0); + } finally { + await engine.quit(); + } + } + ); +}); + +// engine-other read sites — two reads (resolveTaskRunContext, the attemptFailed forceRequeue re-read) +// that must hit the owning primary. Unlike the completion-path cases above (which fabricate STALE reads +// for two `select` shapes via `laggingReadReplica`), these need the whole taskRun row frozen-MISSING on +// the replica, so they use the shared `laggingReplica` ("missing") primitive behind a gate: it forwards +// to the live client during setup so the run reaches EXECUTING, then flips to the frozen replica for +// exactly the read under test. `wasHit()` proves whether the frozen replica was consulted for that read. +function gatedLaggingReplica(prisma: PrismaClient) { + const frozen = { on: false }; + const lagging = laggingReplica(prisma, [{ model: "taskRun", mode: "missing" }]); + const client = new Proxy(prisma, { + get(_t, prop) { + const src: any = frozen.on ? lagging.client : prisma; + const value = src[prop]; + return typeof value === "function" ? value.bind(src) : value; + }, + }) as unknown as PrismaClient; + return { client, frozen, wasHit: (m?: string) => lagging.wasHit(m) }; +} + +// Drive the real engine to a locked, EXECUTING first attempt over a store whose replica is the gated +// frozen-missing proxy. Mirrors `triggerAndStartAttempt` above but returns the gated replica handle. +async function triggerAndStartAttemptGated( + prisma: PrismaClient, + redisOptions: any, + friendlyId: string, + retryOptions?: any +) { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const replica = gatedLaggingReplica(prisma); + const store = new PostgresRunStore({ + prisma, + readOnlyPrisma: replica.client as never, + }); + const engine = new RunEngine({ + prisma, + readOnlyPrisma: replica.client as never, + store, + ...baseEngineOptions(redisOptions), + }); + + try { + const taskIdentifier = "test-task"; + await setupBackgroundWorker(engine, environment, taskIdentifier, undefined, retryOptions); + + const run = await engine.trigger( + triggerParams(friendlyId, environment, taskIdentifier), + prisma + ); + + await setTimeout(500); + const dequeued = await engine.dequeueFromWorkerQueue({ + consumerId: "test_guard", + workerQueue: "main", + }); + expect(dequeued.length).toBe(1); + + const attempt = await engine.startRunAttempt({ + runId: dequeued[0].run.id, + snapshotId: dequeued[0].snapshot.id, + }); + + return { engine, environment, run, attempt, store, replica }; + } catch (e) { + // Setup threw after the engine was constructed: quit it here so a failed helper does not leak the + // engine (the callers only run `await engine.quit()` on the success path). + await engine.quit(); + throw e; + } +} + +describe("RunEngine engine-other reads must hit the owning primary, not a lagging replica", () => { + // resolveTaskRunContext. Under lag the run's row is not visible on the replica, so a client-less + // findRun returns null and resolveTaskRunContext throws ServiceValidationError("Task run not found", + // 404) for a LIVE, EXECUTING run — a spurious 404 on the SpanPresenter's context read. Reading via + // findRunOnPrimary hits the primary and resolves the context; a replica-routed read 404s. + containerTest( + "resolveTaskRunContext resolves a live executing run whose row has not replicated", + async ({ prisma, redisOptions }) => { + const { engine, run, replica } = await triggerAndStartAttemptGated( + prisma, + redisOptions, + "run_nnnnnnnnnnnnnnnnnnnnnn73" + ); + try { + // Freeze the owning replica for the taskRun read: the ONLY thing that can flip the outcome is + // which client serves resolveTaskRunContext's read. + replica.frozen.on = true; + const ctx = await engine.resolveTaskRunContext(run.id); + + // The read routed to the primary → the live run's context resolved (friendlyId echoed), and the + // frozen replica was never consulted for the taskRun read. + expect(ctx.run.id).toBe(run.friendlyId); + expect(replica.wasHit("taskRun")).toBe(false); + } finally { + await engine.quit(); + } + } + ); + + // attemptFailed(forceRequeue) re-read. The forceRequeue branch re-reads the run (status/spanId/... for + // the runAttemptFailed event). Under lag a client-less findRun returns null and attemptFailed throws + // ServiceValidationError("Run not found", 404), aborting the crash-requeue of a LIVE run. Reading via + // findRunOnPrimary hits the primary; a replica-routed read 404s. + containerTest( + "attemptFailed(forceRequeue) re-reads a live run on the primary", + async ({ prisma, redisOptions }) => { + const { engine, run, attempt, replica } = await triggerAndStartAttemptGated( + prisma, + redisOptions, + "run_nnnnnnnnnnnnnnnnnnnnnn83", + { maxAttempts: 3 } + ); + try { + replica.frozen.on = true; + const result = await engine.runAttemptSystem.attemptFailed({ + runId: run.id, + snapshotId: attempt.snapshot.id, + completion: { + ok: false, + id: run.id, + error: { + type: "BUILT_IN_ERROR", + name: "UserError", + message: "boom", + stackTrace: "Error: boom\n at :1:1", + }, + retry: { timestamp: Date.now(), delay: 0 }, + }, + forceRequeue: true, + tx: prisma, + }); + + // The forceRequeue re-read hit the primary → attemptFailed completed (no 404) and returned a + // live result for this run; the frozen replica was never consulted for the taskRun read. + expect(result.run.id).toBe(run.id); + expect(result.attemptStatus).toBeDefined(); + expect(replica.wasHit("taskRun")).toBe(false); + } finally { + await engine.quit(); + } + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.batchDependentAttemptReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.batchDependentAttemptReadView.replicaLag.test.ts new file mode 100644 index 000000000..281559c10 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.batchDependentAttemptReadView.replicaLag.test.ts @@ -0,0 +1,251 @@ +// Property: the batchTriggerV3 dependent-attempt read tolerates a miss under replica lag. The +// client-less findTaskRunAttempt resolves the PARENT run's attempt (by attempt friendlyId, env-scoped) +// to feed a terminal-state pre-check (throw ServiceValidationError if the parent attempt/run is +// terminal). With no `taskRunId` in the where the router fans out UNROUTED (NEW→LEGACY), each leg with a +// null client → readOnlyPrisma → genuinely REPLICA-routed (never defaults to primary). +// +// Under owning-replica lag the read returns null. Tolerated: the parent attempt is a long-committed row +// (the parent is mid-execution when it triggers the batch — never a read-your-write), and its only +// consumer is a best-effort TOCTOU guard that is racy even against the primary and whose resume path is +// a no-op on an already-terminal parent. A null/stale read skips the guard but drives no wrong outcome. +// +// Builds the router as the webapp holds it, seeds the parent run + attempt on the owning LEGACY primary, +// freezes the LEGACY replica with the shared laggingReplica, invokes findTaskRunAttempt with the EXACT +// caller args and NO client. + +import { heteroPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CreateRunInput } from "./types.js"; + +// A cuid (25 chars after the `run_` prefix) classifies LEGACY, so the parent run + attempt are owned +// by the legacy (control-plane) store. The env-scoped, taskRunId-free read then fans out NEW→LEGACY. +const CUID_25 = "c".repeat(25); + +async function seedEnvironment(prisma: PrismaClient, slugSuffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +// Full CreateRunInput for the parent run (mirrors the routing test's builder) — used via +// store.createRun so we don't hand-maintain the TaskRun column list. +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "EXECUTING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "parent-task", + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: `trace_${params.runId}`, + spanId: `span_${params.runId}`, + queue: "task/parent-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "EXECUTING", + description: "Run is executing", + runStatus: "EXECUTING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// Seed a TaskRunAttempt with FK triggers disabled. `session_replication_role` is session-scoped, so the +// `SET` and the insert must share one connection — separate pooled calls can split across connections, +// leaving the insert with FK triggers on. One transaction with `SET LOCAL` keeps them co-connected. +async function seedAttempt( + prisma: PrismaClient, + opts: { + attemptId: string; + friendlyId: string; + runId: string; + runtimeEnvironmentId: string; + status: string; + } +) { + await prisma.$transaction(async (tx) => { + await tx.$executeRawUnsafe(`SET LOCAL session_replication_role = replica`); + await tx.$executeRawUnsafe( + `INSERT INTO "TaskRunAttempt" (id, number, "friendlyId", "taskRunId", "backgroundWorkerId", "backgroundWorkerTaskId", "runtimeEnvironmentId", "queueId", status, "createdAt", "updatedAt", "usageDurationMs", "outputType") + VALUES ($1, 1, $2, $3, 'synthetic-worker', 'synthetic-worker-task', $4, 'synthetic-queue', $5::"TaskRunAttemptStatus", NOW(), NOW(), 0, 'application/json')`, + opts.attemptId, + opts.friendlyId, + opts.runId, + opts.runtimeEnvironmentId, + opts.status + ); + }); +} + +// Mirror of dependentAttemptScope#dependentAttemptWhere — replicated here to keep this test free of a +// webapp import while matching the exact caller shape. +function dependentAttemptWhere(friendlyId: string, environmentId: string) { + return { friendlyId, taskRun: { runtimeEnvironmentId: environmentId } } as const; +} + +// The EXACT args the batchTriggerV3 caller passes (where builder + include), for a given attempt. +function callSiteArgs(attemptFriendlyId: string, environmentId: string) { + return { + where: dependentAttemptWhere(attemptFriendlyId, environmentId), + include: { taskRun: { select: { id: true, status: true } } }, + } as const; +} + +describe("batchTriggerV3 dependentAttempt read — findTaskRunAttempt(no client) under replica lag", () => { + heteroPostgresTest( + "env-scoped dependent-attempt read returns null under replica lag, skipping the terminal-state pre-check", + async ({ prisma14, prisma17 }) => { + // LEGACY store's replica is FROZEN; its primary (prisma14) is real. + const legacyReplica = laggingReplica(prisma14, [ + { model: "taskRunAttempt", mode: "missing" }, + ]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + }); + // NEW store non-lagging (and holds no attempt) — the fan-out probes it first and misses. + const newStore = new PostgresRunStore({ prisma: prisma17, readOnlyPrisma: prisma17 }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, "batch_dep_lag"); + const parentRunId = `run_${CUID_25}`; // cuid → LEGACY-owned + await legacyStore.createRun( + buildCreateRunInput({ + runId: parentRunId, + friendlyId: "run_batch_dep_parent", + runtimeEnvironmentId: seed.environment.id, + organizationId: seed.organization.id, + projectId: seed.project.id, + }) + ); + + const attemptId = "attempt_batch_dep_lag"; + const attemptFriendlyId = "attempt_batch_dep_lag_f"; + // Parent attempt is COMPLETED — a TERMINAL attempt status. If the read saw it, the caller WOULD + // throw the ServiceValidationError. This makes the tolerance question sharp. + await seedAttempt(prisma14, { + attemptId, + friendlyId: attemptFriendlyId, + runId: parentRunId, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED", + }); + + const args = callSiteArgs(attemptFriendlyId, seed.environment.id); + + // HAZARD proof: the call site's exact invocation (no client) misses the attempt under lag. + const underLag = await router.findTaskRunAttempt(args); + expect(underLag).toBeNull(); + // Prove the read was genuinely REPLICA-routed (the frozen owning replica was consulted). + expect(legacyReplica.wasHit()).toBe(true); + + // ROUTING proof (replica, not primary): the SAME store, SAME args, but a WRITER client forces the + // owning primary (#ownPrimary), which sees the row. So the null above is purely replica lag + + // replica routing — not a missing row or a bad query. + const onPrimary = await router.findTaskRunAttempt(args, prisma14); + expect(onPrimary?.id).toBe(attemptId); + expect(onPrimary?.status).toBe("COMPLETED"); + expect(onPrimary?.taskRun.id).toBe(parentRunId); + + // TOLERANCE assertion: reproduce the caller's `dependentAttempt ? throw-if-terminal : proceed` + // branch. Under lag dependentAttempt is null, so the terminal-state guard is SKIPPED. + const dependentAttempt = underLag as { status: string; taskRun: { status: string } } | null; + const wouldThrowTerminal = + !!dependentAttempt && + (["COMPLETED", "FAILED", "CANCELED"].includes(dependentAttempt.status) || + ["COMPLETED_SUCCESSFULLY", "COMPLETED_WITH_ERRORS", "CANCELED", "FAILED"].includes( + dependentAttempt.taskRun.status + )); + expect(wouldThrowTerminal).toBe(false); // guard skipped under lag — tolerated, not a wrong outcome + } + ); + + heteroPostgresTest( + "steady state: the read returns the terminal parent attempt from a caught-up replica", + async ({ prisma14, prisma17 }) => { + // Non-lagging: the legacy store reads its real replica (prisma14 itself as the replica handle). + const legacyStore = new PostgresRunStore({ prisma: prisma14, readOnlyPrisma: prisma14 }); + const newStore = new PostgresRunStore({ prisma: prisma17, readOnlyPrisma: prisma17 }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, "batch_dep_ok"); + const parentRunId = `run_${CUID_25}`; + await legacyStore.createRun( + buildCreateRunInput({ + runId: parentRunId, + friendlyId: "run_batch_dep_parent_ok", + runtimeEnvironmentId: seed.environment.id, + organizationId: seed.organization.id, + projectId: seed.project.id, + }) + ); + + const attemptId = "attempt_batch_dep_ok"; + const attemptFriendlyId = "attempt_batch_dep_ok_f"; + await seedAttempt(prisma14, { + attemptId, + friendlyId: attemptFriendlyId, + runId: parentRunId, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED", + }); + + const found = await router.findTaskRunAttempt( + callSiteArgs(attemptFriendlyId, seed.environment.id) + ); + expect(found?.id).toBe(attemptId); + expect(found?.status).toBe("COMPLETED"); + expect(found?.taskRun.id).toBe(parentRunId); + // Env-scope enforced: a wrong environment id in the where must not resolve the foreign attempt. + const wrongEnv = await router.findTaskRunAttempt( + callSiteArgs(attemptFriendlyId, "env_does_not_exist") + ); + expect(wrongEnv).toBeNull(); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.batchIdempotencyDedupReadAfterWrite.test.ts b/internal-packages/run-store/src/runOpsStore.batchIdempotencyDedupReadAfterWrite.test.ts new file mode 100644 index 000000000..32686f6f8 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.batchIdempotencyDedupReadAfterWrite.test.ts @@ -0,0 +1,161 @@ +// Property: the client-less batch idempotency dedup probe must resolve attempt-1's just-written batch +// from the OWNING store's WRITER, not its lagging replica. batchTriggerV3 dedups via +// findBatchTaskRunByIdempotencyKey(environment.id, idempotencyKey) with NO client, so through the +// router each leg resolves via the owning store's default. Seed attempt-1's batch on the owning WRITER, +// freeze that store's replica (mode "missing") on the real split topology (heteroRunOpsPostgresTest), +// issue the client-less probe, and assert it finds the batch with the replica never consulted — proven +// for BOTH residencies. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { RunStoreSchemaVariant } from "./types.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +// ownerEngine (classifyResidency) routes a run-ops v1 body → NEW, everything else → LEGACY. +const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema) + +// On the dedicated subset there are no Organization/Project/RuntimeEnvironment models (the run-ops +// rows carry FK-free scalar ids), so we mint synthetic owning ids. On legacy we seed the real rows +// the kept FKs require. +async function seedEnvironment( + prisma: AnyClient, + schemaVariant: RunStoreSchemaVariant, + slugSuffix: string +) { + if (schemaVariant === "dedicated") { + return { + organization: { id: `org_${slugSuffix}` }, + project: { id: `proj_${slugSuffix}` }, + environment: { id: `env_${slugSuffix}` }, + }; + } + const organization = await (prisma as PrismaClient).organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await (prisma as PrismaClient).project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await (prisma as PrismaClient).runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +// Seed attempt-1's batch (user-provided idempotencyKey) on the WRITER, exactly as batchTriggerV3's +// create arm would have written it on the first attempt. +async function seedBatch( + store: PostgresRunStore, + params: { id: string; friendlyId: string; environmentId: string; idempotencyKey: string } +) { + await store.createBatchTaskRun({ + id: params.id, + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.environmentId, + idempotencyKey: params.idempotencyKey, + idempotencyKeyExpiresAt: new Date(Date.now() + 60 * 60_000), + }); +} + +describe("run-ops split — batch idempotency dedup reads the OWNING store's WRITER, not its lagging replica", () => { + // The dedup probe issued by batchTriggerV3 on a RETRY: + // findBatchTaskRunByIdempotencyKey(environmentId, idempotencyKey) // NO client + // must resolve attempt-1's just-written batch, so the retry returns the cached batch instead of + // triggering a duplicate one. Proven for BOTH residencies. + + // (a) LEGACY-resident (cuid) batch: attempt-1's batch was committed to the control-plane writer; the + // control-plane replica lags. The client-less dedup must find it on the owning (legacy) store. + heteroRunOpsPostgresTest( + "LEGACY cuid: client-less dedup finds attempt-1's batch despite replica lag", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "batchTaskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, "legacy", "batch_leg"); + const idempotencyKey = "batch-idem-legacy"; + await seedBatch(legacyStore, { + id: `batch_${CUID_25}`, // cuid → LEGACY + friendlyId: "batch_batch_leg", + environmentId: seed.environment.id, + idempotencyKey, + }); + + // The exact dedup probe: no client. + const found = await router.findBatchTaskRunByIdempotencyKey( + seed.environment.id, + idempotencyKey + ); + + // Resolved on the owning store's writer; the lagging replica must never be consulted. + expect(found).not.toBeNull(); + expect(found!.idempotencyKey).toBe(idempotencyKey); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + } + ); + + // (b) NEW-resident (run-ops id) batch on the dedicated subset schema: the NEW replica lags. The + // client-less dedup must resolve on the NEW writer. + heteroRunOpsPostgresTest( + "NEW run-ops id: client-less dedup finds attempt-1's batch despite NEW replica lag", + async ({ prisma14, prisma17 }) => { + const newReplica = laggingReplica(prisma17, [{ model: "batchTaskRun", mode: "missing" }]); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: newReplica.client as never, + schemaVariant: "dedicated", + }); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma17, "dedicated", "batch_new"); + const idempotencyKey = "batch-idem-new"; + await seedBatch(newStore, { + id: `batch_${generateRunOpsId()}`, // run-ops id → NEW + friendlyId: "batch_batch_new", + environmentId: seed.environment.id, + idempotencyKey, + }); + + const found = await router.findBatchTaskRunByIdempotencyKey( + seed.environment.id, + idempotencyKey + ); + + expect(found).not.toBeNull(); + expect(found!.idempotencyKey).toBe(idempotencyKey); + expect(newReplica.wasHit("batchTaskRun")).toBe(false); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.bulkActionReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.bulkActionReadView.replicaLag.test.ts new file mode 100644 index 000000000..392f5a132 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.bulkActionReadView.replicaLag.test.ts @@ -0,0 +1,253 @@ +// Lagging-replica coverage for BULK ACTION member hydration on the run-ops split. BulkActionV2's +// CANCEL and REPLAY branches hydrate each batch's runs via a client-less id-set findRuns, so the bounded +// id-set path reads each OWNING store's REPLICA: a run on its owning PRIMARY but not yet on that store's +// REPLICA is dropped → a PARTIAL merge. Both are tolerated: the id set comes from a ClickHouse list fed +// DOWNSTREAM of Postgres, so a not-yet-replicated run isn't in ClickHouse yet and lands in a later page. +// The tests FORCE the (in-practice-unreachable) stale window at the store, prove the merge drops the +// lagging-store row, and prove that row really exists on the primary. +// Real split topology via heteroRunOpsPostgresTest — NEVER mocked. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CreateRunInput, RunStoreSchemaVariant } from "./types.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +// classifyResidency routes a run-ops v1 body → NEW, everything else → LEGACY. +const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema) + +async function seedEnvironment( + prisma: AnyClient, + schemaVariant: RunStoreSchemaVariant, + slugSuffix: string +) { + if (schemaVariant === "dedicated") { + return { + organization: { id: `org_${slugSuffix}` }, + project: { id: `proj_${slugSuffix}` }, + environment: { id: `env_${slugSuffix}` }, + }; + } + const organization = await (prisma as PrismaClient).organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await (prisma as PrismaClient).project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await (prisma as PrismaClient).runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "COMPLETED_SUCCESSFULLY", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + traceContext: { trace: "ctx" }, + traceId: `trace_${params.runId}`, + spanId: `span_${params.runId}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "FINISHED", + description: "Run finished", + runStatus: "COMPLETED_SUCCESSFULLY", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// Exactly the projection the CANCEL branch reads. +const CANCEL_SELECT = { + id: true, + engine: true, + friendlyId: true, + status: true, + createdAt: true, + completedAt: true, + taskEventStore: true, +} as const; + +describe("run-ops split — bulk-action member hydration (id-set findRuns) vs. a lagging replica", () => { + // CANCEL branch. The id set is ClickHouse-sourced; here we FORCE the (in practice + // unreachable) stale window: freeze the LEGACY replica while a LEGACY-resident run lives on the + // LEGACY primary, and put a second, NEW-resident run whose replica is live in the same batch. + // The no-client findRuns(id-set) drops the lagging-store row and keeps the other → PARTIAL merge. + // Tolerated: the drop is safe because an id only reaches this batch after ClickHouse (which lags + // MORE than the PG replica) has it, by which point the PG replica already has the row. + heteroRunOpsPostgresTest( + "cancel: id-set findRuns (no client) drops the lagging-legacy-replica run, keeps the live-new-replica run", + async ({ prisma14, prisma17 }) => { + // LEGACY replica frozen; NEW replica live. + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const legacySeed = await seedEnvironment(prisma14, "legacy", "bulk_cancel_leg"); + const laggingRunId = `run_${CUID_25}`; // cuid → LEGACY (owning replica is frozen) + await legacyStore.createRun( + buildCreateRunInput({ + runId: laggingRunId, + friendlyId: "run_bulk_cancel_leg", + organizationId: legacySeed.organization.id, + projectId: legacySeed.project.id, + runtimeEnvironmentId: legacySeed.environment.id, + }) + ); + + const newSeed = await seedEnvironment(prisma17, "dedicated", "bulk_cancel_new"); + const liveRunId = `run_${generateRunOpsId()}`; // v1 body → NEW (owning replica is live) + await newStore.createRun( + buildCreateRunInput({ + runId: liveRunId, + friendlyId: "run_bulk_cancel_new", + organizationId: newSeed.organization.id, + projectId: newSeed.project.id, + runtimeEnvironmentId: newSeed.environment.id, + }) + ); + + // EXACT call site — id set from listRunIds, select projection, NO client. + const runIdsToProcess = [laggingRunId, liveRunId]; + const runs = (await router.findRuns({ + where: { id: { in: runIdsToProcess } }, + select: CANCEL_SELECT, + })) as Array<{ id: string }>; + + // Fact: the lagging LEGACY replica returned nothing for its run, so the merge is + // PARTIAL — only the live-replica (NEW) run survives. The cancel branch would iterate just + // that one run this batch; the lagging one is silently absent from `runs`. + const ids = runs.map((r) => r.id).sort(); + expect(ids).toEqual([liveRunId]); + expect(ids).not.toContain(laggingRunId); + expect(legacyReplica.wasHit()).toBe(true); + + // The dropped run genuinely exists on the LEGACY primary — prove the miss is pure lag/routing, + // not a nonexistent row: re-read with the WRITER (→ owning primaries) returns BOTH. + const viaPrimary = (await router.findRuns( + { where: { id: { in: runIdsToProcess } }, select: CANCEL_SELECT }, + prisma14 as never + )) as Array<{ id: string }>; + expect(viaPrimary.map((r) => r.id).sort()).toEqual([laggingRunId, liveRunId].sort()); + // Tolerated: batch ids are ClickHouse-sourced (lags more than the PG replica) — see header. + } + ); + + // REPLAY branch. Same id-set read, full row (no select), roles swapped so the OTHER + // owning store lags: freeze the NEW replica, keep LEGACY live. Same tolerance as above. + heteroRunOpsPostgresTest( + "replay: id-set findRuns (no client, full row) drops the lagging-new-replica run, keeps the live-legacy-replica run", + async ({ prisma14, prisma17 }) => { + // NEW replica frozen; LEGACY replica live. + const newReplica = laggingReplica(prisma17, [{ model: "taskRun", mode: "missing" }]); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: newReplica.client as never, + schemaVariant: "dedicated", + }); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const newSeed = await seedEnvironment(prisma17, "dedicated", "bulk_replay_new"); + const laggingRunId = `run_${generateRunOpsId()}`; // v1 body → NEW (owning replica is frozen) + await newStore.createRun( + buildCreateRunInput({ + runId: laggingRunId, + friendlyId: "run_bulk_replay_new", + organizationId: newSeed.organization.id, + projectId: newSeed.project.id, + runtimeEnvironmentId: newSeed.environment.id, + }) + ); + + const legacySeed = await seedEnvironment(prisma14, "legacy", "bulk_replay_leg"); + const liveRunId = `run_${CUID_25}`; // cuid → LEGACY (owning replica is live) + await legacyStore.createRun( + buildCreateRunInput({ + runId: liveRunId, + friendlyId: "run_bulk_replay_leg", + organizationId: legacySeed.organization.id, + projectId: legacySeed.project.id, + runtimeEnvironmentId: legacySeed.environment.id, + }) + ); + + // EXACT call site — id set from listRunIds, FULL row (no select), NO client. + const runIdsToProcess = [laggingRunId, liveRunId]; + const runs = (await router.findRuns({ + where: { id: { in: runIdsToProcess } }, + })) as Array<{ id: string }>; + + // Fact: the frozen NEW replica returns nothing for its run; only the live LEGACY + // run survives the merge. The replay branch would replay just that one run this batch. + const ids = runs.map((r) => r.id).sort(); + expect(ids).toEqual([liveRunId]); + expect(ids).not.toContain(laggingRunId); + expect(newReplica.wasHit()).toBe(true); + + // The dropped NEW run genuinely exists on the NEW primary: re-read with the WRITER returns BOTH. + const viaPrimary = (await router.findRuns( + { where: { id: { in: runIdsToProcess } } }, + prisma17 as never + )) as Array<{ id: string }>; + expect(viaPrimary.map((r) => r.id).sort()).toEqual([laggingRunId, liveRunId].sort()); + // Tolerated: batch ids are ClickHouse-sourced (lags more than the PG replica) — see header. + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.cancelRunReadAfterWrite.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.cancelRunReadAfterWrite.replicaLag.test.ts new file mode 100644 index 000000000..b99dfa6f6 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.cancelRunReadAfterWrite.replicaLag.test.ts @@ -0,0 +1,223 @@ +// Lagging-replica coverage for the CANCEL route's run lookups on the run-ops split. +// +// The cancel route issues two friendlyId-keyed run reads through the shared runStore — the +// org-resolution read (resolveRunOrganizationId) and the cancel-decision read (action body) — both +// client-less, so both route to the OWNING store's REPLICA. Property: reading with the owning WRITER +// (the primary fallback / findRunOnPrimary) resolves a just-written run the replica has not yet +// received, so the cancel decision does not act on a stale miss. +// +// This is the same read-your-writes class as the other lag guards: a replica read whose result drives +// a decision (cancel vs. not-found). These tests freeze the owning replica with the shared +// laggingReplica primitive and prove, at the store seam, exactly what the route sees — the client-less +// read is null under lag, the writer read finds the row. Real split topology via +// heteroRunOpsPostgresTest — NEVER mocked. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CreateRunInput, RunStoreSchemaVariant } from "./types.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +// ownerEngine (classifyResidency) routes a run-ops v1 body → NEW, everything else → LEGACY. +const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema) +const NEW_RUN_ID = `run_${generateRunOpsId()}`; // valid v1 body → NEW (#new / prisma17, dedicated) + +async function seedEnvironment( + prisma: AnyClient, + schemaVariant: RunStoreSchemaVariant, + slugSuffix: string +) { + if (schemaVariant === "dedicated") { + return { + organization: { id: `org_${slugSuffix}` }, + project: { id: `proj_${slugSuffix}` }, + environment: { id: `env_${slugSuffix}` }, + }; + } + const organization = await (prisma as PrismaClient).organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await (prisma as PrismaClient).project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await (prisma as PrismaClient).runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + taskIdentifier: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "PENDING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: params.taskIdentifier, + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: ["alpha", "beta"], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// Exactly the projection the cancel action reads (subset that exists on both variants). +const CANCEL_SELECT = { + id: true, + engine: true, + status: true, + friendlyId: true, + projectId: true, +} as const; + +describe("run-ops split — cancel-route run lookup vs. a lagging replica (read-your-writes)", () => { + // (a) LEGACY-resident (cuid) run committed to the control-plane WRITER; the control-plane replica + // lags. Mirrors the moment a buffered run has just drained to the primary but not yet replicated. + heteroRunOpsPostgresTest( + "LEGACY cuid: no-client findRun is STALE under lag; the primary re-read finds it", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, "legacy", "cancel_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = "run_cancel_leg"; + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + taskIdentifier: "my-task", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + // The exact cancel-route read — friendlyId, select, NO client → replica. + const staleRead = await router.findRun({ friendlyId }, { select: CANCEL_SELECT }); + + // The run exists on the primary but the replica lags, so the client-less read returns null. + // Re-reading the primary on this miss (as the org-resolution read does) resolves the live run, + // so the cancel does not act on a stale miss. + expect(staleRead).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // Re-read via the control-plane WRITER (the same primary fallback the org-resolution read uses, + // routing to the owning store's primary) — this finds the run. + const primaryRead = await router.findRun( + { friendlyId }, + { select: CANCEL_SELECT }, + prisma14 as never + ); + expect(primaryRead).not.toBeNull(); + expect(primaryRead!.friendlyId).toBe(friendlyId); + expect(primaryRead!.id).toBe(runId); + } + ); + + // (b) NEW-resident (ksuid) run on the dedicated subset schema; the NEW replica lags. + heteroRunOpsPostgresTest( + "NEW ksuid: no-client findRun is STALE under NEW replica lag; the primary re-read finds it", + async ({ prisma14, prisma17 }) => { + const newReplica = laggingReplica(prisma17, [{ model: "taskRun", mode: "missing" }]); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: newReplica.client as never, + schemaVariant: "dedicated", + }); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma17, "dedicated", "cancel_new"); + const runId = NEW_RUN_ID; // v1 body → NEW + // friendlyId classifies identically to the id, so a NEW run's friendlyId must be run-ops-shaped + // for the by-friendlyId read to route to NEW (single-store; no cross-store fan-out). + const friendlyId = `run_${generateRunOpsId()}`; + await newStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + taskIdentifier: "my-task", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const staleRead = await router.findRun({ friendlyId }, { select: CANCEL_SELECT }); + expect(staleRead).toBeNull(); + expect(newReplica.wasHit()).toBe(true); + + const primaryRead = await router.findRun( + { friendlyId }, + { select: CANCEL_SELECT }, + prisma17 as never + ); + expect(primaryRead).not.toBeNull(); + expect(primaryRead!.friendlyId).toBe(friendlyId); + expect(primaryRead!.id).toBe(runId); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.dashboardAgentReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.dashboardAgentReadView.replicaLag.test.ts new file mode 100644 index 000000000..1689009dc --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.dashboardAgentReadView.replicaLag.test.ts @@ -0,0 +1,224 @@ +// Replica-lag coverage for the dashboard-agent run->commit resolver read. dashboardAgent +// resolveRunCommit issues a client-less runStore.findRun({friendlyId, runtimeEnvironmentId}, {select: +// {lockedToVersionId}}) — no client, so readYourWrites() is false and the friendlyId-classified read +// routes to the owning store's REPLICA (miss => probe the other store's replica). Under lag a +// freshly-locked run is not yet visible, so findRun returns null and the resolver returns null. +// +// The stale null is tolerated: the sole consumer is a read-only resolver (the external repo-snapshot +// endpoint), where null becomes a 404 "no deployed commit" / branch-head fallback, not a mutation +// findResource guard. And the runId is dashboard-sourced (a ClickHouse view that lags more than the PG +// replica) and only carries a truthy lockedToVersionId after trigger + dequeue + lock (seconds of +// lifecycle), so by the time this read fires the PG replica has caught up — the stale-null window is +// unreachable for a run that would have returned a commit. +// +// dashboardAgent is a webapp module, so this exercises the exact underlying runStore.findRun pattern +// (same method, no-client arg, friendlyId+environment where, same select) with the owning replica +// frozen by the shared laggingReplica primitive. Asserts both the stale-null under lag and that the +// SAME read on the owning primary recovers the row's lockedToVersionId — so the null is purely replica +// lag. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; + +// A cuid (25 chars after the `run_` prefix) classifies LEGACY, so the friendlyId-keyed read routes +// to the legacy (control-plane) store first — the owning store for this seeded run. +const CUID_25 = "c".repeat(25); +const FRIENDLY_25 = "d".repeat(25); + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +async function seedBackgroundWorker( + prisma: PrismaClient, + opts: { suffix: string; organizationId: string; projectId: string; runtimeEnvironmentId: string } +) { + return prisma.backgroundWorker.create({ + data: { + friendlyId: `worker_${opts.suffix}`, + contentHash: `hash_${opts.suffix}`, + version: "20260717.1", + metadata: {}, + projectId: opts.projectId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + }, + }); +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + lockedToVersionId?: string; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: "PENDING" as const, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + lockedToVersionId: opts.lockedToVersionId ?? null, + }; +} + +// Exactly the resolveRunCommit read: findRun keyed on {friendlyId, runtimeEnvironmentId}, selecting +// lockedToVersionId, with NO client argument. Returns the resolver's early-return decision: +// null when the run (or its lockedToVersionId) is not visible, else the version id it would resolve. +async function resolveRunCommitPgRead( + router: RoutingRunStore, + where: { friendlyId: string; runtimeEnvironmentId: string } +): Promise { + const run = (await router.findRun(where, { select: { lockedToVersionId: true } })) as { + lockedToVersionId: string | null; + } | null; + if (!run?.lockedToVersionId) return null; // dashboardAgent resolveRunCommit early-return + return run.lockedToVersionId; +} + +describe("dashboardAgent resolveRunCommit — findRun (no client) reads the owning replica", () => { + heteroRunOpsPostgresTest( + "resolveRunCommit returns null for a fresh locked run not yet on the lagging replica (tolerated read-only fallback)", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newReplica = laggingReplica(prisma17 as never, [{ model: "taskRun", mode: "missing" }]); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + // Freeze the OTHER store's replica too, so the #findRunRouted miss-probe cannot mask the + // owning-replica staleness with a phantom hit. (The row lives only on legacy anyway.) + readOnlyPrisma: newReplica.client as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironmentLegacy(prisma14, "dash_lag"); + const worker = await seedBackgroundWorker(prisma14, { + suffix: "dash_lag", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }); + const runId = `run_${CUID_25}`; // cuid => LEGACY owner + const friendlyId = `run_${FRIENDLY_25}`; // classifies LEGACY too => routes to owning store first + // The row EXISTS on the primary and is fully locked to a deployed version — resolveRunCommit + // would return a commit for it. Only replica lag hides it. + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + lockedToVersionId: worker.id, + }), + }); + + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + + // HAZARD proof: the caller's read (no client => replica route) misses the fresh row under lag. + const underLag = await resolveRunCommitPgRead(router, where); + expect(underLag).toBeNull(); + // Prove it actually consulted the owning (legacy) replica — i.e. it is replica-routed, and the + // null is replica lag, not a routing accident. + expect(legacyReplica.wasHit()).toBe(true); + + // GROUND TRUTH: the SAME logical read against a healthy store (replica caught up) resolves the + // locked version id — so the null above is PURELY replica lag, and a route-to-primary would + // recover it. The caller tolerates the transient null (read-only 404 / branch-head fallback), so + // the property holds without any primary re-read. + const healthyLegacy = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, // caught-up replica + schemaVariant: "legacy", + }); + const healthyRouter = new RoutingRunStore({ new: newStore, legacy: healthyLegacy }); + const resolved = await resolveRunCommitPgRead(healthyRouter, where); + expect(resolved).toBe(worker.id); + } + ); + + heteroRunOpsPostgresTest( + "resolveRunCommit resolves the locked version id from the replica in steady state", + async ({ prisma14, prisma17 }) => { + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, // caught-up replica handle + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironmentLegacy(prisma14, "dash_ok"); + const worker = await seedBackgroundWorker(prisma14, { + suffix: "dash_ok", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${FRIENDLY_25}`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + lockedToVersionId: worker.id, + }), + }); + + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const resolved = await resolveRunCommitPgRead(router, where); + expect(resolved).toBe(worker.id); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.engineReadViews.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.engineReadViews.replicaLag.test.ts new file mode 100644 index 000000000..7296c6373 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.engineReadViews.replicaLag.test.ts @@ -0,0 +1,359 @@ +// Property: three run-engine read views tolerate stale/absent values under replica lag. Each issues a +// client-less runStore read, so RoutingRunStore routes it to the OWNING store's REPLICA (readYourWrites +// signal absent). With the replica frozen by the shared laggingReplica, this file asserts what each read +// returns under lag and documents the caller that makes the stale/absent value tolerable. Store built as +// the engine holds it (RoutingRunStore over a legacy + dedicated PostgresRunStore); reads invoked +// exactly as the caller does (same method, no client arg). +// +// Reads covered (one case each): +// 1. concurrencySweeper findRuns — #concurrencySweeperCallback (completedAt <= now-10min) +// 2. resolveTaskRunContext findRun +// 3. forceRequeue span-close findRun (telemetry) +// +// Routing recap: a client-less findRun with an id-classifiable `where` → #findRunRouted → owning store +// findRun → readOnlyPrisma (REPLICA); a client-less findRuns over an id set → #findRunsByIdSet → each +// store's findRuns with client undefined → readOnlyPrisma (REPLICA). All three hit the owning REPLICA. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +// A cuid-shaped id (no run-ops v1 marker) classifies LEGACY, so both the create and the id-keyed reads +// route to the legacy (control-plane / prisma14, full schema) store first. +const CUID_25 = "e".repeat(25); + +// Mirror of the engine's final-run-status set (getFinalRunStatuses). Inlined to avoid a +// run-store -> run-engine test dependency; concurrencySweeper's `status: { in: ... }` filter only affects the +// PRIMARY-contrast query — the missing-mode replica ignores the where entirely and returns []. +const FINAL_RUN_STATUSES = [ + "CANCELED", + "INTERRUPTED", + "COMPLETED_SUCCESSFULLY", + "COMPLETED_WITH_ERRORS", + "SYSTEM_FAILURE", + "CRASHED", + "EXPIRED", + "TIMED_OUT", +] as const; + +async function seedEnvironment(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + status?: string; + completedAt?: Date | null; + spanId?: string; + maxAttempts?: number | null; + createdAt?: Date; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: (opts.status ?? "PENDING") as never, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: opts.spanId ?? `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + ...(opts.completedAt !== undefined ? { completedAt: opts.completedAt } : {}), + ...(opts.maxAttempts !== undefined ? { maxAttempts: opts.maxAttempts } : {}), + ...(opts.createdAt !== undefined ? { createdAt: opts.createdAt } : {}), + }; +} + +function buildRouter( + prisma14: PrismaClient, + prisma17: unknown, + legacyReplicaClient: AnyClient +): RoutingRunStore { + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplicaClient, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + return new RoutingRunStore({ new: newStore, legacy: legacyStore }); +} + +describe("run-engine read views under replica lag (client-less -> owning REPLICA)", () => { + // concurrencySweeper findRuns. + // The concurrencySweeper callback is handed a set of runIds (from the Redis concurrency set) and asks + // the store which are FINISHED and completed MORE THAN 10 MINUTES AGO, so it can release their held + // concurrency. The read is a client-less findRuns → owning REPLICA. Two caller facts make a + // stale/absent replica tolerable, and this test proves the store-level fact + asserts both: + // (a) the `completedAt <= now - 10min` filter: any row that MATCHES was committed to the primary at + // least 10 minutes ago, so real replica lag (sub-second..seconds) cannot hide a matching row. + // (b) concurrencySweeper runs periodically: a run transiently missed on one scan is re-evaluated on + // the next — a miss self-heals, it does not leak concurrency permanently. + // The consequence of a miss is at worst "concurrency released one scan later", never a wrong release. + heteroRunOpsPostgresTest( + "concurrencySweeper findRuns misses the finished run under replica lag", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "sweeper_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + // A genuinely-finished run whose completedAt is 30 min in the past — matches the concurrencySweeper filter. + const completedAt = new Date(Date.now() - 1000 * 60 * 30); + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId: "run_sweeper_leg", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED_SUCCESSFULLY", + completedAt, + }), + }); + + const completedAtOffsetMs = 1000 * 60 * 10; // the caller's default + + // Line-for-line mirror of #concurrencySweeperCallback's read + post-filter. + const sweeperCallback = async ( + runStore: RoutingRunStore, + runIds: string[], + readClient?: unknown + ) => { + const runs = (await runStore.findRuns( + { + where: { + id: { in: runIds }, + completedAt: { lte: new Date(Date.now() - completedAtOffsetMs) }, + organizationId: { not: null }, + status: { in: FINAL_RUN_STATUSES as unknown as never }, + }, + select: { id: true, status: true, organizationId: true }, + }, + readClient as never + )) as Array<{ id: string; status: string; organizationId: string | null }>; + return runs + .filter((r) => !!r.organizationId) + .map((r) => ({ id: r.id, orgId: r.organizationId! })); + }; + + // PRIMARY contrast: thread the writer (as read-your-writes callers do) → the run IS seen. Proves + // the run genuinely matches the concurrencySweeper predicate, so any miss below is purely the replica read. + const okReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const routerForPrimary = buildRouter(prisma14, prisma17, okReplica.client); + const viaPrimary = await sweeperCallback(routerForPrimary, [runId], prisma14); + expect(viaPrimary).toEqual([{ id: runId, orgId: seed.organization.id }]); + + // ACTUAL caller behavior: client-less findRuns → owning REPLICA. Freeze the legacy replica so the + // finished run is not visible. + const lagReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const router = buildRouter(prisma14, prisma17, lagReplica.client); + const viaReplica = await sweeperCallback(router, [runId]); + + // Store-level fact under lag: concurrencySweeper's read misses the run entirely and returns []. + expect(lagReplica.wasHit("taskRun")).toBe(true); + expect(viaReplica).toEqual([]); + // TOLERATED because (a) a real row matching `completedAt <= now-10min` (here -30min) committed >=10 + // min ago and has surely replicated, and (b) concurrencySweeper re-scans on schedule so a transient miss + // self-heals — the worst case is concurrency freed one scan later, never a wrong release. + } + ); + + // resolveTaskRunContext findRun. + // resolveTaskRunContext builds a V4 TaskRunContext for a run. Its ONLY non-test caller is + // SpanPresenter #getV4TaskRunContext, a READ-ONLY dashboard presenter that has ALREADY loaded `run` + // for display and passes run.id. The read is client-less → owning REPLICA. Under lag the store returns + // a STALE row (asserted below); the resolver copies its scalar fields straight into the display + // context. Because the sole caller is a read-only span-detail page, a stale field is a cosmetic + // display artifact that self-heals on the next page load — it drives no mutation or control decision. + // The extreme case (row wholly absent on the replica) throws a transient 404, likewise a display miss. + heteroRunOpsPostgresTest( + "resolveTaskRunContext findRun returns a stale run row under replica lag", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "ctx_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = "run_ctx_leg"; + // The run has since PROGRESSED on the primary to EXECUTING… + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + }), + }); + + // …but the replica still shows the OLD PENDING snapshot (frozen row). Provide the fields the site + // asserts on; the legacy read returns this row verbatim (select projection is applied by Postgres, + // which the frozen replica bypasses, so it returns exactly this shape). + const staleRow = { + id: runId, + friendlyId, + status: "PENDING", + runtimeEnvironmentId: seed.environment.id, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }; + const lagReplica = laggingReplica(prisma14, [ + { model: "taskRun", mode: "frozen", rows: [staleRow] }, + ]); + const router = buildRouter(prisma14, prisma17, lagReplica.client); + + // Invoke EXACTLY as resolveTaskRunContext does: client-less findRun({ id }, { select }). + const run = (await router.findRun( + { id: runId }, + { + select: { + id: true, + status: true, + friendlyId: true, + createdAt: true, + runtimeEnvironmentId: true, + }, + } + )) as { id: string; status: string; friendlyId: string } | null; + + expect(lagReplica.wasHit("taskRun")).toBe(true); + // Store-level fact: the client-less read is served by the lagging replica → STALE status. + expect(run).not.toBeNull(); + expect(run!.id).toBe(runId); + expect(run!.status).toBe("PENDING"); // stale — primary is already EXECUTING + + // Proof the staleness is confined to the replica: the primary read returns the fresh status. + const onPrimary = (await router.findRun( + { id: runId }, + { select: { status: true } }, + prisma14 + )) as { status: string } | null; + expect(onPrimary?.status).toBe("EXECUTING"); + // TOLERATED: the only caller (SpanPresenter #getV4TaskRunContext) is a read-only dashboard + // presenter — a stale status on the span-detail page self-heals on refresh and drives no mutation. + } + ); + + // forceRequeue span-close findRun. + // Inside completeRunAttempt's forceRequeue branch, AFTER the retry/requeue decision has already been + // made on the primary (retryOutcomeFromCompletion threads this.$.prisma), this client-less findRun + // re-reads the run purely to shape the `runAttemptFailed` telemetry event (eventBus.emit: status, + // spanId, createdAt, completedAt, updatedAt). The read is client-less → owning REPLICA. Under lag the + // store returns a STALE row (asserted below); the stale scalars only populate the emitted span-close + // event, so a slightly-old status/updatedAt is a cosmetic telemetry artifact — it does not feed the + // requeue control flow (already decided on the primary). The immutable fields the span close needs + // (spanId, createdAt) are set at creation and long replicated, so they are correct even under lag. + heteroRunOpsPostgresTest( + "forceRequeue span-close findRun returns stale status but correct immutable spanId and createdAt under replica lag", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "requeue_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const spanId = "span_requeue_fixed"; + const createdAt = new Date("2024-01-01T00:00:00.000Z"); + // Primary: the run has been requeued back to PENDING with a fresh updatedAt. + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId: "run_requeue_leg", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "PENDING", + spanId, + maxAttempts: 3, + createdAt, + }), + }); + + // Replica still shows the pre-requeue EXECUTING snapshot with an old updatedAt — but the immutable + // spanId/createdAt match the primary (they never change after creation). + const staleRow = { + id: runId, + status: "EXECUTING", + spanId, + maxAttempts: 3, + taskEventStore: "taskEvent", + createdAt, + completedAt: null as Date | null, + updatedAt: new Date("2024-01-01T00:00:05.000Z"), + }; + const lagReplica = laggingReplica(prisma14, [ + { model: "taskRun", mode: "frozen", rows: [staleRow] }, + ]); + const router = buildRouter(prisma14, prisma17, lagReplica.client); + + // Invoke EXACTLY as the forceRequeue branch does: client-less findRun({ id }, { select }). + const minimalRun = (await router.findRun( + { id: runId }, + { + select: { + status: true, + spanId: true, + maxAttempts: true, + taskEventStore: true, + createdAt: true, + completedAt: true, + updatedAt: true, + }, + } + )) as { status: string; spanId: string; createdAt: Date } | null; + + expect(lagReplica.wasHit("taskRun")).toBe(true); + expect(minimalRun).not.toBeNull(); + // Store-level fact: mutable fields are STALE off the replica… + expect(minimalRun!.status).toBe("EXECUTING"); // stale — primary already PENDING (requeued) + // …but the fields the span close actually depends on are immutable and correct even under lag. + expect(minimalRun!.spanId).toBe(spanId); + expect(minimalRun!.createdAt).toEqual(createdAt); + + // Confirm the staleness is replica-only: the primary read shows the requeued PENDING status. + const onPrimary = (await router.findRun( + { id: runId }, + { select: { status: true } }, + prisma14 + )) as { status: string } | null; + expect(onPrimary?.status).toBe("PENDING"); + // TOLERATED: the read's result is only fed to eventBus.emit("runAttemptFailed", ...) (span-close + // telemetry). The requeue decision was already made on the primary; a stale status/updatedAt in + // the emitted event is cosmetic and does not alter control flow. + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.idempotencyGlobalScopeCrossDb.test.ts b/internal-packages/run-store/src/runOpsStore.idempotencyGlobalScopeCrossDb.test.ts new file mode 100644 index 000000000..63a972c63 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.idempotencyGlobalScopeCrossDb.test.ts @@ -0,0 +1,328 @@ +// Property: a GLOBAL-scope idempotency key (per environment+task, NOT per parent) maps to exactly ONE +// child run even when triggered from two parents of DIFFERENT residency. The dedup client is routed by +// the PARENT run's residency (NEW parent → NEW DB, LEGACY parent → LEGACY DB), and the trigger hot path +// probes runStore.findRun({env, idempotencyKey, task}) before minting. Exercised on the REAL +// two-physical-DB split (heteroRunOpsPostgresTest, never mocked): drives the actual +// RoutingRunStore.findRun probe (dedup client chosen by parent residency) + createRun mint in real +// sequential trigger order, asserting one child — the id-less findRun fans out across BOTH DBs, so the +// second probe sees the first child. A routing/topology property, not replica lag. + +import { heteroRunOpsPostgresTest } from "@internal/testcontainers"; +import type { PrismaClient, PrismaClientOrTransaction } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CreateRunInput, RunStoreSchemaVariant } from "./types.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; +type Residency = "NEW" | "LEGACY"; + +// ownerEngine classifies by internal-id LENGTH after stripping a leading `_`: +// 25 chars (no internal underscore) → cuid → LEGACY (#legacy / prisma14), +// a v1 body (version "1" at index 25) → run-ops id → NEW (#new / prisma17). +function cuidLegacy(seed: string): string { + return (seed + "c".repeat(25)).slice(0, 25); // 25 chars → LEGACY +} +function runOpsNew(seed: string): string { + return (seed.replace(/[^0-9a-v]/g, "0") + "k".repeat(24)).slice(0, 24) + "01"; +} + +async function seedEnvironment( + prisma: AnyClient, + schemaVariant: RunStoreSchemaVariant, + suffix: string +) { + if (schemaVariant === "dedicated") { + return { + organization: { id: `org_${suffix}` }, + project: { id: `proj_${suffix}` }, + environment: { id: `env_${suffix}` }, + }; + } + const organization = await (prisma as PrismaClient).organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await (prisma as PrismaClient).project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await (prisma as PrismaClient).runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +// One logical environment whose scalar env/project/org ids are shared by both physical DBs, with real +// owning rows seeded on #legacy (kept FKs) and the same scalar ids valid on the FK-free #new subset. +async function seedSharedEnv(prisma14: PrismaClient, suffix: string) { + const legacy = await seedEnvironment(prisma14, "legacy", suffix); + return { + organizationId: legacy.organization.id, + projectId: legacy.project.id, + runtimeEnvironmentId: legacy.environment.id, + environmentId: legacy.environment.id, + }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + idempotencyKey: string; + taskIdentifier: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "PENDING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + idempotencyKey: params.idempotencyKey, + idempotencyKeyExpiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000), + taskIdentifier: params.taskIdentifier, + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: `trace_${params.runId}`, + spanId: `span_${params.runId}`, + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 1, // a child run (has a parent) + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +function makeSplitRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + return { router: new RoutingRunStore({ new: newStore, legacy: legacyStore }) }; +} + +// A faithful port of resolveIdempotencyDedupClient's parent-residency branch: the dedup client is the +// parent's own DB writer. This is the exact routing decision the code makes. +function resolveDedupClient( + parentResidency: Residency, + newClient: PrismaClientOrTransaction, + legacyClient: PrismaClientOrTransaction +): PrismaClientOrTransaction { + return parentResidency === "NEW" ? newClient : legacyClient; +} + +describe("run-ops split — GLOBAL-scope idempotency dedup across two different-residency parents", () => { + // The real trigger dedup flow for one trigger with an idempotency key, minus the webapp glue: + // 1. resolve the dedup client by the PARENT's residency, + // 2. probe runStore.findRun({ env, idempotencyKey, task }, { include }, dedupClient) — the EXACT + // id-less probe the trigger hot path issues, + // 3. if a run comes back → CACHED hit, mint nothing, + // 4. else createRun a fresh child on the store its (parent-inherited) id-shape names. + // A child inherits its parent's residency, so a NEW parent's child gets a run-ops id (→ #new) and a + // LEGACY parent's child gets a cuid (→ #legacy). + async function triggerChild( + router: RoutingRunStore, + params: { + parentResidency: Residency; + childId: string; + childFriendlyId: string; + idempotencyKey: string; + taskIdentifier: string; + env: { + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + environmentId: string; + }; + newClient: PrismaClientOrTransaction; + legacyClient: PrismaClientOrTransaction; + } + ): Promise<{ cached: boolean; runId: string }> { + const dedupClient = resolveDedupClient( + params.parentResidency, + params.newClient, + params.legacyClient + ); + + const existing = (await router.findRun( + { + runtimeEnvironmentId: params.env.runtimeEnvironmentId, + idempotencyKey: params.idempotencyKey, + taskIdentifier: params.taskIdentifier, + }, + { include: { associatedWaitpoint: true } }, + dedupClient + )) as Record | null; + + if (existing) { + return { cached: true, runId: existing.id as string }; + } + + await router.createRun( + buildCreateRunInput({ + runId: params.childId, + friendlyId: params.childFriendlyId, + organizationId: params.env.organizationId, + projectId: params.env.projectId, + runtimeEnvironmentId: params.env.runtimeEnvironmentId, + idempotencyKey: params.idempotencyKey, + taskIdentifier: params.taskIdentifier, + }) + ); + return { cached: false, runId: params.childId }; + } + + async function countChildrenForKey( + prisma14: PrismaClient, + prisma17: RunOpsPrismaClient, + environmentId: string, + idempotencyKey: string, + taskIdentifier: string + ): Promise<{ legacy: number; new: number; total: number }> { + const where = { + runtimeEnvironmentId: environmentId, + idempotencyKey, + taskIdentifier, + }; + const legacy = await prisma14.taskRun.count({ where }); + const nw = await prisma17.taskRun.count({ where }); + return { legacy, new: nw, total: legacy + nw }; + } + + // (a) NEW parent triggers first, then LEGACY parent — same GLOBAL key. If the LEGACY parent's probe + // only read the LEGACY DB it would miss the NEW child and mint a second one → two children. + heteroRunOpsPostgresTest( + "NEW-parent-first then LEGACY-parent: one global key must yield exactly one child", + async ({ prisma14, prisma17 }) => { + const { router } = makeSplitRouter(prisma14, prisma17); + const env = await seedSharedEnv(prisma14, "gsx_a"); + const idempotencyKey = "global-scope-key-a"; + const taskIdentifier = "child-task"; + + // Trigger 1: from a NEW-resident parent → child born on #new (run-ops id). + const first = await triggerChild(router, { + parentResidency: "NEW", + childId: runOpsNew("gxan"), + childFriendlyId: "run_gsx_a_new_child", + idempotencyKey, + taskIdentifier, + env, + newClient: prisma17 as unknown as PrismaClientOrTransaction, + legacyClient: prisma14, + }); + expect(first.cached).toBe(false); + + // Trigger 2: from a LEGACY-resident parent → child would be born on #legacy (cuid) UNLESS the + // dedup probe sees trigger 1's child on #new and returns it cached. + const second = await triggerChild(router, { + parentResidency: "LEGACY", + childId: cuidLegacy("gxal"), + childFriendlyId: "run_gsx_a_legacy_child", + idempotencyKey, + taskIdentifier, + env, + newClient: prisma17 as unknown as PrismaClientOrTransaction, + legacyClient: prisma14, + }); + + const counts = await countChildrenForKey( + prisma14, + prisma17, + env.environmentId, + idempotencyKey, + taskIdentifier + ); + + // The load-bearing assertion: a GLOBAL idempotency key must map to exactly ONE child run, + // no matter that the two parents live on different physical DBs. + expect(counts.total).toBe(1); + // And the second trigger must be a cached hit resolving to the FIRST child. + expect(second.cached).toBe(true); + expect(second.runId).toBe(first.runId); + } + ); + + // (b) Reverse order: LEGACY parent first, then NEW parent. Symmetric — guards the other fan-out leg. + heteroRunOpsPostgresTest( + "LEGACY-parent-first then NEW-parent: one global key must yield exactly one child", + async ({ prisma14, prisma17 }) => { + const { router } = makeSplitRouter(prisma14, prisma17); + const env = await seedSharedEnv(prisma14, "gsx_b"); + const idempotencyKey = "global-scope-key-b"; + const taskIdentifier = "child-task"; + + const first = await triggerChild(router, { + parentResidency: "LEGACY", + childId: cuidLegacy("gxbl"), + childFriendlyId: "run_gsx_b_legacy_child", + idempotencyKey, + taskIdentifier, + env, + newClient: prisma17 as unknown as PrismaClientOrTransaction, + legacyClient: prisma14, + }); + expect(first.cached).toBe(false); + + const second = await triggerChild(router, { + parentResidency: "NEW", + childId: runOpsNew("gxbn"), + childFriendlyId: "run_gsx_b_new_child", + idempotencyKey, + taskIdentifier, + env, + newClient: prisma17 as unknown as PrismaClientOrTransaction, + legacyClient: prisma14, + }); + + const counts = await countChildrenForKey( + prisma14, + prisma17, + env.environmentId, + idempotencyKey, + taskIdentifier + ); + + expect(counts.total).toBe(1); + expect(second.cached).toBe(true); + expect(second.runId).toBe(first.runId); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.idempotencyResetReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.idempotencyResetReadView.replicaLag.test.ts new file mode 100644 index 000000000..ad48e3d69 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.idempotencyResetReadView.replicaLag.test.ts @@ -0,0 +1,227 @@ +// Lagging-replica coverage for the IDEMPOTENCY-KEY RESET route's source-run lookup on the run-ops split. +// The reset action reads the run by friendlyId with NO client (→ OWNING store's REPLICA, fan-out to the +// other store's replica on a miss) and that result gates the whole reset — a DECISION-DRIVING read. +// Under lag the read returns null for a run that exists on the primary. These tests build the store as +// the route holds it, seed the run on the OWNING primary, freeze the OWNING replica via laggingReplica, +// and invoke the read EXACTLY as the route does, then show the owning-WRITER read escalates to the +// primary and resolves the run + its idempotencyKey. Real split topology via heteroRunOpsPostgresTest — NEVER mocked. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CreateRunInput, RunStoreSchemaVariant } from "./types.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +// ownerEngine (classifyResidency) routes a run-ops v1 body → NEW, everything else → LEGACY. +const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema) +const NEW_RUN_ID = `run_${generateRunOpsId()}`; // valid v1 body → NEW (#new / prisma17, dedicated) + +async function seedEnvironment( + prisma: AnyClient, + schemaVariant: RunStoreSchemaVariant, + slugSuffix: string +) { + if (schemaVariant === "dedicated") { + return { + organization: { id: `org_${slugSuffix}` }, + project: { id: `proj_${slugSuffix}` }, + environment: { id: `env_${slugSuffix}` }, + }; + } + const organization = await (prisma as PrismaClient).organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await (prisma as PrismaClient).project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await (prisma as PrismaClient).runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + taskIdentifier: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + idempotencyKey: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "COMPLETED_SUCCESSFULLY", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: params.taskIdentifier, + // The reset route only makes sense for a run triggered WITH an idempotency key. + idempotencyKey: params.idempotencyKey, + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: ["alpha", "beta"], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// Exactly the projection the reset action reads. +const RESET_SELECT = { + id: true, + idempotencyKey: true, + taskIdentifier: true, + projectId: true, + runtimeEnvironmentId: true, +} as const; + +describe("run-ops split — idempotency-key reset source-run lookup vs. a lagging replica (read-your-writes)", () => { + // LEGACY cuid resident. The run is committed to the control-plane WRITER; the control-plane replica + // lags (the buffer-drained-but-not-yet-replicated window). The route's no-client findRun hits that + // replica and returns null for a run that exists on the primary; the owning-writer re-read resolves it. + heteroRunOpsPostgresTest( + "LEGACY cuid: no-client findRun is stale under lag; owning-primary re-read finds it", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, "legacy", "idem_reset_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = "run_idem_reset_leg"; + const idempotencyKey = "user-supplied-key-leg"; + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + taskIdentifier: "my-task", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + idempotencyKey, + }) + ); + + // The exact reset-route read — friendlyId + RESET_SELECT, NO client → replica. + const staleRead = await router.findRun({ friendlyId }, { select: RESET_SELECT }); + + // The run exists on the primary but the replica lags, so the no-client read returns null. In the + // route this is the "Run not found" short-circuit, before ResetIdempotencyKeyService runs. The + // owning-writer re-read below is the property that resolves the run under lag. + expect(staleRead).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // Pass the control-plane WRITER to findRun so it escalates to findRunOnPrimary. This resolves the + // run + its idempotencyKey under lag. + const primaryRead = await router.findRun( + { friendlyId }, + { select: RESET_SELECT }, + prisma14 as never + ); + expect(primaryRead).not.toBeNull(); + expect(primaryRead!.id).toBe(runId); + // The idempotencyKey the reset service needs is only visible via the primary. + expect(primaryRead!.idempotencyKey).toBe(idempotencyKey); + } + ); + + // Same property on a NEW-resident (ksuid) run; the NEW replica lags. Not legacy-specific: whichever + // store owns the run, the no-client read hits its lagging replica and the owning-writer re-read resolves it. + heteroRunOpsPostgresTest( + "NEW ksuid: no-client findRun is stale under NEW replica lag; owning-primary re-read finds it", + async ({ prisma14, prisma17 }) => { + const newReplica = laggingReplica(prisma17, [{ model: "taskRun", mode: "missing" }]); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: newReplica.client as never, + schemaVariant: "dedicated", + }); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma17, "dedicated", "idem_reset_new"); + const runId = NEW_RUN_ID; // v1 body → NEW + // A NEW run's friendlyId must be run-ops-shaped so the by-friendlyId read routes to NEW + // (single-store; friendlyId classifies identically to the id). + const friendlyId = `run_${generateRunOpsId()}`; + const idempotencyKey = "user-supplied-key-new"; + await newStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + taskIdentifier: "my-task", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + idempotencyKey, + }) + ); + + const staleRead = await router.findRun({ friendlyId }, { select: RESET_SELECT }); + expect(staleRead).toBeNull(); + expect(newReplica.wasHit()).toBe(true); + + const primaryRead = await router.findRun( + { friendlyId }, + { select: RESET_SELECT }, + prisma17 as never + ); + expect(primaryRead).not.toBeNull(); + expect(primaryRead!.id).toBe(runId); + expect(primaryRead!.idempotencyKey).toBe(idempotencyKey); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.modelRuntimeEnvReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.modelRuntimeEnvReadView.replicaLag.test.ts new file mode 100644 index 000000000..45c4eeaf9 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.modelRuntimeEnvReadView.replicaLag.test.ts @@ -0,0 +1,183 @@ +// Lagging-replica coverage for the model-runtime-env read view on the run-ops split. +// +// findEnvironmentFromRun's findRun reads the run-ops scalars {runTags, batchId, runtimeEnvironmentId} +// off a run and resolves the authenticated env from them. The webapp calls it with NO tx from the +// `runMetadataUpdated` engine handler, so the client arg is the branded `$replica` handle and the +// read is REPLICA-routed. +// +// Routing (against runOpsStore.ts): findRun(where {id}, select, BRANDED $replica) → id classifiable → +// owning store first; a branded read-replica client is not a write signal, so the read stays on the +// owning store's REPLICA, then fans out to the OTHER store's replica on a miss. A cuid run owns +// LEGACY, so its frozen legacy replica is the one hit. +// +// Why read-your-writes matters here: the handler treats a null result as "Failed to find environment" +// and RETURNS, dropping BOTH the final-metadata write (updateMetadataService.call, which itself +// reads/writes the PRIMARY within the post-completion grace window) AND the realtime run-changed +// publish. The event is one-shot at attempt completion — no retry loop, no primary fallback. For a +// fast run whose lifetime is shorter than replica lag the owning replica lacks the row, so a +// replica-routed read misses a live, primary-resident run. Routing this read to the owning PRIMARY +// keeps the precheck at least as consistent as the mutation it gates (which already reads the primary). +// +// This proves the store fact at the seam: freeze the OWNING replica with the shared laggingReplica +// primitive, invoke findRun EXACTLY as the caller does (same where + select + branded-replica +// client), assert the null under lag AND that the identical row IS on the owning primary (so the miss +// is provably lag-induced and the primary re-read recovers it). Real split topology via +// heteroRunOpsPostgresTest — NEVER mocked. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import { markReadReplicaClient } from "./readReplicaClient.js"; +import type { CreateRunInput } from "./types.js"; + +// ownerEngine (classifyResidency) routes a run-ops v1 body → NEW, everything else → LEGACY. A cuid +// run owns LEGACY (the control-plane DB), the residency of a run whose completion emits +// runMetadataUpdated, so the owning store is LEGACY and its frozen replica is the one this read hits. +const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema) + +async function seedEnvironment(prisma: PrismaClient, slugSuffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "PENDING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: ["alpha", "beta"], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// The webapp always passes its $replica handle, which is markReadReplicaClient()'d. A branded client +// tells RoutingRunStore "do not escalate to primary" — the read stays on the owning store's replica. +// Its identity is irrelevant; the router discards it and reads the owning store's readOnlyPrisma. +const BRANDED_REPLICA = markReadReplicaClient({}) as never; + +describe("run-ops split — model-runtime-env (findEnvironmentFromRun) read view vs. a lagging replica", () => { + // The findEnvironmentFromRun read (branded $replica). + heteroRunOpsPostgresTest( + "findEnvironmentFromRun findRun(branded $replica) is NULL under owning-replica lag — runMetadataUpdated handler drops the final-metadata write + realtime publish for a live run; primary re-read recovers it", + async ({ prisma14, prisma17 }) => { + // Build the router the way the webapp holds it: a LEGACY store on the control-plane DB whose + // replica LAGS, plus a fresh NEW store. The run is a cuid → LEGACY-resident. + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, "modelenv"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = "run_modelenv"; + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + // The EXACT env-resolve read: where {id}, select {runTags, batchId, runtimeEnvironmentId}, $replica. + const staleRead = (await router.findRun( + { id: runId }, + { select: { runTags: true, batchId: true, runtimeEnvironmentId: true } }, + BRANDED_REPLICA + )) as { runTags: string[]; batchId: string | null; runtimeEnvironmentId: string } | null; + + // Store fact: the read is REPLICA-routed and the owning replica lags → null. The fan-out probes + // the LEGACY replica (owner) first, then the NEW replica; the frozen legacy replica records a hit. + expect(staleRead).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // Caller decision, exercised: findEnvironmentFromRun returns null on the miss, so the + // runMetadataUpdated handler logs "Failed to find environment" and RETURNS — never calling + // updateMetadataService.call (the final-metadata write) or publishChangeRecord (realtime). For + // this LIVE run that is a dropped mutation, not a tolerated stale display value. + const environmentFromRun = staleRead + ? { runtimeEnvironmentId: staleRead.runtimeEnvironmentId } + : null; + expect(environmentFromRun).toBeNull(); // → handler aborts: metadata write + realtime publish dropped + + // Read-your-writes: re-reading the owning PRIMARY (pass the WRITER `prisma14` → the router + // escalates to findRunOnPrimary) returns the live run with the exact scalars the env-resolve + // needs. Routing findEnvironmentFromRun to the owning primary keeps this consistent with + // updateMetadataService.call's own primary read. + const primaryRead = (await router.findRun( + { id: runId }, + { select: { runTags: true, batchId: true, runtimeEnvironmentId: true } }, + prisma14 as never + )) as { runTags: string[]; batchId: string | null; runtimeEnvironmentId: string } | null; + expect(primaryRead).not.toBeNull(); + expect(primaryRead!.runtimeEnvironmentId).toBe(seed.environment.id); + expect(primaryRead!.runTags).toEqual(["alpha", "beta"]); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.presentersRunReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.presentersRunReadView.replicaLag.test.ts new file mode 100644 index 000000000..67baabc15 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.presentersRunReadView.replicaLag.test.ts @@ -0,0 +1,435 @@ +// Lagging-replica coverage for seven run-detail presenter reads. Every one is a client-less (or +// branded-replica) runStore read, so RoutingRunStore serves it from the OWNING store's REPLICA. This +// proves — with the replica frozen by the shared laggingReplica primitive — what each read returns +// under lag, and names the caller mechanism that makes the stale/absent value tolerable. Builds the +// store as the webapp holds it (RoutingRunStore over a legacy + dedicated PostgresRunStore) and invokes +// each read EXACTLY as the presenter does (same method, where, and client arg — none, or a branded +// $replica). +// +// Routing recap (verified against RoutingRunStore): findRun with NO client OR a BRANDED replica has +// readYourWrites false (a branded replica does not escalate), so it reads the owning store's +// readOnlyPrisma (REPLICA) and probes the other store's replica on a miss. findRuns over a bounded +// id-set reads each store's REPLICA; findRuns over an open predicate ({parentSpanId}) queries BOTH +// stores' replicas. So all seven reads hit a REPLICA. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import { markReadReplicaClient } from "./readReplicaClient.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +// A cuid-shaped id/friendlyId (no run-ops v1 marker) classifies LEGACY, so both the create and the +// keyed reads route to the legacy (control-plane / prisma14, full schema) store as the owner. +const CUID_25 = "e".repeat(25); +const FRIENDLY_25 = "f".repeat(25); + +async function seedEnvironment(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + status?: string; + spanId?: string; + parentSpanId?: string; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: (opts.status ?? "EXECUTING") as never, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: opts.spanId ?? `span_${opts.id}`, + ...(opts.parentSpanId !== undefined ? { parentSpanId: opts.parentSpanId } : {}), + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +// Build the router exactly as buildRunStore does (RoutingRunStore over legacy + dedicated stores), +// with the LEGACY store's REPLICA swapped for the frozen `legacyReplicaClient`. The NEW store keeps +// the real prisma17 as its replica — the seeded run lives only on legacy, so the routed miss-probe of +// the new store returns null/[] naturally (no phantom hit masking the legacy-replica staleness). +function buildRouter( + prisma14: PrismaClient, + prisma17: unknown, + legacyReplicaClient: AnyClient +): RoutingRunStore { + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplicaClient, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + return new RoutingRunStore({ new: newStore, legacy: legacyStore }); +} + +describe("run-detail presenter read views route to the owning replica under lag", () => { + // ApiRetrieveRunPresenter findRun (+$replica) — public GET /runs/:runId retrieve. Passes the BRANDED + // $replica, which stays on a replica (no escalation), so the read is served by the owning REPLICA. + // Under lag a freshly triggered run's row is not yet on the replica, so findRun returns null and the + // presenter's `if (pgRow)` guard falls through → not-found for this poll. Tolerated: the SDK's + // runs.retrieve/poll loop re-fetches on the next poll once the replica catches up. Read-only. + heteroRunOpsPostgresTest( + "ApiRetrieveRunPresenter findRun (+$replica) returns null for a fresh run missed on the frozen replica (SDK retrieve/poll re-fetches)", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "retrieve_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${FRIENDLY_25}`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const select = { select: { id: true, friendlyId: true, status: true } }; + + // PRIMARY contrast: an unbranded writer client → readYourWrites true → findRunOnPrimary → the + // run IS seen. Proves the run genuinely exists and any miss below is purely replica lag. + const okReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const primaryRouter = buildRouter(prisma14, prisma17, okReplica.client); + const viaPrimary = (await primaryRouter.findRun(where, select, prisma14 as never)) as { + id: string; + } | null; + expect(viaPrimary?.id).toBe(runId); + + // ACTUAL caller behavior: pass the BRANDED $replica handle → owning REPLICA (frozen). + const lagReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const $replica = markReadReplicaClient(lagReplica.client); + const router = buildRouter(prisma14, prisma17, lagReplica.client); + const pgRow = (await router.findRun(where, select, $replica as never)) as { + id: string; + } | null; + + expect(lagReplica.wasHit("taskRun")).toBe(true); + expect(pgRow).toBeNull(); + } + ); + + // ApiRunResultPresenter findRun (no client) — GET /runs/:runId/result poll (SDK waitForRun result). + // No client → owning REPLICA. Under lag the run is invisible → findRun null → the presenter returns + // undefined, which the SDK treats as "not finished yet, keep polling". A run that has produced a + // result committed that write well before the poll, so the transient undefined only lengthens the + // poll by one tick. Read-only. + heteroRunOpsPostgresTest( + "ApiRunResultPresenter findRun (no client) returns null for a run missed on the frozen replica (result poll retries)", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "result_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${FRIENDLY_25}`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED_SUCCESSFULLY", + }), + }); + + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + // Exact caller shape: an `include` of attempts. Under "missing" mode the replica returns null + // regardless of the projection; routing depends only on where + (absent) client. + const args = { include: { attempts: { orderBy: { createdAt: "desc" as const } } } }; + + const okReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const primaryRouter = buildRouter(prisma14, prisma17, okReplica.client); + const viaPrimary = (await primaryRouter.findRun(where, args, prisma14 as never)) as { + id: string; + } | null; + expect(viaPrimary?.id).toBe(runId); + + const lagReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const router = buildRouter(prisma14, prisma17, lagReplica.client); + // Invoke EXACTLY as the caller does: findRun(where, { include }) — NO client argument. + const taskRun = (await router.findRun(where, args)) as { id: string } | null; + + expect(lagReplica.wasHit("taskRun")).toBe(true); + expect(taskRun).toBeNull(); + // Presenter maps null → undefined: "not finished yet, keep polling". + expect(taskRun ?? undefined).toBeUndefined(); + } + ); + + // RunPresenter findRun (no client) — dashboard run-detail page loader. No client → owning REPLICA. + // Under lag the fresh run is invisible → findRun null → the presenter throws RunNotInPgError, which + // the route CATCHES to fall back to the synthesised mollifier-buffer view. That buffer holds exactly + // the just-triggered runs whose rows have not yet drained/replicated — the read-your-writes safety net + // for this window; the page self-heals on the next poll once the replica catches up. + heteroRunOpsPostgresTest( + "RunPresenter findRun (no client) returns null for a run missed on the frozen replica (mollifier-buffer fallback)", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "detail_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${FRIENDLY_25}`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + const where = { friendlyId }; + const select = { + select: { + id: true, + projectId: true, + friendlyId: true, + status: true, + runtimeEnvironmentId: true, + }, + }; + + const okReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const primaryRouter = buildRouter(prisma14, prisma17, okReplica.client); + const viaPrimary = (await primaryRouter.findRun(where, select, prisma14 as never)) as { + id: string; + } | null; + expect(viaPrimary?.id).toBe(runId); + + const lagReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const router = buildRouter(prisma14, prisma17, lagReplica.client); + const run = (await router.findRun(where, select)) as { id: string } | null; + + expect(lagReplica.wasHit("taskRun")).toBe(true); + expect(run).toBeNull(); // → presenter throws RunNotInPgError → route's mollifier-buffer fallback + } + ); + + // RunStreamPresenter findRun (no client) — run-detail SSE trace stream loader. No client → owning + // REPLICA. Under lag the fresh run is invisible → findRun null → traceId stays null and the presenter + // falls back to the mollifier buffer for a traceId; if still unresolved it 404s, but the SSE loader + // RECONNECTS (closing with 404 forces the dashboard to keep retrying). Worst case is a reconnect one + // tick later — the stream attaches as soon as the replica (or buffer) yields the traceId. Read-only. + heteroRunOpsPostgresTest( + "RunStreamPresenter findRun (no client) returns null for a run missed on the frozen replica (buffer/404-then-reconnect)", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "stream_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${FRIENDLY_25}`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + const where = { friendlyId }; + const select = { select: { traceId: true, projectId: true } }; + + const okReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const primaryRouter = buildRouter(prisma14, prisma17, okReplica.client); + const viaPrimary = (await primaryRouter.findRun(where, select, prisma14 as never)) as { + traceId: string; + } | null; + expect(viaPrimary?.traceId).toBe(`trace_${runId}`); + + const lagReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const router = buildRouter(prisma14, prisma17, lagReplica.client); + const run = (await router.findRun(where, select)) as { traceId: string } | null; + + expect(lagReplica.wasHit("taskRun")).toBe(true); + expect(run).toBeNull(); // traceId stays null → buffer fallback → 404-then-SSE-reconnect + } + ); + + // PlaygroundPresenter findRuns (no client) — agent-playground conversation list. Reads conversations + // off $replica, then resolves each conversation's backing run scalars via a client-less findRuns over + // the id set → owning REPLICA (bounded id-set path). Under lag a run missing on the replica is absent + // from runsById, so that conversation renders runFriendlyId=null / runStatus=null / isActive=false — + // a cosmetic "status unknown" on one list row that self-heals on the next list load; the row itself + // still renders. + heteroRunOpsPostgresTest( + "PlaygroundPresenter findRuns (no client) omits a run missed on the frozen replica (null status on that row)", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "playground_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${FRIENDLY_25}`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + }), + }); + + const findRunsArgs = { + where: { id: { in: [runId] } }, + select: { id: true, friendlyId: true, status: true }, + }; + + const okReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const primaryRouter = buildRouter(prisma14, prisma17, okReplica.client); + const viaPrimary = (await primaryRouter.findRuns(findRunsArgs, prisma14 as never)) as Array<{ + id: string; + }>; + expect(viaPrimary.map((r) => r.id)).toEqual([runId]); + + const lagReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const router = buildRouter(prisma14, prisma17, lagReplica.client); + // Invoke EXACTLY as the caller does: findRuns({ where:{id:{in}}, select }) — NO client argument. + const runs = (await router.findRuns(findRunsArgs)) as Array<{ id: string }>; + + expect(lagReplica.wasHit("taskRun")).toBe(true); + expect(runs).toEqual([]); // run absent → conversation row shows null friendlyId/status + } + ); + + // SpanPresenter findRun (+this._replica) — span-detail (run inspector) panel. Passes the BRANDED + // this._replica → owning REPLICA (no escalation). Under lag the fresh run is invisible → findRun null + // → the panel renders its not-found/loading state for this poll and self-heals on the next tick. + // Read-only. (The alternate `{spanId, runtimeEnvironmentId}` branch is unclassifiable → fans + // NEW→LEGACY, still each store's REPLICA — same replica-served conclusion.) + heteroRunOpsPostgresTest( + "SpanPresenter findRun (+this._replica) returns null for a run missed on the frozen replica (span-detail not-found this poll)", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "span_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${FRIENDLY_25}`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + // originalRunId branch: where = {friendlyId, runtimeEnvironmentId}. Representative select subset + // (the full caller select is ~60 scalar/relation fields; routing depends only on where + client, + // and "missing" mode ignores the projection). + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const select = { select: { id: true, friendlyId: true, status: true, spanId: true } }; + + const okReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const primaryRouter = buildRouter(prisma14, prisma17, okReplica.client); + const viaPrimary = (await primaryRouter.findRun(where, select, prisma14 as never)) as { + id: string; + } | null; + expect(viaPrimary?.id).toBe(runId); + + const lagReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const _replica = markReadReplicaClient(lagReplica.client); + const router = buildRouter(prisma14, prisma17, lagReplica.client); + const run = (await router.findRun(where, select, _replica as never)) as { id: string } | null; + + expect(lagReplica.wasHit("taskRun")).toBe(true); + expect(run).toBeNull(); + } + ); + + // SpanPresenter findRuns {parentSpanId} (+this._replica) — the "triggered runs" list on a span-detail + // panel. Passes the BRANDED this._replica; the where is an OPEN predicate ({parentSpanId}) with no id + // set, so it queries BOTH stores' REPLICAS and dedupes. Under lag a just-triggered child run is not + // yet on the replica, so it is absent from the triggered-runs list for this render — a cosmetic "one + // fewer child shown" that self-heals on the next poll; the child still exists and executes. + heteroRunOpsPostgresTest( + "SpanPresenter findRuns {parentSpanId} (+this._replica) omits a child run missed on the frozen replica (triggered-runs list)", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "triggered_leg"); + const parentSpanId = "span_parent_fixed"; + const childRunId = `run_${CUID_25}`; + const childFriendlyId = `run_${FRIENDLY_25}`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: childRunId, + friendlyId: childFriendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + spanId: "span_child_fixed", + parentSpanId, + }), + }); + + const findRunsArgs = { + where: { parentSpanId }, + select: { + friendlyId: true, + taskIdentifier: true, + spanId: true, + createdAt: true, + status: true, + }, + }; + + // PRIMARY contrast: unbranded writer → each leg reads its primary → child run IS listed. + const okReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const primaryRouter = buildRouter(prisma14, prisma17, okReplica.client); + const viaPrimary = (await primaryRouter.findRuns(findRunsArgs, prisma14 as never)) as Array<{ + friendlyId: string; + }>; + expect(viaPrimary.map((r) => r.friendlyId)).toEqual([childFriendlyId]); + + const lagReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const _replica = markReadReplicaClient(lagReplica.client); + const router = buildRouter(prisma14, prisma17, lagReplica.client); + const triggeredRuns = (await router.findRuns(findRunsArgs, _replica as never)) as Array<{ + friendlyId: string; + }>; + + expect(lagReplica.wasHit("taskRun")).toBe(true); + expect(triggeredRuns).toEqual([]); // child omitted from the triggered-runs list this render + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.presentersSessionBatchReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.presentersSessionBatchReadView.replicaLag.test.ts new file mode 100644 index 000000000..dbd1b59bd --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.presentersSessionBatchReadView.replicaLag.test.ts @@ -0,0 +1,344 @@ +// Property: the four run-store reads issued by the AI-session + batch DASHBOARD presenters behave +// correctly under replica lag. Each passes the webapp's BRANDED `$replica` (markReadReplicaClient) into +// the run store; a branded replica makes the routing store's `#ownPrimary(store, client)` return +// undefined, so every leg reads the OWNING store's REPLICA (no primary escalation). Under lag each read +// misses a row present on the owning primary. Real split topology via heteroRunOpsPostgresTest; the +// branded arg is `markReadReplicaClient({})`, whose object is never forwarded across DBs — only its +// BRAND is read, exactly as in production. +// +// Reads covered (one case each): +// 1. SessionListPresenter findRuns (id-set fan-out, each leg owning REPLICA). Consumer builds +// `runById` and emits `currentRunFriendlyId`. A miss drops the row → the link is simply undefined, +// presenter returns normally. Display-view omission on an env-scoped list that revalidates. +// Tolerated. +// 2. SessionPresenter findRuns (id-set fan-out, each leg owning REPLICA). Consumer maps each +// sessionRun to `run: run ? {...} : null`. A miss yields `run: null` for that history row — the +// detail page still renders. Display GET view, no throw. Tolerated. +// 3. SessionPresenter findRun currentRun fallback (id-classified → owning REPLICA, then fan-out). +// The `runsById.get(currentRunId) ?? findRun(...)` fallback. A miss yields `currentRun: null` — +// the detail page renders with no current run highlighted until revalidation. Tolerated. +// 4. BatchPresenter findBatchTaskRunByFriendlyId (env-scoped friendlyId fan-out, each leg owning +// REPLICA). findBatchTaskRunByFriendlyId defaults to `this.readOnlyPrisma` — the lone batch-family +// read that defaults to the replica; findBatchTaskRunById and findBatchTaskRunByIdempotencyKey +// default to `this.prisma` (primary). On a bare null the presenter throws `Error("Batch not found")` +// → a 400 error page. This case pins the desired property: a LIVE batch's detail resolves under lag +// via a primary re-read (recovered below on the owning primary). +// +// Method: build the router as the webapp holds it, seed on the owning LEGACY primary, freeze the owning +// LEGACY replica with the shared laggingReplica, invoke each read with the EXACT caller's args + the +// branded replica client, assert the under-lag behavior, then prove the row is recoverable on the owning +// PRIMARY (unbranded writer → #ownPrimary → owning primary) so the null is purely replica lag + replica +// routing, not a missing row or bad query. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import { markReadReplicaClient } from "./readReplicaClient.js"; +import type { CreateRunInput } from "./types.js"; + +// A cuid (25 chars after the id prefix) classifies LEGACY, so create + read both route to the legacy +// (control-plane) store first — the store that owns these session/run/batch rows in production. +const CUID_25 = "c".repeat(25); + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "EXECUTING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: `trace_${params.runId}`, + spanId: `span_${params.runId}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "EXECUTING", + description: "Run is executing", + runStatus: "EXECUTING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// Build the router the way the webapp holds it, with the LEGACY (control-plane) replica FROZEN for the +// given models (mode "missing" — the just-written row has not replicated). NEW is non-lagging + empty. +function buildRouter( + prisma14: PrismaClient, + prisma17: RunOpsPrismaClient, + frozenModels: readonly ("taskRun" | "batchTaskRun")[] +) { + const legacyReplica = laggingReplica( + prisma14, + frozenModels.map((model) => ({ model, mode: "missing" as const })) + ); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +describe("session and batch presenter read-views under replica lag", () => { + // SessionListPresenter findRuns (branded $replica) — tolerated. + heteroRunOpsPostgresTest( + "SessionListPresenter findRuns returns empty under replica lag, dropping the currentRunFriendlyId link", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, ["taskRun"]); + const seed = await seedEnvironmentLegacy(prisma14, "sesslist"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = "run_sesslist_current"; + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const brandedReplica = markReadReplicaClient({} as object); + const args = { + where: { + id: { in: [runId] }, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }, + select: { id: true, friendlyId: true }, + } as const; + + // Under lag the owning-replica id-set fan-out misses → empty list. + const underLag = (await router.findRuns(args, brandedReplica)) as Array<{ + id: string; + friendlyId: string; + }>; + expect(underLag).toEqual([]); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + // TOLERANCE: the consumer's map lookup — a missing row just yields an undefined link, no throw. + const runById = new Map(underLag.map((r) => [r.id, r] as const)); + const currentRunFriendlyId = runById.get(runId)?.friendlyId; + expect(currentRunFriendlyId).toBeUndefined(); + + // ROUTING proof (replica, not primary): the unbranded WRITER forces the owning primary and finds it. + const onPrimary = (await router.findRuns(args, prisma14 as never)) as Array<{ + id: string; + friendlyId: string; + }>; + expect(onPrimary.map((r) => r.id)).toEqual([runId]); + expect(onPrimary[0]?.friendlyId).toBe(friendlyId); + } + ); + + // SessionPresenter findRuns (branded $replica) — tolerated. + heteroRunOpsPostgresTest( + "SessionPresenter findRuns returns empty under replica lag, rendering run:null for the history row", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, ["taskRun"]); + const seed = await seedEnvironmentLegacy(prisma14, "sesshist"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = "run_sesshist_row"; + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const brandedReplica = markReadReplicaClient({} as object); + const args = { + where: { id: { in: [runId] } }, + select: { id: true, friendlyId: true, status: true }, + } as const; + + const underLag = (await router.findRuns(args, brandedReplica)) as Array<{ + id: string; + friendlyId: string; + status: string; + }>; + expect(underLag).toEqual([]); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + // TOLERANCE: the consumer's mapping — the history row's `run` is null, page still renders. + const runsById = new Map(underLag.map((r) => [r.id, r] as const)); + const hydratedRow = runsById.get(runId); + const rowRun = hydratedRow + ? { friendlyId: hydratedRow.friendlyId, status: hydratedRow.status } + : null; + expect(rowRun).toBeNull(); + + // ROUTING proof: WRITER → owning primary → hydrated. + const onPrimary = (await router.findRuns(args, prisma14 as never)) as Array<{ + id: string; + friendlyId: string; + status: string; + }>; + expect(onPrimary.map((r) => r.id)).toEqual([runId]); + expect(onPrimary[0]?.status).toBe("EXECUTING"); + } + ); + + // SessionPresenter findRun currentRun fallback (branded $replica) — tolerated. + heteroRunOpsPostgresTest( + "SessionPresenter findRun currentRun fallback returns null under replica lag, emitting currentRun:null", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, ["taskRun"]); + const seed = await seedEnvironmentLegacy(prisma14, "currun"); + const currentRunId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = "run_currun"; + await legacyStore.createRun( + buildCreateRunInput({ + runId: currentRunId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const brandedReplica = markReadReplicaClient({} as object); + + // The EXACT fallback read: { id }, select {id,friendlyId,status}, branded $replica. + const underLag = (await router.findRun( + { id: currentRunId }, + { select: { id: true, friendlyId: true, status: true } }, + brandedReplica + )) as { id: string; friendlyId: string; status: string } | null; + + // Replica-routed (id-classified → owning legacy replica, then other-store probe) → null under lag. + expect(underLag).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + // TOLERANCE: the consumer's emit — currentRun is null, detail page still renders. + const currentRun = underLag + ? { friendlyId: underLag.friendlyId, status: underLag.status } + : null; + expect(currentRun).toBeNull(); + + // ROUTING proof: WRITER → readYourWrites → owning primary (findRunOnPrimary) → found. + const onPrimary = (await router.findRun( + { id: currentRunId }, + { select: { id: true, friendlyId: true, status: true } }, + prisma14 as never + )) as { id: string; friendlyId: string; status: string } | null; + expect(onPrimary).not.toBeNull(); + expect(onPrimary!.id).toBe(currentRunId); + expect(onPrimary!.friendlyId).toBe(friendlyId); + } + ); + + // BatchPresenter findBatchTaskRunByFriendlyId (branded $replica) — recovers via primary re-read. + heteroRunOpsPostgresTest( + "BatchPresenter findBatchTaskRunByFriendlyId resolves a live batch under replica lag", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, [ + "batchTaskRun", + ]); + const seed = await seedEnvironmentLegacy(prisma14, "batchview"); + const batchId = `batch_${CUID_25}`; // cuid → LEGACY + const batchFriendlyId = "batch_batchview"; + await legacyStore.createBatchTaskRun({ + id: batchId, + friendlyId: batchFriendlyId, + runtimeEnvironmentId: seed.environment.id, + }); + + const brandedReplica = markReadReplicaClient({} as object); + + // The EXACT read: friendlyId + environmentId + include, branded $replica. + const underLag = await router.findBatchTaskRunByFriendlyId( + batchFriendlyId, + seed.environment.id, + { include: { errors: true } }, + brandedReplica as never + ); + + // Store fact: findBatchTaskRunByFriendlyId defaults to `this.readOnlyPrisma`, so through the + // router (branded → #ownPrimary undefined) BOTH legs read their replica. Under lag → null. + expect(underLag).toBeNull(); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(true); + + // On a bare null the presenter's guard throws "Batch not found" (→ a 400 error page from the route + // loader) for a batch ALIVE on the owning primary — reproduced here on the stale null. + expect(() => { + if (!underLag) { + throw new Error("Batch not found"); + } + }).toThrowError("Batch not found"); + + // The property: the row EXISTS on the owning primary — reading it there (unbranded WRITER → + // #ownPrimary → owning primary) returns the live batch, so the primary re-read resolves the detail. + const onPrimary = await router.findBatchTaskRunByFriendlyId( + batchFriendlyId, + seed.environment.id, + { include: { errors: true } }, + prisma14 as never + ); + expect(onPrimary).not.toBeNull(); + expect(onPrimary!.id).toBe(batchId); + expect(onPrimary!.friendlyId).toBe(batchFriendlyId); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.presentersWaitpointReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.presentersWaitpointReadView.replicaLag.test.ts new file mode 100644 index 000000000..0c7419b35 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.presentersWaitpointReadView.replicaLag.test.ts @@ -0,0 +1,516 @@ +// Replica-lag properties for the waitpoint-family dashboard/API read presenters. Every read here is +// served by the OWNING store's REPLICA (no client passed) and backs a display/GET/list view, so each +// tolerates the lag. On the real split topology (heteroRunOpsPostgresTest, never mocked) we freeze the +// owning (LEGACY) replica via a local proxy that also freezes the $queryRaw connected-run lookup (which +// the shared laggingReplica primitive can't intercept), invoke each read EXACTLY as its +// caller does, assert the stale-under-lag value, and prove the row exists on the PRIMARY (so the miss +// is pure lag). A caller-passed primary client flips the read to the owning primary. + +import { heteroRunOpsPostgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +const CUID_25 = "e".repeat(25); // cuid id-shape -> LEGACY (#legacy / prisma14, full schema) + +// A recording "replica" that has NOT caught up: `taskRun`, `waitpoint`, and `waitpointTag` MODEL reads +// come back empty, and `$queryRaw`/`$queryRawUnsafe` (the connection-join lookups) come back empty too, +// so any replica-routed read misses the just-written row. Everything else forwards to the real client. +// `wasHit` flips true iff an intercepted read was routed here. Writes always land on the PRIMARY +// (this.prisma), so freezing the readOnly client never affects seeding. +function laggingReplica(real: C): { client: C; wasHit: () => boolean } { + let hit = false; + function wrapModel(target: any) { + return new Proxy(target, { + get(innerTarget, prop) { + if (prop === "findFirst" || prop === "findMany" || prop === "findUnique") { + return async () => { + hit = true; + return prop === "findMany" ? [] : null; + }; + } + if (prop === "findFirstOrThrow" || prop === "findUniqueOrThrow") { + return async () => { + hit = true; + throw new Error("lagging replica: row not visible"); + }; + } + return (innerTarget as any)[prop]; + }, + }); + } + const laggingTaskRun = wrapModel((real as any).taskRun); + const laggingWaitpoint = wrapModel((real as any).waitpoint); + const laggingWaitpointTag = wrapModel((real as any).waitpointTag); + const client = new Proxy(real, { + get(target, prop) { + if (prop === "taskRun") return laggingTaskRun; + if (prop === "waitpoint") return laggingWaitpoint; + if (prop === "waitpointTag") return laggingWaitpointTag; + // The connection-id gather runs a raw JOIN (findWaitpointConnectedRunIds); a lagging replica + // has none of those rows yet, so freeze raw reads to empty as well. + if (prop === "$queryRaw" || prop === "$queryRawUnsafe") { + return async () => { + hit = true; + return []; + }; + } + return (target as any)[prop]; + }, + }) as C; + return { client, wasHit: () => hit }; +} + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +// Build a LEGACY-owning router whose legacy replica is frozen (lagging); the NEW store is real +// (non-lagging) so the on-miss fan-out to the other store's replica also legitimately misses. +function buildRouterWithLaggingLegacyReplica(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyReplica = laggingReplica(prisma14); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +// Seed a standalone MANUAL token waitpoint on the WRITER (exactly as minting a resume token does). +async function seedManualWaitpoint( + store: PostgresRunStore, + params: { + id: string; + friendlyId: string; + projectId: string; + environmentId: string; + tags?: string[]; + } +) { + await store.upsertWaitpoint({ + where: { + environmentId_idempotencyKey: { + environmentId: params.environmentId, + idempotencyKey: params.id, + }, + }, + create: { + id: params.id, + friendlyId: params.friendlyId, + type: "MANUAL", + status: "PENDING", + idempotencyKey: params.id, + userProvidedIdempotencyKey: false, + projectId: params.projectId, + environmentId: params.environmentId, + ...(params.tags ? { tags: params.tags } : {}), + }, + update: {}, + }); +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: "COMPLETED_SUCCESSFULLY" as const, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: JSON.stringify({ hello: "world" }), + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +// ── Exact projections the call sites read ───────────────────────────────────────────────────────── +const API_WAITPOINT_SELECT = { + id: true, + friendlyId: true, + type: true, + status: true, + idempotencyKey: true, + userProvidedIdempotencyKey: true, + idempotencyKeyExpiresAt: true, + inactiveIdempotencyKey: true, + output: true, + outputType: true, + outputIsError: true, + completedAfter: true, + completedAt: true, + createdAt: true, + tags: true, +} as const; + +const LIST_WAITPOINT_SELECT = { + id: true, + friendlyId: true, + status: true, + completedAt: true, + completedAfter: true, + outputIsError: true, + idempotencyKey: true, + idempotencyKeyExpiresAt: true, + inactiveIdempotencyKey: true, + userProvidedIdempotencyKey: true, + tags: true, + createdAt: true, +} as const; + +const DETAIL_WAITPOINT_SELECT = { + id: true, + friendlyId: true, + type: true, + status: true, + idempotencyKey: true, + userProvidedIdempotencyKey: true, + idempotencyKeyExpiresAt: true, + inactiveIdempotencyKey: true, + output: true, + outputType: true, + outputIsError: true, + completedAfter: true, + completedAt: true, + createdAt: true, + tags: true, + environmentId: true, +} as const; + +describe("waitpoint-family dashboard/API read presenters under replica lag", () => { + // ApiWaitpointPresenter (GET retrieve loader). + heteroRunOpsPostgresTest( + "ApiWaitpointPresenter findWaitpoint(id) is null under lag; the owning primary resolves it", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( + prisma14, + prisma17 + ); + const seed = await seedEnvironmentLegacy(prisma14, "api_wp"); + const waitpointId = `waitpoint_${CUID_25}`; // cuid → LEGACY + await seedManualWaitpoint(legacyStore, { + id: waitpointId, + friendlyId: "waitpoint_api_wp", + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + + // Exact caller read: select the api projection, where {id, environmentId}, NO client → owning replica. + const fromReplica = await router.findWaitpoint({ + where: { id: waitpointId, environmentId: seed.environment.id }, + select: API_WAITPOINT_SELECT, + }); + // Stale: owning replica lags → null. The route's `if (!waitpoint) throw "Waitpoint not found"` + // fires — a transient error on a read-only GET loader; the client re-requests the retrieve. + // This gates NO write (the mint POST + complete path use their own primary reads), so tolerated. + expect(fromReplica).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // Prove the null is lag, not absence: the same call on the owning PRIMARY resolves the token. + const fromPrimary = await router.findWaitpoint( + { + where: { id: waitpointId, environmentId: seed.environment.id }, + select: API_WAITPOINT_SELECT, + }, + prisma14 + ); + expect(fromPrimary).not.toBeNull(); + expect(fromPrimary!.friendlyId).toBe("waitpoint_api_wp"); + expect(fromPrimary!.type).toBe("MANUAL"); + } + ); + + // WaitpointListPresenter findManyWaitpoints. + heteroRunOpsPostgresTest( + "WaitpointListPresenter findManyWaitpoints is empty under lag; the primary returns the token", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( + prisma14, + prisma17 + ); + const seed = await seedEnvironmentLegacy(prisma14, "list_many"); + const waitpointId = `waitpoint_${CUID_25}`; + await seedManualWaitpoint(legacyStore, { + id: waitpointId, + friendlyId: "waitpoint_list_many", + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + + // Exact caller read: MANUAL scan, keyset order, over-fetch window, NO client → both replicas. + const fromReplica = await router.findManyWaitpoints({ + where: { environmentId: seed.environment.id, type: "MANUAL" }, + orderBy: { id: "desc" }, + take: 26, + select: LIST_WAITPOINT_SELECT, + }); + // Stale: both replicas empty (new has no row; legacy replica frozen). The list simply omits the + // token this render; the next fetch (once caught up) shows it. No write/decision on the stale set. + expect(fromReplica).toEqual([]); + expect(legacyReplica.wasHit()).toBe(true); + + // Prove pure lag: the same scan on the owning primary returns the token. + const fromPrimary = await router.findManyWaitpoints( + { + where: { environmentId: seed.environment.id, type: "MANUAL" }, + orderBy: { id: "desc" }, + take: 26, + select: LIST_WAITPOINT_SELECT, + }, + prisma14 + ); + expect(fromPrimary).toHaveLength(1); + expect(fromPrimary[0]!.friendlyId).toBe("waitpoint_list_many"); + } + ); + + // WaitpointListPresenter #probeAnyToken findWaitpoint. + heteroRunOpsPostgresTest( + "WaitpointListPresenter probe findWaitpoint(MANUAL) is null under lag; the primary finds the token", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( + prisma14, + prisma17 + ); + const seed = await seedEnvironmentLegacy(prisma14, "probe_any"); + const waitpointId = `waitpoint_${CUID_25}`; + await seedManualWaitpoint(legacyStore, { + id: waitpointId, + friendlyId: "waitpoint_probe_any", + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + + // Exact caller read: no id, no select — fan NEW-then-LEGACY on each store's replica. + const fromReplica = await router.findWaitpoint({ + where: { environmentId: seed.environment.id, type: "MANUAL" }, + }); + // Stale null → hasAnyTokens=false. The only user-visible effect is which empty-state copy renders + // ("get started" vs "no matches"); it drives no write and self-corrects on the next render. + expect(fromReplica).toBeNull(); + const hasAnyTokensUnderLag = Boolean(fromReplica); + expect(hasAnyTokensUnderLag).toBe(false); + expect(legacyReplica.wasHit()).toBe(true); + + // Prove pure lag: the same probe on the owning primary finds a MANUAL token (hasAnyTokens=true). + const fromPrimary = await router.findWaitpoint( + { where: { environmentId: seed.environment.id, type: "MANUAL" } }, + prisma14 + ); + expect(fromPrimary).not.toBeNull(); + } + ); + + // WaitpointPresenter #findWaitpoint(friendlyId). + heteroRunOpsPostgresTest( + "WaitpointPresenter findWaitpoint(friendlyId) is null under lag; the primary resolves it", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( + prisma14, + prisma17 + ); + const seed = await seedEnvironmentLegacy(prisma14, "detail_wp"); + const waitpointId = `waitpoint_${CUID_25}`; + const friendlyId = "waitpoint_detail_wp"; + await seedManualWaitpoint(legacyStore, { + id: waitpointId, + friendlyId, + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + + // Exact caller read: where {friendlyId, environmentId} (no id) + detail select, NO client → replicas. + const fromReplica = await router.findWaitpoint({ + where: { friendlyId, environmentId: seed.environment.id }, + select: DETAIL_WAITPOINT_SELECT, + }); + // Stale null → presenter.call logs "Waitpoint not found" and returns null; the detail page shows + // not-found and the user reloads. Read-only render, no mutation gated. Tolerated. + expect(fromReplica).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + const fromPrimary = await router.findWaitpoint( + { + where: { friendlyId, environmentId: seed.environment.id }, + select: DETAIL_WAITPOINT_SELECT, + }, + prisma14 + ); + expect(fromPrimary).not.toBeNull(); + expect(fromPrimary!.environmentId).toBe(seed.environment.id); + } + ); + + // WaitpointPresenter findWaitpointConnectedRunIds. + heteroRunOpsPostgresTest( + "WaitpointPresenter findWaitpointConnectedRunIds is empty under lag; the primary returns the connection", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( + prisma14, + prisma17 + ); + const seed = await seedEnvironmentLegacy(prisma14, "conn_ids"); + const waitpointId = `waitpoint_${CUID_25}`; + const runId = `run_${CUID_25}`; // cuid → LEGACY, co-located with the token + await seedManualWaitpoint(legacyStore, { + id: waitpointId, + friendlyId: "waitpoint_conn_ids", + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId: "run_conn_ids", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + // The run↔waitpoint join is written on the run's DB (blockRunWithWaitpointEdges, routed by runId). + await router.blockRunWithWaitpointEdges({ + runId, + waitpointIds: [waitpointId], + projectId: seed.project.id, + }); + + // Exact caller read: fan the connection JOIN across both stores' replicas, NO client. + const fromReplica = await router.findWaitpointConnectedRunIds(waitpointId); + // Stale: both replicas' join reads empty (new has none; legacy replica raw frozen). The detail + // page shows an empty/partial "connected runs" list; a refresh fills it. Pure display. + expect(fromReplica).toEqual([]); + expect(legacyReplica.wasHit()).toBe(true); + + // Prove pure lag: the same fan-out on the owning primary returns the connected run id. + const fromPrimary = await router.findWaitpointConnectedRunIds(waitpointId, prisma14); + expect(fromPrimary).toEqual([runId]); + } + ); + + // WaitpointPresenter findRuns({id:{in}}). + heteroRunOpsPostgresTest( + "WaitpointPresenter findRuns(id in) is empty under lag; the primary returns the run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica } = buildRouterWithLaggingLegacyReplica(prisma14, prisma17); + const seed = await seedEnvironmentLegacy(prisma14, "conn_runs"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId: "run_conn_runs", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + // Exact caller read: id-set + friendlyId projection + take, NO client → owning replica(s). + const fromReplica = (await router.findRuns({ + where: { id: { in: [runId] } }, + select: { friendlyId: true }, + take: 5, + })) as Array<{ friendlyId: string }>; + // Stale: owning replica lags → []; the detail page's connected-runs block renders nothing this + // pass. The set is only mapped to friendlyIds for display — no decision on the stale result. + expect(fromReplica).toEqual([]); + expect(legacyReplica.wasHit()).toBe(true); + + // Prove pure lag: the same query on the owning primary returns the run's friendlyId. + const fromPrimary = (await router.findRuns( + { where: { id: { in: [runId] } }, select: { friendlyId: true }, take: 5 }, + prisma14 + )) as Array<{ friendlyId: string }>; + expect(fromPrimary).toHaveLength(1); + expect(fromPrimary[0]!.friendlyId).toBe("run_conn_runs"); + } + ); + + // WaitpointTagListPresenter findManyWaitpointTags. + heteroRunOpsPostgresTest( + "WaitpointTagListPresenter findManyWaitpointTags is empty under lag; the primary returns the tag", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( + prisma14, + prisma17 + ); + const seed = await seedEnvironmentLegacy(prisma14, "tag_list"); + await legacyStore.upsertWaitpointTag({ + environmentId: seed.environment.id, + name: "my-tag", + projectId: seed.project.id, + }); + + // Exact caller read: env + optional name filter, orderBy id desc, over-fetch window, NO client. + const fromReplica = await router.findManyWaitpointTags({ + where: { environmentId: seed.environment.id, name: undefined }, + orderBy: { id: "desc" }, + take: 26, + skip: 0, + }); + // Stale: both replicas' waitpointTag reads empty. The tag filter dropdown omits the new tag this + // render; it appears once the replica catches up. No write/decision. + expect(fromReplica).toEqual([]); + expect(legacyReplica.wasHit()).toBe(true); + + // Prove pure lag: the same query on the owning primary returns the tag. + const fromPrimary = await router.findManyWaitpointTags( + { + where: { environmentId: seed.environment.id, name: undefined }, + orderBy: { id: "desc" }, + take: 26, + skip: 0, + }, + prisma14 + ); + expect(fromPrimary).toHaveLength(1); + expect(fromPrimary[0]!.name).toBe("my-tag"); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.realtimeServicesReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.realtimeServicesReadView.replicaLag.test.ts new file mode 100644 index 000000000..9a3b19e2c --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.realtimeServicesReadView.replicaLag.test.ts @@ -0,0 +1,356 @@ +// Lagging-replica coverage for the REALTIME-SERVICES read views on the run-ops split. Five run lookups +// feed realtime session/feed serialization; none is a read-your-writes mutation gate — each resolves an +// id/friendlyId for DISPLAY and tolerates a stale/missing read by construction. This file freezes the +// OWNING store's replica via laggingReplica, invokes each read EXACTLY as its caller does (same method + +// client arg), and asserts the concrete under-lag value plus a primary re-read that recovers the row (so +// the null/empty is provably lag-induced). No-client and branded-$replica reads both stay on the owning +// replica; a writer/tx read escalates to the primary. Real split topology via heteroRunOpsPostgresTest — NEVER mocked. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import { markReadReplicaClient } from "./readReplicaClient.js"; +import type { CreateRunInput } from "./types.js"; + +// ownerEngine (classifyResidency) routes a run-ops v1 body → NEW, everything else → LEGACY. +// These sites all resolve pre-existing session/feed runs; LEGACY-resident cuid runs exercise the +// owning-store-first → other-store fan-out on the control-plane DB, the common realtime residency. +const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema) + +async function seedEnvironment(prisma: PrismaClient, slugSuffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + taskIdentifier: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "PENDING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: params.taskIdentifier, + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: ["alpha", "beta"], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// Build the router the way the realtime services hold it: a LEGACY store on the control-plane DB +// whose replica LAGS, plus a fresh NEW store. Returns the router + the seeded run + the lagging +// replica probe. All five sites resolve a LEGACY-resident (cuid) run, so the owning store is LEGACY +// and its frozen replica is the one the read hits. +async function setupLaggingLegacy( + prisma14: PrismaClient, + prisma17: RunOpsPrismaClient, + slug: string +) { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, slug); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = `run_${slug}`; + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + taskIdentifier: "my-task", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + return { router, legacyReplica, seed, runId, friendlyId }; +} + +// The webapp always passes its $replica handle, which is markReadReplicaClient()'d. +// A branded client tells RoutingRunStore "do not escalate to primary" — so the read stays on the +// owning store's own replica. We pass a branded stand-in exactly as the hydrator/session manager do; +// its identity is irrelevant because the router discards it and reads the owning store's readOnlyPrisma. +const BRANDED_REPLICA = markReadReplicaClient({}) as never; + +describe("run-ops split — realtime-services read views vs. a lagging replica", () => { + // --- hydrateByIds findRuns (runReader) --------------------------------------------------------- + // The realtime feed hydrates a ClickHouse-resolved id-set for DISPLAY, passing $replica. Under lag + // the owning replica returns no rows. Tolerated: the id-set is sourced from ClickHouse, which lags + // the Postgres replica by MORE (an id can't appear in CH before it is on the PG replica), and wake + // hydrates are additionally held by the envChangeRouter replica-lag gate; a row missed on one + // hydrate tick reappears on the next. hydrateByIds returns whatever rows exist — a missing row is + // simply absent from the feed frame, never a wrong value. Store fact under lag: []. + heteroRunOpsPostgresTest( + "hydrateByIds findRuns(branded $replica) is empty under owning-replica lag; primary re-read hydrates the row", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, runId, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rr_hydrate" + ); + + // Exactly the hydrateByIds read — where {runtimeEnvironmentId, id:{in}}, select, $replica. + const staleRows = await router.findRuns( + { + where: { + runtimeEnvironmentId: seed.environment.id, + id: { in: [runId] }, + }, + select: { id: true, friendlyId: true, status: true }, + }, + BRANDED_REPLICA + ); + expect(staleRows).toEqual([]); + expect(legacyReplica.wasHit()).toBe(true); + + // The row IS on the primary — a writer-client read (escalates to owning primary) hydrates it. + const primaryRows = await router.findRuns( + { + where: { + runtimeEnvironmentId: seed.environment.id, + id: { in: [runId] }, + }, + select: { id: true, friendlyId: true, status: true }, + }, + prisma14 as never + ); + expect(primaryRows).toHaveLength(1); + expect(primaryRows[0]!.friendlyId).toBe(friendlyId); + } + ); + + // --- #fetch / getRunById findRun (runReader) --------------------------------------------------- + // Single-run hydrate for the feed, passing $replica. Under lag the owning replica (then the + // cross-store fan-out on miss) returns null. Tolerated: RunHydror.getRunById caches a null hit for + // only cacheTtlMs (250ms default) and the feed re-fetches; #fetch returns (run ?? null) — a null is + // "not yet visible", rendered as an absent frame in a read-only display feed, never a decision. + // Store fact under lag: null. + heteroRunOpsPostgresTest( + "#fetch findRun(branded $replica) is null under owning-replica lag; primary re-read finds the row", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, runId, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rr_fetch" + ); + + // Exactly the #fetch read — where {id, runtimeEnvironmentId}, select, $replica. + const stale = await router.findRun( + { id: runId, runtimeEnvironmentId: seed.environment.id }, + { select: { id: true, friendlyId: true, status: true } }, + BRANDED_REPLICA + ); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + const primary = await router.findRun( + { id: runId, runtimeEnvironmentId: seed.environment.id }, + { select: { id: true, friendlyId: true, status: true } }, + prisma14 as never + ); + expect(primary).not.toBeNull(); + expect((primary as { friendlyId: string }).friendlyId).toBe(friendlyId); + } + ); + + // --- resolveRunFriendlyId findRun (sessionRunManager) ------------------------------------------ + // Resolves a run cuid → friendlyId for `payload.previousRunId`, passing $replica. Under lag the + // read is null. Tolerated: the caller is `return row?.friendlyId ?? runId` — on a null it falls back + // to the cuid, and previousRunId is customer-visible bookkeeping only, so a stale-but-non-null value + // is acceptable degraded behavior. Store fact under lag: null → caller returns the cuid. + heteroRunOpsPostgresTest( + "resolveRunFriendlyId findRun(branded $replica) is null under lag; caller falls back to the cuid", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, runId, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "srm_resolve" + ); + + // Exactly the resolveRunFriendlyId read — where {id}, select {friendlyId}, $replica. + const stale = await router.findRun( + { id: runId }, + { select: { friendlyId: true } }, + BRANDED_REPLICA + ); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // The caller's tolerance, exercised: `row?.friendlyId ?? runId`. + const resolved = (stale as { friendlyId: string } | null)?.friendlyId ?? runId; + expect(resolved).toBe(runId); // degraded: the cuid, not the friendlyId + + // Proof the row exists on the primary (writer read would have returned the friendlyId). + const primary = await router.findRun( + { id: runId }, + { select: { friendlyId: true } }, + prisma14 as never + ); + expect((primary as { friendlyId: string }).friendlyId).toBe(friendlyId); + } + ); + + // --- serializeSessionWithFriendlyRunId findRun (sessions) -------------------------------------- + // Resolves Session.currentRunId (internal cuid) → friendlyId for single-row session responses, + // NO client → owning replica. Under lag the read is null. Tolerated: serialized on GET/PATCH/close + // routes over a PRE-EXISTING currentRunId (the PATCH never writes currentRunId — the pointer was + // written by an earlier append), so this is a read-only display resolve, not a same-request + // read-your-writes. The caller is `currentRunId: run?.friendlyId ?? null`: null is the SAFE degraded + // direction, and the tenant-scoped where {projectId, runtimeEnvironmentId} guarantees a stale read + // never mis-resolves a run in another env. The client re-fetches. Store fact under lag: null. + heteroRunOpsPostgresTest( + "serializeSessionWithFriendlyRunId findRun(no client) is null under lag → currentRunId null", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, runId, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "sess_one" + ); + + // Exactly the serializeSessionWithFriendlyRunId read — where {id, projectId, runtimeEnvironmentId}, select, NO client. + const stale = await router.findRun( + { + id: runId, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }, + { select: { friendlyId: true } } + ); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // Caller tolerance, exercised. + const currentRunId = (stale as { friendlyId: string } | null)?.friendlyId ?? null; + expect(currentRunId).toBeNull(); + + // Proof the row exists on the primary. + const primary = await router.findRun( + { + id: runId, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }, + { select: { friendlyId: true } }, + prisma14 as never + ); + expect((primary as { friendlyId: string }).friendlyId).toBe(friendlyId); + } + ); + + // --- serializeSessionsWithFriendlyRunIds findRuns (sessions) ----------------------------------- + // Batched currentRunId → friendlyId resolve for the session LIST endpoint, NO client → owning + // replica. Under lag the id-set read returns no rows. Tolerated: same read-only display resolve of + // pre-existing pointers as the single-session case above, batched. The caller is + // `friendlyIdByRunId.get(session.currentRunId) ?? null` over a Map built only from rows that came + // back, so a missing run serializes currentRunId=null — the safe degraded direction. The where is + // tenant-scoped {projectId, runtimeEnvironmentId}, so a stale id never resolves a run in another env. + // Store fact under lag: [] → empty map → currentRunId null. + heteroRunOpsPostgresTest( + "serializeSessionsWithFriendlyRunIds findRuns(no client) is empty under lag → currentRunId null", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, runId, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "sess_list" + ); + + // Exactly the serializeSessionsWithFriendlyRunIds read — where {id:{in}, projectId, runtimeEnvironmentId}, select, NO client. + const staleRuns = await router.findRuns({ + where: { + id: { in: [runId] }, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }, + select: { id: true, friendlyId: true }, + }); + expect(staleRuns).toEqual([]); + expect(legacyReplica.wasHit()).toBe(true); + + // Caller tolerance, exercised. + const friendlyIdByRunId = new Map( + (staleRuns as Array<{ id: string; friendlyId: string }>).map((r) => [r.id, r.friendlyId]) + ); + const currentRunId = friendlyIdByRunId.get(runId) ?? null; + expect(currentRunId).toBeNull(); + + // Proof the row exists on the primary. + const primaryRuns = await router.findRuns( + { + where: { + id: { in: [runId] }, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }, + select: { id: true, friendlyId: true }, + }, + prisma14 as never + ); + expect(primaryRuns).toHaveLength(1); + expect((primaryRuns as Array<{ friendlyId: string }>)[0]!.friendlyId).toBe(friendlyId); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.replayReadAfterWrite.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.replayReadAfterWrite.replicaLag.test.ts new file mode 100644 index 000000000..297449412 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.replayReadAfterWrite.replicaLag.test.ts @@ -0,0 +1,211 @@ +// Replica-lag coverage for the REPLAY route's source-run reads. The loader, resolveRunOrganizationId, +// and the action all call runStore.findRun({ friendlyId }) with NO client, so the read routes to the +// owning store's REPLICA (fan-out to the other store's replica on a miss). Passing the control-plane +// WRITER instead escalates to findRunOnPrimary (read-your-writes). +// +// The route is a Remix loader/action not unit-testable with an injected store here, so we exercise the +// exact findRun pattern each caller uses against a real split topology with the owning replica frozen. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; + +const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema) + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: "COMPLETED_SUCCESSFULLY" as const, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: JSON.stringify({ hello: "world" }), + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +// Build a LEGACY-owning router whose legacy replica is frozen (lagging), and a real (non-lagging) NEW +// store so the on-miss fan-out to the other store's replica also misses. Returns the router + a probe. +function buildRouterWithLaggingLegacyReplica(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyReplica }; +} + +// The replay route reads the source run keyed by friendlyId (runParam) — mirror that exactly. +const REPLAY_SELECT = { + payload: true, + payloadType: true, + runtimeEnvironmentId: true, + projectId: true, + taskIdentifier: true, +} as const; + +describe("replay route source-run reads under replica lag (findRun by friendlyId)", () => { + // The action does `runStore.findRun({ friendlyId })` with NO client, so the read routes to the + // lagging replica and MISSES; its only miss-handler is the mollifier buffer, so once the buffered + // run has drained to the primary but not the replica the action would fall through to "Run not + // found" for a run that exists on the primary. Passing the control-plane WRITER (like + // resolveRunOrganizationId's primary fallback) makes the read resolve the fresh run. + heteroRunOpsPostgresTest( + "action read (no client) hits the lagging replica and MISSES a run that exists on the primary", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironmentLegacy(prisma14, "replay_action"); + const runId = `run_${CUID_25}`; + const friendlyId = "run_replay_action"; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + // Exactly the action's call: findRun({ friendlyId }) with NO client → owning store's replica, + // then fan-out to the other store's replica. Both miss under lag → null → "Run not found". + const { router, legacyReplica } = buildRouterWithLaggingLegacyReplica(prisma14, prisma17); + const viaReplica = await router.findRun({ friendlyId }); + expect(viaReplica).toBeNull(); // replica misses; the action re-reads the primary + expect(legacyReplica.wasHit()).toBe(true); + + // The action passes the control-plane WRITER (like resolveRunOrganizationId's primary fallback) + // → resolves the fresh run on the owning primary, never the replica. + const { router: routerPrimary, legacyReplica: legacyReplicaPrimary } = + buildRouterWithLaggingLegacyReplica(prisma14, prisma17); + const viaPrimary = await routerPrimary.findRun({ friendlyId }, prisma14); + expect(viaPrimary).not.toBeNull(); + expect((viaPrimary as { friendlyId: string }).friendlyId).toBe(friendlyId); + expect(legacyReplicaPrimary.wasHit()).toBe(false); + } + ); + + // resolveRunOrganizationId reads the replica first (misses under lag), checks the buffer, then FALLS + // BACK to the primary via `runStore.findRun(..., prisma)`. This documents that the primary-fallback + // leg resolves the run (and thus runtimeEnvironmentId → organizationId for the RBAC scope) even when + // the replica is frozen and the buffer is drained. + heteroRunOpsPostgresTest( + "resolveRunOrganizationId: replica leg misses under lag, primary-fallback leg resolves the run", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironmentLegacy(prisma14, "replay_org"); + const runId = `run_${CUID_25}`; + const friendlyId = "run_replay_org"; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + const { router, legacyReplica } = buildRouterWithLaggingLegacyReplica(prisma14, prisma17); + + // Leg 1 (the route's first read): replica, no client → miss under lag. + const replicaLeg = await router.findRun( + { friendlyId }, + { select: { runtimeEnvironmentId: true } } + ); + expect(replicaLeg).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // Leg 2 (the route's primary fallback): findRun(..., prisma) → findRunOnPrimary resolves the + // run so the org scope is never left unresolved under replica lag. + const primaryLeg = await router.findRun( + { friendlyId }, + { select: { runtimeEnvironmentId: true } }, + prisma14 + ); + expect(primaryLeg).not.toBeNull(); + expect((primaryLeg as { runtimeEnvironmentId: string }).runtimeEnvironmentId).toBe( + seed.environment.id + ); + } + ); + + // The loader read is a read-view, included only to document the routing (no staleness assertion). + // The loader renders the replay DIALOG from the source run; on a replica miss it uses the mollifier + // buffer, else the user retries. Eventual consistency is the intended contract for rendering a form + // for an (already-existing, historical) run — it drives no write. Kept here so the full group of + // replay reads is represented; the primary-logic coverage is the two cases above. + heteroRunOpsPostgresTest( + "loader read (no client) routes to the replica (read-view — documents routing only)", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironmentLegacy(prisma14, "replay_loader"); + const runId = `run_${CUID_25}`; + const friendlyId = "run_replay_loader"; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + const { router, legacyReplica } = buildRouterWithLaggingLegacyReplica(prisma14, prisma17); + await router.findRun({ friendlyId }, { select: REPLAY_SELECT }); + // No client → the loader's read went to the replica, as designed for a display read. + expect(legacyReplica.wasHit()).toBe(true); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.resolveRunForMutationReplicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.resolveRunForMutationReplicaLag.test.ts new file mode 100644 index 000000000..efa1dab8b --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.resolveRunForMutationReplicaLag.test.ts @@ -0,0 +1,200 @@ +// Store-seam characterization of the read primitive behind mollifier resolveRunForMutation — NOT a +// caller guard. The resolver reads findRun on the branded $replica, then re-probes the writer on a +// miss; here we reproduce just that two-step read against real Postgres with the owning replica frozen, +// to show the branded-replica read genuinely misses a fresh row (readYourWrites=false) while the writer +// re-probe recovers it. The caller contract (resolveRunForMutation returns source:"pg" instead of a +// spurious 404 under lag) is locked separately by the real caller-driven guards that drive the exported +// resolver end-to-end. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import { markReadReplicaClient } from "./readReplicaClient.js"; + +// A cuid (25 chars after the `run_` prefix) classifies LEGACY, so both the create and the +// friendlyId-keyed read route to the legacy (control-plane) store first. +const CUID_25 = "c".repeat(25); +const FRIENDLY_25 = "d".repeat(25); + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: "PENDING" as const, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +// Mimic the mollifier resolver's PG portion exactly: read the BRANDED replica first, then re-probe the +// WRITER on a miss. (The buffer probe between the two is orthogonal to the replica-lag question.) +async function resolveViaRunStorePgReads( + router: RoutingRunStore, + where: { friendlyId: string; runtimeEnvironmentId: string }, + brandedReplica: unknown, + writer: PrismaClient +): Promise<{ friendlyId: string } | null> { + const pgRun = (await router.findRun(where, { select: { friendlyId: true } }, brandedReplica)) as { + friendlyId: string; + } | null; + if (pgRun) return { friendlyId: pgRun.friendlyId }; + + const writerRun = (await router.findRun(where, { select: { friendlyId: true } }, writer)) as { + friendlyId: string; + } | null; + if (writerRun) return { friendlyId: writerRun.friendlyId }; + + return null; +} + +describe("store reads behind mollifier resolveRunForMutation — replica-first, writer-probe recovers", () => { + heteroRunOpsPostgresTest( + "fresh run invisible on the lagging replica is still resolved via the writer probe", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironmentLegacy(prisma14, "mollifier_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = `run_${FRIENDLY_25}`; // classifies LEGACY too → routes to owning store first + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + + // The resolver's phase-1 client: the BRANDED $replica (readYourWrites=false → replica routing). + const brandedReplica = markReadReplicaClient({} as object); + + // HAZARD proof: the phase-1 replica read alone misses the fresh row under lag. + const phase1Only = (await router.findRun( + where, + { select: { friendlyId: true } }, + brandedReplica + )) as { friendlyId: string } | null; + expect(phase1Only).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // TOLERANCE proof: the full resolver pattern (replica-first, then writer-probe) resolves the run + // despite the lagging replica, so the mutation route does NOT wrongly 404. + const legacyReplica2 = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore2 = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica2.client, + schemaVariant: "legacy", + }); + const router2 = new RoutingRunStore({ new: newStore, legacy: legacyStore2 }); + + const resolved = await resolveViaRunStorePgReads(router2, where, brandedReplica, prisma14); + expect(resolved).not.toBeNull(); + expect(resolved?.friendlyId).toBe(friendlyId); + // The replica was consulted (phase 1) but the writer probe (phase 2) is what recovered the row. + expect(legacyReplica2.wasHit()).toBe(true); + } + ); + + heteroRunOpsPostgresTest( + "phase-1 replica read alone resolves the run in steady state (writer probe not needed)", + async ({ prisma14, prisma17 }) => { + // Non-lagging: legacy store reads its real replica (prisma14 itself here as the replica handle). + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironmentLegacy(prisma14, "mollifier_ok"); + const runId = `run_${CUID_25}`; + const friendlyId = `run_${FRIENDLY_25}`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const brandedReplica = markReadReplicaClient({} as object); + + const phase1 = (await router.findRun( + where, + { select: { friendlyId: true } }, + brandedReplica + )) as { friendlyId: string } | null; + expect(phase1).not.toBeNull(); + expect(phase1?.friendlyId).toBe(friendlyId); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.routesBatchGetReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.routesBatchGetReadView.replicaLag.test.ts new file mode 100644 index 000000000..f34101eac --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.routesBatchGetReadView.replicaLag.test.ts @@ -0,0 +1,312 @@ +// Coverage for the batch-GET route/loader reads issued with NO client, grouped as "routes-batch-get". +// Five external-API callers, two store methods routed differently: +// +// api.v1.batches.$batchId findBatchTaskRunByFriendlyId(id, env.id, {errors}) [GET] +// api.v2.batches.$batchId findBatchTaskRunByFriendlyId(id, env.id, {errors}) [GET] +// realtime.v1.batches.$batchId findBatchTaskRunByFriendlyId(id, env.id) [subscribe] +// api.v3.batches findBatchTaskRunById(cachedRequestId) [request-idempotency dedup] +// api.v2.tasks.batch findBatchTaskRunById(cachedRequestId) [request-idempotency dedup] +// +// Routing. The router fans BOTH methods out NEW→LEGACY, each leg via #ownPrimary(store, client). For a +// NO-client call #ownPrimary returns undefined, so each leg falls to the store default — and the two +// methods default DIFFERENTLY in PostgresRunStore: +// - findBatchTaskRunByFriendlyId `client ?? this.readOnlyPrisma` → REPLICA +// - findBatchTaskRunById `client ?? this.prisma` → PRIMARY +// So the three friendlyId GET/subscribe callers read the owning REPLICA; the two byId dedup callers read +// the owning PRIMARY. Every case proves its routing (frozen owning replica consulted, or not). +// +// findBatchTaskRunByFriendlyId (REPLICA) — api.v1 / api.v2 / realtime.v1: +// The friendlyId is looked up on the owning REPLICA. Under lag a just-created batch (committed to the +// owning PRIMARY) is not yet visible → the read returns null → createLoaderApiRoute short-circuits with +// `{ error: "Not found" }, status 404` and header `x-should-retry: false`, which the SDK obeys verbatim +// (no retry). The batch is live on the owning primary, recoverable via a primary re-read (asserted +// below). Contrast the sibling RUN-get routes, which set `shouldRetryNotFound: true` so the SDK retries +// the 404 through lag. +// +// findBatchTaskRunById (PRIMARY) — api.v3.batches / api.v2.tasks.batch — tolerated: +// The request-idempotency dedup (handleRequestIdempotency.findCachedEntity) looks up the cached batch +// by id on the owning PRIMARY. Under owning-replica lag the just-written batch is still found, so the +// retried create returns isCached instead of minting a DUPLICATE batch. Tolerance = PRIMARY routing, +// proven by wasHit("batchTaskRun") === false (the frozen replica is never consulted) AND a correct row. +// +// Build the RoutingRunStore as the webapp holds it (NEW = dedicated subset store / prisma17; LEGACY = +// control-plane store / prisma14 whose REPLICA is the frozen lagging client). Seed the batch (cuid id → +// LEGACY-owned) on the LEGACY PRIMARY, freeze the LEGACY replica (batchTaskRun + batchTaskRunItem, +// "missing") with the shared laggingReplica, then invoke each read EXACTLY as the caller does — same +// method, same args, NO client. Real split topology via heteroRunOpsPostgresTest — never mocked. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CreateBatchTaskRunData } from "./types.js"; + +// A cuid (25 chars after the `batch_` prefix) classifies LEGACY, so the batch is owned by the legacy +// (control-plane) store; the client-less reads then fan out / land on the LEGACY leg. +const CUID_25 = "c".repeat(25); + +async function seedEnvironment(prisma: PrismaClient, slugSuffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +// Build the RoutingRunStore the same way the webapp does. The LEGACY replica is frozen in "missing" mode +// for the batch models: any REPLICA-routed batch read comes back null/[]/0 AND flips wasHit — so +// wasHit(true) + a null result proves REPLICA routing (the findBatchTaskRunByFriendlyId cases above), +// and wasHit(false) + a correct row proves PRIMARY routing (the findBatchTaskRunById cases). +function buildLaggingRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyReplica = laggingReplica(prisma14, [ + { model: "batchTaskRun", mode: "missing" }, + { model: "batchTaskRunItem", mode: "missing" }, + ]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +function batchData(overrides: Partial): CreateBatchTaskRunData { + return { + id: `batch_${CUID_25}`, + friendlyId: "batch_route_get_f", + runtimeEnvironmentId: "PLACEHOLDER", + status: "PENDING", + runCount: 1, + runIds: ["run_child_1"], + expectedCount: 1, + batchVersion: "runengine:v2", + sealed: false, + ...overrides, + }; +} + +// Reproduce the createLoaderApiRoute resource gate + the SDK's header obedience: a null findResource → +// 404 with x-should-retry "false" → the SDK returns { retry: false }. This is the mechanism that turns a +// stale replica null into a NON-RETRYABLE 404. `shouldRetryNotFound` is UNSET on all three batch routes. +function routeOutcomeForResource( + resource: unknown, + opts: { shouldRetryNotFound?: boolean } = {} +): { status: number; xShouldRetry: string | null; sdkWillRetry: boolean } { + if (!resource) { + const xShouldRetry = opts.shouldRetryNotFound ? "true" : "false"; + return { status: 404, xShouldRetry, sdkWillRetry: xShouldRetry === "true" }; + } + return { status: 200, xShouldRetry: null, sdkWillRetry: false }; +} + +describe("batch-GET route reads under replica lag", () => { + // api.v1.batches — findBatchTaskRunByFriendlyId (REPLICA). + heteroRunOpsPostgresTest( + "api.v1.batches findBatchTaskRunByFriendlyId reads stale-null under replica lag, yielding a non-retryable 404", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bget_v1"); + const friendlyId = "batch_bget_v1_f"; + await legacyStore.createBatchTaskRun( + batchData({ friendlyId, runtimeEnvironmentId: seed.environment.id }) + ); + + // Exact caller invocation: (friendlyId, environment.id, { include: { errors: true } }), NO client. + const underLag = await router.findBatchTaskRunByFriendlyId(friendlyId, seed.environment.id, { + include: { errors: true }, + }); + + // Stale null for a batch that exists on the primary → the route's resource gate fires. + expect(underLag).toBeNull(); + // REPLICA routing proof: the frozen owning replica WAS consulted (this is the split hazard). + expect(legacyReplica.wasHit("batchTaskRun")).toBe(true); + + // The replica-routed miss yields a non-retryable 404 of a LIVE batch. + const outcome = routeOutcomeForResource(underLag); // shouldRetryNotFound unset on this route + expect(outcome.status).toBe(404); + expect(outcome.xShouldRetry).toBe("false"); + expect(outcome.sdkWillRetry).toBe(false); + + // The null is PURELY replica lag + replica routing, not a missing row / bad query. Passing the + // owning WRITER escalates the LEGACY leg to its PRIMARY (#ownPrimary) and the batch is found — a + // primary route / primary re-read resolves it. + const onPrimary = await router.findBatchTaskRunByFriendlyId( + friendlyId, + seed.environment.id, + { include: { errors: true } }, + prisma14 as never + ); + expect(onPrimary).not.toBeNull(); + expect(onPrimary!.friendlyId).toBe(friendlyId); + expect(Array.isArray(onPrimary!.errors)).toBe(true); + expect(routeOutcomeForResource(onPrimary).status).toBe(200); + } + ); + + // api.v2.batches — findBatchTaskRunByFriendlyId (REPLICA). + heteroRunOpsPostgresTest( + "api.v2.batches findBatchTaskRunByFriendlyId reads stale-null under replica lag, yielding a non-retryable 404", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bget_v2"); + const friendlyId = "batch_bget_v2_f"; + await legacyStore.createBatchTaskRun( + batchData({ + friendlyId, + runtimeEnvironmentId: seed.environment.id, + processingCompletedAt: new Date(), + }) + ); + + const underLag = await router.findBatchTaskRunByFriendlyId(friendlyId, seed.environment.id, { + include: { errors: true }, + }); + expect(underLag).toBeNull(); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(true); + + const outcome = routeOutcomeForResource(underLag); + expect(outcome.status).toBe(404); + expect(outcome.sdkWillRetry).toBe(false); + + const onPrimary = await router.findBatchTaskRunByFriendlyId( + friendlyId, + seed.environment.id, + { include: { errors: true } }, + prisma14 as never + ); + expect(onPrimary).not.toBeNull(); + expect(onPrimary!.friendlyId).toBe(friendlyId); + } + ); + + // realtime.v1.batches — findBatchTaskRunByFriendlyId (REPLICA). + heteroRunOpsPostgresTest( + "realtime.v1.batches findBatchTaskRunByFriendlyId reads stale-null under replica lag, 404ing before the subscription starts", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bget_rt"); + const friendlyId = "batch_bget_rt_f"; + const batchId = `batch_${CUID_25}`; + await legacyStore.createBatchTaskRun( + batchData({ id: batchId, friendlyId, runtimeEnvironmentId: seed.environment.id }) + ); + + // Exact caller invocation: (friendlyId, environment.id) — NO include, NO client. + const underLag = await router.findBatchTaskRunByFriendlyId(friendlyId, seed.environment.id); + expect(underLag).toBeNull(); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(true); + + // The loader returns 404 at the resource gate BEFORE reaching streamBatch — the subscription + // never starts. shouldRetryNotFound is unset here too → non-retryable. + const outcome = routeOutcomeForResource(underLag); + expect(outcome.status).toBe(404); + expect(outcome.sdkWillRetry).toBe(false); + + // Owning WRITER escalates to the LEGACY primary; batchRun.id (fed to streamBatch) resolves. + const onPrimary = await router.findBatchTaskRunByFriendlyId( + friendlyId, + seed.environment.id, + undefined, + prisma14 as never + ); + expect(onPrimary).not.toBeNull(); + expect(onPrimary!.id).toBe(batchId); + } + ); + + // api.v3.batches — findBatchTaskRunById (PRIMARY) — tolerated. + heteroRunOpsPostgresTest( + "api.v3.batches findBatchTaskRunById reads the primary for idempotency dedup, finding the cached batch under replica lag", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bget_v3"); + const batchId = `batch_${CUID_25}`; + const friendlyId = "batch_bget_v3_f"; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 3, + }) + ); + + // Exact caller invocation: findCachedEntity(cachedRequestId) → findBatchTaskRunById(id), NO client. + const cached = await router.findBatchTaskRunById(batchId); + + // PRIMARY routing proof: the just-written batch is found DESPITE the frozen replica... + expect(cached).not.toBeNull(); + expect(cached!.id).toBe(batchId); + // ...and the frozen owning replica was NEVER consulted (tolerance = primary routing, not lag-toleration). + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + + // Reproduce the caller's dedup branch (findCachedEntity): env-match + non-null → buildResponse + // returns isCached, so the retried create does NOT mint a duplicate batch. + const dedupHit = !!cached && cached.runtimeEnvironmentId === seed.environment.id; + expect(dedupHit).toBe(true); + expect(cached!.friendlyId).toBe(friendlyId); + expect(cached!.runCount).toBe(3); + + // Env-scope guard: a foreign environment must reject the cached entity (→ null → re-create). + const foreignEnv = + !!cached && cached.runtimeEnvironmentId === "env_does_not_exist" ? cached : null; + expect(foreignEnv).toBeNull(); + } + ); + + // api.v2.tasks.batch — findBatchTaskRunById (PRIMARY) — tolerated. + heteroRunOpsPostgresTest( + "api.v2.tasks.batch findBatchTaskRunById reads the primary for dedup, finding the cached batch under replica lag", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bget_tb"); + const batchId = `batch_${CUID_25}`; + const friendlyId = "batch_bget_tb_f"; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId, + runtimeEnvironmentId: seed.environment.id, + runCount: 5, + }) + ); + + const cached = await router.findBatchTaskRunById(batchId); + + expect(cached).not.toBeNull(); + expect(cached!.id).toBe(batchId); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + + // Dedup branch: env-match + non-null → buildResponse returns { id: friendlyId, runCount }. + expect(cached!.runtimeEnvironmentId).toBe(seed.environment.id); + expect(cached!.friendlyId).toBe(friendlyId); + expect(cached!.runCount).toBe(5); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.routesRealtimeStreamReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.routesRealtimeStreamReadView.replicaLag.test.ts new file mode 100644 index 000000000..b1388bd22 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.routesRealtimeStreamReadView.replicaLag.test.ts @@ -0,0 +1,590 @@ +// Replica-lag properties for the realtime-stream route read views on the run-ops split. Fourteen +// route handlers resolve a run by friendlyId via runStore.findRun with a branded $replica handle or no +// client — both of which RoutingRunStore routes to the OWNING store's REPLICA, so under lag each can +// miss a freshly-created run and return null. Built as the webapp holds the RoutingRunStore, with the +// owning (legacy) replica FROZEN (taskRun "missing") via the shared `laggingReplica` primitive; each +// read is invoked EXACTLY as its caller does, asserting the under-lag value plus a primary re-read of +// the same row (so the null is provably lag). Thirteen reads tolerate the stale null (the run is +// already established when the route fires); subscribeToRun's findResource is a genuine +// read-your-writes window that re-reads the owning PRIMARY on a null to recover the just-triggered run. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import { markReadReplicaClient } from "./readReplicaClient.js"; + +// A cuid-shaped id (no run-ops v1 marker) classifies LEGACY, so both the create and the friendlyId-keyed +// reads route to the legacy (control-plane / prisma14, full schema) store first — the common realtime +// residency and the store whose frozen replica the read hits. +const CUID_25 = "c".repeat(25); + +// The webapp always passes its `$replica` handle, which is markReadReplicaClient()'d. A branded client +// tells RoutingRunStore "do not escalate to primary" — so the read stays on the owning store's replica. +// We pass a branded stand-in exactly as the routes do; its identity is irrelevant because the router +// discards it and reads the owning store's readOnlyPrisma (the frozen replica in these tests). +const BRANDED_REPLICA = markReadReplicaClient({}) as never; + +async function seedEnvironment(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: "PENDING" as never, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + realtimeStreamsVersion: "v1", + realtimeStreams: [], + }; +} + +// Build the router the realtime routes hold it as: a LEGACY store on the control-plane DB whose replica +// LAGS (taskRun frozen "missing"), plus a fresh NEW store. Seed a LEGACY-resident (cuid) run on the +// legacy PRIMARY. Returns the router + probe + seeded ids. Every test below resolves this run. +async function setupLaggingLegacy( + prisma14: PrismaClient, + prisma17: RunOpsPrismaClient, + slug: string +) { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, slug); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = `run_${slug}`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + return { router, legacyReplica, seed, runId, friendlyId }; +} + +describe("run-ops split — realtime-STREAM route read views vs. a lagging replica", () => { + // runs.$runId findRun — subscribeToRun's findResource. + // where {friendlyId, runtimeEnvironmentId}, include {batch:{select:{friendlyId}}}, BRANDED $replica. + // The canonical trigger→subscribe read-your-writes window: the replica miss is the first leg (a bare + // null would make apiBuilder 404 → terminal, since Electric retries only 429 and RunSubscription + // closes on FetchError), so findResource re-reads the owning PRIMARY on a null. Proves both legs: + // (a) the store returns null under lag, and (b) the owning-primary re-read recovers the run. + heteroRunOpsPostgresTest( + "runs.$runId findRun(branded $replica) is null under lag; the owning-primary re-read recovers the live run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_run_subscribe" + ); + + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { include: { batch: { select: { friendlyId: true } } } }; + + // The stale replica read the auth findResource performs. + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); // findResource null → apiBuilder returns 404 (x-should-retry:"false") + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + // The run IS live on the primary — findResource re-reads it there (the owning-primary fallback, + // like resolveRunForMutation's writer re-probe) and recovers the just-triggered run. + const recovered = (await router.findRun(where, args, prisma14 as never)) as { + friendlyId: string; + } | null; + expect(recovered).not.toBeNull(); + expect(recovered!.friendlyId).toBe(friendlyId); + } + ); + + // streams.$streamId ingest findRun — plain action ingest. + // where {friendlyId} (NO env scope, back-compat), select {id,friendlyId,streamBasinName, + // runtimeEnvironmentId}, BRANDED $replica. Null → 404. Tolerated: this is the stream write/ingest + // side, invoked by the producer from INSIDE the executing run — the run was dequeued (primary-gated) + // long after creation, so it is replicated by the time it writes to its own stream; friendlyId is + // immutable. + heteroRunOpsPostgresTest( + "streams.$streamId ingest findRun(branded $replica) is null under lag; the primary sees the executing run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, runId, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_ingest_plain" + ); + const where = { friendlyId }; + const args = { + select: { id: true, friendlyId: true, streamBasinName: true, runtimeEnvironmentId: true }, + }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + id: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.id).toBe(runId); + } + ); + + // streams.$streamId GET-SSE findResource — loader (GET SSE). + // where {friendlyId, runtimeEnvironmentId}, select {id,friendlyId,taskIdentifier,runTags, + // realtimeStreamsVersion,streamBasinName,batch:{...}}, BRANDED $replica. Null → 404. Tolerated: the + // public stream read follows a successful subscribeToRun that already resolved the run (the previous + // test covers that gate); stream chunks only exist once the run is executing, so the run is + // replicated by the time a reader arrives. + heteroRunOpsPostgresTest( + "streams.$streamId GET-SSE findResource(branded $replica) is null under lag; the primary sees the run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_stream_get" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { + select: { + id: true, + friendlyId: true, + taskIdentifier: true, + runTags: true, + realtimeStreamsVersion: true, + streamBasinName: true, + batch: { select: { friendlyId: true } }, + }, + }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + friendlyId: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.friendlyId).toBe(friendlyId); + } + ); + + // streams.$target action findRun — action (ingest/PUT). + // where {friendlyId, runtimeEnvironmentId}, select {id,friendlyId,streamBasinName,parentTaskRun:{...}, + // rootTaskRun:{...}}, BRANDED $replica. Null → 404. Tolerated: write side from inside the executing + // run; and the PUT completedAt gate re-reads the PRIMARY (`prisma`), so the authoritative decision + // never depends on this replica read. Immutable ids resolved here. + heteroRunOpsPostgresTest( + "streams.$target action findRun(branded $replica) is null under lag; the completedAt gate re-reads the primary", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_target_action" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { + select: { + id: true, + friendlyId: true, + streamBasinName: true, + parentTaskRun: { select: { friendlyId: true, streamBasinName: true } }, + rootTaskRun: { select: { friendlyId: true, streamBasinName: true } }, + }, + }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + friendlyId: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.friendlyId).toBe(friendlyId); + } + ); + + // streams.$target HEAD findRun — loader (HEAD). + // Same select as the ingest/PUT action above, BRANDED $replica; resolves parent/root friendlyId for a + // HEAD last-chunk-index probe. Null → 404. Tolerated: read side of an established run's stream + // (subscribeToRun gated it first); read-only, immutable ids. + heteroRunOpsPostgresTest( + "streams.$target HEAD findResource(branded $replica) is null under lag; the primary sees the run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_target_head" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { + select: { + id: true, + friendlyId: true, + streamBasinName: true, + parentTaskRun: { select: { friendlyId: true, streamBasinName: true } }, + rootTaskRun: { select: { friendlyId: true, streamBasinName: true } }, + }, + }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + friendlyId: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.friendlyId).toBe(friendlyId); + } + ); + + // streams.$target append findRun — action. + // where {friendlyId, runtimeEnvironmentId}, select {id,friendlyId,parentTaskRun:{select:{friendlyId}}, + // rootTaskRun:{select:{friendlyId}}}, BRANDED $replica. Null → 404. Tolerated: append write from + // inside the executing run; the authoritative target completedAt gate re-reads the PRIMARY. + heteroRunOpsPostgresTest( + "streams.$target append findRun(branded $replica) is null under lag; the completedAt gate re-reads the primary", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_target_append" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { + select: { + id: true, + friendlyId: true, + parentTaskRun: { select: { friendlyId: true } }, + rootTaskRun: { select: { friendlyId: true } }, + }, + }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + friendlyId: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.friendlyId).toBe(friendlyId); + } + ); + + // streams.input send findRun — action (send input). + // where {friendlyId, runtimeEnvironmentId}, select {id,friendlyId,completedAt,realtimeStreamsVersion, + // streamBasinName}, BRANDED $replica. Null → 404. Tolerated: the `.send()` writer targets an + // executing run's input stream, so it is replicated. The completedAt read off the replica fails safe: + // a stale completedAt=null only allows the append to proceed (a harmless write), never wrongly + // blocks a live run. + heteroRunOpsPostgresTest( + "streams.input send findRun(branded $replica) is null under lag; a stale completedAt fails safe by allowing the append", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_input_send" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { + select: { + id: true, + friendlyId: true, + completedAt: true, + realtimeStreamsVersion: true, + streamBasinName: true, + }, + }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + friendlyId: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.friendlyId).toBe(friendlyId); + } + ); + + // streams.input GET-SSE findResource — loader (GET SSE tail). + // where {friendlyId, runtimeEnvironmentId}, include {batch:{select:{friendlyId}}}, BRANDED $replica. + // Null → 404. Tolerated: read side of an established run's input stream; read-only display tail, + // immutable ids, follows a resolved run. + heteroRunOpsPostgresTest( + "streams.input GET-SSE findResource(branded $replica) is null under lag; the primary sees the run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_input_get" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { include: { batch: { select: { friendlyId: true } } } }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + friendlyId: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.friendlyId).toBe(friendlyId); + } + ); + + // input-streams.wait findRun — create waitpoint. + // where {friendlyId, runtimeEnvironmentId}, select {id,friendlyId,realtimeStreamsVersion, + // streamBasinName}, BRANDED $replica. run.id then feeds engine.createManualWaitpoint (a mutation). + // Null → 404 → waitpoint not created. Tolerated: `.wait()` on an input stream is called from inside + // the executing run, so the run is long-since replicated; the mutation reads its own live run's id. + heteroRunOpsPostgresTest( + "input-streams.wait findRun(branded $replica) is null under lag; the primary sees the executing run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, runId, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_input_wait" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { + select: { id: true, friendlyId: true, realtimeStreamsVersion: true, streamBasinName: true }, + }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + id: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.id).toBe(runId); // the run.id createManualWaitpoint would co-locate the waitpoint on + } + ); + + // session-streams.wait findRun — create waitpoint. + // where {friendlyId, runtimeEnvironmentId}, select {id,friendlyId,realtimeStreamsVersion}, BRANDED + // $replica. run.id feeds engine.createManualWaitpoint. Null → 404. Tolerated: same reasoning as the + // input-streams wait above — the agent's `.wait()` on a session stream runs inside the executing run, + // so the run is replicated. + heteroRunOpsPostgresTest( + "session-streams.wait findRun(branded $replica) is null under lag; the primary sees the executing run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, runId, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_session_wait" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { select: { id: true, friendlyId: true, realtimeStreamsVersion: true } }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + id: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.id).toBe(runId); + } + ); + + // resources sessions.$io findRun — dashboard SSE. + // where {friendlyId, runtimeEnvironmentId}, select {id,friendlyId}, BRANDED $replica. Verifies the run + // lives in this env before subscribing to a session channel. Null → 404. Tolerated: this + // dashboard-auth route is opened by a human in the span inspector Agent tab, on a run whose detail + // page already resolved — the navigation latency dwarfs replica lag; read-only display; the error + // self-heals on reconnect/navigation. + heteroRunOpsPostgresTest( + "resources sessions.$io findRun(branded $replica) is null under lag; the primary sees the run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, runId, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_res_session_io" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { select: { id: true, friendlyId: true } }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + id: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.id).toBe(runId); + } + ); + + // resources streams.$streamId findRun — dashboard SSE. + // where {friendlyId, runtimeEnvironmentId}, select {id,friendlyId,realtimeStreamsVersion, + // streamBasinName}, BRANDED $replica. Null → 404. Tolerated: same dashboard-navigation reasoning as + // above (Agent tab output-stream viewer); read-only. + heteroRunOpsPostgresTest( + "resources streams.$streamId findRun(branded $replica) is null under lag; the primary sees the run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_res_stream" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { + select: { + id: true, + friendlyId: true, + realtimeStreamsVersion: true, + streamBasinName: true, + }, + }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + friendlyId: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.friendlyId).toBe(friendlyId); + } + ); + + // resources streams.input findRun — dashboard SSE. + // where {friendlyId, runtimeEnvironmentId}, select {id,friendlyId,realtimeStreamsVersion, + // streamBasinName}, BRANDED $replica. Null → 404. Tolerated: same dashboard-navigation reasoning as + // above (Agent tab input-stream viewer); read-only. + heteroRunOpsPostgresTest( + "resources streams.input findRun(branded $replica) is null under lag; the primary sees the run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_res_input" + ); + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + const args = { + select: { + id: true, + friendlyId: true, + realtimeStreamsVersion: true, + streamBasinName: true, + }, + }; + + const stale = await router.findRun(where, args, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + friendlyId: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.friendlyId).toBe(friendlyId); + } + ); + + // route.tsx findRun — dashboard SSE, no client argument. + // where {friendlyId, projectId} (note: projectId, not env), select {id,friendlyId, + // realtimeStreamsVersion,streamBasinName,runtimeEnvironmentId}, NO client → owning REPLICA. Null → + // throw Response 404. Tolerated: same dashboard-navigation reasoning as above; the run's own detail + // page resolved it already; read-only stream viewer. This caller passes no client at all (versus + // the branded $replica in the tests above) but routes to the same owning replica, so the under-lag + // behavior is identical. + heteroRunOpsPostgresTest( + "route.tsx findRun(no client) is null under lag; the primary sees the run", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica, seed, friendlyId } = await setupLaggingLegacy( + prisma14, + prisma17, + "rt_route_tsx" + ); + const where = { friendlyId, projectId: seed.project.id }; + const args = { + select: { + id: true, + friendlyId: true, + realtimeStreamsVersion: true, + streamBasinName: true, + runtimeEnvironmentId: true, + }, + }; + + // NO client — exactly as the route.tsx caller. + const stale = await router.findRun(where, args); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + const primary = (await router.findRun(where, args, prisma14 as never)) as { + friendlyId: string; + } | null; + expect(primary).not.toBeNull(); + expect(primary!.friendlyId).toBe(friendlyId); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.routesRunGetReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.routesRunGetReadView.replicaLag.test.ts new file mode 100644 index 000000000..32515068c --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.routesRunGetReadView.replicaLag.test.ts @@ -0,0 +1,664 @@ +// Lagging-replica coverage for the run-GET / run-detail read views ("routes-run-get"). Each read is +// exercised against real Postgres with the owning store's replica FROZEN via laggingReplica, driving the +// exact exported route caller that issues it. Every read RoutingRunStore serves from the OWNING store's +// REPLICA — a client-less findRun/findTaskRunAttempt, or a findRun passed the BRANDED $replica (the brand +// keeps the read on a replica, it does not escalate). We build the router exactly as ~/v3/runStore.server +// holds it (legacy + dedicated PostgresRunStore) and invoke each read as the caller does. +// +// Shared tolerance, asserted concretely per read: the fields that drive any control decision +// (routing/auth/redirect/queue keys — projectId, runtimeEnvironmentId, organizationId, spanId, traceId, +// createdAt, engine, queue, concurrencyKey, taskEventStore) are IMMUTABLE, so even a STALE replica row +// carries correct values. The only lag effect is transient ABSENCE → a not-found/error-redirect/empty +// display that self-heals on the next load and drives no mutation. Displayed mutable fields (status, +// cost, completedAt, output) are cosmetic and self-heal. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import { markReadReplicaClient } from "./readReplicaClient.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +// A cuid-shaped id (25 chars, no run-ops v1 marker) classifies LEGACY, so a friendlyId-keyed read +// routes to the legacy (control-plane / prisma14, full schema) store's replica first. +const CUID_25 = "e".repeat(25); +// A v1-marked 26-char body classifies NEW, routing to the dedicated (#new / prisma17) store — used for +// the findTaskRunAttempt case so we can freeze the NEW replica without control-plane FK seeding. +const NEW_ID_26 = "k".repeat(24) + "01"; + +// Final attempt statuses (FINAL_ATTEMPT_STATUSES). Inlined to avoid a run-store→webapp +// test dependency; only used to mirror the caller's findTaskRunAttempt `where`. +const FINAL_ATTEMPT_STATUSES = ["COMPLETED", "FAILED", "CANCELED"] as const; + +async function seedLegacyEnvironment(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + status?: string; + spanId?: string; + engine?: "V1" | "V2"; + createdAt?: Date; + completedAt?: Date | null; + concurrencyKey?: string | null; + traceId?: string; +}) { + return { + id: opts.id, + engine: (opts.engine ?? "V2") as "V1" | "V2", + status: (opts.status ?? "PENDING") as never, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: opts.traceId ?? `trace_${opts.id}`, + spanId: opts.spanId ?? `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + ...(opts.concurrencyKey !== undefined ? { concurrencyKey: opts.concurrencyKey } : {}), + ...(opts.createdAt !== undefined ? { createdAt: opts.createdAt } : {}), + ...(opts.completedAt !== undefined ? { completedAt: opts.completedAt } : {}), + }; +} + +// Build the router exactly as runStore.server holds it: a legacy store (prisma14) + a dedicated store +// (prisma17). Either store's REPLICA (readOnlyPrisma) can be swapped for a frozen client so a +// friendlyId/id-routed read is served stale. +function buildRouter( + prisma14: PrismaClient, + prisma17: unknown, + opts?: { legacyReplica?: AnyClient; newReplica?: AnyClient } +): RoutingRunStore { + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: (opts?.legacyReplica ?? prisma14) as never, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: (opts?.newReplica ?? prisma17) as never, + schemaVariant: "dedicated", + }); + return new RoutingRunStore({ new: newStore, legacy: legacyStore }); +} + +describe("routes-run-get read views under replica lag (owning REPLICA)", () => { + // orgs redirect — legacy canonical redirect. + // Reads a run by friendlyId to get { projectId, runtimeEnvironmentId } and redirects to the v3 run + // path. Both selected fields are IMMUTABLE (set at creation). + // (a) frozen-missing replica → null → the loader throws 404. Tolerated: this is a GET navigation, the + // run list the link came from is served by the same replica, and a transient not-found self-heals + // on the next load; it drives no mutation. + // (b) frozen-STALE replica (run has since progressed to EXECUTING) → the decision fields projectId / + // runtimeEnvironmentId are STILL CORRECT because they never change — so the redirect target is + // right even off a lagging replica. + heteroRunOpsPostgresTest( + "orgs redirect findRun: frozen-missing → null (transient 404, self-heals); frozen-stale → immutable projectId/runtimeEnvironmentId still correct", + async ({ prisma14, prisma17 }) => { + const seed = await seedLegacyEnvironment(prisma14, "site1_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = "run_site1"; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", // primary is already progressed + }), + }); + + const site1 = (router: RoutingRunStore) => + router.findRun( + { friendlyId }, + { select: { projectId: true, runtimeEnvironmentId: true } } + ) as Promise<{ projectId: string; runtimeEnvironmentId: string } | null>; + + // (a) missing: replica has not replicated the run → null → 404. + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const viaMissing = await site1( + buildRouter(prisma14, prisma17, { legacyReplica: missing.client }) + ); + expect(missing.wasHit("taskRun")).toBe(true); + expect(viaMissing).toBeNull(); + + // Primary contrast: the run genuinely exists (so the null above was purely the lagging replica). + const onPrimary = (await buildRouter(prisma14, prisma17).findRun( + { friendlyId }, + { select: { projectId: true, runtimeEnvironmentId: true } }, + prisma14 + )) as { projectId: string } | null; + expect(onPrimary?.projectId).toBe(seed.project.id); + + // (b) stale: replica shows the OLD PENDING snapshot — but the redirect's decision fields are + // immutable, so the stale row still carries the correct projectId / runtimeEnvironmentId. + const staleRow = { + friendlyId, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "PENDING", + }; + const frozen = laggingReplica(prisma14, [ + { model: "taskRun", mode: "frozen", rows: [staleRow] }, + ]); + const viaStale = await site1( + buildRouter(prisma14, prisma17, { legacyReplica: frozen.client }) + ); + expect(frozen.wasHit("taskRun")).toBe(true); + expect(viaStale).not.toBeNull(); + expect(viaStale!.projectId).toBe(seed.project.id); + expect(viaStale!.runtimeEnvironmentId).toBe(seed.environment.id); + } + ); + + // run inspector loader. + // The run inspector fetcher reads a run by friendlyId for a large DISPLAY projection. + // frozen-missing → null → 404 on the fetcher, which self-heals on the next poll/refresh. + // frozen-stale → a stale status/cost is displayed (cosmetic); the immutable id/projectId/ + // runtimeEnvironmentId/createdAt that gate the follow-on auth + attempt reads are still correct. + heteroRunOpsPostgresTest( + "run inspector findRun: frozen-missing → null (fetcher 404 self-heals); frozen-stale → status cosmetic-stale but immutable id/projectId/createdAt correct", + async ({ prisma14, prisma17 }) => { + const seed = await seedLegacyEnvironment(prisma14, "site2_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = "run_site2"; + const createdAt = new Date("2024-01-01T00:00:00.000Z"); + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED_SUCCESSFULLY", // primary finished + createdAt, + }), + }); + + const select = { + id: true, + friendlyId: true, + status: true, + projectId: true, + runtimeEnvironmentId: true, + createdAt: true, + } as const; + const site2 = (router: RoutingRunStore) => + router.findRun({ friendlyId }, { select }) as Promise<{ + id: string; + status: string; + projectId: string; + createdAt: Date; + runtimeEnvironmentId: string; + } | null>; + + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const viaMissing = await site2( + buildRouter(prisma14, prisma17, { legacyReplica: missing.client }) + ); + expect(missing.wasHit("taskRun")).toBe(true); + expect(viaMissing).toBeNull(); + + const staleRow = { + id: runId, + friendlyId, + status: "EXECUTING", // stale — primary is COMPLETED_SUCCESSFULLY + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + createdAt, + }; + const frozen = laggingReplica(prisma14, [ + { model: "taskRun", mode: "frozen", rows: [staleRow] }, + ]); + const viaStale = await site2( + buildRouter(prisma14, prisma17, { legacyReplica: frozen.client }) + ); + expect(frozen.wasHit("taskRun")).toBe(true); + expect(viaStale).not.toBeNull(); + expect(viaStale!.status).toBe("EXECUTING"); // cosmetic-stale (display only) + // Immutable decision fields correct even off the lagging replica: + expect(viaStale!.id).toBe(runId); + expect(viaStale!.projectId).toBe(seed.project.id); + expect(viaStale!.runtimeEnvironmentId).toBe(seed.environment.id); + expect(viaStale!.createdAt).toEqual(createdAt); + + // Primary shows the fresh status — proving the staleness is confined to the replica. + const onPrimary = (await buildRouter(prisma14, prisma17).findRun( + { friendlyId }, + { select: { status: true } }, + prisma14 + )) as { status: string } | null; + expect(onPrimary?.status).toBe("COMPLETED_SUCCESSFULLY"); + } + ); + + // finished-attempt output findTaskRunAttempt. + // On a finished run, the inspector reads the final TaskRunAttempt (client-less) purely to display its + // output/error. A classifiable taskRunId routes to the owning store's REPLICA. Routed here via a + // run-ops id so we can freeze the #new replica without control-plane FK seeding. Under lag the read + // misses the live attempt (returns null) → the panel shows empty output on a finished run, which + // self-heals on the next load; it drives no mutation. + heteroRunOpsPostgresTest( + "finished-attempt findTaskRunAttempt: frozen-missing NEW replica misses the live attempt (null) — empty-output display self-heals; primary sees it", + async ({ prisma14, prisma17 }) => { + const runId = `run_${NEW_ID_26}`; // run-ops id → NEW (#new / prisma17) + const attemptId = `attempt_${NEW_ID_26}`; + // Seed run + attempt on the dedicated (#new) primary. The dedicated subset schema does not enforce + // the control-plane FKs, so synthetic org/proj/env ids are fine (mirrors batchProbeReadClient). + await prisma17.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId: "run_site3", + organizationId: "org_site3", + projectId: "proj_site3", + runtimeEnvironmentId: "env_site3", + status: "COMPLETED_SUCCESSFULLY", + }) as never, + }); + await prisma17.taskRunAttempt.create({ + data: { + id: attemptId, + number: 1, + friendlyId: "attempt_site3", + taskRunId: runId, + backgroundWorkerId: `bw_${NEW_ID_26}`, + backgroundWorkerTaskId: `bwt_${NEW_ID_26}`, + runtimeEnvironmentId: "env_site3", + queueId: `queue_${NEW_ID_26}`, + status: "COMPLETED", + output: '{"ok":true}', + outputType: "application/json", + } as never, + }); + + const site3 = (router: RoutingRunStore) => + router.findTaskRunAttempt({ + select: { output: true, outputType: true, error: true }, + where: { + status: { in: FINAL_ATTEMPT_STATUSES as unknown as never }, + taskRunId: runId, + }, + orderBy: { createdAt: "desc" }, + }) as Promise<{ output: string | null } | null>; + + // Freeze the NEW replica (taskRunAttempt missing) → client-less read served there misses it. + const missing = laggingReplica(prisma17 as AnyClient, [ + { model: "taskRunAttempt", mode: "missing" }, + ]); + const viaReplica = await site3( + buildRouter(prisma14, prisma17, { newReplica: missing.client }) + ); + expect(missing.wasHit("taskRunAttempt")).toBe(true); + expect(viaReplica).toBeNull(); // empty output shown on a finished run; self-heals + + // Primary contrast: threading the writer finds the live attempt (proves the miss was pure lag). + const onPrimary = await site3(buildRouter(prisma14, prisma17)).then(() => + buildRouter(prisma14, prisma17).findTaskRunAttempt( + { + select: { output: true, outputType: true, error: true }, + where: { + status: { in: FINAL_ATTEMPT_STATUSES as unknown as never }, + taskRunId: runId, + }, + orderBy: { createdAt: "desc" }, + }, + prisma17 as never + ) + ); + expect(onPrimary).not.toBeNull(); + expect((onPrimary as { output: string }).output).toBe('{"ok":true}'); + } + ); + + // public short-link redirect. + // Reads a run by friendlyId for { spanId, projectId, runtimeEnvironmentId } and redirects to the v3 + // run path (attaching ?span=). All three fields are IMMUTABLE. frozen-missing → + // null → redirectWithErrorMessage ("run doesn't exist or no permission") — a transient error redirect + // that self-heals; frozen-stale → spanId/projectId/runtimeEnvironmentId still correct. + heteroRunOpsPostgresTest( + "short-link redirect findRun: frozen-missing → null (transient error redirect); frozen-stale → immutable spanId/projectId/runtimeEnvironmentId correct", + async ({ prisma14, prisma17 }) => { + const seed = await seedLegacyEnvironment(prisma14, "site4_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = "run_site4"; + const spanId = "span_site4_fixed"; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + spanId, + }), + }); + + const site4 = (router: RoutingRunStore) => + router.findRun( + { friendlyId }, + { select: { spanId: true, projectId: true, runtimeEnvironmentId: true } } + ) as Promise<{ spanId: string; projectId: string; runtimeEnvironmentId: string } | null>; + + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const viaMissing = await site4( + buildRouter(prisma14, prisma17, { legacyReplica: missing.client }) + ); + expect(missing.wasHit("taskRun")).toBe(true); + expect(viaMissing).toBeNull(); + + const staleRow = { + friendlyId, + spanId, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }; + const frozen = laggingReplica(prisma14, [ + { model: "taskRun", mode: "frozen", rows: [staleRow] }, + ]); + const viaStale = await site4( + buildRouter(prisma14, prisma17, { legacyReplica: frozen.client }) + ); + expect(frozen.wasHit("taskRun")).toBe(true); + expect(viaStale).not.toBeNull(); + expect(viaStale!.spanId).toBe(spanId); + expect(viaStale!.projectId).toBe(seed.project.id); + expect(viaStale!.runtimeEnvironmentId).toBe(seed.environment.id); + + const onPrimary = (await buildRouter(prisma14, prisma17).findRun( + { friendlyId }, + { select: { spanId: true } }, + prisma14 + )) as { spanId: string } | null; + expect(onPrimary?.spanId).toBe(spanId); + } + ); + + // admin queue-debug loader. + // Reads a run by friendlyId for { id, engine, queue, concurrencyKey, runtimeEnvironmentId, projectId } + // and introspects the run-queue Redis sets. frozen-missing → null → 404 on an + // admin debug tool (self-heals); frozen-stale → engine/queue/concurrencyKey (all immutable) still + // correct, so the Redis keys it builds are the right ones even off a lagging replica. + heteroRunOpsPostgresTest( + "debug findRun: frozen-missing → null (admin-tool 404 self-heals); frozen-stale → immutable engine/queue/concurrencyKey correct", + async ({ prisma14, prisma17 }) => { + const seed = await seedLegacyEnvironment(prisma14, "site5_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = "run_site5"; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + engine: "V2", + concurrencyKey: "ck-site5", + }), + }); + + const select = { + id: true, + engine: true, + friendlyId: true, + queue: true, + concurrencyKey: true, + runtimeEnvironmentId: true, + projectId: true, + } as const; + const site5 = (router: RoutingRunStore) => + router.findRun({ friendlyId }, { select }) as Promise<{ + engine: string; + queue: string; + concurrencyKey: string | null; + projectId: string; + } | null>; + + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const viaMissing = await site5( + buildRouter(prisma14, prisma17, { legacyReplica: missing.client }) + ); + expect(missing.wasHit("taskRun")).toBe(true); + expect(viaMissing).toBeNull(); + + const staleRow = { + id: runId, + friendlyId, + engine: "V2", + queue: "task/my-task", + concurrencyKey: "ck-site5", + runtimeEnvironmentId: seed.environment.id, + projectId: seed.project.id, + }; + const frozen = laggingReplica(prisma14, [ + { model: "taskRun", mode: "frozen", rows: [staleRow] }, + ]); + const viaStale = await site5( + buildRouter(prisma14, prisma17, { legacyReplica: frozen.client }) + ); + expect(frozen.wasHit("taskRun")).toBe(true); + expect(viaStale).not.toBeNull(); + expect(viaStale!.engine).toBe("V2"); + expect(viaStale!.queue).toBe("task/my-task"); + expect(viaStale!.concurrencyKey).toBe("ck-site5"); + expect(viaStale!.projectId).toBe(seed.project.id); + + const onPrimary = (await buildRouter(prisma14, prisma17).findRun( + { friendlyId }, + { select: { queue: true } }, + prisma14 + )) as { queue: string } | null; + expect(onPrimary?.queue).toBe("task/my-task"); + } + ); + + // log-detail run-status annotation. + // This caller is DISTINCT: it passes the BRANDED $replica as the 3rd findRun arg. The brand is a + // read-replica signal, so readYourWrites=false and the read STAYS on the owning REPLICA (the brand + // never escalates to the primary). The result is used only as `runStatus = run?.status` — an OPTIONAL + // display annotation on a log-detail row. First prove the branded client does + // NOT route to primary (frozen-stale → the read returns the STALE status), then that missing → null → + // runStatus undefined (the caller optional-chains and tolerates it). Contrast: an UNBRANDED writer + // escalates to the primary → fresh status. + heteroRunOpsPostgresTest( + "log-detail findRun(branded $replica): brand keeps read on REPLICA → stale status (tolerated via run?.status); unbranded writer escalates to fresh", + async ({ prisma14, prisma17 }) => { + const seed = await seedLegacyEnvironment(prisma14, "site6_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = "run_site6"; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED_SUCCESSFULLY", // primary finished + }), + }); + + // Frozen replica shows the stale EXECUTING snapshot and backs the legacy store's readOnlyPrisma. + const staleRow = { + friendlyId, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + }; + const frozen = laggingReplica(prisma14, [ + { model: "taskRun", mode: "frozen", rows: [staleRow] }, + ]); + const router = buildRouter(prisma14, prisma17, { legacyReplica: frozen.client }); + + // The 3rd findRun arg is a ROUTING SIGNAL only — the router reads its presence + brand and never + // forwards it to the store (the store always reads its own readOnlyPrisma). So a standalone + // branded object faithfully stands in for the branded $replica the caller passes, without leaking + // the brand onto prisma14 (branding the frozen Proxy would forward the symbol to its writer target + // and wrongly mark prisma14 as a replica). + const brandedReplicaSignal = markReadReplicaClient({}) as AnyClient; + + // Invoke EXACTLY as the caller does: findRun(where, {select:{status}}, $replica-branded). + const run = (await router.findRun( + { friendlyId, runtimeEnvironmentId: seed.environment.id }, + { select: { status: true } }, + brandedReplicaSignal as never + )) as { status: string } | null; + const runStatus = run?.status; // the caller's runStatus + + expect(frozen.wasHit("taskRun")).toBe(true); + // The BRANDED client did NOT escalate to primary: the read is served by the (stale) replica. + expect(runStatus).toBe("EXECUTING"); // stale — primary is COMPLETED_SUCCESSFULLY + // Tolerated: runStatus is only a display annotation on the log row (`run?.status`), self-heals. + + // Contrast: an UNBRANDED writer IS a read-your-writes signal → escalates to primary → fresh. + const viaWriter = (await router.findRun( + { friendlyId, runtimeEnvironmentId: seed.environment.id }, + { select: { status: true } }, + prisma14 + )) as { status: string } | null; + expect(viaWriter?.status).toBe("COMPLETED_SUCCESSFULLY"); + + // And missing → null → runStatus undefined (optional, tolerated). + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const routerMissing = buildRouter(prisma14, prisma17, { legacyReplica: missing.client }); + const runMissing = (await routerMissing.findRun( + { friendlyId, runtimeEnvironmentId: seed.environment.id }, + { select: { status: true } }, + brandedReplicaSignal as never + )) as { status: string } | null; + expect(missing.wasHit("taskRun")).toBe(true); + expect(runMissing?.status).toBeUndefined(); + } + ); + + // trace-export download. + // Reads a run by friendlyId for the trace-export window + auth: { friendlyId, traceId, organizationId, + // runtimeEnvironmentId, createdAt, completedAt, taskEventStore, taskIdentifier }. + // frozen-missing → null → the buffer fallback (or 404) handles it, self-heals. frozen-stale is SAFE: + // every field the ClickHouse trace query keys on — traceId, organizationId, runtimeEnvironmentId, + // createdAt, taskEventStore — is IMMUTABLE; the only mutable field, completedAt, is used as the export + // END bound (`run.completedAt ?? undefined`), and a stale null just yields an OPEN-ENDED window (a + // superset of events), never a truncated/lossy export. + heteroRunOpsPostgresTest( + "trace-download findRun: frozen-missing → null (buffer fallback self-heals); frozen-stale → immutable traceId/org/createdAt correct, stale null completedAt = open-ended (superset) not lossy", + async ({ prisma14, prisma17 }) => { + const seed = await seedLegacyEnvironment(prisma14, "site7_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = "run_site7"; + const traceId = "trace_site7_fixed"; + const createdAt = new Date("2024-01-01T00:00:00.000Z"); + const completedAt = new Date("2024-01-01T00:05:00.000Z"); + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED_SUCCESSFULLY", + traceId, + createdAt, + completedAt, // primary: the run has completed + }), + }); + + const select = { + friendlyId: true, + traceId: true, + organizationId: true, + runtimeEnvironmentId: true, + createdAt: true, + completedAt: true, + taskEventStore: true, + taskIdentifier: true, + } as const; + const site7 = (router: RoutingRunStore) => + router.findRun({ friendlyId }, { select }) as Promise<{ + traceId: string; + organizationId: string | null; + createdAt: Date; + completedAt: Date | null; + taskEventStore: string; + } | null>; + + const missing = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const viaMissing = await site7( + buildRouter(prisma14, prisma17, { legacyReplica: missing.client }) + ); + expect(missing.wasHit("taskRun")).toBe(true); + expect(viaMissing).toBeNull(); + + // Stale replica: still-RUNNING snapshot, completedAt null (not yet replicated), old status. + const staleRow = { + friendlyId, + traceId, + organizationId: seed.organization.id, + runtimeEnvironmentId: seed.environment.id, + createdAt, + completedAt: null as Date | null, // stale — primary already has a completedAt + taskEventStore: "taskEvent", + taskIdentifier: "my-task", + }; + const frozen = laggingReplica(prisma14, [ + { model: "taskRun", mode: "frozen", rows: [staleRow] }, + ]); + const viaStale = await site7( + buildRouter(prisma14, prisma17, { legacyReplica: frozen.client }) + ); + expect(frozen.wasHit("taskRun")).toBe(true); + expect(viaStale).not.toBeNull(); + // Immutable trace-query keys correct even off the lagging replica: + expect(viaStale!.traceId).toBe(traceId); + expect(viaStale!.organizationId).toBe(seed.organization.id); + expect(viaStale!.createdAt).toEqual(createdAt); + expect(viaStale!.taskEventStore).toBe("taskEvent"); + // The one mutable field is the export END bound: a stale null → open-ended window (superset), the + // export streams MORE events, never fewer — not a lossy/truncated download. + expect(viaStale!.completedAt).toBeNull(); + + const onPrimary = (await buildRouter(prisma14, prisma17).findRun( + { friendlyId }, + { select: { completedAt: true } }, + prisma14 + )) as { completedAt: Date | null } | null; + expect(onPrimary?.completedAt).toEqual(completedAt); // primary has the real end bound + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.routesSpanTraceReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.routesSpanTraceReadView.replicaLag.test.ts new file mode 100644 index 000000000..15754d557 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.routesSpanTraceReadView.replicaLag.test.ts @@ -0,0 +1,322 @@ +// Lagging-replica coverage for the SPAN-DETAIL / TRACE / TRACE-SYNC route read views on the run-ops +// split. Four run lookups back three GET routes; NONE is a read-your-writes mutation gate — every one +// resolves a run for a DISPLAY/GET response and tolerates a stale/missing read by a named caller +// mechanism. This builds the store as each route holds it (`runStore` via a RoutingRunStore), freezes +// the OWNING store's replica with the shared laggingReplica primitive, invokes each read EXACTLY as +// the caller does (same method + branded-$replica client arg + where/select), and asserts the +// under-lag value AND that the SAME read against a caught-up replica recovers the row — so the +// null/empty is provably lag-induced, not a routing accident or true absence. +// +// Routing recap (against runOpsStore.ts on this branch): +// findRun/findRuns with NO client → owning store's REPLICA. +// findRun/findRuns with a BRANDED $replica → stays on the owning REPLICA (a read-replica brand is +// not a write signal). The webapp `$replica` handle is markReadReplicaClient()'d, so passing it +// is routing-equivalent to passing no client. +// findRun/findRuns with a WRITER/tx → owning store's PRIMARY (read-your-writes escalation). +// The routing store never forwards the client across DBs; only its brand/presence is read, so the +// branded stand-in below (markReadReplicaClient({})) faithfully reproduces the webapp arg. +// +// Reads covered (all read the owning REPLICA; each case names the exact tolerance mechanism): +// 1. spans.$spanId findRun({friendlyId, runtimeEnvironmentId}) — classifiable → routed lookup +// (owning replica, miss-probe the other store). +// 2. spans.$spanId findRuns({where:{runtimeEnvironmentId, parentSpanId}, take:50, select}) — open +// predicate → both stores' replicas, merged. +// 3. trace findRun({friendlyId, runtimeEnvironmentId}) — same shape as read 1. +// 4. sync.traces findRun({traceId}, {select:{runtimeEnvironmentId}}) — unclassifiable where → +// unrouted lookup (NEW-first then LEGACY, both replicas). +// +// Real split topology via heteroRunOpsPostgresTest — NEVER mocked. The laggingReplica primitive +// freezes findFirst/findMany/findUnique on a chosen Prisma model so a replica-routed read misses the +// just-written row (see the helper below). + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import { markReadReplicaClient } from "./readReplicaClient.js"; + +// A cuid (25 chars after the `run_` prefix) classifies LEGACY, so id/friendlyId-keyed reads route to +// the legacy (control-plane) store first — the owning store for the runs seeded here. Trace/span +// detail runs are overwhelmingly legacy-resident (draining DB) in the split window. +const CUID_25 = "c".repeat(25); + +// The webapp passes its `$replica` handle, markReadReplicaClient()'d. Its object identity is +// irrelevant to routing (the router discards it and reads the owning store's readOnlyPrisma) — only +// the brand matters. This stand-in reproduces the arg exactly. +const BRANDED_REPLICA = markReadReplicaClient({}) as never; + +async function seedEnvironment(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + traceId: string; + spanId: string; + parentSpanId?: string; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: "PENDING" as const, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: opts.traceId, + spanId: opts.spanId, + parentSpanId: opts.parentSpanId ?? null, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +// Build the router the way the routes hold it: a LEGACY store on the control-plane DB whose replica +// LAGS, plus a NEW store on the (empty) dedicated DB. Also returns a HEALTHY router (identical, but +// the legacy store reads a caught-up replica handle) sharing the SAME seeded rows in prisma14, so a +// ground-truth re-read proves the under-lag null/empty is purely replica lag. The NEW store's replica +// is left un-frozen (the dedicated DB is empty), so the cross-store miss-probe returns null naturally +// and cannot mask the owning-replica staleness with a phantom hit. +function makeRouters(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const laggingLegacy = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const laggingRouter = new RoutingRunStore({ new: newStore, legacy: laggingLegacy }); + + const healthyLegacy = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, // caught-up replica handle + schemaVariant: "legacy", + }); + const healthyRouter = new RoutingRunStore({ new: newStore, legacy: healthyLegacy }); + + return { laggingRouter, healthyRouter, legacyReplica }; +} + +describe("run-ops split — span-detail / trace / trace-sync route read views vs. a lagging replica", () => { + // --- spans.$spanId findPgRun -------------------------------------------------------------------- + // findResource resolves the run for the span-detail GET. Under lag the owning (legacy) replica + // returns null; the route falls through to the mollifier buffer fallback and, failing that, returns + // a retryable 404 (x-should-retry: true). The SDK re-requests, and by the retry the PG replica has + // caught up. No mutation is guarded by this read — a replica-stale null, tolerated by + // eventual-consistency-by-contract (retryable 404 + mollifier buffer fallback). Store fact under + // lag: null. + heteroRunOpsPostgresTest( + "spans.$spanId findPgRun findRun(branded $replica) is NULL under owning-replica lag; caught-up replica recovers the run (retryable 404 + buffer fallback tolerate the stale null)", + async ({ prisma14, prisma17 }) => { + const { laggingRouter, healthyRouter, legacyReplica } = makeRouters(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "spans_find"); + const runId = `run_${CUID_25}`; + const friendlyId = `run_spansfind0000000000000`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + traceId: "trace_spans_find", + spanId: "span_spans_find", + }), + }); + + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + + // Exactly the call site: findRun(where, $replica) — no select, branded replica client. + const stale = await laggingRouter.findRun(where, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); // proves the read was owning-replica-routed + + // Ground truth: the SAME read against a caught-up replica resolves the run — so the null above + // is purely replica lag. This is a read-only span-detail GET (no mutation depends on it), so the + // tolerated stale null is the intended behaviour. + const recovered = (await healthyRouter.findRun(where, BRANDED_REPLICA)) as { + friendlyId: string; + } | null; + expect(recovered).not.toBeNull(); + expect(recovered!.friendlyId).toBe(friendlyId); + } + ); + + // --- spans.$spanId triggeredRuns ---------------------------------------------------------------- + // The span-detail body lists the CHILD runs a span triggered (where {runtimeEnvironmentId, + // parentSpanId}). Under lag freshly-triggered children are not yet on the owning replica, so the + // list is empty/short. This is a DISPLAY list in a polled span-detail response — a missing child + // simply appears on the next poll; nothing decides or mutates on it. Replica-stale empty list, + // tolerated (display list, re-polled). Store fact under lag: []. + heteroRunOpsPostgresTest( + "spans.$spanId triggeredRuns findRuns(branded $replica) is EMPTY under owning-replica lag; caught-up replica lists the child run (display list, re-polled)", + async ({ prisma14, prisma17 }) => { + const { laggingRouter, healthyRouter, legacyReplica } = makeRouters(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "spans_children"); + const parentSpanId = "span_parent_children"; + const childId = `run_${CUID_25}`; + const childFriendlyId = `run_spanschild00000000000`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: childId, + friendlyId: childFriendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + traceId: "trace_spans_children", + spanId: "span_child_children", + parentSpanId, + }), + }); + + const args = { + take: 50, + select: { + friendlyId: true, + taskIdentifier: true, + status: true, + createdAt: true, + }, + where: { + runtimeEnvironmentId: seed.environment.id, + parentSpanId, + }, + }; + + // Exactly the call site: findRuns(args, $replica) — open predicate, branded replica. + const staleRows = await laggingRouter.findRuns(args, BRANDED_REPLICA); + expect(staleRows).toEqual([]); + expect(legacyReplica.wasHit()).toBe(true); + + // Ground truth: the same open-predicate read against a caught-up replica lists the child. + const recoveredRows = (await healthyRouter.findRuns(args, BRANDED_REPLICA)) as Array<{ + friendlyId: string; + }>; + expect(recoveredRows).toHaveLength(1); + expect(recoveredRows[0]!.friendlyId).toBe(childFriendlyId); + } + ); + + // --- trace findPgRun ---------------------------------------------------------------------------- + // Identical read shape to the spans.$spanId findPgRun read above but for the trace GET. findResource + // resolves the run; a null falls through to the mollifier buffer fallback then a retryable 404. No + // mutation is guarded. Replica-stale null, tolerated by eventual-consistency-by-contract (retryable + // 404 + mollifier buffer fallback). Store fact under lag: null. + heteroRunOpsPostgresTest( + "trace route findPgRun findRun(branded $replica) is NULL under owning-replica lag; caught-up replica recovers the run (retryable 404 + buffer fallback tolerate the stale null)", + async ({ prisma14, prisma17 }) => { + const { laggingRouter, healthyRouter, legacyReplica } = makeRouters(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "trace_find"); + const runId = `run_${CUID_25}`; + const friendlyId = `run_tracefind00000000000000`; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + traceId: "trace_trace_find", + spanId: "span_trace_find", + }), + }); + + const where = { friendlyId, runtimeEnvironmentId: seed.environment.id }; + + const stale = await laggingRouter.findRun(where, BRANDED_REPLICA); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + const recovered = (await healthyRouter.findRun(where, BRANDED_REPLICA)) as { + friendlyId: string; + } | null; + expect(recovered).not.toBeNull(); + expect(recovered!.friendlyId).toBe(friendlyId); + } + ); + + // --- sync.traces findRun ------------------------------------------------------------------------ + // The dashboard trace-sync loader resolves a run by traceId to authorize the user + resolve the + // env, then long-polls Electric for the "TaskRun" shape. The where carries ONLY traceId + // (unclassifiable) → unrouted lookup (NEW-first then LEGACY, both on their replicas). Under lag the + // owning replica returns null → the loader returns a 404 "No run found". This is a live-sync loader + // the dashboard re-establishes (long-poll); a run reaching it is user-navigation sourced and has + // existed for seconds, so the stale-null window is transient and re-polled. Tolerated (display/sync + // loader, re-polled; no mutation). Store fact under lag: null. + heteroRunOpsPostgresTest( + "sync.traces findRun(traceId, select, branded $replica) is NULL under owning-replica lag; caught-up replica recovers the run (re-polled sync loader tolerates the stale 404)", + async ({ prisma14, prisma17 }) => { + const { laggingRouter, healthyRouter, legacyReplica } = makeRouters(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "trace_sync"); + const runId = `run_${CUID_25}`; + const traceId = "trace_sync_unique"; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId: `run_tracesync00000000000000`, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + traceId, + spanId: "span_trace_sync", + }), + }); + + // Exactly the call site: findRun({traceId}, {select:{runtimeEnvironmentId}}, $replica). + const stale = await laggingRouter.findRun( + { traceId }, + { select: { runtimeEnvironmentId: true } }, + BRANDED_REPLICA + ); + expect(stale).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + const recovered = (await healthyRouter.findRun( + { traceId }, + { select: { runtimeEnvironmentId: true } }, + BRANDED_REPLICA + )) as { runtimeEnvironmentId: string } | null; + expect(recovered).not.toBeNull(); + expect(recovered!.runtimeEnvironmentId).toBe(seed.environment.id); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.serviceBatchFamilyReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.serviceBatchFamilyReadView.replicaLag.test.ts new file mode 100644 index 000000000..6345b86cd --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.serviceBatchFamilyReadView.replicaLag.test.ts @@ -0,0 +1,346 @@ +// Lagging-replica coverage for the batch-family reads issued by the batch services with NO client: +// findBatchTaskRunByIdempotencyKey (dedup), findBatchTaskRunById (batchTriggerV3 complete arm + +// tryCompleteBatchV3, runEngine batchTrigger, and three streamBatchItems checks), and +// countBatchTaskRunItems (tryCompleteBatchV3). +// +// The property every case proves: these three store methods are the batch-family PRIMARY defaulters. In +// the router, findBatchTaskRunById / findBatchTaskRunByIdempotencyKey fan out NEW→LEGACY and +// countBatchTaskRunItems routes by batchTaskRunId, each leg via #ownPrimary(store, undefined), which +// returns undefined for a null client; each PostgresRunStore method then falls back to +// `client ?? this.prisma` — the OWNING store's PRIMARY, not readOnlyPrisma. (Contrast +// findBatchTaskRunByFriendlyId / findManyBatchTaskRunItems, which default to readOnlyPrisma — covered +// separately.) +// +// So under owning-replica lag the just-written batch/items are found and no downstream decision is +// driven by a stale value: the dedup does not trigger a DUPLICATE batch; a stale null does not DROP +// batch completion, abort sealing, throw a spurious "Batch not found", or defeat the idempotent-retry +// short-circuit; a stale count does not fail to seal a finished batch. Each case asserts the frozen +// owning replica is NEVER consulted (wasHit === false) AND the row/count is correct — proving PRIMARY +// routing, not replica tolerance. +// +// Builds the RoutingRunStore as batchTriggerV3 / RunEngine hold it (a NEW dedicated store + a LEGACY +// control-plane store), seeds the batch (cuid id → LEGACY-owned) on the LEGACY PRIMARY, freezes the +// LEGACY replica with the shared laggingReplica primitive (batchTaskRun + batchTaskRunItem in "missing" +// mode), then invokes each read EXACTLY as the caller does (same method, same args, NO client). + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CreateBatchTaskRunData } from "./types.js"; + +// A cuid (25 chars after the `batch_` prefix) classifies LEGACY, so the batch + its items are owned by +// the legacy (control-plane) store; the client-less reads then fan out / route through the LEGACY leg. +const CUID_25 = "c".repeat(25); + +async function seedEnvironment(prisma: PrismaClient, slugSuffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +// Build the RoutingRunStore exactly as the seam wires it: NEW = dedicated subset store (prisma17), +// LEGACY = control-plane store (prisma14) whose REPLICA is the frozen lagging client. The batch models +// are frozen in "missing" mode: if any batch read were replica-routed it would come back null/[]/0 AND +// flip wasHit — so wasHit(false) + a correct result together prove PRIMARY routing. +function buildLaggingRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyReplica = laggingReplica(prisma14, [ + { model: "batchTaskRun", mode: "missing" }, + { model: "batchTaskRunItem", mode: "missing" }, + ]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +function batchData(overrides: Partial): CreateBatchTaskRunData { + return { + id: `batch_${CUID_25}`, + friendlyId: "batch_svc_family_f", + runtimeEnvironmentId: "PLACEHOLDER", + status: "PENDING", + runCount: 1, + expectedCount: 1, + batchVersion: "runengine:v2", + sealed: false, + ...overrides, + }; +} + +// Seed BatchTaskRunItem rows on the LEGACY primary with FK triggers disabled (no real TaskRun needed — +// countBatchTaskRunItems only matches batchTaskRunId + status). +async function seedItems( + prisma: PrismaClient, + opts: { batchTaskRunId: string; status: string; count: number; idPrefix: string } +) { + // `session_replication_role` is session-scoped, so `SET` and the inserts must share one connection — + // separate pooled Prisma calls can land on different connections, leaving the inserts with FK triggers + // still enabled. Run them in a single transaction with `SET LOCAL`, which applies for the duration of + // this transaction (same connection) and auto-resets on commit. + await prisma.$transaction(async (tx) => { + await tx.$executeRawUnsafe(`SET LOCAL session_replication_role = replica`); + for (let i = 0; i < opts.count; i++) { + await tx.$executeRawUnsafe( + `INSERT INTO "BatchTaskRunItem" (id, status, "batchTaskRunId", "taskRunId", "createdAt", "updatedAt") + VALUES ($1, $2::"BatchTaskRunItemStatus", $3, $4, NOW(), NOW())`, + `${opts.idPrefix}_${i}`, + opts.status, + opts.batchTaskRunId, + `run_item_${opts.idPrefix}_${i}` + ); + } + }); +} + +describe("batch-service family batch reads (no client) route to the owning PRIMARY under replica lag", () => { + // ── findBatchTaskRunByIdempotencyKey (dedup) ────────────────────── + heteroRunOpsPostgresTest( + "findBatchTaskRunByIdempotencyKey dedup hits the LEGACY primary, never the frozen replica (no duplicate batch)", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bf_idem"); + const idempotencyKey = "batch-idem-key"; + await legacyStore.createBatchTaskRun( + batchData({ + friendlyId: "batch_bf_idem_f", + runtimeEnvironmentId: seed.environment.id, + idempotencyKey, + idempotencyKeyExpiresAt: new Date(Date.now() + 60 * 60_000), + }) + ); + + // Exact caller invocation: (environment.id, idempotencyKey), NO client. + const found = await router.findBatchTaskRunByIdempotencyKey( + seed.environment.id, + idempotencyKey + ); + + expect(found).not.toBeNull(); + expect(found!.idempotencyKey).toBe(idempotencyKey); + // Primary routing proof: the frozen LEGACY replica was never consulted. + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + } + ); + + // ── findBatchTaskRunById (complete arm) ──────────────────────────── + heteroRunOpsPostgresTest( + "findBatchTaskRunById complete-arm lookup hits the LEGACY primary, never the frozen replica", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bf_666"); + const batchId = `batch_${CUID_25}`; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId: "batch_bf_666_f", + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const batch = await router.findBatchTaskRunById(batchId); + + expect(batch).not.toBeNull(); + expect(batch!.id).toBe(batchId); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + } + ); + + // ── findBatchTaskRunById (tryCompleteBatchV3) ───────────────────── + heteroRunOpsPostgresTest( + "findBatchTaskRunById in tryCompleteBatchV3 hits the LEGACY primary, never the frozen replica", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bf_1016"); + const batchId = `batch_${CUID_25}`; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId: "batch_bf_1016_f", + runtimeEnvironmentId: seed.environment.id, + status: "PENDING", + sealed: true, + expectedCount: 1, + }) + ); + + const batch = await router.findBatchTaskRunById(batchId); + + expect(batch).not.toBeNull(); + expect(batch!.id).toBe(batchId); + // tryCompleteBatchV3 reads batch.status/sealed/expectedCount off this row — a stale null returns early. + expect(batch!.sealed).toBe(true); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + } + ); + + // ── countBatchTaskRunItems (tryCompleteBatchV3) ─────────────────── + heteroRunOpsPostgresTest( + "countBatchTaskRunItems in tryCompleteBatchV3 counts on the LEGACY primary, never the frozen replica", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bf_1033"); + const batchId = `batch_${CUID_25}`; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId: "batch_bf_1033_f", + runtimeEnvironmentId: seed.environment.id, + expectedCount: 3, + runCount: 3, + }) + ); + // 3 COMPLETED items on the LEGACY primary; the frozen replica would report 0. + await seedItems(prisma14, { + batchTaskRunId: batchId, + status: "COMPLETED", + count: 3, + idPrefix: "bti_1033", + }); + + // Exact caller invocation: ({ batchTaskRunId, status }), NO client. + const completedCount = await router.countBatchTaskRunItems({ + batchTaskRunId: batchId, + status: "COMPLETED", + }); + + expect(completedCount).toBe(3); + // Primary routing proof: the frozen LEGACY item replica was never consulted (else it returns 0). + expect(legacyReplica.wasHit("batchTaskRunItem")).toBe(false); + } + ); + + // ── runEngine batchTrigger findBatchTaskRunById ────────────────────────── + heteroRunOpsPostgresTest( + "findBatchTaskRunById in runEngine batchTrigger hits the LEGACY primary, never the frozen replica", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bf_360"); + const batchId = `batch_${CUID_25}`; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId: "batch_bf_360_f", + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const batch = await router.findBatchTaskRunById(batchId); + + expect(batch).not.toBeNull(); + expect(batch!.id).toBe(batchId); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + } + ); + + // ── streamBatchItems findBatchTaskRunById (validate exists) ─────────────────────── + heteroRunOpsPostgresTest( + "findBatchTaskRunById existence check in streamBatchItems hits the LEGACY primary, never the frozen replica", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bf_134"); + const batchId = `batch_${CUID_25}`; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId: "batch_bf_134_f", + runtimeEnvironmentId: seed.environment.id, + }) + ); + + const batch = await router.findBatchTaskRunById(batchId); + + // The caller does: `if (!batch || batch.runtimeEnvironmentId !== environment.id) throw 404`. + expect(batch).not.toBeNull(); + expect(batch!.runtimeEnvironmentId).toBe(seed.environment.id); + const wouldThrow404 = !batch || batch.runtimeEnvironmentId !== seed.environment.id; + expect(wouldThrow404).toBe(false); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + } + ); + + // ── streamBatchItems findBatchTaskRunById (idempotent-retry recheck) ────────────── + heteroRunOpsPostgresTest( + "findBatchTaskRunById idempotent-retry recheck in streamBatchItems hits the LEGACY primary, never the frozen replica", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bf_206"); + const batchId = `batch_${CUID_25}`; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId: "batch_bf_206_f", + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED", + sealed: true, + processingCompletedAt: new Date(), + }) + ); + + const currentBatch = await router.findBatchTaskRunById(batchId); + + expect(currentBatch).not.toBeNull(); + // isIdempotentRetrySuccess(status, sealed, processingCompletedAt) reads these three off the row. + expect(currentBatch!.status).toBe("COMPLETED"); + expect(currentBatch!.sealed).toBe(true); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + } + ); + + // ── streamBatchItems findBatchTaskRunById (idempotent-retry recheck, second) ────────────── + heteroRunOpsPostgresTest( + "findBatchTaskRunById idempotent-retry recheck (second) in streamBatchItems hits the LEGACY primary, never the frozen replica", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildLaggingRouter(prisma14, prisma17); + const seed = await seedEnvironment(prisma14, "bf_294"); + const batchId = `batch_${CUID_25}`; + await legacyStore.createBatchTaskRun( + batchData({ + id: batchId, + friendlyId: "batch_bf_294_f", + runtimeEnvironmentId: seed.environment.id, + status: "COMPLETED", + sealed: true, + processingCompletedAt: new Date(), + }) + ); + + const currentBatch = await router.findBatchTaskRunById(batchId); + + expect(currentBatch).not.toBeNull(); + expect(currentBatch!.status).toBe("COMPLETED"); + expect(currentBatch!.sealed).toBe(true); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(false); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.sessionMetadataRouteReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.sessionMetadataRouteReadView.replicaLag.test.ts new file mode 100644 index 000000000..17a133bee --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.sessionMetadataRouteReadView.replicaLag.test.ts @@ -0,0 +1,231 @@ +// Coverage for two friendlyId-keyed run reads that pass the webapp's BRANDED `$replica` into +// `runStore.findRun`. A branded replica makes `readYourWrites` return false, so the routing store keeps +// the read on the OWNING store's REPLICA (no primary escalation). Under lag both miss a row present on +// the owning primary. Real split topology via heteroRunOpsPostgresTest; the branded arg is +// `markReadReplicaClient({})`, whose object is never forwarded across DBs — only its BRAND is read. +// +// Reads covered (one case each): +// 1. end-and-continue callingRun lookup +// runStore.findRun({ friendlyId: callingRunId, runtimeEnvironmentId }, { select:{id} }, $replica) +// → owning REPLICA. On null → 404 "callingRunId not found in this environment". +// SAFE: `callingRunId` is `ctx.run.id` of the run CURRENTLY EXECUTING and issuing this API call. +// An executing run was materialised to the primary and dequeued long before it runs user code, so +// its row replicated well before this request — bounded lag cannot miss it. (Contrast the same +// route's read of the just-swapped run, which correctly passes the WRITER `prisma`.) +// 2. metadata GET loader +// runStore.findRun({ friendlyId, runtimeEnvironmentId }, { select:{metadata,metadataType} }, $replica) +// → owning REPLICA. On null the loader's only miss fallback is the mollifier buffer +// (buffer.getEntry — no primary re-read). Once a run has drained from the buffer to the primary +// but the replica has not caught up, it exists on NEITHER the replica NOR the buffer. The primary +// re-read (pass the WRITER `prisma`, i.e. findRunOnPrimary) recovers the live run before the 404. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import { markReadReplicaClient } from "./readReplicaClient.js"; +import type { CreateRunInput } from "./types.js"; + +// A cuid (25 chars after the `run_` prefix) classifies LEGACY, so create + friendlyId-keyed read +// both route to the legacy (control-plane) store first — the store that owns these session/run rows. +const CUID_25 = "c".repeat(25); + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + metadata?: string; + metadataType?: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "PENDING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + metadata: params.metadata, + metadataType: params.metadataType, + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: ["alpha"], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// Build the router the way the webapp holds it, with a lagging legacy (control-plane) replica. +function buildRouterWithLaggingLegacyReplica(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +describe("session and metadata route findRun read-views under replica lag", () => { + // end-and-continue callingRun lookup. + heteroRunOpsPostgresTest( + "end-and-continue callingRun read is stale-null under replica lag", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( + prisma14, + prisma17 + ); + + const seed = await seedEnvironmentLegacy(prisma14, "eac_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = "run_eac_calling"; + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + // The branded $replica client arg (its object is never forwarded; only its brand is read). + const brandedReplica = markReadReplicaClient({} as object); + + // The EXACT :78 read: friendlyId + runtimeEnvironmentId, select {id}, branded $replica. + const staleRead = (await router.findRun( + { friendlyId, runtimeEnvironmentId: seed.environment.id }, + { select: { id: true } }, + brandedReplica + )) as { id: string } | null; + + // Store fact: branded-replica read is REPLICA-routed, so under lag it misses → null. + // A null return leads to a 404 "callingRunId not found in this environment". + expect(staleRead).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // TOLERANCE premise proof: the row DOES exist on the owning primary. In production + // `callingRunId` = ctx.run.id of the CURRENTLY-EXECUTING caller, materialised + dequeued long + // before it runs user code, so it replicated well before the request — bounded lag can't miss + // it. The WRITER read (unbranded → readYourWrites → owning primary) recovers it here. + const primaryRead = (await router.findRun( + { friendlyId, runtimeEnvironmentId: seed.environment.id }, + { select: { id: true } }, + prisma14 as never + )) as { id: string } | null; + expect(primaryRead).not.toBeNull(); + expect(primaryRead!.id).toBe(runId); + } + ); + + // metadata GET loader. + heteroRunOpsPostgresTest( + "metadata loader read is stale-null under replica lag", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( + prisma14, + prisma17 + ); + + const seed = await seedEnvironmentLegacy(prisma14, "meta_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + const friendlyId = "run_meta_view"; + const metadata = '{"phase":"one"}'; + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + metadata, + metadataType: "application/json", + }) + ); + + const brandedReplica = markReadReplicaClient({} as object); + + // The EXACT :43 read: friendlyId + runtimeEnvironmentId, select {metadata, metadataType}, branded $replica. + const staleRead = (await router.findRun( + { friendlyId, runtimeEnvironmentId: seed.environment.id }, + { select: { metadata: true, metadataType: true } }, + brandedReplica + )) as { metadata: string | null; metadataType: string } | null; + + // Store fact: branded-replica read is REPLICA-routed → stale/null under lag. The loader then + // consults ONLY the mollifier buffer (buffer.getEntry, no primary read); once the run has drained + // from the buffer to the primary, the buffer miss + this replica miss together yield a 404 + // "Run not found" for a run that exists on the owning primary. + expect(staleRead).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // The property: re-reading the owning PRIMARY (pass the WRITER `prisma` → findRunOnPrimary) returns + // the live run with its metadata, so the primary re-read on the double-miss resolves it. + const primaryRead = (await router.findRun( + { friendlyId, runtimeEnvironmentId: seed.environment.id }, + { select: { metadata: true, metadataType: true } }, + prisma14 as never + )) as { metadata: string | null; metadataType: string } | null; + expect(primaryRead).not.toBeNull(); + expect(primaryRead!.metadata).toBe(metadata); + expect(primaryRead!.metadataType).toBe("application/json"); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.sessionRunProbeReadAfterWrite.test.ts b/internal-packages/run-store/src/runOpsStore.sessionRunProbeReadAfterWrite.test.ts new file mode 100644 index 000000000..11de9b034 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.sessionRunProbeReadAfterWrite.test.ts @@ -0,0 +1,211 @@ +// Store-seam characterization of the read primitive behind the session-run liveness probe — NOT a +// caller guard. Under owning-replica lag, findRun({id},{select},$replica) misses the just-written run +// (→ null) and the same findRun on the WRITER recovers it. This exercises only the store reads against +// real Postgres. The caller contract (ensureRunForSession reuses the live run and does NOT double- +// trigger the session) is locked separately by the real caller-driven realtime-services replica-lag +// guard, which drives ensureRunForSession end-to-end. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; + +// ownerEngine classifies by internal-id LENGTH: 25 chars → cuid → LEGACY, 26 → run-ops id → NEW. +const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema) +const NEW_ID_26 = "k".repeat(24) + "01"; // → NEW (#new / prisma17, dedicated subset schema) + +// The probe select mirrors getRunStatusAndFriendlyId exactly. +const PROBE_SELECT = { select: { status: true, friendlyId: true } } as const; + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +function seedEnvironmentDedicated(suffix: string) { + return { + organization: { id: `org_${suffix}` }, + project: { id: `proj_${suffix}` }, + environment: { id: `env_${suffix}` }, + }; +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; + status?: "PENDING" | "EXECUTING" | "COMPLETED_SUCCESSFULLY"; +}) { + return { + id: opts.id, + engine: "V2" as const, + // Non-final by default: this is the "run is still alive, reuse it" case the probe must recover. + status: opts.status ?? "EXECUTING", + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +describe("store reads behind the session-run liveness probe — replica misses, writer recovers", () => { + // (a) LEGACY-resident (cuid) session run: triggered moments ago on the control-plane writer; the + // control-plane replica lags. The branded-$replica probe misses; the writer re-probe finds the + // fresh EXECUTING run → ensureRunForSession reuses it instead of double-triggering. + heteroRunOpsPostgresTest( + "LEGACY cuid: branded-replica probe misses under lag; writer re-probe finds the live run", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironmentLegacy(prisma14, "sess_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId: "run_sess_leg", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + }), + }); + + // Step 1 — the getRunStatusAndFriendlyId probe. The call site passes the branded $replica; at + // the routing layer that is identical to omitting the client (both route to the owning store's + // OWN replica and neither escalates to the primary — the brand only suppresses escalation). + // Under lag the owning replica hasn't applied the fresh row → MISS. A naive caller treating this + // null as "run vanished" would double-trigger the session. + const viaReplica = await router.findRun({ id: runId }, PROBE_SELECT); + expect(viaReplica).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // Step 2 — ensureRunForSession's compensation: re-read on the control-plane WRITER before + // deciding liveness. Fresh row is found with its non-final status → the session reuses the + // still-alive run and does NOT double-trigger. + const legacyReplica2 = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore2 = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica2.client, + schemaVariant: "legacy", + }); + const router2 = new RoutingRunStore({ new: newStore, legacy: legacyStore2 }); + const viaWriter = (await router2.findRun({ id: runId }, PROBE_SELECT, prisma14)) as { + status: string; + friendlyId: string; + } | null; + expect(viaWriter).not.toBeNull(); + expect(viaWriter!.friendlyId).toBe("run_sess_leg"); + // Non-final status ⇒ isFinalRunStatus(probe.status) is false ⇒ reuse branch (return existing). + expect(viaWriter!.status).toBe("EXECUTING"); + // The re-probe hit the WRITER, never the replica. + expect(legacyReplica2.wasHit()).toBe(false); + } + ); + + // (b) NEW-resident (run-ops id) session run under split: the session row + currentRunId live on the + // control-plane, but the run itself is on the NEW DB whose replica lags. The probe passes the + // BRANDED control-plane $replica; the router routes by residency to the NEW store's OWN replica → + // miss. The writer re-probe (control-plane writer identity) routes to the NEW store's OWN writer → + // finds the fresh run. Mirrors the live shape (sessions pass the control-plane clients; the run may + // be NEW-resident under split-ON). + heteroRunOpsPostgresTest( + "NEW run-ops id: branded-replica probe misses under NEW-replica lag; writer re-probe finds the live run", + async ({ prisma14, prisma17 }) => { + const newReplica = laggingReplica(prisma17, [{ model: "taskRun", mode: "missing" }]); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: newReplica.client as never, + schemaVariant: "dedicated", + }); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = seedEnvironmentDedicated("sess_new"); + const runId = `run_${NEW_ID_26}`; // run-ops id → NEW + await prisma17.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId: "run_sess_new", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + status: "EXECUTING", + }), + }); + + // Step 1 — the probe routes by residency to the NEW store's OWN (lagging) replica → miss. + const viaReplica = await router.findRun({ id: runId }, PROBE_SELECT); + expect(viaReplica).toBeNull(); + expect(newReplica.wasHit()).toBe(true); + + // Step 2 — writer re-probe. Passing the control-plane WRITER (writer identity) routes to the + // NEW store's OWN writer (never forwarding the control-plane client into the NEW DB) → finds + // the fresh run with its non-final status → reuse, no double-trigger. + const newReplica2 = laggingReplica(prisma17, [{ model: "taskRun", mode: "missing" }]); + const newStore2 = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: newReplica2.client as never, + schemaVariant: "dedicated", + }); + const router2 = new RoutingRunStore({ new: newStore2, legacy: legacyStore }); + const viaWriter = (await router2.findRun({ id: runId }, PROBE_SELECT, prisma14)) as { + status: string; + friendlyId: string; + } | null; + expect(viaWriter).not.toBeNull(); + expect(viaWriter!.friendlyId).toBe("run_sess_new"); + expect(viaWriter!.status).toBe("EXECUTING"); + expect(newReplica2.wasHit()).toBe(false); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.storeReceiverReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.storeReceiverReadView.replicaLag.test.ts new file mode 100644 index 000000000..58f788f9d --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.storeReceiverReadView.replicaLag.test.ts @@ -0,0 +1,288 @@ +// Lagging-replica coverage for two RunStore reads whose receiver is `store` / `deps.store`. Both are +// split-mode reads RoutingRunStore serves from the OWNING store's REPLICA (no writer / branded replica). +// This file freezes the owning replica via laggingReplica, invokes each read with the EXACT caller args, +// asserts the under-lag behavior, then recovers the row on the owning PRIMARY (proving pure lag + routing). +// (1) findBatchTaskRunByFriendlyId (no client): under lag returns null and the caller's `if(batch)` guard +// falls through — no throw — leaving the friendlyId unresolved for a ClickHouse filter that lags more. +// (2) readThroughRun's readLegacy leg (deps.store.findRun): a LEGACY-resident run's pre-cutover row is +// long replicated, so findRun returns the OLD row (stale scalars, correct immutable identity), never +// null; the handler enriches from event-supplied endTime/updatedAt. + +import { + heteroRunOpsPostgresTest, + laggingReplica, + type LaggingModel, +} from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import { markReadReplicaClient } from "./readReplicaClient.js"; + +// A cuid (25 chars after the `run_`/`batch_` prefix) classifies LEGACY, so create + read both route to +// the legacy (control-plane / prisma14, full schema) store — the store that owns these rows in prod. +const CUID_25 = "c".repeat(25); + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +// Build the router the way the webapp holds it, with the LEGACY (control-plane) replica FROZEN per the +// given configs. NEW is non-lagging + empty, so the router's miss-probe of NEW returns null/[] naturally +// (no phantom hit masking the legacy-replica staleness). +function buildRouter( + prisma14: PrismaClient, + prisma17: RunOpsPrismaClient, + configs: readonly LaggingModel[] +) { + const legacyReplica = laggingReplica(prisma14, configs); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +describe("run-ops split — store/deps.store receiver read-views vs. a lagging replica", () => { + // ── findBatchTaskRunByFriendlyId (no client) — runsRepository runs-list batch filter ─────────────── + heteroRunOpsPostgresTest( + "findBatchTaskRunByFriendlyId (no client) → owning REPLICA; under lag returns null, the caller's `if(batch)` guard falls through leaving the friendlyId unresolved for the ClickHouse filter (tolerated: CH lags more, list revalidates). Primary resolves the id.", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouter(prisma14, prisma17, [ + { model: "batchTaskRun", mode: "missing" }, + ]); + const seed = await seedEnvironmentLegacy(prisma14, "batchfilter"); + const batchId = `batch_${CUID_25}`; // cuid → LEGACY-resident + const batchFriendlyId = "batch_batchfilter"; + await legacyStore.createBatchTaskRun({ + id: batchId, + friendlyId: batchFriendlyId, + runtimeEnvironmentId: seed.environment.id, + }); + + // Invoke EXACTLY as convertRunListInputOptionsToFilterRunsOptions:329 does — friendlyId + envId, + // NO client, NO include. + const underLag = await router.findBatchTaskRunByFriendlyId( + batchFriendlyId, + seed.environment.id + ); + + // Store-level fact: default readOnlyPrisma → owning REPLICA (both legs). Frozen → null. + expect(underLag).toBeNull(); + expect(legacyReplica.wasHit("batchTaskRun")).toBe(true); + + // Reproduce the caller's guard and its downstream effect on the ClickHouse filter: on null it + // leaves `batchId` as the unresolved friendlyId — a filter value, not an error (no throw here). + const convertedBatchId = (() => { + let out: string = batchFriendlyId; // convertedOptions.batchId starts as options.batchId + const batch = underLag; // store.findBatchTaskRunByFriendlyId(...) + if (batch) { + out = (batch as { id: string }).id; + } + return out; + })(); + // The friendlyId flows UNRESOLVED into the ClickHouse `batch_id` filter (matches nothing → empty + // list) — consistent with ClickHouse's own laggier state (no child run of a just-created batch is + // ingested yet), and self-heals on the next poll once the replica catches up. + expect(convertedBatchId).toBe(batchFriendlyId); + expect(convertedBatchId.startsWith("batch_")).toBe(true); + + // PRIMARY contrast: an unbranded WRITER client → #ownPrimary → owning primary → the batch IS + // resolved to its internal id. Proves the null above is purely replica lag + replica routing. + const onPrimary = await router.findBatchTaskRunByFriendlyId( + batchFriendlyId, + seed.environment.id, + undefined, + prisma14 as never + ); + expect(onPrimary).not.toBeNull(); + expect(onPrimary!.id).toBe(batchId); + expect(onPrimary!.friendlyId).toBe(batchFriendlyId); + // With the primary read, the caller WOULD resolve the filter to the internal id. + const resolvedWithPrimary = onPrimary ? onPrimary.id : batchFriendlyId; + expect(resolvedWithPrimary).toBe(batchId); + } + ); + + // ── readLegacy-leg findRun({id},{select},replica) — readThroughRun for a LEGACY run ──────────── + // The readLegacy leg of readThroughRun for a LEGACY-resident run. Real behavior: the run row is OLD + // (pre-cutover, long replicated), so findRun returns it with STALE mutable scalars — never null — and + // the event handler enriches from event-supplied time + the row's immutable identifiers. + heteroRunOpsPostgresTest( + "readLegacy-leg findRun({id}) (branded $replica) → owning REPLICA; a LEGACY run's OLD row is present-but-STALE under lag (not a missing-row RYW); immutable spanId/traceId/friendlyId/createdAt are correct and the handler uses event-supplied endTime/updatedAt. Primary shows the fresh status.", + async ({ prisma14, prisma17 }) => { + const runId = `run_${CUID_25}`; // cuid → LEGACY-resident (pre-cutover row) + const friendlyId = "run_evtenrich_leg"; + const spanId = "span_evtenrich_fixed"; + const parentSpanId = "span_evtenrich_parent"; + const traceId = `trace_${runId}`; + const createdAt = new Date("2024-01-01T00:00:00.000Z"); // long before cut-over + + // The exact select the runSucceeded handler uses. + const eventSelect = { + id: true, + friendlyId: true, + traceId: true, + spanId: true, + parentSpanId: true, + createdAt: true, + completedAt: true, + taskIdentifier: true, + projectId: true, + runtimeEnvironmentId: true, + environmentType: true, + isTest: true, + organizationId: true, + taskEventStore: true, + runTags: true, + batchId: true, + } as const; + + // Frozen replica: an OLD snapshot of the SAME row — the selected mutable scalar stale (no + // completedAt yet), immutable identifiers identical to the primary. laggingReplica "frozen" returns + // this verbatim (bypasses Postgres projection), matched on top-level `where` id equality. + const staleRow = { + id: runId, + friendlyId, + traceId, + spanId, + parentSpanId, + createdAt, + completedAt: null as Date | null, // STALE — primary already has a completedAt + taskIdentifier: "my-task", + projectId: "", // filled after seed + runtimeEnvironmentId: "", // filled after seed + environmentType: "DEVELOPMENT", + isTest: false, + organizationId: "", // filled after seed + taskEventStore: "taskEvent", + runTags: [] as string[], + batchId: null as string | null, + }; + + const { router, legacyReplica } = buildRouter(prisma14, prisma17, [ + { model: "taskRun", mode: "frozen", rows: [staleRow] }, + ]); + const seed = await seedEnvironmentLegacy(prisma14, "evtenrich"); + staleRow.projectId = seed.project.id; + staleRow.runtimeEnvironmentId = seed.environment.id; + staleRow.organizationId = seed.organization.id; + + const completedAt = new Date("2024-06-01T00:00:00.000Z"); + await prisma14.taskRun.create({ + data: { + id: runId, + engine: "V2", + status: "COMPLETED_SUCCESSFULLY", // the terminal state the event fires on + friendlyId, + runtimeEnvironmentId: seed.environment.id, + environmentType: "DEVELOPMENT", + organizationId: seed.organization.id, + projectId: seed.project.id, + taskIdentifier: "my-task", + payload: "{}", + payloadType: "application/json", + traceContext: {}, + traceId, + spanId, + parentSpanId, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt, + completedAt, + }, + }); + + // Invoke EXACTLY as the readLegacy closure does: findRun({id},{select}, ). + const brandedReplica = markReadReplicaClient({} as object); + const underLag = (await router.findRun( + { id: runId }, + { select: eventSelect }, + brandedReplica as never + )) as typeof staleRow | null; + + expect(legacyReplica.wasHit("taskRun")).toBe(true); + // KEY REAL-BEHAVIOR FACT: the read returns the OLD row (present), NOT null — a legacy-resident run's + // row was inserted pre-cutover and is long replicated. So this is a stale-row read, not a + // missing-row read-your-write. + expect(underLag).not.toBeNull(); + expect(underLag!.id).toBe(runId); + // The selected mutable scalar is STALE off the frozen replica… + expect(underLag!.completedAt).toBeNull(); + // …but every IMMUTABLE identifier the event enrichment actually depends on is correct under lag. + expect(underLag!.friendlyId).toBe(friendlyId); + expect(underLag!.spanId).toBe(spanId); + expect(underLag!.parentSpanId).toBe(parentSpanId); + expect(underLag!.traceId).toBe(traceId); + expect(underLag!.createdAt).toEqual(createdAt); + expect(underLag!.taskEventStore).toBe("taskEvent"); + expect(underLag!.organizationId).toBe(seed.organization.id); + + // TOLERANCE assertion: reproduce the handler's use of the enrichment read. endTime and updatedAtMs + // come from the EVENT payload (not the stale row); the span-close/publish keys are immutable row + // fields. So the emitted completion event + realtime publish are CORRECT despite the stale status. + const eventTime = new Date("2024-06-01T00:00:01.000Z"); // engine event `time` + const eventRunUpdatedAt = new Date("2024-06-01T00:00:01.500Z"); // engine event `run.updatedAt` + const completionEvent = { + endTime: eventTime, // completeSuccessfulRunEvent({ run, endTime: time }) + runId: underLag!.id, + spanId: underLag!.spanId, + traceId: underLag!.traceId, + friendlyId: underLag!.friendlyId, + taskEventStore: underLag!.taskEventStore, + }; + const publish = { + runId: underLag!.id, + envId: seed.environment.id, + tags: underLag!.runTags, + batchId: underLag!.batchId, + updatedAtMs: eventRunUpdatedAt.getTime(), // from event, not the stale row + }; + expect(completionEvent.endTime).toEqual(eventTime); // not derived from the stale row + expect(completionEvent.spanId).toBe(spanId); // immutable → correct under lag + expect(publish.updatedAtMs).toBe(eventRunUpdatedAt.getTime()); + + // PRIMARY contrast: an unbranded WRITER client → #ownPrimary → owning primary → the FRESH terminal + // status. Proves the staleness above is confined to the replica. + const onPrimary = (await router.findRun( + { id: runId }, + { select: { status: true, completedAt: true } }, + prisma14 as never + )) as { status: string; completedAt: Date | null } | null; + expect(onPrimary?.status).toBe("COMPLETED_SUCCESSFULLY"); + expect(onPrimary?.completedAt).toEqual(completedAt); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.test.ts b/internal-packages/run-store/src/runOpsStore.test.ts index e64bf9f28..75f838deb 100644 --- a/internal-packages/run-store/src/runOpsStore.test.ts +++ b/internal-packages/run-store/src/runOpsStore.test.ts @@ -1643,17 +1643,22 @@ describe("RoutingRunStore.findTaskRunAttempt residency routing", () => { status?: string; } ) { - await prisma.$executeRawUnsafe(`SET session_replication_role = replica`); - await prisma.$executeRawUnsafe( - `INSERT INTO "TaskRunAttempt" (id, number, "friendlyId", "taskRunId", "backgroundWorkerId", "backgroundWorkerTaskId", "runtimeEnvironmentId", "queueId", status, "createdAt", "updatedAt", "usageDurationMs", "outputType") - VALUES ($1, 1, $2, $3, 'synthetic-worker', 'synthetic-worker-task', $4, 'synthetic-queue', $5::"TaskRunAttemptStatus", NOW(), NOW(), 0, 'application/json')`, - opts.attemptId, - opts.friendlyId, - opts.runId, - opts.runtimeEnvironmentId, - opts.status ?? "COMPLETED" - ); - await prisma.$executeRawUnsafe(`SET session_replication_role = DEFAULT`); + // `session_replication_role` is session-scoped, so the `SET` and the insert must share one + // connection — separate pooled calls can split across connections, leaving the insert with FK + // triggers on (the synthetic worker/queue ids have no backing rows). One transaction with + // `SET LOCAL` keeps them co-connected and auto-resets on commit. + await prisma.$transaction(async (tx) => { + await tx.$executeRawUnsafe(`SET LOCAL session_replication_role = replica`); + await tx.$executeRawUnsafe( + `INSERT INTO "TaskRunAttempt" (id, number, "friendlyId", "taskRunId", "backgroundWorkerId", "backgroundWorkerTaskId", "runtimeEnvironmentId", "queueId", status, "createdAt", "updatedAt", "usageDurationMs", "outputType") + VALUES ($1, 1, $2, $3, 'synthetic-worker', 'synthetic-worker-task', $4, 'synthetic-queue', $5::"TaskRunAttemptStatus", NOW(), NOW(), 0, 'application/json')`, + opts.attemptId, + opts.friendlyId, + opts.runId, + opts.runtimeEnvironmentId, + opts.status ?? "COMPLETED" + ); + }); } heteroPostgresTest( diff --git a/internal-packages/run-store/src/runOpsStore.ttlExpireOrphanReplicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.ttlExpireOrphanReplicaLag.test.ts new file mode 100644 index 000000000..bcc0b1e07 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.ttlExpireOrphanReplicaLag.test.ts @@ -0,0 +1,222 @@ +// Guard for the run-ops split read-after-write property on the TTL BATCH EXPIRY path. +// +// TtlSystem.expireRunsBatch reads the runs it is about to expire with +// runStore.findRuns({ where: { id: { in: runIds } }, select: {...} }, this.$.prisma) +// threading the owning primary (as the single-run path expireRun does), so it reads the run it just +// wrote. Why that matters: a client-less findRuns is NOT a read-your-writes signal, so RoutingRunStore +// routes each leg to the owning store's REPLICA. A run whose row is not yet visible there comes back +// absent, is bucketed `not_found`, skipped, and never expired — and since the TTL Lua script has +// ALREADY removed the run from the Redis queue before this guard runs, that miss does not self-heal: +// the run would be permanently ORPHANED (never expired, never emits `runExpired`, never re-queued). +// +// This is the weakest of the read-after-write class: the run row is usually written well before TTL +// fires, so a lagging replica has normally caught up. The window only exists for a very-short-TTL run +// swept moments after creation. This test forces the lag with the shared lagging-replica primitive +// and pins both paths: the primary-threaded read (the production path) expires the run, while the +// client-less path misses it on the replica and buckets it not_found. +// +// Deterministic via heteroRunOpsPostgresTest (real split topology, NEVER mocked): a LEGACY-resident +// (cuid) run is created on the control-plane writer; the control-plane replica lags (its `taskRun` +// reads come back empty). We drive the EXACT guard logic ttlSystem.expireRunsBatch uses. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CreateRunInput, ReadClient } from "./types.js"; + +// ownerEngine (classifyResidency) routes a run-ops v1 body → NEW, everything else → LEGACY. +const CUID_25 = "d".repeat(25); // → LEGACY (control-plane / prisma14, full schema) + +async function seedEnvironment(prisma: PrismaClient, slugSuffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "PENDING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "my-task", + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + ttl: "1s", + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// A line-for-line mirror of the guard in TtlSystem.expireRunsBatch: read the candidate runs, filter +// to PENDING+unlocked, and bucket every candidate id whose row is absent as `not_found`. `readClient` +// models the two paths: the client-less batch guard passes nothing (replica default); the +// read-your-writes path threads the primary (as single-run expireRun does). +async function guardExpireBatch( + runStore: RoutingRunStore, + runIds: string[], + readClient?: ReadClient +): Promise<{ expired: string[]; skipped: { runId: string; reason: string }[] }> { + const expired: string[] = []; + const skipped: { runId: string; reason: string }[] = []; + + const runs = (await runStore.findRuns( + { + where: { id: { in: runIds } }, + select: { + id: true, + spanId: true, + status: true, + lockedAt: true, + ttl: true, + taskEventStore: true, + createdAt: true, + associatedWaitpoint: { select: { id: true } }, + organizationId: true, + projectId: true, + runtimeEnvironmentId: true, + }, + }, + readClient + )) as Array<{ id: string; status: string; lockedAt: Date | null }>; + + const runsToExpire: typeof runs = []; + for (const run of runs) { + if (run.status !== "PENDING") { + skipped.push({ runId: run.id, reason: `status_${run.status}` }); + continue; + } + if (run.lockedAt) { + skipped.push({ runId: run.id, reason: "locked" }); + continue; + } + runsToExpire.push(run); + } + + const foundRunIds = new Set(runs.map((r) => r.id)); + for (const runId of runIds) { + if (!foundRunIds.has(runId)) { + skipped.push({ runId, reason: "not_found" }); + } + } + + // The rest of expireRunsBatch would write EXPIRED + emit runExpired for runsToExpire only. + for (const run of runsToExpire) { + expired.push(run.id); + } + + return { expired, skipped }; +} + +describe("run-ops split — TTL batch expiry guard threads the owning primary; the client-less path orphans the run", () => { + // A LEGACY-resident (cuid) run sits PENDING on the control-plane WRITER; the control-plane REPLICA + // lags (its `taskRun` reads come back empty, as just after a very-short-TTL run is created). The + // TTL Lua script has already dequeued the run, so this guard is its only chance to be expired. + heteroRunOpsPostgresTest( + "LEGACY cuid: primary-threaded guard expires the run; the client-less control path misses it on the lagging replica -> orphaned as not_found", + async ({ prisma14, prisma17 }) => { + // Control-plane replica lags on `taskRun`: the just-created run row is not visible yet. + const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, "ttl_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId: "run_ttl_leg", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + // Sanity: the run really is PENDING on the primary (so the only reason a guard could skip it is + // the misrouted replica read, not a genuinely-missing/non-PENDING run). + const onPrimary = await legacyStore.findRun({ id: runId }, prisma14); + expect(onPrimary).not.toBeNull(); + expect(onPrimary!.status).toBe("PENDING"); + + // Contrast — the read-your-writes path (thread the primary, as single-run expireRun does): the + // run is found on the owning store's writer and IS expired. Establishes the run is genuinely + // expirable, so any miss below is purely the misrouted read. + const viaPrimary = await guardExpireBatch(router, [runId], prisma14); + expect(viaPrimary.expired).toEqual([runId]); + expect(viaPrimary.skipped).toEqual([]); + + // The client-less control path: findRuns with NO client → owning store's REPLICA. + const guard = await guardExpireBatch(router, [runId]); + + // Proves the misrouted read is what happened: the (stale) legacy replica was consulted. + expect(legacyReplica.wasHit("taskRun")).toBe(true); + + // The property the primary read secures: a client-less findRuns misses the run on the lagging + // replica and buckets it `not_found`, so the run would be ORPHANED — never expired. Pinning it + // here documents why expireRunsBatch threads this.$.prisma (the viaPrimary path above). + expect(guard.expired).toEqual([]); + expect(guard.skipped).toEqual([{ runId, reason: "not_found" }]); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.usageCostUndercountReadAfterWrite.test.ts b/internal-packages/run-store/src/runOpsStore.usageCostUndercountReadAfterWrite.test.ts new file mode 100644 index 000000000..fcbe19eea --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.usageCostUndercountReadAfterWrite.test.ts @@ -0,0 +1,253 @@ +// Read-after-write coverage for the USAGE/COST read-modify-write in the attempt-completion path +// (attemptSucceeded, and the same shape in cancelRun / #permanentlyFailRun). The completion path reads +// the run's CUMULATIVE usageDurationMs/costInCents, recomputes, and writes back. Across a run's retries +// usage accumulates on the OWNING store's PRIMARY (each completion is a read-modify-write); if the read +// were served by a lagging replica pinned at a pre-delta snapshot, the recomputed total would be built +// on the STALE cumulative and the prior delta LOST -> usage/cost UNDERCOUNT (a classic lost update). +// Reading via findRunOnPrimary (the owning store's PRIMARY) keeps the total correct. +// +// Deterministic via heteroRunOpsPostgresTest (real split topology, never mocked) + the shared +// laggingReplica primitive (frozen mode: the owning store's replica is pinned at the pre-delta snapshot +// while the primary advances). The property: the primary read sees the fresher cumulative and the write +// preserves it; a client-less (replica) read would clobber it downward. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CompletionSnapshotInput, CreateRunInput, RunStoreSchemaVariant } from "./types.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +// ownerEngine (classifyResidency) routes a cuid → LEGACY (#legacy / prisma14, full schema). +const CUID_25 = "d".repeat(25); + +async function seedEnvironment( + prisma: AnyClient, + schemaVariant: RunStoreSchemaVariant, + slugSuffix: string +) { + if (schemaVariant === "dedicated") { + return { + organization: { id: `org_${slugSuffix}` }, + project: { id: `proj_${slugSuffix}` }, + environment: { id: `env_${slugSuffix}` }, + }; + } + const organization = await (prisma as PrismaClient).organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await (prisma as PrismaClient).project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await (prisma as PrismaClient).runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + taskIdentifier: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "EXECUTING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: params.taskIdentifier, + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: [], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "EXECUTING", + description: "Run is executing", + runStatus: "EXECUTING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// Line-for-line mirror of runAttemptSystem.attemptSucceeded's usage read-modify-write (dev-mode: +// cost is a pure passthrough of the read `currentCostInCents`, per #calculateUpdatedUsage's +// `environmentType !== "DEVELOPMENT"` guard). The cumulative read goes to the owning primary via +// findRunOnPrimary, exactly as the engine issues it. +async function attemptSucceededRecomputeUsage( + router: RoutingRunStore, + params: { + runId: string; + attemptDurationMs: number; + snapshot: CompletionSnapshotInput; + } +): Promise { + // Read current usage totals via findRunOnPrimary (owning store's PRIMARY), as + // attemptSucceeded/cancelRun/#permanentlyFailRun do. + const currentRun = (await router.findRunOnPrimary( + { id: params.runId }, + { + select: { + usageDurationMs: true, + costInCents: true, + machinePreset: true, + }, + } + )) as { usageDurationMs: number; costInCents: number; machinePreset: string | null } | null; + + if (!currentRun) { + throw new Error("Run not found"); + } + + // #calculateUpdatedUsage, DEVELOPMENT branch: usage accumulates, cost passes through unchanged. + const usageDurationMs = currentRun.usageDurationMs + params.attemptDurationMs; + const costInCents = currentRun.costInCents; + + await router.completeAttemptSuccess( + params.runId, + { + completedAt: new Date(), + output: '{"done":true}', + outputType: "application/json", + usageDurationMs, + costInCents, + snapshot: params.snapshot, + }, + { select: { id: true, usageDurationMs: true, costInCents: true } } + ); +} + +describe("run-ops split — attempt-completion usage/cost read-modify-write must read the OWNING store's WRITER, not its lagging replica", () => { + // A LEGACY-resident (cuid) run whose earlier attempts already accumulated usage/cost on the + // control-plane PRIMARY, while the control-plane REPLICA still lags at the pre-delta snapshot. + // The final attemptSucceeded reads the cumulative usage on the owning primary, recomputes, and + // writes back — so the fresher primary total is preserved (no lost update); a replica-routed read + // would clobber it downward. + heteroRunOpsPostgresTest( + "LEGACY cuid: attemptSucceeded accumulates usage on the primary total, not the lagging replica", + async ({ prisma14, prisma17 }) => { + const seed = await seedEnvironment(prisma14, "legacy", "usage_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + + // Build the store on the PRIMARY first so createRun + the "prior attempt" write both land there. + const primaryLegacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + await primaryLegacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId: "run_usage_leg", + taskIdentifier: "my-task", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + + // The pre-delta snapshot the replica is still pinned at: usage/cost as freshly created (both 0). + const staleRun = await prisma14.taskRun.findFirstOrThrow({ where: { id: runId } }); + const PRIOR_USAGE = staleRun.usageDurationMs; // 0 at create + const PRIOR_COST = staleRun.costInCents; // 0 at create + + // A prior attempt's read-modify-write already committed a cumulative delta to the PRIMARY. + const PRIOR_ATTEMPT_USAGE_MS = 1000; + const PRIOR_ATTEMPT_COST = 250; + await prisma14.taskRun.update({ + where: { id: runId }, + data: { + usageDurationMs: PRIOR_USAGE + PRIOR_ATTEMPT_USAGE_MS, + costInCents: PRIOR_COST + PRIOR_ATTEMPT_COST, + }, + }); + + // The control-plane replica lags: frozen at the pre-delta snapshot (usage/cost = 0). + const legacyReplica = laggingReplica(prisma14, [ + { + model: "taskRun", + mode: "frozen", + rows: [staleRun as unknown as Record], + }, + ]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + // The final successful attempt adds its own delta on top of the accumulated total. + const FINAL_ATTEMPT_USAGE_MS = 500; + await attemptSucceededRecomputeUsage(router, { + runId, + attemptDurationMs: FINAL_ATTEMPT_USAGE_MS, + snapshot: { + executionStatus: "FINISHED", + description: "Task completed successfully", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 2, + environmentId: seed.environment.id, + environmentType: "DEVELOPMENT", + projectId: seed.project.id, + organizationId: seed.organization.id, + }, + }); + + // The cumulative-usage read hits the owning PRIMARY, so the lagging replica is never consulted + // (a client-less read would hit it and undercount). + expect(legacyReplica.wasHit("taskRun")).toBe(false); + + // Ground truth lives on the PRIMARY. Correct cumulative total = prior accumulated + this attempt. + const finalRun = await prisma14.taskRun.findFirstOrThrow({ where: { id: runId } }); + + // The primary read sees the accumulated 1000ms/250c, so the RMW writes 1000 + 500 = 1500 (no + // undercount). A client-less (replica) read would recompute 0 + 500 = 500. + expect(finalRun.usageDurationMs).toBe( + PRIOR_USAGE + PRIOR_ATTEMPT_USAGE_MS + FINAL_ATTEMPT_USAGE_MS + ); + expect(finalRun.costInCents).toBe(PRIOR_COST + PRIOR_ATTEMPT_COST); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.waitpointCompleteRouteAndReplayLoaderReadView.replicaLag.test.ts b/internal-packages/run-store/src/runOpsStore.waitpointCompleteRouteAndReplayLoaderReadView.replicaLag.test.ts new file mode 100644 index 000000000..8493c66b0 --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.waitpointCompleteRouteAndReplayLoaderReadView.replicaLag.test.ts @@ -0,0 +1,246 @@ +// Coverage for two dashboard read call sites: +// +// complete-waitpoint route (dashboard "Complete waitpoint" action) +// runStore.findWaitpoint({ select: { projectId, environmentId }, where: { id } }) (no client) +// replay loader (renders the Replay dialog) +// runStore.findRun({ friendlyId }, { select: {...} }) (no client) +// +// Both pass NO client, so per the per-method default they route to the OWNING store's REPLICA (a `{ id }` +// waitpoint lookup even resolves its owning store via a replica probe first — #resolveWaitpointStore(id, +// onPrimary=false) — so a not-yet-replicated row is invisible on BOTH the resolution probe and the read). +// The owning replica is frozen with the shared laggingReplica on the REAL split topology +// (heteroRunOpsPostgresTest — NEVER mocked); each case asserts exactly what the caller sees. +// +// The complete-waitpoint route GATES a mutation in the SAME request: +// const waitpoint = await runStore.findWaitpoint({ ..., where: { id } }); +// if (waitpoint?.projectId !== project.id) return "No waitpoint found"; +// ... engine.completeWaitpoint({ id }) // the write +// Under lag the replica read is null (the first leg). A bare null makes `undefined !== project.id` true, +// which would return "No waitpoint found" for a waitpoint present on the primary. So on a null read the +// route re-reads the owning primary via runStore.findWaitpointOnPrimary({ where: { id } }) before +// deciding. The test pins both legs: the replica read is null, and the owning-primary re-read resolves +// the same waitpoint. +// +// The replay loader read is a tolerated read-view: it only renders a form; on a miss it falls through to +// the mollifier buffer and finally a 404 the user retries. It drives NO write and no decision on stale +// data — eventual consistency is the intended contract for opening a dialog for an (already-existing, +// historical) run. The consequential write path (the replay ACTION) has its OWN primary re-read, so a +// stale loader render never causes a wrong replay. The test asserts the store-level fact (null under +// lag, replica hit) AND proves the run exists on the primary (so the null is pure lag, not absence). + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; + +const CUID_25 = "e".repeat(25); // cuid id-shape -> LEGACY (#legacy / prisma14, full schema) + +async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${suffix}`, + pkApiKey: `pk_dev_${suffix}`, + shortcode: `short_${suffix}`, + }, + }); + return { organization, project, environment }; +} + +// Build a LEGACY-owning router whose legacy replica is frozen (lagging); the NEW store is real +// (non-lagging) so the on-miss fan-out to the other store's replica also legitimately misses. +function buildRouterWithLaggingLegacyReplica(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const legacyReplica = laggingReplica(prisma14, [ + { model: "taskRun", mode: "missing" }, + { model: "waitpoint", mode: "missing" }, + ]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + return { router, legacyStore, legacyReplica }; +} + +// Seed a standalone, still-PENDING MANUAL token waitpoint on the WRITER (exactly as minting a resume +// token does). The complete route looks it up by id right after; under replica lag that read misses. +async function seedPendingTokenWaitpoint( + store: PostgresRunStore, + params: { id: string; friendlyId: string; projectId: string; environmentId: string } +) { + await store.upsertWaitpoint({ + where: { + environmentId_idempotencyKey: { + environmentId: params.environmentId, + idempotencyKey: params.id, + }, + }, + create: { + id: params.id, + friendlyId: params.friendlyId, + type: "MANUAL", + status: "PENDING", + idempotencyKey: params.id, + userProvidedIdempotencyKey: false, + projectId: params.projectId, + environmentId: params.environmentId, + }, + update: {}, + }); +} + +function taskRunData(opts: { + id: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}) { + return { + id: opts.id, + engine: "V2" as const, + status: "COMPLETED_SUCCESSFULLY" as const, + friendlyId: opts.friendlyId, + runtimeEnvironmentId: opts.runtimeEnvironmentId, + environmentType: "DEVELOPMENT" as const, + organizationId: opts.organizationId, + projectId: opts.projectId, + taskIdentifier: "my-task", + payload: JSON.stringify({ hello: "world" }), + payloadType: "application/json", + traceContext: {}, + traceId: `trace_${opts.id}`, + spanId: `span_${opts.id}`, + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + }; +} + +// The exact projection the complete route reads to run its project/env auth guard. +const COMPLETE_WAITPOINT_SELECT = { projectId: true, environmentId: true } as const; + +// The loader's source-run projection, subset that exists on both variants. +const REPLAY_LOADER_SELECT = { + payload: true, + payloadType: true, + runtimeEnvironmentId: true, + projectId: true, + taskIdentifier: true, +} as const; + +describe("waitpoint-complete route + replay loader read-view under replica lag", () => { + // ── complete-waitpoint route ──────────────────────────────────────────────────────────────────── + // findWaitpoint(no client) → owning REPLICA. Under lag it returns null (the first leg); a bare null + // makes the auth guard `waitpoint?.projectId !== project.id` refuse a real waitpoint with + // "No waitpoint found". So the route re-reads the owning primary (findWaitpointOnPrimary), which + // resolves the same waitpoint. + heteroRunOpsPostgresTest( + "complete-waitpoint route findWaitpoint is stale-null under replica lag and resolves on the primary re-read", + async ({ prisma14, prisma17 }) => { + const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( + prisma14, + prisma17 + ); + const seed = await seedEnvironmentLegacy(prisma14, "wpcomplete_leg"); + const waitpointId = `waitpoint_${CUID_25}`; // cuid → LEGACY + await seedPendingTokenWaitpoint(legacyStore, { + id: waitpointId, + friendlyId: "waitpoint_wpcomplete_leg", + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + + // The EXACT complete-route read: select {projectId, environmentId}, where {id}, NO client. + const fromReplica = await router.findWaitpoint({ + select: COMPLETE_WAITPOINT_SELECT, + where: { id: waitpointId }, + }); + + // Store seam: the owning replica lags → null. This is precisely what drives the route to fail: + // (fromReplica?.projectId !== project.id) == (undefined !== ) == true + // → "No waitpoint found", and engine.completeWaitpoint is never called. The waitpoint exists. + expect(fromReplica).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + // Reproduce the route's guard evaluation on the stale value: it wrongly fails. + const guardFailsUnderLag = fromReplica?.projectId !== seed.project.id; + expect(guardFailsUnderLag).toBe(true); + + // The route re-reads the owning primary on the miss. It finds the waitpoint, its projectId + // matches, and the guard passes — the completion proceeds. + const fromPrimary = await router.findWaitpointOnPrimary({ + select: COMPLETE_WAITPOINT_SELECT, + where: { id: waitpointId }, + }); + expect(fromPrimary).not.toBeNull(); + expect(fromPrimary!.projectId).toBe(seed.project.id); + expect(fromPrimary!.environmentId).toBe(seed.environment.id); + } + ); + + // ── replay loader (tolerated read-view) ────────────────────────────────────────────────────────── + // loader findRun(no client) → owning REPLICA. Under lag it returns null. Unlike the complete-waitpoint + // route above, this drives NO write and no decision on stale data: on a null the loader tries the + // mollifier buffer then 404s and the user retries; the consequential replay ACTION re-reads the + // primary on its own. So a stale loader render is safe. We assert the store-level fact (null + replica + // hit) AND prove the run truly exists on the primary — the null is pure lag, tolerated by a display + // render. + heteroRunOpsPostgresTest( + "replay loader findRun is null under replica lag while the run exists on the primary", + async ({ prisma14, prisma17 }) => { + const { router, legacyReplica } = buildRouterWithLaggingLegacyReplica(prisma14, prisma17); + const seed = await seedEnvironmentLegacy(prisma14, "replayloader_leg"); + const runId = `run_${CUID_25}`; + const friendlyId = "run_replayloader_leg"; + await prisma14.taskRun.create({ + data: taskRunData({ + id: runId, + friendlyId, + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }), + }); + + // The EXACT loader read: findRun by friendlyId with the display select, NO client → owning replica. + const fromReplica = await router.findRun({ friendlyId }, { select: REPLAY_LOADER_SELECT }); + expect(fromReplica).toBeNull(); // loader falls to the buffer / 404 (no write is driven) + expect(legacyReplica.wasHit()).toBe(true); + + // Prove the null is lag, not absence: the run is on the primary. The loader tolerates the stale + // miss because it only renders a dialog; the replay ACTION guards its write with its own primary + // re-read, so eventual consistency of this display read is correct-by-design. + const fromPrimary = await router.findRun( + { friendlyId }, + { select: REPLAY_LOADER_SELECT }, + prisma14 + ); + expect(fromPrimary).not.toBeNull(); + expect((fromPrimary as { projectId: string }).projectId).toBe(seed.project.id); + expect((fromPrimary as { taskIdentifier: string }).taskIdentifier).toBe("my-task"); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.waitpointCompleteTokenReadAfterWrite.test.ts b/internal-packages/run-store/src/runOpsStore.waitpointCompleteTokenReadAfterWrite.test.ts new file mode 100644 index 000000000..7cdf434fa --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.waitpointCompleteTokenReadAfterWrite.test.ts @@ -0,0 +1,175 @@ +// Property: the waitpoint-complete token read tolerates replica lag. The three completion routes read +// the waitpoint by id (findWaitpoint with NO client) before completing it, which routes to the OWNING +// store's REPLICA — and a `{id}` lookup resolves the owning store via a replica probe first, so a +// just-minted, not-yet-replicated token is invisible on both. Each route re-reads via +// findWaitpointOnPrimary on a null. Pins both legs with the shared lagging-replica primitive on the +// real split topology (never mocked): findWaitpoint(id) is null under lag; findWaitpointOnPrimary(id) +// resolves it. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { RunStoreSchemaVariant } from "./types.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +const CUID_25 = "d".repeat(25); // waitpoint id shape -> LEGACY (#legacy / prisma14, full schema) + +// On the dedicated subset there are no Organization/Project/RuntimeEnvironment models (run-ops rows +// carry FK-free scalar ids), so mint synthetic owning ids; on legacy seed the real rows the FKs need. +async function seedEnvironment( + prisma: AnyClient, + schemaVariant: RunStoreSchemaVariant, + slugSuffix: string +) { + if (schemaVariant === "dedicated") { + return { + organization: { id: `org_${slugSuffix}` }, + project: { id: `proj_${slugSuffix}` }, + environment: { id: `env_${slugSuffix}` }, + }; + } + const organization = await (prisma as PrismaClient).organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await (prisma as PrismaClient).project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await (prisma as PrismaClient).runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +// Seed a standalone, still-PENDING MANUAL token waitpoint on the WRITER, exactly as minting a resume +// token does. The completion routes look it up by id right after; under replica lag that read misses. +async function seedPendingTokenWaitpoint( + store: PostgresRunStore, + params: { id: string; friendlyId: string; projectId: string; environmentId: string } +) { + await store.upsertWaitpoint({ + where: { + environmentId_idempotencyKey: { + environmentId: params.environmentId, + idempotencyKey: params.id, + }, + }, + create: { + id: params.id, + friendlyId: params.friendlyId, + type: "MANUAL", + status: "PENDING", + idempotencyKey: params.id, + userProvidedIdempotencyKey: false, + projectId: params.projectId, + environmentId: params.environmentId, + }, + update: {}, + }); +} + +describe("run-ops split — waitpoint-complete token read: replica misses a just-minted token, primary resolves it", () => { + // (a) LEGACY-resident (cuid) token: minted on the control-plane writer; its replica lags. The + // completion routes' `findWaitpoint({ where: { id } })` must NOT strand the token. + heteroRunOpsPostgresTest( + "LEGACY cuid: findWaitpoint(id) is null under replica lag; findWaitpointOnPrimary(id) resolves it", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "waitpoint", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, "legacy", "wctok_leg"); + const waitpointId = `waitpoint_${CUID_25}`; // cuid -> LEGACY + await seedPendingTokenWaitpoint(legacyStore, { + id: waitpointId, + friendlyId: "waitpoint_wctok_leg", + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + + // The exact completion-route read (no client, by id). Under lag it misses the just-minted token; + // a bare null would strand a token that exists on the primary. + const fromReplica = await router.findWaitpoint({ + where: { id: waitpointId, environmentId: seed.environment.id }, + }); + expect(fromReplica).toBeNull(); + expect(legacyReplica.wasHit()).toBe(true); + + // The read-your-writes fallback each completion route applies: a re-read on the owning primary + // resolves the token. + const fromPrimary = await router.findWaitpointOnPrimary({ + where: { id: waitpointId, environmentId: seed.environment.id }, + }); + expect(fromPrimary).not.toBeNull(); + expect(fromPrimary!.id).toBe(waitpointId); + expect(fromPrimary!.status).toBe("PENDING"); + } + ); + + // (b) NEW-resident (run-ops id) token on the dedicated subset schema: the NEW replica lags. + heteroRunOpsPostgresTest( + "NEW id: findWaitpoint(id) is null under NEW replica lag; findWaitpointOnPrimary(id) resolves it", + async ({ prisma14, prisma17 }) => { + const newReplica = laggingReplica(prisma17, [{ model: "waitpoint", mode: "missing" }]); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: newReplica.client as never, + schemaVariant: "dedicated", + }); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma17, "dedicated", "wctok_new"); + const waitpointId = `waitpoint_${generateRunOpsId()}`; // run-ops id -> NEW + await seedPendingTokenWaitpoint(newStore, { + id: waitpointId, + friendlyId: "waitpoint_wctok_new", + projectId: seed.project.id, + environmentId: seed.environment.id, + }); + + const fromReplica = await router.findWaitpoint({ + where: { id: waitpointId, environmentId: seed.environment.id }, + }); + expect(fromReplica).toBeNull(); + expect(newReplica.wasHit()).toBe(true); + + const fromPrimary = await router.findWaitpointOnPrimary({ + where: { id: waitpointId, environmentId: seed.environment.id }, + }); + expect(fromPrimary).not.toBeNull(); + expect(fromPrimary!.id).toBe(waitpointId); + expect(fromPrimary!.status).toBe("PENDING"); + } + ); +}); diff --git a/internal-packages/run-store/src/runOpsStore.waitpointDedupReadAfterWrite.test.ts b/internal-packages/run-store/src/runOpsStore.waitpointDedupReadAfterWrite.test.ts new file mode 100644 index 000000000..80247950f --- /dev/null +++ b/internal-packages/run-store/src/runOpsStore.waitpointDedupReadAfterWrite.test.ts @@ -0,0 +1,288 @@ +// Verifies the waitpoint idempotency dedup reads the OWNING store's WRITER, not its lagging replica. +// createDateTimeWaitpoint/createManualWaitpoint dedup on a retry via findWaitpoint({environmentId, +// idempotencyKey}, undefined, {coLocateWithRunId}); colocated with the owning run it must resolve +// attempt-1's just-written waitpoint on the primary so isCached fires and the run does not re-arm. +// Deterministic via heteroRunOpsPostgresTest (real split topology, never mocked) + a lagging-replica +// proxy whose `waitpoint` reads return empty and record that they ran. + +import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; +import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect } from "vitest"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RoutingRunStore } from "./runOpsStore.js"; +import type { CreateRunInput, RunStoreSchemaVariant } from "./types.js"; + +type AnyClient = PrismaClient | RunOpsPrismaClient; + +// ownerEngine (classifyResidency) routes a run-ops v1 body → NEW, everything else → LEGACY. +const CUID_25 = "c".repeat(25); // → LEGACY (#legacy / prisma14, full schema) +const NEW_RUN_ID = `run_${generateRunOpsId()}`; // valid v1 body → NEW (#new / prisma17, dedicated) + +// On the dedicated subset there are no Organization/Project/RuntimeEnvironment models (the run-ops +// rows carry FK-free scalar ids), so we mint synthetic owning ids. On legacy we seed the real rows +// the kept FKs require. +async function seedEnvironment( + prisma: AnyClient, + schemaVariant: RunStoreSchemaVariant, + slugSuffix: string +) { + if (schemaVariant === "dedicated") { + return { + organization: { id: `org_${slugSuffix}` }, + project: { id: `proj_${slugSuffix}` }, + environment: { id: `env_${slugSuffix}` }, + }; + } + const organization = await (prisma as PrismaClient).organization.create({ + data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` }, + }); + const project = await (prisma as PrismaClient).project.create({ + data: { + name: `Project ${slugSuffix}`, + slug: `project-${slugSuffix}`, + externalRef: `proj_${slugSuffix}`, + organizationId: organization.id, + }, + }); + const environment = await (prisma as PrismaClient).runtimeEnvironment.create({ + data: { + type: "DEVELOPMENT", + slug: "dev", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_dev_${slugSuffix}`, + pkApiKey: `pk_dev_${slugSuffix}`, + shortcode: `short_${slugSuffix}`, + }, + }); + return { organization, project, environment }; +} + +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + taskIdentifier: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "PENDING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: params.taskIdentifier, + payload: '{"hello":"world"}', + payloadType: "application/json", + context: { foo: "bar" }, + traceContext: { trace: "ctx" }, + traceId: "trace_1", + spanId: "span_1", + runTags: ["alpha", "beta"], + queue: "task/my-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "DEVELOPMENT", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} + +// Seed attempt-1's already-completed DATETIME waitpoint (user-provided idempotencyKey) on the WRITER, +// exactly as `createDateTimeWaitpoint`'s upsert would have written it on the first attempt. +async function seedCompletedDateTimeWaitpoint( + store: PostgresRunStore, + params: { + id: string; + friendlyId: string; + projectId: string; + environmentId: string; + idempotencyKey: string; + } +) { + const completedAfter = new Date(Date.now() - 60_000); + await store.upsertWaitpoint({ + where: { + environmentId_idempotencyKey: { + environmentId: params.environmentId, + idempotencyKey: params.idempotencyKey, + }, + }, + create: { + id: params.id, + friendlyId: params.friendlyId, + type: "DATETIME", + status: "COMPLETED", + idempotencyKey: params.idempotencyKey, + userProvidedIdempotencyKey: true, + idempotencyKeyExpiresAt: new Date(Date.now() + 60 * 60_000), + projectId: params.projectId, + environmentId: params.environmentId, + completedAfter, + completedAt: completedAfter, + }, + update: {}, + }); +} + +describe("run-ops split — waitpoint idempotency dedup reads the OWNING store's WRITER, not its lagging replica", () => { + // The dedup probe issued by createDateTimeWaitpoint/createManualWaitpoint on a RETRY: + // findWaitpoint({ where: { environmentId, idempotencyKey } }, undefined, { coLocateWithRunId }) + // must resolve attempt-1's just-written waitpoint via the OWNING store's primary, so isCached fires + // and the run does NOT re-arm/re-block. Proven for BOTH residencies + a routing guard. + + // (a) LEGACY-resident (cuid) run: attempt-1's waitpoint was committed to the control-plane writer; + // the control-plane replica lags. The colocated dedup must find it on the owning (legacy) writer. + heteroRunOpsPostgresTest( + "LEGACY cuid: colocated dedup finds attempt-1's waitpoint despite replica lag", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "waitpoint", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma14, "legacy", "wp_leg"); + const runId = `run_${CUID_25}`; // cuid → LEGACY + await legacyStore.createRun( + buildCreateRunInput({ + runId, + friendlyId: "run_wp_leg", + taskIdentifier: "my-task", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + const idempotencyKey = "wuik-legacy"; + await seedCompletedDateTimeWaitpoint(legacyStore, { + id: `waitpoint_${CUID_25}`, + friendlyId: "waitpoint_wp_leg", + projectId: seed.project.id, + environmentId: seed.environment.id, + idempotencyKey, + }); + + // The exact dedup probe: no client, colocated with the owning run. + const found = await router.findWaitpoint( + { where: { environmentId: seed.environment.id, idempotencyKey } }, + undefined, + { coLocateWithRunId: runId } + ); + + // Resolved on the owning store's writer; the replica is never consulted. + expect(found).not.toBeNull(); + expect(found!.idempotencyKey).toBe(idempotencyKey); + expect(found!.status).toBe("COMPLETED"); + expect(legacyReplica.wasHit()).toBe(false); + } + ); + + // (b) NEW-resident (ksuid) run on the dedicated subset schema: the NEW replica lags. The colocated + // dedup must resolve on the NEW writer. + heteroRunOpsPostgresTest( + "NEW ksuid: colocated dedup finds attempt-1's waitpoint despite NEW replica lag", + async ({ prisma14, prisma17 }) => { + const newReplica = laggingReplica(prisma17, [{ model: "waitpoint", mode: "missing" }]); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: newReplica.client as never, + schemaVariant: "dedicated", + }); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + const seed = await seedEnvironment(prisma17, "dedicated", "wp_new"); + const runId = NEW_RUN_ID; // v1 body → NEW + await newStore.createRun( + buildCreateRunInput({ + runId, + friendlyId: "run_wp_new", + taskIdentifier: "my-task", + organizationId: seed.organization.id, + projectId: seed.project.id, + runtimeEnvironmentId: seed.environment.id, + }) + ); + const idempotencyKey = "wuik-new"; + await seedCompletedDateTimeWaitpoint(newStore, { + id: `waitpoint_${generateRunOpsId()}`, + friendlyId: "waitpoint_wp_new", + projectId: seed.project.id, + environmentId: seed.environment.id, + idempotencyKey, + }); + + const found = await router.findWaitpoint( + { where: { environmentId: seed.environment.id, idempotencyKey } }, + undefined, + { coLocateWithRunId: runId } + ); + + expect(found).not.toBeNull(); + expect(found!.idempotencyKey).toBe(idempotencyKey); + expect(found!.status).toBe("COMPLETED"); + expect(newReplica.wasHit()).toBe(false); + } + ); + + // Guard: a NON-colocated, no-client findWaitpoint (no read-your-writes intent) still routes to the + // replica — colocated dedup must not turn every waitpoint read into a primary read (that would defeat + // replica offload). + heteroRunOpsPostgresTest( + "plain non-colocated reads still route to the replica (no read-your-writes escalation)", + async ({ prisma14, prisma17 }) => { + const legacyReplica = laggingReplica(prisma14, [{ model: "waitpoint", mode: "missing" }]); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: legacyReplica.client, + schemaVariant: "legacy", + }); + const newReplica = laggingReplica(prisma17, [{ model: "waitpoint", mode: "missing" }]); + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: newReplica.client as never, + schemaVariant: "dedicated", + }); + const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); + + // No id, no colocation → the cross-DB scan probes each store's replica. + const found = await router.findWaitpoint({ + where: { environmentId: "env_none", idempotencyKey: "does-not-exist" }, + }); + + expect(found).toBeNull(); + // Both legs must be replica-probed — an `||` would hide a primary-routing regression on one leg. + expect(legacyReplica.wasHit()).toBe(true); + expect(newReplica.wasHit()).toBe(true); + } + ); +}); From d5f1696a979b039ef3e036588ab16974dcfd60da Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 20 Jul 2026 12:16:03 +0100 Subject: [PATCH 004/300] ci(publish): unify image scanning across published images (#4306) ## What Run the shared Trivy image scan on every published image through a single reusable workflow. ## Changes - Generalise the image-scan workflow (`trivy-image-webapp.yml` -> `trivy-image.yml`) - it was already parameterised by `image-ref`; only the run-summary label was image-specific. - `publish-worker-v4.yml`: expose `version` + `image_repo` as workflow outputs (single-entry matrix, so unambiguous). - `publish.yml`: run the shared scan from each publish job (`scan-webapp`, `scan-supervisor`). Report-only (writes a table to the run summary), OS packages only (`vuln-type: os` - library deps stay with Dependabot), never blocks the publish. --- .github/workflows/publish-worker-v4.yml | 13 +++++++++++++ .github/workflows/publish.yml | 11 ++++++++++- .../{trivy-image-webapp.yml => trivy-image.yml} | 14 +++++++------- 3 files changed, 30 insertions(+), 8 deletions(-) rename .github/workflows/{trivy-image-webapp.yml => trivy-image.yml} (83%) diff --git a/.github/workflows/publish-worker-v4.yml b/.github/workflows/publish-worker-v4.yml index ab10e7166..2dba27013 100644 --- a/.github/workflows/publish-worker-v4.yml +++ b/.github/workflows/publish-worker-v4.yml @@ -13,6 +13,13 @@ on: type: string required: false default: "" + outputs: + version: + description: The published image tag + value: ${{ jobs.build.outputs.version }} + image_repo: + description: The image repository the build was published to (without tag) + value: ${{ jobs.build.outputs.image_repo }} push: tags: - "re2-test-*" @@ -38,6 +45,11 @@ jobs: matrix: package: [supervisor] runs-on: warp-ubuntu-latest-x64-2x + # Single-entry matrix, so these job outputs are unambiguous (consumed by the + # scan-supervisor job in publish.yml). + outputs: + version: ${{ steps.get_tag.outputs.tag }} + image_repo: ${{ steps.set_tags.outputs.image_repo }} env: DOCKER_BUILDKIT: "1" steps: @@ -81,6 +93,7 @@ jobs: fi echo "image_tags=${image_tags}" >> "$GITHUB_OUTPUT" + echo "image_repo=${ref_without_tag}" >> "$GITHUB_OUTPUT" env: IMAGE_REGISTRY: ${{ inputs.image_registry || vars.IMAGE_REGISTRY || format('ghcr.io/{0}', github.repository_owner) }} STEPS_GET_REPOSITORY_OUTPUTS_REPO: ${{ steps.get_repository.outputs.repo }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a0f32b026..1b9e55c3b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -97,10 +97,19 @@ jobs: permissions: contents: read packages: read # pull the just-published image from GHCR - uses: ./.github/workflows/trivy-image-webapp.yml + uses: ./.github/workflows/trivy-image.yml with: image-ref: ${{ needs.publish-webapp.outputs.image_repo }}:${{ needs.publish-webapp.outputs.version }} + scan-supervisor: + needs: [publish-worker-v4] + permissions: + contents: read + packages: read # pull the just-published image from GHCR + uses: ./.github/workflows/trivy-image.yml + with: + image-ref: ${{ needs.publish-worker-v4.outputs.image_repo }}:${{ needs.publish-worker-v4.outputs.version }} + # Announce the freshly published mutable `main` webapp image to subscriber # repos via repository_dispatch, handing them a digest-pinned ref to build or # deploy from. The repo, ref prefix, and dispatch target all default to the diff --git a/.github/workflows/trivy-image-webapp.yml b/.github/workflows/trivy-image.yml similarity index 83% rename from .github/workflows/trivy-image-webapp.yml rename to .github/workflows/trivy-image.yml index 2307b5a6b..4c85823f0 100644 --- a/.github/workflows/trivy-image-webapp.yml +++ b/.github/workflows/trivy-image.yml @@ -1,7 +1,7 @@ -name: Trivy Image Scan (webapp) +name: Trivy Image Scan -# OS-level CVE scan of a published webapp image. Called by the publish pipeline -# (publish.yml) to scan each build right after it's pushed to GHCR — so every +# OS-level CVE scan of a published image. Called by the publish pipeline +# (publish.yml) to scan each image right after it's pushed to GHCR — so every # main build and every release is scanned, not rebuilt. Also runnable ad-hoc # via workflow_dispatch against any image ref. # @@ -27,7 +27,7 @@ on: permissions: {} concurrency: - group: trivy-image-webapp-${{ inputs.image-ref }} + group: trivy-image-${{ inputs.image-ref }} cancel-in-progress: true jobs: @@ -59,7 +59,7 @@ jobs: ignore-unfixed: true severity: HIGH,CRITICAL format: table - output: trivy-image-webapp.txt + output: trivy-image.txt - name: Job summary if: always() @@ -67,9 +67,9 @@ jobs: IMAGE_REF: ${{ inputs.image-ref }} run: | { - echo "## Trivy Image Scan (webapp) — \`${IMAGE_REF}\`" + echo "## Trivy Image Scan — \`${IMAGE_REF}\`" echo '```' # GitHub step summary is capped at 1 MiB; truncate large reports. - head -c 900000 trivy-image-webapp.txt 2>/dev/null || echo "(no report produced)" + head -c 900000 trivy-image.txt 2>/dev/null || echo "(no report produced)" echo '```' } >> "$GITHUB_STEP_SUMMARY" From 1cbe25bd1d426c8974e4d26ff6c0f0a9e99c7868 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:01:37 +0100 Subject: [PATCH 005/300] chore: release v4.5.5 (#4267) ## Summary 5 improvements, 5 bug fixes. ## Improvements - Add experimental Node.js 24 and 26 task runtimes. Set `runtime` to `experimental-node-24` or `experimental-node-26` in `trigger.config.ts`. ([#4085](https://github.com/triggerdotdev/trigger.dev/pull/4085)) - Add `defaultRegion` to the project GET and list API responses; null when unset. ([#4146](https://github.com/triggerdotdev/trigger.dev/pull/4146)) ## Server changes These changes affect the self-hosted Docker image and Trigger.dev Cloud: - Transient internal sync failures are now retried quietly instead of surfacing as errors. ([#4270](https://github.com/triggerdotdev/trigger.dev/pull/4270)) - Optionally route ClickHouse read traffic to a read replica while writes stay on the primary. Set `CLICKHOUSE_READER_URL` to move all reads, or target the busiest paths with `RUNS_LIST_CLICKHOUSE_URL` (runs list) and `EVENTS_READER_CLICKHOUSE_URL` (traces, spans, logs). All optional; unset keeps current behavior. ([#4081](https://github.com/triggerdotdev/trigger.dev/pull/4081)) - Remove the deprecated realtime stream write endpoint used by retired v3 task clients. ([#4250](https://github.com/triggerdotdev/trigger.dev/pull/4250)) - Fix batchTrigger requests that set a per-item idempotency key failing with an error instead of creating and deduplicating the runs ([#4271](https://github.com/triggerdotdev/trigger.dev/pull/4271)) - Speed up idempotency checks on `batchTrigger` calls that use idempotency keys. Large batches against a task with a big run history no longer degrade to multi-second lookups. ([#4255](https://github.com/triggerdotdev/trigger.dev/pull/4255)) - The "Preview branches" usage on the Limits page now counts only preview branches. ([#4283](https://github.com/triggerdotdev/trigger.dev/pull/4283)) - Avoid opening a redundant database connection pool when the legacy and primary databases are the same server, preventing connection usage from doubling. ([#4253](https://github.com/triggerdotdev/trigger.dev/pull/4253)) - Fix pages occasionally loading unstyled or failing to load during a deploy. The dashboard now reloads automatically to recover. ([#4282](https://github.com/triggerdotdev/trigger.dev/pull/4282))
Raw changeset output # Releases ## @trigger.dev/build@4.5.5 ### Patch Changes - Updated dependencies: - `@trigger.dev/core@4.5.5` ## trigger.dev@4.5.5 ### Patch Changes - Add experimental Node.js 24 and 26 task runtimes. Set `runtime` to `experimental-node-24` or `experimental-node-26` in `trigger.config.ts`. ([#4085](https://github.com/triggerdotdev/trigger.dev/pull/4085)) - Updated dependencies: - `@trigger.dev/core@4.5.5` - `@trigger.dev/build@4.5.5` - `@trigger.dev/schema-to-json@4.5.5` ## @trigger.dev/core@4.5.5 ### Patch Changes - Add experimental Node.js 24 and 26 task runtimes. Set `runtime` to `experimental-node-24` or `experimental-node-26` in `trigger.config.ts`. ([#4085](https://github.com/triggerdotdev/trigger.dev/pull/4085)) - Add `defaultRegion` to the project GET and list API responses; null when unset. ([#4146](https://github.com/triggerdotdev/trigger.dev/pull/4146)) ## @trigger.dev/python@4.5.5 ### Patch Changes - Updated dependencies: - `@trigger.dev/core@4.5.5` - `@trigger.dev/build@4.5.5` - `@trigger.dev/sdk@4.5.5` ## @trigger.dev/react-hooks@4.5.5 ### Patch Changes - Updated dependencies: - `@trigger.dev/core@4.5.5` ## @trigger.dev/redis-worker@4.5.5 ### Patch Changes - Updated dependencies: - `@trigger.dev/core@4.5.5` ## @trigger.dev/rsc@4.5.5 ### Patch Changes - Updated dependencies: - `@trigger.dev/core@4.5.5` ## @trigger.dev/schema-to-json@4.5.5 ### Patch Changes - Updated dependencies: - `@trigger.dev/core@4.5.5` ## @trigger.dev/sdk@4.5.5 ### Patch Changes - Updated dependencies: - `@trigger.dev/core@4.5.5`
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .changeset/experimental-node-runtimes.md | 6 ----- .changeset/project-default-region-response.md | 5 ---- .server-changes/attio-sync-transient-retry.md | 6 ----- .server-changes/clickhouse-read-replica.md | 6 ----- .../fix-batch-idempotency-per-item-keys.md | 6 ----- .../fix-batch-idempotency-point-lookups.md | 6 ----- .../fix-preview-branch-limit-count.md | 6 ----- ...emove-deprecated-realtime-stream-action.md | 6 ----- .server-changes/run-ops-legacy-shared-pool.md | 6 ----- .../stale-deploy-asset-recovery.md | 6 ----- hosting/k8s/helm/Chart.yaml | 4 ++-- packages/build/CHANGELOG.md | 7 ++++++ packages/build/package.json | 4 ++-- packages/cli-v3/CHANGELOG.md | 10 ++++++++ packages/cli-v3/package.json | 8 +++---- packages/core/CHANGELOG.md | 7 ++++++ packages/core/package.json | 2 +- packages/python/CHANGELOG.md | 9 +++++++ packages/python/package.json | 12 +++++----- packages/react-hooks/CHANGELOG.md | 7 ++++++ packages/react-hooks/package.json | 4 ++-- packages/redis-worker/CHANGELOG.md | 7 ++++++ packages/redis-worker/package.json | 4 ++-- packages/rsc/CHANGELOG.md | 7 ++++++ packages/rsc/package.json | 6 ++--- packages/schema-to-json/CHANGELOG.md | 7 ++++++ packages/schema-to-json/package.json | 2 +- packages/trigger-sdk/CHANGELOG.md | 7 ++++++ packages/trigger-sdk/package.json | 4 ++-- pnpm-lock.yaml | 24 +++++++++---------- 30 files changed, 105 insertions(+), 96 deletions(-) delete mode 100644 .changeset/experimental-node-runtimes.md delete mode 100644 .changeset/project-default-region-response.md delete mode 100644 .server-changes/attio-sync-transient-retry.md delete mode 100644 .server-changes/clickhouse-read-replica.md delete mode 100644 .server-changes/fix-batch-idempotency-per-item-keys.md delete mode 100644 .server-changes/fix-batch-idempotency-point-lookups.md delete mode 100644 .server-changes/fix-preview-branch-limit-count.md delete mode 100644 .server-changes/remove-deprecated-realtime-stream-action.md delete mode 100644 .server-changes/run-ops-legacy-shared-pool.md delete mode 100644 .server-changes/stale-deploy-asset-recovery.md diff --git a/.changeset/experimental-node-runtimes.md b/.changeset/experimental-node-runtimes.md deleted file mode 100644 index 39ec16321..000000000 --- a/.changeset/experimental-node-runtimes.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -"@trigger.dev/core": patch -"trigger.dev": patch ---- - -Add experimental Node.js 24 and 26 task runtimes. Set `runtime` to `experimental-node-24` or `experimental-node-26` in `trigger.config.ts`. diff --git a/.changeset/project-default-region-response.md b/.changeset/project-default-region-response.md deleted file mode 100644 index 86d1856ac..000000000 --- a/.changeset/project-default-region-response.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@trigger.dev/core": patch ---- - -Add `defaultRegion` to the project GET and list API responses; null when unset. diff --git a/.server-changes/attio-sync-transient-retry.md b/.server-changes/attio-sync-transient-retry.md deleted file mode 100644 index 9c54416a4..000000000 --- a/.server-changes/attio-sync-transient-retry.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: improvement ---- - -Transient internal sync failures are now retried quietly instead of surfacing as errors. diff --git a/.server-changes/clickhouse-read-replica.md b/.server-changes/clickhouse-read-replica.md deleted file mode 100644 index 334b10d3c..000000000 --- a/.server-changes/clickhouse-read-replica.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: improvement ---- - -Optionally route ClickHouse read traffic to a read replica while writes stay on the primary. Set `CLICKHOUSE_READER_URL` to move all reads, or target the busiest paths with `RUNS_LIST_CLICKHOUSE_URL` (runs list) and `EVENTS_READER_CLICKHOUSE_URL` (traces, spans, logs). All optional; unset keeps current behavior. diff --git a/.server-changes/fix-batch-idempotency-per-item-keys.md b/.server-changes/fix-batch-idempotency-per-item-keys.md deleted file mode 100644 index 9e84d474c..000000000 --- a/.server-changes/fix-batch-idempotency-per-item-keys.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: fix ---- - -Fix batchTrigger requests that set a per-item idempotency key failing with an error instead of creating and deduplicating the runs diff --git a/.server-changes/fix-batch-idempotency-point-lookups.md b/.server-changes/fix-batch-idempotency-point-lookups.md deleted file mode 100644 index 1c3a4aab7..000000000 --- a/.server-changes/fix-batch-idempotency-point-lookups.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: fix ---- - -Speed up idempotency checks on `batchTrigger` calls that use idempotency keys. Large batches against a task with a big run history no longer degrade to multi-second lookups. diff --git a/.server-changes/fix-preview-branch-limit-count.md b/.server-changes/fix-preview-branch-limit-count.md deleted file mode 100644 index d5f5b26d7..000000000 --- a/.server-changes/fix-preview-branch-limit-count.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: fix ---- - -The "Preview branches" usage on the Limits page now counts only preview branches. diff --git a/.server-changes/remove-deprecated-realtime-stream-action.md b/.server-changes/remove-deprecated-realtime-stream-action.md deleted file mode 100644 index 3ad4f19d7..000000000 --- a/.server-changes/remove-deprecated-realtime-stream-action.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: improvement ---- - -Remove the deprecated realtime stream write endpoint used by retired v3 task clients. diff --git a/.server-changes/run-ops-legacy-shared-pool.md b/.server-changes/run-ops-legacy-shared-pool.md deleted file mode 100644 index dca8d5192..000000000 --- a/.server-changes/run-ops-legacy-shared-pool.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: fix ---- - -Avoid opening a redundant database connection pool when the legacy and primary databases are the same server, preventing connection usage from doubling. diff --git a/.server-changes/stale-deploy-asset-recovery.md b/.server-changes/stale-deploy-asset-recovery.md deleted file mode 100644 index ae34351fb..000000000 --- a/.server-changes/stale-deploy-asset-recovery.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: fix ---- - -Fix pages occasionally loading unstyled or failing to load during a deploy. The dashboard now reloads automatically to recover. diff --git a/hosting/k8s/helm/Chart.yaml b/hosting/k8s/helm/Chart.yaml index 3ef6eb2a8..ed45726ab 100644 --- a/hosting/k8s/helm/Chart.yaml +++ b/hosting/k8s/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: trigger description: The official Trigger.dev Helm chart type: application -version: 4.5.4 -appVersion: v4.5.4 +version: 4.5.5 +appVersion: v4.5.5 home: https://trigger.dev sources: - https://github.com/triggerdotdev/trigger.dev diff --git a/packages/build/CHANGELOG.md b/packages/build/CHANGELOG.md index 5fb02a3d5..483ff9a80 100644 --- a/packages/build/CHANGELOG.md +++ b/packages/build/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/build +## 4.5.5 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.5` + ## 4.5.4 ### Patch Changes diff --git a/packages/build/package.json b/packages/build/package.json index 34566788b..d2e21643b 100644 --- a/packages/build/package.json +++ b/packages/build/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/build", - "version": "4.5.4", + "version": "4.5.5", "description": "trigger.dev build extensions", "license": "MIT", "publishConfig": { @@ -79,7 +79,7 @@ }, "dependencies": { "@prisma/config": "^6.10.0", - "@trigger.dev/core": "workspace:4.5.4", + "@trigger.dev/core": "workspace:4.5.5", "mlly": "^1.7.1", "pkg-types": "^1.1.3", "resolve": "^1.22.8", diff --git a/packages/cli-v3/CHANGELOG.md b/packages/cli-v3/CHANGELOG.md index 9e68372a0..f5ca164b1 100644 --- a/packages/cli-v3/CHANGELOG.md +++ b/packages/cli-v3/CHANGELOG.md @@ -1,5 +1,15 @@ # trigger.dev +## 4.5.5 + +### Patch Changes + +- Add experimental Node.js 24 and 26 task runtimes. Set `runtime` to `experimental-node-24` or `experimental-node-26` in `trigger.config.ts`. ([#4085](https://github.com/triggerdotdev/trigger.dev/pull/4085)) +- Updated dependencies: + - `@trigger.dev/core@4.5.5` + - `@trigger.dev/build@4.5.5` + - `@trigger.dev/schema-to-json@4.5.5` + ## 4.5.4 ### Patch Changes diff --git a/packages/cli-v3/package.json b/packages/cli-v3/package.json index 630b2af85..1eb1fe2f1 100644 --- a/packages/cli-v3/package.json +++ b/packages/cli-v3/package.json @@ -1,6 +1,6 @@ { "name": "trigger.dev", - "version": "4.5.4", + "version": "4.5.5", "description": "A Command-Line Interface for Trigger.dev projects", "type": "module", "license": "MIT", @@ -96,9 +96,9 @@ "@opentelemetry/sdk-trace-node": "2.7.1", "@opentelemetry/semantic-conventions": "1.41.1", "@s2-dev/streamstore": "^0.22.10", - "@trigger.dev/build": "workspace:4.5.4", - "@trigger.dev/core": "workspace:4.5.4", - "@trigger.dev/schema-to-json": "workspace:4.5.4", + "@trigger.dev/build": "workspace:4.5.5", + "@trigger.dev/core": "workspace:4.5.5", + "@trigger.dev/schema-to-json": "workspace:4.5.5", "ansi-escapes": "^7.0.0", "braces": "^3.0.3", "c12": "^1.11.1", diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index cf3ddf475..1c54f5cac 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -1,5 +1,12 @@ # internal-platform +## 4.5.5 + +### Patch Changes + +- Add experimental Node.js 24 and 26 task runtimes. Set `runtime` to `experimental-node-24` or `experimental-node-26` in `trigger.config.ts`. ([#4085](https://github.com/triggerdotdev/trigger.dev/pull/4085)) +- Add `defaultRegion` to the project GET and list API responses; null when unset. ([#4146](https://github.com/triggerdotdev/trigger.dev/pull/4146)) + ## 4.5.4 ### Patch Changes diff --git a/packages/core/package.json b/packages/core/package.json index f2b1e62a3..93758fa95 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/core", - "version": "4.5.4", + "version": "4.5.5", "description": "Core code used across the Trigger.dev SDK and platform", "license": "MIT", "publishConfig": { diff --git a/packages/python/CHANGELOG.md b/packages/python/CHANGELOG.md index ba66f42d4..febf81e99 100644 --- a/packages/python/CHANGELOG.md +++ b/packages/python/CHANGELOG.md @@ -1,5 +1,14 @@ # @trigger.dev/python +## 4.5.5 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.5` + - `@trigger.dev/build@4.5.5` + - `@trigger.dev/sdk@4.5.5` + ## 4.5.4 ### Patch Changes diff --git a/packages/python/package.json b/packages/python/package.json index 9d7ab2df2..d5ea3971b 100644 --- a/packages/python/package.json +++ b/packages/python/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/python", - "version": "4.5.4", + "version": "4.5.5", "description": "Python runtime and build extension for Trigger.dev", "license": "MIT", "publishConfig": { @@ -45,7 +45,7 @@ "check-exports": "attw --pack ." }, "dependencies": { - "@trigger.dev/core": "workspace:4.5.4", + "@trigger.dev/core": "workspace:4.5.5", "tinyexec": "^0.3.2" }, "devDependencies": { @@ -56,12 +56,12 @@ "tsx": "4.17.0", "esbuild": "^0.23.0", "@arethetypeswrong/cli": "^0.15.4", - "@trigger.dev/build": "workspace:4.5.4", - "@trigger.dev/sdk": "workspace:4.5.4" + "@trigger.dev/build": "workspace:4.5.5", + "@trigger.dev/sdk": "workspace:4.5.5" }, "peerDependencies": { - "@trigger.dev/sdk": "workspace:^4.5.4", - "@trigger.dev/build": "workspace:^4.5.4" + "@trigger.dev/sdk": "workspace:^4.5.5", + "@trigger.dev/build": "workspace:^4.5.5" }, "engines": { "node": ">=18.20.0" diff --git a/packages/react-hooks/CHANGELOG.md b/packages/react-hooks/CHANGELOG.md index bbb1551d7..c698067ab 100644 --- a/packages/react-hooks/CHANGELOG.md +++ b/packages/react-hooks/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/react-hooks +## 4.5.5 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.5` + ## 4.5.4 ### Patch Changes diff --git a/packages/react-hooks/package.json b/packages/react-hooks/package.json index 013525b54..347f0e2c6 100644 --- a/packages/react-hooks/package.json +++ b/packages/react-hooks/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/react-hooks", - "version": "4.5.4", + "version": "4.5.5", "description": "trigger.dev react hooks", "license": "MIT", "publishConfig": { @@ -37,7 +37,7 @@ "check-exports": "attw --pack ." }, "dependencies": { - "@trigger.dev/core": "workspace:^4.5.4", + "@trigger.dev/core": "workspace:^4.5.5", "swr": "^2.2.5" }, "devDependencies": { diff --git a/packages/redis-worker/CHANGELOG.md b/packages/redis-worker/CHANGELOG.md index 0ef288a11..471e96c31 100644 --- a/packages/redis-worker/CHANGELOG.md +++ b/packages/redis-worker/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/redis-worker +## 4.5.5 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.5` + ## 4.5.4 ### Patch Changes diff --git a/packages/redis-worker/package.json b/packages/redis-worker/package.json index bb4541c1b..43fa77072 100644 --- a/packages/redis-worker/package.json +++ b/packages/redis-worker/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/redis-worker", - "version": "4.5.4", + "version": "4.5.5", "description": "Redis worker for trigger.dev", "license": "MIT", "publishConfig": { @@ -23,7 +23,7 @@ "test": "vitest --sequence.concurrent=false --no-file-parallelism" }, "dependencies": { - "@trigger.dev/core": "workspace:4.5.4", + "@trigger.dev/core": "workspace:4.5.5", "lodash.omit": "^4.5.0", "nanoid": "^5.0.7", "p-limit": "^6.2.0", diff --git a/packages/rsc/CHANGELOG.md b/packages/rsc/CHANGELOG.md index 29842b09e..acd0e1af0 100644 --- a/packages/rsc/CHANGELOG.md +++ b/packages/rsc/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/rsc +## 4.5.5 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.5` + ## 4.5.4 ### Patch Changes diff --git a/packages/rsc/package.json b/packages/rsc/package.json index 329c27ef6..7a5d3c4d2 100644 --- a/packages/rsc/package.json +++ b/packages/rsc/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/rsc", - "version": "4.5.4", + "version": "4.5.5", "description": "trigger.dev rsc", "license": "MIT", "publishConfig": { @@ -37,14 +37,14 @@ "check-exports": "attw --pack ." }, "dependencies": { - "@trigger.dev/core": "workspace:^4.5.4", + "@trigger.dev/core": "workspace:^4.5.5", "mlly": "^1.7.1", "react": "19.0.0-rc.1", "react-dom": "19.0.0-rc.1" }, "devDependencies": { "@arethetypeswrong/cli": "^0.15.4", - "@trigger.dev/build": "workspace:^4.5.4", + "@trigger.dev/build": "workspace:^4.5.5", "@types/node": "^24.13.3", "@types/react": "*", "@types/react-dom": "*", diff --git a/packages/schema-to-json/CHANGELOG.md b/packages/schema-to-json/CHANGELOG.md index 4ecfa98c8..03b024c52 100644 --- a/packages/schema-to-json/CHANGELOG.md +++ b/packages/schema-to-json/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/schema-to-json +## 4.5.5 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.5` + ## 4.5.4 ### Patch Changes diff --git a/packages/schema-to-json/package.json b/packages/schema-to-json/package.json index e11d598a9..d9daf0ca0 100644 --- a/packages/schema-to-json/package.json +++ b/packages/schema-to-json/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/schema-to-json", - "version": "4.5.4", + "version": "4.5.5", "description": "Convert various schema validation libraries to JSON Schema", "license": "MIT", "publishConfig": { diff --git a/packages/trigger-sdk/CHANGELOG.md b/packages/trigger-sdk/CHANGELOG.md index 8e2da32fe..4697678b2 100644 --- a/packages/trigger-sdk/CHANGELOG.md +++ b/packages/trigger-sdk/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/sdk +## 4.5.5 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.5` + ## 4.5.4 ### Patch Changes diff --git a/packages/trigger-sdk/package.json b/packages/trigger-sdk/package.json index 3a28b0b89..daf6e0633 100644 --- a/packages/trigger-sdk/package.json +++ b/packages/trigger-sdk/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/sdk", - "version": "4.5.4", + "version": "4.5.5", "description": "trigger.dev Node.JS SDK", "license": "MIT", "publishConfig": { @@ -77,7 +77,7 @@ "dependencies": { "@opentelemetry/api": "1.9.1", "@opentelemetry/semantic-conventions": "1.41.1", - "@trigger.dev/core": "workspace:4.5.4", + "@trigger.dev/core": "workspace:4.5.5", "chalk": "^5.2.0", "cronstrue": "^2.21.0", "debug": "^4.3.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b8dc0f272..b0e328c76 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1393,7 +1393,7 @@ importers: specifier: ^6.10.0 version: 6.19.0(magicast@0.3.5) '@trigger.dev/core': - specifier: workspace:4.5.4 + specifier: workspace:4.5.5 version: link:../core mlly: specifier: ^1.7.1 @@ -1469,13 +1469,13 @@ importers: specifier: ^0.22.10 version: 0.22.10(supports-color@10.0.0) '@trigger.dev/build': - specifier: workspace:4.5.4 + specifier: workspace:4.5.5 version: link:../build '@trigger.dev/core': - specifier: workspace:4.5.4 + specifier: workspace:4.5.5 version: link:../core '@trigger.dev/schema-to-json': - specifier: workspace:4.5.4 + specifier: workspace:4.5.5 version: link:../schema-to-json ansi-escapes: specifier: ^7.0.0 @@ -1862,7 +1862,7 @@ importers: packages/python: dependencies: '@trigger.dev/core': - specifier: workspace:4.5.4 + specifier: workspace:4.5.5 version: link:../core tinyexec: specifier: ^0.3.2 @@ -1872,10 +1872,10 @@ importers: specifier: ^0.15.4 version: 0.15.4 '@trigger.dev/build': - specifier: workspace:4.5.4 + specifier: workspace:4.5.5 version: link:../build '@trigger.dev/sdk': - specifier: workspace:4.5.4 + specifier: workspace:4.5.5 version: link:../trigger-sdk '@types/node': specifier: 24.13.3 @@ -1899,7 +1899,7 @@ importers: packages/react-hooks: dependencies: '@trigger.dev/core': - specifier: workspace:^4.5.4 + specifier: workspace:^4.5.5 version: link:../core react: specifier: 18.3.1 @@ -1933,7 +1933,7 @@ importers: packages/redis-worker: dependencies: '@trigger.dev/core': - specifier: workspace:4.5.4 + specifier: workspace:4.5.5 version: link:../core cron-parser: specifier: ^4.9.0 @@ -1982,7 +1982,7 @@ importers: packages/rsc: dependencies: '@trigger.dev/core': - specifier: workspace:^4.5.4 + specifier: workspace:^4.5.5 version: link:../core mlly: specifier: ^1.7.1 @@ -1998,7 +1998,7 @@ importers: specifier: ^0.15.4 version: 0.15.4 '@trigger.dev/build': - specifier: workspace:^4.5.4 + specifier: workspace:^4.5.5 version: link:../build '@types/node': specifier: 24.13.3 @@ -2077,7 +2077,7 @@ importers: specifier: 1.41.1 version: 1.41.1 '@trigger.dev/core': - specifier: workspace:4.5.4 + specifier: workspace:4.5.5 version: link:../core chalk: specifier: ^5.2.0 From cc748422d8188833948035281cc50ef4f7ab77e2 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Mon, 20 Jul 2026 19:17:44 +0100 Subject: [PATCH 006/300] test(webapp): replace slow metadata replica guard with unit test (#4312) --- .../metadataRouteReplicaLag.guard.test.ts | 297 +++++------------- 1 file changed, 77 insertions(+), 220 deletions(-) diff --git a/apps/webapp/test/metadataRouteReplicaLag.guard.test.ts b/apps/webapp/test/metadataRouteReplicaLag.guard.test.ts index 9a27055c3..6d53e884d 100644 --- a/apps/webapp/test/metadataRouteReplicaLag.guard.test.ts +++ b/apps/webapp/test/metadataRouteReplicaLag.guard.test.ts @@ -1,38 +1,17 @@ -// Property: the metadata GET loader resolves a live run on a replica+buffer double-miss via its primary -// fallback, returning 200 + the run's metadata. Drives the REAL exported `loader` end-to-end: -// 1. runStore.findRun(..., $replica) → REPLICA (lagging) → null -// 2. findRunByIdWithMollifierFallback(...) → buffer MISS (null) -// 3. runStore.findRunOnPrimary(...) → owning PRIMARY → HIT -// Store is REAL: a split RoutingRunStore over two testcontainer Postgres DBs, the owning -// (legacy/control-plane) REPLICA frozen behind the shared laggingReplica. Only the loader's webapp -// singletons (auth, $replica brand, mollifier buffer, route builder, logging) are stubbed. +import { beforeEach, describe, expect, it, vi } from "vitest"; -import { describe, expect, vi } from "vitest"; -import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers"; -import type { PrismaClient } from "@trigger.dev/database"; -import type { RunOpsPrismaClient } from "@internal/run-ops-database"; -import { PostgresRunStore, RoutingRunStore } from "@internal/run-store"; -import type { CreateRunInput } from "@internal/run-store"; +const { mocks } = vi.hoisted(() => ({ + mocks: { + replica: {}, + environment: { id: "env_meta", organizationId: "org_meta" }, + authenticateApiRequest: vi.fn(), + findRun: vi.fn(), + findRunOnPrimary: vi.fn(), + findRunByIdWithMollifierFallback: vi.fn(), + }, +})); -// vi.mock factories are hoisted above the imports, so anything they reference must be created inside -// vi.hoisted. `holder.store` is filled in per-test with the REAL split router; the mocked -// `~/v3/runStore.server` export is a stable Proxy that forwards every property access to it. The -// branded `$replica` marker uses the global-registry symbol the run-store brands replicas with -// (readReplicaClient.ts) — a branded client makes the routing store keep the read on the owning -// REPLICA (no primary escalation), which under lag is exactly the miss the primary fallback recovers. -const { holder } = vi.hoisted(() => { - const REPLICA_BRAND = Symbol.for("trigger.dev/run-store/read-replica"); - return { - holder: { - store: undefined as unknown, - brandedReplica: { [REPLICA_BRAND]: true } as object, - environment: undefined as unknown, - bufferResult: null as unknown, - }, - }; -}); - -vi.mock("~/db.server", () => ({ prisma: {}, $replica: holder.brandedReplica })); +vi.mock("~/db.server", () => ({ prisma: {}, $replica: mocks.replica })); vi.mock("~/env.server", () => ({ env: { TASK_RUN_METADATA_MAXIMUM_SIZE: 256 * 1024, @@ -41,42 +20,30 @@ vi.mock("~/env.server", () => ({ TRIGGER_MOLLIFIER_METADATA_BACKOFF_STEP_MS: 10, }, })); -// The route module builds its action at import time via createActionApiRoute; stub the builder so the -// heavy platform/auth middleware graph never evaluates. We drive the exported `loader` directly. vi.mock("~/services/routeBuilders/apiBuilder.server", () => ({ createActionApiRoute: () => ({ action: vi.fn() }), })); vi.mock("~/services/apiAuth.server", () => ({ - authenticateApiRequest: vi.fn(async () => ({ environment: holder.environment })), + authenticateApiRequest: mocks.authenticateApiRequest, })); -// Inject the REAL split router. A stable Proxy keeps the named import binding constant while -// forwarding every method to the per-test router set in holder.store. vi.mock("~/v3/runStore.server", () => ({ - runStore: new Proxy( - {}, - { - get(_target, prop) { - const store = holder.store as Record; - if (!store) throw new Error("test bug: holder.store not initialised before loader ran"); - const value = store[prop]; - return typeof value === "function" - ? (value as (...a: unknown[]) => unknown).bind(store) - : value; - }, - } - ), + runStore: { + findRun: mocks.findRun, + findRunOnPrimary: mocks.findRunOnPrimary, + }, })); -// Buffer fallback returns whatever the test set (null = buffer miss, the double-miss scenario). vi.mock("~/v3/mollifier/readFallback.server", () => ({ - findRunByIdWithMollifierFallback: vi.fn(async () => holder.bufferResult), + findRunByIdWithMollifierFallback: mocks.findRunByIdWithMollifierFallback, })); -// Action-only leaf; stub to keep the import graph light. vi.mock("~/v3/mollifier/applyMetadataMutation.server", () => ({ applyMetadataMutationToBufferedRun: vi.fn(), })); vi.mock("~/services/metadata/updateMetadataInstance.server", () => ({ updateMetadataService: { call: vi.fn(async () => undefined) }, })); +vi.mock("~/services/realtime/runChangeNotifierInstance.server", () => ({ + publishChangeRecord: vi.fn(), +})); vi.mock("~/v3/services/common.server", () => ({ ServiceValidationError: class extends Error {}, })); @@ -86,176 +53,66 @@ vi.mock("~/services/logger.server", () => ({ import { loader } from "~/routes/api.v1.runs.$runId.metadata"; -// A cuid (25 chars after `run_`) classifies LEGACY, so both the create and the friendlyId-keyed reads -// route to the legacy (control-plane) store — the store that owns this run. -const CUID_25 = "c".repeat(25); +const friendlyId = "run_meta_live"; +const runWhere = { friendlyId, runtimeEnvironmentId: mocks.environment.id }; +const metadataSelect = { select: { metadata: true, metadataType: true } }; -async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) { - const organization = await prisma.organization.create({ - data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, - }); - const project = await prisma.project.create({ - data: { - name: `Project ${suffix}`, - slug: `project-${suffix}`, - externalRef: `proj_${suffix}`, - organizationId: organization.id, - }, - }); - const environment = await prisma.runtimeEnvironment.create({ - data: { - type: "DEVELOPMENT", - slug: "dev", - projectId: project.id, - organizationId: organization.id, - apiKey: `tr_dev_${suffix}`, - pkApiKey: `pk_dev_${suffix}`, - shortcode: `short_${suffix}`, - }, - }); - return { organization, project, environment }; -} - -function buildCreateRunInput(params: { - runId: string; - friendlyId: string; - organizationId: string; - projectId: string; - runtimeEnvironmentId: string; - metadata?: string; - metadataType?: string; -}): CreateRunInput { - return { - data: { - id: params.runId, - engine: "V2", - status: "PENDING", - friendlyId: params.friendlyId, - runtimeEnvironmentId: params.runtimeEnvironmentId, - environmentType: "DEVELOPMENT", - organizationId: params.organizationId, - projectId: params.projectId, - taskIdentifier: "my-task", - payload: '{"hello":"world"}', - payloadType: "application/json", - metadata: params.metadata, - metadataType: params.metadataType, - context: { foo: "bar" }, - traceContext: { trace: "ctx" }, - traceId: "trace_1", - spanId: "span_1", - runTags: ["alpha"], - queue: "task/my-task", - isTest: false, - taskEventStore: "taskEvent", - depth: 0, - createdAt: new Date("2024-01-01T00:00:00.000Z"), - }, - snapshot: { - engine: "V2", - executionStatus: "RUN_CREATED", - description: "Run was created", - runStatus: "PENDING", - environmentId: params.runtimeEnvironmentId, - environmentType: "DEVELOPMENT", - projectId: params.projectId, - organizationId: params.organizationId, - }, - }; -} - -function buildRouterWithLaggingLegacyReplica(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { - const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]); - const legacyStore = new PostgresRunStore({ - prisma: prisma14, - readOnlyPrisma: legacyReplica.client, - schemaVariant: "legacy", - }); - const newStore = new PostgresRunStore({ - prisma: prisma17 as never, - readOnlyPrisma: prisma17 as never, - schemaVariant: "dedicated", - }); - const router = new RoutingRunStore({ new: newStore, legacy: legacyStore }); - return { router, legacyStore, legacyReplica }; -} - -async function callLoader(friendlyId: string) { - const response = await loader({ - request: new Request("https://api.trigger.dev/api/v1/runs/" + friendlyId + "/metadata", { +async function callLoader(runId = friendlyId) { + return (await loader({ + request: new Request(`https://example.com/api/v1/runs/${runId}/metadata`, { headers: { Authorization: "Bearer tr_dev_meta" }, }), - params: { runId: friendlyId }, + params: { runId }, context: {} as never, - }); - return response as Response; + })) as Response; } -describe("metadata GET loader under replica lag", () => { - heteroRunOpsPostgresTest( - "metadata GET loader resolves a live run on a replica and buffer double-miss", - async ({ prisma14, prisma17 }) => { - const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica( - prisma14, - prisma17 - ); - - const seed = await seedEnvironmentLegacy(prisma14, "meta"); - const runId = `run_${CUID_25}`; // cuid → LEGACY-owned - const friendlyId = "run_meta_live"; - const metadata = '{"phase":"one"}'; - await legacyStore.createRun( - buildCreateRunInput({ - runId, - friendlyId, - organizationId: seed.organization.id, - projectId: seed.project.id, - runtimeEnvironmentId: seed.environment.id, - metadata, - metadataType: "application/json", - }) - ); - - // Wire the loader's module singletons: the real split router, the authenticated env, and a - // buffer that MISSES (the run has drained from the buffer to the primary but the replica has - // not caught up — the exact double-miss under test). - holder.store = router; - holder.environment = { id: seed.environment.id, organizationId: seed.organization.id }; - holder.bufferResult = null; - - const response = await callLoader(friendlyId); - const body = (await response.json()) as { - metadata?: unknown; - metadataType?: unknown; - error?: string; - }; - - // The replica WAS consulted (and, being frozen, missed) — proving the read genuinely went - // through the lagging replica and the recovery is the primary fallback, not a lucky replica hit. - expect(legacyReplica.wasHit()).toBe(true); - - // The property: the loader re-reads the owning primary and returns the live run's metadata. - expect(response.status).toBe(200); - expect(body.metadata).toBe(metadata); - expect(body.metadataType).toBe("application/json"); - } - ); - - // Negative control: when the run truly does not exist anywhere (replica miss + buffer miss + - // primary miss), the loader must still 404. This pins the behavior to "recover a LIVE run" rather - // than "never 404". - heteroRunOpsPostgresTest( - "metadata GET loader 404s when the run is absent on the primary too", - async ({ prisma14, prisma17 }) => { - const { router } = buildRouterWithLaggingLegacyReplica(prisma14, prisma17); - const seed = await seedEnvironmentLegacy(prisma14, "meta_absent"); - - holder.store = router; - holder.environment = { id: seed.environment.id, organizationId: seed.organization.id }; - holder.bufferResult = null; - - const response = await callLoader("run_does_not_exist"); - expect(response.status).toBe(404); - } - ); +beforeEach(() => { + vi.clearAllMocks(); + mocks.authenticateApiRequest.mockResolvedValue({ environment: mocks.environment }); + mocks.findRun.mockResolvedValue(null); + mocks.findRunByIdWithMollifierFallback.mockResolvedValue(null); + mocks.findRunOnPrimary.mockResolvedValue(null); +}); + +describe("metadata GET loader under replica lag", () => { + it("resolves a live run after a replica and buffer double-miss", async () => { + const metadata = '{"phase":"one"}'; + mocks.findRunOnPrimary.mockResolvedValue({ + metadata, + metadataType: "application/json", + }); + + const response = await callLoader(); + + expect(mocks.findRun).toHaveBeenCalledWith(runWhere, metadataSelect, mocks.replica); + expect(mocks.findRunByIdWithMollifierFallback).toHaveBeenCalledWith({ + runId: friendlyId, + environmentId: mocks.environment.id, + organizationId: mocks.environment.organizationId, + }); + expect(mocks.findRunOnPrimary).toHaveBeenCalledWith(runWhere, metadataSelect); + expect(mocks.findRun.mock.invocationCallOrder[0]).toBeLessThan( + mocks.findRunByIdWithMollifierFallback.mock.invocationCallOrder[0] + ); + expect(mocks.findRunByIdWithMollifierFallback.mock.invocationCallOrder[0]).toBeLessThan( + mocks.findRunOnPrimary.mock.invocationCallOrder[0] + ); + await expect(response.json()).resolves.toEqual({ + metadata, + metadataType: "application/json", + }); + expect(response.status).toBe(200); + }); + + it("returns 404 when the run is absent from the primary too", async () => { + const response = await callLoader("run_does_not_exist"); + + expect(mocks.findRunOnPrimary).toHaveBeenCalledWith( + { friendlyId: "run_does_not_exist", runtimeEnvironmentId: mocks.environment.id }, + metadataSelect + ); + await expect(response.json()).resolves.toEqual({ error: "Run not found" }); + expect(response.status).toBe(404); + }); }); From 6997aeb05e27d2db47f9eda01fdc8a17c81a1ae0 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Tue, 21 Jul 2026 12:00:58 +0100 Subject: [PATCH 007/300] fix: security release 2026-07-08 (#4316) --- .changeset/block-metadata-proto-pollution.md | 5 + .changeset/cli-login-poll-window.md | 6 + .../worker-id-deployment-friendly-id.md | 5 + .changeset/workload-deployment-token.md | 5 + .env.example | 1 + .github/workflows/helm-prerelease.yml | 4 +- .github/workflows/release-helm.yml | 4 +- .server-changes/otlp-ingestion-rate-limit.md | 6 + .../require-webapp-auth-secrets.md | 6 + ...ope-deployment-background-worker-lookup.md | 6 + .../scope-github-app-update-to-org.md | 6 + .../scope-schedule-and-env-var-writes.md | 6 + .../secure-compute-snapshot-callbacks.md | 6 + .../session-out-stream-private-auth.md | 6 + .../tighten-realtime-and-trace-sync.md | 6 + .../tsql-window-function-allowlist.md | 6 + .server-changes/workload-token-supervisor.md | 6 + .server-changes/workload-token-webapp.md | 6 + apps/supervisor/.env.example | 4 +- apps/supervisor/src/env.ts | 19 +- apps/supervisor/src/index.ts | 17 +- .../services/computeSnapshotService.test.ts | 120 +++++++- .../src/services/computeSnapshotService.ts | 92 +++++- .../supervisor/src/workloadManager/compute.ts | 4 +- apps/supervisor/src/workloadManager/docker.ts | 4 +- .../src/workloadManager/kubernetes.ts | 5 + apps/supervisor/src/workloadManager/types.ts | 2 + apps/supervisor/src/workloadServer/index.ts | 164 ++++++++++- .../workloadServer.auth.test.ts | 107 +++++++ .../workloadServer.authLog.test.ts | 87 ++++++ apps/supervisor/src/workloadToken.ts | 85 ++++++ apps/webapp/app/entry.server.tsx | 1 + apps/webapp/app/env.server.ts | 80 +++++- apps/webapp/app/models/schedules.server.ts | 47 +++- .../app/routes/_app.github.callback/route.tsx | 34 ++- .../route.tsx | 26 ++ .../route.tsx | 19 ++ .../route.tsx | 7 +- .../route.tsx | 171 ++++++++---- .../app/routes/api.v1.authorization-code.ts | 25 +- ...api.v1.deployments.$deploymentId.cancel.ts | 2 +- .../api.v1.schedules.$scheduleId.activate.ts | 21 +- ...api.v1.schedules.$scheduleId.deactivate.ts | 21 +- .../routes/api.v1.schedules.$scheduleId.ts | 35 ++- apps/webapp/app/routes/api.v1.token.ts | 18 ++ ...s.$snapshotFriendlyId.attempts.complete.ts | 2 + ...hots.$snapshotFriendlyId.attempts.start.ts | 2 + ....snapshots.$snapshotFriendlyId.continue.ts | 8 + ...ns.runs.$runFriendlyId.snapshots.latest.ts | 2 + ...nFriendlyId.snapshots.since.$snapshotId.ts | 2 + apps/webapp/app/routes/logout.tsx | 9 +- apps/webapp/app/routes/realtime.v1.runs.ts | 15 + .../realtime.v1.sessions.$session.$io.ts | 10 + ...deployments.$deploymentShortCode.cancel.ts | 7 +- .../webapp/app/routes/sync.traces.$traceId.ts | 32 ++- .../app/routes/sync.traces.runs.$traceId.ts | 32 ++- .../app/services/apiRateLimit.server.ts | 3 + .../services/authCodeRateLimiter.server.ts | 85 ++++++ apps/webapp/app/services/gitHub.server.ts | 26 +- .../app/services/otlpRateLimit.server.ts | 91 ++++++ .../services/personalAccessToken.server.ts | 41 ++- .../app/services/realtimeClient.server.ts | 6 +- .../routeBuilders/apiBuilder.server.ts | 8 + apps/webapp/app/v3/electricShape.server.ts | 69 +++++ .../environmentVariablesRepository.server.ts | 9 +- apps/webapp/app/v3/otlpTransform.server.ts | 7 +- .../app/v3/services/checkSchedule.server.ts | 15 +- ...eateDeploymentBackgroundWorkerV4.server.ts | 1 + .../v3/services/deleteTaskSchedule.server.ts | 5 +- .../app/v3/services/deployment.server.ts | 14 +- .../services/initializeDeployment.server.ts | 2 +- .../resolveProjectScopedEnvironments.ts | 27 ++ .../setActiveOnTaskSchedule.server.ts | 9 +- .../worker/workerGroupTokenService.server.ts | 141 +++++++++- .../workloadTokenAuthorization.server.ts | 27 ++ apps/webapp/app/v3/workerIdUnwrap.server.ts | 75 +++++ apps/webapp/app/v3/writableEnvironments.ts | 33 +++ apps/webapp/package.json | 3 + apps/webapp/server.ts | 2 + .../test/authorizationCodeConsent.test.ts | 58 ++++ apps/webapp/test/checkSchedule.test.ts | 60 ++++ apps/webapp/test/deleteTaskSchedule.test.ts | 75 +++++ apps/webapp/test/env.server.test.ts | 84 ++++++ .../environmentVariablesRepository.test.ts | 76 +++++ apps/webapp/test/registryConfig.test.ts | 3 + .../resolveProjectScopedEnvironments.test.ts | 44 +++ .../test/schedulesPutEnvScoping.test.ts | 261 ++++++++++++++++++ .../test/setActiveOnTaskSchedule.test.ts | 149 ++++++++++ apps/webapp/test/setup.ts | 3 + .../test/spanTraceRoutes.replicaLag.test.ts | 5 +- apps/webapp/test/workerIdUnwrap.test.ts | 71 +++++ .../test/workloadTokenAuthorization.test.ts | 26 ++ .../workloadTokenGate.integration.test.ts | 88 ++++++ apps/webapp/test/writableEnvironments.test.ts | 55 ++++ docs/self-hosting/docker.mdx | 20 +- docs/self-hosting/env/webapp.mdx | 3 +- docs/self-hosting/kubernetes.mdx | 26 +- hosting/docker/.env.example | 56 ++-- hosting/docker/generate-secrets.sh | 110 ++++++++ hosting/docker/registry/auth.htpasswd | 1 - hosting/docker/webapp/docker-compose.yml | 22 +- hosting/k8s/helm/README.md | 27 +- hosting/k8s/helm/ci/lint-values.yaml | 26 ++ hosting/k8s/helm/templates/NOTES.txt | 20 +- hosting/k8s/helm/templates/_helpers.tpl | 35 ++- .../k8s/helm/templates/datastore-secret.yaml | 39 +++ hosting/k8s/helm/templates/electric.yaml | 7 + hosting/k8s/helm/templates/s2.yaml | 5 +- hosting/k8s/helm/templates/secrets.yaml | 30 +- .../templates/validate-external-config.yaml | 17 +- hosting/k8s/helm/templates/webapp.yaml | 23 ++ .../k8s/helm/values-production-example.yaml | 9 + hosting/k8s/helm/values.yaml | 66 +++-- .../run-engine/src/engine/index.ts | 23 +- .../src/engine/systems/checkpointSystem.ts | 9 +- .../engine/systems/executionSnapshotSystem.ts | 31 ++- .../src/engine/systems/runAttemptSystem.ts | 31 ++- .../run-store/src/PostgresRunStore.ts | 8 +- .../run-store/src/runOpsStore.ts | 6 +- internal-packages/run-store/src/types.ts | 5 +- .../testcontainers/src/webapp.ts | 5 + internal-packages/tsql/src/query/printer.ts | 10 +- packages/cli-v3/src/commands/login.ts | 14 +- .../src/entryPoints/managed/controller.ts | 2 +- .../cli-v3/src/entryPoints/managed/env.ts | 11 + .../src/entryPoints/managed/execution.ts | 2 +- .../managed/taskRunProcessProvider.ts | 5 +- packages/cli-v3/src/mcp/auth.ts | 5 +- packages/core/src/v3/apiClient/core.ts | 8 + packages/core/src/v3/index.ts | 1 + packages/core/src/v3/jwt.ts | 16 +- .../core/src/v3/runEngineWorker/consts.ts | 3 + .../src/v3/runEngineWorker/supervisor/http.ts | 33 ++- packages/core/src/v3/runEngineWorker/types.ts | 2 + .../core/src/v3/runMetadata/operations.ts | 9 +- packages/core/src/v3/schemas/common.ts | 29 +- .../core/src/v3/utils/flattenAttributes.ts | 15 +- .../src/v3/workloadDeploymentToken.test.ts | 127 +++++++++ .../core/src/v3/workloadDeploymentToken.ts | 99 +++++++ packages/core/test/taskExecutor.test.ts | 6 +- pnpm-lock.yaml | 15 + 141 files changed, 3939 insertions(+), 324 deletions(-) create mode 100644 .changeset/block-metadata-proto-pollution.md create mode 100644 .changeset/cli-login-poll-window.md create mode 100644 .changeset/worker-id-deployment-friendly-id.md create mode 100644 .changeset/workload-deployment-token.md create mode 100644 .server-changes/otlp-ingestion-rate-limit.md create mode 100644 .server-changes/require-webapp-auth-secrets.md create mode 100644 .server-changes/scope-deployment-background-worker-lookup.md create mode 100644 .server-changes/scope-github-app-update-to-org.md create mode 100644 .server-changes/scope-schedule-and-env-var-writes.md create mode 100644 .server-changes/secure-compute-snapshot-callbacks.md create mode 100644 .server-changes/session-out-stream-private-auth.md create mode 100644 .server-changes/tighten-realtime-and-trace-sync.md create mode 100644 .server-changes/tsql-window-function-allowlist.md create mode 100644 .server-changes/workload-token-supervisor.md create mode 100644 .server-changes/workload-token-webapp.md create mode 100644 apps/supervisor/src/workloadServer/workloadServer.auth.test.ts create mode 100644 apps/supervisor/src/workloadServer/workloadServer.authLog.test.ts create mode 100644 apps/supervisor/src/workloadToken.ts create mode 100644 apps/webapp/app/services/authCodeRateLimiter.server.ts create mode 100644 apps/webapp/app/services/otlpRateLimit.server.ts create mode 100644 apps/webapp/app/v3/electricShape.server.ts create mode 100644 apps/webapp/app/v3/services/resolveProjectScopedEnvironments.ts create mode 100644 apps/webapp/app/v3/services/worker/workloadTokenAuthorization.server.ts create mode 100644 apps/webapp/app/v3/workerIdUnwrap.server.ts create mode 100644 apps/webapp/app/v3/writableEnvironments.ts create mode 100644 apps/webapp/test/authorizationCodeConsent.test.ts create mode 100644 apps/webapp/test/checkSchedule.test.ts create mode 100644 apps/webapp/test/deleteTaskSchedule.test.ts create mode 100644 apps/webapp/test/env.server.test.ts create mode 100644 apps/webapp/test/resolveProjectScopedEnvironments.test.ts create mode 100644 apps/webapp/test/schedulesPutEnvScoping.test.ts create mode 100644 apps/webapp/test/setActiveOnTaskSchedule.test.ts create mode 100644 apps/webapp/test/workerIdUnwrap.test.ts create mode 100644 apps/webapp/test/workloadTokenAuthorization.test.ts create mode 100644 apps/webapp/test/workloadTokenGate.integration.test.ts create mode 100644 apps/webapp/test/writableEnvironments.test.ts create mode 100755 hosting/docker/generate-secrets.sh create mode 100644 hosting/k8s/helm/ci/lint-values.yaml create mode 100644 hosting/k8s/helm/templates/datastore-secret.yaml create mode 100644 packages/core/src/v3/workloadDeploymentToken.test.ts create mode 100644 packages/core/src/v3/workloadDeploymentToken.ts diff --git a/.changeset/block-metadata-proto-pollution.md b/.changeset/block-metadata-proto-pollution.md new file mode 100644 index 000000000..aa38beab4 --- /dev/null +++ b/.changeset/block-metadata-proto-pollution.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +Prevent prototype pollution when applying run metadata operations or reconstructing nested telemetry attributes, while preserving legitimate `constructor` and `prototype` fields. diff --git a/.changeset/cli-login-poll-window.md b/.changeset/cli-login-poll-window.md new file mode 100644 index 000000000..e0453895a --- /dev/null +++ b/.changeset/cli-login-poll-window.md @@ -0,0 +1,6 @@ +--- +"@trigger.dev/core": patch +"trigger.dev": patch +--- + +Require explicit browser approval for CLI and MCP login, with resilient polling while approval is pending. diff --git a/.changeset/worker-id-deployment-friendly-id.md b/.changeset/worker-id-deployment-friendly-id.md new file mode 100644 index 000000000..2e62fee4b --- /dev/null +++ b/.changeset/worker-id-deployment-friendly-id.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +Deployed task telemetry now reports the deployment identifier (e.g. `deployment_abc123`) in the `worker.id` attribute, instead of an opaque internal value. Upgrade to get the readable identifier in your own OpenTelemetry exporters. diff --git a/.changeset/workload-deployment-token.md b/.changeset/workload-deployment-token.md new file mode 100644 index 000000000..6a3c08cfc --- /dev/null +++ b/.changeset/workload-deployment-token.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +Add helpers to mint and verify the deployment-scoped token used to authenticate run controllers to the platform. diff --git a/.env.example b/.env.example index f37a1a519..901fbd7b3 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,7 @@ SESSION_SECRET=abcdef1234 MAGIC_LINK_SECRET=abcdef1234 ENCRYPTION_KEY=ae13021afef0819c3a307ad487071c06 # Must be a random 16 byte hex string. You can generate an encryption key by running `openssl rand -hex 16` in your terminal +MANAGED_WORKER_SECRET=abcdef1234 # Must match the supervisor's MANAGED_WORKER_SECRET LOGIN_ORIGIN=http://localhost:3030 DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres?schema=public # This sets the URL used for direct connections to the database and should only be needed in limited circumstances diff --git a/.github/workflows/helm-prerelease.yml b/.github/workflows/helm-prerelease.yml index 37b909e41..7d49868ee 100644 --- a/.github/workflows/helm-prerelease.yml +++ b/.github/workflows/helm-prerelease.yml @@ -52,12 +52,14 @@ jobs: - name: Lint Helm Chart run: | - helm lint ./hosting/k8s/helm/ + helm lint ./hosting/k8s/helm/ \ + --values ./hosting/k8s/helm/ci/lint-values.yaml - name: Render templates run: | helm template test-release ./hosting/k8s/helm/ \ --values ./hosting/k8s/helm/values.yaml \ + --values ./hosting/k8s/helm/ci/lint-values.yaml \ --output-dir ./helm-output - name: Validate manifests diff --git a/.github/workflows/release-helm.yml b/.github/workflows/release-helm.yml index a5afd8b1b..0ce968ebc 100644 --- a/.github/workflows/release-helm.yml +++ b/.github/workflows/release-helm.yml @@ -47,12 +47,14 @@ jobs: - name: Lint Helm Chart run: | - helm lint ./hosting/k8s/helm/ + helm lint ./hosting/k8s/helm/ \ + --values ./hosting/k8s/helm/ci/lint-values.yaml - name: Render templates run: | helm template test-release ./hosting/k8s/helm/ \ --values ./hosting/k8s/helm/values.yaml \ + --values ./hosting/k8s/helm/ci/lint-values.yaml \ --output-dir ./helm-output - name: Validate manifests diff --git a/.server-changes/otlp-ingestion-rate-limit.md b/.server-changes/otlp-ingestion-rate-limit.md new file mode 100644 index 000000000..acb9c6231 --- /dev/null +++ b/.server-changes/otlp-ingestion-rate-limit.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Added optional request rate limiting for telemetry ingestion endpoints. diff --git a/.server-changes/require-webapp-auth-secrets.md b/.server-changes/require-webapp-auth-secrets.md new file mode 100644 index 000000000..52cb61d87 --- /dev/null +++ b/.server-changes/require-webapp-auth-secrets.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: breaking +--- + +Self-hosted deployments no longer ship shared default credentials; fresh installs generate their own. If yours still uses a previously published default, set a unique value before upgrading, or set `ALLOW_INSECURE_DEFAULT_SECRETS=true` to keep booting while you migrate. diff --git a/.server-changes/scope-deployment-background-worker-lookup.md b/.server-changes/scope-deployment-background-worker-lookup.md new file mode 100644 index 000000000..74c2aa7ee --- /dev/null +++ b/.server-changes/scope-deployment-background-worker-lookup.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Background-worker deployment lookups are now scoped to the authenticated environment. diff --git a/.server-changes/scope-github-app-update-to-org.md b/.server-changes/scope-github-app-update-to-org.md new file mode 100644 index 000000000..df417e52d --- /dev/null +++ b/.server-changes/scope-github-app-update-to-org.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Updating a GitHub App installation from the callback flow is now scoped to your own organization, so an installation ID belonging to another organization can no longer be used to refresh that organization's installation record. The GitHub App installation session is also now single-use, so completing an installation callback invalidates its state and it can no longer be replayed. diff --git a/.server-changes/scope-schedule-and-env-var-writes.md b/.server-changes/scope-schedule-and-env-var-writes.md new file mode 100644 index 000000000..3ddcd6a1a --- /dev/null +++ b/.server-changes/scope-schedule-and-env-var-writes.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Scope schedule and environment-variable writes to the caller's project and environment diff --git a/.server-changes/secure-compute-snapshot-callbacks.md b/.server-changes/secure-compute-snapshot-callbacks.md new file mode 100644 index 000000000..5f362f3fa --- /dev/null +++ b/.server-changes/secure-compute-snapshot-callbacks.md @@ -0,0 +1,6 @@ +--- +area: supervisor +type: fix +--- + +Reject compute snapshot callbacks that do not match the snapshot request that created them. diff --git a/.server-changes/session-out-stream-private-auth.md b/.server-changes/session-out-stream-private-auth.md new file mode 100644 index 000000000..bfc95c662 --- /dev/null +++ b/.server-changes/session-out-stream-private-auth.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Require secret-key authentication to initialize the session out (agent→client) stream, matching the append route. diff --git a/.server-changes/tighten-realtime-and-trace-sync.md b/.server-changes/tighten-realtime-and-trace-sync.md new file mode 100644 index 000000000..51b4b2d41 --- /dev/null +++ b/.server-changes/tighten-realtime-and-trace-sync.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Live run and trace subscriptions now validate their identifiers more strictly and only return data from your own organization. diff --git a/.server-changes/tsql-window-function-allowlist.md b/.server-changes/tsql-window-function-allowlist.md new file mode 100644 index 000000000..a56a881ae --- /dev/null +++ b/.server-changes/tsql-window-function-allowlist.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Window-function names in the query compiler are now validated against the allowlist, matching how other function calls are handled. diff --git a/.server-changes/workload-token-supervisor.md b/.server-changes/workload-token-supervisor.md new file mode 100644 index 000000000..b7c82c888 --- /dev/null +++ b/.server-changes/workload-token-supervisor.md @@ -0,0 +1,6 @@ +--- +area: supervisor +type: fix +--- + +Authenticate run controllers to the platform with a signed, deployment-scoped token. diff --git a/.server-changes/workload-token-webapp.md b/.server-changes/workload-token-webapp.md new file mode 100644 index 000000000..b743c4463 --- /dev/null +++ b/.server-changes/workload-token-webapp.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Verify that worker actions (starting, completing, and continuing a run, and reading its snapshots) target a run belonging to the caller's environment. diff --git a/apps/supervisor/.env.example b/apps/supervisor/.env.example index 4355a4eea..f71cb88f3 100644 --- a/apps/supervisor/.env.example +++ b/apps/supervisor/.env.example @@ -1,8 +1,8 @@ # This needs to match the token of the worker group you want to connect to TRIGGER_WORKER_TOKEN= -# This needs to match the MANAGED_WORKER_SECRET env var on the webapp -MANAGED_WORKER_SECRET=managed-secret +# Must match the webapp's MANAGED_WORKER_SECRET. Generate with: openssl rand -hex 16 +MANAGED_WORKER_SECRET= # Point this at the webapp in prod TRIGGER_API_URL=http://localhost:3030 diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index 32e669b59..6c954b4e7 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -14,8 +14,17 @@ export const Env = z // Required settings TRIGGER_API_URL: z.string().url(), - TRIGGER_WORKER_TOKEN: z.string(), // accepts file:// path to read from a file + TRIGGER_WORKER_TOKEN: z.string().min(1), // accepts file:// path to read from a file MANAGED_WORKER_SECRET: z.string(), + + // Deployment token: sign a token into TRIGGER_DEPLOYMENT_ID at pod creation and verify it on + // inbound workload calls. "disabled" = off; "log" = mint + verify + metrics only; "enforce" = + // also reject invalid tokens. + WORKLOAD_TOKEN_SECRET: z.string().optional(), + WORKLOAD_TOKEN_ENFORCEMENT: z.enum(["disabled", "log", "enforce"]).default("disabled"), + // Absolute expiry for minted deployment tokens. Deterministic (no wall-clock issued-at) so every + // pod of a deployment carries an identical token; bump before this date. Must outlive any run. + WORKLOAD_TOKEN_EXP: z.string().datetime().default("2032-01-01T00:00:00.000Z"), OTEL_EXPORTER_OTLP_ENDPOINT: z.string().url(), // set on the runners // Workload API settings (coordinator mode) - the workload API is what the run controller connects to @@ -365,6 +374,14 @@ export const Env = z path: ["TRIGGER_WORKLOAD_API_DOMAIN"], }); } + if (data.WORKLOAD_TOKEN_ENFORCEMENT !== "disabled" && !data.WORKLOAD_TOKEN_SECRET) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + "WORKLOAD_TOKEN_SECRET is required when WORKLOAD_TOKEN_ENFORCEMENT is not disabled", + path: ["WORKLOAD_TOKEN_SECRET"], + }); + } if ( data.TRIGGER_DEQUEUE_BACKPRESSURE_ENABLED && !data.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_HOST diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts index 701b8a25e..cf73be90e 100644 --- a/apps/supervisor/src/index.ts +++ b/apps/supervisor/src/index.ts @@ -27,6 +27,7 @@ import { register } from "./metrics.js"; import { PodCleaner } from "./services/podCleaner.js"; import { FailedPodHandler } from "./services/failedPodHandler.js"; import { getWorkerToken } from "./workerToken.js"; +import { mintDeploymentToken } from "./workloadToken.js"; import { OtlpTraceService } from "./services/otlpTraceService.js"; import { WarmStartVerificationService, @@ -96,6 +97,7 @@ class ManagedSupervisor { COMPUTE_GATEWAY_AUTH_TOKEN, DOCKER_REGISTRY_PASSWORD, TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_PASSWORD, + WORKLOAD_TOKEN_SECRET, ...envWithoutSecrets } = env; @@ -290,8 +292,10 @@ class ManagedSupervisor { }); } + const workerToken = getWorkerToken(); + this.workerSession = new SupervisorSession({ - workerToken: getWorkerToken(), + workerToken, apiUrl: env.TRIGGER_API_URL, instanceName: env.TRIGGER_WORKER_INSTANCE_NAME, managedWorkerSecret: env.MANAGED_WORKER_SECRET, @@ -569,6 +573,7 @@ class ManagedSupervisor { checkpointClient: this.checkpointClient, computeManager: this.computeManager, tracing: this.tracing, + snapshotCallbackSecret: workerToken, wideEventOpts: this.wideEventOpts, wideEventsNoisyRoutes: this.wideEventsNoisyRoutes, }); @@ -603,6 +608,15 @@ class ManagedSupervisor { throw new Error("Image is missing"); } + const deploymentToken = await mintDeploymentToken({ + deployment: message.deployment.friendlyId, + deployment_version: message.backgroundWorker.version, + environment_id: message.environment.id, + environment_type: message.environment.type, + org_id: message.organization.id, + project_id: message.project.id, + }); + await this.workloadManager.create({ dequeuedAt: message.dequeuedAt, dequeueResponseMs: timings.dequeueResponseMs, @@ -617,6 +631,7 @@ class ManagedSupervisor { deploymentFriendlyId: message.deployment.friendlyId, deploymentVersion: message.backgroundWorker.version, runtime: message.backgroundWorker.runtime, + deploymentToken, runId: message.run.id, runFriendlyId: message.run.friendlyId, version: message.version, diff --git a/apps/supervisor/src/services/computeSnapshotService.test.ts b/apps/supervisor/src/services/computeSnapshotService.test.ts index 44a13a4bd..0877f5e9d 100644 --- a/apps/supervisor/src/services/computeSnapshotService.test.ts +++ b/apps/supervisor/src/services/computeSnapshotService.test.ts @@ -20,13 +20,26 @@ function createService() { snapshot, } as unknown as ComputeWorkloadManager; + const submitSuspendCompletion = vi.fn(async () => ({ success: true })); + const service = new ComputeSnapshotService({ computeManager, - workerClient: {} as SupervisorHttpClient, + workerClient: { submitSuspendCompletion } as unknown as SupervisorHttpClient, wideEventOpts: { service: "supervisor-test", env: {}, enabled: false }, + snapshotCallbackSecret: "test-secret", }); - return { service, snapshot }; + return { service, snapshot, submitSuspendCompletion }; +} + +function dispatchedMetadata(snapshot: { + mock: { calls: Array }>> }; +}) { + const metadata = snapshot.mock.calls[0]?.[0]?.metadata; + if (!metadata) { + throw new Error("Snapshot was not dispatched"); + } + return metadata; } function delayedSnapshot(runnerId = "runner-1") { @@ -38,6 +51,24 @@ function delayedSnapshot(runnerId = "runner-1") { } describe("ComputeSnapshotService", () => { + it("refuses to construct with an empty callback secret", () => { + const computeManager = { + snapshotDelayMs: DELAY_MS, + snapshotDispatchLimit: 1, + snapshot: vi.fn(async () => true), + } as unknown as ComputeWorkloadManager; + + expect( + () => + new ComputeSnapshotService({ + computeManager, + workerClient: {} as SupervisorHttpClient, + wideEventOpts: { service: "supervisor-test", env: {}, enabled: false }, + snapshotCallbackSecret: "", + }) + ).toThrow(); + }); + it("dispatches a scheduled snapshot after the delay", async () => { const { service, snapshot } = createService(); try { @@ -46,7 +77,12 @@ describe("ComputeSnapshotService", () => { await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(1), { timeout: 2_000 }); expect(snapshot).toHaveBeenCalledWith({ runnerId: "runner-1", - metadata: { runId: "run_1", snapshotFriendlyId: "snapshot_1" }, + metadata: expect.objectContaining({ + runId: "run_1", + snapshotFriendlyId: "snapshot_1", + snapshotCallbackNonce: expect.any(String), + snapshotCallbackToken: expect.any(String), + }), }); } finally { service.stop(); @@ -121,10 +157,86 @@ describe("ComputeSnapshotService", () => { expect(snapshot).toHaveBeenCalledTimes(1); expect(snapshot).toHaveBeenCalledWith({ runnerId: "runner-1", - metadata: { runId: "run_1", snapshotFriendlyId: "snapshot_2" }, + metadata: expect.objectContaining({ + runId: "run_1", + snapshotFriendlyId: "snapshot_2", + snapshotCallbackNonce: expect.any(String), + snapshotCallbackToken: expect.any(String), + }), }); } finally { service.stop(); } }); + + it("accepts a snapshot callback with the dispatched token", async () => { + const { service, snapshot, submitSuspendCompletion } = createService(); + try { + service.schedule("run_1", delayedSnapshot()); + + await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(1), { timeout: 2_000 }); + const metadata = dispatchedMetadata(snapshot); + + const result = await service.handleCallback({ + status: "completed", + instance_id: "instance_1", + snapshot_id: "compute_snapshot_1", + metadata, + }); + + expect(result).toEqual({ ok: true, status: 200 }); + expect(submitSuspendCompletion).toHaveBeenCalledWith({ + runId: "run_1", + snapshotId: "snapshot_1", + body: { + success: true, + checkpoint: { + type: "COMPUTE", + location: "compute_snapshot_1", + }, + }, + }); + } finally { + service.stop(); + } + }); + + it("rejects a snapshot callback without a valid token", async () => { + const { service, submitSuspendCompletion } = createService(); + try { + const result = await service.handleCallback({ + status: "completed", + instance_id: "instance_1", + snapshot_id: "compute_snapshot_1", + metadata: { runId: "run_1", snapshotFriendlyId: "snapshot_1" }, + }); + + expect(result).toEqual({ ok: false, status: 401 }); + expect(submitSuspendCompletion).not.toHaveBeenCalled(); + } finally { + service.stop(); + } + }); + + it("rejects a snapshot callback whose token is for a different snapshot", async () => { + const { service, snapshot, submitSuspendCompletion } = createService(); + try { + service.schedule("run_1", delayedSnapshot()); + + await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(1), { timeout: 2_000 }); + const metadata = dispatchedMetadata(snapshot); + + const result = await service.handleCallback({ + status: "completed", + instance_id: "instance_1", + snapshot_id: "compute_snapshot_1", + metadata: { ...metadata, snapshotFriendlyId: "snapshot_2" }, + }); + + expect(result).toEqual({ ok: false, status: 401 }); + expect(submitSuspendCompletion).not.toHaveBeenCalled(); + } finally { + service.stop(); + } + }); }); diff --git a/apps/supervisor/src/services/computeSnapshotService.ts b/apps/supervisor/src/services/computeSnapshotService.ts index 216753fc1..6af307d0e 100644 --- a/apps/supervisor/src/services/computeSnapshotService.ts +++ b/apps/supervisor/src/services/computeSnapshotService.ts @@ -1,3 +1,4 @@ +import { createHmac, randomBytes, timingSafeEqual } from "node:crypto"; import pLimit from "p-limit"; import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; import { parseTraceparent } from "@trigger.dev/core/v3/isomorphic"; @@ -16,6 +17,13 @@ import { type WideEventOptions, } from "../wideEvents/index.js"; +const SNAPSHOT_CALLBACK_NONCE_METADATA_KEY = "snapshotCallbackNonce"; +const SNAPSHOT_CALLBACK_TOKEN_METADATA_KEY = "snapshotCallbackToken"; + +// Domain-separation label so the callback-signing key is derived from, rather +// than equal to, the secret used for other protocols. Bump the suffix to rotate. +const SNAPSHOT_CALLBACK_KEY_INFO = "compute-snapshot-callback-v1"; + type DelayedSnapshot = { runnerId: string; runFriendlyId: string; @@ -34,6 +42,7 @@ export type ComputeSnapshotServiceOptions = { workerClient: SupervisorHttpClient; tracing?: OtlpTraceService; wideEventOpts: WideEventOptions; + snapshotCallbackSecret: string; }; export class ComputeSnapshotService { @@ -48,6 +57,7 @@ export class ComputeSnapshotService { private readonly workerClient: SupervisorHttpClient; private readonly tracing?: OtlpTraceService; private readonly wideEventOpts: WideEventOptions; + private readonly snapshotCallbackKey: Buffer; constructor(opts: ComputeSnapshotServiceOptions) { this.computeManager = opts.computeManager; @@ -55,6 +65,18 @@ export class ComputeSnapshotService { this.tracing = opts.tracing; this.wideEventOpts = opts.wideEventOpts; + // Reject an empty secret up front: an empty HMAC key would make callback + // tokens forgeable by anyone. Guarding here (rather than only at env parse) + // also covers the case where the secret is read from an empty file. + if (!opts.snapshotCallbackSecret) { + throw new Error("snapshotCallbackSecret must not be empty"); + } + // Derive a dedicated key by domain separation so the raw secret is never + // used directly as a MAC key for this protocol. + this.snapshotCallbackKey = createHmac("sha256", opts.snapshotCallbackSecret) + .update(SNAPSHOT_CALLBACK_KEY_INFO) + .digest(); + this.dispatchLimit = pLimit(this.computeManager.snapshotDispatchLimit); this.timerWheel = new TimerWheel({ delayMs: this.computeManager.snapshotDelayMs, @@ -146,15 +168,29 @@ export class ComputeSnapshotService { instanceId: body.instance_id, status: body.status, error: body.status === "failed" ? body.error : undefined, - metadata: body.metadata, + runId, + snapshotFriendlyId, durationMs: body.duration_ms, }); if (!runId || !snapshotFriendlyId) { - this.logger.error("Snapshot callback missing metadata", { body }); + this.logger.error("Snapshot callback missing metadata", { + status: body.status, + instanceId: body.instance_id, + metadataKeys: Object.keys(body.metadata ?? {}), + }); return { ok: false as const, status: 400 }; } + if (!this.#verifyCallbackToken(body.metadata, runId, snapshotFriendlyId)) { + this.logger.error("Snapshot callback failed token verification", { + runId, + snapshotFriendlyId, + instanceId: body.instance_id, + }); + return { ok: false as const, status: 401 }; + } + this.#emitSnapshotSpan(runId, body.duration_ms, snapshotId); if (body.status === "completed") { @@ -266,11 +302,18 @@ export class ComputeSnapshotService { }, }, async () => { + const callbackNonce = randomBytes(16).toString("hex"); const result = await this.computeManager.snapshot({ runnerId: snapshot.runnerId, metadata: { runId: snapshot.runFriendlyId, snapshotFriendlyId: snapshot.snapshotFriendlyId, + [SNAPSHOT_CALLBACK_NONCE_METADATA_KEY]: callbackNonce, + [SNAPSHOT_CALLBACK_TOKEN_METADATA_KEY]: this.#createCallbackToken( + callbackNonce, + snapshot.runFriendlyId, + snapshot.snapshotFriendlyId + ), }, }); @@ -281,6 +324,51 @@ export class ComputeSnapshotService { ); } + #createCallbackToken(nonce: string, runFriendlyId: string, snapshotFriendlyId: string): string { + return createHmac("sha256", this.snapshotCallbackKey) + .update(nonce) + .update("\0") + .update(runFriendlyId) + .update("\0") + .update(snapshotFriendlyId) + .digest("hex"); + } + + /** + * Verify that a callback carries a token this supervisor issued for the given + * run and snapshot. The token binds only the identifiers known at dispatch + * time (nonce, run, snapshot); it intentionally does not cover result fields + * such as the snapshot location or status/error, which are produced by the + * gateway after the snapshot and so cannot be signed in advance. Verification + * is also stateless, so a token is not single-use. + * + * This closes the primary risk (a caller that can merely reach the endpoint + * cannot mint a valid token, so cannot forge a result for an arbitrary run). + * It does not defend against an attacker who can observe a genuine callback + * and then replay it or alter its unsigned result fields - that relies on the + * gateway->supervisor callback channel being authenticated and encrypted. + */ + #verifyCallbackToken( + metadata: Record | undefined, + runFriendlyId: string, + snapshotFriendlyId: string + ): boolean { + const nonce = metadata?.[SNAPSHOT_CALLBACK_NONCE_METADATA_KEY]; + const token = metadata?.[SNAPSHOT_CALLBACK_TOKEN_METADATA_KEY]; + + if (!nonce || !token) { + return false; + } + + const expected = this.#createCallbackToken(nonce, runFriendlyId, snapshotFriendlyId); + const expectedBuffer = Buffer.from(expected, "hex"); + const tokenBuffer = Buffer.from(token, "hex"); + + return ( + expectedBuffer.length === tokenBuffer.length && timingSafeEqual(expectedBuffer, tokenBuffer) + ); + } + #emitSnapshotSpan(runFriendlyId: string, durationMs?: number, snapshotId?: string) { if (!this.tracing) return; diff --git a/apps/supervisor/src/workloadManager/compute.ts b/apps/supervisor/src/workloadManager/compute.ts index 857e129a8..89360437d 100644 --- a/apps/supervisor/src/workloadManager/compute.ts +++ b/apps/supervisor/src/workloadManager/compute.ts @@ -151,7 +151,9 @@ export class ComputeWorkloadManager implements WorkloadManager { TRIGGER_DEQUEUED_AT_MS: String(opts.dequeuedAt.getTime()), TRIGGER_POD_SCHEDULED_AT_MS: String(Date.now()), TRIGGER_ENV_ID: opts.envId, - TRIGGER_DEPLOYMENT_ID: opts.deploymentFriendlyId, + TRIGGER_DEPLOYMENT_ID: opts.deploymentToken ?? opts.deploymentFriendlyId, + // Plain friendlyId for telemetry (worker.id), so it isn't the opaque token in DEPLOYMENT_ID. + TRIGGER_DEPLOYMENT_FRIENDLY_ID: opts.deploymentFriendlyId, TRIGGER_DEPLOYMENT_VERSION: opts.deploymentVersion, TRIGGER_RUN_ID: opts.runFriendlyId, TRIGGER_SNAPSHOT_ID: opts.snapshotFriendlyId, diff --git a/apps/supervisor/src/workloadManager/docker.ts b/apps/supervisor/src/workloadManager/docker.ts index 1d8fec1df..c8049873b 100644 --- a/apps/supervisor/src/workloadManager/docker.ts +++ b/apps/supervisor/src/workloadManager/docker.ts @@ -72,7 +72,9 @@ export class DockerWorkloadManager implements WorkloadManager { `TRIGGER_DEQUEUED_AT_MS=${opts.dequeuedAt.getTime()}`, `TRIGGER_POD_SCHEDULED_AT_MS=${Date.now()}`, `TRIGGER_ENV_ID=${opts.envId}`, - `TRIGGER_DEPLOYMENT_ID=${opts.deploymentFriendlyId}`, + `TRIGGER_DEPLOYMENT_ID=${opts.deploymentToken ?? opts.deploymentFriendlyId}`, + // Plain friendlyId for telemetry (worker.id), so it isn't the opaque token in DEPLOYMENT_ID. + `TRIGGER_DEPLOYMENT_FRIENDLY_ID=${opts.deploymentFriendlyId}`, `TRIGGER_DEPLOYMENT_VERSION=${opts.deploymentVersion}`, `TRIGGER_RUN_ID=${opts.runFriendlyId}`, `TRIGGER_SNAPSHOT_ID=${opts.snapshotFriendlyId}`, diff --git a/apps/supervisor/src/workloadManager/kubernetes.ts b/apps/supervisor/src/workloadManager/kubernetes.ts index 282ac80b4..0263b229e 100644 --- a/apps/supervisor/src/workloadManager/kubernetes.ts +++ b/apps/supervisor/src/workloadManager/kubernetes.ts @@ -158,6 +158,11 @@ export class KubernetesWorkloadManager implements WorkloadManager { }, { name: "TRIGGER_DEPLOYMENT_ID", + value: opts.deploymentToken ?? opts.deploymentFriendlyId, + }, + { + // Plain friendlyId for telemetry (worker.id), not the opaque token in DEPLOYMENT_ID. + name: "TRIGGER_DEPLOYMENT_FRIENDLY_ID", value: opts.deploymentFriendlyId, }, { diff --git a/apps/supervisor/src/workloadManager/types.ts b/apps/supervisor/src/workloadManager/types.ts index 4945b1382..eda9162c7 100644 --- a/apps/supervisor/src/workloadManager/types.ts +++ b/apps/supervisor/src/workloadManager/types.ts @@ -44,6 +44,8 @@ export interface WorkloadManagerCreateOptions { deploymentVersion: string; // Canonical runtime identifier (e.g. "node", "node-22", "node-24") runtime?: string; + // When set, overrides the TRIGGER_DEPLOYMENT_ID value the runner forwards as its identity header. + deploymentToken?: string; runId: string; runFriendlyId: string; snapshotId: string; diff --git a/apps/supervisor/src/workloadServer/index.ts b/apps/supervisor/src/workloadServer/index.ts index bacdbbe06..86717438c 100644 --- a/apps/supervisor/src/workloadServer/index.ts +++ b/apps/supervisor/src/workloadServer/index.ts @@ -25,6 +25,11 @@ import { type Namespace, Server, type Socket } from "socket.io"; import { z } from "zod"; import { env } from "../env.js"; import { register } from "../metrics.js"; +import { + verifyDeploymentIdHeader, + workloadTokenEnforced, + workloadTokensEnabled, +} from "../workloadToken.js"; import { ComputeSnapshotService, type RunTraceContext, @@ -86,6 +91,7 @@ type WorkloadServerOptions = { checkpointClient?: CheckpointClient; computeManager?: ComputeWorkloadManager; tracing?: OtlpTraceService; + snapshotCallbackSecret: string; wideEventOpts: WideEventOptions; /** When true, high-frequency HTTP routes also emit wide events. */ wideEventsNoisyRoutes: boolean; @@ -136,6 +142,7 @@ export class WorkloadServer extends EventEmitter { workerClient: opts.workerClient, tracing: opts.tracing, wideEventOpts: this.wideEventOpts, + snapshotCallbackSecret: opts.snapshotCallbackSecret, }); } @@ -169,6 +176,34 @@ export class WorkloadServer extends EventEmitter { return this.headerValueFromRequest(req, WORKLOAD_HEADERS.PROJECT_REF); } + /** + * Verify the deployment token from the workload deployment-id header and return the verified + * environment_id to forward upstream. The env id is only forwarded in enforce mode: in log mode + * we still verify + record metrics but attach no header (so the platform never scopes). Only + * enforce fails a request, and only for a present-but-invalid token; absent and legacy ids pass. + */ + private async authorizeWorkloadRequest( + req: IncomingMessage + ): Promise<{ ok: true; environmentId?: string } | { ok: false }> { + if (!workloadTokensEnabled) { + return { ok: true }; + } + + const result = await verifyDeploymentIdHeader(this.deploymentIdFromRequest(req), "http"); + + if (result.outcome === "jwt_invalid" && workloadTokenEnforced) { + return { ok: false }; + } + + return { + ok: true, + environmentId: + workloadTokenEnforced && result.outcome === "jwt_valid" + ? result.claims.environment_id + : undefined, + }; + } + /** * Sets common route meta on the wide-event state from URL params. */ @@ -250,11 +285,17 @@ export class WorkloadServer extends EventEmitter { "POST", async () => { const { req, reply, params, body } = ctx; + const auth = await this.authorizeWorkloadRequest(req); + if (!auth.ok) { + reply.empty(401); + return; + } const startResponse = await this.workerClient.startRunAttempt( params.runFriendlyId, params.snapshotFriendlyId, body, - this.runnerIdFromRequest(req) + this.runnerIdFromRequest(req), + auth.environmentId ); if (!startResponse.success) { @@ -286,6 +327,11 @@ export class WorkloadServer extends EventEmitter { "POST", async () => { const { req, reply, params, body } = ctx; + const auth = await this.authorizeWorkloadRequest(req); + if (!auth.ok) { + reply.empty(401); + return; + } const runnerId = this.runnerIdFromRequest(req); // A completion attempt invalidates any pending delayed snapshot @@ -304,7 +350,8 @@ export class WorkloadServer extends EventEmitter { params.runFriendlyId, params.snapshotFriendlyId, body, - runnerId + runnerId, + auth.environmentId ); if (!completeResponse.success) { @@ -336,6 +383,11 @@ export class WorkloadServer extends EventEmitter { "POST", async () => { const { req, reply, params, body } = ctx; + const auth = await this.authorizeWorkloadRequest(req); + if (!auth.ok) { + reply.empty(401); + return; + } const heartbeatResponse = await this.workerClient.heartbeatRun( params.runFriendlyId, params.snapshotFriendlyId, @@ -373,6 +425,11 @@ export class WorkloadServer extends EventEmitter { "GET", async () => { const { reply, params, req } = ctx; + const auth = await this.authorizeWorkloadRequest(req); + if (!auth.ok) { + reply.empty(401); + return; + } const runnerId = this.runnerIdFromRequest(req); const deploymentVersion = this.deploymentVersionFromRequest(req); const projectRef = this.projectRefFromRequest(req); @@ -469,6 +526,11 @@ export class WorkloadServer extends EventEmitter { "GET", async () => { const { req, reply, params } = ctx; + const auth = await this.authorizeWorkloadRequest(req); + if (!auth.ok) { + reply.empty(401); + return; + } this.logger.debug("Run continuation request", { params }); // Cancel any pending delayed snapshot for this run @@ -477,7 +539,8 @@ export class WorkloadServer extends EventEmitter { const continuationResult = await this.workerClient.continueRunExecution( params.runFriendlyId, params.snapshotFriendlyId, - this.runnerIdFromRequest(req) + this.runnerIdFromRequest(req), + auth.environmentId ); if (!continuationResult.success) { @@ -511,10 +574,16 @@ export class WorkloadServer extends EventEmitter { "GET", async () => { const { req, reply, params } = ctx; + const auth = await this.authorizeWorkloadRequest(req); + if (!auth.ok) { + reply.empty(401); + return; + } const sinceSnapshotResponse = await this.workerClient.getSnapshotsSince( params.runFriendlyId, params.snapshotFriendlyId, - this.runnerIdFromRequest(req) + this.runnerIdFromRequest(req), + auth.environmentId ); if (!sinceSnapshotResponse.success) { @@ -585,9 +654,18 @@ export class WorkloadServer extends EventEmitter { const { req, reply, params, body } = ctx; reply.empty(204); + // Redact TRIGGER_DEPLOYMENT_ID before relaying to the platform. + const sanitizedBody = + body.properties && "TRIGGER_DEPLOYMENT_ID" in body.properties + ? { + ...body, + properties: { ...body.properties, TRIGGER_DEPLOYMENT_ID: "[redacted]" }, + } + : body; + await this.workerClient.sendDebugLog( params.runFriendlyId, - body, + sanitizedBody, this.runnerIdFromRequest(req) ); }, @@ -681,7 +759,31 @@ export class WorkloadServer extends EventEmitter { return; } - this.logger.debug("[WS] auth success", socket.data); + if (workloadTokensEnabled) { + const result = await verifyDeploymentIdHeader(socket.data.deploymentId, "ws"); + + if (result.outcome === "jwt_invalid" && workloadTokenEnforced) { + this.logger.error("[WS] deployment token verification failed", { + runnerId: socket.data.runnerId, + }); + socket.disconnect(true); + return; + } + + // Re-source the deployment id from the verified claim; the raw header may be an opaque token. + // A legacy bare id is itself the friendlyId, so it's safe to keep. + socket.data.deploymentFriendlyId = + result.outcome === "jwt_valid" + ? result.claims.deployment + : result.outcome === "legacy_bare" + ? socket.data.deploymentId + : undefined; + } + + this.logger.debug("[WS] handshake complete", { + runnerId: socket.data.runnerId, + deploymentFriendlyId: socket.data.deploymentFriendlyId, + }); next(); }); @@ -693,7 +795,7 @@ export class WorkloadServer extends EventEmitter { const getSocketMetadata = () => { return { - deploymentId: socket.data.deploymentId, + deploymentId: socket.data.deploymentFriendlyId ?? socket.data.deploymentId, runId: socket.data.runFriendlyId, snapshotId: socket.data.snapshotId, runnerId: socket.data.runnerId, @@ -712,8 +814,9 @@ export class WorkloadServer extends EventEmitter { populate: (state) => { state.extras.event = event; setMeta(state, "run_id", friendlyId); - if (socket.data.deploymentId) { - setMeta(state, "deployment_id", socket.data.deploymentId); + const deploymentId = socket.data.deploymentFriendlyId ?? socket.data.deploymentId; + if (deploymentId) { + setMeta(state, "deployment_id", deploymentId); } if (socket.data.runnerId) setMeta(state, "runner_id", socket.data.runnerId); state.extras.socket_id = socket.id; @@ -725,6 +828,33 @@ export class WorkloadServer extends EventEmitter { const runConnected = (friendlyId: string) => { socketLogger.debug("runConnected", { ...getSocketMetadata() }); + // Only the owning runner may (re)bind a run. A live socket from a *different* + // runner keeps its binding so an unrelated connection can't hijack the run. But + // the newest socket for the *same* runner is a legitimate reconnection/handoff and + // is allowed to take over even while the stale socket still reports connected - + // otherwise, during a reconnect race the fresh socket would silently stay unbound + // (missing continue/cancel/suspend notifications) until the dead socket times out. + const existing = this.runSockets.get(friendlyId); + if (existing && existing.id !== socket.id && existing.connected) { + const sameRunner = + !!socket.data.runnerId && existing.data.runnerId === socket.data.runnerId; + + if (!sameRunner) { + socketLogger.warn("runConnected: run already bound to another socket", { + ...getSocketMetadata(), + friendlyId, + existingSocketId: existing.id, + }); + return; + } + + socketLogger.debug("runConnected: replacing stale socket for same runner", { + ...getSocketMetadata(), + friendlyId, + existingSocketId: existing.id, + }); + } + // If there's already a run ID set, we should "disconnect" it from this socket if (socket.data.runFriendlyId && socket.data.runFriendlyId !== friendlyId) { socketLogger.debug("runConnected: disconnecting existing run", { @@ -744,6 +874,22 @@ export class WorkloadServer extends EventEmitter { const runDisconnected = (friendlyId: string, reason: string) => { socketLogger.debug("runDisconnected", { ...getSocketMetadata() }); + // A newer socket may have taken over this run (same-runner reconnect race). If the + // run is now bound to a different socket, this stale socket must not clear the fresh + // binding or emit a spurious disconnect - just drop its own reference and bail. + const bound = this.runSockets.get(friendlyId); + if (bound && bound.id !== socket.id) { + socketLogger.debug("runDisconnected: run rebound to another socket, skipping", { + ...getSocketMetadata(), + friendlyId, + boundSocketId: bound.id, + }); + if (socket.data.runFriendlyId === friendlyId) { + socket.data.runFriendlyId = undefined; + } + return; + } + // The run is gone from this runner (crash, exit, or replaced by a new // run), so a pending delayed snapshot for it is stale. Genuine // waitpoint suspensions keep the socket connected, so this doesn't diff --git a/apps/supervisor/src/workloadServer/workloadServer.auth.test.ts b/apps/supervisor/src/workloadServer/workloadServer.auth.test.ts new file mode 100644 index 000000000..3e8fb6e9d --- /dev/null +++ b/apps/supervisor/src/workloadServer/workloadServer.auth.test.ts @@ -0,0 +1,107 @@ +import { mintWorkloadDeploymentToken } from "@trigger.dev/core/v3"; +import { WORKLOAD_HEADERS } from "@trigger.dev/core/v3/workers"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +// Set enforce mode + secret before env.ts parses (vi.mock is hoisted above imports, so the secret +// must be a literal here). SECRET below mirrors it for use in the test body. +vi.mock("std-env", () => ({ + env: { + TRIGGER_API_URL: "http://localhost:3030", + TRIGGER_WORKER_TOKEN: "test-token", + MANAGED_WORKER_SECRET: "test-secret", + OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318", + WORKLOAD_TOKEN_SECRET: "integration-test-secret", + WORKLOAD_TOKEN_ENFORCEMENT: "enforce", + }, +})); + +const SECRET = "integration-test-secret"; +const EXP = Math.floor(Date.UTC(2032, 0, 1) / 1000); + +const { WorkloadServer } = await import("./index.js"); + +const PORT = 18732; +const BASE = `http://127.0.0.1:${PORT}`; + +function claims(environmentId = "env_test_123") { + return { + deployment: "deployment_test", + deployment_version: "20260710.1", + environment_id: environmentId, + environment_type: "PRODUCTION", + org_id: "org_1", + project_id: "proj_1", + }; +} + +// Records the args each relay method is called with so we can assert the forwarded claim. +const calls: { getSnapshotsSince: any[][] } = { getSnapshotsSince: [] }; + +const workerClient = { + getSnapshotsSince: vi.fn(async (...args: any[]) => { + calls.getSnapshotsSince.push(args); + return { success: true as const, data: { snapshots: [] } }; + }), +} as any; + +let server: InstanceType; + +beforeAll(async () => { + server = new WorkloadServer({ + port: PORT, + workerClient, + snapshotCallbackSecret: "snapshot-callback-secret", + wideEventOpts: { service: "supervisor", env: { nodeId: "test" }, enabled: false }, + wideEventsNoisyRoutes: false, + }); + await server.start(); +}); + +afterAll(async () => { + await server.stop(); +}); + +function snapshotsSince(deploymentIdHeader?: string) { + const headers: Record = { + [WORKLOAD_HEADERS.RUNNER_ID]: "runner_1", + }; + if (deploymentIdHeader !== undefined) { + headers[WORKLOAD_HEADERS.DEPLOYMENT_ID] = deploymentIdHeader; + } + return fetch(`${BASE}/api/v1/workload-actions/runs/run_1/snapshots/since/snap_1`, { headers }); +} + +describe("WorkloadServer auth (enforce mode)", () => { + it("allows a valid token and forwards the verified environment_id", async () => { + const token = await mintWorkloadDeploymentToken(claims("env_forwarded_42"), SECRET, EXP); + const res = await snapshotsSince(token); + + expect(res.status).toBe(200); + const lastCall = calls.getSnapshotsSince.at(-1)!; + // getSnapshotsSince(runId, snapshotId, runnerId, environmentId) + expect(lastCall[3]).toBe("env_forwarded_42"); + }); + + it("rejects a token signed with the wrong secret (401) and does not relay", async () => { + const before = calls.getSnapshotsSince.length; + const badToken = await mintWorkloadDeploymentToken(claims(), "wrong-secret", EXP); + const res = await snapshotsSince(badToken); + + expect(res.status).toBe(401); + expect(calls.getSnapshotsSince.length).toBe(before); + }); + + it("allows a legacy bare friendlyId and forwards no environment_id", async () => { + const res = await snapshotsSince("deployment_legacy_bare"); + + expect(res.status).toBe(200); + expect(calls.getSnapshotsSince.at(-1)![3]).toBeUndefined(); + }); + + it("allows an absent token and forwards no environment_id", async () => { + const res = await snapshotsSince(undefined); + + expect(res.status).toBe(200); + expect(calls.getSnapshotsSince.at(-1)![3]).toBeUndefined(); + }); +}); diff --git a/apps/supervisor/src/workloadServer/workloadServer.authLog.test.ts b/apps/supervisor/src/workloadServer/workloadServer.authLog.test.ts new file mode 100644 index 000000000..8e66f738f --- /dev/null +++ b/apps/supervisor/src/workloadServer/workloadServer.authLog.test.ts @@ -0,0 +1,87 @@ +import { mintWorkloadDeploymentToken } from "@trigger.dev/core/v3"; +import { WORKLOAD_HEADERS } from "@trigger.dev/core/v3/workers"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +// Log mode: mint + verify + metrics, but the platform must NOT be scoped, so no environment_id is +// forwarded even for a valid token. (vi.mock is hoisted; secret literal here, mirrored below.) +vi.mock("std-env", () => ({ + env: { + TRIGGER_API_URL: "http://localhost:3030", + TRIGGER_WORKER_TOKEN: "test-token", + MANAGED_WORKER_SECRET: "test-secret", + OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318", + WORKLOAD_TOKEN_SECRET: "integration-test-secret", + WORKLOAD_TOKEN_ENFORCEMENT: "log", + }, +})); + +const SECRET = "integration-test-secret"; +const EXP = Math.floor(Date.UTC(2032, 0, 1) / 1000); + +const { WorkloadServer } = await import("./index.js"); + +const PORT = 18733; +const BASE = `http://127.0.0.1:${PORT}`; + +const claims = { + deployment: "deployment_test", + deployment_version: "20260710.1", + environment_id: "env_should_not_forward", + environment_type: "PRODUCTION", + org_id: "org_1", + project_id: "proj_1", +}; + +const calls: { getSnapshotsSince: any[][] } = { getSnapshotsSince: [] }; + +const workerClient = { + getSnapshotsSince: vi.fn(async (...args: any[]) => { + calls.getSnapshotsSince.push(args); + return { success: true as const, data: { snapshots: [] } }; + }), +} as any; + +let server: InstanceType; + +beforeAll(async () => { + server = new WorkloadServer({ + port: PORT, + workerClient, + snapshotCallbackSecret: "snapshot-callback-secret", + wideEventOpts: { service: "supervisor", env: { nodeId: "test" }, enabled: false }, + wideEventsNoisyRoutes: false, + }); + await server.start(); +}); + +afterAll(async () => { + await server.stop(); +}); + +describe("WorkloadServer auth (log mode)", () => { + it("allows a valid token but forwards no environment_id", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const res = await fetch(`${BASE}/api/v1/workload-actions/runs/run_1/snapshots/since/snap_1`, { + headers: { + [WORKLOAD_HEADERS.RUNNER_ID]: "runner_1", + [WORKLOAD_HEADERS.DEPLOYMENT_ID]: token, + }, + }); + + expect(res.status).toBe(200); + expect(calls.getSnapshotsSince.at(-1)![3]).toBeUndefined(); + }); + + it("does not reject an invalid token in log mode", async () => { + const badToken = await mintWorkloadDeploymentToken(claims, "wrong-secret", EXP); + const res = await fetch(`${BASE}/api/v1/workload-actions/runs/run_1/snapshots/since/snap_1`, { + headers: { + [WORKLOAD_HEADERS.RUNNER_ID]: "runner_1", + [WORKLOAD_HEADERS.DEPLOYMENT_ID]: badToken, + }, + }); + + expect(res.status).toBe(200); + expect(calls.getSnapshotsSince.at(-1)![3]).toBeUndefined(); + }); +}); diff --git a/apps/supervisor/src/workloadToken.ts b/apps/supervisor/src/workloadToken.ts new file mode 100644 index 000000000..914411dd7 --- /dev/null +++ b/apps/supervisor/src/workloadToken.ts @@ -0,0 +1,85 @@ +import { + classifyDeploymentIdHeader, + mintWorkloadDeploymentToken, + type WorkloadDeploymentTokenClaims, + type WorkloadDeploymentTokenInput, +} from "@trigger.dev/core/v3"; +import { Counter } from "prom-client"; +import { env } from "./env.js"; +import { register } from "./metrics.js"; + +const secret = env.WORKLOAD_TOKEN_SECRET; + +// Absolute expiry (epoch seconds) shared by every mint, so tokens stay byte-deterministic per +// deployment regardless of when/where a pod is created. +const tokenExpSeconds = Math.floor(new Date(env.WORKLOAD_TOKEN_EXP).getTime() / 1000); + +/** Mint + verify run in "log" (dry-run) and "enforce"; the env superRefine guarantees a secret then. */ +export const workloadTokensEnabled = env.WORKLOAD_TOKEN_ENFORCEMENT !== "disabled"; + +/** Only "enforce" rejects a present-but-invalid token; "log" observes and always allows. */ +export const workloadTokenEnforced = env.WORKLOAD_TOKEN_ENFORCEMENT === "enforce"; + +const mintCounter = new Counter({ + name: "workload_token_minted_total", + help: "Deployment tokens minted and injected into TRIGGER_DEPLOYMENT_ID at pod creation", + labelNames: ["env_type"] as const, + registers: [register], +}); + +export type WorkloadAuthTransport = "http" | "ws"; +export type WorkloadAuthOutcome = "jwt_valid" | "jwt_invalid" | "legacy_bare" | "token_absent"; + +const verifyCounter = new Counter({ + name: "workload_auth_verify_total", + help: "Runner-boundary token verification outcomes at the supervisor workload server", + labelNames: ["outcome", "transport", "env_type"] as const, + registers: [register], +}); + +export async function mintDeploymentToken( + claims: WorkloadDeploymentTokenInput +): Promise { + if (!workloadTokensEnabled || !secret) { + return undefined; + } + + const token = await mintWorkloadDeploymentToken(claims, secret, tokenExpSeconds); + mintCounter.inc({ env_type: claims.environment_type }); + return token; +} + +export type VerifiedDeploymentHeader = + | { outcome: "jwt_valid"; claims: WorkloadDeploymentTokenClaims } + | { outcome: "jwt_invalid" | "legacy_bare" | "token_absent"; claims?: undefined }; + +/** + * Verify the deployment-id header value and record the outcome. "jwt_valid" returns the claims so the + * caller can forward the verified environment_id upstream; other outcomes carry no trusted data. + */ +export async function verifyDeploymentIdHeader( + value: string | undefined, + transport: WorkloadAuthTransport +): Promise { + const result = await classify(value); + verifyCounter.inc({ + outcome: result.outcome, + transport, + env_type: result.outcome === "jwt_valid" ? result.claims.environment_type : "unknown", + }); + return result; +} + +async function classify(value: string | undefined): Promise { + if (!value || !secret) { + return { outcome: "token_absent" }; + } + + const result = await classifyDeploymentIdHeader(value, secret); + + if (result.outcome === "jwt_valid" && result.claims) { + return { outcome: "jwt_valid", claims: result.claims }; + } + + return { outcome: result.outcome === "jwt_valid" ? "jwt_invalid" : result.outcome }; +} diff --git a/apps/webapp/app/entry.server.tsx b/apps/webapp/app/entry.server.tsx index 672916d81..da1ab5a43 100644 --- a/apps/webapp/app/entry.server.tsx +++ b/apps/webapp/app/entry.server.tsx @@ -300,6 +300,7 @@ singleton("SentryTenantContextProcessor", () => { export { apiRateLimiter } from "./services/apiRateLimit.server"; export { engineRateLimiter } from "./services/engineRateLimit.server"; +export { otlpRateLimiter } from "./services/otlpRateLimit.server"; export { runWithHttpContext } from "./services/httpAsyncStorage.server"; export { tenantContextMiddleware } from "./services/tenantContextResolver.server"; export { socketIo } from "./v3/handleSocketIo.server"; diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 973560ca0..53b5edf0a 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -85,6 +85,29 @@ const S2EnvSchema = z.preprocess( ]) ); +// Previously published secret values must never be accepted, including when +// an existing deployment or external secret manager still supplies one. +const INSECURE_SECRET_VALUES = [ + "managed-secret", + "2818143646516f6fffd707b36f334bbb", + "44da78b7bbb0dfe709cf38931d25dcdd", + "f686147ab967943ebbe9ed3b496e465a", + "447c29678f9eaf289e9c4b70d3dd8a7f", +]; + +// Escape hatch for deployments that can't rotate a published default yet (e.g. +// ENCRYPTION_KEY protects existing data). Read raw: a refine can't see the +// sibling parsed flag. +const allowInsecureDefaultSecrets = ["true", "1"].includes( + (process.env.ALLOW_INSECURE_DEFAULT_SECRETS ?? "").toLowerCase().trim() +); + +const isNotInsecureSecret = (value: string) => + allowInsecureDefaultSecrets || !INSECURE_SECRET_VALUES.includes(value); + +const INSECURE_SECRET_MESSAGE = + "must not be a known-insecure published default; set a strong, unique value. If you cannot rotate it yet (e.g. it protects existing encrypted data or active sessions), set ALLOW_INSECURE_DEFAULT_SECRETS=1 to boot while you migrate."; + const EnvironmentSchema = z .object({ NODE_ENV: z.union([z.literal("development"), z.literal("production"), z.literal("test")]), @@ -188,14 +211,15 @@ const EnvironmentSchema = z // Control-plane cache relax knobs. Unset -> defaults (DEFAULT_CP_CACHE_TTL_MS / _MAX_ENTRIES). CONTROL_PLANE_CACHE_TTL_MS: z.coerce.number().int().optional(), CONTROL_PLANE_CACHE_MAX_ENTRIES: z.coerce.number().int().optional(), - SESSION_SECRET: z.string(), - MAGIC_LINK_SECRET: z.string(), + SESSION_SECRET: z.string().min(1).refine(isNotInsecureSecret, INSECURE_SECRET_MESSAGE), + MAGIC_LINK_SECRET: z.string().min(1).refine(isNotInsecureSecret, INSECURE_SECRET_MESSAGE), ENCRYPTION_KEY: z .string() .refine( (val) => Buffer.from(val, "utf8").length === 32, "ENCRYPTION_KEY must be exactly 32 bytes" - ), + ) + .refine(isNotInsecureSecret, INSECURE_SECRET_MESSAGE), WHITELISTED_EMAILS: z .string() .refine(isValidRegex, "WHITELISTED_EMAILS must be a valid regex.") @@ -547,6 +571,22 @@ const EnvironmentSchema = z API_RATE_LIMIT_JWT_WINDOW: z.string().default("1m"), API_RATE_LIMIT_JWT_TOKENS: z.coerce.number().int().default(60), + // Per-IP rate limit for the unauthenticated OTLP ingestion endpoints + // (/otel/*). Bounds unauthenticated request rates. Opt-in + // (disabled by default): because it keys on the source IP, it is only + // safe to enable when each client presents a distinct IP through a proxy + // that appends the real client IP to X-Forwarded-For. Enabling it where + // many clients share one egress IP (e.g. behind NAT or a shared proxy) + // would collapse that traffic into a single bucket and could throttle + // legitimate telemetry. Set OTLP_RATE_LIMIT_ENABLED=1 to enable, then tune + // OTLP_RATE_LIMIT_MAX / OTLP_RATE_LIMIT_WINDOW for expected volume. + OTLP_RATE_LIMIT_ENABLED: z.string().default("0"), + OTLP_RATE_LIMIT_WINDOW: z + .string() + .regex(/^\d+ ?(?:ms|s|m|h|d)$/) + .default("1m"), + OTLP_RATE_LIMIT_MAX: z.coerce.number().int().positive().default(3000), + DEPOT_TOKEN: z.string().optional(), DEPOT_ORG_ID: z.string().optional(), DEPOT_REGION: z.string().default("us-east-1"), @@ -668,7 +708,18 @@ const EnvironmentSchema = z EVENTS_LOAD_SHEDDING_THRESHOLD: z.coerce.number().int().default(100000), EVENTS_LOAD_SHEDDING_ENABLED: z.string().default("1"), - MANAGED_WORKER_SECRET: z.string().default("managed-secret"), + MANAGED_WORKER_SECRET: z.string().min(1).refine(isNotInsecureSecret, INSECURE_SECRET_MESSAGE), + + // Allow booting with a known-insecure published default secret. Temporary + // bridge for deployments that can't rotate yet; rotate as soon as possible. + ALLOW_INSECURE_DEFAULT_SECRETS: BoolEnv.default(false), + + // Tenant scoping on worker actions is header-driven (folded into the engine snapshot read) and + // needs no flag. This is only the no-header fallback: when "1", a worker action on a run created + // after WORKLOAD_TOKEN_CUTOFF without a verified env header is rejected; runs on or before the + // cutoff pass (grandfathered). Default off = no run-row read, byte-for-byte today's behavior. + WORKLOAD_CREATED_AT_GATE_ENABLED: z.string().default("0"), + WORKLOAD_TOKEN_CUTOFF: z.string().datetime().optional(), // Development OTEL environment variables DEV_OTEL_EXPORTER_OTLP_ENDPOINT: z.string().optional(), @@ -2090,3 +2141,24 @@ const EnvironmentSchema = z export type Environment = z.infer; export const env = EnvironmentSchema.parse(process.env); + +if (env.ALLOW_INSECURE_DEFAULT_SECRETS) { + const insecure = ( + [ + ["SESSION_SECRET", env.SESSION_SECRET], + ["MAGIC_LINK_SECRET", env.MAGIC_LINK_SECRET], + ["ENCRYPTION_KEY", env.ENCRYPTION_KEY], + ["MANAGED_WORKER_SECRET", env.MANAGED_WORKER_SECRET], + ] as const + ) + .filter(([, value]) => INSECURE_SECRET_VALUES.includes(value)) + .map(([name]) => name); + + if (insecure.length > 0) { + console.warn( + `⚠️ ALLOW_INSECURE_DEFAULT_SECRETS is enabled and these secrets still use a known-insecure published default: ${insecure.join( + ", " + )}. This is insecure - rotate them as soon as you can.` + ); + } +} diff --git a/apps/webapp/app/models/schedules.server.ts b/apps/webapp/app/models/schedules.server.ts index 72d08c4fb..b9d72665c 100644 --- a/apps/webapp/app/models/schedules.server.ts +++ b/apps/webapp/app/models/schedules.server.ts @@ -1,4 +1,4 @@ -import type { Prisma } from "~/db.server"; +import type { Prisma, PrismaClientOrTransaction, TaskSchedule } from "@trigger.dev/database"; export function scheduleUniqWhereClause( projectId: string, @@ -35,3 +35,48 @@ export function scheduleWhereClause( deduplicationKey: scheduleId, }; } + +/** + * Resolve a schedule's visibility for an environment-scoped caller. + * + * - "visible": the schedule exists in the project and has at least one + * instance bound to `environmentId` (or has no instances yet). + * - "hidden": the schedule exists but none of its instances live in the + * caller's environment. + * - "missing": no schedule exists for the (project, scheduleId) pair. + * + * A schedule can be bound to several environments at once, so visibility + * mirrors the "some instance is in this environment" rule the schedule + * list uses: a schedule that is listed for a key must also be readable + * and mutable by that key. This still rejects cross-environment access to + * schedules the caller has no instance in, and `scheduleWhereClause` + * already confines the lookup to the caller's project. + * + * The tri-state lets PUT (upsert) disambiguate "hidden" (refuse) from + * "missing" (fall through to create). DELETE/GET treat hidden and + * missing the same way. + */ +export type ScheduleEnvVisibility = + | { status: "visible"; schedule: TaskSchedule } + | { status: "hidden" } + | { status: "missing" }; + +export async function getScheduleEnvVisibility( + prisma: PrismaClientOrTransaction, + projectId: string, + scheduleId: string, + environmentId: string +): Promise { + const schedule = await prisma.taskSchedule.findFirst({ + where: scheduleWhereClause(projectId, scheduleId), + include: { instances: { select: { environmentId: true } } }, + }); + + if (!schedule) return { status: "missing" }; + + const { instances, ...rest } = schedule; + if (instances.length === 0) return { status: "visible", schedule: rest }; + const scoped = instances.some((i) => i.environmentId === environmentId); + if (!scoped) return { status: "hidden" }; + return { status: "visible", schedule: rest }; +} diff --git a/apps/webapp/app/routes/_app.github.callback/route.tsx b/apps/webapp/app/routes/_app.github.callback/route.tsx index cd67f27f5..94e13ae18 100644 --- a/apps/webapp/app/routes/_app.github.callback/route.tsx +++ b/apps/webapp/app/routes/_app.github.callback/route.tsx @@ -1,6 +1,9 @@ import { type LoaderFunctionArgs } from "@remix-run/node"; import { z } from "zod"; -import { validateGitHubAppInstallSession } from "~/services/gitHubSession.server"; +import { + destroyGitHubAppInstallSession, + validateGitHubAppInstallSession, +} from "~/services/gitHubSession.server"; import { linkGitHubAppInstallation, updateGitHubAppInstallation } from "~/services/gitHub.server"; import { logger } from "~/services/logger.server"; import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server"; @@ -75,6 +78,15 @@ export async function loader({ request }: LoaderFunctionArgs) { return redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub app"); } + // The install session is single-use: once a callback consumes an + // installation_id, invalidate the state cookie so the same initiation + // cannot be replayed against other installation_ids. + const clearInstallSession = await destroyGitHubAppInstallSession(cookieHeader); + const consumingSession = (response: Response) => { + response.headers.append("Set-Cookie", clearInstallSession); + return response; + }; + switch (callbackData.setup_action) { case "install": { const [error] = await tryCatch( @@ -85,23 +97,33 @@ export async function loader({ request }: LoaderFunctionArgs) { logger.error("Failed to link GitHub App installation", { error, }); - return redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub app"); + return consumingSession( + await redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub app") + ); } - return redirectWithSuccessMessage(redirectTo, request, "GitHub App installed successfully"); + return consumingSession( + await redirectWithSuccessMessage(redirectTo, request, "GitHub App installed successfully") + ); } case "update": { - const [error] = await tryCatch(updateGitHubAppInstallation(callbackData.installation_id)); + const [error] = await tryCatch( + updateGitHubAppInstallation(callbackData.installation_id, organizationId) + ); if (error) { logger.error("Failed to update GitHub App installation", { error, }); - return redirectWithErrorMessage(redirectTo, request, "Failed to update GitHub App"); + return consumingSession( + await redirectWithErrorMessage(redirectTo, request, "Failed to update GitHub App") + ); } - return redirectWithSuccessMessage(redirectTo, request, "GitHub App updated successfully"); + return consumingSession( + await redirectWithSuccessMessage(redirectTo, request, "GitHub App updated successfully") + ); } case "request": { diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx index d552b3104..02d6ef00c 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx @@ -55,6 +55,7 @@ import { } from "~/utils/pathBuilder"; import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; import { EnvironmentVariableKey } from "~/v3/environmentVariables/repository"; +import { findUnauthorizedEnvironmentId } from "~/v3/writableEnvironments"; const Variable = z.object({ key: EnvironmentVariableKey, @@ -164,6 +165,31 @@ export const action = dashboardAction( return json(submission.reply({ formErrors: ["Project not found"] })); } + // The submitted `environmentIds` are user-supplied. Shared env types are + // writable by any member; a DEV env only by its owner. See + // findUnauthorizedEnvironmentId. + const submittedEnvs = await prisma.runtimeEnvironment.findMany({ + where: { + projectId: project.id, + id: { in: submission.value.environmentIds }, + }, + select: { id: true, type: true, orgMember: { select: { userId: true } } }, + }); + const unauthorizedEnvironmentId = findUnauthorizedEnvironmentId( + submittedEnvs, + submission.value.environmentIds, + userId + ); + if (unauthorizedEnvironmentId) { + return json( + submission.reply({ + fieldErrors: { + environmentIds: ["One or more of the selected environments is not writable by you."], + }, + }) + ); + } + const repository = new EnvironmentVariablesRepository(prisma); const result = await repository.create(project.id, { ...submission.value, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx index b0b2a8931..cdbe364d8 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx @@ -78,6 +78,7 @@ import { v3NewEnvironmentVariablesPath, } from "~/utils/pathBuilder"; import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; +import { findUnauthorizedEnvironmentId } from "~/v3/writableEnvironments"; import { DeleteEnvironmentVariableValue, EditEnvironmentVariableValue, @@ -267,6 +268,24 @@ export const action = dashboardAction( return json(submission.reply({ formErrors: ["Project not found"] })); } + // Per-env write gate for the mutating value actions: `environmentId` is a + // user-supplied hidden field and the repository only checks project + // membership. Mirrors the create route's check. + if (submission.value.action === "edit" || submission.value.action === "delete") { + const submittedEnvs = await prisma.runtimeEnvironment.findMany({ + where: { projectId: project.id, id: submission.value.environmentId }, + select: { id: true, type: true, orgMember: { select: { userId: true } } }, + }); + const unauthorizedEnvironmentId = findUnauthorizedEnvironmentId( + submittedEnvs, + [submission.value.environmentId], + userId + ); + if (unauthorizedEnvironmentId) { + return json(submission.reply({ formErrors: ["This environment is not writable by you."] })); + } + } + switch (submission.value.action) { case "edit": { const repository = new EnvironmentVariablesRepository(prisma); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules.$scheduleParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules.$scheduleParam/route.tsx index cd433467c..a3a0ef054 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules.$scheduleParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules.$scheduleParam/route.tsx @@ -6,7 +6,6 @@ import { z } from "zod"; import { ExitIcon } from "~/assets/icons/ExitIcon"; import { LinkButton } from "~/components/primitives/Buttons"; import { ScheduleInspector } from "~/components/schedules/ScheduleInspector"; -import { prisma } from "~/db.server"; import { useEnvironment } from "~/hooks/useEnvironment"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; @@ -78,11 +77,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { // `_format=json` → return JSON instead of redirecting; caller stays put. const wantsJson = formData.get("_format") === "json"; - const project = await prisma.project.findFirst({ - where: { - slug: projectParam, - }, - }); + const project = await findProjectBySlug(organizationSlug, projectParam, userId); if (!project) { const message = `No project found with slug ${projectParam}`; diff --git a/apps/webapp/app/routes/account.authorization-code.$authorizationCode/route.tsx b/apps/webapp/app/routes/account.authorization-code.$authorizationCode/route.tsx index 5bd9550e6..067f6af44 100644 --- a/apps/webapp/app/routes/account.authorization-code.$authorizationCode/route.tsx +++ b/apps/webapp/app/routes/account.authorization-code.$authorizationCode/route.tsx @@ -1,14 +1,19 @@ import { CheckCircleIcon } from "@heroicons/react/24/solid"; -import type { LoaderFunctionArgs } from "@remix-run/server-runtime"; -import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { Form } from "@remix-run/react"; +import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { typedjson, useTypedActionData, useTypedLoaderData } from "remix-typedjson"; import { z } from "zod"; import { AppContainer, MainCenteredContainer } from "~/components/layout/AppLayout"; +import { Button } from "~/components/primitives/Buttons"; import { Callout } from "~/components/primitives/Callout"; import { Header1 } from "~/components/primitives/Headers"; import { Icon } from "~/components/primitives/Icon"; import { Paragraph } from "~/components/primitives/Paragraph"; import { logger } from "~/services/logger.server"; -import { createPersonalAccessTokenFromAuthorizationCode } from "~/services/personalAccessToken.server"; +import { + createPersonalAccessTokenFromAuthorizationCode, + isAuthorizationCodeMintable, +} from "~/services/personalAccessToken.server"; import { requireUserId } from "~/services/session.server"; const ParamsSchema = z.object({ @@ -20,49 +25,57 @@ const SearchParamsSchema = z.object({ clientName: z.string().optional(), }); -export const loader = async ({ request, params }: LoaderFunctionArgs) => { - const userId = await requireUserId(request); - +function parseParams(params: unknown) { const parsedParams = ParamsSchema.safeParse(params); - if (!parsedParams.success) { logger.info("Invalid params", { params }); - throw new Response(undefined, { - status: 400, - statusText: "Invalid params", - }); + throw new Response(undefined, { status: 400, statusText: "Invalid params" }); } + return parsedParams.data; +} +function parseSearch(request: Request) { const url = new URL(request.url); const searchObject = Object.fromEntries(url.searchParams.entries()); - const searchParams = SearchParamsSchema.safeParse(searchObject); - const source = (searchParams.success ? searchParams.data.source : undefined) ?? "cli"; const clientName = (searchParams.success ? searchParams.data.clientName : undefined) ?? "unknown"; + return { source, clientName }; +} + +// The loader only renders a consent screen; minting/binding a PAT happens in +// the `action`, behind an explicit "Authorize" POST. +export const loader = async ({ request, params }: LoaderFunctionArgs) => { + await requireUserId(request); + + const { authorizationCode } = parseParams(params); + const { source, clientName } = parseSearch(request); + + const mintable = await isAuthorizationCodeMintable(authorizationCode); + + return typedjson({ + status: mintable ? ("consent" as const) : ("invalid" as const), + source, + clientName, + }); +}; + +export const action = async ({ request, params }: ActionFunctionArgs) => { + const userId = await requireUserId(request); + + const { authorizationCode } = parseParams(params); + const { source, clientName } = parseSearch(request); try { - const _personalAccessToken = await createPersonalAccessTokenFromAuthorizationCode( - parsedParams.data.authorizationCode, - userId - ); - return typedjson({ - success: true as const, - source, - clientName, - }); + await createPersonalAccessTokenFromAuthorizationCode(authorizationCode, userId); + return typedjson({ success: true as const, source, clientName }); } catch (error) { if (error instanceof Response) { throw error; } if (error instanceof Error) { - return typedjson({ - success: false as const, - error: error.message, - source, - clientName, - }); + return typedjson({ success: false as const, error: error.message, source, clientName }); } logger.error(JSON.stringify(error)); @@ -74,32 +87,78 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }; export default function Page() { - const result = useTypedLoaderData(); + const loaderData = useTypedLoaderData(); + const actionData = useTypedActionData(); + // After the consent POST: success or failure. + if (actionData) { + return ( + + {actionData.success ? ( +
+ + Successfully + authenticated + + + {getInstructionsForSource(actionData.source, actionData.clientName)} + +
+ ) : ( +
+ Authentication failed + + {actionData.error} + + + There was a problem authenticating you, please try logging in with your CLI again. + +
+ )} + + ); + } + + // Initial GET: invalid/expired code, or the consent prompt. + if (loaderData.status === "invalid") { + return ( + +
+ Authentication failed + + This login link is invalid or has expired. + + + Please try logging in with your CLI again to get a fresh link. + +
+ + ); + } + + return ( + +
+ Authorize login + {getConsentPrompt(loaderData.source, loaderData.clientName)} +
+ +
+ + Only authorize if you started this login yourself. If you didn't, close this page. + +
+ + ); +} + +function AuthShell({ children }: { children: React.ReactNode }) { return ( -
- {result.success ? ( -
- - Successfully - authenticated - - {getInstructionsForSource(result.source, result.clientName)} -
- ) : ( -
- Authentication failed - - {result.error} - - - There was a problem authenticating you, please try logging in with your CLI again. - -
- )} -
+
{children}
 ); @@ -113,6 +172,18 @@ const prettyClientNames: Record = { "claude-ai": "Claude Desktop", }; +function getConsentPrompt(source: string, clientName: string) { + if (source === "mcp") { + const pretty = prettyClientNames[clientName] ?? clientName; + if (pretty && pretty !== "unknown") { + return `Authorize ${pretty} to access your Trigger.dev account?`; + } + return `Authorize this MCP client to access your Trigger.dev account?`; + } + + return `Authorize the Trigger.dev CLI to access your account?`; +} + function getInstructionsForSource(source: string, clientName: string) { if (source === "mcp") { if (clientName) { diff --git a/apps/webapp/app/routes/api.v1.authorization-code.ts b/apps/webapp/app/routes/api.v1.authorization-code.ts index 1a56ef040..fa7a36673 100644 --- a/apps/webapp/app/routes/api.v1.authorization-code.ts +++ b/apps/webapp/app/routes/api.v1.authorization-code.ts @@ -3,7 +3,12 @@ import { json } from "@remix-run/server-runtime"; import type { CreateAuthorizationCodeResponse } from "@trigger.dev/core/v3"; import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; +import { + AuthorizationCodeRateLimitError, + checkAuthorizationCodeMintRateLimit, +} from "~/services/authCodeRateLimiter.server"; import { createAuthorizationCode } from "~/services/personalAccessToken.server"; +import { extractClientIp } from "~/utils/extractClientIp.server"; /** Used to create an AuthorizationCode, that can then be used to obtain a Personal Access Token by logging in with the provided URL */ export async function action({ request }: ActionFunctionArgs) { @@ -14,8 +19,24 @@ export async function action({ request }: ActionFunctionArgs) { return { status: 405, body: "Method Not Allowed" }; } - //there is no authentication on this endpoint, anyone can create an AuthorizationCode. - //they're only used to allow a user to login, when they'll then receive a Personal Access Token + //this endpoint is unauthenticated (codes only allow a user to log in), so it's + //rate-limited per client IP. Keyed by X-Forwarded-For; if there's no trustworthy + //client IP we skip the limit rather than bucket everyone together. Self-hosters + //wanting per-IP limiting should front the app with a proxy that sets X-Forwarded-For. + const clientIp = extractClientIp(request.headers.get("x-forwarded-for")); + if (clientIp) { + try { + await checkAuthorizationCodeMintRateLimit(clientIp); + } catch (error) { + if (error instanceof AuthorizationCodeRateLimitError) { + return json( + { error: "Too many requests, please try again later." }, + { status: 429, headers: { "Retry-After": Math.ceil(error.retryAfter / 1000).toString() } } + ); + } + throw error; + } + } try { const authorizationCode = await createAuthorizationCode(); diff --git a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.cancel.ts b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.cancel.ts index 56b93fe17..d14f7a8bc 100644 --- a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.cancel.ts +++ b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.cancel.ts @@ -45,7 +45,7 @@ export async function action({ request, params }: ActionFunctionArgs) { const deploymentService = new DeploymentService(); return await deploymentService - .cancelDeployment(authenticatedEnv, deploymentId, { + .cancelDeployment({ id: authenticatedEnv.id }, deploymentId, { canceledReason: body.data.reason, }) .match( diff --git a/apps/webapp/app/routes/api.v1.schedules.$scheduleId.activate.ts b/apps/webapp/app/routes/api.v1.schedules.$scheduleId.activate.ts index 9cc8b7173..99ca31599 100644 --- a/apps/webapp/app/routes/api.v1.schedules.$scheduleId.activate.ts +++ b/apps/webapp/app/routes/api.v1.schedules.$scheduleId.activate.ts @@ -2,7 +2,7 @@ import type { ActionFunctionArgs } from "@remix-run/server-runtime"; import { json } from "@remix-run/server-runtime"; import { z } from "zod"; import { prisma } from "~/db.server"; -import { scheduleUniqWhereClause, scheduleWhereClause } from "~/models/schedules.server"; +import { getScheduleEnvVisibility, scheduleUniqWhereClause } from "~/models/schedules.server"; import { ViewSchedulePresenter } from "~/presenters/v3/ViewSchedulePresenter.server"; import { authenticateApiRequest } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; @@ -34,14 +34,17 @@ export async function action({ request, params }: ActionFunctionArgs) { } try { - const existingSchedule = await prisma.taskSchedule.findFirst({ - where: scheduleWhereClause( - authenticationResult.environment.projectId, - parsedParams.data.scheduleId - ), - }); - - if (!existingSchedule) { + // Env-scoped API keys can only toggle schedules that have an instance in + // their own environment. Without this a key scoped to one environment + // could enable/disable a schedule that only runs in another environment + // of the same project. + const visibility = await getScheduleEnvVisibility( + prisma, + authenticationResult.environment.projectId, + parsedParams.data.scheduleId, + authenticationResult.environment.id + ); + if (visibility.status !== "visible") { return json({ error: "Schedule not found" }, { status: 404 }); } diff --git a/apps/webapp/app/routes/api.v1.schedules.$scheduleId.deactivate.ts b/apps/webapp/app/routes/api.v1.schedules.$scheduleId.deactivate.ts index eb985bea7..3c9514ef8 100644 --- a/apps/webapp/app/routes/api.v1.schedules.$scheduleId.deactivate.ts +++ b/apps/webapp/app/routes/api.v1.schedules.$scheduleId.deactivate.ts @@ -2,7 +2,7 @@ import type { ActionFunctionArgs } from "@remix-run/server-runtime"; import { json } from "@remix-run/server-runtime"; import { z } from "zod"; import { prisma } from "~/db.server"; -import { scheduleUniqWhereClause, scheduleWhereClause } from "~/models/schedules.server"; +import { getScheduleEnvVisibility, scheduleUniqWhereClause } from "~/models/schedules.server"; import { ViewSchedulePresenter } from "~/presenters/v3/ViewSchedulePresenter.server"; import { authenticateApiRequest } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; @@ -34,14 +34,17 @@ export async function action({ request, params }: ActionFunctionArgs) { } try { - const existingSchedule = await prisma.taskSchedule.findFirst({ - where: scheduleWhereClause( - authenticationResult.environment.projectId, - parsedParams.data.scheduleId - ), - }); - - if (!existingSchedule) { + // Env-scoped API keys can only toggle schedules that have an instance in + // their own environment. Without this a key scoped to one environment + // could enable/disable a schedule that only runs in another environment + // of the same project. + const visibility = await getScheduleEnvVisibility( + prisma, + authenticationResult.environment.projectId, + parsedParams.data.scheduleId, + authenticationResult.environment.id + ); + if (visibility.status !== "visible") { return json({ error: "Schedule not found" }, { status: 404 }); } diff --git a/apps/webapp/app/routes/api.v1.schedules.$scheduleId.ts b/apps/webapp/app/routes/api.v1.schedules.$scheduleId.ts index 9da9ffcb1..4f7e8d8c1 100644 --- a/apps/webapp/app/routes/api.v1.schedules.$scheduleId.ts +++ b/apps/webapp/app/routes/api.v1.schedules.$scheduleId.ts @@ -5,7 +5,7 @@ import { UpdateScheduleOptions } from "@trigger.dev/core/v3"; import { z } from "zod"; import { Prisma, prisma } from "~/db.server"; import { clientSafeErrorMessage } from "~/utils/prismaErrors"; -import { scheduleUniqWhereClause } from "~/models/schedules.server"; +import { getScheduleEnvVisibility, scheduleUniqWhereClause } from "~/models/schedules.server"; import { ViewSchedulePresenter } from "~/presenters/v3/ViewSchedulePresenter.server"; import { authenticateApiRequest } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; @@ -38,6 +38,16 @@ export async function action({ request, params }: ActionFunctionArgs) { switch (method) { case "DELETE": { + const visibility = await getScheduleEnvVisibility( + prisma, + authenticationResult.environment.projectId, + parsedParams.data.scheduleId, + authenticationResult.environment.id + ); + if (visibility.status !== "visible") { + return json({ error: "Schedule not found" }, { status: 404 }); + } + try { const deletedSchedule = await prisma.taskSchedule.delete({ where: scheduleUniqWhereClause( @@ -76,6 +86,19 @@ export async function action({ request, params }: ActionFunctionArgs) { return json({ error: "Invalid request body", issues: body.error.issues }, { status: 400 }); } + // Env-scoped API keys can't see or mutate a schedule whose + // instances live in a different environment. "hidden" → refuse; + // "missing" → fall through to the upsert's create path. + const visibility = await getScheduleEnvVisibility( + prisma, + authenticationResult.environment.projectId, + parsedParams.data.scheduleId, + authenticationResult.environment.id + ); + if (visibility.status === "hidden") { + return json({ error: "Schedule not found" }, { status: 404 }); + } + const service = new UpsertTaskScheduleService(); try { @@ -137,6 +160,16 @@ export async function loader({ request, params }: LoaderFunctionArgs) { ); } + const visibility = await getScheduleEnvVisibility( + prisma, + authenticationResult.environment.projectId, + parsedParams.data.scheduleId, + authenticationResult.environment.id + ); + if (visibility.status !== "visible") { + return json({ error: "Schedule not found" }, { status: 404 }); + } + const presenter = new ViewSchedulePresenter(); const result = await presenter.call({ diff --git a/apps/webapp/app/routes/api.v1.token.ts b/apps/webapp/app/routes/api.v1.token.ts index 7763f9bb5..e3c856520 100644 --- a/apps/webapp/app/routes/api.v1.token.ts +++ b/apps/webapp/app/routes/api.v1.token.ts @@ -4,6 +4,10 @@ import type { GetPersonalAccessTokenResponse } from "@trigger.dev/core/v3"; import { GetPersonalAccessTokenRequestSchema } from "@trigger.dev/core/v3"; import { generateErrorMessage } from "zod-error"; import { logger } from "~/services/logger.server"; +import { + AuthorizationCodeRateLimitError, + checkAuthorizationCodeTokenPollRateLimit, +} from "~/services/authCodeRateLimiter.server"; import { getPersonalAccessTokenFromAuthorizationCode } from "~/services/personalAccessToken.server"; import { clientSafeErrorMessage } from "~/utils/prismaErrors"; @@ -25,6 +29,20 @@ export async function action({ request }: ActionFunctionArgs) { return json({ error: generateErrorMessage(body.error.issues) }, { status: 422 }); } + // Per-code rate limit (keyed by the code, not the IP, so the CLI's poll loop + // isn't broken behind a shared NAT). + try { + await checkAuthorizationCodeTokenPollRateLimit(body.data.authorizationCode); + } catch (error) { + if (error instanceof AuthorizationCodeRateLimitError) { + return json( + { error: "Too many requests, please try again later." }, + { status: 429, headers: { "Retry-After": Math.ceil(error.retryAfter / 1000).toString() } } + ); + } + throw error; + } + try { const personalAccessToken = await getPersonalAccessTokenFromAuthorizationCode( body.data.authorizationCode diff --git a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.complete.ts b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.complete.ts index 051b6d4d0..342ca3f28 100644 --- a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.complete.ts +++ b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.complete.ts @@ -18,6 +18,7 @@ export const action = createActionWorkerApiRoute( body, params, runnerId, + environmentId, }): Promise> => { const { completion } = body; const { runFriendlyId, snapshotFriendlyId } = params; @@ -27,6 +28,7 @@ export const action = createActionWorkerApiRoute( snapshotFriendlyId, completion, runnerId, + environmentId, }); return json({ result: completeResult }); diff --git a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts index a28a362ed..558fa0a0c 100644 --- a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts +++ b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts @@ -18,6 +18,7 @@ export const action = createActionWorkerApiRoute( body, params, runnerId, + environmentId, }): Promise> => { const { runFriendlyId, snapshotFriendlyId } = params; @@ -26,6 +27,7 @@ export const action = createActionWorkerApiRoute( snapshotFriendlyId, isWarmStart: body.isWarmStart, runnerId, + environmentId, }); return json(runExecutionData); diff --git a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.continue.ts b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.continue.ts index 3c264adcd..1aef207aa 100644 --- a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.continue.ts +++ b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.continue.ts @@ -17,6 +17,7 @@ export const loader = createLoaderWorkerApiRoute( authenticatedWorker, params, runnerId, + environmentId, }): Promise> => { const { runFriendlyId, snapshotFriendlyId } = params; @@ -27,10 +28,17 @@ export const loader = createLoaderWorkerApiRoute( runFriendlyId, snapshotFriendlyId, runnerId, + environmentId, }); return json(continuationResult); } catch (error) { + // An authorization rejection is thrown as a Response; propagate it as-is rather than + // masking it as a generic 422. + if (error instanceof Response) { + throw error; + } + logger.warn("Failed to continue run execution", { runFriendlyId, snapshotFriendlyId, diff --git a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.latest.ts b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.latest.ts index 73df07748..46d847410 100644 --- a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.latest.ts +++ b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.latest.ts @@ -13,11 +13,13 @@ export const loader = createLoaderWorkerApiRoute( async ({ authenticatedWorker, params, + environmentId, }): Promise> => { const { runFriendlyId } = params; const executionData = await authenticatedWorker.getLatestSnapshot({ runFriendlyId, + environmentId, }); if (!executionData) { diff --git a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.since.$snapshotId.ts b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.since.$snapshotId.ts index 87e1ed5ee..7b91dc476 100644 --- a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.since.$snapshotId.ts +++ b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.since.$snapshotId.ts @@ -14,12 +14,14 @@ export const loader = createLoaderWorkerApiRoute( async ({ authenticatedWorker, params, + environmentId, }): Promise> => { const { runFriendlyId, snapshotId } = params; const snapshots = await authenticatedWorker.getSnapshotsSince({ runFriendlyId, snapshotId, + environmentId, }); if (!snapshots) { diff --git a/apps/webapp/app/routes/logout.tsx b/apps/webapp/app/routes/logout.tsx index 20b4705f3..fe6368ee3 100644 --- a/apps/webapp/app/routes/logout.tsx +++ b/apps/webapp/app/routes/logout.tsx @@ -1,6 +1,8 @@ -import { type ActionFunction, type LoaderFunction } from "@remix-run/node"; +import { redirect, type ActionFunction, type LoaderFunction } from "@remix-run/node"; +import { env } from "~/env.server"; import { authenticator } from "~/services/auth.server"; import { sanitizeRedirectPath } from "~/utils"; +import { isSameOriginNavigation } from "~/utils/sameOriginNavigation"; import { SSO_SESSION_EXPIRED_REASON } from "~/utils/ssoSession"; function logoutRedirectTo(request: Request): string { @@ -18,5 +20,10 @@ export const action: ActionFunction = async ({ request }) => { }; export const loader: LoaderFunction = async ({ request }) => { + // GET /logout is state-changing, so reject cross-site navigations. + if (!isSameOriginNavigation(request, env.LOGIN_ORIGIN)) { + throw redirect("/"); + } + return await authenticator.logout(request, { redirectTo: logoutRedirectTo(request) }); }; diff --git a/apps/webapp/app/routes/realtime.v1.runs.ts b/apps/webapp/app/routes/realtime.v1.runs.ts index ca2998f29..465e7ab4f 100644 --- a/apps/webapp/app/routes/realtime.v1.runs.ts +++ b/apps/webapp/app/routes/realtime.v1.runs.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { getRequestAbortSignal } from "~/services/httpAsyncStorage.server"; import { resolveRealtimeStreamClient } from "~/services/realtime/resolveRealtimeStreamClient.server"; import { anyResource, createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { UNSAFE_REALTIME_TAG_CHARS } from "~/v3/electricShape.server"; const SearchParamsSchema = z.object({ tags: z @@ -9,6 +10,20 @@ const SearchParamsSchema = z.object({ .optional() .transform((value) => { return value ? value.split(",") : undefined; + }) + .superRefine((tags, ctx) => { + if (!tags) return; + for (const tag of tags) { + // Mirror the runtime sanitiser's reject list so the API returns 400 + // instead of a 500. Single quotes are allowed — escaped downstream. + if (UNSAFE_REALTIME_TAG_CHARS.test(tag) || tag.length === 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Invalid tag: ${JSON.stringify(tag)}`, + }); + return; + } + } }), createdAt: z.string().optional(), }); diff --git a/apps/webapp/app/routes/realtime.v1.sessions.$session.$io.ts b/apps/webapp/app/routes/realtime.v1.sessions.$session.$io.ts index e846e851b..b4ff0ba95 100644 --- a/apps/webapp/app/routes/realtime.v1.sessions.$session.$io.ts +++ b/apps/webapp/app/routes/realtime.v1.sessions.$session.$io.ts @@ -34,6 +34,16 @@ const { action } = createActionApiRoute( }, }, async ({ params, authentication }) => { + // `.out` is the agent→client channel. Only PRIVATE (secret key) auth — + // i.e. the agent run itself — may initialize it. Session-scoped JWTs carry + // `write:sessions:` for `.in`; without this gate they could obtain + // credentials to forge assistant chunks on their own session's `.out`. + if (params.io === "out" && authentication.type !== "PRIVATE") { + return new Response("Initializing the out channel requires secret key authentication", { + status: 403, + }); + } + // Row-optional addressing. The agent calls PUT initialize as part // of `session.out.writer()`, by which time it has already created // the row at bind, so a missing row here is an unusual case diff --git a/apps/webapp/app/routes/resources.$projectId.deployments.$deploymentShortCode.cancel.ts b/apps/webapp/app/routes/resources.$projectId.deployments.$deploymentShortCode.cancel.ts index dd1c452c5..239d0982f 100644 --- a/apps/webapp/app/routes/resources.$projectId.deployments.$deploymentShortCode.cancel.ts +++ b/apps/webapp/app/routes/resources.$projectId.deployments.$deploymentShortCode.cancel.ts @@ -75,7 +75,7 @@ export const action = dashboardAction( prisma.workerDeployment.findUnique({ select: { friendlyId: true, - projectId: true, + environmentId: true, }, where: { projectId_shortCode: { @@ -96,10 +96,7 @@ export const action = dashboardAction( const result = await verifyProjectMembership() .andThen(findDeploymentFriendlyId) .andThen((deployment) => - deploymentService.cancelDeployment( - { projectId: deployment.projectId }, - deployment.friendlyId - ) + deploymentService.cancelDeployment({ id: deployment.environmentId }, deployment.friendlyId) ); if (result.isErr()) { diff --git a/apps/webapp/app/routes/sync.traces.$traceId.ts b/apps/webapp/app/routes/sync.traces.$traceId.ts index 1a06dd488..d7695014b 100644 --- a/apps/webapp/app/routes/sync.traces.$traceId.ts +++ b/apps/webapp/app/routes/sync.traces.$traceId.ts @@ -1,15 +1,33 @@ import type { LoaderFunctionArgs } from "@remix-run/node"; +import { z } from "zod"; import { $replica } from "~/db.server"; import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; import { getUserId } from "~/services/session.server"; import { longPollingFetch } from "~/utils/longPollingFetch"; +import { + OtelTraceIdSchema, + RESERVED_ELECTRIC_SHAPE_PARAMS, + buildElectricTraceWhereClause, +} from "~/v3/electricShape.server"; + +const Params = z.object({ + traceId: OtelTraceIdSchema, +}); export async function loader({ params, request }: LoaderFunctionArgs) { try { const userId = await getUserId(request); - logger.log(`/sync/traces/${params.traceId}`, { userId }); + const parsedParams = Params.safeParse(params); + if (!parsedParams.success) { + // Treat a malformed traceId as not-found rather than 400 to avoid + // signalling the validator. + return new Response("Not found", { status: 404 }); + } + const { traceId } = parsedParams.data; + + logger.log(`/sync/traces/${traceId}`, { userId }); if (!userId) { return new Response("No user found in cookie", { status: 401 }); @@ -20,7 +38,7 @@ export async function loader({ params, request }: LoaderFunctionArgs) { organizationId: true, }, where: { - traceId: params.traceId, + traceId, }, }); @@ -41,11 +59,19 @@ export async function loader({ params, request }: LoaderFunctionArgs) { const url = new URL(request.url); const originUrl = new URL(`${env.ELECTRIC_ORIGIN}/v1/shape/public."TaskEvent"`); + // Strip params we set ourselves so the caller can't override them. url.searchParams.forEach((value, key) => { + if (RESERVED_ELECTRIC_SHAPE_PARAMS.has(key)) return; originUrl.searchParams.set(key, value); }); - originUrl.searchParams.set("where", `"traceId"='${params.traceId}'`); + originUrl.searchParams.set( + "where", + buildElectricTraceWhereClause({ + traceId, + scope: { column: "organizationId", id: trace.organizationId }, + }) + ); const finalUrl = originUrl.toString(); diff --git a/apps/webapp/app/routes/sync.traces.runs.$traceId.ts b/apps/webapp/app/routes/sync.traces.runs.$traceId.ts index 5fd1f7c65..24870cf2f 100644 --- a/apps/webapp/app/routes/sync.traces.runs.$traceId.ts +++ b/apps/webapp/app/routes/sync.traces.runs.$traceId.ts @@ -5,17 +5,27 @@ import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; import { getUserId } from "~/services/session.server"; import { longPollingFetch } from "~/utils/longPollingFetch"; -import { runStore } from "~/v3/runStore.server"; +import { + OtelTraceIdSchema, + RESERVED_ELECTRIC_SHAPE_PARAMS, + buildElectricTraceWhereClause, +} from "~/v3/electricShape.server"; import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; +import { runStore } from "~/v3/runStore.server"; const Params = z.object({ - traceId: z.string(), + traceId: OtelTraceIdSchema, }); export async function loader({ params, request }: LoaderFunctionArgs) { try { const userId = await getUserId(request); - const { traceId } = Params.parse(params); + + const parsedParams = Params.safeParse(params); + if (!parsedParams.success) { + return new Response("Not found", { status: 404 }); + } + const { traceId } = parsedParams.data; logger.log(`/sync/runs/${traceId}`, { userId }); @@ -29,6 +39,7 @@ export async function loader({ params, request }: LoaderFunctionArgs) { }, { select: { + projectId: true, runtimeEnvironmentId: true, }, }, @@ -40,7 +51,7 @@ export async function loader({ params, request }: LoaderFunctionArgs) { // primary before 404ing so a live run's realtime trace feed isn't spuriously not-found. run = await runStore.findRunOnPrimary( { traceId }, - { select: { runtimeEnvironmentId: true } } + { select: { projectId: true, runtimeEnvironmentId: true } } ); } @@ -67,11 +78,22 @@ export async function loader({ params, request }: LoaderFunctionArgs) { const url = new URL(request.url); const originUrl = new URL(`${env.ELECTRIC_ORIGIN}/v1/shape/public."TaskRun"`); + // Strip params we set ourselves so the caller can't override them. url.searchParams.forEach((value, key) => { + if (RESERVED_ELECTRIC_SHAPE_PARAMS.has(key)) return; originUrl.searchParams.set(key, value); }); - originUrl.searchParams.set("where", `"traceId"='${traceId}'`); + originUrl.searchParams.set( + "where", + // Scope by non-null projectId, not the nullable organizationId (legacy + // rows would vanish). Tenant-safe: membership was verified against this + // project's org and a trace's runs all live in one project. + buildElectricTraceWhereClause({ + traceId, + scope: { column: "projectId", id: run.projectId }, + }) + ); const finalUrl = originUrl.toString(); diff --git a/apps/webapp/app/services/apiRateLimit.server.ts b/apps/webapp/app/services/apiRateLimit.server.ts index fbd59cc74..837d3ee58 100644 --- a/apps/webapp/app/services/apiRateLimit.server.ts +++ b/apps/webapp/app/services/apiRateLimit.server.ts @@ -48,6 +48,9 @@ export const apiRateLimiter = authorizationRateLimitMiddleware({ // Allow /api/v1/tasks/:id/callback/:secret pathWhiteList: [ "/api/internal/stripe_webhooks", + // Keep allowlisted: these CLI endpoints are intentionally unauthenticated, + // so this Authorization-header-keyed limiter would 401 them. They are + // throttled separately by authCodeRateLimiter.server.ts. "/api/v1/authorization-code", "/api/v1/token", "/api/v1/usage/ingest", diff --git a/apps/webapp/app/services/authCodeRateLimiter.server.ts b/apps/webapp/app/services/authCodeRateLimiter.server.ts new file mode 100644 index 000000000..5c88c094c --- /dev/null +++ b/apps/webapp/app/services/authCodeRateLimiter.server.ts @@ -0,0 +1,85 @@ +import { Ratelimit } from "@upstash/ratelimit"; +import { createHash } from "node:crypto"; +import { env } from "~/env.server"; +import { createRedisRateLimitClient, RateLimiter } from "~/services/rateLimiter.server"; +import { singleton } from "~/utils/singleton"; + +/** + * Rate limiting for the unauthenticated CLI auth-code endpoints + * (`/api/v1/authorization-code` mint + `/api/v1/token` poll). The global + * limiter keys on the Authorization header, which these endpoints don't carry, + * so it can't throttle them — this module does. + */ +export class AuthorizationCodeRateLimitError extends Error { + public readonly retryAfter: number; + + constructor(retryAfter: number) { + super("Authorization code rate limit exceeded."); + this.name = "AuthorizationCodeRateLimitError"; + this.retryAfter = retryAfter; + } +} + +function getRedisClient() { + return createRedisRateLimitClient({ + port: env.RATE_LIMIT_REDIS_PORT, + host: env.RATE_LIMIT_REDIS_HOST, + username: env.RATE_LIMIT_REDIS_USERNAME, + password: env.RATE_LIMIT_REDIS_PASSWORD, + tlsDisabled: env.RATE_LIMIT_REDIS_TLS_DISABLED === "true", + clusterMode: env.RATE_LIMIT_REDIS_CLUSTER_MODE_ENABLED === "1", + }); +} + +// Minting is unauthenticated and a real login mints one code. Cap per IP, with +// headroom for many users behind a shared NAT. +const authorizationCodeMintIpRateLimiter = singleton( + "authorizationCodeMintIpRateLimiter", + () => + new RateLimiter({ + redisClient: getRedisClient(), + keyPrefix: "auth:authcode:mint:ip", + limiter: Ratelimit.slidingWindow(30, "1 m"), // 30 code mints / min / IP + logSuccess: false, + logFailure: true, + }) +); + +// Keyed by the code, not the IP: the CLI polls this endpoint ~1/s per login, so +// IP-keying would break logins behind a shared NAT. The ~60/min cadence stays +// under the cap. The code is hashed first so it never lands in a Redis key or log. +const authorizationCodeTokenPollRateLimiter = singleton( + "authorizationCodeTokenPollRateLimiter", + () => + new RateLimiter({ + redisClient: getRedisClient(), + keyPrefix: "auth:authcode:token:code", + limiter: Ratelimit.slidingWindow(100, "1 m"), // 100 polls / min / code (CLI polls ~60/min) + logSuccess: false, + logFailure: false, + }) +); + +function hashCode(code: string): string { + return createHash("sha256").update(code).digest("hex").slice(0, 32); +} + +export async function checkAuthorizationCodeMintRateLimit(ip: string): Promise { + const result = await authorizationCodeMintIpRateLimiter.limit(ip); + + if (!result.success) { + const retryAfter = new Date(result.reset).getTime() - Date.now(); + throw new AuthorizationCodeRateLimitError(retryAfter); + } +} + +export async function checkAuthorizationCodeTokenPollRateLimit( + authorizationCode: string +): Promise { + const result = await authorizationCodeTokenPollRateLimiter.limit(hashCode(authorizationCode)); + + if (!result.success) { + const retryAfter = new Date(result.reset).getTime() - Date.now(); + throw new AuthorizationCodeRateLimitError(retryAfter); + } +} diff --git a/apps/webapp/app/services/gitHub.server.ts b/apps/webapp/app/services/gitHub.server.ts index e67895fe0..00ec2087f 100644 --- a/apps/webapp/app/services/gitHub.server.ts +++ b/apps/webapp/app/services/gitHub.server.ts @@ -58,26 +58,32 @@ export async function linkGitHubAppInstallation( } /** - * Links a GitHub App installation to a Trigger organization + * Updates a GitHub App installation owned by the given Trigger organization */ -export async function updateGitHubAppInstallation(installationId: number): Promise { +export async function updateGitHubAppInstallation( + installationId: number, + organizationId: string +): Promise { if (!githubApp) { throw new Error("GitHub App is not enabled"); } + // Scope the lookup to the caller's organization so a cross-tenant + // installation_id cannot update another org's record. Resolve ownership + // before calling GitHub to avoid burning the victim's API rate limit. + const existingInstallation = await prisma.githubAppInstallation.findFirst({ + where: { appInstallationId: installationId, organizationId }, + }); + + if (!existingInstallation) { + throw new Error("GitHub App installation not found"); + } + const octokit = await githubApp.getInstallationOctokit(installationId); const { data: installation } = await octokit.rest.apps.getInstallation({ installation_id: installationId, }); - const existingInstallation = await prisma.githubAppInstallation.findFirst({ - where: { appInstallationId: installationId }, - }); - - if (!existingInstallation) { - throw new Error("GitHub App installation not found"); - } - const repositorySelection = installation.repository_selection === "all" ? "ALL" : "SELECTED"; // repos are updated asynchronously via webhook events diff --git a/apps/webapp/app/services/otlpRateLimit.server.ts b/apps/webapp/app/services/otlpRateLimit.server.ts new file mode 100644 index 000000000..eef07990e --- /dev/null +++ b/apps/webapp/app/services/otlpRateLimit.server.ts @@ -0,0 +1,91 @@ +import { Ratelimit } from "@upstash/ratelimit"; +import type { NextFunction, Request, Response } from "express"; +import { env } from "~/env.server"; +import { createRedisRateLimitClient, RateLimiter } from "~/services/rateLimiter.server"; +import { extractClientIp } from "~/utils/extractClientIp.server"; +import { singleton } from "~/utils/singleton"; +import { logger } from "./logger.server"; + +const OTLP_PATH = /^\/otel\//i; + +function getOtlpIpRateLimiter() { + return singleton( + "otlpIpRateLimiter", + () => + new RateLimiter({ + redisClient: createRedisRateLimitClient({ + port: env.RATE_LIMIT_REDIS_PORT, + host: env.RATE_LIMIT_REDIS_HOST, + username: env.RATE_LIMIT_REDIS_USERNAME, + password: env.RATE_LIMIT_REDIS_PASSWORD, + tlsDisabled: env.RATE_LIMIT_REDIS_TLS_DISABLED === "true", + clusterMode: env.RATE_LIMIT_REDIS_CLUSTER_MODE_ENABLED === "1", + }), + keyPrefix: "otlp:ip", + limiter: Ratelimit.slidingWindow( + env.OTLP_RATE_LIMIT_MAX, + env.OTLP_RATE_LIMIT_WINDOW as Parameters[1] + ), + logSuccess: false, + logFailure: true, + }) + ); +} + +/** + * Per-IP rate limiter for the OTLP ingestion endpoints (`/otel/*`). + * + * These endpoints are currently unauthenticated (see SEC-98), so the source IP + * is the only identity available to key on. This bounds unauthenticated + * request rates and is NOT a substitute for authenticating the + * endpoints. It fails open (allows the request) whenever the source cannot be + * identified or the limiter backend errors, so a limiter outage never drops + * legitimate telemetry. + * + * Opt-in (disabled unless `OTLP_RATE_LIMIT_ENABLED=1`). Because it keys on the + * source IP, two preconditions must hold before enabling it, or it can drop + * legitimate telemetry: + * + * 1. Each client must present a distinct IP. Where many clients share one + * egress IP (e.g. behind NAT or a shared proxy) their traffic collapses + * into a single bucket and can be throttled together. Size + * `OTLP_RATE_LIMIT_MAX` for the aggregate volume of a shared source, not a + * single client. + * 2. The IP must be trustworthy. `extractClientIp` takes the last + * `X-Forwarded-For` hop, which is only spoof-resistant behind a proxy that + * appends the real client IP. Without such a proxy the value is + * client-controlled and the per-IP bound is bypassable. + */ +export async function otlpRateLimiter(req: Request, res: Response, next: NextFunction) { + if (env.OTLP_RATE_LIMIT_ENABLED !== "1") { + return next(); + } + + if (req.method.toUpperCase() === "OPTIONS" || !OTLP_PATH.test(req.path)) { + return next(); + } + + const xff = req.headers["x-forwarded-for"]; + const ip = extractClientIp(Array.isArray(xff) ? xff.join(",") : (xff ?? null)) ?? req.ip; + + if (!ip) { + // Fail open: without a source we cannot fairly rate limit. + return next(); + } + + try { + const { success, reset } = await getOtlpIpRateLimiter().limit(ip); + + if (!success) { + const retryAfterSeconds = Math.max(1, Math.ceil((reset - Date.now()) / 1000)); + res.setHeader("Retry-After", retryAfterSeconds.toString()); + res.status(429).send("Too Many Requests"); + return; + } + } catch (error) { + // Fail open: a rate-limiter backend outage must not drop telemetry. + logger.warn("otlpRateLimiter: limiter error, allowing request", { error }); + } + + return next(); +} diff --git a/apps/webapp/app/services/personalAccessToken.server.ts b/apps/webapp/app/services/personalAccessToken.server.ts index a0821d660..d80ca6fa3 100644 --- a/apps/webapp/app/services/personalAccessToken.server.ts +++ b/apps/webapp/app/services/personalAccessToken.server.ts @@ -18,6 +18,10 @@ const tokenGenerator = customAlphabet("123456789abcdefghijkmnopqrstuvwxyz", toke // staleness is fine. export const PAT_LAST_ACCESSED_THROTTLE_MS = 5 * 60 * 1000; +// How long an unconsumed CLI authorization code stays valid. Shared constant so +// the mint and read paths can't drift. +export const AUTHORIZATION_CODE_TTL_MS = 10 * 60 * 1000; + type CreatePersonalAccessTokenOptions = { name: string; userId: string; @@ -60,16 +64,16 @@ export type ObfuscatedPersonalAccessToken = Awaited< /** Gets a PersonalAccessToken from an Auth Code, this only works within 10 mins of the auth code being created */ export async function getPersonalAccessTokenFromAuthorizationCode(authorizationCode: string) { - //only allow authorization codes that were created less than 10 mins ago - const tenMinutesAgo = new Date(Date.now() - 10 * 60 * 1000); - const code = await prisma.authorizationCode.findUnique({ + // Only allow authorization codes created within the short consent window. + const validAfter = new Date(Date.now() - AUTHORIZATION_CODE_TTL_MS); + const code = await prisma.authorizationCode.findFirst({ select: { personalAccessToken: true, }, where: { code: authorizationCode, createdAt: { - gte: tenMinutesAgo, + gte: validAfter, }, }, }); @@ -280,6 +284,27 @@ export function isPersonalAccessToken(token: string) { return token.startsWith(tokenPrefix); } +/** + * Read-only check that an authorization code is still mintable: it exists, is + * unconsumed (`personalAccessTokenId: null`), and within the TTL. Lets the + * consent-screen loader show Authorize vs expired/invalid without minting a PAT. + */ +export async function isAuthorizationCodeMintable( + authorizationCode: string, + prismaClient = prisma +): Promise { + const validAfter = new Date(Date.now() - AUTHORIZATION_CODE_TTL_MS); + const code = await prismaClient.authorizationCode.findFirst({ + where: { + code: authorizationCode, + personalAccessTokenId: null, + createdAt: { gte: validAfter }, + }, + select: { id: true }, + }); + return code !== null; +} + export function createAuthorizationCode() { return prisma.authorizationCode.create({ data: { @@ -293,14 +318,14 @@ export async function createPersonalAccessTokenFromAuthorizationCode( authorizationCode: string, userId: string ) { - //only allow authorization codes that were created less than 10 mins ago - const tenMinutesAgo = new Date(Date.now() - 10 * 60 * 1000); - const code = await prisma.authorizationCode.findUnique({ + // Only allow authorization codes created within the short consent window. + const validAfter = new Date(Date.now() - AUTHORIZATION_CODE_TTL_MS); + const code = await prisma.authorizationCode.findFirst({ where: { code: authorizationCode, personalAccessTokenId: null, createdAt: { - gte: tenMinutesAgo, + gte: validAfter, }, }, }); diff --git a/apps/webapp/app/services/realtimeClient.server.ts b/apps/webapp/app/services/realtimeClient.server.ts index 9797eeebf..bf8634064 100644 --- a/apps/webapp/app/services/realtimeClient.server.ts +++ b/apps/webapp/app/services/realtimeClient.server.ts @@ -15,6 +15,7 @@ import { RedisCacheStore } from "./unkey/redisCacheStore.server"; import { env } from "~/env.server"; import type { API_VERSIONS } from "~/api/versions"; import { CURRENT_API_VERSION } from "~/api/versions"; +import { sanitizeRealtimeTagsForSql } from "~/v3/electricShape.server"; export interface CachedLimitProvider { getCachedLimit: (organizationId: string, defaultValue: number) => Promise; @@ -171,7 +172,10 @@ export class RealtimeClient { const whereClauses: string[] = [`"runtimeEnvironmentId"='${environment.id}'`]; if (params.tags) { - whereClauses.push(`"runTags" @> ARRAY[${params.tags.map((t) => `'${t}'`).join(",")}]`); + // Reject unsafe chars and escape single quotes so tag values can't + // break out of the Electric SQL string literal. + const safeTags = sanitizeRealtimeTagsForSql(params.tags); + whereClauses.push(`"runTags" @> ARRAY[${safeTags.map((t) => `'${t}'`).join(",")}]`); } const createdAtFilter = await this.#calculateCreatedAtFilter(url, params.createdAt); diff --git a/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts b/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts index a74d49d22..94c2929f5 100644 --- a/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts +++ b/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts @@ -1537,6 +1537,7 @@ type WorkerLoaderHandlerFunction< ? z.infer : undefined; runnerId?: string; + environmentId?: string; }) => Promise; export function createLoaderWorkerApiRoute< @@ -1601,6 +1602,8 @@ export function createLoaderWorkerApiRoute< } const runnerId = request.headers.get(WORKER_HEADERS.RUNNER_ID) ?? undefined; + // `|| undefined` so a blank header can't become a zero-match snapshot filter (→ false reject). + const environmentId = request.headers.get(WORKER_HEADERS.ENVIRONMENT_ID) || undefined; const result = await handler({ params: parsedParams, @@ -1609,6 +1612,7 @@ export function createLoaderWorkerApiRoute< request, headers: parsedHeaders, runnerId, + environmentId, }); return result; } catch (error) { @@ -1660,6 +1664,7 @@ type WorkerActionHandlerFunction< ? z.infer : undefined; runnerId?: string; + environmentId?: string; }) => Promise; export function createActionWorkerApiRoute< @@ -1758,6 +1763,8 @@ export function createActionWorkerApiRoute< } const runnerId = request.headers.get(WORKER_HEADERS.RUNNER_ID) ?? undefined; + // `|| undefined` so a blank header can't become a zero-match snapshot filter (→ false reject). + const environmentId = request.headers.get(WORKER_HEADERS.ENVIRONMENT_ID) || undefined; const result = await handler({ params: parsedParams, @@ -1767,6 +1774,7 @@ export function createActionWorkerApiRoute< body: parsedBody, headers: parsedHeaders, runnerId, + environmentId, }); return result; } catch (error) { diff --git a/apps/webapp/app/v3/electricShape.server.ts b/apps/webapp/app/v3/electricShape.server.ts new file mode 100644 index 000000000..c430cc027 --- /dev/null +++ b/apps/webapp/app/v3/electricShape.server.ts @@ -0,0 +1,69 @@ +import { z } from "zod"; + +/** + * OTel trace IDs are 32 lowercase hex chars. The traceparent parser only + * checks the dash-delimited format, so crafted ids can be persisted and later + * interpolated into shape `where` clauses. Validate here to close the SQLi vector. + */ +export const OtelTraceIdSchema = z + .string() + .regex(/^[0-9a-f]{32}$/, "traceId must be 32 lowercase hex characters"); + +/** Params the sync routes set themselves; stripped from incoming requests. */ +export const RESERVED_ELECTRIC_SHAPE_PARAMS = new Set(["where", "table", "columns"]); + +const CUID_LIKE = /^[a-z][a-z0-9_]*$/i; + +/** + * Tenant column a trace shape is scoped by. TaskEvent scopes by non-null + * organizationId; TaskRun scopes by non-null projectId (its organizationId is + * nullable). The column is from this fixed union, never user input, so it's + * safe to interpolate. + */ +export type TraceScope = + | { column: "organizationId"; id: string } + | { column: "projectId"; id: string }; + +/** + * Build the Electric Shape `where` clause for the trace sync routes. Both ids + * are re-validated as defense-in-depth so a missed call site can't bypass scope. + */ +export function buildElectricTraceWhereClause(args: { + traceId: string; + scope: TraceScope; +}): string { + const { traceId, scope } = args; + if (!OtelTraceIdSchema.safeParse(traceId).success) { + throw new Error("buildElectricTraceWhereClause: unsafe traceId"); + } + if (!CUID_LIKE.test(scope.id)) { + throw new Error("buildElectricTraceWhereClause: unsafe scope id"); + } + return `"traceId"='${traceId}' AND "${scope.column}"='${scope.id}'`; +} + +/** + * Characters rejected in realtime tag values — the single source of truth + * shared by the apiBuilder Zod refine (`realtime.v1.runs.ts`) and the runtime + * sanitiser. Rejects control chars/DEL, backslash, and double-quote. Single + * quotes are allowed and escaped (`'` → `''`) in `sanitizeRealtimeTagForSql`. + */ +export const UNSAFE_REALTIME_TAG_CHARS = /[\x00-\x1f\x7f\\"]/; + +/** + * Sanitise a tag value for interpolation into an Electric Shape `where` clause: + * reject unsafe chars, escape single quotes per SQL standard. + */ +export function sanitizeRealtimeTagForSql(tag: string): string { + if (typeof tag !== "string" || tag.length === 0) { + throw new Error("Invalid realtime tag: empty"); + } + if (UNSAFE_REALTIME_TAG_CHARS.test(tag)) { + throw new Error(`Invalid realtime tag: ${JSON.stringify(tag)} — contains unsafe character`); + } + return tag.replace(/'/g, "''"); +} + +export function sanitizeRealtimeTagsForSql(tags: string[]): string[] { + return tags.map(sanitizeRealtimeTagForSql); +} diff --git a/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts b/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts index a8f2ce701..1a0b82473 100644 --- a/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts +++ b/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts @@ -82,7 +82,10 @@ export class EnvironmentVariablesRepository implements Repository { return { success: false as const, error: "Project not found" }; } - if (options.environmentIds.every((v) => !project.environments.some((e) => e.id === v))) { + // Reject if ANY supplied environmentId is outside the caller's project. + // `.some` (not `.every`) so one in-project id can't let a mixed array + // through. + if (options.environmentIds.some((v) => !project.environments.some((e) => e.id === v))) { return { success: false as const, error: `Environment not found` }; } @@ -291,7 +294,9 @@ export class EnvironmentVariablesRepository implements Repository { return { success: false as const, error: "Project not found" }; } - if (options.values.every((v) => !project.environments.some((e) => e.id === v.environmentId))) { + // Same guard as `create()`: reject if ANY supplied environmentId is + // outside the caller's project (`.some`, not `.every`). + if (options.values.some((v) => !project.environments.some((e) => e.id === v.environmentId))) { return { success: false as const, error: `Environment not found` }; } diff --git a/apps/webapp/app/v3/otlpTransform.server.ts b/apps/webapp/app/v3/otlpTransform.server.ts index d66f1a8b8..4de989b0a 100644 --- a/apps/webapp/app/v3/otlpTransform.server.ts +++ b/apps/webapp/app/v3/otlpTransform.server.ts @@ -12,6 +12,7 @@ import type { import { SeverityNumber, Span_SpanKind, Status_StatusCode } from "@trigger.dev/otlp-importer"; import type { MetricsV1Input } from "@internal/clickhouse"; import { generateSpanId } from "./eventRepository/common.server"; +import { unwrapWorkerIdInMetadata } from "./workerIdUnwrap.server"; import type { CreatableEventKind, CreatableEventStatus, @@ -468,7 +469,11 @@ function resolveDataPointContext( function extractEventProperties(attributes: KeyValue[], prefix?: string) { return { - metadata: convertSelectedKeyValueItemsToMap(attributes, [SemanticInternalAttributes.METADATA]), + // Decode a deployment-token worker.id back to its friendlyId so the credential never lands in + // stored span/log metadata (the metrics path unwraps its own worker_id separately). + metadata: unwrapWorkerIdInMetadata( + convertSelectedKeyValueItemsToMap(attributes, [SemanticInternalAttributes.METADATA]) + ), environmentId: extractStringAttribute(attributes, [ prefix, SemanticInternalAttributes.ENVIRONMENT_ID, diff --git a/apps/webapp/app/v3/services/checkSchedule.server.ts b/apps/webapp/app/v3/services/checkSchedule.server.ts index e6429a2ad..beb5de4b5 100644 --- a/apps/webapp/app/v3/services/checkSchedule.server.ts +++ b/apps/webapp/app/v3/services/checkSchedule.server.ts @@ -1,6 +1,7 @@ import { ZodError } from "zod"; import { CronPattern } from "../schedules"; import { BaseService, ServiceValidationError } from "./baseService.server"; +import { resolveProjectScopedEnvironments } from "./resolveProjectScopedEnvironments"; import { getLimit } from "~/services/platform.v3.server"; import { getTimezones } from "~/utils/timezones.server"; import { env } from "~/env.server"; @@ -82,7 +83,19 @@ export class CheckScheduleService extends BaseService { throw new ServiceValidationError("Project not found"); } - const environments = project.environments.filter((env) => environmentIds.includes(env.id)); + // Reject (don't silently drop) any environmentId that doesn't belong to the + // authorized project. + const scopedEnvironments = resolveProjectScopedEnvironments( + environmentIds, + project.environments + ); + if (scopedEnvironments.kind === "foreign") { + throw new ServiceValidationError( + `Environment ${scopedEnvironments.foreignEnvironmentId} does not belong to this project.` + ); + } + + const environments = scopedEnvironments.environments; if (environments.some((env) => env.archivedAt)) { throw new ServiceValidationError("Can't add or edit a schedule for an archived branch"); } diff --git a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts index a53dee423..66a17bb9f 100644 --- a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts +++ b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts @@ -51,6 +51,7 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { const deployment = await this._prisma.workerDeployment.findFirst({ where: { friendlyId: deploymentId, + environmentId: environment.id, }, }); diff --git a/apps/webapp/app/v3/services/deleteTaskSchedule.server.ts b/apps/webapp/app/v3/services/deleteTaskSchedule.server.ts index a696fd1ff..601f7470a 100644 --- a/apps/webapp/app/v3/services/deleteTaskSchedule.server.ts +++ b/apps/webapp/app/v3/services/deleteTaskSchedule.server.ts @@ -1,3 +1,4 @@ +import { scheduleWhereClause } from "~/models/schedules.server"; import { BaseService } from "./baseService.server"; type Options = { @@ -28,9 +29,7 @@ export class DeleteTaskScheduleService extends BaseService { try { const schedule = await this._prisma.taskSchedule.findFirst({ - where: { - friendlyId, - }, + where: scheduleWhereClause(projectId, friendlyId), }); if (!schedule) { diff --git a/apps/webapp/app/v3/services/deployment.server.ts b/apps/webapp/app/v3/services/deployment.server.ts index 71292f409..f726bba3d 100644 --- a/apps/webapp/app/v3/services/deployment.server.ts +++ b/apps/webapp/app/v3/services/deployment.server.ts @@ -169,7 +169,7 @@ export class DeploymentService extends BaseService { }) ); - return this.getDeployment(authenticatedEnv.projectId, friendlyId) + return this.getDeployment(authenticatedEnv.id, friendlyId) .andThen(validateDeployment) .andThen((deployment) => { if (deployment.status === "PENDING") { @@ -191,7 +191,7 @@ export class DeploymentService extends BaseService { * @param data Cancelation reason. */ public cancelDeployment( - authenticatedEnv: Pick, + authenticatedEnv: Pick, friendlyId: string, data?: Partial> ) { @@ -246,7 +246,7 @@ export class DeploymentService extends BaseService { cause: error, })); - return this.getDeployment(authenticatedEnv.projectId, friendlyId) + return this.getDeployment(authenticatedEnv.id, friendlyId) .andThen(validateDeployment) .andThen(cancelDeployment) .andThen(({ deployment }) => @@ -278,7 +278,7 @@ export class DeploymentService extends BaseService { * @param friendlyId The friendly deployment ID. */ public generateRegistryCredentials( - authenticatedEnv: Pick, + authenticatedEnv: Pick, friendlyId: string ) { const validateDeployment = ( @@ -326,7 +326,7 @@ export class DeploymentService extends BaseService { }); }); - return this.getDeployment(authenticatedEnv.projectId, friendlyId) + return this.getDeployment(authenticatedEnv.id, friendlyId) .andThen(validateDeployment) .andThen(getDeploymentRegion) .andThen(generateCredentials); @@ -472,12 +472,12 @@ export class DeploymentService extends BaseService { ); } - private getDeployment(projectId: string, friendlyId: string) { + private getDeployment(environmentId: string, friendlyId: string) { return fromPromise( this._prisma.workerDeployment.findFirst({ where: { friendlyId, - projectId, + environmentId, }, select: { status: true, diff --git a/apps/webapp/app/v3/services/initializeDeployment.server.ts b/apps/webapp/app/v3/services/initializeDeployment.server.ts index 1eef2e12b..abb99082d 100644 --- a/apps/webapp/app/v3/services/initializeDeployment.server.ts +++ b/apps/webapp/app/v3/services/initializeDeployment.server.ts @@ -286,7 +286,7 @@ export class InitializeDeploymentService extends BaseService { }); return deploymentService - .cancelDeployment(environment, deployment.friendlyId, { + .cancelDeployment({ id: environment.id }, deployment.friendlyId, { canceledReason: "Failed to enqueue build, please try again shortly.", }) .orTee((cancelError) => diff --git a/apps/webapp/app/v3/services/resolveProjectScopedEnvironments.ts b/apps/webapp/app/v3/services/resolveProjectScopedEnvironments.ts new file mode 100644 index 000000000..631eefac7 --- /dev/null +++ b/apps/webapp/app/v3/services/resolveProjectScopedEnvironments.ts @@ -0,0 +1,27 @@ +// Resolve a caller-supplied list of environment ids against the environments +// that belong to the authorized project. Any id not in the project is reported +// as `foreign` so the caller can reject the request rather than silently drop +// it. Returns a discriminated result instead of throwing so it stays +// dependency-free and unit-testable. + +export type ProjectScopedEnvironmentsResult = + | { kind: "foreign"; foreignEnvironmentId: string } + | { kind: "ok"; environments: E[] }; + +export function resolveProjectScopedEnvironments( + environmentIds: string[], + projectEnvironments: ReadonlyArray +): ProjectScopedEnvironmentsResult { + const byId = new Map(projectEnvironments.map((e) => [e.id, e])); + + const foreignEnvironmentId = environmentIds.find((id) => !byId.has(id)); + // Explicit undefined check: an empty-string id is a foreign id that must be + // rejected, but it is falsy, so a truthiness test would silently drop it + // instead. + if (foreignEnvironmentId !== undefined) { + return { kind: "foreign", foreignEnvironmentId }; + } + + const environments = environmentIds.map((id) => byId.get(id)).filter((e): e is E => Boolean(e)); + return { kind: "ok", environments }; +} diff --git a/apps/webapp/app/v3/services/setActiveOnTaskSchedule.server.ts b/apps/webapp/app/v3/services/setActiveOnTaskSchedule.server.ts index 5afa96d7c..a78e4722e 100644 --- a/apps/webapp/app/v3/services/setActiveOnTaskSchedule.server.ts +++ b/apps/webapp/app/v3/services/setActiveOnTaskSchedule.server.ts @@ -1,3 +1,4 @@ +import { scheduleUniqWhereClause, scheduleWhereClause } from "~/models/schedules.server"; import { BaseService } from "./baseService.server"; type Options = { @@ -29,9 +30,7 @@ export class SetActiveOnTaskScheduleService extends BaseService { try { const schedule = await this._prisma.taskSchedule.findFirst({ - where: { - friendlyId, - }, + where: scheduleWhereClause(projectId, friendlyId), }); if (!schedule) { @@ -43,9 +42,7 @@ export class SetActiveOnTaskScheduleService extends BaseService { } await this._prisma.taskSchedule.update({ - where: { - friendlyId, - }, + where: scheduleUniqWhereClause(projectId, friendlyId), data: { active, }, diff --git a/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts b/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts index b16600656..23d20e108 100644 --- a/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts +++ b/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts @@ -18,11 +18,14 @@ import { fromFriendlyId } from "@trigger.dev/core/v3/isomorphic"; import { WORKER_HEADERS, type WorkerQueueClass } from "@trigger.dev/core/v3/workers"; import type { RuntimeEnvironment, WorkerInstanceGroup } from "@trigger.dev/database"; import { Prisma, WorkerInstanceGroupType } from "@trigger.dev/database"; -import { SENSITIVE_WORKER_HEADERS, sanitizeWorkerHeaders } from "./sanitizeWorkerHeaders"; +import { json } from "@remix-run/server-runtime"; import { createHash, timingSafeEqual } from "crypto"; import { customAlphabet } from "nanoid"; +import { Counter } from "prom-client"; import { z } from "zod"; import { env } from "~/env.server"; +import { metricsRegister } from "~/metrics.server"; +import { evaluateCreatedAtGate } from "./workloadTokenAuthorization.server"; import { isWorkerQueueDequeueDisabled, recordBlockedDequeue, @@ -42,6 +45,30 @@ const authenticatedWorkerInstanceCache = singleton( createAuthenticatedWorkerInstanceCache ); +// Opt-in suppression of untokened worker actions on runs created after the cutoff. Only the +// no-header path ever reads a run row, and only when this is on - default off means feature-off is +// byte-for-byte today's behavior (no extra reads). Tenant scoping itself is header-driven (folded +// into the engine snapshot read) and needs no platform flag. +const workloadCreatedAtGateEnabled = env.WORKLOAD_CREATED_AT_GATE_ENABLED === "1"; +const workloadTokenCutoff = env.WORKLOAD_TOKEN_CUTOFF + ? new Date(env.WORKLOAD_TOKEN_CUTOFF) + : undefined; + +if (workloadCreatedAtGateEnabled && !workloadTokenCutoff) { + logger.warn( + "WORKLOAD_CREATED_AT_GATE_ENABLED is set but WORKLOAD_TOKEN_CUTOFF is missing; the created-at gate stays off until a cutoff is configured" + ); +} + +type WorkloadGateAction = "start" | "complete" | "continue" | "snapshots_since"; + +const workloadAuthGateCounter = new Counter({ + name: "workload_auth_gate_total", + help: "Deployment token authorization outcomes on worker actions", + labelNames: ["outcome", "action"] as const, + registers: [metricsRegister], +}); + function createAuthenticatedWorkerInstanceCache() { return createCache({ authenticatedWorkerInstance: new Namespace( @@ -188,6 +215,7 @@ export class WorkerGroupTokenService extends WithRunEngine { if (a.byteLength !== b.byteLength) { logger.error("[WorkerGroupTokenService] Managed secret length mismatch", { + managedWorkerSecret, headers: this.sanitizeHeaders(request), }); return; @@ -195,6 +223,7 @@ export class WorkerGroupTokenService extends WithRunEngine { if (!timingSafeEqual(a, b)) { logger.error("[WorkerGroupTokenService] Managed secret mismatch", { + managedWorkerSecret, headers: this.sanitizeHeaders(request), }); return; @@ -316,10 +345,16 @@ export class WorkerGroupTokenService extends WithRunEngine { } } - // Strip sensitive headers before logging request headers — see - // `sanitizeWorkerHeaders`. - private sanitizeHeaders(request: Request, denylist = SENSITIVE_WORKER_HEADERS) { - return sanitizeWorkerHeaders(request.headers, denylist); + private sanitizeHeaders(request: Request, skipHeaders = ["authorization"]) { + const sanitizedHeaders: Partial> = {}; + + for (const [key, value] of request.headers.entries()) { + if (!skipHeaders.includes(key.toLowerCase())) { + sanitizedHeaders[key] = value; + } + } + + return sanitizedHeaders; } } @@ -398,6 +433,55 @@ export class AuthenticatedWorkerInstance extends WithRunEngine { }); } + /** + * The no-header fallback. When the env header is present the engine scopes the snapshot read by it + * (nothing to do here). When it's absent, this optionally suppresses runs created after the cutoff - + * a run that new enough should have carried a token, so a missing one is treated as out-of-scope. + * Only runs when the gate is enabled AND a cutoff is set; that's the ONLY path that reads a run row. + */ + private async assertCreatedAtGate({ + runId, + environmentId, + action, + }: { + runId: string; + environmentId?: string; + action: WorkloadGateAction; + }): Promise { + if (environmentId) { + // Scoping is delegated to the engine snapshot read; no run-row read here. Recorded so the + // platform can see how much traffic is env-scoped as enforcement rolls out. + workloadAuthGateCounter.inc({ outcome: "env_scoped", action }); + return; + } + + if (!workloadCreatedAtGateEnabled || !workloadTokenCutoff) { + return; + } + + const run = await this._engine.runStore.findRun({ id: runId }, { select: { createdAt: true } }); + + if (!run) { + // Let the engine method surface the canonical not-found error. + return; + } + + const { allow, outcome } = evaluateCreatedAtGate({ + runCreatedAt: run.createdAt, + cutoff: workloadTokenCutoff, + }); + + workloadAuthGateCounter.inc({ outcome, action }); + + if (!allow) { + logger.warn("[workload-auth] rejecting untokened worker action created after cutoff", { + action, + runId, + }); + throw json({ error: "Run does not belong to this worker" }, { status: 403 }); + } + } + async heartbeatRun({ runFriendlyId, snapshotFriendlyId, @@ -420,22 +504,31 @@ export class AuthenticatedWorkerInstance extends WithRunEngine { snapshotFriendlyId, isWarmStart, runnerId, + environmentId, }: { runFriendlyId: string; snapshotFriendlyId: string; isWarmStart?: boolean; runnerId?: string; + environmentId?: string; }): Promise< StartRunAttemptResult & { envVars: Record; } > { + await this.assertCreatedAtGate({ + runId: fromFriendlyId(runFriendlyId), + environmentId, + action: "start", + }); + const engineResult = await this._engine.startRunAttempt({ runId: fromFriendlyId(runFriendlyId), snapshotId: fromFriendlyId(snapshotFriendlyId), isWarmStart, workerId: this.workerInstanceId, runnerId, + environmentId, }); const defaultMachinePreset = machinePresetFromName(defaultMachine); @@ -470,24 +563,42 @@ export class AuthenticatedWorkerInstance extends WithRunEngine { snapshotFriendlyId, completion, runnerId, + environmentId, }: { runFriendlyId: string; snapshotFriendlyId: string; completion: TaskRunExecutionResult; runnerId?: string; + environmentId?: string; }): Promise { + await this.assertCreatedAtGate({ + runId: fromFriendlyId(runFriendlyId), + environmentId, + action: "complete", + }); + return await this._engine.completeRunAttempt({ runId: fromFriendlyId(runFriendlyId), snapshotId: fromFriendlyId(snapshotFriendlyId), completion, workerId: this.workerInstanceId, runnerId, + environmentId, }); } - async getLatestSnapshot({ runFriendlyId }: { runFriendlyId: string }) { + async getLatestSnapshot({ + runFriendlyId, + environmentId, + }: { + runFriendlyId: string; + environmentId?: string; + }) { + // No created-at gate: the only untokened caller is an internal warm-start poll that legitimately + // has no token, so an absent header must not reject. When a header is present the engine scopes. return await this._engine.getRunExecutionData({ runId: fromFriendlyId(runFriendlyId), + environmentId, }); } @@ -515,29 +626,47 @@ export class AuthenticatedWorkerInstance extends WithRunEngine { runFriendlyId, snapshotFriendlyId, runnerId, + environmentId, }: { runFriendlyId: string; snapshotFriendlyId: string; runnerId?: string; + environmentId?: string; }) { + await this.assertCreatedAtGate({ + runId: fromFriendlyId(runFriendlyId), + environmentId, + action: "continue", + }); + return await this._engine.continueRunExecution({ runId: fromFriendlyId(runFriendlyId), snapshotId: fromFriendlyId(snapshotFriendlyId), workerId: this.workerInstanceId, runnerId, + environmentId, }); } async getSnapshotsSince({ runFriendlyId, snapshotId, + environmentId, }: { runFriendlyId: string; snapshotId: string; + environmentId?: string; }) { + await this.assertCreatedAtGate({ + runId: fromFriendlyId(runFriendlyId), + environmentId, + action: "snapshots_since", + }); + return await this._engine.getSnapshotsSince({ runId: fromFriendlyId(runFriendlyId), snapshotId: fromFriendlyId(snapshotId), + environmentId, }); } diff --git a/apps/webapp/app/v3/services/worker/workloadTokenAuthorization.server.ts b/apps/webapp/app/v3/services/worker/workloadTokenAuthorization.server.ts new file mode 100644 index 000000000..a38097b62 --- /dev/null +++ b/apps/webapp/app/v3/services/worker/workloadTokenAuthorization.server.ts @@ -0,0 +1,27 @@ +/** + * The no-header created-at gate. Tenant scoping is handled by the env-scoped snapshot read in the + * engine; this only covers the fallback where no verified env header was forwarded (caller predates + * tokens, or the supervisor isn't enforcing): + * + * run created on/before cutoff -> allow (grandfather legacy untokened runs) + * run created after cutoff -> reject (a new-enough run should have carried a token) + * + * Pure and env-import-free so it stays trivially testable. + */ + +export type CreatedAtGateOutcome = "grandfathered" | "suppressed"; + +export type CreatedAtGateEvaluation = { + outcome: CreatedAtGateOutcome; + allow: boolean; +}; + +export function evaluateCreatedAtGate(params: { + runCreatedAt: Date; + cutoff: Date; +}): CreatedAtGateEvaluation { + const createdAfterCutoff = params.runCreatedAt.getTime() > params.cutoff.getTime(); + return createdAfterCutoff + ? { outcome: "suppressed", allow: false } + : { outcome: "grandfathered", allow: true }; +} diff --git a/apps/webapp/app/v3/workerIdUnwrap.server.ts b/apps/webapp/app/v3/workerIdUnwrap.server.ts new file mode 100644 index 000000000..7300b7406 --- /dev/null +++ b/apps/webapp/app/v3/workerIdUnwrap.server.ts @@ -0,0 +1,75 @@ +// Decodes a `worker_id` that may be a deployment token back to its friendlyId, so the stored/queried +// value stays a short, stable id. Runs on the OTEL ingest hot path: base64url + JSON decode only (no +// verify), LRU-memoized. Fails safe — a non-token value passes through unchanged. + +import { LRUCache } from "lru-cache"; +import { SemanticInternalAttributes } from "@trigger.dev/core/v3"; + +// The runner flattens `worker.id` (the raw TRIGGER_DEPLOYMENT_ID) into span/log metadata under this +// key, mirroring how the runner composes it, so it can't drift. +const WORKER_ID_METADATA_KEY = `${SemanticInternalAttributes.METADATA}.${SemanticInternalAttributes.WORKER_ID}`; + +/** + * Unwrap, in place, the `worker.id` a runner stamps into span/log metadata: a deployment-token value + * becomes its friendlyId, everything else is left untouched. Keeps the credential out of stored + * telemetry and the value a stable deployment id (the metrics path unwraps its own worker_id). + */ +export function unwrapWorkerIdInMetadata< + T extends Record, +>(metadata: T | undefined): T | undefined { + if (metadata) { + const value = metadata[WORKER_ID_METADATA_KEY]; + if (typeof value === "string") { + (metadata as Record)[WORKER_ID_METADATA_KEY] = + unwrapWorkerId(value); + } + } + return metadata; +} + +// LRU, not FIFO: the active-deployment working set is unbounded, so an insertion-order memo thrashes +// once it exceeds the cap. Raw lru-cache (not @internal/cache's async wrapper) since this is sync. +const memo = new LRUCache({ max: 32_768 }); + +export function unwrapWorkerId(value: string | undefined): string | undefined { + if (!value) { + return value; + } + + // Fast path: a minted token is a JWT, whose base64url header always begins "eyJ". Anything else — + // a bare deployment friendlyId, "unmanaged", a dev id — returns immediately with no decode and no + // memo entry, so the non-token case costs nothing (and stays free once telemetry no longer carries + // tokens at all). + if (!value.startsWith("eyJ")) { + return value; + } + + const cached = memo.get(value); + if (cached !== undefined) { + return cached; + } + + const unwrapped = decodeDeploymentFriendlyId(value) ?? value; + memo.set(value, unwrapped); + + return unwrapped; +} + +function decodeDeploymentFriendlyId(value: string): string | undefined { + const parts = value.split("."); + // Not a JWT (three non-empty segments) → a legacy bare friendlyId; leave it as-is. + if (parts.length !== 3 || !parts[1]) { + return undefined; + } + + try { + const payload = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")) as { + deployment?: unknown; + }; + return typeof payload.deployment === "string" && payload.deployment.length > 0 + ? payload.deployment + : undefined; + } catch { + return undefined; + } +} diff --git a/apps/webapp/app/v3/writableEnvironments.ts b/apps/webapp/app/v3/writableEnvironments.ts new file mode 100644 index 000000000..af3ff12e8 --- /dev/null +++ b/apps/webapp/app/v3/writableEnvironments.ts @@ -0,0 +1,33 @@ +// Which environments a user may write env vars to. Shared env types +// (preview/staging/production) are writable by any project member; DEVELOPMENT +// environments are per-user and only writable by their owner. + +export type WriteCheckEnvironment = { + id: string; + type: string; + orgMember: { userId: string } | null; +}; + +const SHARED_ENV_TYPES = new Set(["PREVIEW", "STAGING", "PRODUCTION"]); + +/** + * Return the first submitted id the user may NOT write to — either it isn't one + * of the project's environments or it's a DEV env owned by someone else. + * Returns null when every submitted id is writable by `userId`. + */ +export function findUnauthorizedEnvironmentId( + projectEnvironments: ReadonlyArray, + submittedIds: ReadonlyArray, + userId: string +): string | null { + const byId = new Map(projectEnvironments.map((e) => [e.id, e])); + for (const id of submittedIds) { + const env = byId.get(id); + if (!env) return id; + const writable = + SHARED_ENV_TYPES.has(env.type) || + (env.type === "DEVELOPMENT" && env.orgMember?.userId === userId); + if (!writable) return id; + } + return null; +} diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 31b2bb9f8..60495a471 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -154,6 +154,9 @@ "isbot": "^3.6.5", "jose": "^5.4.0", "json-stable-stringify": "^1.3.0", + "jsonpointer": "^5.0.1", + "lodash.omit": "^4.5.0", + "lru-cache": "^11.2.4", "lucide-react": "^0.229.0", "marked": "^4.0.18", "match-sorter": "^6.3.4", diff --git a/apps/webapp/server.ts b/apps/webapp/server.ts index 3794441ab..fb4e20359 100644 --- a/apps/webapp/server.ts +++ b/apps/webapp/server.ts @@ -147,6 +147,7 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { const wss: WebSocketServer | undefined = build.entry.module.wss; const apiRateLimiter: RateLimitMiddleware = build.entry.module.apiRateLimiter; const engineRateLimiter: RateLimitMiddleware = build.entry.module.engineRateLimiter; + const otlpRateLimiter: RequestHandler = build.entry.module.otlpRateLimiter; const runWithHttpContext: RunWithHttpContextFunction = build.entry.module.runWithHttpContext; const tenantContextMiddleware: RequestHandler = build.entry.module.tenantContextMiddleware; @@ -198,6 +199,7 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { app.use(apiRateLimiter); app.use(engineRateLimiter); + app.use(otlpRateLimiter); app.use(tenantContextMiddleware); diff --git a/apps/webapp/test/authorizationCodeConsent.test.ts b/apps/webapp/test/authorizationCodeConsent.test.ts new file mode 100644 index 000000000..85cda08da --- /dev/null +++ b/apps/webapp/test/authorizationCodeConsent.test.ts @@ -0,0 +1,58 @@ +import { containerTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { + AUTHORIZATION_CODE_TTL_MS, + isAuthorizationCodeMintable, +} from "~/services/personalAccessToken.server"; + +vi.setConfig({ testTimeout: 30_000 }); + +function randomCode() { + return `code_${Math.random().toString(36).slice(2)}${Math.random().toString(36).slice(2)}`; +} + +// Lock the read-only + TTL properties `isAuthorizationCodeMintable` relies on: +// the loader must not bind a PAT, and expired codes must not be mintable. +describe("authorization code consent gate", () => { + containerTest( + "a fresh, unconsumed code is mintable and checking it does NOT bind a PAT", + async ({ prisma }) => { + const created = await prisma.authorizationCode.create({ data: { code: randomCode() } }); + + expect(await isAuthorizationCodeMintable(created.code, prisma)).toBe(true); + + // The check is read-only — it must not bind a Personal Access Token. + const after = await prisma.authorizationCode.findFirst({ where: { id: created.id } }); + expect(after?.personalAccessTokenId).toBeNull(); + } + ); + + containerTest("a code older than the TTL is not mintable", async ({ prisma }) => { + const created = await prisma.authorizationCode.create({ data: { code: randomCode() } }); + + await prisma.authorizationCode.update({ + where: { id: created.id }, + data: { createdAt: new Date(Date.now() - AUTHORIZATION_CODE_TTL_MS - 1_000) }, + }); + + expect(await isAuthorizationCodeMintable(created.code, prisma)).toBe(false); + }); + + containerTest( + "a code created just inside the TTL is still mintable (CLI flow not broken)", + async ({ prisma }) => { + const created = await prisma.authorizationCode.create({ data: { code: randomCode() } }); + + await prisma.authorizationCode.update({ + where: { id: created.id }, + data: { createdAt: new Date(Date.now() - (AUTHORIZATION_CODE_TTL_MS - 30_000)) }, + }); + + expect(await isAuthorizationCodeMintable(created.code, prisma)).toBe(true); + } + ); + + containerTest("an unknown code is not mintable", async ({ prisma }) => { + expect(await isAuthorizationCodeMintable(randomCode(), prisma)).toBe(false); + }); +}); diff --git a/apps/webapp/test/checkSchedule.test.ts b/apps/webapp/test/checkSchedule.test.ts new file mode 100644 index 000000000..bd872bbce --- /dev/null +++ b/apps/webapp/test/checkSchedule.test.ts @@ -0,0 +1,60 @@ +import { containerTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; +import { resolveProjectScopedEnvironments } from "~/v3/services/resolveProjectScopedEnvironments"; + +vi.setConfig({ testTimeout: 60_000 }); + +// Exercises the environment-scoping primitive CheckScheduleService relies on +// (`resolveProjectScopedEnvironments`) with real RuntimeEnvironment rows, +// imported directly to avoid `~/db.server` and its eager global-prisma connect. + +async function seedProjectWithEnv(prisma: PrismaClient, slugBase: string) { + const slug = `${slugBase}_${Math.random().toString(36).slice(2, 10)}`; + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: slug }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + slug: `${slug}-prod`, + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_prod_${slug}`, + pkApiKey: `pk_prod_${slug}`, + shortcode: slug.slice(0, 6), + }, + }); + return { organization, project, environment }; +} + +function projectEnvironments(prisma: PrismaClient, projectId: string) { + return prisma.runtimeEnvironment.findMany({ where: { projectId }, select: { id: true } }); +} + +describe("resolveProjectScopedEnvironments (schedule env scoping)", () => { + containerTest("rejects an environment id that belongs to another project", async ({ prisma }) => { + const a = await seedProjectWithEnv(prisma, "orga"); + const b = await seedProjectWithEnv(prisma, "orgb"); + + const result = resolveProjectScopedEnvironments( + [a.environment.id, b.environment.id], + await projectEnvironments(prisma, a.project.id) + ); + + expect(result.kind).toBe("foreign"); + expect(result).toMatchObject({ foreignEnvironmentId: b.environment.id }); + }); + + containerTest("accepts environment ids that belong to the project", async ({ prisma }) => { + const a = await seedProjectWithEnv(prisma, "orga"); + + const result = resolveProjectScopedEnvironments( + [a.environment.id], + await projectEnvironments(prisma, a.project.id) + ); + + expect(result.kind).toBe("ok"); + }); +}); diff --git a/apps/webapp/test/deleteTaskSchedule.test.ts b/apps/webapp/test/deleteTaskSchedule.test.ts new file mode 100644 index 000000000..320b44bf4 --- /dev/null +++ b/apps/webapp/test/deleteTaskSchedule.test.ts @@ -0,0 +1,75 @@ +import { containerTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; +import { scheduleWhereClause } from "~/models/schedules.server"; + +vi.setConfig({ testTimeout: 60_000 }); + +// Exercises the project-scoping primitive DeleteTaskScheduleService relies on +// (`scheduleWhereClause`) directly against a real database, to avoid importing +// `~/db.server` and its eager global-prisma connect. + +async function seedProject(prisma: PrismaClient, slugBase: string) { + const slug = `${slugBase}_${Math.random().toString(36).slice(2, 10)}`; + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: slug }, + }); + return { organization, project }; +} + +function seedSchedule(prisma: PrismaClient, projectId: string, friendlyId: string) { + return prisma.taskSchedule.create({ + data: { + friendlyId, + taskIdentifier: "my-task", + projectId, + generatorExpression: "0 * * * *", + generatorDescription: "every hour", + type: "IMPERATIVE", + }, + }); +} + +describe("scheduleWhereClause (delete lookup scoping)", () => { + containerTest( + "a schedule from another project is not found when scoped to the caller's project", + async ({ prisma }) => { + const a = await seedProject(prisma, "orga"); + const b = await seedProject(prisma, "orgb"); + const victim = await seedSchedule( + prisma, + b.project.id, + `sched_${Math.random().toString(36).slice(2, 10)}` + ); + + // Scoped to A's project: the cross-tenant schedule is invisible (the + // `projectId` in the where is what prevents a cross-project delete). + const fromA = await prisma.taskSchedule.findFirst({ + where: scheduleWhereClause(a.project.id, victim.friendlyId), + }); + expect(fromA).toBeNull(); + + // Scoped to its own project: found. + const fromB = await prisma.taskSchedule.findFirst({ + where: scheduleWhereClause(b.project.id, victim.friendlyId), + }); + expect(fromB?.id).toBe(victim.id); + } + ); + + containerTest("the where pins projectId for both id shapes", async ({ prisma }) => { + const a = await seedProject(prisma, "orga"); + + // friendlyId shape + expect(scheduleWhereClause(a.project.id, "sched_abc")).toMatchObject({ + friendlyId: "sched_abc", + projectId: a.project.id, + }); + // deduplicationKey shape + expect(scheduleWhereClause(a.project.id, "my-dedup-key")).toMatchObject({ + projectId: a.project.id, + deduplicationKey: "my-dedup-key", + }); + }); +}); diff --git a/apps/webapp/test/env.server.test.ts b/apps/webapp/test/env.server.test.ts new file mode 100644 index 000000000..0fe13efbe --- /dev/null +++ b/apps/webapp/test/env.server.test.ts @@ -0,0 +1,84 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +const originalEnv = process.env; + +const requiredEnv = { + NODE_ENV: "test", + DATABASE_URL: "postgresql://test:test@localhost:5432/test", + DIRECT_URL: "postgresql://test:test@localhost:5432/test", + SESSION_SECRET: "test-session-secret", + MAGIC_LINK_SECRET: "test-magic-link-secret", + ENCRYPTION_KEY: "test-encryption-keeeeey-32-bytes", + CLICKHOUSE_URL: "http://localhost:8123", + DEPLOY_REGISTRY_HOST: "registry.example.com", + MANAGED_WORKER_SECRET: "test-managed-worker-secret", +}; + +describe("webapp environment secrets", () => { + afterEach(() => { + process.env = originalEnv; + vi.resetModules(); + }); + + it.each(["SESSION_SECRET", "MAGIC_LINK_SECRET", "ENCRYPTION_KEY", "MANAGED_WORKER_SECRET"])( + "requires %s to be explicitly set", + async (key) => { + process.env = { ...requiredEnv }; + delete process.env[key]; + + await expect(import("../app/env.server")).rejects.toThrow(key); + } + ); + + it.each(["SESSION_SECRET", "MAGIC_LINK_SECRET", "ENCRYPTION_KEY", "MANAGED_WORKER_SECRET"])( + "rejects an empty %s", + async (key) => { + process.env = { ...requiredEnv, [key]: "" }; + + await expect(import("../app/env.server")).rejects.toThrow(key); + } + ); + + it.each([ + ["SESSION_SECRET", "2818143646516f6fffd707b36f334bbb"], + ["MAGIC_LINK_SECRET", "44da78b7bbb0dfe709cf38931d25dcdd"], + ["ENCRYPTION_KEY", "f686147ab967943ebbe9ed3b496e465a"], + ["MANAGED_WORKER_SECRET", "managed-secret"], + ["MANAGED_WORKER_SECRET", "447c29678f9eaf289e9c4b70d3dd8a7f"], + ])("rejects the known-insecure default value for %s", async (key, insecureValue) => { + process.env = { ...requiredEnv, [key]: insecureValue }; + + await expect(import("../app/env.server")).rejects.toThrow(key); + }); + + it("accepts explicitly configured secrets", async () => { + process.env = { ...requiredEnv }; + + const { env } = await import("../app/env.server"); + + expect(env.SESSION_SECRET).toBe(requiredEnv.SESSION_SECRET); + expect(env.MAGIC_LINK_SECRET).toBe(requiredEnv.MAGIC_LINK_SECRET); + expect(env.ENCRYPTION_KEY).toBe(requiredEnv.ENCRYPTION_KEY); + expect(env.MANAGED_WORKER_SECRET).toBe(requiredEnv.MANAGED_WORKER_SECRET); + }); + + it("allows a known-insecure default when ALLOW_INSECURE_DEFAULT_SECRETS is set", async () => { + process.env = { + ...requiredEnv, + ALLOW_INSECURE_DEFAULT_SECRETS: "1", + ENCRYPTION_KEY: "f686147ab967943ebbe9ed3b496e465a", + MANAGED_WORKER_SECRET: "managed-secret", + }; + + const { env } = await import("../app/env.server"); + + expect(env.ENCRYPTION_KEY).toBe("f686147ab967943ebbe9ed3b496e465a"); + expect(env.MANAGED_WORKER_SECRET).toBe("managed-secret"); + }); + + it("still rejects an empty secret even with ALLOW_INSECURE_DEFAULT_SECRETS", async () => { + process.env = { ...requiredEnv, ALLOW_INSECURE_DEFAULT_SECRETS: "1", SESSION_SECRET: "" }; + + await expect(import("../app/env.server")).rejects.toThrow("SESSION_SECRET"); + }); +}); diff --git a/apps/webapp/test/environmentVariablesRepository.test.ts b/apps/webapp/test/environmentVariablesRepository.test.ts index 4025d8fd2..b5661c89c 100644 --- a/apps/webapp/test/environmentVariablesRepository.test.ts +++ b/apps/webapp/test/environmentVariablesRepository.test.ts @@ -180,4 +180,80 @@ describe("EnvironmentVariablesRepository.getVariableValuesForKeys", () => { expect(crossProjectRequest.size).toBe(0); }); + + postgresTest( + "create() rejects a mix of in-project and foreign environmentIds without writing foreign values", + async ({ prisma }) => { + const { + user, + organization, + project: projectA, + } = await createTestOrgProjectWithMember(prisma); + + const projectB = await prisma.project.create({ + data: { + name: "Project B", + slug: `proj-b-${Date.now()}`, + organizationId: organization.id, + externalRef: `ext-b-${Date.now()}`, + }, + }); + + const envA = await createRuntimeEnvironment(prisma, { + projectId: projectA.id, + organizationId: organization.id, + type: "PRODUCTION", + }); + const envB = await createRuntimeEnvironment(prisma, { + projectId: projectB.id, + organizationId: organization.id, + type: "PRODUCTION", + }); + + const repository = new EnvironmentVariablesRepository(prisma, prisma); + + // Caller scoped to projectA supplies a mixed array: an in-project env + // (envA) plus a foreign one (envB). The whole request must be refused. + const result = await repository.create(projectA.id, { + override: true, + environmentIds: [envA.id, envB.id], + variables: [{ key: "CROSS_TENANT", value: "x" }], + isSecret: false, + lastUpdatedBy: { type: "user", userId: user.id }, + }); + + expect(result.success).toBe(false); + + // No value row may have been written against the foreign environment. + const foreignValues = await prisma.environmentVariableValue.findMany({ + where: { environmentId: envB.id }, + }); + expect(foreignValues).toHaveLength(0); + } + ); + + postgresTest( + "create() still succeeds for an all-in-project environmentIds array", + async ({ prisma }) => { + const { user, organization, project } = await createTestOrgProjectWithMember(prisma); + + const environment = await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "PRODUCTION", + }); + + const repository = new EnvironmentVariablesRepository(prisma, prisma); + + const result = await repository.create(project.id, { + override: true, + environmentIds: [environment.id], + variables: [{ key: "OK_KEY", value: "v" }], + isSecret: false, + lastUpdatedBy: { type: "user", userId: user.id }, + }); + + expect(result.success).toBe(true); + } + ); }); diff --git a/apps/webapp/test/registryConfig.test.ts b/apps/webapp/test/registryConfig.test.ts index 341761d01..b9a8c9661 100644 --- a/apps/webapp/test/registryConfig.test.ts +++ b/apps/webapp/test/registryConfig.test.ts @@ -10,6 +10,9 @@ describe("getRegistryConfig", () => { MAGIC_LINK_SECRET: "test-magic-link-secret", ENCRYPTION_KEY: "test-encryption-keeeeey-32-bytes", CLICKHOUSE_URL: "http://localhost:8123", + PROVIDER_SECRET: "test-provider-secret", + COORDINATOR_SECRET: "test-coordinator-secret", + MANAGED_WORKER_SECRET: "test-managed-worker-secret", }; beforeEach(() => { diff --git a/apps/webapp/test/resolveProjectScopedEnvironments.test.ts b/apps/webapp/test/resolveProjectScopedEnvironments.test.ts new file mode 100644 index 000000000..f04a623d6 --- /dev/null +++ b/apps/webapp/test/resolveProjectScopedEnvironments.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { resolveProjectScopedEnvironments } from "../app/v3/services/resolveProjectScopedEnvironments.js"; + +const projectEnvs = [{ id: "env_prod" }, { id: "env_staging" }, { id: "env_dev" }]; + +// A submitted environment id that doesn't belong to the project must be +// rejected, not silently dropped. +describe("resolveProjectScopedEnvironments", () => { + it("resolves ids that all belong to the project", () => { + const r = resolveProjectScopedEnvironments(["env_prod", "env_dev"], projectEnvs); + expect(r.kind).toBe("ok"); + if (r.kind === "ok") expect(r.environments.map((e) => e.id)).toEqual(["env_prod", "env_dev"]); + }); + + it("rejects a foreign environment id (the cross-tenant vector)", () => { + const r = resolveProjectScopedEnvironments(["env_someone_elses"], projectEnvs); + expect(r.kind).toBe("foreign"); + if (r.kind === "foreign") expect(r.foreignEnvironmentId).toBe("env_someone_elses"); + }); + + it("rejects when a foreign id is mixed in with valid ones (not silently dropped)", () => { + const r = resolveProjectScopedEnvironments(["env_prod", "env_foreign"], projectEnvs); + expect(r.kind).toBe("foreign"); + if (r.kind === "foreign") expect(r.foreignEnvironmentId).toBe("env_foreign"); + }); + + it("returns an empty set for no ids", () => { + const r = resolveProjectScopedEnvironments([], projectEnvs); + expect(r.kind).toBe("ok"); + if (r.kind === "ok") expect(r.environments).toEqual([]); + }); + + it("rejects an empty-string id rather than dropping it (falsy edge case)", () => { + const r = resolveProjectScopedEnvironments([""], projectEnvs); + expect(r.kind).toBe("foreign"); + if (r.kind === "foreign") expect(r.foreignEnvironmentId).toBe(""); + }); + + it("rejects an empty-string id mixed in with valid ids", () => { + const r = resolveProjectScopedEnvironments(["env_prod", ""], projectEnvs); + expect(r.kind).toBe("foreign"); + if (r.kind === "foreign") expect(r.foreignEnvironmentId).toBe(""); + }); +}); diff --git a/apps/webapp/test/schedulesPutEnvScoping.test.ts b/apps/webapp/test/schedulesPutEnvScoping.test.ts new file mode 100644 index 000000000..4bc937f60 --- /dev/null +++ b/apps/webapp/test/schedulesPutEnvScoping.test.ts @@ -0,0 +1,261 @@ +import { containerTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; +import { getScheduleEnvVisibility } from "~/models/schedules.server"; + +vi.setConfig({ testTimeout: 60_000 }); + +async function seedProjectWithEnvs(prisma: PrismaClient, slugBase: string) { + const slug = `${slugBase}_${Math.random().toString(36).slice(2, 10)}`; + const organization = await prisma.organization.create({ + data: { title: slug, slug }, + }); + const project = await prisma.project.create({ + data: { + name: slug, + slug, + organizationId: organization.id, + externalRef: slug, + }, + }); + const prodEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_prod_${slug}`, + pkApiKey: `pk_prod_${slug}`, + shortcode: `p${slug.slice(0, 4)}`, + }, + }); + const stagingEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "staging", + type: "STAGING", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_staging_${slug}`, + pkApiKey: `pk_staging_${slug}`, + shortcode: `s${slug.slice(0, 4)}`, + }, + }); + return { organization, project, prodEnv, stagingEnv }; +} + +async function seedScheduleWithInstance( + prisma: PrismaClient, + projectId: string, + environmentId: string, + opts: { friendlyId?: string; deduplicationKey?: string } = {} +) { + const schedule = await prisma.taskSchedule.create({ + data: { + friendlyId: opts.friendlyId ?? `sched_${Math.random().toString(36).slice(2, 10)}`, + taskIdentifier: "my-task", + projectId, + generatorExpression: "0 * * * *", + generatorDescription: "every hour", + type: "IMPERATIVE", + ...(opts.deduplicationKey + ? { deduplicationKey: opts.deduplicationKey, userProvidedDeduplicationKey: true } + : {}), + }, + }); + await prisma.taskScheduleInstance.create({ + data: { + taskScheduleId: schedule.id, + environmentId, + projectId, + }, + }); + return schedule; +} + +describe("getScheduleEnvVisibility", () => { + containerTest( + "returns 'hidden' when an instance lives in a different env (by friendlyId)", + async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + const schedule = await seedScheduleWithInstance(prisma, env.project.id, env.prodEnv.id); + + const visibility = await getScheduleEnvVisibility( + prisma, + env.project.id, + schedule.friendlyId, + env.stagingEnv.id + ); + expect(visibility.status).toBe("hidden"); + } + ); + + containerTest("returns 'visible' when every instance is in caller env", async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + const schedule = await seedScheduleWithInstance(prisma, env.project.id, env.prodEnv.id); + + const visibility = await getScheduleEnvVisibility( + prisma, + env.project.id, + schedule.friendlyId, + env.prodEnv.id + ); + expect(visibility.status).toBe("visible"); + if (visibility.status === "visible") { + expect(visibility.schedule.id).toBe(schedule.id); + } + }); + + containerTest("returns 'missing' when no schedule exists", async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + + const visibility = await getScheduleEnvVisibility( + prisma, + env.project.id, + "sched_does_not_exist", + env.prodEnv.id + ); + expect(visibility.status).toBe("missing"); + }); + + containerTest("returns 'visible' when no instances exist yet", async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + const schedule = await prisma.taskSchedule.create({ + data: { + friendlyId: `sched_${Math.random().toString(36).slice(2, 10)}`, + taskIdentifier: "my-task", + projectId: env.project.id, + generatorExpression: "0 * * * *", + generatorDescription: "every hour", + type: "IMPERATIVE", + }, + }); + + const visibility = await getScheduleEnvVisibility( + prisma, + env.project.id, + schedule.friendlyId, + env.prodEnv.id + ); + expect(visibility.status).toBe("visible"); + }); + + containerTest( + "returns 'visible' from every environment a multi-env schedule spans", + async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + const schedule = await prisma.taskSchedule.create({ + data: { + friendlyId: `sched_${Math.random().toString(36).slice(2, 10)}`, + taskIdentifier: "my-task", + projectId: env.project.id, + generatorExpression: "0 * * * *", + generatorDescription: "every hour", + type: "IMPERATIVE", + }, + }); + await prisma.taskScheduleInstance.createMany({ + data: [ + { taskScheduleId: schedule.id, environmentId: env.prodEnv.id, projectId: env.project.id }, + { + taskScheduleId: schedule.id, + environmentId: env.stagingEnv.id, + projectId: env.project.id, + }, + ], + }); + + // The schedule list surfaces a schedule for any environment it has an + // instance in, so per-schedule reads/mutations must resolve the same + // way. A multi-env schedule is visible from each environment it spans. + const fromProd = await getScheduleEnvVisibility( + prisma, + env.project.id, + schedule.friendlyId, + env.prodEnv.id + ); + expect(fromProd.status).toBe("visible"); + + const fromStaging = await getScheduleEnvVisibility( + prisma, + env.project.id, + schedule.friendlyId, + env.stagingEnv.id + ); + expect(fromStaging.status).toBe("visible"); + } + ); + + containerTest( + "returns 'hidden' from an environment the schedule has no instance in", + async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + // A third environment with no instance of the schedule. + const devEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "dev", + type: "DEVELOPMENT", + projectId: env.project.id, + organizationId: env.organization.id, + apiKey: `tr_dev_${env.project.slug}`, + pkApiKey: `pk_dev_${env.project.slug}`, + shortcode: `d${env.project.slug.slice(0, 4)}`, + }, + }); + const schedule = await prisma.taskSchedule.create({ + data: { + friendlyId: `sched_${Math.random().toString(36).slice(2, 10)}`, + taskIdentifier: "my-task", + projectId: env.project.id, + generatorExpression: "0 * * * *", + generatorDescription: "every hour", + type: "IMPERATIVE", + }, + }); + await prisma.taskScheduleInstance.createMany({ + data: [ + { taskScheduleId: schedule.id, environmentId: env.prodEnv.id, projectId: env.project.id }, + { + taskScheduleId: schedule.id, + environmentId: env.stagingEnv.id, + projectId: env.project.id, + }, + ], + }); + + const fromDev = await getScheduleEnvVisibility( + prisma, + env.project.id, + schedule.friendlyId, + devEnv.id + ); + expect(fromDev.status).toBe("hidden"); + } + ); + + containerTest( + "resolves by user-provided deduplicationKey (the non-sched_ prefix branch)", + async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + const dedupKey = `my-daily-cleanup-${Math.random().toString(36).slice(2, 8)}`; + await seedScheduleWithInstance(prisma, env.project.id, env.prodEnv.id, { + deduplicationKey: dedupKey, + }); + + const hiddenFromStaging = await getScheduleEnvVisibility( + prisma, + env.project.id, + dedupKey, + env.stagingEnv.id + ); + expect(hiddenFromStaging.status).toBe("hidden"); + + const visibleFromProd = await getScheduleEnvVisibility( + prisma, + env.project.id, + dedupKey, + env.prodEnv.id + ); + expect(visibleFromProd.status).toBe("visible"); + } + ); +}); diff --git a/apps/webapp/test/setActiveOnTaskSchedule.test.ts b/apps/webapp/test/setActiveOnTaskSchedule.test.ts new file mode 100644 index 000000000..6c3b6e9ab --- /dev/null +++ b/apps/webapp/test/setActiveOnTaskSchedule.test.ts @@ -0,0 +1,149 @@ +import { containerTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; +import { + getScheduleEnvVisibility, + scheduleUniqWhereClause, + scheduleWhereClause, +} from "~/models/schedules.server"; + +vi.setConfig({ testTimeout: 60_000 }); + +// Exercises the project-scoping primitives SetActiveOnTaskScheduleService relies +// on (`scheduleWhereClause` + `scheduleUniqWhereClause`) directly against a real +// database, to avoid importing `~/db.server` and its eager global-prisma connect. + +async function seedProject(prisma: PrismaClient, slugBase: string) { + const slug = `${slugBase}_${Math.random().toString(36).slice(2, 10)}`; + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: slug }, + }); + return { organization, project }; +} + +function seedSchedule( + prisma: PrismaClient, + projectId: string, + friendlyId: string, + active: boolean = true +) { + return prisma.taskSchedule.create({ + data: { + friendlyId, + taskIdentifier: "my-task", + projectId, + generatorExpression: "0 * * * *", + generatorDescription: "every hour", + type: "IMPERATIVE", + active, + }, + }); +} + +describe("schedule scoping (enable/disable)", () => { + containerTest( + "toggling scoped to another project does not touch the victim schedule", + async ({ prisma }) => { + const a = await seedProject(prisma, "orga"); + const b = await seedProject(prisma, "orgb"); + const victim = await seedSchedule( + prisma, + b.project.id, + `sched_${Math.random().toString(36).slice(2, 10)}`, + true + ); + + // Project A cannot toggle B's schedule: the where pins projectId, so the + // update matches zero rows. + const result = await prisma.taskSchedule.updateMany({ + where: scheduleWhereClause(a.project.id, victim.friendlyId), + data: { active: false }, + }); + expect(result.count).toBe(0); + + const unchanged = await prisma.taskSchedule.findUnique({ where: { id: victim.id } }); + expect(unchanged?.active).toBe(true); + + // The owning project can toggle it. + const owned = await prisma.taskSchedule.updateMany({ + where: scheduleWhereClause(b.project.id, victim.friendlyId), + data: { active: false }, + }); + expect(owned.count).toBe(1); + } + ); + + containerTest("the unique-where pins projectId", async ({ prisma }) => { + const a = await seedProject(prisma, "orga"); + expect(scheduleUniqWhereClause(a.project.id, "sched_abc")).toMatchObject({ + friendlyId: "sched_abc", + projectId: a.project.id, + }); + }); + + // The public activate/deactivate endpoints gate on getScheduleEnvVisibility + // before toggling `active`. A key scoped to one environment must not be able + // to enable/disable a schedule that only runs in another environment of the + // same project. + containerTest( + "an env-scoped caller cannot toggle a schedule that only runs in another env", + async ({ prisma }) => { + const a = await seedProject(prisma, "orga"); + const prodEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: a.project.id, + organizationId: a.organization.id, + apiKey: `tr_prod_${a.project.slug}`, + pkApiKey: `pk_prod_${a.project.slug}`, + shortcode: `p${a.project.slug.slice(0, 4)}`, + }, + }); + const stagingEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "staging", + type: "STAGING", + projectId: a.project.id, + organizationId: a.organization.id, + apiKey: `tr_staging_${a.project.slug}`, + pkApiKey: `pk_staging_${a.project.slug}`, + shortcode: `s${a.project.slug.slice(0, 4)}`, + }, + }); + const schedule = await seedSchedule( + prisma, + a.project.id, + `sched_${Math.random().toString(36).slice(2, 10)}`, + true + ); + // The schedule only runs in staging. + await prisma.taskScheduleInstance.create({ + data: { + taskScheduleId: schedule.id, + environmentId: stagingEnv.id, + projectId: a.project.id, + }, + }); + + // A prod-scoped key is refused (the route returns 404 and never updates). + const fromProd = await getScheduleEnvVisibility( + prisma, + a.project.id, + schedule.friendlyId, + prodEnv.id + ); + expect(fromProd.status).toBe("hidden"); + + // The staging-scoped key that owns an instance can toggle it. + const fromStaging = await getScheduleEnvVisibility( + prisma, + a.project.id, + schedule.friendlyId, + stagingEnv.id + ); + expect(fromStaging.status).toBe("visible"); + } + ); +}); diff --git a/apps/webapp/test/setup.ts b/apps/webapp/test/setup.ts index db07cb140..521a9624b 100644 --- a/apps/webapp/test/setup.ts +++ b/apps/webapp/test/setup.ts @@ -13,6 +13,9 @@ config({ path: path.resolve(__dirname, "../.env") }); // the pair — the ioredis mock below forces lazyConnect, so nothing ever dials. process.env.REDIS_HOST ??= "localhost"; process.env.REDIS_PORT ??= "6379"; +process.env.PROVIDER_SECRET ??= "test-provider-secret"; +process.env.COORDINATOR_SECRET ??= "test-coordinator-secret"; +process.env.MANAGED_WORKER_SECRET ??= "test-managed-worker-secret"; // Worker singletons construct a RedisWorker at import time whose ioredis client // connects eagerly, so any test importing the service graph opens real Redis diff --git a/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts b/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts index f3df2d2ac..6b8dac279 100644 --- a/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts +++ b/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts @@ -475,7 +475,7 @@ describe("run-trace/span-detail route loaders under a lagging replica", () => { const seed = await seedTenant(prisma14, suffix); const runId = `run_${CUID_25}`; const friendlyId = `run_${suffix}`; - const traceId = `trace_${suffix}`; + const traceId = "a".repeat(32); const userId = `user_${suffix}`; // The dashboard user, joined to the org so the route's real orgMember check passes. @@ -538,7 +538,8 @@ describe("run-trace/span-detail route loaders under a lagging replica", () => { holder.resolvedEnv = { organizationId: seed.organization.id }; holder.replicaMarker = { orgMember: prisma14.orgMember }; - const res = (await syncTraceRunsLoader(syncRequest("trace_does_not_exist"))) as Response; + const res = (await syncTraceRunsLoader(syncRequest("b".repeat(32)))) as Response; + expect(lagged.legacyReplica.wasHit("taskRun")).toBe(true); expect(res.status).toBe(404); } ); diff --git a/apps/webapp/test/workerIdUnwrap.test.ts b/apps/webapp/test/workerIdUnwrap.test.ts new file mode 100644 index 000000000..ea787a811 --- /dev/null +++ b/apps/webapp/test/workerIdUnwrap.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; +import { mintWorkloadDeploymentToken, SemanticInternalAttributes } from "@trigger.dev/core/v3"; +import { unwrapWorkerId, unwrapWorkerIdInMetadata } from "~/v3/workerIdUnwrap.server"; + +const EXP = Math.floor(Date.UTC(2032, 0, 1) / 1000); +const WORKER_ID_KEY = `${SemanticInternalAttributes.METADATA}.${SemanticInternalAttributes.WORKER_ID}`; + +function mint(deployment: string) { + return mintWorkloadDeploymentToken( + { + deployment, + deployment_version: "20260709.1", + environment_id: "env_1", + environment_type: "PRODUCTION", + org_id: "org_1", + project_id: "proj_1", + }, + "any-secret", + EXP + ); +} + +describe("unwrapWorkerId", () => { + it("unwraps a real minted token to its deployment friendlyId", async () => { + // Decode is signature-independent (display-only), so the secret is irrelevant here. + expect(unwrapWorkerId(await mint("deployment_abc123"))).toBe("deployment_abc123"); + }); + + it("passes a legacy bare friendlyId through unchanged", () => { + expect(unwrapWorkerId("deployment_legacy")).toBe("deployment_legacy"); + }); + + it("passes undefined and garbage through unchanged (fail-safe)", () => { + expect(unwrapWorkerId(undefined)).toBeUndefined(); + expect(unwrapWorkerId("")).toBe(""); + expect(unwrapWorkerId("a.b.c")).toBe("a.b.c"); + expect(unwrapWorkerId("not-a-token")).toBe("not-a-token"); + }); + + it("is stable across repeated calls (memoized)", async () => { + const token = await mint("deployment_memo"); + expect(unwrapWorkerId(token)).toBe("deployment_memo"); + expect(unwrapWorkerId(token)).toBe("deployment_memo"); + }); +}); + +describe("unwrapWorkerIdInMetadata", () => { + it("unwraps a token worker.id in the metadata bag, leaving other keys untouched", async () => { + const metadata = { + [WORKER_ID_KEY]: await mint("deployment_span"), + "$metadata.custom": "keep-me", + "$metadata.worker.version": "20260709.1", + }; + + const result = unwrapWorkerIdInMetadata(metadata); + + expect(result?.[WORKER_ID_KEY]).toBe("deployment_span"); + expect(result?.["$metadata.custom"]).toBe("keep-me"); + expect(result?.["$metadata.worker.version"]).toBe("20260709.1"); + }); + + it("leaves a legacy bare friendlyId worker.id unchanged", () => { + const metadata = { [WORKER_ID_KEY]: "deployment_legacy" }; + expect(unwrapWorkerIdInMetadata(metadata)?.[WORKER_ID_KEY]).toBe("deployment_legacy"); + }); + + it("handles absent worker.id and undefined metadata", () => { + expect(unwrapWorkerIdInMetadata(undefined)).toBeUndefined(); + expect(unwrapWorkerIdInMetadata({ "$metadata.other": "x" })?.["$metadata.other"]).toBe("x"); + }); +}); diff --git a/apps/webapp/test/workloadTokenAuthorization.test.ts b/apps/webapp/test/workloadTokenAuthorization.test.ts new file mode 100644 index 000000000..205db2632 --- /dev/null +++ b/apps/webapp/test/workloadTokenAuthorization.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { evaluateCreatedAtGate } from "~/v3/services/worker/workloadTokenAuthorization.server"; + +const cutoff = new Date("2026-07-09T00:00:00.000Z"); +const before = new Date("2026-07-01T00:00:00.000Z"); +const after = new Date("2026-07-10T00:00:00.000Z"); + +describe("evaluateCreatedAtGate", () => { + it("grandfathers a run created before the cutoff", () => { + const result = evaluateCreatedAtGate({ runCreatedAt: before, cutoff }); + expect(result.outcome).toBe("grandfathered"); + expect(result.allow).toBe(true); + }); + + it("suppresses a run created after the cutoff", () => { + const result = evaluateCreatedAtGate({ runCreatedAt: after, cutoff }); + expect(result.outcome).toBe("suppressed"); + expect(result.allow).toBe(false); + }); + + it("treats a run created exactly at the cutoff as grandfathered (not after)", () => { + const result = evaluateCreatedAtGate({ runCreatedAt: cutoff, cutoff }); + expect(result.outcome).toBe("grandfathered"); + expect(result.allow).toBe(true); + }); +}); diff --git a/apps/webapp/test/workloadTokenGate.integration.test.ts b/apps/webapp/test/workloadTokenGate.integration.test.ts new file mode 100644 index 000000000..ea6458b57 --- /dev/null +++ b/apps/webapp/test/workloadTokenGate.integration.test.ts @@ -0,0 +1,88 @@ +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; + +// The env-scoped snapshot read the platform now performs for worker actions. Mirrors the where clause +// in getLatestExecutionSnapshot so this exercises the real select against the schema. +async function readLatestSnapshot(prisma: PrismaClient, runId: string, environmentId?: string) { + return prisma.taskRunExecutionSnapshot.findFirst({ + where: { runId, isValid: true, ...(environmentId ? { environmentId } : {}) }, + orderBy: { createdAt: "desc" }, + }); +} + +async function seed(prisma: PrismaClient) { + const org = await prisma.organization.create({ + data: { title: "Org", slug: `org-${Date.now()}` }, + }); + const project = await prisma.project.create({ + data: { + name: "Project", + slug: `proj-${Date.now()}`, + externalRef: `proj_${Date.now()}`, + organizationId: org.id, + }, + }); + const env = await prisma.runtimeEnvironment.create({ + data: { + type: "PRODUCTION", + slug: "prod", + projectId: project.id, + organizationId: org.id, + apiKey: "api_key", + pkApiKey: "pk_api_key", + shortcode: "short", + }, + }); + + const run = await prisma.taskRun.create({ + data: { + friendlyId: `run_${Date.now()}`, + taskIdentifier: "test-task", + payload: "{}", + payloadType: "application/json", + traceId: "trace_1", + spanId: "span_1", + queue: "task/test-task", + runtimeEnvironmentId: env.id, + projectId: project.id, + organizationId: org.id, + }, + }); + + await prisma.taskRunExecutionSnapshot.create({ + data: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "seed", + runId: run.id, + runStatus: "PENDING", + environmentId: env.id, + environmentType: "PRODUCTION", + projectId: project.id, + organizationId: org.id, + }, + }); + + return { env, run }; +} + +describe("env-scoped snapshot read against a real DB row", () => { + postgresTest( + "returns the snapshot for the matching env and nothing for another", + async ({ prisma }) => { + const { env, run } = await seed(prisma as PrismaClient); + + // No env scoping -> found (internal callers) + expect(await readLatestSnapshot(prisma as PrismaClient, run.id)).not.toBeNull(); + + // Matching env -> found + expect(await readLatestSnapshot(prisma as PrismaClient, run.id, env.id)).not.toBeNull(); + + // Different env -> not found (rejected as a tenant boundary) + expect( + await readLatestSnapshot(prisma as PrismaClient, run.id, "clenvdoesnotexist000000000") + ).toBeNull(); + } + ); +}); diff --git a/apps/webapp/test/writableEnvironments.test.ts b/apps/webapp/test/writableEnvironments.test.ts new file mode 100644 index 000000000..ed26fcde9 --- /dev/null +++ b/apps/webapp/test/writableEnvironments.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { + findUnauthorizedEnvironmentId, + type WriteCheckEnvironment, +} from "../app/v3/writableEnvironments.js"; + +const prod: WriteCheckEnvironment = { id: "env_prod", type: "PRODUCTION", orgMember: null }; +const staging: WriteCheckEnvironment = { id: "env_staging", type: "STAGING", orgMember: null }; +const myDev: WriteCheckEnvironment = { + id: "env_dev_me", + type: "DEVELOPMENT", + orgMember: { userId: "user_me" }, +}; +const otherDev: WriteCheckEnvironment = { + id: "env_dev_other", + type: "DEVELOPMENT", + orgMember: { userId: "user_other" }, +}; +const projectEnvs = [prod, staging, myDev, otherDev]; + +// Shared env types are writable by any project member; a DEVELOPMENT env only by +// its owner; an id not in the project is never writable. +describe("findUnauthorizedEnvironmentId", () => { + it("allows shared env types for any member", () => { + expect( + findUnauthorizedEnvironmentId(projectEnvs, ["env_prod", "env_staging"], "user_me") + ).toBeNull(); + }); + + it("allows a caller's own DEV env", () => { + expect(findUnauthorizedEnvironmentId(projectEnvs, ["env_dev_me"], "user_me")).toBeNull(); + }); + + it("rejects another user's DEV env (the cross-user injection vector)", () => { + expect(findUnauthorizedEnvironmentId(projectEnvs, ["env_dev_other"], "user_me")).toBe( + "env_dev_other" + ); + }); + + it("rejects a foreign DEV env even when mixed with allowed ones", () => { + expect( + findUnauthorizedEnvironmentId( + projectEnvs, + ["env_prod", "env_dev_me", "env_dev_other"], + "user_me" + ) + ).toBe("env_dev_other"); + }); + + it("rejects an id that isn't one of the project's environments", () => { + expect(findUnauthorizedEnvironmentId(projectEnvs, ["env_not_in_project"], "user_me")).toBe( + "env_not_in_project" + ); + }); +}); diff --git a/docs/self-hosting/docker.mdx b/docs/self-hosting/docker.mdx index ead4281a6..6c8de292f 100644 --- a/docs/self-hosting/docker.mdx +++ b/docs/self-hosting/docker.mdx @@ -75,12 +75,23 @@ git clone --depth=1 https://github.com/triggerdotdev/trigger.dev cd trigger.dev/hosting/docker ``` -2. Create a `.env` file +2. Create a `.env` file and generate secrets ```bash cp .env.example .env + +# Fills the required secrets in .env with strong, unique values. +# Safe to re-run - it never overwrites a secret you've already set. +./generate-secrets.sh ``` + + The stack ships no working default credentials. `generate-secrets.sh` fills the + application secrets and the bundled datastore passwords with strong, unique values. + Keep them safe - rotating the encryption key or session secret later will invalidate + existing sessions and encrypted data. + + 3. Start the webapp ```bash @@ -130,6 +141,13 @@ docker compose up -d 4. Configure the supervisor using the [environment variables](/self-hosting/env/supervisor) in your `.env` file, including the [worker token](#worker-token). + + For a split webapp/worker setup, set `MANAGED_WORKER_SECRET` on the worker to + the **same** value as the webapp's `MANAGED_WORKER_SECRET`. Don't run + `generate-secrets.sh` on the worker host - it would create a mismatched value + and the worker would fail to authenticate. + + 5. Apply the changes: ```bash diff --git a/docs/self-hosting/env/webapp.mdx b/docs/self-hosting/env/webapp.mdx index 1f629718b..09416f0f3 100644 --- a/docs/self-hosting/env/webapp.mdx +++ b/docs/self-hosting/env/webapp.mdx @@ -11,7 +11,8 @@ mode: "wide" | `SESSION_SECRET` | Yes | — | Session encryption secret. Run: `openssl rand -hex 16` | | `MAGIC_LINK_SECRET` | Yes | — | Magic link encryption secret. Run: `openssl rand -hex 16` | | `ENCRYPTION_KEY` | Yes | — | Secret store encryption key. Run: `openssl rand -hex 16` | -| `MANAGED_WORKER_SECRET` | No | managed-secret | Managed worker secret. Should be changed and match supervisor. | +| `MANAGED_WORKER_SECRET` | Yes | — | Managed worker secret. Must be set and match supervisor. Run: `openssl rand -hex 32` | +| `ALLOW_INSECURE_DEFAULT_SECRETS` | No | false | Boot even if a secret is still a known-insecure published default. Temporary escape hatch for values you can't safely rotate yet (see [Secret generation and rotation](/self-hosting/kubernetes#secret-generation-and-rotation)). | | **Domains & ports** | | | | | `REMIX_APP_PORT` | No | 3030 | Remix app port. | | `APP_ORIGIN` | Yes | http://localhost:3030 | App origin URL. | diff --git a/docs/self-hosting/kubernetes.mdx b/docs/self-hosting/kubernetes.mdx index 5e24b675f..33d06f308 100644 --- a/docs/self-hosting/kubernetes.mdx +++ b/docs/self-hosting/kubernetes.mdx @@ -121,8 +121,10 @@ The default values are insecure and are only suitable for testing. You will need Create a `values-custom.yaml` file to override the defaults. For example: ```yaml -# Generate new secrets with `openssl rand -hex 16` -# WARNING: You should probably use an existingSecret instead +# Leave these unset to have the chart auto-generate strong values on first +# install (retained across upgrades). Set them explicitly only if you need to +# control the value - e.g. sharing MANAGED_WORKER_SECRET with an external +# supervisor - or use an existingSecret. secrets: enabled: true sessionSecret: "your-32-char-hex-secret-1" @@ -133,6 +135,8 @@ secrets: # - SESSION_SECRET # - MAGIC_LINK_SECRET # - ENCRYPTION_KEY +# - PROVIDER_SECRET +# - COORDINATOR_SECRET # - MANAGED_WORKER_SECRET # - OBJECT_STORE_ACCESS_KEY_ID # - OBJECT_STORE_SECRET_ACCESS_KEY @@ -176,6 +180,24 @@ helm upgrade -n trigger --install trigger \ -f values-custom.yaml ``` +### Secret generation and rotation + +Application, control-plane, and bundled-datastore secrets left unset are generated on +first install and **retained across `helm upgrade`** - they are never rotated +automatically, so sessions, encrypted data, and datastore volumes survive upgrades. + + + GitOps tools that render with `helm template` (e.g. Argo CD) cannot read the existing + secret, so they regenerate these values on every sync - which rotates them. If you + deploy via GitOps, always supply your own `secrets.existingSecret` (and datastore + credentials) so nothing is generated in-cluster. + + +There is no clean migration for a compromised `ENCRYPTION_KEY`: changing it makes +existing encrypted data unreadable. If a deployment is still running a previously +published default and cannot rotate yet, set `ALLOW_INSECURE_DEFAULT_SECRETS=true` on +the webapp to keep booting while you plan a migration. + ### Extra env You can set extra environment variables on all services. For example: diff --git a/hosting/docker/.env.example b/hosting/docker/.env.example index 383b6f79d..4c7cf11bc 100644 --- a/hosting/docker/.env.example +++ b/hosting/docker/.env.example @@ -3,13 +3,16 @@ # - You should change them to suit your needs, especially the secrets # - See the docs for more information: https://trigger.dev/docs/self-hosting/overview -# Secrets -# - Do NOT use these defaults in production -# - Generate your own by running `openssl rand -hex 16` for each secret -SESSION_SECRET=2818143646516f6fffd707b36f334bbb -MAGIC_LINK_SECRET=44da78b7bbb0dfe709cf38931d25dcdd -ENCRYPTION_KEY=f686147ab967943ebbe9ed3b496e465a -MANAGED_WORKER_SECRET=447c29678f9eaf289e9c4b70d3dd8a7f +# Secrets — REQUIRED, no defaults. The stack will not boot until each is set to a unique value. +# Generate each with: openssl rand -hex 16 +SESSION_SECRET= +MAGIC_LINK_SECRET= +ENCRYPTION_KEY= +# These authenticate the internal control-plane connections. Generate each with: openssl rand -hex 16 +# COORDINATOR_SECRET must match the coordinator's PLATFORM_SECRET; MANAGED_WORKER_SECRET the supervisor's. +PROVIDER_SECRET= +COORDINATOR_SECRET= +MANAGED_WORKER_SECRET= # Worker token # - This is the token for the worker to connect to the webapp @@ -24,13 +27,14 @@ MANAGED_WORKER_SECRET=447c29678f9eaf289e9c4b70d3dd8a7f # OTEL_EXPORTER_OTLP_ENDPOINT=https://trigger.example.com/otel # Postgres -# - Do NOT use these defaults in production -# - Especially if you decide to expose the database to the internet +# - Password is REQUIRED, no default. Run ./generate-secrets.sh to fill it (or openssl rand -hex 16). +# - DATABASE_URL / DIRECT_URL are derived from POSTGRES_PASSWORD automatically - only set them +# below to point at an external Postgres (POSTGRES_PASSWORD is then unused). # POSTGRES_USER=postgres -POSTGRES_PASSWORD=unsafe-postgres-pw +POSTGRES_PASSWORD= # POSTGRES_DB=postgres -DATABASE_URL=postgresql://postgres:unsafe-postgres-pw@postgres:5432/main?schema=public&sslmode=disable -DIRECT_URL=postgresql://postgres:unsafe-postgres-pw@postgres:5432/main?schema=public&sslmode=disable +# DATABASE_URL=postgresql://user:password@host:5432/main?schema=public&sslmode=disable +# DIRECT_URL=postgresql://user:password@host:5432/main?schema=public&sslmode=disable # Trigger image tag # - This is the version of the webapp and worker images to use, they should be locked to a specific version in production @@ -59,19 +63,21 @@ DEV_OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:8030/otel # NODE_MAX_OLD_SPACE_SIZE=8192 # ClickHouse -# - Do NOT use these defaults in production +# - Password is REQUIRED, no default. Run ./generate-secrets.sh to fill it. +# - CLICKHOUSE_URL / RUN_REPLICATION_CLICKHOUSE_URL are derived from CLICKHOUSE_PASSWORD +# automatically - only set them below to point at an external ClickHouse. CLICKHOUSE_USER=default -CLICKHOUSE_PASSWORD=password -CLICKHOUSE_URL=http://default:password@clickhouse:8123?secure=false -RUN_REPLICATION_CLICKHOUSE_URL=http://default:password@clickhouse:8123 +CLICKHOUSE_PASSWORD= +# CLICKHOUSE_URL=http://user:password@host:8123?secure=false +# RUN_REPLICATION_CLICKHOUSE_URL=http://user:password@host:8123 # Docker Registry -# - When testing locally, the default values should be fine -# - When deploying to production, you will have to change these, especially the password and URL +# - Password is REQUIRED, no default. Run ./generate-secrets.sh to fill it - it also writes +# the matching registry/auth.htpasswd (bcrypt) the bundled registry authenticates against. # - See the docs for more information: https://trigger.dev/docs/self-hosting/docker#registry-setup DOCKER_REGISTRY_URL=localhost:5000 DOCKER_REGISTRY_USERNAME=registry-user -DOCKER_REGISTRY_PASSWORD=very-secure-indeed +DOCKER_REGISTRY_PASSWORD= # When using an external registry you will have to change this # On Docker Hub it should generally be the same as your username DOCKER_REGISTRY_NAMESPACE=trigger @@ -80,8 +86,10 @@ DOCKER_REGISTRY_NAMESPACE=trigger # - You need to log into the Minio dashboard and create a bucket called "packets" # - See the docs for more information: https://trigger.dev/docs/self-hosting/docker#object-storage # Default provider (backward compatible - no protocol prefix) +# - Secret access key is REQUIRED, no default. Run ./generate-secrets.sh to fill it. +# - For the bundled MinIO, these ARE its root credentials (MINIO_ROOT_USER/PASSWORD derive from them). OBJECT_STORE_ACCESS_KEY_ID=admin -OBJECT_STORE_SECRET_ACCESS_KEY=very-safe-password +OBJECT_STORE_SECRET_ACCESS_KEY= # You will have to uncomment and configure this for production # OBJECT_STORE_BASE_URL=http://localhost:9000 # OBJECT_STORE_REGION=auto @@ -100,11 +108,11 @@ OBJECT_STORE_SECRET_ACCESS_KEY=very-safe-password # OBJECT_STORE_R2_SECRET_ACCESS_KEY= # OBJECT_STORE_R2_REGION=auto # OBJECT_STORE_R2_SERVICE=s3 -# Credentials to access the Minio dashboard at http://localhost:9001 -# - You should change these credentials and not use them for the `OBJECT_STORE_` env vars above -# - Instead, setup a non-root user with access the "packets" bucket +# Minio dashboard at http://localhost:9001 +# - The bundled Minio's root credentials default to OBJECT_STORE_ACCESS_KEY_ID / OBJECT_STORE_SECRET_ACCESS_KEY. +# - For production, set a separate root user here and create a non-root user scoped to the "packets" bucket for OBJECT_STORE_*. # MINIO_ROOT_USER=admin -# MINIO_ROOT_PASSWORD=very-safe-password +# MINIO_ROOT_PASSWORD= # Realtime streams # - Realtime streams power AI-agent token streaming and run streams diff --git a/hosting/docker/generate-secrets.sh b/hosting/docker/generate-secrets.sh new file mode 100755 index 000000000..679dc129c --- /dev/null +++ b/hosting/docker/generate-secrets.sh @@ -0,0 +1,110 @@ +#!/bin/sh +# Generate strong, unique secrets and datastore passwords into the self-hosting .env. +# +# Safe to re-run: only fills values that are missing or empty, and never overwrites +# one that is already set. Rotating a live secret would orphan encrypted data, log +# everyone out, or break an already-initialised datastore volume - so rotation is +# opt-in only, via --force (which WILL break existing data/sessions). +set -eu + +FORCE=0 +for arg in "$@"; do + case "$arg" in + -f | --force) FORCE=1 ;; + -h | --help) + echo "Usage: $0 [--force] [env-file]" + echo " --force Regenerate every secret, overwriting existing values." + echo " WARNING: rotates live secrets - breaks encrypted data, sessions," + echo " and already-initialised datastore volumes." + exit 0 + ;; + *) env_file="$arg" ;; + esac +done + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +env_file="${env_file:-$script_dir/.env}" +env_example="$script_dir/.env.example" +htpasswd_file="$script_dir/registry/auth.htpasswd" + +# openssl rand -hex 16 -> 32 hex chars: URL-safe (no @ : / etc. to break connection +# strings) and satisfies the webapp's exact-32-byte ENCRYPTION_KEY check. +gen() { openssl rand -hex 16; } + +if [ ! -f "$env_file" ]; then + cp "$env_example" "$env_file" + echo "Created $(basename "$env_file") from $(basename "$env_example")" +fi + +sed_inplace() { + if [ "$(uname)" = "Darwin" ]; then + sed -i '' "$@" + else + sed -i "$@" + fi +} + +# Value of KEY= in the env file, trailing inline comment/whitespace stripped, so +# "KEY=" and "KEY= # todo" both read as unset. +current_value() { + grep -E "^$1=" "$env_file" | head -n1 | cut -d= -f2- \ + | sed -e 's/[[:space:]]*#.*$//' -e 's/[[:space:]]*$//' +} + +set_var() { + key="$1" + value="$2" + if grep -qE "^$key=" "$env_file"; then + sed_inplace -e "s|^$key=.*|$key=$value|" "$env_file" + else + printf '%s=%s\n' "$key" "$value" >>"$env_file" + fi +} + +# Fill KEY with a fresh secret unless it already has a value (respecting --force). +# Returns 0 if it wrote, 1 if it skipped. +fill() { + key="$1" + if [ "$FORCE" -eq 0 ] && [ -n "$(current_value "$key")" ]; then + return 1 + fi + set_var "$key" "$(gen)" + echo "Generated $key" + return 0 +} + +generated=0 + +# App + control-plane secrets, and the bundled-datastore passwords. All plain +# values consumed directly (or, for datastores, woven into the connection URLs by +# docker-compose interpolation - see .env.example). +for key in \ + SESSION_SECRET MAGIC_LINK_SECRET ENCRYPTION_KEY \ + PROVIDER_SECRET COORDINATOR_SECRET MANAGED_WORKER_SECRET \ + POSTGRES_PASSWORD CLICKHOUSE_PASSWORD OBJECT_STORE_SECRET_ACCESS_KEY; do + if fill "$key"; then generated=$((generated + 1)); fi +done + +# Registry is special: the bundled registry authenticates against a bcrypt htpasswd +# file, so when we set its password we must regenerate that file to match. bcrypt is +# the only hash the registry accepts, and openssl can't produce it - use httpd's +# htpasswd via docker (already required for this stack). +if [ "$FORCE" -eq 1 ] || [ -z "$(current_value DOCKER_REGISTRY_PASSWORD)" ]; then + registry_user=$(current_value DOCKER_REGISTRY_USERNAME) + registry_user=${registry_user:-registry-user} + registry_pass=$(gen) + if ! command -v docker >/dev/null 2>&1; then + echo "ERROR: docker is required to hash the registry password (bcrypt). Install docker and re-run." >&2 + exit 1 + fi + docker run --rm httpd:2 htpasswd -Bbn "$registry_user" "$registry_pass" >"$htpasswd_file" + set_var DOCKER_REGISTRY_PASSWORD "$registry_pass" + echo "Generated DOCKER_REGISTRY_PASSWORD (and wrote $(basename "$htpasswd_file"))" + generated=$((generated + 1)) +fi + +if [ "$generated" -eq 0 ]; then + echo "All secrets already set in $(basename "$env_file"); nothing to do. Use --force to rotate." +else + echo "Wrote $generated secret(s) to $(basename "$env_file")." +fi diff --git a/hosting/docker/registry/auth.htpasswd b/hosting/docker/registry/auth.htpasswd index 3f659261e..e69de29bb 100644 --- a/hosting/docker/registry/auth.htpasswd +++ b/hosting/docker/registry/auth.htpasswd @@ -1 +0,0 @@ -registry-user:$2y$05$6ingYqw0.3j13dxHY4w3neMSvKhF3pvRmc0AFifScWsVA9JpuLwNK diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index 611ce4939..8d133cbee 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -54,11 +54,13 @@ services: REALTIME_STREAMS_S2_ENDPOINT: ${REALTIME_STREAMS_S2_ENDPOINT:-http://s2/v1} REALTIME_STREAMS_S2_SKIP_ACCESS_TOKENS: ${REALTIME_STREAMS_S2_SKIP_ACCESS_TOKENS:-true} REALTIME_STREAMS_S2_ACCESS_TOKEN: ${REALTIME_STREAMS_S2_ACCESS_TOKEN:-} - DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable} - DIRECT_URL: ${DIRECT_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable} + DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:${POSTGRES_PASSWORD}@postgres:5432/main?schema=public&sslmode=disable} + DIRECT_URL: ${DIRECT_URL:-postgresql://postgres:${POSTGRES_PASSWORD}@postgres:5432/main?schema=public&sslmode=disable} SESSION_SECRET: ${SESSION_SECRET} MAGIC_LINK_SECRET: ${MAGIC_LINK_SECRET} ENCRYPTION_KEY: ${ENCRYPTION_KEY} + PROVIDER_SECRET: ${PROVIDER_SECRET} + COORDINATOR_SECRET: ${COORDINATOR_SECRET} MANAGED_WORKER_SECRET: ${MANAGED_WORKER_SECRET} REDIS_HOST: redis REDIS_PORT: 6379 @@ -78,11 +80,11 @@ services: TRIGGER_BOOTSTRAP_WORKER_GROUP_NAME: bootstrap TRIGGER_BOOTSTRAP_WORKER_TOKEN_PATH: /home/node/shared/worker_token # ClickHouse configuration - CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://default:password@clickhouse:8123?secure=false} + CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://default:${CLICKHOUSE_PASSWORD}@clickhouse:8123?secure=false} CLICKHOUSE_LOG_LEVEL: ${CLICKHOUSE_LOG_LEVEL:-info} # Run replication RUN_REPLICATION_ENABLED: ${RUN_REPLICATION_ENABLED:-1} - RUN_REPLICATION_CLICKHOUSE_URL: ${RUN_REPLICATION_CLICKHOUSE_URL:-http://default:password@clickhouse:8123} + RUN_REPLICATION_CLICKHOUSE_URL: ${RUN_REPLICATION_CLICKHOUSE_URL:-http://default:${CLICKHOUSE_PASSWORD}@clickhouse:8123} RUN_REPLICATION_LOG_LEVEL: ${RUN_REPLICATION_LOG_LEVEL:-info} # Limits # TASK_PAYLOAD_OFFLOAD_THRESHOLD: 524288 # 512KB @@ -109,7 +111,7 @@ services: - wal_level=logical environment: POSTGRES_USER: ${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env - run ./generate-secrets.sh} POSTGRES_DB: ${POSTGRES_DB:-postgres} healthcheck: test: ["CMD", "pg_isready", "-U", "postgres"] @@ -144,7 +146,7 @@ services: networks: - webapp environment: - DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable} + DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:${POSTGRES_PASSWORD}@postgres:5432/main?schema=public&sslmode=disable} ELECTRIC_INSECURE: true ELECTRIC_USAGE_REPORTING: false healthcheck: @@ -163,7 +165,7 @@ services: - ${CLICKHOUSE_PUBLISH_IP:-127.0.0.1}:9090:9000 environment: CLICKHOUSE_ADMIN_USER: ${CLICKHOUSE_USER:-default} - CLICKHOUSE_ADMIN_PASSWORD: ${CLICKHOUSE_PASSWORD:-password} + CLICKHOUSE_ADMIN_PASSWORD: ${CLICKHOUSE_PASSWORD:?Set CLICKHOUSE_PASSWORD in .env - run ./generate-secrets.sh} volumes: - clickhouse:/bitnami/clickhouse - ../clickhouse/override.xml:/bitnami/clickhouse/etc/config.d/override.xml:ro @@ -181,7 +183,7 @@ services: "--user", "${CLICKHOUSE_USER:-default}", "--password", - "${CLICKHOUSE_PASSWORD:-password}", + "${CLICKHOUSE_PASSWORD}", "--query", "SELECT 1", ] @@ -224,8 +226,8 @@ services: volumes: - minio:/bitnami/minio/data environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER:-admin} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-very-safe-password} + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-${OBJECT_STORE_ACCESS_KEY_ID:-admin}} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-${OBJECT_STORE_SECRET_ACCESS_KEY:?Set OBJECT_STORE_SECRET_ACCESS_KEY in .env - run ./generate-secrets.sh}} MINIO_DEFAULT_BUCKETS: packets MINIO_BROWSER: "on" healthcheck: diff --git a/hosting/k8s/helm/README.md b/hosting/k8s/helm/README.md index 33e64a93a..79c491382 100644 --- a/hosting/k8s/helm/README.md +++ b/hosting/k8s/helm/README.md @@ -20,7 +20,10 @@ helm template trigger . --dependency-update ### Installation ```bash -# Deploy with default values (testing/development only) +# A bare install works: application, control-plane, and bundled-datastore +# secrets are auto-generated on first install (and retained across upgrades) +# when you don't set them. Provide a values file to pin any of them or to +# point at external datastores. helm install trigger . # Deploy to specific namespace @@ -59,24 +62,28 @@ npx trigger.dev@latest deploy --push ### Secrets Configuration -**IMPORTANT**: The default secrets are for **TESTING ONLY** and must be changed for production. +**IMPORTANT**: The chart ships no working secret defaults. Application, control-plane, and bundled-datastore secrets (postgres/clickhouse/minio/registry) are **auto-generated per install** (and retained across upgrades) when left unset. You can still set them explicitly - e.g. to share `managedWorkerSecret` with an external supervisor, or to manage everything via `secrets.existingSecret`. Explicit values always win over generation. -#### Required Secrets +#### Auto-generated secrets -All secrets must be exactly **32 hexadecimal characters** (16 bytes): +Left unset, these are generated on first install and preserved on `helm upgrade` (rotating them would invalidate sessions and orphan encrypted data, so they are never regenerated once set): - `sessionSecret` - User authentication sessions -- `magicLinkSecret` - Passwordless login tokens +- `magicLinkSecret` - Passwordless login tokens - `encryptionKey` - Sensitive data encryption -- `managedWorkerSecret` - Worker authentication +- `managedWorkerSecret` - Worker authentication (set explicitly if an external supervisor must share it) +- `providerSecret` - Provider control-plane socket authentication +- `coordinatorSecret` - Coordinator control-plane socket authentication -#### Generate Production Secrets +#### Generating secrets yourself + +If you prefer to set them explicitly (e.g. to reuse across clusters): ```bash -for i in {1..4}; do openssl rand -hex 16; done +for i in {1..6}; do openssl rand -hex 16; done ``` -#### Configure Production Secrets +#### Configure secrets explicitly ```yaml # values-production.yaml @@ -85,6 +92,8 @@ secrets: magicLinkSecret: "your-generated-secret-2" encryptionKey: "your-generated-secret-3" managedWorkerSecret: "your-generated-secret-4" + providerSecret: "your-generated-secret-5" + coordinatorSecret: "your-generated-secret-6" objectStore: accessKeyId: "your-s3-access-key" secretAccessKey: "your-s3-secret-key" diff --git a/hosting/k8s/helm/ci/lint-values.yaml b/hosting/k8s/helm/ci/lint-values.yaml new file mode 100644 index 000000000..967cc6c50 --- /dev/null +++ b/hosting/k8s/helm/ci/lint-values.yaml @@ -0,0 +1,26 @@ +# Throwaway credentials used ONLY to render the chart during CI (helm lint / helm template). +# The shipped values.yaml no longer contains working default credentials for the bundled +# datastores (see SEC-70): an unconfigured install now fails closed. CI must therefore +# supply dummy secrets here so template rendering can be validated. These values are never +# deployed and must never be used in a real environment. +secrets: + sessionSecret: "ci-render-only-session" + magicLinkSecret: "ci-render-only-magic-link" + encryptionKey: "ci-render-only-encryption-key" + providerSecret: "ci-render-only-provider" + coordinatorSecret: "ci-render-only-coordinator" + managedWorkerSecret: "ci-render-only-managed-worker" +postgres: + auth: + postgresPassword: "ci-render-only-postgres" + password: "ci-render-only-postgres" +clickhouse: + auth: + password: "ci-render-only-clickhouse" +s3: + auth: + rootPassword: "ci-render-only-minio" +registry: + deploy: true + auth: + password: "ci-render-only-registry" diff --git a/hosting/k8s/helm/templates/NOTES.txt b/hosting/k8s/helm/templates/NOTES.txt index e70d18dd7..4571eb9f4 100644 --- a/hosting/k8s/helm/templates/NOTES.txt +++ b/hosting/k8s/helm/templates/NOTES.txt @@ -2,21 +2,11 @@ Thank you for installing {{ .Chart.Name }}. Your release is named {{ .Release.Name }}. -🔐 SECURITY WARNING: -{{- if or (eq .Values.secrets.sessionSecret "2818143646516f6fffd707b36f334bbb") (eq .Values.secrets.magicLinkSecret "44da78b7bbb0dfe709cf38931d25dcdd") (eq .Values.secrets.encryptionKey "f686147ab967943ebbe9ed3b496e465a") (eq .Values.secrets.managedWorkerSecret "447c29678f9eaf289e9c4b70d3dd8a7f") }} - You are using DEFAULT SECRETS which are NOT SECURE for production! - - For production deployments, generate new secrets: - 1. Run: openssl rand -hex 16 (repeat for each secret) - 2. Override in your values.yaml: - secrets: - sessionSecret: "your-new-32-char-hex-secret" - magicLinkSecret: "your-new-32-char-hex-secret" - encryptionKey: "your-new-32-char-hex-secret" - managedWorkerSecret: "your-new-32-char-hex-secret" -{{- else }} - Custom secrets detected - good for production deployment! -{{- end }} +🔐 SECURITY: + This chart ships no working secret defaults. Application and control-plane + secrets are auto-generated per install (and retained across upgrades) unless + you set them explicitly or supply secrets.existingSecret. Bundled-datastore + passwords must still be provided. To get started: diff --git a/hosting/k8s/helm/templates/_helpers.tpl b/hosting/k8s/helm/templates/_helpers.tpl index 2ce273c01..0ecc17b98 100644 --- a/hosting/k8s/helm/templates/_helpers.tpl +++ b/hosting/k8s/helm/templates/_helpers.tpl @@ -141,7 +141,7 @@ PostgreSQL connection string (fallback when not using secrets) {{- if .Values.postgres.external.databaseUrl -}} {{ .Values.postgres.external.databaseUrl }} {{- else if .Values.postgres.deploy -}} -postgresql://{{ .Values.postgres.auth.username }}:{{ .Values.postgres.auth.password }}@{{ include "trigger-v4.postgres.hostname" . }}:5432/{{ .Values.postgres.auth.database }}?schema={{ .Values.postgres.connection.schema | default "public" }}&sslmode={{ .Values.postgres.connection.sslMode | default "prefer" }} +postgresql://{{ .Values.postgres.auth.username }}:$(POSTGRES_PASSWORD)@{{ include "trigger-v4.postgres.hostname" . }}:5432/{{ .Values.postgres.auth.database }}?schema={{ .Values.postgres.connection.schema | default "public" }}&sslmode={{ .Values.postgres.connection.sslMode | default "prefer" }} {{- end -}} {{- end }} @@ -439,7 +439,7 @@ hex-encoded password or percent-encode before storing in the Secret. {{- if .Values.clickhouse.deploy -}} {{- $protocol := ternary "https" "http" .Values.clickhouse.secure -}} {{- $secure := ternary "true" "false" .Values.clickhouse.secure -}} -{{ $protocol }}://{{ .Values.clickhouse.auth.username }}:{{ .Values.clickhouse.auth.password }}@{{ include "trigger-v4.clickhouse.hostname" . }}:8123?secure={{ $secure }} +{{ $protocol }}://{{ .Values.clickhouse.auth.username }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:8123?secure={{ $secure }} {{- else if .Values.clickhouse.external.host -}} {{- $protocol := ternary "https" "http" .Values.clickhouse.external.secure -}} {{- $secure := ternary "true" "false" .Values.clickhouse.external.secure -}} @@ -460,7 +460,7 @@ applies to the replication URL. {{- define "trigger-v4.clickhouse.replication.url" -}} {{- if .Values.clickhouse.deploy -}} {{- $protocol := ternary "https" "http" .Values.clickhouse.secure -}} -{{ $protocol }}://{{ .Values.clickhouse.auth.username }}:{{ .Values.clickhouse.auth.password }}@{{ include "trigger-v4.clickhouse.hostname" . }}:8123 +{{ $protocol }}://{{ .Values.clickhouse.auth.username }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:8123 {{- else if .Values.clickhouse.external.host -}} {{- $protocol := ternary "https" "http" .Values.clickhouse.external.secure -}} {{- if .Values.clickhouse.external.existingSecret -}} @@ -511,6 +511,35 @@ Get the secrets name - either existing secret or generated name {{- end -}} {{- end }} +{{/* +Fixed name of the chart-managed datastore-credentials Secret. Fixed (not release- +scoped) because bundled Bitnami subcharts read it via `*.auth.existingSecret`, which +is set in values.yaml and cannot be templated - so the name must be a literal that +both this chart and those values agree on. One release per namespace. +*/}} +{{- define "trigger-v4.datastore.secretName" -}} +trigger-datastore +{{- end }} + +{{/* +Resolve a chart-managed secret: an explicit value wins; otherwise reuse the +value already stored in the cluster Secret so `helm upgrade` never rotates it +(rotating ENCRYPTION_KEY/SESSION_SECRET would orphan encrypted data and +invalidate every session); otherwise generate a fresh 32-hex-char value. +Note: `lookup` returns nothing during `helm template`/`--dry-run`, so those +render a throwaway value each time - only real installs/upgrades retain. +Args: dict "existingData" "key" "value" +*/}} +{{- define "trigger-v4.resolveSecret" -}} +{{- if .value -}} +{{- .value -}} +{{- else if (index .existingData .key) -}} +{{- index .existingData .key | b64dec -}} +{{- else -}} +{{- sha256sum (randAlphaNum 32) | trunc 32 -}} +{{- end -}} +{{- end }} + {{/* Registry connection details */}} diff --git a/hosting/k8s/helm/templates/datastore-secret.yaml b/hosting/k8s/helm/templates/datastore-secret.yaml new file mode 100644 index 000000000..392a2e7af --- /dev/null +++ b/hosting/k8s/helm/templates/datastore-secret.yaml @@ -0,0 +1,39 @@ +{{/* +Chart-managed credentials for the bundled datastores (postgres/clickhouse/minio). +Each password is generated ONCE here (retained across upgrades via lookup) and +consumed in two places from this single source: the Bitnami subchart reads it via +its `auth.existingSecret`, and the webapp reads it via secretKeyRef + `$(VAR)` +runtime interpolation in the connection URL. Generating it once and reading it back +is what keeps the server credential and the app's URL identical. + +The registry password is NOT here - it is consumed at template render time +(htpasswd + dockerconfigjson) so it is generated and retained inside secrets.yaml. +*/}} +{{- if or .Values.postgres.deploy .Values.clickhouse.deploy .Values.s3.deploy }} +{{- $name := include "trigger-v4.datastore.secretName" . }} +{{- $existing := (lookup "v1" "Secret" .Release.Namespace $name) | default dict }} +{{- $existingData := (get $existing "data") | default dict }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }} + labels: + {{- include "trigger-v4.labels" . | nindent 4 }} + annotations: + # Never lose these - the password is the only copy the datastore volumes accept. + helm.sh/resource-policy: keep +type: Opaque +data: + {{- if .Values.postgres.deploy }} + {{- $pg := include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "postgres-password" "value" .Values.postgres.auth.password) }} + postgres-password: {{ $pg | b64enc | quote }} + password: {{ $pg | b64enc | quote }} + {{- end }} + {{- if .Values.clickhouse.deploy }} + clickhouse-admin-password: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "clickhouse-admin-password" "value" .Values.clickhouse.auth.password) | b64enc | quote }} + {{- end }} + {{- if .Values.s3.deploy }} + minio-root-user: {{ .Values.s3.auth.rootUser | default "admin" | b64enc | quote }} + minio-root-password: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "minio-root-password" "value" (.Values.s3.auth.rootPassword | default .Values.s3.auth.secretAccessKey)) | b64enc | quote }} + {{- end }} +{{- end }} diff --git a/hosting/k8s/helm/templates/electric.yaml b/hosting/k8s/helm/templates/electric.yaml index ac2d7582f..5bb1964a0 100644 --- a/hosting/k8s/helm/templates/electric.yaml +++ b/hosting/k8s/helm/templates/electric.yaml @@ -44,6 +44,13 @@ spec: name: {{ include "trigger-v4.postgres.external.secretName" . }} key: {{ include "trigger-v4.postgres.external.databaseUrlKey" . }} {{- else }} + {{- if .Values.postgres.deploy }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "trigger-v4.datastore.secretName" . }} + key: password + {{- end }} - name: DATABASE_URL value: {{ include "trigger-v4.postgres.connectionString" . | quote }} {{- end }} diff --git a/hosting/k8s/helm/templates/s2.yaml b/hosting/k8s/helm/templates/s2.yaml index 54e75b13b..eccfe4c14 100644 --- a/hosting/k8s/helm/templates/s2.yaml +++ b/hosting/k8s/helm/templates/s2.yaml @@ -56,7 +56,10 @@ spec: {{- end }} image: "{{ .Values.s2.image.registry }}/{{ .Values.s2.image.repository }}:{{ .Values.s2.image.tag }}{{ with .Values.s2.image.digest }}@{{ . }}{{ end }}" imagePullPolicy: {{ .Values.s2.image.pullPolicy }} - command: + # args, not command: the image ENTRYPOINT is ["./s2"] and these are its + # subcommand/flags. Using command would override the entrypoint and try to + # exec "lite" directly (StartError). + args: - "lite" - "--init-file" - "/config/s2-spec.json" diff --git a/hosting/k8s/helm/templates/secrets.yaml b/hosting/k8s/helm/templates/secrets.yaml index c122f89d0..3c19df0d3 100644 --- a/hosting/k8s/helm/templates/secrets.yaml +++ b/hosting/k8s/helm/templates/secrets.yaml @@ -1,4 +1,6 @@ {{- if and .Values.secrets.enabled (not .Values.secrets.existingSecret) }} +{{- $existing := (lookup "v1" "Secret" .Release.Namespace (printf "%s-secrets" (include "trigger-v4.fullname" .))) | default dict }} +{{- $existingData := (get $existing "data") | default dict }} apiVersion: v1 kind: Secret metadata: @@ -7,10 +9,12 @@ metadata: {{- include "trigger-v4.labels" . | nindent 4 }} type: Opaque data: - SESSION_SECRET: {{ .Values.secrets.sessionSecret | b64enc | quote }} - MAGIC_LINK_SECRET: {{ .Values.secrets.magicLinkSecret | b64enc | quote }} - ENCRYPTION_KEY: {{ .Values.secrets.encryptionKey | b64enc | quote }} - MANAGED_WORKER_SECRET: {{ .Values.secrets.managedWorkerSecret | b64enc | quote }} + SESSION_SECRET: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "SESSION_SECRET" "value" .Values.secrets.sessionSecret) | b64enc | quote }} + MAGIC_LINK_SECRET: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "MAGIC_LINK_SECRET" "value" .Values.secrets.magicLinkSecret) | b64enc | quote }} + ENCRYPTION_KEY: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "ENCRYPTION_KEY" "value" .Values.secrets.encryptionKey) | b64enc | quote }} + PROVIDER_SECRET: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "PROVIDER_SECRET" "value" .Values.secrets.providerSecret) | b64enc | quote }} + COORDINATOR_SECRET: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "COORDINATOR_SECRET" "value" .Values.secrets.coordinatorSecret) | b64enc | quote }} + MANAGED_WORKER_SECRET: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "MANAGED_WORKER_SECRET" "value" .Values.secrets.managedWorkerSecret) | b64enc | quote }} {{- if and .Values.s3.external.accessKeyId (not .Values.s3.external.existingSecret) }} s3-access-key-id: {{ .Values.s3.external.accessKeyId | b64enc | quote }} s3-secret-access-key: {{ .Values.s3.external.secretAccessKey | b64enc | quote }} @@ -36,15 +40,27 @@ data: {{- end }} --- {{- if and .Values.registry.deploy .Values.registry.auth.enabled }} +{{/* Generated here (not in datastore-secret) because it is consumed at render time by + htpasswd and the dockerconfigjson below. Retained via the plaintext `password` key + on this secret; htpasswd bcrypt is one-way so it can't be the retain source. */}} +{{- $regAuthName := printf "%s-registry-auth" (include "trigger-v4.fullname" .) }} +{{- $regExisting := (lookup "v1" "Secret" .Release.Namespace $regAuthName) | default dict }} +{{- $regExistingData := (get $regExisting "data") | default dict }} +{{- $regPass := include "trigger-v4.resolveSecret" (dict "existingData" $regExistingData "key" "password" "value" .Values.registry.auth.password) }} +{{- $regAuth := printf "%s:%s" .Values.registry.auth.username $regPass | b64enc }} +{{- $regConfig := dict "auths" (dict (include "trigger-v4.registry.host" .) (dict "username" .Values.registry.auth.username "password" $regPass "auth" $regAuth)) }} apiVersion: v1 kind: Secret metadata: - name: {{ include "trigger-v4.fullname" . }}-registry-auth + name: {{ $regAuthName }} labels: {{- include "trigger-v4.labels" . | nindent 4 }} + annotations: + helm.sh/resource-policy: keep type: Opaque data: - htpasswd: {{ htpasswd .Values.registry.auth.username .Values.registry.auth.password | trim | b64enc | quote }} + htpasswd: {{ htpasswd .Values.registry.auth.username $regPass | trim | b64enc | quote }} + password: {{ $regPass | b64enc | quote }} --- apiVersion: v1 kind: Secret @@ -54,7 +70,7 @@ metadata: {{- include "trigger-v4.labels" . | nindent 4 }} type: kubernetes.io/dockerconfigjson data: - .dockerconfigjson: {{ include "trigger-v4.imagePullSecret" . | b64enc }} + .dockerconfigjson: {{ $regConfig | toJson | b64enc }} {{- else if and (not .Values.registry.deploy) .Values.registry.external.auth.enabled }} apiVersion: v1 kind: Secret diff --git a/hosting/k8s/helm/templates/validate-external-config.yaml b/hosting/k8s/helm/templates/validate-external-config.yaml index 063aeecbc..939e699a4 100644 --- a/hosting/k8s/helm/templates/validate-external-config.yaml +++ b/hosting/k8s/helm/templates/validate-external-config.yaml @@ -7,6 +7,7 @@ This template will fail the Helm deployment if external config is missing for re {{- fail "PostgreSQL external configuration is required when postgres.deploy=false. Please provide either postgres.external.databaseUrl or postgres.external.existingSecret" }} {{- end }} {{- end }} +{{/* When postgres.deploy=true the password is auto-generated into the datastore Secret (templates/datastore-secret.yaml) and consumed by the subchart via auth.existingSecret; set postgres.auth.password to pin it. */}} {{- if not .Values.redis.deploy }} {{- if not .Values.redis.external.host }} @@ -19,12 +20,9 @@ This template will fail the Helm deployment if external config is missing for re {{- fail "ClickHouse external configuration is required when clickhouse.deploy=false. Please provide clickhouse.external.host and clickhouse.external.username" }} {{- end }} {{- end }} +{{/* When clickhouse.deploy=true the password is auto-generated into the datastore Secret and consumed via auth.existingSecret; set clickhouse.auth.password to pin it. */}} -{{- if .Values.s3.deploy }} -{{- if and (not .Values.s3.auth.existingSecret) (not .Values.s3.auth.accessKeyId) (not .Values.s3.auth.rootUser) }} -{{- fail "S3 auth credentials are required when s3.deploy=true. Please provide either s3.auth.accessKeyId, s3.auth.existingSecret, or s3.auth.rootUser" }} -{{- end }} -{{- else }} +{{- if not .Values.s3.deploy }} {{- if not .Values.s3.external.endpoint }} {{- fail "S3 external configuration is required when s3.deploy=false. Please provide s3.external.endpoint" }} {{- end }} @@ -32,6 +30,7 @@ This template will fail the Helm deployment if external config is missing for re {{- fail "S3 credentials are required when s3.deploy=false. Please provide either s3.external.existingSecret or both s3.external.accessKeyId and s3.external.secretAccessKey" }} {{- end }} {{- end }} +{{/* When s3.deploy=true the MinIO root password is auto-generated into the datastore Secret and consumed via auth.existingSecret; set s3.auth.rootPassword to pin it. */}} {{- if not .Values.electric.deploy }} {{- if not .Values.electric.external.url }} @@ -48,6 +47,14 @@ This template will fail the Helm deployment if external config is missing for re {{- fail "Registry external configuration is required when registry.deploy=false. Please provide registry.external.host" }} {{- end }} {{- end }} +{{/* When registry.deploy=true the password is auto-generated + retained inside secrets.yaml; set registry.auth.password to pin it. */}} + +{{/* +Application and control-plane secrets are auto-generated by templates/secrets.yaml +when left unset (retained across upgrades via lookup), so they need no fail-closed +guard here. The webapp still rejects previously published values at startup, even +when supplied via secrets.existingSecret. +*/}} {{/* This template produces no output but will fail the deployment if validation fails diff --git a/hosting/k8s/helm/templates/webapp.yaml b/hosting/k8s/helm/templates/webapp.yaml index f83ff0ecb..49acee98f 100644 --- a/hosting/k8s/helm/templates/webapp.yaml +++ b/hosting/k8s/helm/templates/webapp.yaml @@ -221,6 +221,13 @@ spec: name: {{ include "trigger-v4.postgres.external.secretName" . }} key: {{ include "trigger-v4.postgres.external.directUrlKey" . }} {{- else }} + {{- if .Values.postgres.deploy }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "trigger-v4.datastore.secretName" . }} + key: password + {{- end }} - name: DATABASE_URL value: {{ include "trigger-v4.postgres.connectionString" . | quote }} - name: DIRECT_URL @@ -306,6 +313,16 @@ spec: secretKeyRef: name: {{ include "trigger-v4.secretsName" . }} key: ENCRYPTION_KEY + - name: PROVIDER_SECRET + valueFrom: + secretKeyRef: + name: {{ include "trigger-v4.secretsName" . }} + key: PROVIDER_SECRET + - name: COORDINATOR_SECRET + valueFrom: + secretKeyRef: + name: {{ include "trigger-v4.secretsName" . }} + key: COORDINATOR_SECRET - name: MANAGED_WORKER_SECRET valueFrom: secretKeyRef: @@ -401,6 +418,12 @@ spec: secretKeyRef: name: {{ include "trigger-v4.clickhouse.external.secretName" . }} key: {{ include "trigger-v4.clickhouse.external.passwordKey" . }} + {{- else if .Values.clickhouse.deploy }} + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "trigger-v4.datastore.secretName" . }} + key: clickhouse-admin-password {{- end }} - name: CLICKHOUSE_URL value: {{ include "trigger-v4.clickhouse.url" . | quote }} diff --git a/hosting/k8s/helm/values-production-example.yaml b/hosting/k8s/helm/values-production-example.yaml index 65bdc0a6f..ee99de58a 100644 --- a/hosting/k8s/helm/values-production-example.yaml +++ b/hosting/k8s/helm/values-production-example.yaml @@ -7,6 +7,8 @@ secrets: magicLinkSecret: "YOUR_32_CHAR_HEX_SECRET_HERE_002" encryptionKey: "YOUR_32_CHAR_HEX_SECRET_HERE_003" managedWorkerSecret: "YOUR_32_CHAR_HEX_SECRET_HERE_004" + providerSecret: "YOUR_32_CHAR_HEX_SECRET_HERE_005" + coordinatorSecret: "YOUR_32_CHAR_HEX_SECRET_HERE_006" # Production webapp configuration webapp: @@ -42,6 +44,10 @@ webapp: # Production PostgreSQL (or use external) postgres: + auth: + # Required — no built-in default. The webapp connection string uses postgres.auth.password. + postgresPassword: "your-strong-postgres-password" + password: "your-strong-postgres-password" primary: persistence: enabled: true @@ -72,6 +78,9 @@ redis: # Production ClickHouse clickhouse: + auth: + # Required — no built-in default. The webapp connection string uses clickhouse.auth.password. + password: "your-strong-clickhouse-password" # Set to true to enable TLS/secure connections in production secure: true persistence: diff --git a/hosting/k8s/helm/values.yaml b/hosting/k8s/helm/values.yaml index 968419d98..e76a92127 100644 --- a/hosting/k8s/helm/values.yaml +++ b/hosting/k8s/helm/values.yaml @@ -10,11 +10,10 @@ nameOverride: "" fullnameOverride: "" # Secrets configuration -# IMPORTANT: The default values below are for TESTING ONLY and should NOT be used in production -# For production deployments: -# 1. Generate new secrets using: openssl rand -hex 16 -# 2. Override these values in your values.yaml or use external secret management -# 3. Each secret must be exactly 32 hex characters (16 bytes) +# No working defaults are shipped. Leave a secret empty and the chart generates a +# strong random value on first install, retained across upgrades (so sessions and +# encrypted data survive). Set a value explicitly to control it yourself, or use +# secrets.existingSecret / external secret management. Explicit values win. secrets: # Enable/disable creation of secrets # Set to false to use external secret management (Vault, Infisical, External Secrets, etc.) @@ -27,17 +26,23 @@ secrets: # - SESSION_SECRET # - MAGIC_LINK_SECRET # - ENCRYPTION_KEY + # - PROVIDER_SECRET + # - COORDINATOR_SECRET # - MANAGED_WORKER_SECRET existingSecret: "" - # Session secret for user authentication (32 hex chars) - sessionSecret: "2818143646516f6fffd707b36f334bbb" - # Magic link secret for passwordless login (32 hex chars) - magicLinkSecret: "44da78b7bbb0dfe709cf38931d25dcdd" - # Encryption key for sensitive data (32 hex chars) - encryptionKey: "f686147ab967943ebbe9ed3b496e465a" - # Worker secret for managed worker authentication (32 hex chars) - managedWorkerSecret: "447c29678f9eaf289e9c4b70d3dd8a7f" + # Session secret for user authentication + sessionSecret: "" # Leave empty to auto-generate, or set a strong value. + # Magic link secret for passwordless login + magicLinkSecret: "" # Leave empty to auto-generate, or set a strong value. + # Encryption key for sensitive data + encryptionKey: "" # Leave empty to auto-generate, or set a strong value. + # Provider socket secret + providerSecret: "" # Leave empty to auto-generate, or set a strong value. + # Coordinator socket secret + coordinatorSecret: "" # Leave empty to auto-generate, or set a strong value. + # Worker secret for managed worker authentication + managedWorkerSecret: "" # Leave empty to auto-generate, or set a strong value. # Object store credentials moved to s3.auth and s3.external section # Webapp configuration @@ -397,10 +402,16 @@ postgres: # Bitnami PostgreSQL chart configuration (when deploy: true) auth: enablePostgresUser: true - postgresPassword: "postgres" + postgresPassword: "" # Leave empty to auto-generate into the datastore secret, or set to pin. username: "postgres" - password: "postgres" + password: "" # Leave empty to auto-generate into the datastore secret, or set to pin. database: "main" + # Read the auto-generated password from the chart-managed datastore secret + # (templates/datastore-secret.yaml). Must equal trigger-v4.datastore.secretName. + existingSecret: "trigger-datastore" + secretKeys: + adminPasswordKey: "postgres-password" + userPasswordKey: "password" primary: persistence: @@ -632,7 +643,10 @@ clickhouse: # Bitnami ClickHouse chart configuration (when deploy: true) auth: username: "default" - password: "password" + password: "" # Leave empty to auto-generate into the datastore secret, or set to pin. + # Read the auto-generated password from the chart-managed datastore secret. + existingSecret: "trigger-datastore" + existingSecretKey: "clickhouse-admin-password" # Single-node configuration (disable clustering for dev/test) keeper: @@ -709,14 +723,18 @@ s3: # MinIO provides S3-compatible storage when deployed internally auth: rootUser: "admin" - rootPassword: "very-safe-password" + rootPassword: "" # Leave empty to auto-generate into the datastore secret, or set to pin. # Webapp credentials for S3 access (defaults to root credentials if not specified) accessKeyId: "" # Defaults to rootUser if empty secretAccessKey: "" # Defaults to rootPassword if empty - # Existing secret support for webapp credentials - existingSecret: "" # If set, accessKeyId/secretAccessKey will be ignored - accessKeyIdSecretKey: "access-key-id" # Key in existingSecret containing access key ID - secretAccessKeySecretKey: "secret-access-key" # Key in existingSecret containing secret access key + # Read the auto-generated root credentials from the chart-managed datastore secret. + # The same keys serve the MinIO subchart (rootUser/rootPassword) and the webapp's + # S3 credentials, so both stay in sync. Must equal trigger-v4.datastore.secretName. + existingSecret: "trigger-datastore" + rootUserSecretKey: "minio-root-user" + rootPasswordSecretKey: "minio-root-password" + accessKeyIdSecretKey: "minio-root-user" # Key in existingSecret containing access key ID + secretAccessKeySecretKey: "minio-root-password" # Key in existingSecret containing secret access key # The required "packets" bucket is created by default. defaultBuckets: "packets" @@ -729,8 +747,8 @@ s3: # External S3 connection (when deploy: false) external: endpoint: "" # e.g., "https://s3.amazonaws.com" or "https://your-minio.com:9000" - accessKeyId: "admin" # Default for internal MinIO - change for production - secretAccessKey: "very-safe-password" # Default for internal MinIO - change for production + accessKeyId: "" # Required when s3.deploy=false — no default. Set explicitly or use existingSecret. + secretAccessKey: "" # Required when s3.deploy=false — no default. Set explicitly or use existingSecret. # # Secure credential management existingSecret: "" # Name of existing secret containing S3 credentials @@ -769,7 +787,7 @@ registry: auth: enabled: true username: "registry-user" - password: "very-secure-indeed" + password: "" # Required when registry.deploy=true and auth.enabled — no default. Set a strong value. # External Registry connection (when deploy: false) external: diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index bf9612ba1..3c1f4330a 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -1497,6 +1497,7 @@ export class RunEngine { workerId, runnerId, isWarmStart, + environmentId, tx, }: { runId: string; @@ -1504,6 +1505,7 @@ export class RunEngine { workerId?: string; runnerId?: string; isWarmStart?: boolean; + environmentId?: string; tx?: PrismaClientOrTransaction; }): Promise { return this.runAttemptSystem.startRunAttempt({ @@ -1512,6 +1514,7 @@ export class RunEngine { workerId, runnerId, isWarmStart, + environmentId, tx, }); } @@ -1523,12 +1526,14 @@ export class RunEngine { completion, workerId, runnerId, + environmentId, }: { runId: string; snapshotId: string; completion: TaskRunExecutionResult; workerId?: string; runnerId?: string; + environmentId?: string; }): Promise { return this.runAttemptSystem.completeRunAttempt({ runId, @@ -1536,6 +1541,7 @@ export class RunEngine { completion, workerId, runnerId, + environmentId, }); } @@ -2014,12 +2020,14 @@ export class RunEngine { snapshotId, workerId, runnerId, + environmentId, tx, }: { runId: string; snapshotId: string; workerId?: string; runnerId?: string; + environmentId?: string; tx?: PrismaClientOrTransaction; }): Promise { return this.checkpointSystem.continueRunExecution({ @@ -2027,6 +2035,7 @@ export class RunEngine { snapshotId, workerId, runnerId, + environmentId, tx, }); } @@ -2062,14 +2071,21 @@ export class RunEngine { /** Get required data to execute the run */ async getRunExecutionData({ runId, + environmentId, tx, }: { runId: string; + environmentId?: string; tx?: PrismaClientOrTransaction; }): Promise { const prisma = tx ?? this.prisma; try { - const snapshot = await getLatestExecutionSnapshot(prisma, runId, this.runStore); + const snapshot = await getLatestExecutionSnapshot( + prisma, + runId, + this.runStore, + environmentId + ); return executionDataFromSnapshot(snapshot); } catch (e) { this.logger.error("Failed to getRunExecutionData", { @@ -2086,10 +2102,12 @@ export class RunEngine { async getSnapshotsSince({ runId, snapshotId, + environmentId, tx, }: { runId: string; snapshotId: string; + environmentId?: string; tx?: PrismaClientOrTransaction; }): Promise { const useReplica = @@ -2107,7 +2125,8 @@ export class RunEngine { runId, snapshotId, this.runStore, - repairClient + repairClient, + environmentId ); return snapshots.map(executionDataFromSnapshot); }; diff --git a/internal-packages/run-engine/src/engine/systems/checkpointSystem.ts b/internal-packages/run-engine/src/engine/systems/checkpointSystem.ts index 7bac40838..a8f4eab1f 100644 --- a/internal-packages/run-engine/src/engine/systems/checkpointSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/checkpointSystem.ts @@ -271,18 +271,25 @@ export class CheckpointSystem { snapshotId, workerId, runnerId, + environmentId, tx, }: { runId: string; snapshotId: string; workerId?: string; runnerId?: string; + environmentId?: string; tx?: PrismaClientOrTransaction; }): Promise { const prisma = tx ?? this.$.prisma; return await this.$.runLock.lock("continueRunExecution", [runId], async () => { - const snapshot = await getLatestExecutionSnapshot(prisma, runId, this.$.runStore); + const snapshot = await getLatestExecutionSnapshot( + prisma, + runId, + this.$.runStore, + environmentId + ); if (snapshot.id !== snapshotId) { throw new ServiceValidationError( diff --git a/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts b/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts index b98f2920b..dca4c66b2 100644 --- a/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts @@ -11,7 +11,7 @@ import type { Waitpoint, } from "@trigger.dev/database"; import type { RunStore } from "@internal/run-store"; -import { ExecutionSnapshotNotFoundError } from "../errors.js"; +import { ExecutionSnapshotNotFoundError, ServiceValidationError } from "../errors.js"; import type { HeartbeatTimeouts } from "../types.js"; import type { SystemResources } from "./systems.js"; @@ -195,16 +195,21 @@ async function fetchWaitpointsInChunks( return allWaitpoints; } -/* Gets the most recent valid snapshot for a run */ +/** + * Gets the most recent valid snapshot for a run. When `environmentId` is provided the read is scoped + * to that environment (tenant boundary): a run in another environment reads as not-found and rejects + * with a 404 rather than leaking existence. Internal callers omit it to read regardless of env. + */ export async function getLatestExecutionSnapshot( prisma: PrismaClientOrTransaction, runId: string, - runStore?: RunStore + runStore?: RunStore, + environmentId?: string ): Promise { const snapshot = runStore - ? await runStore.findLatestExecutionSnapshot(runId, prisma) + ? await runStore.findLatestExecutionSnapshot(runId, prisma, environmentId) : await prisma.taskRunExecutionSnapshot.findFirst({ - where: { runId, isValid: true }, + where: { runId, isValid: true, ...(environmentId ? { environmentId } : {}) }, include: { completedWaitpoints: true, checkpoint: true, @@ -213,6 +218,9 @@ export async function getLatestExecutionSnapshot( }); if (!snapshot) { + if (environmentId) { + throw new ServiceValidationError(`No execution snapshot found for TaskRun ${runId}`, 404); + } throw new Error(`No execution snapshot found for TaskRun ${runId}`); } @@ -295,19 +303,24 @@ export async function getExecutionSnapshotsSince( runStore?: RunStore, // The primary, for read-repair when `prisma` is a lagging read replica (see Step 3). Omit when // `prisma` is already the primary. - repairClient?: PrismaClientOrTransaction + repairClient?: PrismaClientOrTransaction, + // When set, scopes both reads to this environment (tenant boundary): a run in another env reads as + // not-found. Omit to read regardless of environment (internal callers). + environmentId?: string ): Promise { + const envScope = environmentId ? { environmentId } : {}; + // Step 1: Find the createdAt of the sinceSnapshotId const sinceSnapshot = runStore ? await runStore.findExecutionSnapshot( { - where: { id: sinceSnapshotId, runId }, + where: { id: sinceSnapshotId, runId, ...envScope }, select: { createdAt: true }, }, prisma ) : await prisma.taskRunExecutionSnapshot.findFirst({ - where: { id: sinceSnapshotId, runId }, + where: { id: sinceSnapshotId, runId, ...envScope }, select: { createdAt: true }, }); @@ -323,6 +336,7 @@ export async function getExecutionSnapshotsSince( runId, isValid: true, createdAt: { gt: sinceSnapshot.createdAt }, + ...envScope, }, include: { checkpoint: true, @@ -338,6 +352,7 @@ export async function getExecutionSnapshotsSince( runId, isValid: true, createdAt: { gt: sinceSnapshot.createdAt }, + ...envScope, }, include: { checkpoint: true, diff --git a/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts b/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts index 922b41cd2..c7a331fe9 100644 --- a/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts @@ -300,6 +300,7 @@ export class RunAttemptSystem { workerId, runnerId, isWarmStart, + environmentId, tx, }: { runId: string; @@ -307,6 +308,7 @@ export class RunAttemptSystem { workerId?: string; runnerId?: string; isWarmStart?: boolean; + environmentId?: string; tx?: PrismaClientOrTransaction; }): Promise { const prisma = tx ?? this.$.prisma; @@ -316,7 +318,12 @@ export class RunAttemptSystem { "startRunAttempt", async (span) => { return this.$.runLock.lock("startRunAttempt", [runId], async () => { - const latestSnapshot = await getLatestExecutionSnapshot(prisma, runId, this.$.runStore); + const latestSnapshot = await getLatestExecutionSnapshot( + prisma, + runId, + this.$.runStore, + environmentId + ); if (latestSnapshot.id !== snapshotId) { //if there is a big delay between the snapshot and the attempt, the snapshot might have changed @@ -643,12 +650,14 @@ export class RunAttemptSystem { completion, workerId, runnerId, + environmentId, }: { runId: string; snapshotId: string; completion: TaskRunExecutionResult; workerId?: string; runnerId?: string; + environmentId?: string; }): Promise { await this.#notifyMetadataUpdated(runId, completion); @@ -661,6 +670,7 @@ export class RunAttemptSystem { tx: this.$.prisma, workerId, runnerId, + environmentId, }); } case false: { @@ -671,6 +681,7 @@ export class RunAttemptSystem { tx: this.$.prisma, workerId, runnerId, + environmentId, }); } } @@ -683,6 +694,7 @@ export class RunAttemptSystem { tx, workerId, runnerId, + environmentId, }: { runId: string; snapshotId: string; @@ -690,6 +702,7 @@ export class RunAttemptSystem { tx: PrismaClientOrTransaction; workerId?: string; runnerId?: string; + environmentId?: string; }): Promise { const prisma = tx ?? this.$.prisma; @@ -698,7 +711,12 @@ export class RunAttemptSystem { "#completeRunAttemptSuccess", async (span) => { return this.$.runLock.lock("attemptSucceeded", [runId], async () => { - const latestSnapshot = await getLatestExecutionSnapshot(prisma, runId, this.$.runStore); + const latestSnapshot = await getLatestExecutionSnapshot( + prisma, + runId, + this.$.runStore, + environmentId + ); if (latestSnapshot.id !== snapshotId) { throw new ServiceValidationError("Snapshot ID doesn't match the latest snapshot", 400); @@ -868,6 +886,7 @@ export class RunAttemptSystem { runnerId, completion, forceRequeue, + environmentId, tx, }: { runId: string; @@ -876,6 +895,7 @@ export class RunAttemptSystem { runnerId?: string; completion: TaskRunFailedExecutionResult; forceRequeue?: boolean; + environmentId?: string; tx: PrismaClientOrTransaction; }): Promise { const prisma = this.$.prisma; @@ -885,7 +905,12 @@ export class RunAttemptSystem { "completeRunAttemptFailure", async (span) => { return this.$.runLock.lock("attemptFailed", [runId], async () => { - const latestSnapshot = await getLatestExecutionSnapshot(prisma, runId, this.$.runStore); + const latestSnapshot = await getLatestExecutionSnapshot( + prisma, + runId, + this.$.runStore, + environmentId + ); if (latestSnapshot.id !== snapshotId) { throw new ServiceValidationError("Snapshot ID doesn't match the latest snapshot", 400); diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index 5f636ada9..1b0d2f89c 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -1708,15 +1708,17 @@ export class PostgresRunStore implements RunStore { async findLatestExecutionSnapshot( runId: string, - client?: ReadClient + client?: ReadClient, + environmentId?: string ): Promise | null> { const prisma = client ?? this.readOnlyPrisma; + const where = { runId, isValid: true, ...(environmentId ? { environmentId } : {}) }; if (this.schemaVariant === "dedicated") { const snapshot = await prisma.taskRunExecutionSnapshot.findFirst({ - where: { runId, isValid: true }, + where, include: { checkpoint: true }, orderBy: { createdAt: "desc" }, }); @@ -1728,7 +1730,7 @@ export class PostgresRunStore implements RunStore { } return prisma.taskRunExecutionSnapshot.findFirst({ - where: { runId, isValid: true }, + where, include: { completedWaitpoints: true, checkpoint: true, diff --git a/internal-packages/run-store/src/runOpsStore.ts b/internal-packages/run-store/src/runOpsStore.ts index d14ae5252..ba4969807 100644 --- a/internal-packages/run-store/src/runOpsStore.ts +++ b/internal-packages/run-store/src/runOpsStore.ts @@ -841,14 +841,16 @@ export class RoutingRunStore implements RunStore { // OUTPUT is silently missing from the resume payload — re-resolve them across BOTH DBs. async findLatestExecutionSnapshot( runId: string, - client?: ReadClient + client?: ReadClient, + environmentId?: string ): Promise | null> { const owningStore = this.#routeOrNew(runId); const snapshot = await owningStore.findLatestExecutionSnapshot( runId, - RoutingRunStore.#ownPrimary(owningStore, client) + RoutingRunStore.#ownPrimary(owningStore, client), + environmentId ); if (snapshot) { await this.#reresolveCompletedWaitpointsCrossDb( diff --git a/internal-packages/run-store/src/types.ts b/internal-packages/run-store/src/types.ts index b55e8f968..374a550e4 100644 --- a/internal-packages/run-store/src/types.ts +++ b/internal-packages/run-store/src/types.ts @@ -673,7 +673,10 @@ export interface RunStore { // Snapshot group findLatestExecutionSnapshot( runId: string, - client?: ReadClient + client?: ReadClient, + // When set, scopes the read to this environment (tenant boundary); a run in another env reads as + // not-found. Omit to read regardless of environment (internal callers). + environmentId?: string ): Promise | null>; diff --git a/internal-packages/testcontainers/src/webapp.ts b/internal-packages/testcontainers/src/webapp.ts index 2d2774df4..fc587abd6 100644 --- a/internal-packages/testcontainers/src/webapp.ts +++ b/internal-packages/testcontainers/src/webapp.ts @@ -104,6 +104,11 @@ export async function startWebapp( SESSION_SECRET: "test-session-secret-for-e2e-tests", MAGIC_LINK_SECRET: "test-magic-link-secret-32chars!!", ENCRYPTION_KEY: "test-encryption-key-for-e2e!!!!!", // exactly 32 bytes + // Control-plane auth secrets are required (no defaults) and reject the + // known-insecure placeholder strings, so supply strong test values here. + PROVIDER_SECRET: "test-provider-secret-for-e2e-tests", + COORDINATOR_SECRET: "test-coordinator-secret-for-e2e-tests", + MANAGED_WORKER_SECRET: "test-managed-worker-secret-for-e2e-tests", CLICKHOUSE_URL: "http://localhost:19123", // dummy, auth paths never connect DEPLOY_REGISTRY_HOST: "registry.example.com", // dummy, not needed for auth tests ELECTRIC_ORIGIN: "http://localhost:3060", diff --git a/internal-packages/tsql/src/query/printer.ts b/internal-packages/tsql/src/query/printer.ts index 6f1af07d7..00ff8eabd 100644 --- a/internal-packages/tsql/src/query/printer.ts +++ b/internal-packages/tsql/src/query/printer.ts @@ -3127,8 +3127,16 @@ export class ClickHousePrinter { } private visitWindowFunction(node: WindowFunction): string { + // Validate the function name against the allowlist and resolve it to its + // safe ClickHouse name. Without this, an attacker-controlled name would be + // emitted raw into the query (mirrors the validation in visitCall). + const funcMeta = findTSQLFunction(node.name) ?? findTSQLAggregation(node.name); + if (!funcMeta) { + throw new QueryError(`Unknown function: ${node.name}`); + } + const args = node.args ? node.args.map((a) => this.visit(a)) : []; - const funcCall = `${node.name}(${args.join(", ")})`; + const funcCall = `${funcMeta.clickhouseName}(${args.join(", ")})`; if (node.over_identifier) { return `${funcCall} OVER ${this.printIdentifier(node.over_identifier)}`; diff --git a/packages/cli-v3/src/commands/login.ts b/packages/cli-v3/src/commands/login.ts index 0171286d4..83576b0d0 100644 --- a/packages/cli-v3/src/commands/login.ts +++ b/packages/cli-v3/src/commands/login.ts @@ -295,9 +295,10 @@ export async function login(options?: LoginOptions): Promise { const indexResult = await pRetry( () => getPersonalAccessToken(apiClient, authorizationCodeResult.authorizationCode), { - //this means we're polling, same distance between each attempt + //poll at a fixed 1s interval. ~5 min window so the user has time to + //approve the consent screen; stays within the code's 10-min validity. factor: 1, - retries: 60, + retries: 300, minTimeout: 1000, } ); @@ -404,6 +405,15 @@ export async function getPersonalAccessToken(apiClient: CliApiClient, authorizat const token = await apiClient.getPersonalAccessToken(authorizationCode); if (!token.success) { + // A 429 from the per-code poll rate limiter is transient: the auth code + // is still valid and the user may just not have approved the consent + // screen yet. Throw a regular (retryable) error so the poll loop backs + // off and keeps polling, rather than an AbortError, which pRetry treats + // as fatal and would abandon the whole login. + if (token.statusCode === 429) { + throw new Error(token.error); + } + throw new AbortError(token.error); } diff --git a/packages/cli-v3/src/entryPoints/managed/controller.ts b/packages/cli-v3/src/entryPoints/managed/controller.ts index 5c63a4082..ae1f1d22a 100644 --- a/packages/cli-v3/src/entryPoints/managed/controller.ts +++ b/packages/cli-v3/src/entryPoints/managed/controller.ts @@ -76,7 +76,7 @@ export class ManagedRunController { }); const properties = { - ...env.raw, + ...env.rawForLogging, TRIGGER_POD_SCHEDULED_AT_MS: env.TRIGGER_POD_SCHEDULED_AT_MS.toISOString(), TRIGGER_DEQUEUED_AT_MS: env.TRIGGER_DEQUEUED_AT_MS.toISOString(), }; diff --git a/packages/cli-v3/src/entryPoints/managed/env.ts b/packages/cli-v3/src/entryPoints/managed/env.ts index 4da8361e8..825e83e15 100644 --- a/packages/cli-v3/src/entryPoints/managed/env.ts +++ b/packages/cli-v3/src/entryPoints/managed/env.ts @@ -27,6 +27,9 @@ const Env = z.object({ // Set at runtime TRIGGER_DEPLOYMENT_ID: z.string(), + // Plain deployment friendlyId for telemetry. Optional: older supervisors don't set it, in which + // case we fall back to TRIGGER_DEPLOYMENT_ID. + TRIGGER_DEPLOYMENT_FRIENDLY_ID: z.string().optional(), TRIGGER_DEPLOYMENT_VERSION: z.string(), TRIGGER_WORKLOAD_CONTROLLER_ID: z.string().default(`controller_${randomUUID()}`), TRIGGER_ENV_ID: z.string(), @@ -74,6 +77,11 @@ export class RunnerEnv { return this.env; } + // TRIGGER_DEPLOYMENT_ID carries the deployment token; redact it before logging. + get rawForLogging() { + return { ...this.env, TRIGGER_DEPLOYMENT_ID: "[redacted]" }; + } + // Base environment variables get NODE_ENV() { return this.env.NODE_ENV; @@ -93,6 +101,9 @@ export class RunnerEnv { get TRIGGER_DEPLOYMENT_ID() { return this.env.TRIGGER_DEPLOYMENT_ID; } + get TRIGGER_DEPLOYMENT_FRIENDLY_ID() { + return this.env.TRIGGER_DEPLOYMENT_FRIENDLY_ID; + } get TRIGGER_DEPLOYMENT_VERSION() { return this.env.TRIGGER_DEPLOYMENT_VERSION; } diff --git a/packages/cli-v3/src/entryPoints/managed/execution.ts b/packages/cli-v3/src/entryPoints/managed/execution.ts index 8dca48fe2..c5bb4875b 100644 --- a/packages/cli-v3/src/entryPoints/managed/execution.ts +++ b/packages/cli-v3/src/entryPoints/managed/execution.ts @@ -951,7 +951,7 @@ export class RunExecution { this.sendDebugLog(`[override] processing: ${reason}`, { overrides, - currentEnv: this.env.raw, + currentEnv: this.env.rawForLogging, }); // Override the env with the new values diff --git a/packages/cli-v3/src/entryPoints/managed/taskRunProcessProvider.ts b/packages/cli-v3/src/entryPoints/managed/taskRunProcessProvider.ts index 4e0a56122..aaf09862f 100644 --- a/packages/cli-v3/src/entryPoints/managed/taskRunProcessProvider.ts +++ b/packages/cli-v3/src/entryPoints/managed/taskRunProcessProvider.ts @@ -252,11 +252,14 @@ export class TaskRunProcessProvider { workerManifest: this.workerManifest, env: processEnv, serverWorker: { - id: this.env.TRIGGER_DEPLOYMENT_ID, + // Telemetry-only (becomes OTel worker.id). Prefer the plain friendlyId; fall back to + // DEPLOYMENT_ID (which may be an opaque token) only when an older supervisor didn't set it. + id: this.env.TRIGGER_DEPLOYMENT_FRIENDLY_ID ?? this.env.TRIGGER_DEPLOYMENT_ID, contentHash: this.env.TRIGGER_CONTENT_HASH, version: this.env.TRIGGER_DEPLOYMENT_VERSION, engine: "V2", }, + machineResources: { cpu: Number(this.env.TRIGGER_MACHINE_CPU), memory: Number(this.env.TRIGGER_MACHINE_MEMORY), diff --git a/packages/cli-v3/src/mcp/auth.ts b/packages/cli-v3/src/mcp/auth.ts index b948f2cb8..fa5f62708 100644 --- a/packages/cli-v3/src/mcp/auth.ts +++ b/packages/cli-v3/src/mcp/auth.ts @@ -123,9 +123,10 @@ export async function mcpAuth(options: McpAuthOptions): Promise { const indexResult = await pRetry( () => getPersonalAccessToken(apiClient, authorizationCodeResult.authorizationCode), { - //this means we're polling, same distance between each attempt + //poll at a fixed 1s interval. ~5 min window so the user has time to + //approve the consent screen; stays within the code's 10-min validity. factor: 1, - retries: 60, + retries: 300, minTimeout: 1000, } ); diff --git a/packages/core/src/v3/apiClient/core.ts b/packages/core/src/v3/apiClient/core.ts index 3c9ae47cf..c6425e7ee 100644 --- a/packages/core/src/v3/apiClient/core.ts +++ b/packages/core/src/v3/apiClient/core.ts @@ -725,6 +725,13 @@ export type ApiResult = | { success: false; error: string; + /** + * HTTP status code, when the failure originated from an API response + * (e.g. 429 for rate limiting). Undefined for connection/transport + * errors that never reached the server. Lets callers distinguish + * transient failures worth retrying from fatal ones. + */ + statusCode?: number; }; export async function wrapZodFetch( @@ -754,6 +761,7 @@ export async function wrapZodFetch( return { success: false, error: error.message, + statusCode: error.status, }; } else if (error instanceof Error) { return { diff --git a/packages/core/src/v3/index.ts b/packages/core/src/v3/index.ts index 9052884bb..35dbfa287 100644 --- a/packages/core/src/v3/index.ts +++ b/packages/core/src/v3/index.ts @@ -30,6 +30,7 @@ export * from "./resource-catalog-api.js"; export * from "./types/index.js"; export { links } from "./links.js"; export * from "./jwt.js"; +export * from "./workloadDeploymentToken.js"; export * from "./idempotencyKeys.js"; export * from "./streams/asyncIterableStream.js"; export * from "./utils/getEnv.js"; diff --git a/packages/core/src/v3/jwt.ts b/packages/core/src/v3/jwt.ts index d71f1e7f1..6845225be 100644 --- a/packages/core/src/v3/jwt.ts +++ b/packages/core/src/v3/jwt.ts @@ -4,6 +4,10 @@ export type GenerateJWTOptions = { secretKey: string; payload: Record; expirationTime?: number | Date | string; + // Skip the `iat` claim. Combined with an absolute `expirationTime`, this makes the signed token a + // pure function of its payload — the same claims mint byte-identical tokens. Off by default so + // ordinary short-lived tokens keep their issued-at. + omitIssuedAt?: boolean; }; export const JWT_ALGORITHM = "HS256"; @@ -15,13 +19,17 @@ export async function generateJWT(options: GenerateJWTOptions): Promise const secret = new TextEncoder().encode(options.secretKey); - return new SignJWT(options.payload) + const jwt = new SignJWT(options.payload) .setIssuer(JWT_ISSUER) .setAudience(JWT_AUDIENCE) .setProtectedHeader({ alg: JWT_ALGORITHM }) - .setIssuedAt() - .setExpirationTime(options.expirationTime ?? "15m") - .sign(secret); + .setExpirationTime(options.expirationTime ?? "15m"); + + if (!options.omitIssuedAt) { + jwt.setIssuedAt(); + } + + return jwt.sign(secret); } export type ValidationResult = diff --git a/packages/core/src/v3/runEngineWorker/consts.ts b/packages/core/src/v3/runEngineWorker/consts.ts index 29dc76b53..e04e83875 100644 --- a/packages/core/src/v3/runEngineWorker/consts.ts +++ b/packages/core/src/v3/runEngineWorker/consts.ts @@ -3,6 +3,9 @@ export const WORKER_HEADERS = { DEPLOYMENT_ID: "x-trigger-worker-deployment-id", MANAGED_SECRET: "x-trigger-worker-managed-secret", RUNNER_ID: "x-trigger-worker-runner-id", + // Verified environment id recovered from the deployment token. Set by the supervisor only when + // enforcing; the platform scopes the worker-action's snapshot read by it. + ENVIRONMENT_ID: "x-trigger-worker-environment-id", }; export const WORKLOAD_HEADERS = { diff --git a/packages/core/src/v3/runEngineWorker/supervisor/http.ts b/packages/core/src/v3/runEngineWorker/supervisor/http.ts index bbb1fe227..d6d51b1c2 100644 --- a/packages/core/src/v3/runEngineWorker/supervisor/http.ts +++ b/packages/core/src/v3/runEngineWorker/supervisor/http.ts @@ -143,7 +143,8 @@ export class SupervisorHttpClient { runId: string, snapshotId: string, body: WorkerApiRunAttemptStartRequestBody, - runnerId?: string + runnerId?: string, + environmentId?: string ) { return wrapZodFetch( WorkerApiRunAttemptStartResponseBody, @@ -153,6 +154,7 @@ export class SupervisorHttpClient { headers: { ...this.defaultHeaders, ...this.runnerIdHeader(runnerId), + ...this.environmentIdHeader(environmentId), }, body: JSON.stringify(body), } @@ -163,7 +165,8 @@ export class SupervisorHttpClient { runId: string, snapshotId: string, body: WorkerApiRunAttemptCompleteRequestBody, - runnerId?: string + runnerId?: string, + environmentId?: string ) { return wrapZodFetch( WorkerApiRunAttemptCompleteResponseBody, @@ -173,13 +176,14 @@ export class SupervisorHttpClient { headers: { ...this.defaultHeaders, ...this.runnerIdHeader(runnerId), + ...this.environmentIdHeader(environmentId), }, body: JSON.stringify(body), } ); } - async getLatestSnapshot(runId: string, runnerId?: string) { + async getLatestSnapshot(runId: string, runnerId?: string, environmentId?: string) { return wrapZodFetch( WorkerApiRunLatestSnapshotResponseBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/latest`, @@ -188,12 +192,18 @@ export class SupervisorHttpClient { headers: { ...this.defaultHeaders, ...this.runnerIdHeader(runnerId), + ...this.environmentIdHeader(environmentId), }, } ); } - async getSnapshotsSince(runId: string, snapshotId: string, runnerId?: string) { + async getSnapshotsSince( + runId: string, + snapshotId: string, + runnerId?: string, + environmentId?: string + ) { return wrapZodFetch( WorkerApiRunSnapshotsSinceResponseBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/since/${snapshotId}`, @@ -202,6 +212,7 @@ export class SupervisorHttpClient { headers: { ...this.defaultHeaders, ...this.runnerIdHeader(runnerId), + ...this.environmentIdHeader(environmentId), }, } ); @@ -235,7 +246,12 @@ export class SupervisorHttpClient { } } - async continueRunExecution(runId: string, snapshotId: string, runnerId?: string) { + async continueRunExecution( + runId: string, + snapshotId: string, + runnerId?: string, + environmentId?: string + ) { return wrapZodFetch( WorkerApiContinueRunExecutionRequestBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/continue`, @@ -244,6 +260,7 @@ export class SupervisorHttpClient { headers: { ...this.defaultHeaders, ...this.runnerIdHeader(runnerId), + ...this.environmentIdHeader(environmentId), }, }, { @@ -295,4 +312,10 @@ export class SupervisorHttpClient { [WORKER_HEADERS.RUNNER_ID]: runnerId, }); } + + private environmentIdHeader(environmentId?: string): Record { + return createHeaders({ + [WORKER_HEADERS.ENVIRONMENT_ID]: environmentId, + }); + } } diff --git a/packages/core/src/v3/runEngineWorker/types.ts b/packages/core/src/v3/runEngineWorker/types.ts index b54d23ccd..7a76af3f4 100644 --- a/packages/core/src/v3/runEngineWorker/types.ts +++ b/packages/core/src/v3/runEngineWorker/types.ts @@ -29,4 +29,6 @@ export type WorkloadClientSocketData = { runnerId: string; runFriendlyId?: string; snapshotId?: string; + // Deployment friendlyId resolved from the verified token; prefer over the raw deploymentId. + deploymentFriendlyId?: string; }; diff --git a/packages/core/src/v3/runMetadata/operations.ts b/packages/core/src/v3/runMetadata/operations.ts index b1cc68672..fff98c50b 100644 --- a/packages/core/src/v3/runMetadata/operations.ts +++ b/packages/core/src/v3/runMetadata/operations.ts @@ -1,5 +1,5 @@ import { JSONHeroPath } from "@jsonhero/path"; -import type { RunMetadataChangeOperation } from "../schemas/common.js"; +import { isSafeMetadataKey, type RunMetadataChangeOperation } from "../schemas/common.js"; import { dequal } from "dequal"; export type ApplyOperationResult = { @@ -16,6 +16,13 @@ export function applyMetadataOperations( let newMetadata: Record = structuredClone(currentMetadata); for (const operation of Array.isArray(operations) ? operations : [operations]) { + // Prevent unsafe JSON paths and direct __proto__ assignments from changing Object.prototype. + // ("update" carries no key.) + if (operation.type !== "update" && !isSafeMetadataKey(operation.key)) { + unappliedOperations.push(operation); + continue; + } + switch (operation.type) { case "set": { if (operation.key.startsWith("$.")) { diff --git a/packages/core/src/v3/schemas/common.ts b/packages/core/src/v3/schemas/common.ts index 4de2ddb58..a6ec4b100 100644 --- a/packages/core/src/v3/schemas/common.ts +++ b/packages/core/src/v3/schemas/common.ts @@ -4,6 +4,25 @@ import type { RuntimeEnvironmentType as DBRuntimeEnvironmentType } from "@trigge export type Enum = { [K in T]: K }; +const DANGEROUS_METADATA_KEY_SEGMENTS = new Set(["__proto__", "constructor", "prototype"]); + +/** + * Prototype-pollution guard for run metadata operation keys. JSON paths are applied via + * JSONHeroPath, so dangerous path segments must be rejected. Literal keys are assigned directly, + * where only __proto__ can change the target object's prototype. + */ +export function isSafeMetadataKey(key: string): boolean { + if (!key.startsWith("$.")) { + return key !== "__proto__"; + } + + return !key.split(/[.[\]'"]+/).some((segment) => DANGEROUS_METADATA_KEY_SEGMENTS.has(segment)); +} + +const MetadataOperationKey = z.string().refine(isSafeMetadataKey, { + message: "Metadata key may not reference __proto__, constructor, or prototype", +}); + export const RunMetadataUpdateOperation = z.object({ type: z.literal("update"), value: z.record(z.unknown()), @@ -13,7 +32,7 @@ export type RunMetadataUpdateOperation = z.infer; export const RunMetadataAppendKeyOperation = z.object({ type: z.literal("append"), - key: z.string(), + key: MetadataOperationKey, value: DeserializedJsonSchema, }); @@ -36,7 +55,7 @@ export type RunMetadataAppendKeyOperation = z.infer /^\d+$/.test(k))) { + // Convert the result to an array if all top-level keys are numeric indices. + // Guard against an empty result (e.g. every key was skipped as unsafe), which + // would otherwise produce Array(-Infinity) and throw. + if (Object.keys(result).length > 0 && Object.keys(result).every((k) => /^\d+$/.test(k))) { const maxIndex = Math.max(...Object.keys(result).map((k) => parseInt(k))); const arrayResult = Array(maxIndex + 1); for (const key in result) { diff --git a/packages/core/src/v3/workloadDeploymentToken.test.ts b/packages/core/src/v3/workloadDeploymentToken.test.ts new file mode 100644 index 000000000..4a807b002 --- /dev/null +++ b/packages/core/src/v3/workloadDeploymentToken.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it } from "vitest"; +import { generateJWT } from "./jwt.js"; +import { + classifyDeploymentIdHeader, + looksLikeWorkloadDeploymentToken, + mintWorkloadDeploymentToken, + verifyWorkloadDeploymentToken, + WORKLOAD_DEPLOYMENT_TOKEN_VERSION, + type WorkloadDeploymentTokenInput, +} from "./workloadDeploymentToken.js"; + +const SECRET = "test-workload-token-secret"; +const EXP = Math.floor(Date.UTC(2032, 0, 1) / 1000); + +const claims: WorkloadDeploymentTokenInput = { + deployment: "deployment_abc123", + deployment_version: "20260709.1", + environment_id: "env_1", + environment_type: "PRODUCTION", + org_id: "org_1", + project_id: "proj_1", +}; + +describe("workloadDeploymentToken", () => { + it("mints a token that verifies and round-trips every claim", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + + const result = await verifyWorkloadDeploymentToken(token, SECRET); + expect(result.ok).toBe(true); + if (!result.ok) return; + + expect(result.claims.ver).toBe(WORKLOAD_DEPLOYMENT_TOKEN_VERSION); + expect(result.claims.deployment).toBe(claims.deployment); + expect(result.claims.deployment_version).toBe(claims.deployment_version); + expect(result.claims.environment_id).toBe(claims.environment_id); + expect(result.claims.environment_type).toBe(claims.environment_type); + expect(result.claims.org_id).toBe(claims.org_id); + expect(result.claims.project_id).toBe(claims.project_id); + }); + + it("sets the caller-supplied absolute exp", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const result = await verifyWorkloadDeploymentToken(token, SECRET); + expect(result.ok).toBe(true); + if (!result.ok) return; + + expect(result.claims.exp).toBe(EXP); + }); + + it("mints byte-identical tokens for identical claims (deterministic, no iat)", async () => { + const a = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const b = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + expect(a).toBe(b); + + const result = await verifyWorkloadDeploymentToken(a, SECRET); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect((result.claims as Record).iat).toBeUndefined(); + }); + + it("rejects a token signed with a different secret", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const result = await verifyWorkloadDeploymentToken(token, "wrong-secret"); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.reason).toBe("invalid_signature"); + }); + + it("rejects a tampered token", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const [header, payload, signature] = token.split("."); + const tampered = `${header}.${payload}x.${signature}`; + const result = await verifyWorkloadDeploymentToken(tampered, SECRET); + expect(result.ok).toBe(false); + }); + + it("rejects a valid JWT that is missing required claims", async () => { + // Signed with the right secret, but not a workload deployment token. + const token = await generateJWT({ + secretKey: SECRET, + payload: { foo: "bar" }, + expirationTime: "1825d", + }); + const result = await verifyWorkloadDeploymentToken(token, SECRET); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.reason).toBe("malformed_claims"); + }); + + describe("looksLikeWorkloadDeploymentToken", () => { + it("is true for a minted token", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + expect(looksLikeWorkloadDeploymentToken(token)).toBe(true); + }); + + it("is false for a legacy bare friendlyId", () => { + expect(looksLikeWorkloadDeploymentToken("deployment_abc123")).toBe(false); + }); + + it("is false for empty and malformed values", () => { + expect(looksLikeWorkloadDeploymentToken("")).toBe(false); + expect(looksLikeWorkloadDeploymentToken("a.b")).toBe(false); + expect(looksLikeWorkloadDeploymentToken("a..c")).toBe(false); + }); + }); + + describe("classifyDeploymentIdHeader", () => { + it("classifies a minted token as jwt_valid and returns claims", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const result = await classifyDeploymentIdHeader(token, SECRET); + expect(result.outcome).toBe("jwt_valid"); + expect(result.claims?.deployment).toBe(claims.deployment); + }); + + it("classifies a JWT with a bad signature as jwt_invalid", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const result = await classifyDeploymentIdHeader(token, "wrong-secret"); + expect(result.outcome).toBe("jwt_invalid"); + expect(result.claims).toBeUndefined(); + }); + + it("classifies a bare friendlyId as legacy_bare", async () => { + const result = await classifyDeploymentIdHeader("deployment_abc123", SECRET); + expect(result.outcome).toBe("legacy_bare"); + }); + }); +}); diff --git a/packages/core/src/v3/workloadDeploymentToken.ts b/packages/core/src/v3/workloadDeploymentToken.ts new file mode 100644 index 000000000..4f0d36703 --- /dev/null +++ b/packages/core/src/v3/workloadDeploymentToken.ts @@ -0,0 +1,99 @@ +import { z } from "zod"; +import { generateJWT, validateJWT } from "./jwt.js"; + +/** + * Signed, deployment-scoped token carried in the TRIGGER_DEPLOYMENT_ID env var to identify a run + * controller. Long-lived identity token, not a rotating credential; the `ver` claim allows a future + * format revision. + */ +export const WORKLOAD_DEPLOYMENT_TOKEN_VERSION = 1 as const; + +export const WorkloadDeploymentTokenClaims = z.object({ + ver: z.literal(WORKLOAD_DEPLOYMENT_TOKEN_VERSION), + /** Deployment friendlyId (deployment_xxxx). */ + deployment: z.string(), + deployment_version: z.string(), + environment_id: z.string(), + environment_type: z.string(), + org_id: z.string(), + project_id: z.string(), + exp: z.number(), +}); + +export type WorkloadDeploymentTokenClaims = z.infer; + +/** Claims the caller supplies; `ver` and `exp` are set by the minter. */ +export type WorkloadDeploymentTokenInput = Omit; + +/** + * `expiresAtSeconds` is an absolute epoch (caller-supplied — the supervisor drives it). Combined with + * the omitted `iat`, the token is byte-deterministic per deployment: identical claims mint identical + * bytes, so the OTel `worker.id` the runner derives from it stays one value per deployment. + */ +export async function mintWorkloadDeploymentToken( + claims: WorkloadDeploymentTokenInput, + secret: string, + expiresAtSeconds: number +): Promise { + return generateJWT({ + secretKey: secret, + payload: { ver: WORKLOAD_DEPLOYMENT_TOKEN_VERSION, ...claims }, + expirationTime: expiresAtSeconds, + omitIssuedAt: true, + }); +} + +export type WorkloadDeploymentTokenVerification = + | { ok: true; claims: WorkloadDeploymentTokenClaims } + | { ok: false; reason: "invalid_signature" | "malformed_claims"; error: string }; + +export async function verifyWorkloadDeploymentToken( + token: string, + secret: string +): Promise { + const result = await validateJWT(token, secret); + + if (!result.ok) { + return { ok: false, reason: "invalid_signature", error: result.error }; + } + + const parsed = WorkloadDeploymentTokenClaims.safeParse(result.payload); + + if (!parsed.success) { + return { ok: false, reason: "malformed_claims", error: parsed.error.message }; + } + + return { ok: true, claims: parsed.data }; +} + +/** + * A legacy TRIGGER_DEPLOYMENT_ID is a bare friendlyId (deployment_xxxx); a minted token is a JWT of + * three non-empty base64url segments. Used by the dry-run to tell a pre-upgrade runner from a token. + */ +export function looksLikeWorkloadDeploymentToken(value: string): boolean { + const parts = value.split("."); + return parts.length === 3 && parts.every((part) => part.length > 0); +} + +export type DeploymentIdHeaderOutcome = "jwt_valid" | "jwt_invalid" | "legacy_bare"; + +/** + * Classify the value carried in the deployment-id header for the rollout metric, verifying the + * signature when it is JWT-shaped. Returns the claims on a valid token so callers avoid a second verify. + */ +export async function classifyDeploymentIdHeader( + value: string, + secret: string +): Promise<{ outcome: DeploymentIdHeaderOutcome; claims?: WorkloadDeploymentTokenClaims }> { + if (!looksLikeWorkloadDeploymentToken(value)) { + return { outcome: "legacy_bare" }; + } + + const result = await verifyWorkloadDeploymentToken(value, secret); + + if (result.ok) { + return { outcome: "jwt_valid", claims: result.claims }; + } + + return { outcome: "jwt_invalid" }; +} diff --git a/packages/core/test/taskExecutor.test.ts b/packages/core/test/taskExecutor.test.ts index 9bbda935f..d46a3fbe6 100644 --- a/packages/core/test/taskExecutor.test.ts +++ b/packages/core/test/taskExecutor.test.ts @@ -1802,8 +1802,10 @@ describe("TaskExecutor", () => { // Rate limit errors should use the rate limit retry delay expect((result.result as any).retry.delay).toBeGreaterThan(0); } else { - // Other retryable errors should use the exponential backoff - expect((result.result as any).retry.delay).toBeGreaterThan(1000); + // Other retryable errors should use the exponential backoff. The first retry uses + // minDelay (1000ms) as its base, and jitter can land exactly on the floor, so this + // is >= rather than > to avoid a flaky boundary failure. + expect((result.result as any).retry.delay).toBeGreaterThanOrEqual(1000); expect((result.result as any).retry.delay).toBeLessThan(5000); } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b0e328c76..29450d4b1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -585,6 +585,15 @@ importers: json-stable-stringify: specifier: ^1.3.0 version: 1.3.0 + jsonpointer: + specifier: ^5.0.1 + version: 5.0.1 + lodash.omit: + specifier: ^4.5.0 + version: 4.5.0 + lru-cache: + specifier: ^11.2.4 + version: 11.2.4 lucide-react: specifier: ^0.229.0 version: 0.229.0(react@18.3.1) @@ -11391,6 +11400,10 @@ packages: engines: {node: '>=18.0.0'} hasBin: true + jsonpointer@5.0.1: + resolution: {integrity: sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==} + engines: {node: '>=0.10.0'} + jsonwebtoken@9.0.2: resolution: {integrity: sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==} engines: {node: '>=12', npm: '>=6'} @@ -26634,6 +26647,8 @@ snapshots: '@jsep-plugin/regex': 1.0.4(jsep@1.4.0) jsep: 1.4.0 + jsonpointer@5.0.1: {} + jsonwebtoken@9.0.2: dependencies: jws: 3.2.3 From 325b906319be1cb14b3ba8eb1ca7ac1ef1a30f01 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 12:07:52 +0100 Subject: [PATCH 008/300] chore: release v4.5.6 (#4317) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary 5 improvements, 9 bug fixes. ## Breaking changes - Self-hosted deployments no longer ship shared default credentials; fresh installs generate their own. If yours still uses a previously published default, set a unique value before upgrading, or set `ALLOW_INSECURE_DEFAULT_SECRETS=true` to keep booting while you migrate. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) ## Improvements - Require explicit browser approval for CLI and MCP login, with resilient polling while approval is pending. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) - Deployed task telemetry now reports the deployment identifier (e.g. `deployment_abc123`) in the `worker.id` attribute, instead of an opaque internal value. Upgrade to get the readable identifier in your own OpenTelemetry exporters. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) - Prevent prototype pollution when applying run metadata operations or reconstructing nested telemetry attributes, while preserving legitimate `constructor` and `prototype` fields. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) - Add helpers to mint and verify the deployment-scoped token used to authenticate run controllers to the platform. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) ## Server changes These changes affect the self-hosted Docker image and Trigger.dev Cloud: - Added optional request rate limiting for telemetry ingestion endpoints. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) - Background-worker deployment lookups are now scoped to the authenticated environment. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) - Updating a GitHub App installation from the callback flow is now scoped to your own organization, so an installation ID belonging to another organization can no longer be used to refresh that organization's installation record. The GitHub App installation session is also now single-use, so completing an installation callback invalidates its state and it can no longer be replayed. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) - Scope schedule and environment-variable writes to the caller's project and environment ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) - Reject compute snapshot callbacks that do not match the snapshot request that created them. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) - Require secret-key authentication to initialize the session out (agent→client) stream, matching the append route. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) - Live run and trace subscriptions now validate their identifiers more strictly and only return data from your own organization. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) - Window-function names in the query compiler are now validated against the allowlist, matching how other function calls are handled. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) - Authenticate run controllers to the platform with a signed, deployment-scoped token. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) - Verify that worker actions (starting, completing, and continuing a run, and reading its snapshots) target a run belonging to the caller's environment. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316))
Raw changeset output # Releases ## @trigger.dev/build@4.5.6 ### Patch Changes - Updated dependencies: - `@trigger.dev/core@4.5.6` ## trigger.dev@4.5.6 ### Patch Changes - Require explicit browser approval for CLI and MCP login, with resilient polling while approval is pending. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) - Deployed task telemetry now reports the deployment identifier (e.g. `deployment_abc123`) in the `worker.id` attribute, instead of an opaque internal value. Upgrade to get the readable identifier in your own OpenTelemetry exporters. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) - Updated dependencies: - `@trigger.dev/core@4.5.6` - `@trigger.dev/build@4.5.6` - `@trigger.dev/schema-to-json@4.5.6` ## @trigger.dev/core@4.5.6 ### Patch Changes - Prevent prototype pollution when applying run metadata operations or reconstructing nested telemetry attributes, while preserving legitimate `constructor` and `prototype` fields. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) - Require explicit browser approval for CLI and MCP login, with resilient polling while approval is pending. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) - Add helpers to mint and verify the deployment-scoped token used to authenticate run controllers to the platform. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) ## @trigger.dev/python@4.5.6 ### Patch Changes - Updated dependencies: - `@trigger.dev/core@4.5.6` - `@trigger.dev/build@4.5.6` - `@trigger.dev/sdk@4.5.6` ## @trigger.dev/react-hooks@4.5.6 ### Patch Changes - Updated dependencies: - `@trigger.dev/core@4.5.6` ## @trigger.dev/redis-worker@4.5.6 ### Patch Changes - Updated dependencies: - `@trigger.dev/core@4.5.6` ## @trigger.dev/rsc@4.5.6 ### Patch Changes - Updated dependencies: - `@trigger.dev/core@4.5.6` ## @trigger.dev/schema-to-json@4.5.6 ### Patch Changes - Updated dependencies: - `@trigger.dev/core@4.5.6` ## @trigger.dev/sdk@4.5.6 ### Patch Changes - Updated dependencies: - `@trigger.dev/core@4.5.6`
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .changeset/block-metadata-proto-pollution.md | 5 ---- .changeset/cli-login-poll-window.md | 6 ----- .../worker-id-deployment-friendly-id.md | 5 ---- .changeset/workload-deployment-token.md | 5 ---- .server-changes/otlp-ingestion-rate-limit.md | 6 ----- .../require-webapp-auth-secrets.md | 6 ----- ...ope-deployment-background-worker-lookup.md | 6 ----- .../scope-github-app-update-to-org.md | 6 ----- .../scope-schedule-and-env-var-writes.md | 6 ----- .../secure-compute-snapshot-callbacks.md | 6 ----- .../session-out-stream-private-auth.md | 6 ----- .../tighten-realtime-and-trace-sync.md | 6 ----- .../tsql-window-function-allowlist.md | 6 ----- .server-changes/workload-token-supervisor.md | 6 ----- .server-changes/workload-token-webapp.md | 6 ----- hosting/k8s/helm/Chart.yaml | 4 ++-- packages/build/CHANGELOG.md | 7 ++++++ packages/build/package.json | 4 ++-- packages/cli-v3/CHANGELOG.md | 11 +++++++++ packages/cli-v3/package.json | 8 +++---- packages/core/CHANGELOG.md | 8 +++++++ packages/core/package.json | 2 +- packages/python/CHANGELOG.md | 9 +++++++ packages/python/package.json | 12 +++++----- packages/react-hooks/CHANGELOG.md | 7 ++++++ packages/react-hooks/package.json | 4 ++-- packages/redis-worker/CHANGELOG.md | 7 ++++++ packages/redis-worker/package.json | 4 ++-- packages/rsc/CHANGELOG.md | 7 ++++++ packages/rsc/package.json | 6 ++--- packages/schema-to-json/CHANGELOG.md | 7 ++++++ packages/schema-to-json/package.json | 2 +- packages/trigger-sdk/CHANGELOG.md | 7 ++++++ packages/trigger-sdk/package.json | 4 ++-- pnpm-lock.yaml | 24 +++++++++---------- 35 files changed, 107 insertions(+), 124 deletions(-) delete mode 100644 .changeset/block-metadata-proto-pollution.md delete mode 100644 .changeset/cli-login-poll-window.md delete mode 100644 .changeset/worker-id-deployment-friendly-id.md delete mode 100644 .changeset/workload-deployment-token.md delete mode 100644 .server-changes/otlp-ingestion-rate-limit.md delete mode 100644 .server-changes/require-webapp-auth-secrets.md delete mode 100644 .server-changes/scope-deployment-background-worker-lookup.md delete mode 100644 .server-changes/scope-github-app-update-to-org.md delete mode 100644 .server-changes/scope-schedule-and-env-var-writes.md delete mode 100644 .server-changes/secure-compute-snapshot-callbacks.md delete mode 100644 .server-changes/session-out-stream-private-auth.md delete mode 100644 .server-changes/tighten-realtime-and-trace-sync.md delete mode 100644 .server-changes/tsql-window-function-allowlist.md delete mode 100644 .server-changes/workload-token-supervisor.md delete mode 100644 .server-changes/workload-token-webapp.md diff --git a/.changeset/block-metadata-proto-pollution.md b/.changeset/block-metadata-proto-pollution.md deleted file mode 100644 index aa38beab4..000000000 --- a/.changeset/block-metadata-proto-pollution.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@trigger.dev/core": patch ---- - -Prevent prototype pollution when applying run metadata operations or reconstructing nested telemetry attributes, while preserving legitimate `constructor` and `prototype` fields. diff --git a/.changeset/cli-login-poll-window.md b/.changeset/cli-login-poll-window.md deleted file mode 100644 index e0453895a..000000000 --- a/.changeset/cli-login-poll-window.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -"@trigger.dev/core": patch -"trigger.dev": patch ---- - -Require explicit browser approval for CLI and MCP login, with resilient polling while approval is pending. diff --git a/.changeset/worker-id-deployment-friendly-id.md b/.changeset/worker-id-deployment-friendly-id.md deleted file mode 100644 index 2e62fee4b..000000000 --- a/.changeset/worker-id-deployment-friendly-id.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"trigger.dev": patch ---- - -Deployed task telemetry now reports the deployment identifier (e.g. `deployment_abc123`) in the `worker.id` attribute, instead of an opaque internal value. Upgrade to get the readable identifier in your own OpenTelemetry exporters. diff --git a/.changeset/workload-deployment-token.md b/.changeset/workload-deployment-token.md deleted file mode 100644 index 6a3c08cfc..000000000 --- a/.changeset/workload-deployment-token.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@trigger.dev/core": patch ---- - -Add helpers to mint and verify the deployment-scoped token used to authenticate run controllers to the platform. diff --git a/.server-changes/otlp-ingestion-rate-limit.md b/.server-changes/otlp-ingestion-rate-limit.md deleted file mode 100644 index acb9c6231..000000000 --- a/.server-changes/otlp-ingestion-rate-limit.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: improvement ---- - -Added optional request rate limiting for telemetry ingestion endpoints. diff --git a/.server-changes/require-webapp-auth-secrets.md b/.server-changes/require-webapp-auth-secrets.md deleted file mode 100644 index 52cb61d87..000000000 --- a/.server-changes/require-webapp-auth-secrets.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: breaking ---- - -Self-hosted deployments no longer ship shared default credentials; fresh installs generate their own. If yours still uses a previously published default, set a unique value before upgrading, or set `ALLOW_INSECURE_DEFAULT_SECRETS=true` to keep booting while you migrate. diff --git a/.server-changes/scope-deployment-background-worker-lookup.md b/.server-changes/scope-deployment-background-worker-lookup.md deleted file mode 100644 index 74c2aa7ee..000000000 --- a/.server-changes/scope-deployment-background-worker-lookup.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: fix ---- - -Background-worker deployment lookups are now scoped to the authenticated environment. diff --git a/.server-changes/scope-github-app-update-to-org.md b/.server-changes/scope-github-app-update-to-org.md deleted file mode 100644 index df417e52d..000000000 --- a/.server-changes/scope-github-app-update-to-org.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: fix ---- - -Updating a GitHub App installation from the callback flow is now scoped to your own organization, so an installation ID belonging to another organization can no longer be used to refresh that organization's installation record. The GitHub App installation session is also now single-use, so completing an installation callback invalidates its state and it can no longer be replayed. diff --git a/.server-changes/scope-schedule-and-env-var-writes.md b/.server-changes/scope-schedule-and-env-var-writes.md deleted file mode 100644 index 3ddcd6a1a..000000000 --- a/.server-changes/scope-schedule-and-env-var-writes.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: fix ---- - -Scope schedule and environment-variable writes to the caller's project and environment diff --git a/.server-changes/secure-compute-snapshot-callbacks.md b/.server-changes/secure-compute-snapshot-callbacks.md deleted file mode 100644 index 5f362f3fa..000000000 --- a/.server-changes/secure-compute-snapshot-callbacks.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: supervisor -type: fix ---- - -Reject compute snapshot callbacks that do not match the snapshot request that created them. diff --git a/.server-changes/session-out-stream-private-auth.md b/.server-changes/session-out-stream-private-auth.md deleted file mode 100644 index bfc95c662..000000000 --- a/.server-changes/session-out-stream-private-auth.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: fix ---- - -Require secret-key authentication to initialize the session out (agent→client) stream, matching the append route. diff --git a/.server-changes/tighten-realtime-and-trace-sync.md b/.server-changes/tighten-realtime-and-trace-sync.md deleted file mode 100644 index 51b4b2d41..000000000 --- a/.server-changes/tighten-realtime-and-trace-sync.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: fix ---- - -Live run and trace subscriptions now validate their identifiers more strictly and only return data from your own organization. diff --git a/.server-changes/tsql-window-function-allowlist.md b/.server-changes/tsql-window-function-allowlist.md deleted file mode 100644 index a56a881ae..000000000 --- a/.server-changes/tsql-window-function-allowlist.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: fix ---- - -Window-function names in the query compiler are now validated against the allowlist, matching how other function calls are handled. diff --git a/.server-changes/workload-token-supervisor.md b/.server-changes/workload-token-supervisor.md deleted file mode 100644 index b7c82c888..000000000 --- a/.server-changes/workload-token-supervisor.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: supervisor -type: fix ---- - -Authenticate run controllers to the platform with a signed, deployment-scoped token. diff --git a/.server-changes/workload-token-webapp.md b/.server-changes/workload-token-webapp.md deleted file mode 100644 index b743c4463..000000000 --- a/.server-changes/workload-token-webapp.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -area: webapp -type: fix ---- - -Verify that worker actions (starting, completing, and continuing a run, and reading its snapshots) target a run belonging to the caller's environment. diff --git a/hosting/k8s/helm/Chart.yaml b/hosting/k8s/helm/Chart.yaml index ed45726ab..899346acf 100644 --- a/hosting/k8s/helm/Chart.yaml +++ b/hosting/k8s/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: trigger description: The official Trigger.dev Helm chart type: application -version: 4.5.5 -appVersion: v4.5.5 +version: 4.5.6 +appVersion: v4.5.6 home: https://trigger.dev sources: - https://github.com/triggerdotdev/trigger.dev diff --git a/packages/build/CHANGELOG.md b/packages/build/CHANGELOG.md index 483ff9a80..def8d5fe7 100644 --- a/packages/build/CHANGELOG.md +++ b/packages/build/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/build +## 4.5.6 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.6` + ## 4.5.5 ### Patch Changes diff --git a/packages/build/package.json b/packages/build/package.json index d2e21643b..cdee00705 100644 --- a/packages/build/package.json +++ b/packages/build/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/build", - "version": "4.5.5", + "version": "4.5.6", "description": "trigger.dev build extensions", "license": "MIT", "publishConfig": { @@ -79,7 +79,7 @@ }, "dependencies": { "@prisma/config": "^6.10.0", - "@trigger.dev/core": "workspace:4.5.5", + "@trigger.dev/core": "workspace:4.5.6", "mlly": "^1.7.1", "pkg-types": "^1.1.3", "resolve": "^1.22.8", diff --git a/packages/cli-v3/CHANGELOG.md b/packages/cli-v3/CHANGELOG.md index f5ca164b1..6cb6c8b30 100644 --- a/packages/cli-v3/CHANGELOG.md +++ b/packages/cli-v3/CHANGELOG.md @@ -1,5 +1,16 @@ # trigger.dev +## 4.5.6 + +### Patch Changes + +- Require explicit browser approval for CLI and MCP login, with resilient polling while approval is pending. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) +- Deployed task telemetry now reports the deployment identifier (e.g. `deployment_abc123`) in the `worker.id` attribute, instead of an opaque internal value. Upgrade to get the readable identifier in your own OpenTelemetry exporters. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) +- Updated dependencies: + - `@trigger.dev/core@4.5.6` + - `@trigger.dev/build@4.5.6` + - `@trigger.dev/schema-to-json@4.5.6` + ## 4.5.5 ### Patch Changes diff --git a/packages/cli-v3/package.json b/packages/cli-v3/package.json index 1eb1fe2f1..0aa3b9aa2 100644 --- a/packages/cli-v3/package.json +++ b/packages/cli-v3/package.json @@ -1,6 +1,6 @@ { "name": "trigger.dev", - "version": "4.5.5", + "version": "4.5.6", "description": "A Command-Line Interface for Trigger.dev projects", "type": "module", "license": "MIT", @@ -96,9 +96,9 @@ "@opentelemetry/sdk-trace-node": "2.7.1", "@opentelemetry/semantic-conventions": "1.41.1", "@s2-dev/streamstore": "^0.22.10", - "@trigger.dev/build": "workspace:4.5.5", - "@trigger.dev/core": "workspace:4.5.5", - "@trigger.dev/schema-to-json": "workspace:4.5.5", + "@trigger.dev/build": "workspace:4.5.6", + "@trigger.dev/core": "workspace:4.5.6", + "@trigger.dev/schema-to-json": "workspace:4.5.6", "ansi-escapes": "^7.0.0", "braces": "^3.0.3", "c12": "^1.11.1", diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index 1c54f5cac..0696d5123 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -1,5 +1,13 @@ # internal-platform +## 4.5.6 + +### Patch Changes + +- Prevent prototype pollution when applying run metadata operations or reconstructing nested telemetry attributes, while preserving legitimate `constructor` and `prototype` fields. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) +- Require explicit browser approval for CLI and MCP login, with resilient polling while approval is pending. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) +- Add helpers to mint and verify the deployment-scoped token used to authenticate run controllers to the platform. ([#4316](https://github.com/triggerdotdev/trigger.dev/pull/4316)) + ## 4.5.5 ### Patch Changes diff --git a/packages/core/package.json b/packages/core/package.json index 93758fa95..87292e0dc 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/core", - "version": "4.5.5", + "version": "4.5.6", "description": "Core code used across the Trigger.dev SDK and platform", "license": "MIT", "publishConfig": { diff --git a/packages/python/CHANGELOG.md b/packages/python/CHANGELOG.md index febf81e99..48ca36c0c 100644 --- a/packages/python/CHANGELOG.md +++ b/packages/python/CHANGELOG.md @@ -1,5 +1,14 @@ # @trigger.dev/python +## 4.5.6 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.6` + - `@trigger.dev/build@4.5.6` + - `@trigger.dev/sdk@4.5.6` + ## 4.5.5 ### Patch Changes diff --git a/packages/python/package.json b/packages/python/package.json index d5ea3971b..f230c8289 100644 --- a/packages/python/package.json +++ b/packages/python/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/python", - "version": "4.5.5", + "version": "4.5.6", "description": "Python runtime and build extension for Trigger.dev", "license": "MIT", "publishConfig": { @@ -45,7 +45,7 @@ "check-exports": "attw --pack ." }, "dependencies": { - "@trigger.dev/core": "workspace:4.5.5", + "@trigger.dev/core": "workspace:4.5.6", "tinyexec": "^0.3.2" }, "devDependencies": { @@ -56,12 +56,12 @@ "tsx": "4.17.0", "esbuild": "^0.23.0", "@arethetypeswrong/cli": "^0.15.4", - "@trigger.dev/build": "workspace:4.5.5", - "@trigger.dev/sdk": "workspace:4.5.5" + "@trigger.dev/build": "workspace:4.5.6", + "@trigger.dev/sdk": "workspace:4.5.6" }, "peerDependencies": { - "@trigger.dev/sdk": "workspace:^4.5.5", - "@trigger.dev/build": "workspace:^4.5.5" + "@trigger.dev/sdk": "workspace:^4.5.6", + "@trigger.dev/build": "workspace:^4.5.6" }, "engines": { "node": ">=18.20.0" diff --git a/packages/react-hooks/CHANGELOG.md b/packages/react-hooks/CHANGELOG.md index c698067ab..dc99b3b45 100644 --- a/packages/react-hooks/CHANGELOG.md +++ b/packages/react-hooks/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/react-hooks +## 4.5.6 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.6` + ## 4.5.5 ### Patch Changes diff --git a/packages/react-hooks/package.json b/packages/react-hooks/package.json index 347f0e2c6..3702022c4 100644 --- a/packages/react-hooks/package.json +++ b/packages/react-hooks/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/react-hooks", - "version": "4.5.5", + "version": "4.5.6", "description": "trigger.dev react hooks", "license": "MIT", "publishConfig": { @@ -37,7 +37,7 @@ "check-exports": "attw --pack ." }, "dependencies": { - "@trigger.dev/core": "workspace:^4.5.5", + "@trigger.dev/core": "workspace:^4.5.6", "swr": "^2.2.5" }, "devDependencies": { diff --git a/packages/redis-worker/CHANGELOG.md b/packages/redis-worker/CHANGELOG.md index 471e96c31..db6f152a2 100644 --- a/packages/redis-worker/CHANGELOG.md +++ b/packages/redis-worker/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/redis-worker +## 4.5.6 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.6` + ## 4.5.5 ### Patch Changes diff --git a/packages/redis-worker/package.json b/packages/redis-worker/package.json index 43fa77072..ed7c16d3c 100644 --- a/packages/redis-worker/package.json +++ b/packages/redis-worker/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/redis-worker", - "version": "4.5.5", + "version": "4.5.6", "description": "Redis worker for trigger.dev", "license": "MIT", "publishConfig": { @@ -23,7 +23,7 @@ "test": "vitest --sequence.concurrent=false --no-file-parallelism" }, "dependencies": { - "@trigger.dev/core": "workspace:4.5.5", + "@trigger.dev/core": "workspace:4.5.6", "lodash.omit": "^4.5.0", "nanoid": "^5.0.7", "p-limit": "^6.2.0", diff --git a/packages/rsc/CHANGELOG.md b/packages/rsc/CHANGELOG.md index acd0e1af0..17bf179f3 100644 --- a/packages/rsc/CHANGELOG.md +++ b/packages/rsc/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/rsc +## 4.5.6 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.6` + ## 4.5.5 ### Patch Changes diff --git a/packages/rsc/package.json b/packages/rsc/package.json index 7a5d3c4d2..b52c5408d 100644 --- a/packages/rsc/package.json +++ b/packages/rsc/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/rsc", - "version": "4.5.5", + "version": "4.5.6", "description": "trigger.dev rsc", "license": "MIT", "publishConfig": { @@ -37,14 +37,14 @@ "check-exports": "attw --pack ." }, "dependencies": { - "@trigger.dev/core": "workspace:^4.5.5", + "@trigger.dev/core": "workspace:^4.5.6", "mlly": "^1.7.1", "react": "19.0.0-rc.1", "react-dom": "19.0.0-rc.1" }, "devDependencies": { "@arethetypeswrong/cli": "^0.15.4", - "@trigger.dev/build": "workspace:^4.5.5", + "@trigger.dev/build": "workspace:^4.5.6", "@types/node": "^24.13.3", "@types/react": "*", "@types/react-dom": "*", diff --git a/packages/schema-to-json/CHANGELOG.md b/packages/schema-to-json/CHANGELOG.md index 03b024c52..68fbd531a 100644 --- a/packages/schema-to-json/CHANGELOG.md +++ b/packages/schema-to-json/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/schema-to-json +## 4.5.6 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.6` + ## 4.5.5 ### Patch Changes diff --git a/packages/schema-to-json/package.json b/packages/schema-to-json/package.json index d9daf0ca0..fa00d69ac 100644 --- a/packages/schema-to-json/package.json +++ b/packages/schema-to-json/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/schema-to-json", - "version": "4.5.5", + "version": "4.5.6", "description": "Convert various schema validation libraries to JSON Schema", "license": "MIT", "publishConfig": { diff --git a/packages/trigger-sdk/CHANGELOG.md b/packages/trigger-sdk/CHANGELOG.md index 4697678b2..6a4ebef26 100644 --- a/packages/trigger-sdk/CHANGELOG.md +++ b/packages/trigger-sdk/CHANGELOG.md @@ -1,5 +1,12 @@ # @trigger.dev/sdk +## 4.5.6 + +### Patch Changes + +- Updated dependencies: + - `@trigger.dev/core@4.5.6` + ## 4.5.5 ### Patch Changes diff --git a/packages/trigger-sdk/package.json b/packages/trigger-sdk/package.json index daf6e0633..051c955e6 100644 --- a/packages/trigger-sdk/package.json +++ b/packages/trigger-sdk/package.json @@ -1,6 +1,6 @@ { "name": "@trigger.dev/sdk", - "version": "4.5.5", + "version": "4.5.6", "description": "trigger.dev Node.JS SDK", "license": "MIT", "publishConfig": { @@ -77,7 +77,7 @@ "dependencies": { "@opentelemetry/api": "1.9.1", "@opentelemetry/semantic-conventions": "1.41.1", - "@trigger.dev/core": "workspace:4.5.5", + "@trigger.dev/core": "workspace:4.5.6", "chalk": "^5.2.0", "cronstrue": "^2.21.0", "debug": "^4.3.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 29450d4b1..9f1242f68 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1402,7 +1402,7 @@ importers: specifier: ^6.10.0 version: 6.19.0(magicast@0.3.5) '@trigger.dev/core': - specifier: workspace:4.5.5 + specifier: workspace:4.5.6 version: link:../core mlly: specifier: ^1.7.1 @@ -1478,13 +1478,13 @@ importers: specifier: ^0.22.10 version: 0.22.10(supports-color@10.0.0) '@trigger.dev/build': - specifier: workspace:4.5.5 + specifier: workspace:4.5.6 version: link:../build '@trigger.dev/core': - specifier: workspace:4.5.5 + specifier: workspace:4.5.6 version: link:../core '@trigger.dev/schema-to-json': - specifier: workspace:4.5.5 + specifier: workspace:4.5.6 version: link:../schema-to-json ansi-escapes: specifier: ^7.0.0 @@ -1871,7 +1871,7 @@ importers: packages/python: dependencies: '@trigger.dev/core': - specifier: workspace:4.5.5 + specifier: workspace:4.5.6 version: link:../core tinyexec: specifier: ^0.3.2 @@ -1881,10 +1881,10 @@ importers: specifier: ^0.15.4 version: 0.15.4 '@trigger.dev/build': - specifier: workspace:4.5.5 + specifier: workspace:4.5.6 version: link:../build '@trigger.dev/sdk': - specifier: workspace:4.5.5 + specifier: workspace:4.5.6 version: link:../trigger-sdk '@types/node': specifier: 24.13.3 @@ -1908,7 +1908,7 @@ importers: packages/react-hooks: dependencies: '@trigger.dev/core': - specifier: workspace:^4.5.5 + specifier: workspace:^4.5.6 version: link:../core react: specifier: 18.3.1 @@ -1942,7 +1942,7 @@ importers: packages/redis-worker: dependencies: '@trigger.dev/core': - specifier: workspace:4.5.5 + specifier: workspace:4.5.6 version: link:../core cron-parser: specifier: ^4.9.0 @@ -1991,7 +1991,7 @@ importers: packages/rsc: dependencies: '@trigger.dev/core': - specifier: workspace:^4.5.5 + specifier: workspace:^4.5.6 version: link:../core mlly: specifier: ^1.7.1 @@ -2007,7 +2007,7 @@ importers: specifier: ^0.15.4 version: 0.15.4 '@trigger.dev/build': - specifier: workspace:^4.5.5 + specifier: workspace:^4.5.6 version: link:../build '@types/node': specifier: 24.13.3 @@ -2086,7 +2086,7 @@ importers: specifier: 1.41.1 version: 1.41.1 '@trigger.dev/core': - specifier: workspace:4.5.5 + specifier: workspace:4.5.6 version: link:../core chalk: specifier: ^5.2.0 From cbec61309a4d9b4494a76c689059c469cdd712dd Mon Sep 17 00:00:00 2001 From: DKP <8297864+D-K-P@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:48:05 +0100 Subject: [PATCH 009/300] fix(webapp): fix promo page heading typography (#4311) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What The `/promo` page heading rendered with overlapping lines — the two lines of "Promo codes are for new accounts" collided. ## Why The page used `Header2` stretched to display sizes (`sm:text-2xl md:text-3xl lg:text-4xl`), but `Header2` bakes in a fixed `leading-6` (24px). A 36px font in a 24px line box makes wrapped lines overlap. It only showed at `sm`+ widths and only on headings that wrap to 2+ lines, which is why it slipped through — the short single-line headings on the same page looked fine. ## Fix Switch both headings to `Header1` — the page-title primitive the sibling login pages (`login._index`, `login.magic`) already use for exactly this size. Add `leading-tight` (relative line-height, scales with font size, and this heading uniquely wraps to two lines) and `pb-4` to match the login pages' spacing convention. ## Testing Manually verified the signed-in view (`/promo` while logged in) renders as two clean, non-overlapping lines across breakpoints. Pure CSS/layout change — no automated test. --- apps/webapp/app/routes/promo.tsx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/webapp/app/routes/promo.tsx b/apps/webapp/app/routes/promo.tsx index 1ae47f573..cd8b8325c 100644 --- a/apps/webapp/app/routes/promo.tsx +++ b/apps/webapp/app/routes/promo.tsx @@ -8,7 +8,7 @@ import { LoginPageLayout } from "~/components/LoginPageLayout"; import { Button, LinkButton } from "~/components/primitives/Buttons"; import { Callout } from "~/components/primitives/Callout"; import { Fieldset } from "~/components/primitives/Fieldset"; -import { Header2 } from "~/components/primitives/Headers"; +import { Header1 } from "~/components/primitives/Headers"; import { Paragraph } from "~/components/primitives/Paragraph"; import { TextLink } from "~/components/primitives/TextLink"; import { isGithubAuthSupported, isGoogleAuthSupported } from "~/services/auth.server"; @@ -139,9 +139,9 @@ export default function PromoPage() {
{data.view === "signed_in" ? ( <> - + Promo codes are for new accounts - + You're already signed in. Promo credits can only be added to a brand-new account. @@ -151,11 +151,11 @@ export default function PromoPage() { ) : ( <> - + {data.view === "valid" ? `Claim ${formatDollars(data.amountInCents)} credits` : "Create your account"} - + {data.view === "valid" ? ( These are only available for new accounts on the Free plan. From dc87b884e71bd4168af1e05d9a28453edce3b637 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Tue, 21 Jul 2026 13:57:52 +0100 Subject: [PATCH 010/300] chore: upgrade to typescript 6 (#4310) ## Summary Upgrades the workspace to TypeScript 6.0.3 and applies the compiler, type, and build configuration changes required to preserve package layouts and existing runtime behavior, apart from correcting the HTTP status field used for deployment connection errors. ## Compatibility - Centralizes TypeScript 6.0.3 through the pnpm workspace catalog. - Replaces compiler options and module resolution modes that TypeScript 6 no longer accepts. - Restores explicit Node types where TypeScript 6 no longer includes them transitively. - Adds explicit declaration build roots that preserve each package's existing output layout. - Patches tsup to stop injecting the removed `baseUrl` option during declaration builds. - Uses type-only assertions for stricter typed-array and stream definitions without changing runtime behavior. - Reads the EventSource v3 HTTP status from `code`, so deployment connection errors include it correctly. - Keeps standalone CLI compatibility fixtures pinned to their existing TypeScript version and lockfiles. `turbo run typecheck` and the complete PR test suite are green. --- .configs/tsconfig.base.json | 1 - apps/webapp/app/routes/otel.v1.logs.ts | 13 +- apps/webapp/app/routes/otel.v1.metrics.ts | 13 +- apps/webapp/app/routes/otel.v1.traces.ts | 13 +- .../realtime/redisRealtimeStreams.server.ts | 4 +- apps/webapp/tsconfig.check.json | 2 +- apps/webapp/tsconfig.json | 1 - internal-packages/cache/tsconfig.json | 2 +- .../clickhouse/tsconfig.build.json | 2 + .../clickhouse/tsconfig.test.json | 2 +- .../dashboard-agent-db/tsconfig.json | 3 +- internal-packages/database/package.json | 2 +- .../database/tsconfig.build.json | 12 + internal-packages/database/tsconfig.json | 5 +- internal-packages/emails/tsconfig.json | 4 +- internal-packages/otlp-importer/tsconfig.json | 3 +- internal-packages/rbac/package.json | 2 +- internal-packages/rbac/tsconfig.build.json | 12 + internal-packages/rbac/tsconfig.json | 2 +- internal-packages/redis/tsconfig.json | 2 +- .../replication/tsconfig.build.json | 2 + .../replication/tsconfig.test.json | 2 +- .../run-engine/tsconfig.build.json | 2 + .../run-engine/tsconfig.test.json | 2 +- .../run-ops-database/package.json | 2 +- .../run-ops-database/tsconfig.build.json | 12 + .../run-ops-database/tsconfig.json | 3 +- .../run-store/tsconfig.build.json | 2 + .../run-store/tsconfig.test.json | 2 +- .../schedule-engine/tsconfig.build.json | 2 + .../schedule-engine/tsconfig.test.json | 2 +- .../sdk-compat-tests/package.json | 2 +- .../src/fixtures/typescript/tsconfig.json | 3 +- internal-packages/sso/package.json | 2 +- internal-packages/sso/tsconfig.build.json | 12 + internal-packages/sso/tsconfig.json | 2 +- .../testcontainers/tsconfig.json | 6 +- internal-packages/tracing/tsconfig.json | 2 +- internal-packages/tsql/tsconfig.json | 7 +- package.json | 5 +- packages/build/package.json | 2 +- packages/build/tsconfig.src.json | 3 +- packages/cli-v3/src/apiClient.ts | 4 +- packages/cli-v3/tsconfig.e2e.test.json | 2 +- packages/cli-v3/tsconfig.test.json | 2 +- packages/core/package.json | 2 +- packages/core/src/v3/isomorphic/friendlyId.ts | 2 +- packages/core/tsconfig.test.json | 2 +- packages/plugins/package.json | 2 +- packages/python/package.json | 4 +- packages/python/tsconfig.src.json | 3 +- packages/react-hooks/package.json | 2 +- packages/react-hooks/tsconfig.json | 3 +- packages/redis-worker/tsconfig.test.json | 2 +- packages/rsc/package.json | 2 +- packages/rsc/tsconfig.json | 3 +- packages/schema-to-json/package.json | 2 +- packages/schema-to-json/tsconfig.src.json | 3 +- packages/schema-to-json/tsconfig.test.json | 2 +- packages/trigger-sdk/package.json | 2 +- patches/tsup@8.4.0.patch | 12 + pnpm-lock.yaml | 360 +++++++++--------- pnpm-workspace.yaml | 3 + 63 files changed, 356 insertions(+), 242 deletions(-) create mode 100644 internal-packages/database/tsconfig.build.json create mode 100644 internal-packages/rbac/tsconfig.build.json create mode 100644 internal-packages/run-ops-database/tsconfig.build.json create mode 100644 internal-packages/sso/tsconfig.build.json create mode 100644 patches/tsup@8.4.0.patch diff --git a/.configs/tsconfig.base.json b/.configs/tsconfig.base.json index 2d560d22d..891021047 100644 --- a/.configs/tsconfig.base.json +++ b/.configs/tsconfig.base.json @@ -26,7 +26,6 @@ "esModuleInterop": true, "emitDecoratorMetadata": false, "experimentalDecorators": false, - "downlevelIteration": true, "isolatedModules": true, "noUncheckedIndexedAccess": true, diff --git a/apps/webapp/app/routes/otel.v1.logs.ts b/apps/webapp/app/routes/otel.v1.logs.ts index 8b177f524..f73b99a64 100644 --- a/apps/webapp/app/routes/otel.v1.logs.ts +++ b/apps/webapp/app/routes/otel.v1.logs.ts @@ -21,7 +21,9 @@ export async function action({ request }: ActionFunctionArgs) { await exporter.exportLogsRaw(new Uint8Array(buffer)); return new Response( - ExportLogsServiceResponse.encode(ExportLogsServiceResponse.create()).finish(), + ExportLogsServiceResponse.encode( + ExportLogsServiceResponse.create() + ).finish() as Uint8Array, { status: 200 } ); } @@ -30,9 +32,12 @@ export async function action({ request }: ActionFunctionArgs) { const exportResponse = await exporter.exportLogs(exportRequest); - return new Response(ExportLogsServiceResponse.encode(exportResponse).finish(), { - status: 200, - }); + return new Response( + ExportLogsServiceResponse.encode(exportResponse).finish() as Uint8Array, + { + status: 200, + } + ); } else { return new Response( "Unsupported content type. Must be either application/x-protobuf or application/json", diff --git a/apps/webapp/app/routes/otel.v1.metrics.ts b/apps/webapp/app/routes/otel.v1.metrics.ts index 8b3964956..5338bebd0 100644 --- a/apps/webapp/app/routes/otel.v1.metrics.ts +++ b/apps/webapp/app/routes/otel.v1.metrics.ts @@ -25,7 +25,9 @@ export async function action({ request }: ActionFunctionArgs) { await exporter.exportMetricsRaw(new Uint8Array(buffer)); return new Response( - ExportMetricsServiceResponse.encode(ExportMetricsServiceResponse.create()).finish(), + ExportMetricsServiceResponse.encode( + ExportMetricsServiceResponse.create() + ).finish() as Uint8Array, { status: 200 } ); } @@ -34,9 +36,12 @@ export async function action({ request }: ActionFunctionArgs) { const exportResponse = await exporter.exportMetrics(exportRequest); - return new Response(ExportMetricsServiceResponse.encode(exportResponse).finish(), { - status: 200, - }); + return new Response( + ExportMetricsServiceResponse.encode(exportResponse).finish() as Uint8Array, + { + status: 200, + } + ); } else { return new Response( "Unsupported content type. Must be either application/x-protobuf or application/json", diff --git a/apps/webapp/app/routes/otel.v1.traces.ts b/apps/webapp/app/routes/otel.v1.traces.ts index 313077ac0..bd99ccb8b 100644 --- a/apps/webapp/app/routes/otel.v1.traces.ts +++ b/apps/webapp/app/routes/otel.v1.traces.ts @@ -21,7 +21,9 @@ export async function action({ request }: ActionFunctionArgs) { await exporter.exportTracesRaw(new Uint8Array(buffer)); return new Response( - ExportTraceServiceResponse.encode(ExportTraceServiceResponse.create()).finish(), + ExportTraceServiceResponse.encode( + ExportTraceServiceResponse.create() + ).finish() as Uint8Array, { status: 200 } ); } @@ -30,9 +32,12 @@ export async function action({ request }: ActionFunctionArgs) { const exportResponse = await exporter.exportTraces(exportRequest); - return new Response(ExportTraceServiceResponse.encode(exportResponse).finish(), { - status: 200, - }); + return new Response( + ExportTraceServiceResponse.encode(exportResponse).finish() as Uint8Array, + { + status: 200, + } + ); } else { return new Response( "Unsupported content type. Must be either application/x-protobuf or application/json", diff --git a/apps/webapp/app/services/realtime/redisRealtimeStreams.server.ts b/apps/webapp/app/services/realtime/redisRealtimeStreams.server.ts index 1952c59af..085ac8e03 100644 --- a/apps/webapp/app/services/realtime/redisRealtimeStreams.server.ts +++ b/apps/webapp/app/services/realtime/redisRealtimeStreams.server.ts @@ -321,7 +321,9 @@ export class RedisRealtimeStreams implements StreamIngestor, StreamResponder { let currentChunkIndex = startChunk; try { - const textStream = stream.pipeThrough(new TextDecoderStream()); + const textStream = stream.pipeThrough( + new TextDecoderStream() as ReadableWritablePair + ); const reader = textStream.getReader(); while (true) { diff --git a/apps/webapp/tsconfig.check.json b/apps/webapp/tsconfig.check.json index cb73affed..332645b8f 100644 --- a/apps/webapp/tsconfig.check.json +++ b/apps/webapp/tsconfig.check.json @@ -10,5 +10,5 @@ }, "customConditions": [] }, - "exclude": ["**/*.test.ts", "**/*.test.tsx"] + "exclude": ["**/*.test.ts", "**/*.test.tsx", "test/setup-test-env.ts"] } diff --git a/apps/webapp/tsconfig.json b/apps/webapp/tsconfig.json index 36944e395..ee4123287 100644 --- a/apps/webapp/tsconfig.json +++ b/apps/webapp/tsconfig.json @@ -17,7 +17,6 @@ "skipLibCheck": true, "experimentalDecorators": true, "emitDecoratorMetadata": true, - "baseUrl": ".", "paths": { "~/*": ["./app/*"], "@/*": ["./*"] diff --git a/internal-packages/cache/tsconfig.json b/internal-packages/cache/tsconfig.json index 010433962..a826c8f67 100644 --- a/internal-packages/cache/tsconfig.json +++ b/internal-packages/cache/tsconfig.json @@ -6,7 +6,7 @@ "moduleResolution": "Node16", "moduleDetection": "force", "verbatimModuleSyntax": false, - "types": ["vitest/globals"], + "types": ["vitest/globals", "node"], "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "isolatedModules": true, diff --git a/internal-packages/clickhouse/tsconfig.build.json b/internal-packages/clickhouse/tsconfig.build.json index 6a71c854e..8dfd96dfa 100644 --- a/internal-packages/clickhouse/tsconfig.build.json +++ b/internal-packages/clickhouse/tsconfig.build.json @@ -2,6 +2,8 @@ "include": ["src/**/*.ts"], "exclude": ["src/**/*.test.ts"], "compilerOptions": { + "types": ["node"], + "rootDir": ".", "composite": true, "target": "ES2019", "lib": ["ES2019", "DOM", "DOM.Iterable", "DOM.AsyncIterable"], diff --git a/internal-packages/clickhouse/tsconfig.test.json b/internal-packages/clickhouse/tsconfig.test.json index 99db8eb7c..dbf1dfec3 100644 --- a/internal-packages/clickhouse/tsconfig.test.json +++ b/internal-packages/clickhouse/tsconfig.test.json @@ -9,7 +9,7 @@ "moduleResolution": "Node16", "moduleDetection": "force", "verbatimModuleSyntax": false, - "types": ["vitest/globals"], + "types": ["vitest/globals", "node"], "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "isolatedModules": true, diff --git a/internal-packages/dashboard-agent-db/tsconfig.json b/internal-packages/dashboard-agent-db/tsconfig.json index 858f299a6..5d15bfc0e 100644 --- a/internal-packages/dashboard-agent-db/tsconfig.json +++ b/internal-packages/dashboard-agent-db/tsconfig.json @@ -12,7 +12,8 @@ "preserveWatchOutput": true, "skipLibCheck": true, "noEmit": true, - "strict": true + "strict": true, + "types": ["node"] }, "exclude": ["node_modules", "drizzle"] } diff --git a/internal-packages/database/package.json b/internal-packages/database/package.json index 3a396e8a8..d2fc05131 100644 --- a/internal-packages/database/package.json +++ b/internal-packages/database/package.json @@ -23,7 +23,7 @@ "db:studio": "prisma studio", "db:reset": "prisma migrate reset", "typecheck": "tsc --noEmit", - "build": "pnpm run clean && tsc --noEmit false --outDir dist --declaration", + "build": "pnpm run clean && tsc -p tsconfig.build.json", "dev": "tsc --noEmit false --outDir dist --declaration --watch" } } diff --git a/internal-packages/database/tsconfig.build.json b/internal-packages/database/tsconfig.build.json new file mode 100644 index 000000000..37a9fb0a9 --- /dev/null +++ b/internal-packages/database/tsconfig.build.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.test.ts"], + "compilerOptions": { + "noEmit": false, + "declaration": true, + "outDir": "dist", + "rootDir": "src", + "types": ["node"] + } +} diff --git a/internal-packages/database/tsconfig.json b/internal-packages/database/tsconfig.json index 67f916782..8851e8705 100644 --- a/internal-packages/database/tsconfig.json +++ b/internal-packages/database/tsconfig.json @@ -3,11 +3,12 @@ "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "isolatedModules": true, - "moduleResolution": "node", + "module": "nodenext", + "moduleResolution": "nodenext", "preserveWatchOutput": true, "skipLibCheck": true, "noEmit": true, "strict": true }, - "exclude": ["node_modules"] + "exclude": ["node_modules", "dist"] } diff --git a/internal-packages/emails/tsconfig.json b/internal-packages/emails/tsconfig.json index 6d7e4a1b4..fec5444a2 100644 --- a/internal-packages/emails/tsconfig.json +++ b/internal-packages/emails/tsconfig.json @@ -8,8 +8,8 @@ "noEmit": true, "incremental": true, "esModuleInterop": true, - "module": "esnext", - "moduleResolution": "node", + "module": "nodenext", + "moduleResolution": "nodenext", "resolveJsonModule": true, "isolatedModules": true, "jsx": "preserve", diff --git a/internal-packages/otlp-importer/tsconfig.json b/internal-packages/otlp-importer/tsconfig.json index ec09e52a4..b721b8cee 100644 --- a/internal-packages/otlp-importer/tsconfig.json +++ b/internal-packages/otlp-importer/tsconfig.json @@ -4,7 +4,8 @@ "isolatedDeclarations": false, "composite": true, "sourceMap": true, - "stripInternal": true + "stripInternal": true, + "types": ["node"] }, "include": ["./src/**/*.ts"] } diff --git a/internal-packages/rbac/package.json b/internal-packages/rbac/package.json index 41d18ebf8..53374670f 100644 --- a/internal-packages/rbac/package.json +++ b/internal-packages/rbac/package.json @@ -16,7 +16,7 @@ "scripts": { "clean": "rimraf dist", "typecheck": "tsc --noEmit", - "build": "pnpm run clean && tsc --noEmit false --outDir dist --declaration", + "build": "pnpm run clean && tsc -p tsconfig.build.json", "dev": "tsc --noEmit false --outDir dist --declaration --watch", "test": "vitest run", "test:watch": "vitest" diff --git a/internal-packages/rbac/tsconfig.build.json b/internal-packages/rbac/tsconfig.build.json new file mode 100644 index 000000000..1f858270c --- /dev/null +++ b/internal-packages/rbac/tsconfig.build.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.test.ts"], + "compilerOptions": { + "noEmit": false, + "declaration": true, + "outDir": "dist", + "rootDir": ".", + "types": ["node"] + } +} diff --git a/internal-packages/rbac/tsconfig.json b/internal-packages/rbac/tsconfig.json index 8da0857b4..9ab7d8135 100644 --- a/internal-packages/rbac/tsconfig.json +++ b/internal-packages/rbac/tsconfig.json @@ -13,5 +13,5 @@ "strict": true, "customConditions": ["@triggerdotdev/source"] }, - "exclude": ["node_modules"] + "exclude": ["node_modules", "dist"] } diff --git a/internal-packages/redis/tsconfig.json b/internal-packages/redis/tsconfig.json index 010433962..a826c8f67 100644 --- a/internal-packages/redis/tsconfig.json +++ b/internal-packages/redis/tsconfig.json @@ -6,7 +6,7 @@ "moduleResolution": "Node16", "moduleDetection": "force", "verbatimModuleSyntax": false, - "types": ["vitest/globals"], + "types": ["vitest/globals", "node"], "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "isolatedModules": true, diff --git a/internal-packages/replication/tsconfig.build.json b/internal-packages/replication/tsconfig.build.json index 89c87a3dc..845200814 100644 --- a/internal-packages/replication/tsconfig.build.json +++ b/internal-packages/replication/tsconfig.build.json @@ -2,6 +2,8 @@ "include": ["src/**/*.ts"], "exclude": ["src/**/*.test.ts"], "compilerOptions": { + "types": ["node"], + "rootDir": ".", "composite": true, "target": "ES2020", "lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"], diff --git a/internal-packages/replication/tsconfig.test.json b/internal-packages/replication/tsconfig.test.json index 37b885fc8..ab2640edc 100644 --- a/internal-packages/replication/tsconfig.test.json +++ b/internal-packages/replication/tsconfig.test.json @@ -9,7 +9,7 @@ "moduleResolution": "Node16", "moduleDetection": "force", "verbatimModuleSyntax": false, - "types": ["vitest/globals"], + "types": ["vitest/globals", "node"], "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "isolatedModules": true, diff --git a/internal-packages/run-engine/tsconfig.build.json b/internal-packages/run-engine/tsconfig.build.json index e5327e934..558055464 100644 --- a/internal-packages/run-engine/tsconfig.build.json +++ b/internal-packages/run-engine/tsconfig.build.json @@ -2,6 +2,8 @@ "include": ["src/**/*.ts"], "exclude": ["src/**/*.test.ts", "src/engine/tests/utils/*.ts"], "compilerOptions": { + "types": ["node"], + "rootDir": ".", "composite": true, "target": "ES2020", "lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"], diff --git a/internal-packages/run-engine/tsconfig.test.json b/internal-packages/run-engine/tsconfig.test.json index d33ed305b..24e7a661d 100644 --- a/internal-packages/run-engine/tsconfig.test.json +++ b/internal-packages/run-engine/tsconfig.test.json @@ -9,7 +9,7 @@ "moduleResolution": "Node16", "moduleDetection": "force", "verbatimModuleSyntax": false, - "types": ["vitest/globals"], + "types": ["vitest/globals", "node"], "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "isolatedModules": true, diff --git a/internal-packages/run-ops-database/package.json b/internal-packages/run-ops-database/package.json index 841d908eb..fef3bec52 100644 --- a/internal-packages/run-ops-database/package.json +++ b/internal-packages/run-ops-database/package.json @@ -33,7 +33,7 @@ "db:push": "prisma db push", "test": "vitest run", "typecheck": "tsc --noEmit", - "build": "pnpm run clean && tsc --noEmit false --outDir dist --declaration", + "build": "pnpm run clean && tsc -p tsconfig.build.json", "dev": "tsc --noEmit false --outDir dist --declaration --watch", "db:studio": "prisma studio", "db:reset": "prisma migrate reset" diff --git a/internal-packages/run-ops-database/tsconfig.build.json b/internal-packages/run-ops-database/tsconfig.build.json new file mode 100644 index 000000000..37a9fb0a9 --- /dev/null +++ b/internal-packages/run-ops-database/tsconfig.build.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.test.ts"], + "compilerOptions": { + "noEmit": false, + "declaration": true, + "outDir": "dist", + "rootDir": "src", + "types": ["node"] + } +} diff --git a/internal-packages/run-ops-database/tsconfig.json b/internal-packages/run-ops-database/tsconfig.json index 8b889b0db..c431a1e69 100644 --- a/internal-packages/run-ops-database/tsconfig.json +++ b/internal-packages/run-ops-database/tsconfig.json @@ -3,7 +3,8 @@ "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "isolatedModules": true, - "moduleResolution": "node", + "module": "nodenext", + "moduleResolution": "nodenext", "preserveWatchOutput": true, "skipLibCheck": true, "noEmit": true, diff --git a/internal-packages/run-store/tsconfig.build.json b/internal-packages/run-store/tsconfig.build.json index 89c87a3dc..845200814 100644 --- a/internal-packages/run-store/tsconfig.build.json +++ b/internal-packages/run-store/tsconfig.build.json @@ -2,6 +2,8 @@ "include": ["src/**/*.ts"], "exclude": ["src/**/*.test.ts"], "compilerOptions": { + "types": ["node"], + "rootDir": ".", "composite": true, "target": "ES2020", "lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"], diff --git a/internal-packages/run-store/tsconfig.test.json b/internal-packages/run-store/tsconfig.test.json index 4c06c9f57..0d70db3e3 100644 --- a/internal-packages/run-store/tsconfig.test.json +++ b/internal-packages/run-store/tsconfig.test.json @@ -9,7 +9,7 @@ "moduleResolution": "Node16", "moduleDetection": "force", "verbatimModuleSyntax": false, - "types": ["vitest/globals"], + "types": ["vitest/globals", "node"], "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "isolatedModules": true, diff --git a/internal-packages/schedule-engine/tsconfig.build.json b/internal-packages/schedule-engine/tsconfig.build.json index 89c87a3dc..845200814 100644 --- a/internal-packages/schedule-engine/tsconfig.build.json +++ b/internal-packages/schedule-engine/tsconfig.build.json @@ -2,6 +2,8 @@ "include": ["src/**/*.ts"], "exclude": ["src/**/*.test.ts"], "compilerOptions": { + "types": ["node"], + "rootDir": ".", "composite": true, "target": "ES2020", "lib": ["ES2020", "DOM", "DOM.Iterable", "DOM.AsyncIterable"], diff --git a/internal-packages/schedule-engine/tsconfig.test.json b/internal-packages/schedule-engine/tsconfig.test.json index 4c06c9f57..0d70db3e3 100644 --- a/internal-packages/schedule-engine/tsconfig.test.json +++ b/internal-packages/schedule-engine/tsconfig.test.json @@ -9,7 +9,7 @@ "moduleResolution": "Node16", "moduleDetection": "force", "verbatimModuleSyntax": false, - "types": ["vitest/globals"], + "types": ["vitest/globals", "node"], "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "isolatedModules": true, diff --git a/internal-packages/sdk-compat-tests/package.json b/internal-packages/sdk-compat-tests/package.json index 445569c95..568f3ad77 100644 --- a/internal-packages/sdk-compat-tests/package.json +++ b/internal-packages/sdk-compat-tests/package.json @@ -14,7 +14,7 @@ "devDependencies": { "esbuild": "^0.24.0", "execa": "^9.3.0", - "typescript": "^5.5.0", + "typescript": "catalog:", "vitest": "4.1.7" } } diff --git a/internal-packages/sdk-compat-tests/src/fixtures/typescript/tsconfig.json b/internal-packages/sdk-compat-tests/src/fixtures/typescript/tsconfig.json index 432fff32a..a5456882a 100644 --- a/internal-packages/sdk-compat-tests/src/fixtures/typescript/tsconfig.json +++ b/internal-packages/sdk-compat-tests/src/fixtures/typescript/tsconfig.json @@ -6,7 +6,8 @@ "strict": true, "esModuleInterop": true, "skipLibCheck": true, - "noEmit": true + "noEmit": true, + "types": ["node"] }, "include": ["test.ts"] } diff --git a/internal-packages/sso/package.json b/internal-packages/sso/package.json index f31a3c4bf..187338f18 100644 --- a/internal-packages/sso/package.json +++ b/internal-packages/sso/package.json @@ -17,7 +17,7 @@ "scripts": { "clean": "rimraf dist", "typecheck": "tsc --noEmit", - "build": "pnpm run clean && tsc --noEmit false --outDir dist --declaration", + "build": "pnpm run clean && tsc -p tsconfig.build.json", "dev": "tsc --noEmit false --outDir dist --declaration --watch", "test": "vitest run", "test:watch": "vitest" diff --git a/internal-packages/sso/tsconfig.build.json b/internal-packages/sso/tsconfig.build.json new file mode 100644 index 000000000..1f858270c --- /dev/null +++ b/internal-packages/sso/tsconfig.build.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.test.ts"], + "compilerOptions": { + "noEmit": false, + "declaration": true, + "outDir": "dist", + "rootDir": ".", + "types": ["node"] + } +} diff --git a/internal-packages/sso/tsconfig.json b/internal-packages/sso/tsconfig.json index 8da0857b4..9ab7d8135 100644 --- a/internal-packages/sso/tsconfig.json +++ b/internal-packages/sso/tsconfig.json @@ -13,5 +13,5 @@ "strict": true, "customConditions": ["@triggerdotdev/source"] }, - "exclude": ["node_modules"] + "exclude": ["node_modules", "dist"] } diff --git a/internal-packages/testcontainers/tsconfig.json b/internal-packages/testcontainers/tsconfig.json index 15d4754cb..3f863511e 100644 --- a/internal-packages/testcontainers/tsconfig.json +++ b/internal-packages/testcontainers/tsconfig.json @@ -2,10 +2,10 @@ "compilerOptions": { "target": "es2022", "lib": ["ES2022", "DOM", "DOM.Iterable", "DOM.AsyncIterable"], - "module": "CommonJS", - "moduleResolution": "Node", + "module": "nodenext", + "moduleResolution": "nodenext", "moduleDetection": "force", - "types": ["vitest/globals"], + "types": ["vitest/globals", "node"], "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "isolatedModules": true, diff --git a/internal-packages/tracing/tsconfig.json b/internal-packages/tracing/tsconfig.json index 010433962..a826c8f67 100644 --- a/internal-packages/tracing/tsconfig.json +++ b/internal-packages/tracing/tsconfig.json @@ -6,7 +6,7 @@ "moduleResolution": "Node16", "moduleDetection": "force", "verbatimModuleSyntax": false, - "types": ["vitest/globals"], + "types": ["vitest/globals", "node"], "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "isolatedModules": true, diff --git a/internal-packages/tsql/tsconfig.json b/internal-packages/tsql/tsconfig.json index 98c833a18..60312e394 100644 --- a/internal-packages/tsql/tsconfig.json +++ b/internal-packages/tsql/tsconfig.json @@ -2,11 +2,14 @@ "compilerOptions": { "target": "ES2019", "lib": ["ES2019", "DOM", "DOM.Iterable", "DOM.AsyncIterable"], + // antlr4ts@0.5.0-alpha.4 uses deep-subpath CJS imports that resolve + // consistently under bundler resolution; nodenext's stricter ESM path + // duplicates its types. tsql is always bundled downstream. "module": "ESNext", - "moduleResolution": "node", + "moduleResolution": "bundler", "moduleDetection": "force", "verbatimModuleSyntax": false, - "types": ["vitest/globals"], + "types": ["vitest/globals", "node"], "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "isolatedModules": true, diff --git a/package.json b/package.json index e5ec30f0d..dddb22753 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,7 @@ "pkg-types": "1.1.3", "tsx": "^3.7.1", "turbo": "^1.10.3", - "typescript": "5.5.4", + "typescript": "catalog:", "vite-tsconfig-paths": "^4.0.5", "vitest": "4.1.7" }, @@ -90,10 +90,11 @@ "antlr4ts@0.5.0-alpha.4": "patches/antlr4ts@0.5.0-alpha.4.patch", "@window-splitter/state@1.1.3": "patches/@window-splitter__state@1.1.3.patch", "streamdown@2.5.0": "patches/streamdown@2.5.0.patch", + "tsup@8.4.0": "patches/tsup@8.4.0.patch", "@remix-run/router@1.23.3": "patches/@remix-run__router@1.23.3.patch" }, "overrides": { - "typescript": "5.5.4", + "typescript": "catalog:", "@types/node": "24.13.3", "react@^18": "18.3.1", "react-dom@^18": "18.3.1", diff --git a/packages/build/package.json b/packages/build/package.json index cdee00705..dcadb68db 100644 --- a/packages/build/package.json +++ b/packages/build/package.json @@ -87,7 +87,7 @@ "tsconfck": "3.1.3" }, "devDependencies": { - "@arethetypeswrong/cli": "^0.15.4", + "@arethetypeswrong/cli": "^0.18.5", "@types/resolve": "^1.20.6", "esbuild": "^0.23.0", "rimraf": "6.0.1", diff --git a/packages/build/tsconfig.src.json b/packages/build/tsconfig.src.json index db06c5331..7b57ad5e1 100644 --- a/packages/build/tsconfig.src.json +++ b/packages/build/tsconfig.src.json @@ -5,6 +5,7 @@ "isolatedDeclarations": false, "composite": true, "sourceMap": true, - "customConditions": ["@triggerdotdev/source"] + "customConditions": ["@triggerdotdev/source"], + "types": ["node"] } } diff --git a/packages/cli-v3/src/apiClient.ts b/packages/cli-v3/src/apiClient.ts index 682e3cd6d..d01062440 100644 --- a/packages/cli-v3/src/apiClient.ts +++ b/packages/cli-v3/src/apiClient.ts @@ -510,8 +510,8 @@ export class CliApiClient { source.onConnectionError((error) => { let message = error.message ?? "Unknown error"; - if (error.status !== undefined) { - message = `HTTP ${error.status} ${message}`; + if (error.code !== undefined) { + message = `HTTP ${error.code} ${message}`; } resolvePromise({ diff --git a/packages/cli-v3/tsconfig.e2e.test.json b/packages/cli-v3/tsconfig.e2e.test.json index 8bd111181..71705e17b 100644 --- a/packages/cli-v3/tsconfig.e2e.test.json +++ b/packages/cli-v3/tsconfig.e2e.test.json @@ -6,6 +6,6 @@ "isolatedDeclarations": false, "composite": true, "sourceMap": true, - "types": ["vitest/globals"] + "types": ["vitest/globals", "node"] } } diff --git a/packages/cli-v3/tsconfig.test.json b/packages/cli-v3/tsconfig.test.json index 52b05f928..a997c78c7 100644 --- a/packages/cli-v3/tsconfig.test.json +++ b/packages/cli-v3/tsconfig.test.json @@ -6,6 +6,6 @@ "isolatedDeclarations": false, "composite": true, "customConditions": ["@triggerdotdev/source"], - "types": ["vitest/globals"] + "types": ["vitest/globals", "node"] } } diff --git a/packages/core/package.json b/packages/core/package.json index 87292e0dc..ca8279856 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -227,7 +227,7 @@ }, "devDependencies": { "@ai-sdk/provider-utils": "^1.0.22", - "@arethetypeswrong/cli": "^0.15.4", + "@arethetypeswrong/cli": "^0.18.5", "@epic-web/test-server": "^0.1.0", "@trigger.dev/database": "workspace:*", "@types/humanize-duration": "^3.27.1", diff --git a/packages/core/src/v3/isomorphic/friendlyId.ts b/packages/core/src/v3/isomorphic/friendlyId.ts index 4a466dfd8..3141b75ae 100644 --- a/packages/core/src/v3/isomorphic/friendlyId.ts +++ b/packages/core/src/v3/isomorphic/friendlyId.ts @@ -46,7 +46,7 @@ export function regionCharForRegion(region: string | undefined): string { // globalThis.crypto is absent on Node 18.20 (a supported engine) without a flag, so fall back to // node:crypto's webcrypto, loaded only when the global is missing to stay isomorphic. -type RandomFiller = (array: Uint8Array) => void; +type RandomFiller = (array: Uint8Array) => void; function resolveGetRandomValues(): RandomFiller { const globalCrypto = (globalThis as { crypto?: Crypto }).crypto; diff --git a/packages/core/tsconfig.test.json b/packages/core/tsconfig.test.json index fb90f380e..57fc1734b 100644 --- a/packages/core/tsconfig.test.json +++ b/packages/core/tsconfig.test.json @@ -6,6 +6,6 @@ "isolatedDeclarations": false, "composite": true, "sourceMap": true, - "types": ["vitest/globals"] + "types": ["vitest/globals", "node"] } } diff --git a/packages/plugins/package.json b/packages/plugins/package.json index 8f0fbee5d..ff871a2d8 100644 --- a/packages/plugins/package.json +++ b/packages/plugins/package.json @@ -27,7 +27,7 @@ "@types/node": "^24.13.3", "rimraf": "6.0.1", "tsup": "^8.4.0", - "typescript": "^5.3.0" + "typescript": "catalog:" }, "engines": { "node": ">=18.20.0" diff --git a/packages/python/package.json b/packages/python/package.json index f230c8289..635f74ef3 100644 --- a/packages/python/package.json +++ b/packages/python/package.json @@ -52,10 +52,10 @@ "@types/node": "^24.13.3", "rimraf": "6.0.1", "tshy": "^3.0.2", - "typescript": "^5.5.4", + "typescript": "catalog:", "tsx": "4.17.0", "esbuild": "^0.23.0", - "@arethetypeswrong/cli": "^0.15.4", + "@arethetypeswrong/cli": "^0.18.5", "@trigger.dev/build": "workspace:4.5.6", "@trigger.dev/sdk": "workspace:4.5.6" }, diff --git a/packages/python/tsconfig.src.json b/packages/python/tsconfig.src.json index db06c5331..7b57ad5e1 100644 --- a/packages/python/tsconfig.src.json +++ b/packages/python/tsconfig.src.json @@ -5,6 +5,7 @@ "isolatedDeclarations": false, "composite": true, "sourceMap": true, - "customConditions": ["@triggerdotdev/source"] + "customConditions": ["@triggerdotdev/source"], + "types": ["node"] } } diff --git a/packages/react-hooks/package.json b/packages/react-hooks/package.json index 3702022c4..9a348b8ad 100644 --- a/packages/react-hooks/package.json +++ b/packages/react-hooks/package.json @@ -41,7 +41,7 @@ "swr": "^2.2.5" }, "devDependencies": { - "@arethetypeswrong/cli": "^0.15.4", + "@arethetypeswrong/cli": "^0.18.5", "@types/react": "*", "@types/react-dom": "*", "rimraf": "^6.0.1", diff --git a/packages/react-hooks/tsconfig.json b/packages/react-hooks/tsconfig.json index f73f5bea4..333267a14 100644 --- a/packages/react-hooks/tsconfig.json +++ b/packages/react-hooks/tsconfig.json @@ -4,7 +4,8 @@ "isolatedDeclarations": false, "composite": true, "sourceMap": true, - "stripInternal": true + "stripInternal": true, + "types": ["node"] }, "include": ["./src/**/*.ts", "./src/**/*.tsx"] } diff --git a/packages/redis-worker/tsconfig.test.json b/packages/redis-worker/tsconfig.test.json index fb90f380e..57fc1734b 100644 --- a/packages/redis-worker/tsconfig.test.json +++ b/packages/redis-worker/tsconfig.test.json @@ -6,6 +6,6 @@ "isolatedDeclarations": false, "composite": true, "sourceMap": true, - "types": ["vitest/globals"] + "types": ["vitest/globals", "node"] } } diff --git a/packages/rsc/package.json b/packages/rsc/package.json index b52c5408d..98624b20e 100644 --- a/packages/rsc/package.json +++ b/packages/rsc/package.json @@ -43,7 +43,7 @@ "react-dom": "19.0.0-rc.1" }, "devDependencies": { - "@arethetypeswrong/cli": "^0.15.4", + "@arethetypeswrong/cli": "^0.18.5", "@trigger.dev/build": "workspace:^4.5.6", "@types/node": "^24.13.3", "@types/react": "*", diff --git a/packages/rsc/tsconfig.json b/packages/rsc/tsconfig.json index f73f5bea4..333267a14 100644 --- a/packages/rsc/tsconfig.json +++ b/packages/rsc/tsconfig.json @@ -4,7 +4,8 @@ "isolatedDeclarations": false, "composite": true, "sourceMap": true, - "stripInternal": true + "stripInternal": true, + "types": ["node"] }, "include": ["./src/**/*.ts", "./src/**/*.tsx"] } diff --git a/packages/schema-to-json/package.json b/packages/schema-to-json/package.json index fa00d69ac..5148d5f0d 100644 --- a/packages/schema-to-json/package.json +++ b/packages/schema-to-json/package.json @@ -56,7 +56,7 @@ "valibot": "^1.1.0", "yup": "^1.7.0", "rimraf": "6.0.1", - "@arethetypeswrong/cli": "^0.15.4" + "@arethetypeswrong/cli": "^0.18.5" }, "peerDependencies": { "arktype": ">=2.0.0", diff --git a/packages/schema-to-json/tsconfig.src.json b/packages/schema-to-json/tsconfig.src.json index 93f59a20f..5ddfa12c3 100644 --- a/packages/schema-to-json/tsconfig.src.json +++ b/packages/schema-to-json/tsconfig.src.json @@ -6,6 +6,7 @@ "isolatedDeclarations": false, "composite": true, "sourceMap": true, - "customConditions": ["@triggerdotdev/source"] + "customConditions": ["@triggerdotdev/source"], + "types": ["node"] } } diff --git a/packages/schema-to-json/tsconfig.test.json b/packages/schema-to-json/tsconfig.test.json index c68227ee3..9307dd5e3 100644 --- a/packages/schema-to-json/tsconfig.test.json +++ b/packages/schema-to-json/tsconfig.test.json @@ -6,6 +6,6 @@ "isolatedDeclarations": false, "composite": true, "sourceMap": true, - "types": ["vitest/globals"] + "types": ["vitest/globals", "node"] } } diff --git a/packages/trigger-sdk/package.json b/packages/trigger-sdk/package.json index 051c955e6..77889cb09 100644 --- a/packages/trigger-sdk/package.json +++ b/packages/trigger-sdk/package.json @@ -89,7 +89,7 @@ }, "devDependencies": { "@ai-sdk/provider": "3.0.8", - "@arethetypeswrong/cli": "^0.15.4", + "@arethetypeswrong/cli": "^0.18.5", "@types/debug": "^4.1.7", "@types/react": "^19.2.14", "@types/slug": "^5.0.3", diff --git a/patches/tsup@8.4.0.patch b/patches/tsup@8.4.0.patch new file mode 100644 index 000000000..4b2dc8757 --- /dev/null +++ b/patches/tsup@8.4.0.patch @@ -0,0 +1,12 @@ +diff --git a/dist/rollup.js b/dist/rollup.js +index cc82363..924f57f 100644 +--- a/dist/rollup.js ++++ b/dist/rollup.js +@@ -6791,7 +6791,6 @@ var getRollupConfig = async (options) => { + tsconfig: options.tsconfig, + compilerOptions: { + ...compilerOptions, +- baseUrl: compilerOptions.baseUrl || ".", + // Ensure ".d.ts" modules are generated + declaration: true, + // Skip ".js" generation diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9f1242f68..fb98ad91c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5,7 +5,7 @@ settings: excludeLinksFromLockfile: false overrides: - typescript: 5.5.4 + typescript: 6.0.3 '@types/node': 24.13.3 react@^18: 18.3.1 react-dom@^18: 18.3.1 @@ -83,6 +83,9 @@ patchedDependencies: streamdown@2.5.0: hash: 36211d09153a59c880b6a2bce2a0a0f011c99c73c20c8ceca78cc77e47623f06 path: patches/streamdown@2.5.0.patch + tsup@8.4.0: + hash: bb8871e353e0582d16a3fdef093af6d43be84f0957b610d51e43cc2bc80add27 + path: patches/tsup@8.4.0.patch importers: @@ -141,11 +144,11 @@ importers: specifier: ^1.10.3 version: 1.10.3 typescript: - specifier: 5.5.4 - version: 5.5.4 + specifier: 6.0.3 + version: 6.0.3 vite-tsconfig-paths: specifier: ^4.0.5 - version: 4.0.5(typescript@5.5.4) + version: 4.0.5(typescript@6.0.3) vitest: specifier: 4.1.7 version: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(@vitest/coverage-v8@4.1.7)(vite@6.4.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@3.12.2)(yaml@2.9.0)) @@ -419,25 +422,25 @@ importers: version: 3.7.1(react@18.3.1) '@remix-run/express': specifier: 2.17.5 - version: 2.17.5(express@4.20.0)(typescript@5.5.4) + version: 2.17.5(express@4.20.0)(typescript@6.0.3) '@remix-run/node': specifier: 2.17.5 - version: 2.17.5(typescript@5.5.4) + version: 2.17.5(typescript@6.0.3) '@remix-run/react': specifier: 2.17.5 - version: 2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4) + version: 2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3) '@remix-run/router': specifier: ^1.23.3 version: 1.23.3(patch_hash=33e2966e9ef36aa09955cec9922c5063227b83429157a71ba226369a08627ac9) '@remix-run/server-runtime': specifier: 2.17.5 - version: 2.17.5(typescript@5.5.4) + version: 2.17.5(typescript@6.0.3) '@s2-dev/streamstore': specifier: ^0.22.10 version: 0.22.10(supports-color@10.0.0) '@sentry/remix': specifier: 9.46.0 - version: 9.46.0(patch_hash=146126b032581925294aaed63ab53ce3f5e0356a755f1763d7a9a76b9846943b)(@remix-run/node@2.17.5(typescript@5.5.4))(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/server-runtime@2.17.5(typescript@5.5.4))(encoding@0.1.13)(react@18.3.1) + version: 9.46.0(patch_hash=146126b032581925294aaed63ab53ce3f5e0356a755f1763d7a9a76b9846943b)(@remix-run/node@2.17.5(typescript@6.0.3))(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/server-runtime@2.17.5(typescript@6.0.3))(encoding@0.1.13)(react@18.3.1) '@slack/web-api': specifier: 7.16.0 version: 7.16.0 @@ -521,7 +524,7 @@ importers: version: 1.0.18 class-variance-authority: specifier: ^0.5.2 - version: 0.5.2(typescript@5.5.4) + version: 0.5.2(typescript@6.0.3) clsx: specifier: ^1.2.1 version: 1.2.1 @@ -689,22 +692,22 @@ importers: version: 2.0.1 remix-auth: specifier: ^3.6.0 - version: 3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/server-runtime@2.17.5(typescript@5.5.4)) + version: 3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/server-runtime@2.17.5(typescript@6.0.3)) remix-auth-email-link: specifier: 2.0.2 - version: 2.0.2(@remix-run/server-runtime@2.17.5(typescript@5.5.4))(remix-auth@3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/server-runtime@2.17.5(typescript@5.5.4))) + version: 2.0.2(@remix-run/server-runtime@2.17.5(typescript@6.0.3))(remix-auth@3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/server-runtime@2.17.5(typescript@6.0.3))) remix-auth-github: specifier: ^1.6.0 - version: 1.6.0(@remix-run/server-runtime@2.17.5(typescript@5.5.4))(remix-auth@3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/server-runtime@2.17.5(typescript@5.5.4))) + version: 1.6.0(@remix-run/server-runtime@2.17.5(typescript@6.0.3))(remix-auth@3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/server-runtime@2.17.5(typescript@6.0.3))) remix-auth-google: specifier: ^2.0.0 - version: 2.0.0(@remix-run/server-runtime@2.17.5(typescript@5.5.4))(remix-auth@3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/server-runtime@2.17.5(typescript@5.5.4))) + version: 2.0.0(@remix-run/server-runtime@2.17.5(typescript@6.0.3))(remix-auth@3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/server-runtime@2.17.5(typescript@6.0.3))) remix-typedjson: specifier: 0.3.1 - version: 0.3.1(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/server-runtime@2.17.5(typescript@5.5.4))(react@18.3.1) + version: 0.3.1(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/server-runtime@2.17.5(typescript@6.0.3))(react@18.3.1) remix-utils: specifier: ^7.7.0 - version: 7.7.0(@remix-run/node@2.17.5(typescript@5.5.4))(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/router@1.23.3(patch_hash=33e2966e9ef36aa09955cec9922c5063227b83429157a71ba226369a08627ac9))(crypto-js@4.2.0)(intl-parse-accept-language@1.0.0)(react@18.3.1)(zod@3.25.76) + version: 7.7.0(@remix-run/node@2.17.5(typescript@6.0.3))(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/router@1.23.3(patch_hash=33e2966e9ef36aa09955cec9922c5063227b83429157a71ba226369a08627ac9))(crypto-js@4.2.0)(intl-parse-accept-language@1.0.0)(react@18.3.1)(zod@3.25.76) simplur: specifier: ^3.0.1 version: 3.0.1 @@ -768,10 +771,10 @@ importers: version: link:../../internal-packages/testcontainers '@remix-run/dev': specifier: 2.17.5 - version: 2.17.5(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/serve@2.17.5(typescript@5.5.4))(@types/node@24.13.3)(bufferutil@4.0.9)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.20.6)(typescript@5.5.4)(vite@6.4.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.20.6)(yaml@2.9.0))(yaml@2.9.0) + version: 2.17.5(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/serve@2.17.5(typescript@6.0.3))(@types/node@24.13.3)(bufferutil@4.0.9)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.20.6)(typescript@6.0.3)(vite@6.4.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.20.6)(yaml@2.9.0))(yaml@2.9.0) '@remix-run/testing': specifier: ^2.17.5 - version: 2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4) + version: 2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3) '@sentry/cli': specifier: 2.50.2 version: 2.50.2(encoding@0.1.13) @@ -873,7 +876,7 @@ importers: version: 16.0.1(postcss@8.5.15) postcss-loader: specifier: ^8.1.1 - version: 8.1.1(postcss@8.5.15)(typescript@5.5.4)(webpack@5.102.1(@swc/core@1.3.26)(esbuild@0.15.18)) + version: 8.1.1(postcss@8.5.15)(typescript@6.0.3)(webpack@5.102.1(@swc/core@1.3.26)(esbuild@0.15.18)) prop-types: specifier: ^15.8.1 version: 15.8.1 @@ -900,7 +903,7 @@ importers: version: 4.20.6 vite-tsconfig-paths: specifier: ^4.0.5 - version: 4.0.5(typescript@5.5.4) + version: 4.0.5(typescript@6.0.3) docs: {} @@ -998,7 +1001,7 @@ importers: dependencies: drizzle-orm: specifier: ^0.45.0 - version: 0.45.2(@opentelemetry/api@1.9.1)(@prisma/client@6.14.0(prisma@6.14.0(magicast@0.3.5)(typescript@5.5.4))(typescript@5.5.4))(@types/pg@8.11.14)(better-sqlite3@11.10.0)(pg@8.15.6)(postgres@3.4.9)(prisma@6.14.0(magicast@0.3.5)(typescript@5.5.4)) + version: 0.45.2(@opentelemetry/api@1.9.1)(@prisma/client@6.14.0(prisma@6.14.0(magicast@0.3.5)(typescript@6.0.3))(typescript@6.0.3))(@types/pg@8.11.14)(better-sqlite3@11.10.0)(pg@8.15.6)(postgres@3.4.9)(prisma@6.14.0(magicast@0.3.5)(typescript@6.0.3)) postgres: specifier: ^3.4.0 version: 3.4.9 @@ -1011,13 +1014,13 @@ importers: dependencies: '@prisma/client': specifier: 6.14.0 - version: 6.14.0(prisma@6.14.0(magicast@0.3.5)(typescript@5.5.4))(typescript@5.5.4) + version: 6.14.0(prisma@6.14.0(magicast@0.3.5)(typescript@6.0.3))(typescript@6.0.3) decimal.js: specifier: ^10.6.0 version: 10.6.0 prisma: specifier: 6.14.0 - version: 6.14.0(magicast@0.3.5)(typescript@5.5.4) + version: 6.14.0(magicast@0.3.5)(typescript@6.0.3) devDependencies: '@types/decimal.js': specifier: ^7.4.3 @@ -1219,10 +1222,10 @@ importers: dependencies: '@prisma/client': specifier: 6.14.0 - version: 6.14.0(prisma@6.14.0(magicast@0.3.5)(typescript@5.5.4))(typescript@5.5.4) + version: 6.14.0(prisma@6.14.0(magicast@0.3.5)(typescript@6.0.3))(typescript@6.0.3) prisma: specifier: 6.14.0 - version: 6.14.0(magicast@0.3.5)(typescript@5.5.4) + version: 6.14.0(magicast@0.3.5)(typescript@6.0.3) devDependencies: rimraf: specifier: 6.0.1 @@ -1300,8 +1303,8 @@ importers: specifier: ^9.3.0 version: 9.6.1 typescript: - specifier: 5.5.4 - version: 5.5.4 + specifier: 6.0.3 + version: 6.0.3 vitest: specifier: 4.1.7 version: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(@vitest/coverage-v8@4.1.7)(vite@6.4.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.22.4)(yaml@2.9.0)) @@ -1418,11 +1421,11 @@ importers: version: 0.2.2 tsconfck: specifier: 3.1.3 - version: 3.1.3(typescript@5.5.4) + version: 3.1.3(typescript@6.0.3) devDependencies: '@arethetypeswrong/cli': - specifier: ^0.15.4 - version: 0.15.4 + specifier: ^0.18.5 + version: 0.18.5 '@types/resolve': specifier: ^1.20.6 version: 1.20.6 @@ -1681,7 +1684,7 @@ importers: version: 6.0.1 ts-essentials: specifier: 10.0.1 - version: 10.0.1(typescript@5.5.4) + version: 10.0.1(typescript@6.0.3) tshy: specifier: ^3.0.2 version: 3.0.2 @@ -1801,8 +1804,8 @@ importers: specifier: ^1.0.22 version: 1.0.22(zod@3.25.76) '@arethetypeswrong/cli': - specifier: ^0.15.4 - version: 0.15.4 + specifier: ^0.18.5 + version: 0.18.5 '@epic-web/test-server': specifier: ^0.1.0 version: 0.1.0(bufferutil@4.0.9) @@ -1838,7 +1841,7 @@ importers: version: 2.2.1 ts-essentials: specifier: 10.0.1 - version: 10.0.1(typescript@5.5.4) + version: 10.0.1(typescript@6.0.3) tshy: specifier: ^3.0.2 version: 3.0.2 @@ -1863,10 +1866,10 @@ importers: version: 6.0.1 tsup: specifier: ^8.4.0 - version: 8.4.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.7.0)(postcss@8.5.15)(tsx@4.22.4)(typescript@5.5.4)(yaml@2.9.0) + version: 8.4.0(patch_hash=bb8871e353e0582d16a3fdef093af6d43be84f0957b610d51e43cc2bc80add27)(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.7.0)(postcss@8.5.15)(tsx@4.22.4)(typescript@6.0.3)(yaml@2.9.0) typescript: - specifier: 5.5.4 - version: 5.5.4 + specifier: 6.0.3 + version: 6.0.3 packages/python: dependencies: @@ -1878,8 +1881,8 @@ importers: version: 0.3.2 devDependencies: '@arethetypeswrong/cli': - specifier: ^0.15.4 - version: 0.15.4 + specifier: ^0.18.5 + version: 0.18.5 '@trigger.dev/build': specifier: workspace:4.5.6 version: link:../build @@ -1902,8 +1905,8 @@ importers: specifier: 4.17.0 version: 4.17.0 typescript: - specifier: 5.5.4 - version: 5.5.4 + specifier: 6.0.3 + version: 6.0.3 packages/react-hooks: dependencies: @@ -1921,8 +1924,8 @@ importers: version: 2.2.5(react@18.3.1) devDependencies: '@arethetypeswrong/cli': - specifier: ^0.15.4 - version: 0.15.4 + specifier: ^0.18.5 + version: 0.18.5 '@types/react': specifier: '*' version: 18.3.1 @@ -1983,7 +1986,7 @@ importers: version: 6.0.1 tsup: specifier: ^8.4.0 - version: 8.4.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.7.0)(postcss@8.5.15)(tsx@4.17.0)(typescript@5.5.4)(yaml@2.9.0) + version: 8.4.0(patch_hash=bb8871e353e0582d16a3fdef093af6d43be84f0957b610d51e43cc2bc80add27)(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.7.0)(postcss@8.5.15)(tsx@4.17.0)(typescript@6.0.3)(yaml@2.9.0) tsx: specifier: 4.17.0 version: 4.17.0 @@ -2004,8 +2007,8 @@ importers: version: 19.0.0-rc.1(react@19.0.0-rc.1) devDependencies: '@arethetypeswrong/cli': - specifier: ^0.15.4 - version: 0.15.4 + specifier: ^0.18.5 + version: 0.18.5 '@trigger.dev/build': specifier: workspace:^4.5.6 version: link:../build @@ -2047,8 +2050,8 @@ importers: version: 3.24.6(zod@3.25.76) devDependencies: '@arethetypeswrong/cli': - specifier: ^0.15.4 - version: 0.15.4 + specifier: ^0.18.5 + version: 0.18.5 '@sinclair/typebox': specifier: ^0.34.3 version: 0.34.38 @@ -2069,7 +2072,7 @@ importers: version: 3.0.2 valibot: specifier: ^1.1.0 - version: 1.1.0(typescript@5.5.4) + version: 1.1.0(typescript@6.0.3) yup: specifier: ^1.7.0 version: 1.7.0 @@ -2120,8 +2123,8 @@ importers: specifier: 3.0.8 version: 3.0.8 '@arethetypeswrong/cli': - specifier: ^0.15.4 - version: 0.15.4 + specifier: ^0.18.5 + version: 0.18.5 '@types/debug': specifier: ^4.1.7 version: 4.1.7 @@ -2265,14 +2268,14 @@ packages: '@antfu/utils@9.3.0': resolution: {integrity: sha512-9hFT4RauhcUzqOE4f1+frMKLZrgNog5b06I7VmZQV1BkvwvqrbC8EBZf3L1eEL2AKb6rNKjER0sEvJiSP1FXEA==} - '@arethetypeswrong/cli@0.15.4': - resolution: {integrity: sha512-YDbImAi1MGkouT7f2yAECpUMFhhA1J0EaXzIqoC5GGtK0xDgauLtcsZezm8tNq7d3wOFXH7OnY+IORYcG212rw==} - engines: {node: '>=18'} + '@arethetypeswrong/cli@0.18.5': + resolution: {integrity: sha512-gM+8vRsQOD/Uc7EnBedUhkG5OCsDWE4uoak5QvomGpMpaky0Eh41p04nIMgrWb8EOmqZUJGc6zz9hsP6E56R7g==} + engines: {node: '>=20'} hasBin: true - '@arethetypeswrong/core@0.15.1': - resolution: {integrity: sha512-FYp6GBAgsNz81BkfItRz8RLZO03w5+BaeiPma1uCfmxTnxbtuMrI/dbzGiOk8VghO108uFI0oJo0OkewdSHw7g==} - engines: {node: '>=18'} + '@arethetypeswrong/core@0.18.5': + resolution: {integrity: sha512-9ytjzGwxjm9Uz7I9avfbt5vlQt6uk9uRRESzJjqrznl6WKvI6dwYTo+vJ3U02Wrq/mR3iql/PzhvHhKdJIAjDQ==} + engines: {node: '>=20'} '@ariakit/core@0.4.6': resolution: {integrity: sha512-L2WIZZlxDs611m3YLSv2xvJyQrkkVQJlxn8Y4DlI1G65VLTEH7hysw3RYUNdXsl0gP6S20id3zBMJCHT9BCRcg==} @@ -2937,6 +2940,9 @@ packages: '@better-auth/utils@0.2.6': resolution: {integrity: sha512-3y/vaL5Ox33dBwgJ6ub3OPkVqr6B5xL2kgxNHG8eHZuryLyG/4JSPGqjbdRSgjuy9kALUZYDFl+ORIAxlWMSuA==} + '@braidai/lang@1.1.2': + resolution: {integrity: sha512-qBcknbBufNHlui137Hft8xauQMTZDKdophmLFv05r2eNmdIv/MlPuP4TdUknHG68UdWLgVZwgxVe735HzJNIwA==} + '@braintree/sanitize-url@7.1.1': resolution: {integrity: sha512-i1L7noDNxtFyL5DmZafWy1wRVhGehQmzZaz1HiN5e7iylJMSZR7ekOV7NsIqa5qBldlLrsKv4HbgFUVlQrz8Mw==} @@ -4516,6 +4522,9 @@ packages: '@lezer/lr@1.4.3': resolution: {integrity: sha512-yenN5SqAxAPv/qMnpWW0AT7l+SxVrgG+u0tNsRQWqbrz66HIl8DnEbBObvy21J5K7+I1v7gsAnlE2VQ5yYVSeA==} + '@loaderkit/resolve@1.0.6': + resolution: {integrity: sha512-G8FdIoF5CypfwmD9rl8BXod5HDn8JqB0CCNBXDTaRZ+yRYhARrrSToX1zg1zy9jX3zLqigsELwhT4gNtkdQAUg==} + '@lukeed/ms@2.0.2': resolution: {integrity: sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==} engines: {node: '>=8'} @@ -5561,7 +5570,7 @@ packages: engines: {node: '>=18.18'} peerDependencies: prisma: '*' - typescript: 5.5.4 + typescript: 6.0.3 peerDependenciesMeta: prisma: optional: true @@ -6504,7 +6513,7 @@ packages: peerDependencies: '@remix-run/react': ^2.17.0 '@remix-run/serve': ^2.17.0 - typescript: 5.5.4 + typescript: 6.0.3 vite: ^6.4.2 wrangler: ^3.28.2 peerDependenciesMeta: @@ -6522,7 +6531,7 @@ packages: engines: {node: '>=18.0.0'} peerDependencies: express: ^4.20.0 - typescript: 5.5.4 + typescript: 6.0.3 peerDependenciesMeta: typescript: optional: true @@ -6531,7 +6540,7 @@ packages: resolution: {integrity: sha512-CwOUCDJqh9o8r/n5N1l+vz4Jh7DFU52/jo7MN56+OL9gM+14aQKdq1aLAh+4V6GuI/qtki9PPk02GmULimmDkw==} engines: {node: '>=18.0.0'} peerDependencies: - typescript: 5.5.4 + typescript: 6.0.3 peerDependenciesMeta: typescript: optional: true @@ -6542,7 +6551,7 @@ packages: peerDependencies: react: 18.3.1 react-dom: 18.3.1 - typescript: 5.5.4 + typescript: 6.0.3 peerDependenciesMeta: typescript: optional: true @@ -6560,7 +6569,7 @@ packages: resolution: {integrity: sha512-SyJ5n2pQyo4ERGvjjLvL2PpRWBzlC2qzO5KkkOE2ygOiNcgOu9WQnnom9GI8KgsTRCO8JnIeLHFRcmxZnVBjfw==} engines: {node: '>=18.0.0'} peerDependencies: - typescript: 5.5.4 + typescript: 6.0.3 peerDependenciesMeta: typescript: optional: true @@ -6570,7 +6579,7 @@ packages: engines: {node: '>=18.0.0'} peerDependencies: react: 18.3.1 - typescript: 5.5.4 + typescript: 6.0.3 peerDependenciesMeta: typescript: optional: true @@ -8907,7 +8916,7 @@ packages: class-variance-authority@0.5.2: resolution: {integrity: sha512-j7Qqw3NPbs4IpO80gvdACWmVvHiLLo5MECacUBLnJG17CrLpWaQ7/4OaWX6P0IO1j2nvZ7AuSfBS/ImtEUZJGA==} peerDependencies: - typescript: 5.5.4 + typescript: 6.0.3 peerDependenciesMeta: typescript: optional: true @@ -9143,7 +9152,7 @@ packages: resolution: {integrity: sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==} engines: {node: '>=14'} peerDependencies: - typescript: 5.5.4 + typescript: 6.0.3 peerDependenciesMeta: typescript: optional: true @@ -10421,8 +10430,8 @@ packages: fflate@0.4.8: resolution: {integrity: sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA==} - fflate@0.8.2: - resolution: {integrity: sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==} + fflate@0.8.3: + resolution: {integrity: sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==} fft.js@4.0.4: resolution: {integrity: sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==} @@ -13202,7 +13211,7 @@ packages: engines: {node: '>=18.18'} hasBin: true peerDependencies: - typescript: 5.5.4 + typescript: 6.0.3 peerDependenciesMeta: typescript: optional: true @@ -14700,14 +14709,11 @@ packages: ts-essentials@10.0.1: resolution: {integrity: sha512-HPH+H2bkkO8FkMDau+hFvv7KYozzned9Zr1Urn7rRPXMF4mZmCKOq+u4AI1AAW+2bofIOXTuSdKo9drQuni2dQ==} peerDependencies: - typescript: 5.5.4 + typescript: 6.0.3 peerDependenciesMeta: typescript: optional: true - ts-expose-internals-conditionally@1.0.0-empty.0: - resolution: {integrity: sha512-F8m9NOF6ZhdOClDVdlM8gj3fDCav4ZIFSs/EI3ksQbAAXVSCN/Jh5OCJDDZWBuBy9psFc6jULGDlPwjMYMhJDw==} - ts-interface-checker@0.1.13: resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} @@ -14733,7 +14739,7 @@ packages: deprecated: unmaintained hasBin: true peerDependencies: - typescript: 5.5.4 + typescript: 6.0.3 peerDependenciesMeta: typescript: optional: true @@ -14744,7 +14750,7 @@ packages: deprecated: unmaintained hasBin: true peerDependencies: - typescript: 5.5.4 + typescript: 6.0.3 peerDependenciesMeta: typescript: optional: true @@ -14781,7 +14787,7 @@ packages: '@microsoft/api-extractor': ^7.36.0 '@swc/core': ^1 postcss: ^8.5.10 - typescript: 5.5.4 + typescript: 6.0.3 peerDependenciesMeta: '@microsoft/api-extractor': optional: true @@ -14920,8 +14926,8 @@ packages: typed-emitter@2.1.0: resolution: {integrity: sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==} - typescript@5.5.4: - resolution: {integrity: sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==} + typescript@6.0.3: + resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} engines: {node: '>=14.17'} hasBin: true @@ -15151,7 +15157,7 @@ packages: valibot@1.1.0: resolution: {integrity: sha512-Nk8lX30Qhu+9txPYTwM0cFlWLdPFsFr6LblzqIySfbZph9+BFsAHsNvHOymEviUepeIW6KFHzpX8TKhbptBXXw==} peerDependencies: - typescript: 5.5.4 + typescript: 6.0.3 peerDependenciesMeta: typescript: optional: true @@ -15159,7 +15165,7 @@ packages: valibot@1.3.1: resolution: {integrity: sha512-sfdRir/QFM0JaF22hqTroPc5xy4DimuGQVKFrzF1YfGwaS1nJot3Y8VqMdLO2Lg27fMzat2yD3pY5PbAYO39Gg==} peerDependencies: - typescript: 5.5.4 + typescript: 6.0.3 peerDependenciesMeta: typescript: optional: true @@ -15725,23 +15731,25 @@ snapshots: '@antfu/utils@9.3.0': {} - '@arethetypeswrong/cli@0.15.4': + '@arethetypeswrong/cli@0.18.5': dependencies: - '@arethetypeswrong/core': 0.15.1 + '@arethetypeswrong/core': 0.18.5 chalk: 4.1.2 cli-table3: 0.6.5 commander: 10.0.1 marked: 9.1.6 marked-terminal: 7.1.0(marked@9.1.6) - semver: 7.7.3 + semver: 7.8.1 - '@arethetypeswrong/core@0.15.1': + '@arethetypeswrong/core@0.18.5': dependencies: '@andrewbranch/untar.js': 1.0.3 - fflate: 0.8.2 + '@loaderkit/resolve': 1.0.6 + cjs-module-lexer: 1.4.3 + fflate: 0.8.3 + lru-cache: 11.2.4 semver: 7.8.1 - ts-expose-internals-conditionally: 1.0.0-empty.0 - typescript: 5.5.4 + typescript: 6.0.3 validate-npm-package-name: 5.0.0 '@ariakit/core@0.4.6': {} @@ -17510,6 +17518,8 @@ snapshots: dependencies: uncrypto: 0.1.3 + '@braidai/lang@1.1.2': {} + '@braintree/sanitize-url@7.1.1': {} '@bufbuild/protobuf@1.10.0': {} @@ -18798,6 +18808,10 @@ snapshots: dependencies: '@lezer/common': 1.3.0 + '@loaderkit/resolve@1.0.6': + dependencies: + '@braidai/lang': 1.1.2 + '@lukeed/ms@2.0.2': {} '@manypkg/cli@0.19.2': @@ -19820,10 +19834,10 @@ snapshots: '@posthog/types@1.376.4': {} - '@prisma/client@6.14.0(prisma@6.14.0(magicast@0.3.5)(typescript@5.5.4))(typescript@5.5.4)': + '@prisma/client@6.14.0(prisma@6.14.0(magicast@0.3.5)(typescript@6.0.3))(typescript@6.0.3)': optionalDependencies: - prisma: 6.14.0(magicast@0.3.5)(typescript@5.5.4) - typescript: 5.5.4 + prisma: 6.14.0(magicast@0.3.5)(typescript@6.0.3) + typescript: 6.0.3 '@prisma/config@6.14.0(magicast@0.3.5)': dependencies: @@ -20881,7 +20895,7 @@ snapshots: transitivePeerDependencies: - encoding - '@remix-run/dev@2.17.5(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/serve@2.17.5(typescript@5.5.4))(@types/node@24.13.3)(bufferutil@4.0.9)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.20.6)(typescript@5.5.4)(vite@6.4.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.20.6)(yaml@2.9.0))(yaml@2.9.0)': + '@remix-run/dev@2.17.5(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/serve@2.17.5(typescript@6.0.3))(@types/node@24.13.3)(bufferutil@4.0.9)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.20.6)(typescript@6.0.3)(vite@6.4.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.20.6)(yaml@2.9.0))(yaml@2.9.0)': dependencies: '@babel/core': 7.22.17 '@babel/generator': 7.29.7 @@ -20893,10 +20907,10 @@ snapshots: '@babel/types': 7.29.7 '@mdx-js/mdx': 2.3.0 '@npmcli/package-json': 4.0.1 - '@remix-run/node': 2.17.5(typescript@5.5.4) - '@remix-run/react': 2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4) + '@remix-run/node': 2.17.5(typescript@6.0.3) + '@remix-run/react': 2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3) '@remix-run/router': 1.23.3(patch_hash=33e2966e9ef36aa09955cec9922c5063227b83429157a71ba226369a08627ac9) - '@remix-run/server-runtime': 2.17.5(typescript@5.5.4) + '@remix-run/server-runtime': 2.17.5(typescript@6.0.3) '@types/mdx': 2.0.5 '@vanilla-extract/integration': 6.2.1(@types/node@24.13.3)(lightningcss@1.32.0)(terser@5.46.1) arg: 5.0.2 @@ -20937,12 +20951,12 @@ snapshots: set-cookie-parser: 2.6.0 tar-fs: 2.1.4 tsconfig-paths: 4.2.0 - valibot: 1.3.1(typescript@5.5.4) + valibot: 1.3.1(typescript@6.0.3) vite-node: 3.1.4(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.20.6)(yaml@2.9.0) ws: 7.5.11(bufferutil@4.0.9) optionalDependencies: - '@remix-run/serve': 2.17.5(typescript@5.5.4) - typescript: 5.5.4 + '@remix-run/serve': 2.17.5(typescript@6.0.3) + typescript: 6.0.3 vite: 6.4.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.20.6)(yaml@2.9.0) transitivePeerDependencies: - '@types/node' @@ -20962,16 +20976,16 @@ snapshots: - utf-8-validate - yaml - '@remix-run/express@2.17.5(express@4.20.0)(typescript@5.5.4)': + '@remix-run/express@2.17.5(express@4.20.0)(typescript@6.0.3)': dependencies: - '@remix-run/node': 2.17.5(typescript@5.5.4) + '@remix-run/node': 2.17.5(typescript@6.0.3) express: 4.20.0 optionalDependencies: - typescript: 5.5.4 + typescript: 6.0.3 - '@remix-run/node@2.17.5(typescript@5.5.4)': + '@remix-run/node@2.17.5(typescript@6.0.3)': dependencies: - '@remix-run/server-runtime': 2.17.5(typescript@5.5.4) + '@remix-run/server-runtime': 2.17.5(typescript@6.0.3) '@remix-run/web-fetch': 4.4.2 '@web3-storage/multipart-parser': 1.0.0 cookie-signature: 1.2.2 @@ -20979,26 +20993,26 @@ snapshots: stream-slice: 0.1.2 undici: 6.27.0 optionalDependencies: - typescript: 5.5.4 + typescript: 6.0.3 - '@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4)': + '@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3)': dependencies: '@remix-run/router': 1.23.3(patch_hash=33e2966e9ef36aa09955cec9922c5063227b83429157a71ba226369a08627ac9) - '@remix-run/server-runtime': 2.17.5(typescript@5.5.4) + '@remix-run/server-runtime': 2.17.5(typescript@6.0.3) react: 18.3.1 react-dom: 18.3.1(react@18.3.1) react-router: 6.30.4(react@18.3.1) react-router-dom: 6.30.4(react-dom@18.3.1(react@18.3.1))(react@18.3.1) turbo-stream: 2.4.1 optionalDependencies: - typescript: 5.5.4 + typescript: 6.0.3 '@remix-run/router@1.23.3(patch_hash=33e2966e9ef36aa09955cec9922c5063227b83429157a71ba226369a08627ac9)': {} - '@remix-run/serve@2.17.5(typescript@5.5.4)': + '@remix-run/serve@2.17.5(typescript@6.0.3)': dependencies: - '@remix-run/express': 2.17.5(express@4.20.0)(typescript@5.5.4) - '@remix-run/node': 2.17.5(typescript@5.5.4) + '@remix-run/express': 2.17.5(express@4.20.0)(typescript@6.0.3) + '@remix-run/node': 2.17.5(typescript@6.0.3) chokidar: 3.6.0 compression: 1.8.1 express: 4.20.0 @@ -21010,7 +21024,7 @@ snapshots: - typescript optional: true - '@remix-run/server-runtime@2.17.5(typescript@5.5.4)': + '@remix-run/server-runtime@2.17.5(typescript@6.0.3)': dependencies: '@remix-run/router': 1.23.3(patch_hash=33e2966e9ef36aa09955cec9922c5063227b83429157a71ba226369a08627ac9) '@types/cookie': 0.6.0 @@ -21020,17 +21034,17 @@ snapshots: source-map: 0.7.4 turbo-stream: 2.4.1 optionalDependencies: - typescript: 5.5.4 + typescript: 6.0.3 - '@remix-run/testing@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4)': + '@remix-run/testing@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3)': dependencies: - '@remix-run/node': 2.17.5(typescript@5.5.4) - '@remix-run/react': 2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4) + '@remix-run/node': 2.17.5(typescript@6.0.3) + '@remix-run/react': 2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3) '@remix-run/router': 1.23.3(patch_hash=33e2966e9ef36aa09955cec9922c5063227b83429157a71ba226369a08627ac9) react: 18.3.1 react-router-dom: 6.30.4(react-dom@18.3.1(react@18.3.1))(react@18.3.1) optionalDependencies: - typescript: 5.5.4 + typescript: 6.0.3 transitivePeerDependencies: - react-dom @@ -21295,15 +21309,15 @@ snapshots: hoist-non-react-statics: 3.3.2 react: 18.3.1 - '@sentry/remix@9.46.0(patch_hash=146126b032581925294aaed63ab53ce3f5e0356a755f1763d7a9a76b9846943b)(@remix-run/node@2.17.5(typescript@5.5.4))(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/server-runtime@2.17.5(typescript@5.5.4))(encoding@0.1.13)(react@18.3.1)': + '@sentry/remix@9.46.0(patch_hash=146126b032581925294aaed63ab53ce3f5e0356a755f1763d7a9a76b9846943b)(@remix-run/node@2.17.5(typescript@6.0.3))(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/server-runtime@2.17.5(typescript@6.0.3))(encoding@0.1.13)(react@18.3.1)': dependencies: '@opentelemetry/api': 1.9.1 '@opentelemetry/instrumentation': 0.57.2(@opentelemetry/api@1.9.1) '@opentelemetry/semantic-conventions': 1.41.1 - '@remix-run/node': 2.17.5(typescript@5.5.4) - '@remix-run/react': 2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4) + '@remix-run/node': 2.17.5(typescript@6.0.3) + '@remix-run/react': 2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3) '@remix-run/router': 1.23.3(patch_hash=33e2966e9ef36aa09955cec9922c5063227b83429157a71ba226369a08627ac9) - '@remix-run/server-runtime': 2.17.5(typescript@5.5.4) + '@remix-run/server-runtime': 2.17.5(typescript@6.0.3) '@sentry/cli': 2.50.2(encoding@0.1.13) '@sentry/core': 9.46.0 '@sentry/node': 9.46.0 @@ -23804,9 +23818,9 @@ snapshots: cjs-module-lexer@2.2.0: {} - class-variance-authority@0.5.2(typescript@5.5.4): + class-variance-authority@0.5.2(typescript@6.0.3): optionalDependencies: - typescript: 5.5.4 + typescript: 6.0.3 clean-stack@2.2.0: {} @@ -24051,14 +24065,14 @@ snapshots: dependencies: layout-base: 2.0.1 - cosmiconfig@9.0.0(typescript@5.5.4): + cosmiconfig@9.0.0(typescript@6.0.3): dependencies: env-paths: 2.2.1 import-fresh: 3.3.0 js-yaml: 4.1.1 parse-json: 5.2.0 optionalDependencies: - typescript: 5.5.4 + typescript: 6.0.3 cp-file@10.0.0: dependencies: @@ -24640,15 +24654,15 @@ snapshots: esbuild: 0.25.12 tsx: 4.22.4 - drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@prisma/client@6.14.0(prisma@6.14.0(magicast@0.3.5)(typescript@5.5.4))(typescript@5.5.4))(@types/pg@8.11.14)(better-sqlite3@11.10.0)(pg@8.15.6)(postgres@3.4.9)(prisma@6.14.0(magicast@0.3.5)(typescript@5.5.4)): + drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@prisma/client@6.14.0(prisma@6.14.0(magicast@0.3.5)(typescript@6.0.3))(typescript@6.0.3))(@types/pg@8.11.14)(better-sqlite3@11.10.0)(pg@8.15.6)(postgres@3.4.9)(prisma@6.14.0(magicast@0.3.5)(typescript@6.0.3)): optionalDependencies: '@opentelemetry/api': 1.9.1 - '@prisma/client': 6.14.0(prisma@6.14.0(magicast@0.3.5)(typescript@5.5.4))(typescript@5.5.4) + '@prisma/client': 6.14.0(prisma@6.14.0(magicast@0.3.5)(typescript@6.0.3))(typescript@6.0.3) '@types/pg': 8.11.14 better-sqlite3: 11.10.0 pg: 8.15.6 postgres: 3.4.9 - prisma: 6.14.0(magicast@0.3.5)(typescript@5.5.4) + prisma: 6.14.0(magicast@0.3.5)(typescript@6.0.3) dunder-proto@1.0.1: dependencies: @@ -25608,7 +25622,7 @@ snapshots: fflate@0.4.8: {} - fflate@0.8.2: {} + fflate@0.8.3: {} fft.js@4.0.4: {} @@ -28702,9 +28716,9 @@ snapshots: tsx: 4.22.4 yaml: 2.9.0 - postcss-loader@8.1.1(postcss@8.5.15)(typescript@5.5.4)(webpack@5.102.1(@swc/core@1.3.26)(esbuild@0.15.18)): + postcss-loader@8.1.1(postcss@8.5.15)(typescript@6.0.3)(webpack@5.102.1(@swc/core@1.3.26)(esbuild@0.15.18)): dependencies: - cosmiconfig: 9.0.0(typescript@5.5.4) + cosmiconfig: 9.0.0(typescript@6.0.3) jiti: 1.21.0 postcss: 8.5.15 semver: 7.6.3 @@ -28853,12 +28867,12 @@ snapshots: clsx: 2.1.1 react: 18.3.1 - prisma@6.14.0(magicast@0.3.5)(typescript@5.5.4): + prisma@6.14.0(magicast@0.3.5)(typescript@6.0.3): dependencies: '@prisma/config': 6.14.0(magicast@0.3.5) '@prisma/engines': 6.14.0 optionalDependencies: - typescript: 5.5.4 + typescript: 6.0.3 transitivePeerDependencies: - magicast @@ -29494,54 +29508,54 @@ snapshots: remend@1.3.0: {} - remix-auth-email-link@2.0.2(@remix-run/server-runtime@2.17.5(typescript@5.5.4))(remix-auth@3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/server-runtime@2.17.5(typescript@5.5.4))): + remix-auth-email-link@2.0.2(@remix-run/server-runtime@2.17.5(typescript@6.0.3))(remix-auth@3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/server-runtime@2.17.5(typescript@6.0.3))): dependencies: - '@remix-run/server-runtime': 2.17.5(typescript@5.5.4) + '@remix-run/server-runtime': 2.17.5(typescript@6.0.3) crypto-js: 4.1.1 - remix-auth: 3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/server-runtime@2.17.5(typescript@5.5.4)) + remix-auth: 3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/server-runtime@2.17.5(typescript@6.0.3)) - remix-auth-github@1.6.0(@remix-run/server-runtime@2.17.5(typescript@5.5.4))(remix-auth@3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/server-runtime@2.17.5(typescript@5.5.4))): + remix-auth-github@1.6.0(@remix-run/server-runtime@2.17.5(typescript@6.0.3))(remix-auth@3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/server-runtime@2.17.5(typescript@6.0.3))): dependencies: - '@remix-run/server-runtime': 2.17.5(typescript@5.5.4) - remix-auth: 3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/server-runtime@2.17.5(typescript@5.5.4)) - remix-auth-oauth2: 1.11.0(@remix-run/server-runtime@2.17.5(typescript@5.5.4))(remix-auth@3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/server-runtime@2.17.5(typescript@5.5.4))) + '@remix-run/server-runtime': 2.17.5(typescript@6.0.3) + remix-auth: 3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/server-runtime@2.17.5(typescript@6.0.3)) + remix-auth-oauth2: 1.11.0(@remix-run/server-runtime@2.17.5(typescript@6.0.3))(remix-auth@3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/server-runtime@2.17.5(typescript@6.0.3))) transitivePeerDependencies: - supports-color - remix-auth-google@2.0.0(@remix-run/server-runtime@2.17.5(typescript@5.5.4))(remix-auth@3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/server-runtime@2.17.5(typescript@5.5.4))): + remix-auth-google@2.0.0(@remix-run/server-runtime@2.17.5(typescript@6.0.3))(remix-auth@3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/server-runtime@2.17.5(typescript@6.0.3))): dependencies: - '@remix-run/server-runtime': 2.17.5(typescript@5.5.4) - remix-auth: 3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/server-runtime@2.17.5(typescript@5.5.4)) - remix-auth-oauth2: 1.11.0(@remix-run/server-runtime@2.17.5(typescript@5.5.4))(remix-auth@3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/server-runtime@2.17.5(typescript@5.5.4))) + '@remix-run/server-runtime': 2.17.5(typescript@6.0.3) + remix-auth: 3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/server-runtime@2.17.5(typescript@6.0.3)) + remix-auth-oauth2: 1.11.0(@remix-run/server-runtime@2.17.5(typescript@6.0.3))(remix-auth@3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/server-runtime@2.17.5(typescript@6.0.3))) transitivePeerDependencies: - supports-color - remix-auth-oauth2@1.11.0(@remix-run/server-runtime@2.17.5(typescript@5.5.4))(remix-auth@3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/server-runtime@2.17.5(typescript@5.5.4))): + remix-auth-oauth2@1.11.0(@remix-run/server-runtime@2.17.5(typescript@6.0.3))(remix-auth@3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/server-runtime@2.17.5(typescript@6.0.3))): dependencies: - '@remix-run/server-runtime': 2.17.5(typescript@5.5.4) + '@remix-run/server-runtime': 2.17.5(typescript@6.0.3) debug: 4.4.3(supports-color@10.0.0) - remix-auth: 3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/server-runtime@2.17.5(typescript@5.5.4)) + remix-auth: 3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/server-runtime@2.17.5(typescript@6.0.3)) transitivePeerDependencies: - supports-color - remix-auth@3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/server-runtime@2.17.5(typescript@5.5.4)): + remix-auth@3.6.0(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/server-runtime@2.17.5(typescript@6.0.3)): dependencies: - '@remix-run/react': 2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4) - '@remix-run/server-runtime': 2.17.5(typescript@5.5.4) + '@remix-run/react': 2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3) + '@remix-run/server-runtime': 2.17.5(typescript@6.0.3) uuid: 8.3.2 - remix-typedjson@0.3.1(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/server-runtime@2.17.5(typescript@5.5.4))(react@18.3.1): + remix-typedjson@0.3.1(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/server-runtime@2.17.5(typescript@6.0.3))(react@18.3.1): dependencies: - '@remix-run/react': 2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4) - '@remix-run/server-runtime': 2.17.5(typescript@5.5.4) + '@remix-run/react': 2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3) + '@remix-run/server-runtime': 2.17.5(typescript@6.0.3) react: 18.3.1 - remix-utils@7.7.0(@remix-run/node@2.17.5(typescript@5.5.4))(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4))(@remix-run/router@1.23.3(patch_hash=33e2966e9ef36aa09955cec9922c5063227b83429157a71ba226369a08627ac9))(crypto-js@4.2.0)(intl-parse-accept-language@1.0.0)(react@18.3.1)(zod@3.25.76): + remix-utils@7.7.0(@remix-run/node@2.17.5(typescript@6.0.3))(@remix-run/react@2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3))(@remix-run/router@1.23.3(patch_hash=33e2966e9ef36aa09955cec9922c5063227b83429157a71ba226369a08627ac9))(crypto-js@4.2.0)(intl-parse-accept-language@1.0.0)(react@18.3.1)(zod@3.25.76): dependencies: type-fest: 4.33.0 optionalDependencies: - '@remix-run/node': 2.17.5(typescript@5.5.4) - '@remix-run/react': 2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.5.4) + '@remix-run/node': 2.17.5(typescript@6.0.3) + '@remix-run/react': 2.17.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@6.0.3) '@remix-run/router': 1.23.3(patch_hash=33e2966e9ef36aa09955cec9922c5063227b83429157a71ba226369a08627ac9) crypto-js: 4.2.0 intl-parse-accept-language: 1.0.0 @@ -30673,11 +30687,9 @@ snapshots: ts-easing@0.2.0: {} - ts-essentials@10.0.1(typescript@5.5.4): + ts-essentials@10.0.1(typescript@6.0.3): optionalDependencies: - typescript: 5.5.4 - - ts-expose-internals-conditionally@1.0.0-empty.0: {} + typescript: 6.0.3 ts-interface-checker@0.1.13: {} @@ -30701,13 +30713,13 @@ snapshots: tsafe@1.4.1: {} - tsconfck@2.1.2(typescript@5.5.4): + tsconfck@2.1.2(typescript@6.0.3): optionalDependencies: - typescript: 5.5.4 + typescript: 6.0.3 - tsconfck@3.1.3(typescript@5.5.4): + tsconfck@3.1.3(typescript@6.0.3): optionalDependencies: - typescript: 5.5.4 + typescript: 6.0.3 tsconfig-paths@3.14.1: dependencies: @@ -30733,7 +30745,7 @@ snapshots: resolve-import: 2.0.0 rimraf: 6.0.1 sync-content: 2.0.1 - typescript: 5.5.4 + typescript: 6.0.3 walk-up-path: 4.0.0 tslib@2.4.1: {} @@ -30744,7 +30756,7 @@ snapshots: tslib@2.8.1: {} - tsup@8.4.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.7.0)(postcss@8.5.15)(tsx@4.17.0)(typescript@5.5.4)(yaml@2.9.0): + tsup@8.4.0(patch_hash=bb8871e353e0582d16a3fdef093af6d43be84f0957b610d51e43cc2bc80add27)(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.7.0)(postcss@8.5.15)(tsx@4.17.0)(typescript@6.0.3)(yaml@2.9.0): dependencies: bundle-require: 5.1.0(esbuild@0.25.1) cac: 6.7.14 @@ -30765,14 +30777,14 @@ snapshots: optionalDependencies: '@swc/core': 1.3.101(@swc/helpers@0.5.15) postcss: 8.5.15 - typescript: 5.5.4 + typescript: 6.0.3 transitivePeerDependencies: - jiti - supports-color - tsx - yaml - tsup@8.4.0(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.7.0)(postcss@8.5.15)(tsx@4.22.4)(typescript@5.5.4)(yaml@2.9.0): + tsup@8.4.0(patch_hash=bb8871e353e0582d16a3fdef093af6d43be84f0957b610d51e43cc2bc80add27)(@swc/core@1.3.101(@swc/helpers@0.5.15))(jiti@2.7.0)(postcss@8.5.15)(tsx@4.22.4)(typescript@6.0.3)(yaml@2.9.0): dependencies: bundle-require: 5.1.0(esbuild@0.25.1) cac: 6.7.14 @@ -30793,7 +30805,7 @@ snapshots: optionalDependencies: '@swc/core': 1.3.101(@swc/helpers@0.5.15) postcss: 8.5.15 - typescript: 5.5.4 + typescript: 6.0.3 transitivePeerDependencies: - jiti - supports-color @@ -30949,7 +30961,7 @@ snapshots: optionalDependencies: rxjs: 7.8.2 - typescript@5.5.4: {} + typescript@6.0.3: {} ufo@1.5.4: {} @@ -31160,13 +31172,13 @@ snapshots: kleur: 4.1.5 sade: 1.8.1 - valibot@1.1.0(typescript@5.5.4): + valibot@1.1.0(typescript@6.0.3): optionalDependencies: - typescript: 5.5.4 + typescript: 6.0.3 - valibot@1.3.1(typescript@5.5.4): + valibot@1.3.1(typescript@6.0.3): optionalDependencies: - typescript: 5.5.4 + typescript: 6.0.3 validate-npm-package-license@3.0.4: dependencies: @@ -31277,11 +31289,11 @@ snapshots: - tsx - yaml - vite-tsconfig-paths@4.0.5(typescript@5.5.4): + vite-tsconfig-paths@4.0.5(typescript@6.0.3): dependencies: debug: 4.3.7(supports-color@10.0.0) globrex: 0.1.2 - tsconfck: 2.1.2(typescript@5.5.4) + tsconfck: 2.1.2(typescript@6.0.3) transitivePeerDependencies: - supports-color - typescript diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index d083a589e..66a0c4b49 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -5,6 +5,9 @@ packages: - "docs" - "!packages/cli-v3/e2e/**" +catalog: + typescript: 6.0.3 + minimumReleaseAge: 4320 minimumReleaseAgeExclude: - "@trigger.dev/platform" From d05f1a739814b3ef712757e1c114aa679d718e20 Mon Sep 17 00:00:00 2001 From: Katia Bulatova Date: Tue, 21 Jul 2026 15:57:13 +0200 Subject: [PATCH 011/300] chore(webapp): migrate from Remix compiler to Vite (#4188) Replaces Remix compiler with the Vite plugin. The Express server (cluster, socket.io, ws) and the Docker image contract are unchanged. --- .changeset/vite-ignore-optional-imports.md | 5 + .../app/components/StaleAssetRecovery.tsx | 10 +- .../app/components/primitives/Avatar.tsx | 3 +- .../presenters/v3/UsagePresenter.server.ts | 5 +- apps/webapp/app/root.tsx | 8 +- .../routes/api.v1.errors.$errorId.ignore.ts | 5 +- .../routes/api.v1.errors.$errorId.resolve.ts | 5 +- .../api.v1.errors.$errorId.unresolve.ts | 5 +- .../api.v1.idempotencyKeys.$key.reset.ts | 6 +- .../routes/api.v1.orgs.$orgParam.projects.ts | 3 +- ...queues.$queueParam.concurrency.override.ts | 6 +- ...v1.queues.$queueParam.concurrency.reset.ts | 6 +- .../routes/api.v1.queues.$queueParam.pause.ts | 6 +- .../app/routes/api.v1.runs.$runId.metadata.ts | 115 +--------------- ...api.v1.sessions.$sessionId.snapshot-url.ts | 4 +- .../app/routes/auth.github.callback.tsx | 4 +- apps/webapp/app/routes/auth.github.ts | 13 +- .../app/routes/auth.google.callback.tsx | 4 +- apps/webapp/app/routes/auth.google.ts | 13 +- .../app/services/redirectCookies.server.ts | 19 +++ apps/webapp/app/tailwind.css | 3 - .../app/utils/reloadingRegistry.server.ts | 49 +++---- .../mollifier/routeOperationsToRun.server.ts | 116 +++++++++++++++++ apps/webapp/app/v3/querySchemas.ts | 6 +- apps/webapp/package.json | 12 +- apps/webapp/remix.config.js | 48 ------- apps/webapp/remix.env.d.ts | 5 + apps/webapp/server.ts | 76 +++++++---- .../metadataRouteOperationsLogging.test.ts | 2 +- apps/webapp/vite.config.ts | 66 ++++++++++ apps/webapp/vite/node-globals-shim.js | 10 ++ docker/Dockerfile | 6 + docker/docker-compose.yml | 8 +- internal-packages/rbac/src/index.ts | 3 +- .../run-engine/src/engine/locking.ts | 24 ++-- internal-packages/sso/src/index.ts | 3 +- .../trigger-sdk/src/v3/aiAutoTelemetry.ts | 6 +- pnpm-lock.yaml | 123 ++++++++++++------ 38 files changed, 489 insertions(+), 322 deletions(-) create mode 100644 .changeset/vite-ignore-optional-imports.md create mode 100644 apps/webapp/app/services/redirectCookies.server.ts create mode 100644 apps/webapp/app/v3/mollifier/routeOperationsToRun.server.ts delete mode 100644 apps/webapp/remix.config.js create mode 100644 apps/webapp/vite.config.ts create mode 100644 apps/webapp/vite/node-globals-shim.js diff --git a/.changeset/vite-ignore-optional-imports.md b/.changeset/vite-ignore-optional-imports.md new file mode 100644 index 000000000..286f80132 --- /dev/null +++ b/.changeset/vite-ignore-optional-imports.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/sdk": patch +--- + +Suppress a build-time warning that could appear in Vite-based projects when the optional `@ai-sdk/otel` package is not installed. diff --git a/apps/webapp/app/components/StaleAssetRecovery.tsx b/apps/webapp/app/components/StaleAssetRecovery.tsx index 7827d9ac8..32b792392 100644 --- a/apps/webapp/app/components/StaleAssetRecovery.tsx +++ b/apps/webapp/app/components/StaleAssetRecovery.tsx @@ -1,7 +1,7 @@ -// Recovers from a rolling deploy rotating the content-hashed /build assets out from +// Recovers from a rolling deploy rotating the content-hashed /assets files out from // under a page. Each image serves only its own build and hard-404s unknown hashes, so // a client can request a hash the serving replica doesn't have and get missing styles -// or a failed asset load. On such a /build load failure we do a bounded full document +// or a failed asset load. On such an asset load failure we do a bounded full document // reload: the fresh document (and, under sticky routing, all of its assets) lands on a // single live build, so the asset resolves. Bounded via sessionStorage so it can never // loop; when the budget is spent it stops rather than reloading forever. @@ -39,7 +39,7 @@ export function staleAssetRecoveryScript() { } function recover() { - // One recovery per page: a broken load fails several /build assets at once and each + // One recovery per page: a broken load fails several hashed assets at once and each // fires its own error event before location.reload() commits — without this guard a // single incident would burn the entire reload budget. if (recovering) return; @@ -62,7 +62,9 @@ export function staleAssetRecoveryScript() { : el.tagName === "SCRIPT" ? (el as HTMLScriptElement).src : null; - if (url && url.indexOf("/build/") !== -1) recover(); + // Match the pathname, not the full URL — a query string or third-party + // URL containing /assets/ must not burn the reload budget. + if (url && new URL(url, location.href).pathname.indexOf("/assets/") !== -1) recover(); }, true ); diff --git a/apps/webapp/app/components/primitives/Avatar.tsx b/apps/webapp/app/components/primitives/Avatar.tsx index 0d000a9a3..9fc6832fc 100644 --- a/apps/webapp/app/components/primitives/Avatar.tsx +++ b/apps/webapp/app/components/primitives/Avatar.tsx @@ -10,7 +10,6 @@ import { } from "@heroicons/react/20/solid"; import type { Prisma } from "@trigger.dev/database"; import { z } from "zod"; -import { logger } from "~/services/logger.server"; import { cn } from "~/utils/cn"; export const AvatarType = z.enum(["icon", "letters", "image"]); @@ -45,7 +44,7 @@ export function parseAvatar(json: Prisma.JsonValue, defaultAvatar: Avatar): Avat const parsed = AvatarData.safeParse(json); if (!parsed.success) { - logger.error("Invalid org avatar", { json, error: parsed.error }); + console.error("Invalid org avatar", { json, error: parsed.error }); return defaultAvatar; } diff --git a/apps/webapp/app/presenters/v3/UsagePresenter.server.ts b/apps/webapp/app/presenters/v3/UsagePresenter.server.ts index fe5ab1ec3..a3b96cf6a 100644 --- a/apps/webapp/app/presenters/v3/UsagePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/UsagePresenter.server.ts @@ -1,5 +1,8 @@ import type { DataPoint } from "regression"; -import { linear } from "regression"; +// Default-import: regression is CJS and its named exports aren't statically +// analyzable under ESM interop. +import regression from "regression"; +const { linear } = regression; import type { PrismaClientOrTransaction } from "~/db.server"; import { env } from "~/env.server"; import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; diff --git a/apps/webapp/app/root.tsx b/apps/webapp/app/root.tsx index f7c89258a..1c3ea631b 100644 --- a/apps/webapp/app/root.tsx +++ b/apps/webapp/app/root.tsx @@ -1,11 +1,14 @@ import type { LinksFunction, LoaderFunctionArgs, MetaFunction } from "@remix-run/node"; import type { ShouldRevalidateFunction } from "@remix-run/react"; -import { Links, LiveReload, Meta, Outlet, Scripts, ScrollRestoration } from "@remix-run/react"; +import { Links, Meta, Outlet, Scripts, ScrollRestoration } from "@remix-run/react"; import { type UseDataFunctionReturn, typedjson, useTypedLoaderData } from "remix-typedjson"; import { ExternalScripts } from "remix-utils/external-scripts"; import type { ToastMessage } from "~/models/message.server"; import { commitSession, getSession } from "~/models/message.server"; -import tailwindStylesheetUrl from "~/tailwind.css"; +// Fonts imported here so Vite rebases the urls and emits the woff2 assets +import "non.geist"; +import "non.geist/mono"; +import tailwindStylesheetUrl from "~/tailwind.css?url"; import { RouteErrorDisplay } from "./components/ErrorDisplay"; import { StaleAssetRecovery } from "./components/StaleAssetRecovery"; import { AppContainer, MainCenteredContainer } from "./components/layout/AppLayout"; @@ -145,7 +148,6 @@ export default function App() { - diff --git a/apps/webapp/app/routes/api.v1.errors.$errorId.ignore.ts b/apps/webapp/app/routes/api.v1.errors.$errorId.ignore.ts index 4b0ad2ab6..9aba43042 100644 --- a/apps/webapp/app/routes/api.v1.errors.$errorId.ignore.ts +++ b/apps/webapp/app/routes/api.v1.errors.$errorId.ignore.ts @@ -9,7 +9,7 @@ const ParamsSchema = z.object({ errorId: z.string(), }); -export const { action, loader } = createActionApiRoute( +const route = createActionApiRoute( { params: ParamsSchema, body: IgnoreErrorRequestBody, @@ -56,3 +56,6 @@ export const { action, loader } = createActionApiRoute( return json(updated); } ); + +export const action = route.action; +export const loader = route.loader; diff --git a/apps/webapp/app/routes/api.v1.errors.$errorId.resolve.ts b/apps/webapp/app/routes/api.v1.errors.$errorId.resolve.ts index e0818c89f..68d4a094a 100644 --- a/apps/webapp/app/routes/api.v1.errors.$errorId.resolve.ts +++ b/apps/webapp/app/routes/api.v1.errors.$errorId.resolve.ts @@ -9,7 +9,7 @@ const ParamsSchema = z.object({ errorId: z.string(), }); -export const { action, loader } = createActionApiRoute( +const route = createActionApiRoute( { params: ParamsSchema, body: ResolveErrorRequestBody, @@ -48,3 +48,6 @@ export const { action, loader } = createActionApiRoute( return json(updated); } ); + +export const action = route.action; +export const loader = route.loader; diff --git a/apps/webapp/app/routes/api.v1.errors.$errorId.unresolve.ts b/apps/webapp/app/routes/api.v1.errors.$errorId.unresolve.ts index 9362b7c4c..ba16118c8 100644 --- a/apps/webapp/app/routes/api.v1.errors.$errorId.unresolve.ts +++ b/apps/webapp/app/routes/api.v1.errors.$errorId.unresolve.ts @@ -8,7 +8,7 @@ const ParamsSchema = z.object({ errorId: z.string(), }); -export const { action, loader } = createActionApiRoute( +const route = createActionApiRoute( { params: ParamsSchema, method: "POST", @@ -40,3 +40,6 @@ export const { action, loader } = createActionApiRoute( return json(updated); } ); + +export const action = route.action; +export const loader = route.loader; diff --git a/apps/webapp/app/routes/api.v1.idempotencyKeys.$key.reset.ts b/apps/webapp/app/routes/api.v1.idempotencyKeys.$key.reset.ts index feec6dd65..2943c21cc 100644 --- a/apps/webapp/app/routes/api.v1.idempotencyKeys.$key.reset.ts +++ b/apps/webapp/app/routes/api.v1.idempotencyKeys.$key.reset.ts @@ -13,7 +13,7 @@ const BodySchema = z.object({ taskIdentifier: z.string().min(1, "Task identifier is required"), }); -export const { action } = createActionApiRoute( +const route = createActionApiRoute( { params: ParamsSchema, body: BodySchema, @@ -50,3 +50,7 @@ export const { action } = createActionApiRoute( } } ); + +export const action = route.action; +// The builder's loader handles CORS OPTIONS preflight +export const loader = route.loader; diff --git a/apps/webapp/app/routes/api.v1.orgs.$orgParam.projects.ts b/apps/webapp/app/routes/api.v1.orgs.$orgParam.projects.ts index f20bf045c..844c1e6da 100644 --- a/apps/webapp/app/routes/api.v1.orgs.$orgParam.projects.ts +++ b/apps/webapp/app/routes/api.v1.orgs.$orgParam.projects.ts @@ -10,7 +10,8 @@ import { createLoaderPATApiRoute, } from "~/services/routeBuilders/apiBuilder.server"; import { ServiceValidationError } from "~/v3/services/common.server"; -import { isCuid } from "cuid"; +import cuid from "cuid"; +const { isCuid } = cuid; const ParamsSchema = z.object({ orgParam: z.string(), diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts index 3223a2a60..3296e7fa7 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.override.ts @@ -10,7 +10,7 @@ const BodySchema = z.object({ concurrencyLimit: z.number().int().min(0).max(100000), }); -export const { action } = createActionApiRoute( +const route = createActionApiRoute( { body: BodySchema, params: z.object({ @@ -73,3 +73,7 @@ export const { action } = createActionApiRoute( ); } ); + +export const action = route.action; +// The builder's loader answers non-POST methods with a 405 +export const loader = route.loader; diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts index dbeea591a..9a83f3ed8 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.concurrency.reset.ts @@ -9,7 +9,7 @@ const BodySchema = z.object({ type: RetrieveQueueType.default("id"), }); -export const { action } = createActionApiRoute( +const route = createActionApiRoute( { body: BodySchema, params: z.object({ @@ -73,3 +73,7 @@ export const { action } = createActionApiRoute( ); } ); + +export const action = route.action; +// The builder's loader answers non-POST methods with a 405 +export const loader = route.loader; diff --git a/apps/webapp/app/routes/api.v1.queues.$queueParam.pause.ts b/apps/webapp/app/routes/api.v1.queues.$queueParam.pause.ts index 452bd8174..a0f910fe5 100644 --- a/apps/webapp/app/routes/api.v1.queues.$queueParam.pause.ts +++ b/apps/webapp/app/routes/api.v1.queues.$queueParam.pause.ts @@ -9,7 +9,7 @@ const BodySchema = z.object({ action: z.enum(["pause", "resume"]), }); -export const { action } = createActionApiRoute( +const route = createActionApiRoute( { body: BodySchema, params: z.object({ @@ -44,3 +44,7 @@ export const { action } = createActionApiRoute( return json(q); } ); + +export const action = route.action; +// The builder's loader answers non-POST methods with a 405 +export const loader = route.loader; diff --git a/apps/webapp/app/routes/api.v1.runs.$runId.metadata.ts b/apps/webapp/app/routes/api.v1.runs.$runId.metadata.ts index ac26302f8..ef0aded2f 100644 --- a/apps/webapp/app/routes/api.v1.runs.$runId.metadata.ts +++ b/apps/webapp/app/routes/api.v1.runs.$runId.metadata.ts @@ -1,21 +1,18 @@ import type { LoaderFunctionArgs } from "@remix-run/server-runtime"; import { json } from "@remix-run/server-runtime"; import { tryCatch } from "@trigger.dev/core/utils"; -import type { RunMetadataChangeOperation } from "@trigger.dev/core/v3/schemas"; import { UpdateMetadataRequestBody } from "@trigger.dev/core/v3"; import { z } from "zod"; import { $replica } from "~/db.server"; -// Aliased to avoid shadowing the local `env: AuthenticatedEnvironment` -// parameter the route handler and `routeOperationsToRun` use. +// Aliased to avoid shadowing the local `env` parameter in the handler. import { env as appEnv } from "~/env.server"; -import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { authenticateApiRequest } from "~/services/apiAuth.server"; -import { logger } from "~/services/logger.server"; import { updateMetadataService } from "~/services/metadata/updateMetadataInstance.server"; import { publishChangeRecord } from "~/services/realtime/runChangeNotifierInstance.server"; import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; import { ServiceValidationError } from "~/v3/services/common.server"; import { applyMetadataMutationToBufferedRun } from "~/v3/mollifier/applyMetadataMutation.server"; +import { routeOperationsToRun } from "~/v3/mollifier/routeOperationsToRun.server"; import { findRunByIdWithMollifierFallback } from "~/v3/mollifier/readFallback.server"; import { runStore } from "~/v3/runStore.server"; @@ -80,114 +77,6 @@ export async function loader({ request, params }: LoaderFunctionArgs) { return json({ error: "Run not found" }, { status: 404 }); } -// Route parent/root operations to the existing PG service by directly -// invoking it against the parent/root runId. The service ingests via -// its batching worker, which targets PG by id. If the parent/root is -// itself buffered we recurse through our buffered-mutation helper. -// `_ingestion_only` flag: a synthetic body that has the operations -// promoted to top-level `operations` so the service applies them to -// `targetRunId` directly. -// Exported so the silent-failure logging behaviour can be unit-tested. -// The route handler itself isn't an attractive test target (createActionApiRoute -// wraps it in auth + body parsing + error-handler middleware), but the -// fan-out helper carries the load-bearing logic — including the ops- -// visibility branch this change adds. -export async function routeOperationsToRun( - targetRunId: string | undefined, - operations: RunMetadataChangeOperation[] | undefined, - env: AuthenticatedEnvironment -): Promise { - if (!targetRunId || !operations || operations.length === 0) return; - - // Try PG first via the existing service (this is how parent/root - // operations have always landed; preserve that). Accepts the full - // AuthenticatedEnvironment so we don't have to recover the unsafe - // `as unknown` cast that the previous narrowed `{ id, organizationId }` - // signature forced on us. - // - // Two non-success outcomes from `call`: - // * throws — PG threw (e.g. "Cannot update metadata for a completed - // run", or a transient PG outage). - // * resolves with undefined — PG row didn't exist (the target may be - // buffered, not yet materialised). - // Either way we want to try the buffer fallback below; treating the - // undefined-return as success would make the fallback unreachable. - const [error, result] = await tryCatch( - updateMetadataService.call(targetRunId, { operations }, env) - ); - if (!error && result !== undefined) { - // The parent/root run changed too — wake its live feeds (only when something was - // actually written here; buffered writes publish from the flusher). - if (result.updatedAtMs !== undefined) { - publishChangeRecord({ - runId: result.runId, - envId: env.id, - tags: result.runTags, - batchId: result.batchId, - updatedAtMs: result.updatedAtMs, - }); - } - return; - } - - if (error) { - // PG threw — auxiliary op, stay best-effort and don't surface this - // to the caller (the caller's primary mutation already landed). But - // warn so a genuine PG outage on these ops isn't invisible. - logger.warn("metadata route: parent/root PG op failed", { - targetRunId, - error: error instanceof Error ? error.message : String(error), - }); - } - - // Buffer fallback only makes sense for friendlyId-keyed entries. The - // PG-side parent/root IDs are internal cuids; the buffer keys entries - // by friendlyId, so passing the internal id would silently no-op. - // Skip explicitly — a buffered child's parent is always materialised - // in PG already (a buffered run hasn't executed, so it can't have - // triggered the child), so the buffered-parent branch isn't actually - // reachable. Treating the no-op as intentional rather than incidental. - if (!targetRunId.startsWith("run_")) return; - - // Best-effort buffer fallback. Wrap so a transient Redis throw on - // this auxiliary op can't 500 the request after the primary mutation - // already succeeded. - const [bufferError, bufferOutcome] = await tryCatch( - applyMetadataMutationToBufferedRun({ - runId: targetRunId, - environmentId: env.id, - organizationId: env.organizationId, - maximumSize: appEnv.TASK_RUN_METADATA_MAXIMUM_SIZE, - maxRetries: appEnv.TRIGGER_MOLLIFIER_METADATA_MAX_RETRIES, - backoffBaseMs: appEnv.TRIGGER_MOLLIFIER_METADATA_BACKOFF_BASE_MS, - backoffStepMs: appEnv.TRIGGER_MOLLIFIER_METADATA_BACKOFF_STEP_MS, - body: { operations }, - }) - ); - if (bufferError) { - logger.warn("metadata route: buffer fallback for parent/root op failed", { - targetRunId, - error: bufferError instanceof Error ? bufferError.message : String(bufferError), - }); - return; - } - // `applyMetadataMutationToBufferedRun` reports non-throw failures via - // its returned outcome kind: `not_found`, `busy`, `version_exhausted`, - // `metadata_too_large`. Without inspecting `.kind`, the parent/root - // operation can silently disappear — no PG row landed it (handled - // above) and the buffer rejected it for one of these reasons but the - // helper returned cleanly. Surface a warn log per non-success branch - // so ops can trace why a parent/root op went missing. The customer's - // primary mutation has already succeeded by this point; this remains - // best-effort, so we still don't bubble these to the response. - if (bufferOutcome && bufferOutcome.kind !== "applied") { - logger.warn("metadata route: parent/root buffer op did not apply", { - targetRunId, - kind: bufferOutcome.kind, - }); - } -} - const { action } = createActionApiRoute( { params: ParamsSchema, diff --git a/apps/webapp/app/routes/api.v1.sessions.$sessionId.snapshot-url.ts b/apps/webapp/app/routes/api.v1.sessions.$sessionId.snapshot-url.ts index ba70f08da..bfe609b7a 100644 --- a/apps/webapp/app/routes/api.v1.sessions.$sessionId.snapshot-url.ts +++ b/apps/webapp/app/routes/api.v1.sessions.$sessionId.snapshot-url.ts @@ -40,7 +40,7 @@ function sessionResource( return anyResource([...ids].map((id) => ({ type: "sessions" as const, id }))); } -export const { action } = createActionApiRoute( +const route = createActionApiRoute( { ...routeConfig, method: "PUT", @@ -94,3 +94,5 @@ export const loader = createLoaderApiRoute( return json({ presignedUrl: signed.url }); } ); + +export const action = route.action; diff --git a/apps/webapp/app/routes/auth.github.callback.tsx b/apps/webapp/app/routes/auth.github.callback.tsx index 32c23ab66..28d5deb70 100644 --- a/apps/webapp/app/routes/auth.github.callback.tsx +++ b/apps/webapp/app/routes/auth.github.callback.tsx @@ -9,12 +9,12 @@ import { commitAuthenticatedSession } from "~/services/sessionDuration.server"; import { trackAndClearReferralSource } from "~/services/referralSource.server"; import { appendRedirectTo, ssoRedirectFromAuthError } from "~/services/ssoAutoDiscovery.server"; import type { AuthUser } from "~/services/authUser"; -import { redirectCookie } from "./auth.github"; +import { githubRedirectCookie } from "~/services/redirectCookies.server"; import { sanitizeRedirectPath } from "~/utils"; export let loader: LoaderFunction = async ({ request }) => { const cookie = request.headers.get("Cookie"); - const redirectValue = await redirectCookie.parse(cookie); + const redirectValue = await githubRedirectCookie.parse(cookie); const redirectTo = sanitizeRedirectPath(redirectValue); // The SSO auto-discovery gate runs inside the strategy's verify diff --git a/apps/webapp/app/routes/auth.github.ts b/apps/webapp/app/routes/auth.github.ts index 8c464e0e5..04bd5fe12 100644 --- a/apps/webapp/app/routes/auth.github.ts +++ b/apps/webapp/app/routes/auth.github.ts @@ -1,6 +1,6 @@ -import { type ActionFunction, type LoaderFunction, redirect, createCookie } from "@remix-run/node"; +import { type ActionFunction, type LoaderFunction, redirect } from "@remix-run/node"; import { authenticator } from "~/services/auth.server"; -import { env } from "~/env.server"; +import { githubRedirectCookie } from "~/services/redirectCookies.server"; import { sanitizeRedirectPath } from "~/utils"; export let loader: LoaderFunction = () => redirect("/login"); @@ -23,15 +23,8 @@ export let action: ActionFunction = async ({ request }) => { if (error instanceof Response) { // we need to append a Set-Cookie header with a cookie storing the // returnTo value (store the sanitized path) - error.headers.append("Set-Cookie", await redirectCookie.serialize(safeRedirect)); + error.headers.append("Set-Cookie", await githubRedirectCookie.serialize(safeRedirect)); } throw error; } }; - -export const redirectCookie = createCookie("redirect-to", { - maxAge: 60 * 60, // 1 hour - httpOnly: true, - sameSite: "lax", - secure: env.NODE_ENV === "production", -}); diff --git a/apps/webapp/app/routes/auth.google.callback.tsx b/apps/webapp/app/routes/auth.google.callback.tsx index 593b9d40f..e32dd4338 100644 --- a/apps/webapp/app/routes/auth.google.callback.tsx +++ b/apps/webapp/app/routes/auth.google.callback.tsx @@ -9,12 +9,12 @@ import { commitAuthenticatedSession } from "~/services/sessionDuration.server"; import { trackAndClearReferralSource } from "~/services/referralSource.server"; import { appendRedirectTo, ssoRedirectFromAuthError } from "~/services/ssoAutoDiscovery.server"; import type { AuthUser } from "~/services/authUser"; -import { redirectCookie } from "./auth.google"; +import { googleRedirectCookie } from "~/services/redirectCookies.server"; import { sanitizeRedirectPath } from "~/utils"; export let loader: LoaderFunction = async ({ request }) => { const cookie = request.headers.get("Cookie"); - const redirectValue = await redirectCookie.parse(cookie); + const redirectValue = await googleRedirectCookie.parse(cookie); const redirectTo = sanitizeRedirectPath(redirectValue); // The SSO auto-discovery gate runs inside the strategy's verify diff --git a/apps/webapp/app/routes/auth.google.ts b/apps/webapp/app/routes/auth.google.ts index 95fb4ff7b..09ab022bd 100644 --- a/apps/webapp/app/routes/auth.google.ts +++ b/apps/webapp/app/routes/auth.google.ts @@ -1,6 +1,6 @@ -import { type ActionFunction, type LoaderFunction, redirect, createCookie } from "@remix-run/node"; +import { type ActionFunction, type LoaderFunction, redirect } from "@remix-run/node"; import { authenticator } from "~/services/auth.server"; -import { env } from "~/env.server"; +import { googleRedirectCookie } from "~/services/redirectCookies.server"; import { sanitizeRedirectPath } from "~/utils"; export let loader: LoaderFunction = () => redirect("/login"); @@ -23,15 +23,8 @@ export let action: ActionFunction = async ({ request }) => { if (error instanceof Response) { // we need to append a Set-Cookie header with a cookie storing the // returnTo value (store the sanitized path) - error.headers.append("Set-Cookie", await redirectCookie.serialize(safeRedirect)); + error.headers.append("Set-Cookie", await googleRedirectCookie.serialize(safeRedirect)); } throw error; } }; - -export const redirectCookie = createCookie("google-redirect-to", { - maxAge: 60 * 60, // 1 hour - httpOnly: true, - sameSite: "lax", - secure: env.NODE_ENV === "production", -}); diff --git a/apps/webapp/app/services/redirectCookies.server.ts b/apps/webapp/app/services/redirectCookies.server.ts new file mode 100644 index 000000000..58cecbc33 --- /dev/null +++ b/apps/webapp/app/services/redirectCookies.server.ts @@ -0,0 +1,19 @@ +import { createCookie } from "@remix-run/node"; +import { env } from "~/env.server"; + +// Post-auth redirect cookies. Kept in a .server module: Vite can't strip +// non-standard route exports that pull in server-only code. + +export const githubRedirectCookie = createCookie("redirect-to", { + maxAge: 60 * 60, // 1 hour + httpOnly: true, + sameSite: "lax", + secure: env.NODE_ENV === "production", +}); + +export const googleRedirectCookie = createCookie("google-redirect-to", { + maxAge: 60 * 60, // 1 hour + httpOnly: true, + sameSite: "lax", + secure: env.NODE_ENV === "production", +}); diff --git a/apps/webapp/app/tailwind.css b/apps/webapp/app/tailwind.css index f7b05430c..5a6429fce 100644 --- a/apps/webapp/app/tailwind.css +++ b/apps/webapp/app/tailwind.css @@ -1,6 +1,3 @@ -@import url("non.geist"); -@import url("non.geist/mono"); - @import "react-grid-layout/css/styles.css" layer(base); @import "react-resizable/css/styles.css" layer(base); diff --git a/apps/webapp/app/utils/reloadingRegistry.server.ts b/apps/webapp/app/utils/reloadingRegistry.server.ts index 81eb67234..3bbb9b4e8 100644 --- a/apps/webapp/app/utils/reloadingRegistry.server.ts +++ b/apps/webapp/app/utils/reloadingRegistry.server.ts @@ -3,29 +3,34 @@ import { Counter, Gauge } from "prom-client"; import { metricsRegister } from "~/metrics.server"; import { logger } from "~/services/logger.server"; import { signalsEmitter } from "~/services/signals.server"; +import { singleton } from "~/utils/singleton"; -const loadFailures = new Counter({ - name: "reloading_registry_load_failures_total", - help: "Failed loads of a reloading registry", - labelNames: ["name"], - registers: [metricsRegister], -}); - -const lastSuccessfulLoadAt = new Gauge({ - name: "reloading_registry_last_successful_load_timestamp_seconds", - help: "Unix time of the last successful registry load (staleness signal)", - labelNames: ["name"], - registers: [metricsRegister], -}); - -// 0 until the first successful load, then 1. Starts at 0 (not absent) so a -// never-loaded registry is an alertable series, distinct from "feature off". -const registryLoaded = new Gauge({ - name: "reloading_registry_loaded", - help: "1 once the registry has loaded at least once, else 0 (0 = serving cold fallback)", - labelNames: ["name"], - registers: [metricsRegister], -}); +// singleton: module-scope registrations double-register under Vite dev HMR +const { loadFailures, lastSuccessfulLoadAt, registryLoaded } = singleton( + "reloadingRegistryMetrics", + () => ({ + loadFailures: new Counter({ + name: "reloading_registry_load_failures_total", + help: "Failed loads of a reloading registry", + labelNames: ["name"], + registers: [metricsRegister], + }), + lastSuccessfulLoadAt: new Gauge({ + name: "reloading_registry_last_successful_load_timestamp_seconds", + help: "Unix time of the last successful registry load (staleness signal)", + labelNames: ["name"], + registers: [metricsRegister], + }), + // 0 until the first successful load, then 1. Starts at 0 (not absent) so a + // never-loaded registry is an alertable series, distinct from "feature off". + registryLoaded: new Gauge({ + name: "reloading_registry_loaded", + help: "1 once the registry has loaded at least once, else 0 (0 = serving cold fallback)", + labelNames: ["name"], + registers: [metricsRegister], + }), + }) +); export type ReloadingRegistry = { isReady: Promise; diff --git a/apps/webapp/app/v3/mollifier/routeOperationsToRun.server.ts b/apps/webapp/app/v3/mollifier/routeOperationsToRun.server.ts new file mode 100644 index 000000000..387759ff9 --- /dev/null +++ b/apps/webapp/app/v3/mollifier/routeOperationsToRun.server.ts @@ -0,0 +1,116 @@ +import { tryCatch } from "@trigger.dev/core/utils"; +import type { RunMetadataChangeOperation } from "@trigger.dev/core/v3/schemas"; +import { env as appEnv } from "~/env.server"; +import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; +import { logger } from "~/services/logger.server"; +import { updateMetadataService } from "~/services/metadata/updateMetadataInstance.server"; +import { publishChangeRecord } from "~/services/realtime/runChangeNotifierInstance.server"; +import { applyMetadataMutationToBufferedRun } from "./applyMetadataMutation.server"; + +// Route parent/root operations to the existing PG service by directly +// invoking it against the parent/root runId. The service ingests via +// its batching worker, which targets PG by id. If the parent/root is +// itself buffered we recurse through our buffered-mutation helper. +// `_ingestion_only` flag: a synthetic body that has the operations +// promoted to top-level `operations` so the service applies them to +// `targetRunId` directly. +// Exported so the silent-failure logging behaviour can be unit-tested. +// The route handler itself isn't an attractive test target (createActionApiRoute +// wraps it in auth + body parsing + error-handler middleware), but the +// fan-out helper carries the load-bearing logic — including the ops- +// visibility branch this change adds. +export async function routeOperationsToRun( + targetRunId: string | undefined, + operations: RunMetadataChangeOperation[] | undefined, + env: AuthenticatedEnvironment +): Promise { + if (!targetRunId || !operations || operations.length === 0) return; + + // Try PG first via the existing service (this is how parent/root + // operations have always landed; preserve that). Accepts the full + // AuthenticatedEnvironment so we don't have to recover the unsafe + // `as unknown` cast that the previous narrowed `{ id, organizationId }` + // signature forced on us. + // + // Two non-success outcomes from `call`: + // * throws — PG threw (e.g. "Cannot update metadata for a completed + // run", or a transient PG outage). + // * resolves with undefined — PG row didn't exist (the target may be + // buffered, not yet materialised). + // Either way we want to try the buffer fallback below; treating the + // undefined-return as success would make the fallback unreachable. + const [error, result] = await tryCatch( + updateMetadataService.call(targetRunId, { operations }, env) + ); + if (!error && result !== undefined) { + // The parent/root run changed too — wake its live feeds (only when something was + // actually written here; buffered writes publish from the flusher). + if (result.updatedAtMs !== undefined) { + publishChangeRecord({ + runId: result.runId, + envId: env.id, + tags: result.runTags, + batchId: result.batchId, + updatedAtMs: result.updatedAtMs, + }); + } + return; + } + + if (error) { + // PG threw — auxiliary op, stay best-effort and don't surface this + // to the caller (the caller's primary mutation already landed). But + // warn so a genuine PG outage on these ops isn't invisible. + logger.warn("metadata route: parent/root PG op failed", { + targetRunId, + error: error instanceof Error ? error.message : String(error), + }); + } + + // Buffer fallback only makes sense for friendlyId-keyed entries. The + // PG-side parent/root IDs are internal cuids; the buffer keys entries + // by friendlyId, so passing the internal id would silently no-op. + // Skip explicitly — a buffered child's parent is always materialised + // in PG already (a buffered run hasn't executed, so it can't have + // triggered the child), so the buffered-parent branch isn't actually + // reachable. Treating the no-op as intentional rather than incidental. + if (!targetRunId.startsWith("run_")) return; + + // Best-effort buffer fallback. Wrap so a transient Redis throw on + // this auxiliary op can't 500 the request after the primary mutation + // already succeeded. + const [bufferError, bufferOutcome] = await tryCatch( + applyMetadataMutationToBufferedRun({ + runId: targetRunId, + environmentId: env.id, + organizationId: env.organizationId, + maximumSize: appEnv.TASK_RUN_METADATA_MAXIMUM_SIZE, + maxRetries: appEnv.TRIGGER_MOLLIFIER_METADATA_MAX_RETRIES, + backoffBaseMs: appEnv.TRIGGER_MOLLIFIER_METADATA_BACKOFF_BASE_MS, + backoffStepMs: appEnv.TRIGGER_MOLLIFIER_METADATA_BACKOFF_STEP_MS, + body: { operations }, + }) + ); + if (bufferError) { + logger.warn("metadata route: buffer fallback for parent/root op failed", { + targetRunId, + error: bufferError instanceof Error ? bufferError.message : String(bufferError), + }); + return; + } + // `applyMetadataMutationToBufferedRun` reports non-throw failures via + // its returned outcome kind: `not_found`, `busy`, `version_exhausted`, + // `metadata_too_large`. Without inspecting `.kind`, the parent/root + // operation can silently disappear — no PG row landed it (handled + // above) and the buffer rejected it for one of these reasons but the + // helper returned cleanly. Surface a warn log per non-success branch + // so ops can trace why a parent/root op went missing. The customer's + // primary mutation has already succeeded by this point; this remains + // best-effort, so we still don't bubble these to the response. + if (bufferOutcome && bufferOutcome.kind !== "applied") { + logger.warn("metadata route: parent/root buffer op did not apply", { + targetRunId, + kind: bufferOutcome.kind, + }); + } +} diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 4784ad756..a860d411a 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -2,7 +2,6 @@ import { column, type BucketThreshold, type TableSchema } from "@internal/tsql"; import { z } from "zod"; import { autoFormatSQL } from "~/components/code/TSQLEditor"; import { runFriendlyStatus, runStatusTitleFromStatus } from "~/components/runs/v3/TaskRunStatus"; -import { logger } from "~/services/logger.server"; export const QueryScopeSchema = z.enum(["organization", "project", "environment"]); export type QueryScope = z.infer; @@ -443,10 +442,7 @@ export const runsSchema: TableSchema = { ...column("Array(String)", { description: "Any bulk actions that operated on this run.", example: '["bulk_12345678", "bulk_34567890"]', - whereTransform: (value: string) => { - logger.log(`WHERE TRANSFORM: ${value}`); - return value.replace(/^bulk_/, ""); - }, + whereTransform: (value: string) => value.replace(/^bulk_/, ""), }), }, }, diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 60495a471..678eeb760 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -5,11 +5,11 @@ "sideEffects": false, "scripts": { "build": "run-s build:** && pnpm run upload:sourcemaps", - "build:remix": "remix build --sourcemap", + "build:remix": "remix vite:build", "build:server": "esbuild --platform=node --format=cjs ./server.ts --outdir=build --sourcemap", "build:otlpworker": "esbuild --platform=node --format=cjs --bundle ./app/v3/otlpTransformWorker.ts --outfile=build/otlpTransformWorker.cjs --sourcemap", "build:sentry": "esbuild --platform=node --format=cjs --outbase=. ./sentry.server.ts ./app/utils/sentryTraceContext.server.ts --outdir=build --sourcemap", - "dev": "cross-env PORT=3030 remix dev -c \"node ./build/server.js\"", + "dev": "cross-env NODE_ENV=development PORT=3030 tsx ./server.ts", "dev:worker": "cross-env NODE_PATH=../../node_modules/.pnpm/node_modules node ./build/server.js", "format": "oxfmt .", "lint": "oxlint -c ../../.oxlintrc.json", @@ -130,6 +130,7 @@ "@vercel/sdk": "^1.19.1", "@window-splitter/react": "1.1.3", "ai": "^6.0.116", + "assert": "^2.1.0", "assert-never": "^1.2.1", "aws4fetch": "^1.0.18", "class-variance-authority": "^0.5.2", @@ -170,6 +171,7 @@ "p-map": "^6.0.0", "p-retry": "^4.6.1", "parse-duration": "^2.1.0", + "pg": "8.15.6", "posthog-js": "^1.93.3", "posthog-node": "5.35.6", "prism-react-renderer": "^2.3.1", @@ -204,8 +206,9 @@ "superjson": "^2.2.1", "tailwind-merge": "^3.6.0", "tailwind-scrollbar-hide": "^4.0.0", - "tw-animate-css": "^1.4.0", "tiny-invariant": "^1.2.0", + "tw-animate-css": "^1.4.0", + "util": "^0.12.5", "uuid": "^14.0.0", "ws": "^8.11.0", "zod": "3.25.76", @@ -260,7 +263,8 @@ "tailwindcss": "^4.3.1", "tsconfig-paths": "^3.14.1", "tsx": "^4.20.6", - "vite-tsconfig-paths": "^4.0.5" + "vite-tsconfig-paths": "^5.1.4", + "vite": "^6.4.2" }, "engines": { "node": ">=18.19.0 || >=20.6.0" diff --git a/apps/webapp/remix.config.js b/apps/webapp/remix.config.js deleted file mode 100644 index f6bad31aa..000000000 --- a/apps/webapp/remix.config.js +++ /dev/null @@ -1,48 +0,0 @@ -/** @type {import('@remix-run/dev').AppConfig} */ -module.exports = { - dev: { - port: 8002, - }, - // Tailwind v4 runs through PostCSS (@tailwindcss/postcss), not the built-in Remix integration - tailwind: false, - postcss: true, - cacheDirectory: "./node_modules/.cache/remix", - ignoredRouteFiles: ["**/.*"], - serverModuleFormat: "cjs", - serverDependenciesToBundle: [ - /^remix-utils.*/, - /^@internal\//, // Bundle all internal packages - /^@trigger\.dev\//, // Bundle all trigger packages - "marked", - "agentcrumbs", - "axios", - "p-limit", - "p-map", - "yocto-queue", - "@unkey/cache", - "@unkey/cache/stores", - "emails", - "highlight.run", - "random-words", - "superjson", - "copy-anything", - "is-what", - "prismjs/components/prism-json", - "prismjs/components/prism-typescript", - "redlock", - "parse-duration", - "uncrypto", - "std-env", - "uuid", - ], - browserNodeBuiltinsPolyfill: { - modules: { - path: true, - os: true, - crypto: true, - http2: true, - assert: true, - util: true, - }, - }, -}; diff --git a/apps/webapp/remix.env.d.ts b/apps/webapp/remix.env.d.ts index 72e2affe3..6438f031f 100644 --- a/apps/webapp/remix.env.d.ts +++ b/apps/webapp/remix.env.d.ts @@ -1,2 +1,7 @@ /// +/// /// + +// Font packages that resolve to CSS — imported for their side effect only. +declare module "non.geist"; +declare module "non.geist/mono"; diff --git a/apps/webapp/server.ts b/apps/webapp/server.ts index fb4e20359..a5dbfb06b 100644 --- a/apps/webapp/server.ts +++ b/apps/webapp/server.ts @@ -1,13 +1,13 @@ import "./sentry.server"; import { createRequestHandler } from "@remix-run/express"; -import { broadcastDevReady, logDevReady } from "@remix-run/server-runtime"; import compression from "compression"; import type { Server as EngineServer } from "engine.io"; import express, { type RequestHandler } from "express"; import morgan from "morgan"; import { nanoid } from "nanoid"; import path from "path"; +import { pathToFileURL } from "node:url"; import type { Server as IoServer } from "socket.io"; import type { WebSocketServer } from "ws"; import type { RateLimitMiddleware } from "~/services/apiRateLimit.server"; @@ -74,6 +74,12 @@ function installPrimarySignalHandlers() { }); } +// Bundled to CJS (esbuild rewrites import() to require); vite and the Remix +// server bundle are ESM, so load them via a real dynamic import. +const dynamicImport = new Function("specifier", "return import(specifier)") as ( + specifier: string +) => Promise; + if (ENABLE_CLUSTER && cluster.isPrimary) { process.title = `node webapp-server primary`; console.log(`[cluster] Primary ${process.pid} is starting with ${WORKERS} workers`); @@ -92,6 +98,13 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { installPrimarySignalHandlers(); } else { + startServer().catch((error) => { + console.error("Failed to start server:", error); + process.exit(1); + }); +} + +async function startServer() { const app = express(); if (process.env.DISABLE_COMPRESSION !== "1") { @@ -101,16 +114,32 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { // http://expressjs.com/en/advanced/best-practice-security.html#at-a-minimum-disable-x-powered-by-header app.disable("x-powered-by"); - // Remix fingerprints its assets so we can cache forever. - app.use("/build", express.static("public/build", { immutable: true, maxAge: "1y" })); - // Stale dev builds can request an old hashed manifest; don't fall through to Remix. - app.use("/build", (_req, res) => { - res.status(404).end(); - }); + const MODE = process.env.NODE_ENV; - // Everything else (like favicon.ico) is cached for an hour. You may want to be - // more aggressive with this caching. - app.use(express.static("public", { maxAge: "1h" })); + // In development, Vite serves assets (and handles HMR) via middleware. + // Only NODE_ENV=development boots Vite — scripts that run the built server + // without NODE_ENV (start:local, dev:worker) must serve the build. + const viteDevServer = + MODE === "development" + ? await dynamicImport("vite").then((vite) => + vite.createServer({ server: { middlewareMode: true } }) + ) + : undefined; + + if (viteDevServer) { + app.use(viteDevServer.middlewares); + } else { + // Vite fingerprints its assets so we can cache forever. + app.use("/assets", express.static("build/client/assets", { immutable: true, maxAge: "1y" })); + // Stale clients can request an old hashed asset; hard-404 instead of falling + // through to Remix and answering a .js request with HTML. + app.use("/assets", (_req, res) => { + res.status(404).end(); + }); + // Everything else (like favicon.ico) is cached for an hour. You may want to be + // more aggressive with this caching. + app.use(express.static("build/client", { maxAge: "1h" })); + } // On high-volume machine-ingest services (e.g. otel) the per-request access // log dominates log volume. HTTP_ACCESS_LOG_DISABLED suppresses successful @@ -127,9 +156,17 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { ? `node webapp-worker-${cluster.isWorker ? cluster.worker?.id : "solo"}` : "node webapp-server"; - const MODE = process.env.NODE_ENV; - const BUILD_DIR = path.join(process.cwd(), "build"); - const build = require(BUILD_DIR); + const loadBuild = () => { + if (viteDevServer) { + return viteDevServer.ssrLoadModule("virtual:remix/server-build"); + } + return dynamicImport( + pathToFileURL(path.join(process.cwd(), "build", "server", "index.mjs")).href + ); + }; + + // Boots the entry.server singletons (socket.io, wss, rate limiters). + const build = await loadBuild(); const port = process.env.REMIX_APP_PORT || process.env.PORT || 3000; @@ -207,7 +244,7 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { "*", // @ts-ignore createRequestHandler({ - build, + build: viteDevServer ? loadBuild : build, mode: MODE, }) ); @@ -220,7 +257,7 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { "/healthcheck", // @ts-ignore createRequestHandler({ - build, + build: viteDevServer ? loadBuild : build, mode: MODE, }) ); @@ -232,12 +269,6 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { ENABLE_CLUSTER && cluster.isWorker ? ` [worker ${cluster.worker?.id}/${process.pid}]` : "" }` ); - - if (MODE === "development") { - broadcastDevReady(build) - .then(() => logDevReady(build)) - .catch(console.error); - } }); server.keepAliveTimeout = HTTP_KEEPALIVE_TIMEOUT_MS; @@ -259,6 +290,8 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { console.log("Express server closed gracefully."); } }); + // Dev-only: release Vite's file watchers and HMR websocket + viteDevServer?.close(); } process.on("SIGTERM", closeServer); @@ -304,7 +337,6 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { }); }); } else { - require(BUILD_DIR); console.log(`✅ app ready (skipping http server)`); } } diff --git a/apps/webapp/test/metadataRouteOperationsLogging.test.ts b/apps/webapp/test/metadataRouteOperationsLogging.test.ts index b7e5f8601..588c1547e 100644 --- a/apps/webapp/test/metadataRouteOperationsLogging.test.ts +++ b/apps/webapp/test/metadataRouteOperationsLogging.test.ts @@ -52,7 +52,7 @@ vi.mock("~/services/logger.server", () => ({ }, })); -import { routeOperationsToRun } from "~/routes/api.v1.runs.$runId.metadata"; +import { routeOperationsToRun } from "~/v3/mollifier/routeOperationsToRun.server"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; const env = { diff --git a/apps/webapp/vite.config.ts b/apps/webapp/vite.config.ts new file mode 100644 index 000000000..b008ae04e --- /dev/null +++ b/apps/webapp/vite.config.ts @@ -0,0 +1,66 @@ +import { vitePlugin as remix } from "@remix-run/dev"; +import { defaultClientConditions, defaultServerConditions, defineConfig } from "vite"; +import tsconfigPaths from "vite-tsconfig-paths"; + +export default defineConfig({ + plugins: [ + remix({ + ignoredRouteFiles: ["**/.*"], + // .mjs so the CJS server.ts wrapper can dynamic-import it + serverBuildFile: "index.mjs", + }), + tsconfigPaths(), + ], + resolve: { + // Resolve workspace packages to TS source (same condition the CLI uses) + conditions: ["@triggerdotdev/source", ...defaultClientConditions], + // Browser polyfills for node builtins used by client deps (antlr4ts) + alias: [ + { find: /^assert$/, replacement: "assert/" }, + { find: /^util$/, replacement: "util/" }, + ], + }, + optimizeDeps: { + // Crawl all routes up front - mid-session re-optimization duplicates React + entries: ["./app/entry.client.tsx", "./app/root.tsx", "./app/routes/**/*.{ts,tsx}"], + esbuildOptions: { + // node globals for prebundled CJS deps (client-only by construction) + define: { global: "globalThis" }, + inject: ["./vite/node-globals-shim.js"], + }, + }, + server: { + warmup: { + clientFiles: ["./app/entry.client.tsx", "./app/root.tsx", "./app/components/**/*.tsx"], + ssrFiles: ["./app/entry.server.tsx", "./app/root.tsx"], + }, + }, + build: { + sourcemap: true, + rollupOptions: { + // Prisma wrappers and pg have CJS/native pieces Rollup can't inline + external: [/^@trigger\.dev\/database$/, /^@internal\/run-ops-database$/, /^pg$/], + }, + }, + ssr: { + resolve: { + conditions: ["@triggerdotdev/source", ...defaultServerConditions], + externalConditions: ["@triggerdotdev/source", "node"], + }, + // CJS Prisma clients and native pg must load through node + external: ["@trigger.dev/database", "@internal/run-ops-database", "pg"], + // CJS deps whose named exports node's ESM interop can't detect + noExternal: [ + /^@radix-ui\//, + "react-use", + "cron-parser", + "@fingerprintjs/fingerprintjs-pro-react", + "@kapaai/react-sdk", + "@fingerprintjs/fingerprintjs-pro", + "@fingerprintjs/fingerprintjs-pro-spa", + ], + optimizeDeps: { + include: ["cron-parser"], + }, + }, +}); diff --git a/apps/webapp/vite/node-globals-shim.js b/apps/webapp/vite/node-globals-shim.js new file mode 100644 index 000000000..68d6826c8 --- /dev/null +++ b/apps/webapp/vite/node-globals-shim.js @@ -0,0 +1,10 @@ +// Minimal `process` stand-in injected into prebundled browser deps +// (see vite.config.ts optimizeDeps). Client-only. +export const process = { + env: {}, + browser: true, + version: "", + platform: "browser", + cwd: () => "/", + nextTick: (fn, ...args) => setTimeout(() => fn(...args), 0), +}; diff --git a/docker/Dockerfile b/docker/Dockerfile index 4290ab922..5fb11d319 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -91,6 +91,12 @@ COPY --from=dev-deps --chown=node:node /triggerdotdev/internal-packages/database # Run-ops Prisma client (query engine + client). Only constructed when the split is enabled, # but the image must carry it so a split-on deployment doesn't hit a missing query engine. COPY --from=dev-deps --chown=node:node /triggerdotdev/internal-packages/run-ops-database/generated ./internal-packages/run-ops-database/generated +# These workspace packages stay external to the Vite SSR bundle, so their +# built entry points must ship in the image (core is the sdk's runtime dep). +COPY --from=builder --chown=node:node /triggerdotdev/internal-packages/database/dist ./internal-packages/database/dist +COPY --from=builder --chown=node:node /triggerdotdev/internal-packages/run-ops-database/dist ./internal-packages/run-ops-database/dist +COPY --from=builder --chown=node:node /triggerdotdev/packages/trigger-sdk/dist ./packages/trigger-sdk/dist +COPY --from=builder --chown=node:node /triggerdotdev/packages/core/dist ./packages/core/dist COPY --from=builder --chown=node:node /triggerdotdev/apps/webapp/build/server.js ./apps/webapp/build/server.js COPY --from=builder --chown=node:node /triggerdotdev/apps/webapp/build ./apps/webapp/build COPY --from=builder --chown=node:node /triggerdotdev/apps/webapp/public ./apps/webapp/public diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 46574c267..bc682658d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -231,10 +231,10 @@ services: "--query", "SELECT 1", ] - interval: "3s" - timeout: "5s" - retries: "5" - start_period: "10s" + interval: 3s + timeout: 5s + retries: 5 + start_period: 10s clickhouse_migrator: build: diff --git a/internal-packages/rbac/src/index.ts b/internal-packages/rbac/src/index.ts index b9ba140ed..eb34da4f0 100644 --- a/internal-packages/rbac/src/index.ts +++ b/internal-packages/rbac/src/index.ts @@ -91,7 +91,8 @@ class LazyController implements RoleBaseAccessController { } const moduleName = "@triggerdotdev/plugins/rbac"; try { - const module = await import(moduleName); + // Optional plugin, resolved at runtime only + const module = await import(/* @vite-ignore */ moduleName); const plugin: RoleBasedAccessControlPlugin = module.default; console.log("RBAC: using plugin implementation"); return plugin.create({ diff --git a/internal-packages/run-engine/src/engine/locking.ts b/internal-packages/run-engine/src/engine/locking.ts index 0c86da80c..900b964be 100644 --- a/internal-packages/run-engine/src/engine/locking.ts +++ b/internal-packages/run-engine/src/engine/locking.ts @@ -1,8 +1,12 @@ -// import { default: Redlock } from "redlock"; -const { default: Redlock } = require("redlock"); import { AsyncLocalStorage } from "async_hooks"; import type { Redis } from "@internal/redis"; -import type * as redlock from "redlock"; +import * as redlockModule from "redlock"; + +// redlock is CJS with `exports.default`; probe the interop shapes instead of +// a bare require(), which breaks in ESM module runners. +const Redlock = ((redlockModule as any).default?.default ?? + (redlockModule as any).default ?? + redlockModule) as typeof redlockModule.default; import { tryCatch } from "@trigger.dev/core"; import type { Logger } from "@trigger.dev/core/logger"; import type { Tracer, Meter, ObservableResult, Attributes, Histogram } from "@internal/tracing"; @@ -34,12 +38,12 @@ export class LockAcquisitionTimeoutError extends Error { interface LockContext { resources: string; - signal: redlock.RedlockAbortSignal; + signal: redlockModule.RedlockAbortSignal; lockType: string; } interface ManualLockContext { - lock: redlock.Lock; + lock: redlockModule.Lock; timeout: NodeJS.Timeout | null | undefined; extension: Promise | undefined; } @@ -60,7 +64,7 @@ export interface LockRetryConfig { } export class RunLocker { - private redlock: InstanceType; + private redlock: InstanceType; private asyncLocalStorage: AsyncLocalStorage; private logger: Logger; private tracer: Tracer; @@ -216,7 +220,7 @@ export class RunLocker { let totalWaitTime = 0; // Retry the lock acquisition with exponential backoff - let lock: redlock.Lock | undefined; + let lock: redlockModule.Lock | undefined; let lastError: Error | undefined; for (let attempt = 0; attempt <= maxAttempts; attempt++) { @@ -346,7 +350,7 @@ export class RunLocker { // Create an AbortController for our signal const controller = new AbortController(); - const signal = controller.signal as redlock.RedlockAbortSignal; + const signal = controller.signal as redlockModule.RedlockAbortSignal; const manualContext: ManualLockContext = { lock, @@ -425,7 +429,7 @@ export class RunLocker { #setupAutoExtension( context: ManualLockContext, duration: number, - signal: redlock.RedlockAbortSignal, + signal: redlockModule.RedlockAbortSignal, controller: AbortController ): void { if (this.automaticExtensionThreshold > duration - 100) { @@ -460,7 +464,7 @@ export class RunLocker { async #extendLock( context: ManualLockContext, duration: number, - signal: redlock.RedlockAbortSignal, + signal: redlockModule.RedlockAbortSignal, controller: AbortController, scheduleNext: () => void ): Promise { diff --git a/internal-packages/sso/src/index.ts b/internal-packages/sso/src/index.ts index b329bfaa0..a90543a72 100644 --- a/internal-packages/sso/src/index.ts +++ b/internal-packages/sso/src/index.ts @@ -56,7 +56,8 @@ export class LazyController implements SsoController { } const moduleName = "@triggerdotdev/plugins/sso"; const importer = - options?.importer ?? ((m: string) => import(m) as Promise<{ default: SsoPlugin }>); + options?.importer ?? + ((m: string) => import(/* @vite-ignore */ m) as Promise<{ default: SsoPlugin }>); try { const module = await importer(moduleName); const plugin: SsoPlugin = module.default; diff --git a/packages/trigger-sdk/src/v3/aiAutoTelemetry.ts b/packages/trigger-sdk/src/v3/aiAutoTelemetry.ts index 7533dc9bf..04bf4b952 100644 --- a/packages/trigger-sdk/src/v3/aiAutoTelemetry.ts +++ b/packages/trigger-sdk/src/v3/aiAutoTelemetry.ts @@ -42,10 +42,10 @@ async function register(): Promise { if (typeof aiMod.registerTelemetry !== "function") { return; // v5 / v6 — `ai` core emits spans itself, nothing to wire. } - // Computed specifier keeps the optional peer out of static bundler - // resolution; resolves at runtime only when the customer installed it. + // Computed specifier + @vite-ignore keep the optional peer out of static + // bundler resolution; resolves at runtime only when installed. const otelSpecifier = ["@ai-sdk", "otel"].join("/"); - const otelMod: any = await import(otelSpecifier).catch(() => null); + const otelMod: any = await import(/* @vite-ignore */ otelSpecifier).catch(() => null); if (typeof otelMod?.OpenTelemetry !== "function") { return; // optional peer not installed } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fb98ad91c..f505b0ab0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -516,6 +516,9 @@ importers: ai: specifier: 6.0.116 version: 6.0.116(zod@3.25.76) + assert: + specifier: ^2.1.0 + version: 2.1.0 assert-never: specifier: ^1.2.1 version: 1.2.1 @@ -636,6 +639,9 @@ importers: parse-duration: specifier: ^2.1.0 version: 2.1.4 + pg: + specifier: 8.15.6 + version: 8.15.6 posthog-js: specifier: ^1.93.3 version: 1.93.3 @@ -744,6 +750,9 @@ importers: tw-animate-css: specifier: ^1.4.0 version: 1.4.0 + util: + specifier: ^0.12.5 + version: 0.12.5 uuid: specifier: ^14.0.0 version: 14.0.0 @@ -901,9 +910,12 @@ importers: tsx: specifier: ^4.20.6 version: 4.20.6 + vite: + specifier: ^6.4.2 + version: 6.4.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.20.6)(yaml@2.9.0) vite-tsconfig-paths: - specifier: ^4.0.5 - version: 4.0.5(typescript@6.0.3) + specifier: ^5.1.4 + version: 5.1.4(typescript@6.0.3)(vite@6.4.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.20.6)(yaml@2.9.0)) docs: {} @@ -8495,6 +8507,9 @@ packages: assert-never@1.2.1: resolution: {integrity: sha512-TaTivMB6pYI1kXwrFlEhLeGfOqoDNdTxjCdwRfFFkEA30Eu+k48W34nlok2EYWJfFFzqaEmichdNM7th6M5HNw==} + assert@2.1.0: + resolution: {integrity: sha512-eLHpSK/Y4nhMJ07gDaAzoX/XAKS8PSaojml3M0DM4JpV1LAi5JOJ/p6H/XWrl8L+DzVEvVCW1z3vWAaB9oTsQw==} + assertion-error@2.0.1: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} @@ -8786,10 +8801,6 @@ packages: resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} engines: {node: '>= 0.4'} - call-bind@1.0.7: - resolution: {integrity: sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==} - engines: {node: '>= 0.4'} - call-bind@1.0.8: resolution: {integrity: sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==} engines: {node: '>= 0.4'} @@ -9581,10 +9592,6 @@ packages: resolution: {integrity: sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==} engines: {node: '>=12'} - define-properties@1.1.4: - resolution: {integrity: sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==} - engines: {node: '>= 0.4'} - define-properties@1.2.1: resolution: {integrity: sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==} engines: {node: '>= 0.4'} @@ -11139,6 +11146,10 @@ packages: resolution: {integrity: sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==} engines: {node: '>=8'} + is-nan@1.3.2: + resolution: {integrity: sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==} + engines: {node: '>= 0.4'} + is-negative-zero@2.0.2: resolution: {integrity: sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==} engines: {node: '>= 0.4'} @@ -12510,6 +12521,10 @@ packages: resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==} engines: {node: '>= 0.4'} + object-is@1.1.6: + resolution: {integrity: sha512-F8cZ+KfGlSGi09lJT7/Nd6KJZ9ygtvYC0/UYYLI9nmQKLMnydpB9yvbv9K1uSkEu7FU9vYPmVwLg328tX+ot3Q==} + engines: {node: '>= 0.4'} + object-keys@1.1.1: resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==} engines: {node: '>= 0.4'} @@ -12873,8 +12888,8 @@ packages: pg-cloudflare@1.2.7: resolution: {integrity: sha512-YgCtzMH0ptvZJslLM1ffsY4EuGaU0cx4XSdXLRFae8bPP4dS5xL1tNB3k2o/N64cHJpwU7dxKli/nZ2lUa5fLg==} - pg-connection-string@2.8.5: - resolution: {integrity: sha512-Ni8FuZ8yAF+sWZzojvtLE2b03cqjO5jNULcHFfM9ZZ0/JXrgom5pBREbtnAw7oxsxJqHw9Nz/XWORUEL3/IFow==} + pg-connection-string@2.9.1: + resolution: {integrity: sha512-nkc6NpDcvPVpZXxrreI/FOtX3XemeLl8E0qFr6F2Lrm/I8WOnaWNhIPK2Z7OHpw7gh5XJThi6j6ppgNoaT1w4w==} pg-int8@1.0.1: resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} @@ -12884,8 +12899,8 @@ packages: resolution: {integrity: sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==} engines: {node: '>=4'} - pg-pool@3.9.6: - resolution: {integrity: sha512-rFen0G7adh1YmgvrmE5IPIqbb+IgEzENUm+tzm6MLLDSlPRoZVhzU1WdML9PV2W5GOdRA9qBKURlbt1OsXOsPw==} + pg-pool@3.10.1: + resolution: {integrity: sha512-Tu8jMlcX+9d8+QVzKIvM/uJtp07PKr82IUOYEphaWcoBhIYkoHpLXN3qO59nAI11ripznDsEzEv8nUxBVWajGg==} peerDependencies: pg: '>=8.0' @@ -15224,6 +15239,14 @@ packages: vite-tsconfig-paths@4.0.5: resolution: {integrity: sha512-/L/eHwySFYjwxoYt1WRJniuK/jPv+WGwgRGBYx3leciR5wBeqntQpUE6Js6+TJemChc+ter7fDBKieyEWDx4yQ==} + vite-tsconfig-paths@5.1.4: + resolution: {integrity: sha512-cYj0LRuLV2c2sMqhqhGpaO3LretdtMn/BVX4cPLanIZuwwrkVl+lK84E/miEXkCHWXuq65rhNN4rXsBcOB3S4w==} + peerDependencies: + vite: ^6.4.2 + peerDependenciesMeta: + vite: + optional: true + vite@4.4.9: resolution: {integrity: sha512-2mbUn2LlUmNASWwSCNSJ/EG2HuSRTnVNaydp6vMCm5VIqJsjMfbIWtbH2kDuwUVW5mMUKKZvGPX/rqeqVvv1XA==} engines: {node: ^14.18.0 || >=16.0.0} @@ -23336,6 +23359,14 @@ snapshots: assert-never@1.2.1: {} + assert@2.1.0: + dependencies: + call-bind: 1.0.8 + is-nan: 1.3.2 + object-is: 1.1.6 + object.assign: 4.1.5 + util: 0.12.5 + assertion-error@2.0.1: {} ast-v8-to-istanbul@1.0.2: @@ -23700,14 +23731,6 @@ snapshots: es-errors: 1.3.0 function-bind: 1.1.2 - call-bind@1.0.7: - dependencies: - es-define-property: 1.0.1 - es-errors: 1.3.0 - function-bind: 1.1.2 - get-intrinsic: 1.3.0 - set-function-length: 1.2.2 - call-bind@1.0.8: dependencies: call-bind-apply-helpers: 1.0.2 @@ -24502,11 +24525,6 @@ snapshots: define-lazy-prop@3.0.0: {} - define-properties@1.1.4: - dependencies: - has-property-descriptors: 1.0.2 - object-keys: 1.1.1 - define-properties@1.2.1: dependencies: define-data-property: 1.1.4 @@ -26112,7 +26130,7 @@ snapshots: hast-util-to-jsx-runtime@2.3.6: dependencies: - '@types/estree': 1.0.8 + '@types/estree': 1.0.9 '@types/hast': 3.0.4 '@types/unist': 3.0.3 comma-separated-tokens: 2.0.3 @@ -26442,6 +26460,11 @@ snapshots: is-interactive@1.0.0: {} + is-nan@1.3.2: + dependencies: + call-bind: 1.0.8 + define-properties: 1.2.1 + is-negative-zero@2.0.2: {} is-negative-zero@2.0.3: {} @@ -28122,6 +28145,11 @@ snapshots: object-inspect@1.13.4: {} + object-is@1.1.6: + dependencies: + call-bind: 1.0.8 + define-properties: 1.2.1 + object-keys@1.1.1: {} object.assign@4.1.5: @@ -28553,13 +28581,13 @@ snapshots: pg-cloudflare@1.2.7: optional: true - pg-connection-string@2.8.5: {} + pg-connection-string@2.9.1: {} pg-int8@1.0.1: {} pg-numeric@1.0.2: {} - pg-pool@3.9.6(pg@8.15.6): + pg-pool@3.10.1(pg@8.15.6): dependencies: pg: 8.15.6 @@ -28587,9 +28615,9 @@ snapshots: pg@8.15.6: dependencies: - pg-connection-string: 2.8.5 - pg-pool: 3.9.6(pg@8.15.6) - pg-protocol: 1.9.5 + pg-connection-string: 2.9.1 + pg-pool: 3.10.1(pg@8.15.6) + pg-protocol: 1.10.3 pg-types: 2.2.0 pgpass: 1.0.5 optionalDependencies: @@ -30245,8 +30273,8 @@ snapshots: string.prototype.padend@3.1.4: dependencies: - call-bind: 1.0.7 - define-properties: 1.1.4 + call-bind: 1.0.8 + define-properties: 1.2.1 es-abstract: 1.21.1 string.prototype.trim@1.2.9: @@ -31298,10 +31326,21 @@ snapshots: - supports-color - typescript + vite-tsconfig-paths@5.1.4(typescript@6.0.3)(vite@6.4.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.20.6)(yaml@2.9.0)): + dependencies: + debug: 4.4.3(supports-color@10.0.0) + globrex: 0.1.2 + tsconfck: 3.1.3(typescript@6.0.3) + optionalDependencies: + vite: 6.4.2(@types/node@24.13.3)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.20.6)(yaml@2.9.0) + transitivePeerDependencies: + - supports-color + - typescript + vite@4.4.9(@types/node@24.13.3)(lightningcss@1.32.0)(terser@5.46.1): dependencies: esbuild: 0.18.20 - postcss: 8.5.10 + postcss: 8.5.15 rollup: 3.29.1 optionalDependencies: '@types/node': 24.13.3 @@ -31314,9 +31353,9 @@ snapshots: esbuild: 0.25.12 fdir: 6.5.0(picomatch@4.0.4) picomatch: 4.0.4 - postcss: 8.5.10 + postcss: 8.5.15 rollup: 4.60.1 - tinyglobby: 0.2.16 + tinyglobby: 0.2.17 optionalDependencies: '@types/node': 24.13.3 fsevents: 2.3.3 @@ -31331,9 +31370,9 @@ snapshots: esbuild: 0.25.12 fdir: 6.5.0(picomatch@4.0.4) picomatch: 4.0.4 - postcss: 8.5.10 + postcss: 8.5.15 rollup: 4.60.1 - tinyglobby: 0.2.16 + tinyglobby: 0.2.17 optionalDependencies: '@types/node': 24.13.3 fsevents: 2.3.3 @@ -31348,9 +31387,9 @@ snapshots: esbuild: 0.25.12 fdir: 6.5.0(picomatch@4.0.4) picomatch: 4.0.4 - postcss: 8.5.10 + postcss: 8.5.15 rollup: 4.60.1 - tinyglobby: 0.2.16 + tinyglobby: 0.2.17 optionalDependencies: '@types/node': 24.13.3 fsevents: 2.3.3 From 6642c8b7850fc32dcfd3a349170d996d8107f589 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Tue, 21 Jul 2026 15:40:48 +0100 Subject: [PATCH 012/300] fix(webapp): prevent duplicate envs from provision race (#4261) Fixes TRI-12078 ## Summary Prevents concurrent environment setup requests from creating duplicate Staging and Preview environments. ## Fix Adds database-enforced uniqueness for root Staging and Preview environments. If two requests race, the losing request loads the environment created by the winner and continues successfully instead of creating a duplicate or returning an error. --- .../prevent-duplicate-root-environments.md | 6 ++++ ...gs.$organizationId.environments.staging.ts | 33 ++++++++++++++++--- .../migration.sql | 4 +++ .../database/prisma/schema.prisma | 2 ++ 4 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 .server-changes/prevent-duplicate-root-environments.md create mode 100644 internal-packages/database/prisma/migrations/20260714140000_runtime_environment_unique_staging_preview_root/migration.sql diff --git a/.server-changes/prevent-duplicate-root-environments.md b/.server-changes/prevent-duplicate-root-environments.md new file mode 100644 index 000000000..0710595de --- /dev/null +++ b/.server-changes/prevent-duplicate-root-environments.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Prevent duplicate Staging and Preview environments when account setup requests overlap diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.environments.staging.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.environments.staging.ts index f3c215bfd..cf6d1abce 100644 --- a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.environments.staging.ts +++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.environments.staging.ts @@ -1,5 +1,6 @@ import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; import { + Prisma, type RuntimeEnvironment, type Organization, type Project, @@ -61,9 +62,16 @@ async function upsertEnvironment( type: RuntimeEnvironmentType, isBranchableEnvironment: boolean ) { - const existingEnvironment = project.environments.find((env) => env.type === type); + const existingEnvironment = project.environments.find( + (env) => env.type === type && env.parentEnvironmentId === null + ); - if (!existingEnvironment) { + if (existingEnvironment) { + await updateEnvConcurrencyLimits({ ...existingEnvironment, organization, project }); + return { status: "updated", environment: existingEnvironment }; + } + + try { const newEnvironment = await createEnvironment({ organization, project, @@ -72,8 +80,23 @@ async function upsertEnvironment( }); await updateEnvConcurrencyLimits({ ...newEnvironment, organization, project }); return { status: "created", environment: newEnvironment }; - } else { - await updateEnvConcurrencyLimits({ ...existingEnvironment, organization, project }); - return { status: "updated", environment: existingEnvironment }; + } catch (error) { + if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") { + const existingAfterConflict = await prisma.runtimeEnvironment.findFirst({ + where: { + organizationId: organization.id, + projectId: project.id, + type, + parentEnvironmentId: null, + }, + }); + + if (existingAfterConflict) { + await updateEnvConcurrencyLimits({ ...existingAfterConflict, organization, project }); + return { status: "updated", environment: existingAfterConflict }; + } + } + + throw error; } } diff --git a/internal-packages/database/prisma/migrations/20260714140000_runtime_environment_unique_staging_preview_root/migration.sql b/internal-packages/database/prisma/migrations/20260714140000_runtime_environment_unique_staging_preview_root/migration.sql new file mode 100644 index 000000000..153db8c2d --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260714140000_runtime_environment_unique_staging_preview_root/migration.sql @@ -0,0 +1,4 @@ +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "RuntimeEnvironment_projectId_type_staging_preview_root_key" +ON "RuntimeEnvironment" ("projectId", "type") +WHERE "parentEnvironmentId" IS NULL + AND "type" IN ('STAGING', 'PREVIEW'); diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 1b8642e86..2800dd5dd 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -394,6 +394,8 @@ model RuntimeEnvironment { taskIdentifiers TaskIdentifier[] revokedApiKeys RevokedApiKey[] + // A partial unique index also enforces one STAGING/PREVIEW root per project and type. + // It is defined in SQL because Prisma does not support partial indexes in the schema. @@unique([projectId, slug, orgMemberId]) @@unique([projectId, shortcode]) @@index([parentEnvironmentId]) From e2d3b8388c3c1ea964b4e932fc825a8d5918575b Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 21 Jul 2026 16:08:30 +0100 Subject: [PATCH 013/300] feat(sdk): return lastEventId from writeTurnComplete and typed capture result (#4304) ## Summary Two ergonomic additions for custom chat-agent loops that own the turn loop (`chat.customAgent`, `chat.createSession`, and the hand-rolled primitives). `chat.writeTurnComplete()` now resolves to `{ lastEventId }`, the resume cursor for the start of the next turn. A custom loop can persist it straight from the task instead of round-tripping it back from the client after the turn ends. The value was already produced internally by the turn-complete write; the public wrapper simply discarded it. `chat.pipeAndCapture()` no longer throws when a stream is stopped or fails. It now resolves to a `PipeAndCaptureResult` carrying any partial `message` captured before the stop or failure, a typed `status` (`"complete" | "aborted" | "error"`), and the `error` on failure. Previously a failed stream threw and the partial was lost, and an abort was captured only when the AI SDK happened to fire `onFinish` in time. ```ts const { message, status, error } = await chat.pipeAndCapture(result, { signal }); if (message) conversation.addResponse(message); if (status === "error") logger.error("turn failed", { error }); const { lastEventId } = await chat.writeTurnComplete(); await db.chats.update(chatId, { lastEventId }); ``` ## Design `pipeAndCapture` wraps the pipe in a `try/catch` and classifies the outcome from the abort signal (a stop drains the source stream cleanly rather than throwing) versus a thrown error. It also races the `onFinish` capture against a timeout so a hard stop that prevents `onFinish` from firing can't hang the caller. This mirrors the capture path `chat.agent` already uses internally. The `finishReason` from `onFinish` is surfaced too, since it was already captured on the built-in path. The internal `turn.complete()` helper keeps its existing contract: it still returns `UIMessage | undefined`, still throws on a genuine stream failure, and still discards output on a full run cancel. ## Breaking change `chat.pipeAndCapture` previously resolved to `UIMessage | undefined`. Call sites now read `.message` off the result. This is a young, low-level API; the docs examples are updated in this PR. --- .changeset/chat-custom-agent-capture.md | 22 ++ docs/ai-chat/changelog.mdx | 4 +- docs/ai-chat/compaction.mdx | 4 +- docs/ai-chat/custom-agents.mdx | 35 ++- docs/ai-chat/fast-starts.mdx | 4 +- docs/ai-chat/pending-messages.mdx | 8 +- docs/ai-chat/reference.mdx | 4 +- packages/trigger-sdk/src/v3/ai.ts | 212 +++++++++++++--- .../src/v3/test/test-session-handle.ts | 26 +- .../test/chat-pipe-and-capture.test.ts | 230 ++++++++++++++++++ 10 files changed, 483 insertions(+), 66 deletions(-) create mode 100644 .changeset/chat-custom-agent-capture.md create mode 100644 packages/trigger-sdk/test/chat-pipe-and-capture.test.ts diff --git a/.changeset/chat-custom-agent-capture.md b/.changeset/chat-custom-agent-capture.md new file mode 100644 index 000000000..8729cda8c --- /dev/null +++ b/.changeset/chat-custom-agent-capture.md @@ -0,0 +1,22 @@ +--- +"@trigger.dev/sdk": patch +--- + +Custom chat agent loops get two ergonomic wins for owning the turn loop. + +`chat.writeTurnComplete()` now returns the turn boundary's resume cursors (`lastEventId` for the output stream and `sessionInEventId` for the input stream), so you can persist them straight from the task instead of round-tripping them back from the client. + +```ts +const { lastEventId, sessionInEventId } = await chat.writeTurnComplete(); +await db.chats.update(chatId, { lastEventId, sessionInEventId }); +``` + +`chat.pipeAndCapture()` no longer throws when a stream is stopped or fails. It now returns a `PipeAndCaptureResult` whose `message` holds any partial output captured before the stop or failure, alongside a typed `status` (`"complete" | "aborted" | "error"`) and, on failure, the `error`. Read the message off the result: + +```ts +const { message, status, error } = await chat.pipeAndCapture(result, { signal }); +if (message) conversation.addResponse(message); +if (status === "error") logger.error("turn failed", { error }); +``` + +Note: `pipeAndCapture` previously resolved to `UIMessage | undefined`. Update call sites to read `.message` from the returned result. diff --git a/docs/ai-chat/changelog.mdx b/docs/ai-chat/changelog.mdx index 4bd667a41..22f496b91 100644 --- a/docs/ai-chat/changelog.mdx +++ b/docs/ai-chat/changelog.mdx @@ -55,10 +55,10 @@ if (isFinal) { const result = streamText({ model, messages: conversation.modelMessages, tools }); // Pass originalMessages so the handed-over tool round merges into the // step-1 assistant instead of starting a new message. - const response = await chat.pipeAndCapture(result, { + const { message } = await chat.pipeAndCapture(result, { originalMessages: conversation.uiMessages, }); - if (response) await conversation.addResponse(response); + if (message) await conversation.addResponse(message); } ``` diff --git a/docs/ai-chat/compaction.mdx b/docs/ai-chat/compaction.mdx index 02db7d7b1..0a8ed48c7 100644 --- a/docs/ai-chat/compaction.mdx +++ b/docs/ai-chat/compaction.mdx @@ -365,8 +365,8 @@ for (let turn = 0; turn < 100; turn++) { stopWhen: stepCountIs(15), }); - const response = await chat.pipeAndCapture(result); - if (response) await conversation.addResponse(response); + const { message } = await chat.pipeAndCapture(result); + if (message) await conversation.addResponse(message); // Outer-loop compaction const usage = await result.totalUsage; diff --git a/docs/ai-chat/custom-agents.mdx b/docs/ai-chat/custom-agents.mdx index 1cac571ad..197bff6b5 100644 --- a/docs/ai-chat/custom-agents.mdx +++ b/docs/ai-chat/custom-agents.mdx @@ -160,11 +160,11 @@ for await (const turn of session) { }); // Manual: pipe and capture separately - const response = await chat.pipeAndCapture(result, { signal: turn.signal }); + const { message } = await chat.pipeAndCapture(result, { signal: turn.signal }); - if (response) { + if (message) { // Custom processing before accumulating - await turn.addResponse(response); + await turn.addResponse(message); } // Custom persistence, analytics, etc. @@ -215,8 +215,8 @@ For full control, skip `createSession` and compose the primitives directly: | ------------------------------- | -------------------------------------------------------------------------------------------- | | `chat.messages` | Input stream for incoming messages — use `.waitWithIdleTimeout()` to wait for the next turn | | `chat.createStopSignal()` | Create a managed stop signal wired to the stop input stream | -| `chat.pipeAndCapture(result)` | Pipe a `StreamTextResult` to the chat stream and capture the response | -| `chat.writeTurnComplete()` | Signal the frontend that the current turn is complete | +| `chat.pipeAndCapture(result)` | Pipe a stream and capture the response; returns `{ message, status, error }` | +| `chat.writeTurnComplete()` | Signal turn complete; returns `{ lastEventId, sessionInEventId }` resume cursors | | `chat.MessageAccumulator` | Accumulates conversation messages across turns | | `chat.pipe(stream)` | Pipe a stream to the frontend (no response capture) | | `chat.cleanupAbortedParts(msg)` | Clean up incomplete parts from a stopped response | @@ -285,21 +285,16 @@ export const myChat = chat.customAgent({ stopWhen: stepCountIs(15), }); - let response; - try { - response = await chat.pipeAndCapture(result, { signal: combinedSignal }); - } catch (error) { - if (error instanceof Error && error.name === "AbortError") { - if (runSignal.aborted) break; - // Stop — fall through to accumulate partial - } else { - throw error; - } - } + const { message, status, error } = await chat.pipeAndCapture(result, { + signal: combinedSignal, + }); + // pipeAndCapture never throws: a user stop returns status "aborted" with + // the partial message, and a failure returns status "error" with `error`. + if (status === "error") throw error; - if (response) { + if (message) { const cleaned = - stop.signal.aborted && !runSignal.aborted ? chat.cleanupAbortedParts(response) : response; + stop.signal.aborted && !runSignal.aborted ? chat.cleanupAbortedParts(message) : message; await conversation.addResponse(cleaned); } @@ -346,8 +341,8 @@ const messages = await conversation.addIncoming( ); // After piping, add the response -const response = await chat.pipeAndCapture(result); -if (response) await conversation.addResponse(response); +const { message } = await chat.pipeAndCapture(result); +if (message) await conversation.addResponse(message); // Access accumulated messages for persistence conversation.uiMessages; // UIMessage[] diff --git a/docs/ai-chat/fast-starts.mdx b/docs/ai-chat/fast-starts.mdx index 40d7b62fd..78bb7ed9f 100644 --- a/docs/ai-chat/fast-starts.mdx +++ b/docs/ai-chat/fast-starts.mdx @@ -589,10 +589,10 @@ if (turn === 0 && payload.trigger === "handover-prepare") { messages: conversation.modelMessages, stopWhen: stepCountIs(10), }); - const response = await chat.pipeAndCapture(result, { + const { message } = await chat.pipeAndCapture(result, { originalMessages: conversation.uiMessages, }); - if (response) await conversation.addResponse(response); + if (message) await conversation.addResponse(message); } await chat.writeTurnComplete(); // on isFinal the warm partial is already the response return; diff --git a/docs/ai-chat/pending-messages.mdx b/docs/ai-chat/pending-messages.mdx index 5aa668099..c77600f47 100644 --- a/docs/ai-chat/pending-messages.mdx +++ b/docs/ai-chat/pending-messages.mdx @@ -179,10 +179,14 @@ for (let turn = 0; turn < 100; turn++) { stopWhen: stepCountIs(15), }); - const response = await chat.pipeAndCapture(result); + const { message, status, error } = await chat.pipeAndCapture(result); sub.off(); - if (response) await conversation.addResponse(response); + // Keep any partial output, then branch on the outcome: pipeAndCapture no + // longer throws, so rethrow a real failure and don't complete a failed turn. + if (message) await conversation.addResponse(message); + if (status === "error") throw error; + await chat.writeTurnComplete(); } ``` diff --git a/docs/ai-chat/reference.mdx b/docs/ai-chat/reference.mdx index 340422050..da08f9a04 100644 --- a/docs/ai-chat/reference.mdx +++ b/docs/ai-chat/reference.mdx @@ -503,8 +503,8 @@ All methods available on the `chat` object from `@trigger.dev/sdk/ai`. | `chat.agent(options)` | Create a chat agent | | `chat.createSession(payload, options)` | Create an async iterator for chat turns | | `chat.pipe(source, options?)` | Pipe a stream to the frontend (from anywhere inside a task) | -| `chat.pipeAndCapture(source, options?)` | Pipe and capture the response `UIMessage` | -| `chat.writeTurnComplete(options?)` | Signal the frontend that the current turn is complete | +| `chat.pipeAndCapture(source, options?)` | Pipe and capture the response; returns `{ message, status, error }` | +| `chat.writeTurnComplete(options?)` | Signal turn complete; returns `{ lastEventId, sessionInEventId }` resume cursors | | `chat.createStopSignal()` | Create a managed stop signal wired to the stop input stream | | `chat.messages` | Input stream for incoming messages — use `.waitWithIdleTimeout()` | | `chat.local({ id })` | Create a per-run typed local (see [`chat.local`](/ai-chat/chat-local)) | diff --git a/packages/trigger-sdk/src/v3/ai.ts b/packages/trigger-sdk/src/v3/ai.ts index ee70c79c7..4612302c4 100644 --- a/packages/trigger-sdk/src/v3/ai.ts +++ b/packages/trigger-sdk/src/v3/ai.ts @@ -8828,14 +8828,117 @@ function createStopSignal(): { * The `TriggerChatTransport` intercepts this to close the ReadableStream * for the current turn. Call after piping the response stream. * + * Returns two resume cursors for the turn boundary, both saveable from the + * task instead of round-tripping them back from the client: + * - `lastEventId` — the turn-complete control record's seq_num on + * `session.out`; where the next turn's output stream resumes. + * - `sessionInEventId` — the committed-consume cursor on `session.in` as of + * this turn-complete, letting a raw loop correlate the boundary with the + * exact input record it acknowledged. Trigger owns input-cursor recovery, + * so this is for correlation / out-of-sync detection, not required. + * + * Either is `undefined` when the corresponding cursor isn't available. + * * @example * ```ts * await chat.pipe(result); - * await chat.writeTurnComplete(); + * const { lastEventId, sessionInEventId } = await chat.writeTurnComplete(); + * await db.chats.update(chatId, { lastEventId, sessionInEventId }); * ``` */ -async function chatWriteTurnComplete(options?: { publicAccessToken?: string }): Promise { - await writeTurnCompleteChunk(undefined, options?.publicAccessToken); +async function chatWriteTurnComplete(options?: { + publicAccessToken?: string; +}): Promise<{ lastEventId?: string; sessionInEventId?: string }> { + const result = await writeTurnCompleteChunk(undefined, options?.publicAccessToken); + // Same cursor written to the `session-in-event-id` header inside + // `writeTurnCompleteChunk`; surfaced here so the caller can persist it. + const inCursor = getChatSession().in.lastDispatchedSeqNum(); + return { + lastEventId: result?.lastEventId, + ...(inCursor !== undefined ? { sessionInEventId: String(inCursor) } : {}), + }; +} + +/** + * The outcome of a turn's stream, reported by {@link pipeChatAndCapture}. + * + * - `complete` — the stream finished on its own. + * - `aborted` — the stream was stopped via the `signal` (user stop / cancel). + * - `error` — the stream threw; `error` carries what was thrown. + * + * `message` holds whatever the assistant produced and is present for every + * status — including `aborted` and `error` — as long as any output streamed + * before the stop or failure, so partial responses are never lost. + */ +export type PipeAndCaptureResult = { + /** The captured assistant message, or `undefined` if nothing streamed. */ + message: UIMessage | undefined; + /** Coarse outcome of the stream. */ + status: "complete" | "aborted" | "error"; + /** The AI SDK finish reason, when the stream reported one. */ + finishReason?: FinishReason; + /** What the stream threw, present only when `status === "error"`. */ + error?: unknown; +}; + +/** + * Pass every chunk through untouched while recording it in `buffer`. Handles + * both the `AsyncIterable` and `ReadableStream` shapes `toUIMessageStream()` + * can return, and propagates a source error to the consumer after buffering + * whatever streamed first. See {@link pipeChatAndCapture} for why. + */ +async function* tapUIMessageChunks( + source: AsyncIterable | ReadableStream, + buffer: UIMessageChunk[] +): AsyncGenerator { + if (isReadableStream(source)) { + const reader = source.getReader(); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buffer.push(value as UIMessageChunk); + yield value; + } + } finally { + reader.releaseLock(); + } + } else { + for await (const chunk of source) { + buffer.push(chunk as UIMessageChunk); + yield chunk; + } + } +} + +/** + * Reconstruct a partial assistant `UIMessage` from the raw chunks that + * streamed before a failure — the fallback for {@link pipeChatAndCapture} + * when a transport error abandons the stream before `onFinish` runs. Uses the + * same `readUIMessageStream` reducer as the boot-time replay path. Returns + * `undefined` if there's nothing to assemble or the reducer throws. + */ +async function assemblePartialFromChunks(chunks: UIMessageChunk[]): Promise { + const relevant = chunks.filter((c) => { + const type = (c as { type?: unknown }).type; + return typeof type === "string" && !type.startsWith("trigger:"); + }); + if (relevant.length === 0) return undefined; + try { + const stream = new ReadableStream({ + start(controller) { + for (const chunk of relevant) controller.enqueue(chunk); + controller.close(); + }, + }); + let last: UIMessage | undefined; + for await (const message of readUIMessageStream({ stream })) { + last = message; + } + return last; + } catch { + return undefined; + } } /** @@ -8843,20 +8946,26 @@ async function chatWriteTurnComplete(options?: { publicAccessToken?: string }): * the assistant's response message via `onFinish`. * * Combines `toUIMessageStream()` + `onFinish` callback + `chat.pipe()`. - * Returns the captured `UIMessage`, or `undefined` if capture failed. + * Never throws on a stopped or failed stream: it returns a + * {@link PipeAndCaptureResult} whose `message` holds any partial output + * captured before the stop/failure, alongside a typed `status` (and `error` + * on failure). Save the partial after a stop or error without separate + * capture logic. * * @example * ```ts * const result = streamText({ model, messages, abortSignal: signal }); - * const response = await chat.pipeAndCapture(result, { signal }); - * if (response) conversation.addResponse(response); + * const { message, status, error } = await chat.pipeAndCapture(result, { signal }); + * if (message) conversation.addResponse(message); + * if (status === "error") logger.error("turn failed", { error }); * ``` */ async function pipeChatAndCapture( source: UIMessageStreamable, options?: { signal?: AbortSignal; spanName?: string; originalMessages?: UIMessage[] } -): Promise { +): Promise { let captured: UIMessage | undefined; + let capturedFinishReason: FinishReason | undefined; let resolveOnFinish: () => void; const onFinishPromise = new Promise((r) => { resolveOnFinish = r; @@ -8875,19 +8984,66 @@ async function pipeChatAndCapture( // the frontend replaces the partial message — wiping the // pre-injection text from the UI and the captured response. generateMessageId: resolvedOptions.generateMessageId ?? generateMessageId, - onFinish: ({ responseMessage }: { responseMessage: UIMessage }) => { + onFinish: ({ + responseMessage, + finishReason, + }: { + responseMessage: UIMessage; + finishReason?: FinishReason; + }) => { captured = responseMessage; + capturedFinishReason = finishReason; resolveOnFinish!(); }, }); - await pipeChat(uiStream, { - signal: options?.signal, - spanName: options?.spanName ?? "stream response", - }); - await onFinishPromise; + // Buffer chunks as they flow to the pipe so a transport failure that + // abandons the UI stream before `onFinish` fires — the one termination path + // that skips `onFinish` — can still reconstruct the partial. This retains + // chunk references only; the reassembly runs solely in the failure fallback + // below, so the happy path pays nothing extra. + const bufferedChunks: UIMessageChunk[] = []; + const tappedStream = tapUIMessageChunks(uiStream, bufferedChunks); - return captured; + let status: PipeAndCaptureResult["status"] = "complete"; + let error: unknown; + try { + await pipeChat(tappedStream, { + signal: options?.signal, + spanName: options?.spanName ?? "stream response", + }); + // The pipe can drain cleanly on a stop — the source stream just ends + // early — so classify by the signal rather than relying on a throw. + if (options?.signal?.aborted) { + status = "aborted"; + } + } catch (err) { + if ((err instanceof Error && err.name === "AbortError") || options?.signal?.aborted) { + status = "aborted"; + } else { + status = "error"; + error = err; + } + } + + // `onFinish` fires even on abort, carrying the partial — but a hard stop can + // prevent it from firing at all, so race it against a timeout to avoid + // hanging the caller. Mirrors chat.agent's capture path. + await Promise.race([onFinishPromise, new Promise((r) => setTimeout(r, 2_000))]); + + // A transport failure can abandon the UI stream before `onFinish` fires, so + // reconstruct the partial from the buffered chunks rather than losing output + // that already streamed. Only runs when `onFinish` produced nothing. + if (!captured && bufferedChunks.length > 0) { + captured = await assemblePartialFromChunks(bufferedChunks); + } + + return { + message: captured, + status, + finishReason: capturedFinishReason, + ...(error !== undefined ? { error } : {}), + }; } /** @@ -8903,8 +9059,8 @@ async function pipeChatAndCapture( * for (let turn = 0; turn < 100; turn++) { * const messages = await conversation.addIncoming(payload.messages, payload.trigger, turn); * const result = streamText({ model, messages }); - * const response = await chat.pipeAndCapture(result); - * if (response) await conversation.addResponse(response); + * const { message } = await chat.pipeAndCapture(result); + * if (message) await conversation.addResponse(message); * } * ``` */ @@ -9570,7 +9726,7 @@ function createChatSession( } let response: UIMessage | undefined; try { - response = await pipeChatAndCapture(source, { + const captured = await pipeChatAndCapture(source, { signal: combinedSignal, // On a non-final handover turn, thread the spliced partial so a // resumed tool round's tool-output chunks merge into the @@ -9579,18 +9735,18 @@ function createChatSession( // fresh response into the prior assistant message). ...(handoverThisTurn ? { originalMessages: accumulator.uiMessages } : {}), }); - } catch (error) { - if (error instanceof Error && error.name === "AbortError") { - if (runSignal.aborted) { - // Full cancel — don't accumulate - sessionMsgSub.off(); - await chatWriteTurnComplete(); - return undefined; - } - // Stop — fall through to accumulate partial response - } else { - throw error; + if (runSignal.aborted) { + // Full cancel — don't accumulate + sessionMsgSub.off(); + await chatWriteTurnComplete(); + return undefined; } + // Surface a genuine stream failure to the caller. A user stop + // (status "aborted") falls through so the partial is accumulated. + if (captured.status === "error") { + throw captured.error; + } + response = captured.message; } finally { // Detach at stream end (like the agent loop): the steering queue // can't inject anymore, so later arrivals must buffer for the next turn. diff --git a/packages/trigger-sdk/src/v3/test/test-session-handle.ts b/packages/trigger-sdk/src/v3/test/test-session-handle.ts index f97cdbcc0..860b7694e 100644 --- a/packages/trigger-sdk/src/v3/test/test-session-handle.ts +++ b/packages/trigger-sdk/src/v3/test/test-session-handle.ts @@ -124,32 +124,37 @@ export class TestSessionOutputChannel extends SessionOutputChannel { ): PipeStreamResult { const state = this.state; const readChunks: T[] = []; + let pipeError: unknown; let resolveDone!: () => void; const done = new Promise((resolve) => { resolveDone = resolve; }); (async () => { - const readable = ensureReadableStream(value); - const reader = readable.getReader(); + let reader: ReadableStreamDefaultReader | undefined; try { + const readable = ensureReadableStream(value); + reader = readable.getReader(); while (true) { const { done: d, value: v } = await reader.read(); - if (d) return; + if (d) break; readChunks.push(v as T); notify(state, v); } + } catch (err) { + // Mirror production: a source-stream error (or a throw from stream + // setup) rejects waitUntilComplete instead of being silently + // swallowed, so callers (e.g. chat.pipeAndCapture) can observe it. + pipeError = err; } finally { try { - reader.releaseLock(); + reader?.releaseLock(); } catch { // ignore } resolveDone(); } - })().catch(() => { - resolveDone(); - }); + })(); const replayStream = new ReadableStream({ async start(controller) { @@ -167,6 +172,7 @@ export class TestSessionOutputChannel extends SessionOutputChannel { }, waitUntilComplete: async () => { await done; + if (pipeError) throw pipeError; return emptyResult; }, }; @@ -262,7 +268,11 @@ export class TestSessionOutputChannel extends SessionOutputChannel { } } notify(this.state, synthetic); - return {}; + // Project a synthetic monotonic seq_num as the ack's `lastEventId`, + // mirroring what S2 returns in production (there it's the control + // record's seq_num). Using the running `.out` record count lets + // `chat.writeTurnComplete()` surface a real resume cursor in tests. + return { lastEventId: String(this.state.chunks.length) }; } /** diff --git a/packages/trigger-sdk/test/chat-pipe-and-capture.test.ts b/packages/trigger-sdk/test/chat-pipe-and-capture.test.ts new file mode 100644 index 000000000..a25cb2b21 --- /dev/null +++ b/packages/trigger-sdk/test/chat-pipe-and-capture.test.ts @@ -0,0 +1,230 @@ +// Import the test harness FIRST — this installs the resource catalog so +// `chat.customAgent()` calls below register their task functions correctly. +import { mockChatAgent } from "../src/v3/test/index.js"; + +import { describe, expect, it } from "vitest"; +import type { UIMessage } from "ai"; +import { simulateReadableStream, streamText } from "ai"; +import { MockLanguageModelV3 } from "ai/test"; +import type { LanguageModelV3StreamPart } from "@ai-sdk/provider"; +import { chat } from "../src/v3/ai.js"; +import type { PipeAndCaptureResult } from "../src/v3/ai.js"; + +// ── Helpers ──────────────────────────────────────────────────────────── + +function userMessage(text: string, id: string): UIMessage { + return { id, role: "user", parts: [{ type: "text", text }] }; +} + +function textChunks(text: string, opts?: { split?: boolean }): LanguageModelV3StreamPart[] { + const deltas = opts?.split ? text.split(" ").map((w, i) => (i === 0 ? w : ` ${w}`)) : [text]; + return [ + { type: "text-start", id: "t1" }, + ...deltas.map((delta) => ({ type: "text-delta" as const, id: "t1", delta })), + { type: "text-end", id: "t1" }, + { + type: "finish", + finishReason: { unified: "stop", raw: "stop" }, + usage: { + inputTokens: { total: 5, noCache: 5, cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: 5, text: 5, reasoning: undefined }, + }, + }, + ]; +} + +/** Model that streams `text` in one fast pass with a `stop` finish. */ +function fastModel(text: string) { + return new MockLanguageModelV3({ + doStream: async () => ({ stream: simulateReadableStream({ chunks: textChunks(text) }) }), + }); +} + +/** Model that streams `text` word-by-word with a wide gap before the final + * chunk, leaving a window to abort mid-stream after the first delta. */ +function slowModel(text: string) { + return new MockLanguageModelV3({ + doStream: async () => ({ + stream: simulateReadableStream({ + chunks: textChunks(text, { split: true }), + initialDelayInMs: 0, + chunkDelayInMs: 500, + }), + }), + }); +} + +function extractText(message: UIMessage | undefined): string { + if (!message) return ""; + return (message.parts as Array<{ type: string; text?: string }>) + .filter((p) => p.type === "text") + .map((p) => p.text ?? "") + .join(""); +} + +async function waitFor(check: () => boolean, timeoutMs = 5_000) { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + if (check()) return; + await new Promise((r) => setTimeout(r, 20)); + } + throw new Error("waitFor timed out"); +} + +function deltaCount(harness: { allChunks: unknown[] }): number { + return (harness.allChunks as { type?: string }[]).filter((c) => c.type === "text-delta").length; +} + +// ── Tests ────────────────────────────────────────────────────────────── + +describe("chat.pipeAndCapture", () => { + it("returns status 'complete' with the message and finish reason on a normal turn", async () => { + const captures: PipeAndCaptureResult[] = []; + const turnCompletes: Array<{ lastEventId?: string; sessionInEventId?: string }> = []; + + const agent = chat.customAgent({ + id: "pipe-capture.complete", + run: async () => { + const conversation = new chat.MessageAccumulator(); + const next = await chat.messages.waitWithIdleTimeout({ + idleTimeoutInSeconds: 60, + timeout: "1h", + }); + if (!next.ok) return; + const wire = next.output as { message?: UIMessage; trigger: string }; + const incoming = wire.message ? [wire.message] : []; + const messages = await conversation.addIncoming(incoming, wire.trigger, 0); + const result = streamText({ model: fastModel("hello world"), messages }); + const captured = await chat.pipeAndCapture(result); + captures.push(captured); + if (captured.message) await conversation.addResponse(captured.message); + turnCompletes.push(await chat.writeTurnComplete()); + }, + }); + + const harness = mockChatAgent(agent, { chatId: "pc-complete" }); + try { + await harness.sendMessage(userMessage("hi", "u-1")); + await waitFor(() => captures.length >= 1 && turnCompletes.length >= 1); + + expect(captures[0]!.status).toBe("complete"); + expect(extractText(captures[0]!.message)).toBe("hello world"); + expect(captures[0]!.finishReason).toBe("stop"); + expect(captures[0]!.error).toBeUndefined(); + // chat.writeTurnComplete() surfaces the .out resume cursor for the next + // turn. (sessionInEventId is a passthrough of the same value written to + // the session-in-event-id header; the in-memory harness doesn't track + // the .in dispatch cursor, so its value isn't asserted here.) + expect(typeof turnCompletes[0]!.lastEventId).toBe("string"); + expect(turnCompletes[0]!.lastEventId!.length).toBeGreaterThan(0); + } finally { + await harness.close(); + } + }); + + it("returns status 'error' with the thrown error and does not throw when the stream fails", async () => { + const captures: PipeAndCaptureResult[] = []; + let runThrew = false; + + // Synthetic source whose UI stream errors after emitting a partial. This + // deterministically drives the pipe-failure path without depending on the + // AI SDK's model-error handling. Chunks are delivered one-per-pull before + // the error so they aren't discarded — calling controller.error() in the + // same tick as enqueue() would reset the queue and drop them. + const partialChunks = [ + { type: "start", messageId: "a-err" }, + { type: "text-start", id: "t1" }, + { type: "text-delta", id: "t1", delta: "partial" }, + ]; + const erroringSource = { + toUIMessageStream() { + let i = 0; + return new ReadableStream({ + pull(controller) { + if (i < partialChunks.length) { + controller.enqueue(partialChunks[i++]); + } else { + controller.error(new Error("boom")); + } + }, + }); + }, + }; + + const agent = chat.customAgent({ + id: "pipe-capture.error", + run: async () => { + const next = await chat.messages.waitWithIdleTimeout({ + idleTimeoutInSeconds: 60, + timeout: "1h", + }); + if (!next.ok) return; + try { + captures.push(await chat.pipeAndCapture(erroringSource as never)); + } catch { + runThrew = true; + } + await chat.writeTurnComplete(); + }, + }); + + const harness = mockChatAgent(agent, { chatId: "pc-error" }); + try { + await harness.sendMessage(userMessage("go", "u-1")); + await waitFor(() => captures.length >= 1); + + expect(runThrew).toBe(false); + expect(captures[0]!.status).toBe("error"); + expect(captures[0]!.error).toBeInstanceOf(Error); + expect((captures[0]!.error as Error).message).toBe("boom"); + // The partial that streamed before the failure is reconstructed from the + // buffered chunks even though onFinish never fired on this hard-error path. + expect(extractText(captures[0]!.message)).toBe("partial"); + } finally { + await harness.close(); + } + }); + + it("returns status 'aborted' and preserves the partial message on a mid-stream stop", async () => { + const captures: PipeAndCaptureResult[] = []; + + const agent = chat.customAgent({ + id: "pipe-capture.aborted", + run: async () => { + const stop = chat.createStopSignal(); + const conversation = new chat.MessageAccumulator(); + const next = await chat.messages.waitWithIdleTimeout({ + idleTimeoutInSeconds: 60, + timeout: "1h", + }); + if (!next.ok) return; + const wire = next.output as { message?: UIMessage; trigger: string }; + const incoming = wire.message ? [wire.message] : []; + const messages = await conversation.addIncoming(incoming, wire.trigger, 0); + const result = streamText({ + model: slowModel("one two three four"), + messages, + abortSignal: stop.signal, + }); + captures.push(await chat.pipeAndCapture(result, { signal: stop.signal })); + await chat.writeTurnComplete(); + stop.cleanup(); + }, + }); + + const harness = mockChatAgent(agent, { chatId: "pc-aborted" }); + try { + void harness.sendMessage(userMessage("hi", "u-1")); + // Stop once the first delta has streamed but before the turn finishes. + await waitFor(() => deltaCount(harness) >= 1); + await harness.sendStop(); + await waitFor(() => captures.length >= 1); + + expect(captures[0]!.status).toBe("aborted"); + // The partial that streamed before the stop is preserved. + expect(extractText(captures[0]!.message).startsWith("one")).toBe(true); + } finally { + await harness.close(); + } + }); +}); From 0b2919465ccfed5b7a99e1a1c836e6c25979fded Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Tue, 21 Jul 2026 17:25:17 +0100 Subject: [PATCH 014/300] feat(webapp): redesign the side menu project and organization menus (#4066) Redesign of the main side menu: separates Projects and Accounts from the Organization menu and makes the menu resizable. **Main changes** - **Organization & Account menus**: the top-left is now a dedicated organization menu (Settings, Usage, Billing, Team, SSO, integrations), with a separate account menu beside it (Profile, PATs, Security, Logout). - **Project switcher**: a new Project section above the Environment selector. - **Resizable side menu**: drag the right edge to set a custom width (saved per user), or click the edge to collapse/expand. - **Environment selector**: reworked to match the Project menu, including dev-branch handling. - **Account Profile page**: redesigned into the Security page's row-and-divider layout. Preview URL: https://samejr-org-menu-update.triggerlabs.dev/ https://github.com/user-attachments/assets/9b199576-6037-4ea6-9bdb-3ee15265b8c2 --- .../side-menu-project-and-org-menus.md | 6 + .../app/assets/icons/AvatarCircleIcon.tsx | 23 +- .../webapp/app/assets/icons/ChainLinkIcon.tsx | 27 + .../icons/LeftSideMenuCollapsedIcon.tsx | 22 + .../app/assets/icons/LeftSideMenuIcon.tsx | 44 + apps/webapp/app/components/AskAI.tsx | 133 +- .../webapp/app/components/GlobalShortcuts.tsx | 40 + apps/webapp/app/components/Shortcuts.tsx | 19 +- .../app/components/UserProfilePhoto.tsx | 61 +- .../app/components/billing/FreePlanUsage.tsx | 8 +- .../environments/EnvironmentLabel.tsx | 10 +- .../components/navigation/AccountSideMenu.tsx | 5 +- .../navigation/EnvironmentSelector.tsx | 308 ++-- .../navigation/HelpAndFeedbackPopover.tsx | 278 +-- .../navigation/NotificationPanel.tsx | 2 +- .../OrganizationSettingsSideMenu.tsx | 25 +- .../app/components/navigation/SideMenu.tsx | 1629 ++++++++++++----- .../components/navigation/SideMenuHeader.tsx | 11 +- .../components/navigation/SideMenuItem.tsx | 94 +- .../components/navigation/SideMenuSection.tsx | 57 +- .../app/components/primitives/Tooltip.tsx | 6 +- apps/webapp/app/root.tsx | 2 + .../route.tsx | 3 +- .../_app.orgs.$organizationSlug/route.tsx | 38 +- .../app/routes/account._index/route.tsx | 117 +- .../routes/resources.preferences.sidemenu.tsx | 2 + .../app/routes/storybook.icons/route.tsx | 2 + .../services/dashboardPreferences.server.ts | 12 +- apps/webapp/app/tailwind.css | 44 +- 29 files changed, 2130 insertions(+), 898 deletions(-) create mode 100644 .server-changes/side-menu-project-and-org-menus.md create mode 100644 apps/webapp/app/assets/icons/ChainLinkIcon.tsx create mode 100644 apps/webapp/app/assets/icons/LeftSideMenuCollapsedIcon.tsx create mode 100644 apps/webapp/app/assets/icons/LeftSideMenuIcon.tsx create mode 100644 apps/webapp/app/components/GlobalShortcuts.tsx diff --git a/.server-changes/side-menu-project-and-org-menus.md b/.server-changes/side-menu-project-and-org-menus.md new file mode 100644 index 000000000..5558a1cec --- /dev/null +++ b/.server-changes/side-menu-project-and-org-menus.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Refreshed the side menu: separate organization and account menus, a new project switcher, and the menu is now resizable by dragging its edge. The account Profile page has also been redesigned. diff --git a/apps/webapp/app/assets/icons/AvatarCircleIcon.tsx b/apps/webapp/app/assets/icons/AvatarCircleIcon.tsx index 83e67c846..30240b51c 100644 --- a/apps/webapp/app/assets/icons/AvatarCircleIcon.tsx +++ b/apps/webapp/app/assets/icons/AvatarCircleIcon.tsx @@ -1,4 +1,4 @@ -export function AvatarCircleIcon({ className }: { className?: string }) { +function AvatarCircle({ className, strokeWidth }: { className?: string; strokeWidth: number }) { return ( - - + + ); } + +/** User avatar placeholder with a 2px stroke (the default). */ +export function AvatarCircleIcon({ className }: { className?: string }) { + return ; +} + +/** Thinner 1.5px-stroke variant of {@link AvatarCircleIcon}. */ +export function AvatarCircleIconThin({ className }: { className?: string }) { + return ; +} + +/** Thinnest 1.25px-stroke variant of {@link AvatarCircleIcon}. */ +export function AvatarCircleIconExtraThin({ className }: { className?: string }) { + return ; +} diff --git a/apps/webapp/app/assets/icons/ChainLinkIcon.tsx b/apps/webapp/app/assets/icons/ChainLinkIcon.tsx new file mode 100644 index 000000000..f2e002454 --- /dev/null +++ b/apps/webapp/app/assets/icons/ChainLinkIcon.tsx @@ -0,0 +1,27 @@ +export function ChainLinkIcon({ className }: { className?: string }) { + return ( + + + + + ); +} diff --git a/apps/webapp/app/assets/icons/LeftSideMenuCollapsedIcon.tsx b/apps/webapp/app/assets/icons/LeftSideMenuCollapsedIcon.tsx new file mode 100644 index 000000000..aa229af92 --- /dev/null +++ b/apps/webapp/app/assets/icons/LeftSideMenuCollapsedIcon.tsx @@ -0,0 +1,22 @@ +export function LeftSideMenuCollapsedIcon({ className }: { className?: string }) { + return ( + + + + + + ); +} diff --git a/apps/webapp/app/assets/icons/LeftSideMenuIcon.tsx b/apps/webapp/app/assets/icons/LeftSideMenuIcon.tsx new file mode 100644 index 000000000..759ff962b --- /dev/null +++ b/apps/webapp/app/assets/icons/LeftSideMenuIcon.tsx @@ -0,0 +1,44 @@ +import { motion } from "framer-motion"; +import { useState } from "react"; + +export function LeftSideMenuIcon({ + className, + hovered: controlledHovered, +}: { + className?: string; + /** Drives the animation when provided (e.g. parent hover); otherwise the icon uses its own hover. */ + hovered?: boolean; +}) { + const [internalHovered, setInternalHovered] = useState(false); + const isControlled = controlledHovered !== undefined; + const hovered = isControlled ? controlledHovered : internalHovered; + + return ( + setInternalHovered(true)} + onMouseLeave={isControlled ? undefined : () => setInternalHovered(false)} + > + + {/* Animate a transform (scaleX), not the SVG `width` attr — framer snaps the first animation + of an idle SVG geometry attribute. Left origin collapses the panel right-to-left. */} + + + ); +} diff --git a/apps/webapp/app/components/AskAI.tsx b/apps/webapp/app/components/AskAI.tsx index d62ffa5b3..0d32265c2 100644 --- a/apps/webapp/app/components/AskAI.tsx +++ b/apps/webapp/app/components/AskAI.tsx @@ -11,11 +11,12 @@ import { useSearchParams } from "@remix-run/react"; import DOMPurify from "dompurify"; import { motion } from "framer-motion"; import { marked } from "marked"; -import { useCallback, useEffect, useRef, useState } from "react"; +import { type ReactNode, useCallback, useEffect, useRef, useState } from "react"; import { useTypedRouteLoaderData } from "remix-typedjson"; import { AISparkleIcon } from "~/assets/icons/AISparkleIcon"; import { SparkleListIcon } from "~/assets/icons/SparkleListIcon"; import { useFeatures } from "~/hooks/useFeatures"; +import { useShortcutKeys } from "~/hooks/useShortcutKeys"; import { type loader } from "~/root"; import { Button } from "./primitives/Buttons"; import { Callout } from "./primitives/Callout"; @@ -38,6 +39,104 @@ function useKapaWebsiteId() { return routeMatch?.kapa.websiteId; } +/** Open/close state for the Ask AI dialog, including the `?aiHelp=` deep-link handling. */ +function useAskAIState() { + const [isOpen, setIsOpen] = useState(false); + const [initialQuery, setInitialQuery] = useState(); + const [searchParams, setSearchParams] = useSearchParams(); + + const openAskAI = useCallback((question?: string) => { + if (question) { + setInitialQuery(question); + } else { + setInitialQuery(undefined); + } + setIsOpen(true); + }, []); + + const closeAskAI = useCallback(() => { + setIsOpen(false); + setInitialQuery(undefined); + }, []); + + // Handle URL param functionality + useEffect(() => { + const aiHelp = searchParams.get("aiHelp"); + if (aiHelp) { + // Delay to avoid hCaptcha bot detection + window.setTimeout(() => openAskAI(aiHelp), 1000); + + // Clone instead of mutating in place + const next = new URLSearchParams(searchParams); + next.delete("aiHelp"); + setSearchParams(next); + } + }, [searchParams, openAskAI]); + + return { isOpen, setIsOpen, initialQuery, openAskAI, closeAskAI }; +} + +/** + * Hosts Ask AI (Kapa provider, ⌘I shortcut, dialog) for a menu that renders its own trigger. Wrap + * it around the popover, not inside, so the dialog and shortcut survive the popover closing. + * `children` receives the open function, or undefined when Ask AI is unavailable (self-hosted, no + * Kapa website id, or SSR). + */ +export function AskAIRoot({ + children, +}: { + children: (openAskAI: (() => void) | undefined) => ReactNode; +}) { + const { isManagedCloud } = useFeatures(); + const websiteId = useKapaWebsiteId(); + + if (!isManagedCloud || !websiteId) { + return <>{children(undefined)}; + } + + return ( + {children(undefined)}}> + {() => {children}} + + ); +} + +function AskAIRootProvider({ + websiteId, + children, +}: { + websiteId: string; + children: (openAskAI: () => void) => ReactNode; +}) { + const { isOpen, setIsOpen, initialQuery, openAskAI, closeAskAI } = useAskAIState(); + + useShortcutKeys({ + shortcut: { modifiers: ["mod"], key: "i", enabledOnInputElements: true }, + action: () => openAskAI(), + }); + + return ( + openAskAI(), + onAnswerGenerationCompleted: () => openAskAI(), + }, + }} + botProtectionMechanism="hcaptcha" + > + {children(() => openAskAI())} + + + ); +} + export function AskAI({ isCollapsed = false }: { isCollapsed?: boolean }) { const { isManagedCloud } = useFeatures(); const websiteId = useKapaWebsiteId(); @@ -72,37 +171,7 @@ type AskAIProviderProps = { }; function AskAIProvider({ websiteId, isCollapsed = false }: AskAIProviderProps) { - const [isOpen, setIsOpen] = useState(false); - const [initialQuery, setInitialQuery] = useState(); - const [searchParams, setSearchParams] = useSearchParams(); - - const openAskAI = useCallback((question?: string) => { - if (question) { - setInitialQuery(question); - } else { - setInitialQuery(undefined); - } - setIsOpen(true); - }, []); - - const closeAskAI = useCallback(() => { - setIsOpen(false); - setInitialQuery(undefined); - }, []); - - // Handle URL param functionality - useEffect(() => { - const aiHelp = searchParams.get("aiHelp"); - if (aiHelp) { - // Delay to avoid hCaptcha bot detection - window.setTimeout(() => openAskAI(aiHelp), 1000); - - // Clone instead of mutating in place - const next = new URLSearchParams(searchParams); - next.delete("aiHelp"); - setSearchParams(next); - } - }, [searchParams, openAskAI]); + const { isOpen, setIsOpen, initialQuery, openAskAI, closeAskAI } = useAskAIState(); return ( { + if (!isAdmin) return; + + const onKeyDown = (event: KeyboardEvent) => { + // Admin escape hatch: Cmd+Option+A (Ctrl+Alt+A on Windows) opens the admin dashboard, or stops + // impersonating. Avoids Escape — Chrome/macOS never delivers a keydown for Escape+modifier (why + // the old Cmd+Esc did nothing). Matched on `event.code`, not `event.key`, because Option makes + // "A" report "å" (so a raw listener, not the `event.key`-based useShortcutKeys hook). + if (event.code !== "KeyA" || !event.altKey || !(event.metaKey || event.ctrlKey)) { + return; + } + event.preventDefault(); + if (isImpersonating) { + submit(null, { action: "/resources/impersonation", method: "delete" }); + } else { + navigate(adminPath()); + } + }; + + document.addEventListener("keydown", onKeyDown); + return () => document.removeEventListener("keydown", onKeyDown); + }, [isAdmin, isImpersonating, navigate, submit]); + + return null; +} diff --git a/apps/webapp/app/components/Shortcuts.tsx b/apps/webapp/app/components/Shortcuts.tsx index 63b5fddf7..87e5d08f3 100644 --- a/apps/webapp/app/components/Shortcuts.tsx +++ b/apps/webapp/app/components/Shortcuts.tsx @@ -1,8 +1,8 @@ import { KeyboardIcon } from "~/assets/icons/KeyboardIcon"; import { useState } from "react"; import { useShortcutKeys } from "~/hooks/useShortcutKeys"; -import { Button } from "./primitives/Buttons"; import { Header3 } from "./primitives/Headers"; +import { SideMenuItemButton } from "./navigation/SideMenuItem"; import { Paragraph } from "./primitives/Paragraph"; import { Sheet, SheetContent, SheetHeader, SheetTitle, SheetTrigger } from "./primitives/SheetV3"; import { ShortcutKey } from "./primitives/ShortcutKey"; @@ -11,19 +11,12 @@ export function Shortcuts() { return ( - + trailing={} + /> diff --git a/apps/webapp/app/components/UserProfilePhoto.tsx b/apps/webapp/app/components/UserProfilePhoto.tsx index 4676594db..92c435aa1 100644 --- a/apps/webapp/app/components/UserProfilePhoto.tsx +++ b/apps/webapp/app/components/UserProfilePhoto.tsx @@ -1,31 +1,62 @@ -import { UserCircleIcon } from "@heroicons/react/24/solid"; +import { + AvatarCircleIcon, + AvatarCircleIconExtraThin, + AvatarCircleIconThin, +} from "~/assets/icons/AvatarCircleIcon"; import { useOptionalUser } from "~/hooks/useUser"; import { cn } from "~/utils/cn"; -export function UserProfilePhoto({ className }: { className?: string }) { +/** Stroke width (px) of the placeholder avatar icon shown when there is no photo. */ +type AvatarStrokeWidth = 1.25 | 1.5 | 2; + +const PLACEHOLDER_BY_STROKE_WIDTH = { + 1.25: AvatarCircleIconExtraThin, + 1.5: AvatarCircleIconThin, + 2: AvatarCircleIcon, +} as const; + +export function UserProfilePhoto({ + className, + strokeWidth = 2, +}: { + className?: string; + strokeWidth?: AvatarStrokeWidth; +}) { const user = useOptionalUser(); - return ; + return ( + + ); } export function UserAvatar({ avatarUrl, name, className, + strokeWidth = 2, }: { avatarUrl?: string | null; name?: string | null; className?: string; + strokeWidth?: AvatarStrokeWidth; }) { - return avatarUrl ? ( -
- {name -
- ) : ( - - ); + if (avatarUrl) { + return ( +
+ {name +
+ ); + } + + const PlaceholderIcon = PLACEHOLDER_BY_STROKE_WIDTH[strokeWidth]; + return ; } diff --git a/apps/webapp/app/components/billing/FreePlanUsage.tsx b/apps/webapp/app/components/billing/FreePlanUsage.tsx index 4add40087..d1f071656 100644 --- a/apps/webapp/app/components/billing/FreePlanUsage.tsx +++ b/apps/webapp/app/components/billing/FreePlanUsage.tsx @@ -27,11 +27,11 @@ export function FreePlanUsage({ to, percentage }: { to: string; percentage: numb )} >
-
- - Free Plan +
+ + Free Plan
- + Upgrade
diff --git a/apps/webapp/app/components/environments/EnvironmentLabel.tsx b/apps/webapp/app/components/environments/EnvironmentLabel.tsx index 8143e5e5e..3fd5d526f 100644 --- a/apps/webapp/app/components/environments/EnvironmentLabel.tsx +++ b/apps/webapp/app/components/environments/EnvironmentLabel.tsx @@ -81,12 +81,15 @@ export function EnvironmentLabel({ tooltipSideOffset = 34, tooltipSide = "right", disableTooltip = false, + truncate = true, }: { environment: Environment; className?: string; tooltipSideOffset?: number; tooltipSide?: "top" | "right" | "bottom" | "left"; disableTooltip?: boolean; + /** When false, the label clips without an ellipsis (side menu fades it in place). Defaults true. */ + truncate?: boolean; }) { const spanRef = useRef(null); const [isTruncated, setIsTruncated] = useState(false); @@ -113,7 +116,12 @@ export function EnvironmentLabel({ const content = ( {text} diff --git a/apps/webapp/app/components/navigation/AccountSideMenu.tsx b/apps/webapp/app/components/navigation/AccountSideMenu.tsx index ba7938add..f48de39e2 100644 --- a/apps/webapp/app/components/navigation/AccountSideMenu.tsx +++ b/apps/webapp/app/components/navigation/AccountSideMenu.tsx @@ -7,7 +7,6 @@ import { personalAccessTokensPath, rootPath, } from "~/utils/pathBuilder"; -import { AskAI } from "../AskAI"; import { LinkButton } from "../primitives/Buttons"; import { HelpAndFeedback } from "./HelpAndFeedbackPopover"; import { SideMenuHeader } from "./SideMenuHeader"; @@ -34,7 +33,7 @@ export function AccountSideMenu({ user }: { user: User }) { Back to app
-
+
-
); diff --git a/apps/webapp/app/components/navigation/EnvironmentSelector.tsx b/apps/webapp/app/components/navigation/EnvironmentSelector.tsx index 7e2a3efdb..0f51b2254 100644 --- a/apps/webapp/app/components/navigation/EnvironmentSelector.tsx +++ b/apps/webapp/app/components/navigation/EnvironmentSelector.tsx @@ -35,18 +35,26 @@ import { V4Badge } from "../V4Badge"; import { type SideMenuEnvironment, type SideMenuProject } from "./SideMenu"; import { Badge } from "../primitives/Badge"; +// Size this Env popover's items to match the Project popover (SIDE_MENU_POPOVER_ITEM_* in +// SideMenu.tsx). Only at these call sites, so shared EnvironmentLabel/EnvironmentCombo defaults stay. +const ENV_POPOVER_ITEM_ICON = "size-5"; +const ENV_POPOVER_ITEM_LABEL = "text-[0.90625rem] font-medium tracking-[-0.01em]"; + export function EnvironmentSelector({ organization, project, environment, className, isCollapsed = false, + isDragging = false, }: { organization: MatchedOrganization; project: SideMenuProject; environment: SideMenuEnvironment; className?: string; isCollapsed?: boolean; + /** True while the side menu is being drag-resized; keeps the row in its expanded arrangement. */ + isDragging?: boolean; }) { const { isManagedCloud } = useFeatures(); const [isMenuOpen, setIsMenuOpen] = useState(false); @@ -73,42 +81,58 @@ export function EnvironmentSelector({ button={ + {/* + In the side menu, opacity + max-width follow --sm-label-opacity (1 → 0): the label + fades in place and scales its width to 0 so it never holds width mid-drag. The + selector is also reused outside the side menu (BlankStatePanels, limits) where the var + is unset — the 0.2 max-width fallback pins a ~200px cap (0.2 * 1000px) so long names + ellipsis-truncate there instead of widening the control, while opacity stays 1. + */} + {/* + Chevron's 16px width follows --sm-label-opacity so an invisible span never holds width + mid-drag and pushes the row's clip edge into the icon. + */} - + } - content={environmentFullTitle(environment)} + content={`${environmentFullTitle(environment)} environment`} side="right" sideOffset={8} + // Tooltip only on the collapsed rail (expanded shows the label; this selector is also reused + // outside the side menu, where a hover tooltip is unwanted). hidden={!isCollapsed} + delayDuration={0} buttonClassName="h-8!" asChild + tabbable disableHoverableContent /> } + title={ + + } isSelected={env.id === environment.id} /> ); @@ -162,8 +192,12 @@ export function EnvironmentSelector({ )} title={
- - Upgrade + + Upgrade
} isSelected={false} @@ -176,8 +210,12 @@ export function EnvironmentSelector({ )} title={
- - Upgrade + + Upgrade
} isSelected={false} @@ -199,10 +237,6 @@ function Branches({ branchEnvironments: SideMenuEnvironment[]; currentEnvironment: SideMenuEnvironment; }) { - const organization = useOrganization(); - const project = useProject(); - const environment = useEnvironment(); - const { urlForEnvironment } = useEnvironmentSwitcher(); const navigation = useNavigation(); const [isMenuOpen, setMenuOpen] = useState(false); const timeoutRef = useRef(null); @@ -234,23 +268,6 @@ function Branches({ }, 150); }; - const activeBranches = branchEnvironments.filter((env) => env.archivedAt === null); - const state = - branchEnvironments.length === 0 - ? "no-branches" - : activeBranches.length === 0 - ? "no-active-branches" - : "has-branches"; - - // Only surface the active environment's archived-branch item in the submenu it - // actually belongs to. Both Development and Preview render this component, so - // without the parent check an archived dev branch would leak into the Preview - // submenu (and vice-versa). - const currentBranchIsArchived = - environment.archivedAt !== null && environment.parentEnvironmentId === parentEnvironment.id; - - const envTextClassName = environmentTextClassName(parentEnvironment); - return ( setMenuOpen(open)} open={isMenuOpen}>
@@ -263,7 +280,11 @@ function Branches({ textAlignLeft fullWidth > - + -
- {currentBranchIsArchived && ( - - - {environment.branchName} - - Archived - - } - icon={ - - } - isSelected={environment.id === currentEnvironment.id} - /> - )} - {state === "has-branches" ? ( - <> - {branchEnvironments - .filter((env) => env.archivedAt === null) - .map((env) => ( - - {env.branchName ?? DEFAULT_DEV_BRANCH} - - } - icon={ - - } - isSelected={env.id === currentEnvironment.id} - /> - ))} - - ) : state === "no-branches" ? ( -
-
- - Create your first branch -
- - Branches are a way to test new features in isolation before merging them into the - main environment. - - - Branches are only available when using or above. Read our{" "} - v4 upgrade guide to learn - more. - -
- ) : ( -
- All branches are archived. -
- )} -
-
- {parentEnvironment.type === "DEVELOPMENT" ? ( - } - leadingIconClassName="text-text-dimmed" - /> - ) : ( - } - leadingIconClassName="text-text-dimmed" - /> - )} -
+
); } + +/** + * Inner content of the branches popover (list, empty states, "Manage branches" footer). Shared by + * the `Branches` hover submenu and the side-menu Preview popover. + */ +export function BranchesPopoverContent({ + parentEnvironment, + branchEnvironments, + currentEnvironment, +}: { + parentEnvironment: SideMenuEnvironment; + branchEnvironments: SideMenuEnvironment[]; + currentEnvironment: SideMenuEnvironment; +}) { + const organization = useOrganization(); + const project = useProject(); + const environment = useEnvironment(); + const { urlForEnvironment } = useEnvironmentSwitcher(); + + const activeBranches = branchEnvironments.filter((env) => env.archivedAt === null); + const state = + branchEnvironments.length === 0 + ? "no-branches" + : activeBranches.length === 0 + ? "no-active-branches" + : "has-branches"; + + // Show the archived-branch item only in the submenu it belongs to: both Development and Preview + // render this, so without the parent check an archived dev branch leaks into Preview (and vice-versa). + const currentBranchIsArchived = + environment.archivedAt !== null && environment.parentEnvironmentId === parentEnvironment.id; + + const envTextClassName = environmentTextClassName(parentEnvironment); + + return ( + <> +
+ {parentEnvironment.type === "DEVELOPMENT" ? ( + } + leadingIconClassName="text-text-dimmed" + className={ENV_POPOVER_ITEM_LABEL} + /> + ) : ( + } + leadingIconClassName="text-text-dimmed" + className={ENV_POPOVER_ITEM_LABEL} + /> + )} +
+
+ {currentBranchIsArchived && ( + + + {environment.branchName} + + Archived + + } + icon={ + + } + isSelected={environment.id === currentEnvironment.id} + /> + )} + {state === "has-branches" ? ( + <> + {branchEnvironments + .filter((env) => env.archivedAt === null) + .map((env) => ( + + {env.branchName ?? DEFAULT_DEV_BRANCH} + + } + icon={ + + } + isSelected={env.id === currentEnvironment.id} + /> + ))} + + ) : state === "no-branches" ? ( +
+
+ + Create your first branch +
+ + Branches are a way to test new features in isolation before merging them into the main + environment. + + + Branches are only available when using or above. Read our{" "} + v4 upgrade guide to learn more. + +
+ ) : ( +
+ All branches are archived. +
+ )} +
+ + ); +} diff --git a/apps/webapp/app/components/navigation/HelpAndFeedbackPopover.tsx b/apps/webapp/app/components/navigation/HelpAndFeedbackPopover.tsx index cf3cb62aa..669271fe7 100644 --- a/apps/webapp/app/components/navigation/HelpAndFeedbackPopover.tsx +++ b/apps/webapp/app/components/navigation/HelpAndFeedbackPopover.tsx @@ -1,25 +1,27 @@ import { ArrowUpRightIcon } from "@heroicons/react/20/solid"; import { motion } from "framer-motion"; import { Fragment, useState } from "react"; +import { AISparkleIcon } from "~/assets/icons/AISparkleIcon"; import { BookIcon } from "~/assets/icons/BookIcon"; import { BulbIcon } from "~/assets/icons/BulbIcon"; +import { DropdownIcon } from "~/assets/icons/DropdownIcon"; import { EnvelopeIcon } from "~/assets/icons/EnvelopeIcon"; import { QuestionMarkIcon } from "~/assets/icons/QuestionMarkIcon"; import { RadarPulseIcon } from "~/assets/icons/RadarPulseIcon"; import { StarIcon } from "~/assets/icons/StarIcon"; import { useShortcutKeys } from "~/hooks/useShortcutKeys"; -import { sanitizeHttpUrl } from "~/utils/sanitizeUrl"; import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route"; import { useRecentChangelogs } from "~/routes/resources.platform-changelogs"; import { cn } from "~/utils/cn"; +import { sanitizeHttpUrl } from "~/utils/sanitizeUrl"; +import { AskAIRoot } from "../AskAI"; import { Feedback } from "../Feedback"; import { Shortcuts } from "../Shortcuts"; -import { Button } from "../primitives/Buttons"; import { Paragraph } from "../primitives/Paragraph"; import { Popover, PopoverContent, PopoverTrigger } from "../primitives/Popover"; import { ShortcutKey } from "../primitives/ShortcutKey"; import { SimpleTooltip } from "../primitives/Tooltip"; -import { SideMenuItem } from "./SideMenuItem"; +import { SideMenuItem, SideMenuItemButton } from "./SideMenuItem"; export function HelpAndFeedback({ disableShortcut = false, @@ -49,135 +51,161 @@ export function HelpAndFeedback({ - - - - - + {(openAskAI) => ( + + + + + {/* + Width + opacity follow --sm-label-opacity so the label tracks a drag both + directions (no CSS transition — it would lag the per-frame writes). + */} + + Help & Feedback + + + {/* + Hover chevron, only when expanded. Its 16px width follows --sm-label-opacity so + an invisible chevron never holds width mid-drag and clips the help icon. + */} + {!isCollapsed && ( + + + + )} + + } + content={ + Help & Feedback + - - + + + {openAskAI !== undefined && ( +
+ + } + onClick={() => { + setHelpMenuOpen(false); + openAskAI(); + }} + /> +
)} - shortcut={{ key: "h" }} - variant="medium/bright" - /> - - } - content={ - - Help & Feedback - - - } - side="right" - sideOffset={8} - hidden={!isCollapsed} - buttonClassName="h-8! w-full" - asChild - disableHoverableContent - /> - - -
- -
-
- - - - - Contact us… - - } - /> -
-
- What's new - {changelogs.map((entry) => ( - - ))} - -
- - - +
+ +
+
+ + + + + } + /> +
+
+ What's new + {changelogs.map((entry) => ( + + ))} + +
+ + + + )} + ); } diff --git a/apps/webapp/app/components/navigation/NotificationPanel.tsx b/apps/webapp/app/components/navigation/NotificationPanel.tsx index bb07b54da..78cb52ca5 100644 --- a/apps/webapp/app/components/navigation/NotificationPanel.tsx +++ b/apps/webapp/app/components/navigation/NotificationPanel.tsx @@ -121,7 +121,7 @@ export function NotificationPanel({ return ( -
+
{isCollapsed ? ( Back to app
-
+
+ {isManagedCloud && ( <> )} -
@@ -241,7 +241,6 @@ export function OrganizationSettingsSideMenu({
-
); diff --git a/apps/webapp/app/components/navigation/SideMenu.tsx b/apps/webapp/app/components/navigation/SideMenu.tsx index 6ddb54135..31001d05f 100644 --- a/apps/webapp/app/components/navigation/SideMenu.tsx +++ b/apps/webapp/app/components/navigation/SideMenu.tsx @@ -2,12 +2,18 @@ import { ArrowTopRightOnSquareIcon, ChevronRightIcon, ExclamationTriangleIcon, - PencilSquareIcon, } from "@heroicons/react/24/outline"; -import { Link, useFetcher, useNavigation } from "@remix-run/react"; +import { useFetcher, useNavigation, useSubmit } from "@remix-run/react"; import { LayoutGroup, motion } from "framer-motion"; -import { type ReactNode, useCallback, useEffect, useRef, useState } from "react"; -import simplur from "simplur"; +import { + type CSSProperties, + type PointerEvent as ReactPointerEvent, + type ReactNode, + useCallback, + useEffect, + useRef, + useState, +} from "react"; import { AIChatIcon } from "~/assets/icons/AIChatIcon"; import { AIPenIcon } from "~/assets/icons/AIPenIcon"; import { ArrowLeftRightIcon } from "~/assets/icons/ArrowLeftRightIcon"; @@ -17,6 +23,7 @@ import { BatchesIcon } from "~/assets/icons/BatchesIcon"; import { BellIcon } from "~/assets/icons/BellIcon"; import { Box3DIcon } from "~/assets/icons/Box3DIcon"; import { BugIcon } from "~/assets/icons/BugIcon"; +import { ChainLinkIcon } from "~/assets/icons/ChainLinkIcon"; import { ChartBarIcon } from "~/assets/icons/ChartBarIcon"; import { CodeSquareIcon } from "~/assets/icons/CodeSquareIcon"; import { ConcurrencyIcon } from "~/assets/icons/ConcurrencyIcon"; @@ -31,30 +38,48 @@ import { HomeIcon } from "~/assets/icons/HomeIcon"; import { IDIcon } from "~/assets/icons/IDIcon"; import { IntegrationsIcon } from "~/assets/icons/IntegrationsIcon"; import { KeyIcon } from "~/assets/icons/KeyIcon"; +import { LeftSideMenuCollapsedIcon } from "~/assets/icons/LeftSideMenuCollapsedIcon"; +import { LeftSideMenuIcon } from "~/assets/icons/LeftSideMenuIcon"; import { ListCheckedIcon } from "~/assets/icons/ListCheckedIcon"; import { LogsIcon } from "~/assets/icons/LogsIcon"; import { PlusIcon } from "~/assets/icons/PlusIcon"; import { QueuesIcon } from "~/assets/icons/QueuesIcon"; import { RunsIcon } from "~/assets/icons/RunsIcon"; +import { ShieldIcon } from "~/assets/icons/ShieldIcon"; import { SlidersIcon } from "~/assets/icons/SlidersIcon"; import { TasksIcon } from "~/assets/icons/TasksIcon"; import { UsageIcon } from "~/assets/icons/UsageIcon"; import { WaitpointTokenIcon } from "~/assets/icons/WaitpointTokenIcon"; +import { CreditCardIcon } from "~/assets/icons/CreditCardIcon"; +import { UserCrossIcon } from "~/assets/icons/UserCrossIcon"; +import { UserGroupIcon } from "~/assets/icons/UserGroupIcon"; +import { RolesIcon } from "~/assets/icons/RolesIcon"; +import { PadlockIcon } from "~/assets/icons/PadlockIcon"; +import { SlackIcon } from "~/assets/icons/SlackIcon"; +import { VercelLogo } from "~/components/integrations/VercelLogo"; import { Avatar } from "~/components/primitives/Avatar"; +import { UserProfilePhoto } from "~/components/UserProfilePhoto"; import { type MatchedEnvironment } from "~/hooks/useEnvironment"; import { useFeatureFlags } from "~/hooks/useFeatureFlags"; import { useFeatures } from "~/hooks/useFeatures"; import { type MatchedOrganization } from "~/hooks/useOrganizations"; import { type MatchedProject } from "~/hooks/useProject"; import { useShortcutKeys } from "~/hooks/useShortcutKeys"; +import { useShowSelfServe } from "~/hooks/useShowSelfServe"; import { useHasAdminAccess } from "~/hooks/useUser"; import { type UserWithDashboardPreferences } from "~/models/user.server"; -import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route"; +import { + useCurrentPlan, + useIsUsingRbacPlugin, + useIsUsingSsoPlugin, +} from "~/routes/_app.orgs.$organizationSlug/route"; import { type FeedbackType } from "~/routes/resources.feedback"; import { IncidentStatusPanel, useIncidentStatus } from "~/routes/resources.incidents"; import { cn } from "~/utils/cn"; import { accountPath, + accountSecurityPath, + personalAccessTokensPath, adminPath, branchesPath, concurrencyPath, @@ -63,13 +88,19 @@ import { newOrganizationPath, newProjectPath, organizationPath, + organizationRolesPath, organizationSettingsPath, + organizationSlackIntegrationPath, + organizationSsoPath, organizationTeamPath, + organizationVercelIntegrationPath, queryPath, regionsPath, v3ApiKeysPath, v3BatchesPath, + v3BillingLimitsPath, v3BillingPath, + v3PrivateConnectionsPath, v3BulkActionsPath, v3DashboardsLandingPath, v3DeploymentsPath, @@ -89,17 +120,22 @@ import { v3UsagePath, v3WaitpointTokensPath, } from "~/utils/pathBuilder"; -import { AskAI } from "../AskAI"; import { FreePlanUsage } from "../billing/FreePlanUsage"; import { ConnectionIcon, DevPresencePanel, useDevPresence } from "../DevPresence"; import { AlphaBadge, NewBadge } from "../FeatureBadges"; -import { ImpersonationBanner } from "../ImpersonationBanner"; import { Button, ButtonContent, LinkButton } from "../primitives/Buttons"; import { Dialog, DialogTrigger } from "../primitives/Dialog"; +import { type RenderIcon } from "../primitives/Icon"; import { Paragraph } from "../primitives/Paragraph"; -import { Popover, PopoverContent, PopoverMenuItem, PopoverTrigger } from "../primitives/Popover"; +import { Badge } from "../primitives/Badge"; +import { + Popover, + PopoverContent, + PopoverMenuItem, + PopoverSectionHeader, + PopoverTrigger, +} from "../primitives/Popover"; import { ShortcutKey } from "../primitives/ShortcutKey"; -import { TextLink } from "../primitives/TextLink"; import { SimpleTooltip, Tooltip, @@ -126,6 +162,109 @@ function getSectionCollapsed( return sideMenu?.collapsedSections?.[sectionId] ?? false; } +// Size popover items (org/project menus) to match the side-menu items, overriding the smaller +// small-menu-item defaults via tailwind-merge; icon carries the default dimmed color. +const SIDE_MENU_POPOVER_ITEM_ICON = "h-5 w-5 text-text-dimmed"; +const SIDE_MENU_POPOVER_ITEM_LABEL = "text-[0.90625rem] font-medium tracking-[-0.01em]"; + +// Impersonation accent (menu border + "Stop impersonating"). Full class strings so Tailwind's +// static scanner picks them up. +const IMPERSONATION_ACCENT = { + border: "border-yellow-500/80", + text: "text-yellow-500/80", +}; + +// --- Resizable side menu ----------------------------------------------------- +// A drag handle on the right edge resizes the menu. Width-driven visuals read two CSS variables +// written to the root each frame (no React re-render, so drags stay smooth): +// --sm-collapse: 0 (>= default width) → 1 (collapsed) +// --sm-label-opacity: 1 → 0, a faster fade curve of --sm-collapse + +/** Collapsed rail width in px (matches the previous `w-11`). */ +const COLLAPSED_WIDTH = 44; +/** The default/again-expanded width in px (matches the previous `w-56`). */ +const DEFAULT_WIDTH = 224; +/** The widest the menu can be dragged, in px. */ +const MAX_WIDTH = 400; +/** Duration of the collapse/expand/snap animation, in ms. */ +const COLLAPSE_ANIM_MS = 200; +/** Fraction of the collapse range over which labels fade to 0 (0.6 = fully faded at 60% collapsed). */ +const LABEL_FADE_FRACTION = 0.6; +/** + * Snap thresholds as collapse progress (0 = default, 1 = collapsed): release at <= threshold springs + * open, past it collapses. Separate per direction so releasing early continues the gesture. + */ +const COLLAPSE_SNAP_THRESHOLD = 0.25; +const EXPAND_SNAP_THRESHOLD = 0.9; +/** Pointer travel (px) below which a press on the handle counts as a click (toggle), not a drag. */ +const DRAG_CLICK_THRESHOLD = 4; + +/** Left/right padding of the pinned top section + scroll body, interpolated 10px → 4px by --sm-collapse. */ +const SIDE_MENU_PAD_X = `calc(0.625rem - 0.375rem * var(--sm-collapse, 0))`; +/** + * Scroll-body right padding DURING a transition (settled-open uses the reserved gutter instead). + * Interpolates from the measured gutter width (seamless handoff) to 4px collapsed; the 8px fallback + * is only for the first paint before `--sm-sb-gutter` is measured. + */ +const SIDE_MENU_SCROLL_PAD_RIGHT = `calc(var(--sm-sb-gutter, 8px) - (var(--sm-sb-gutter, 8px) - 0.25rem) * var(--sm-collapse, 0))`; +/** + * Hover chevron: its 16px width follows --sm-label-opacity so an invisible chevron never holds width + * mid-drag and pushes the row's clip edge into the icon. Opacity stays class-driven (hover-only). + */ +const SIDE_MENU_CHEVRON_STYLE = { + maxWidth: "calc(var(--sm-label-opacity, 1) * 16px)", +} as const; +/** + * Selector row label (org/project/env): opacity follows --sm-label-opacity to fade both directions + * without popping in on drag-open. The generous max-width cap fades the text in place rather than + * truncating it, but still scales to 0 so an invisible label never holds width and clips the icon. + */ +const SIDE_MENU_SELECTOR_LABEL_STYLE = { + maxWidth: "calc(var(--sm-label-opacity, 1) * 1000px)", + opacity: "var(--sm-label-opacity, 1)", +} as const; + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +/** Collapse progress (0 = at/above default width, 1 = collapsed) for a given px width. */ +function widthToProgress(width: number) { + return clamp((DEFAULT_WIDTH - width) / (DEFAULT_WIDTH - COLLAPSED_WIDTH), 0, 1); +} + +/** Label opacity (1 → 0) for a given collapse progress, using the faster fade curve. */ +function progressToLabelOpacity(progress: number) { + return clamp((LABEL_FADE_FRACTION - progress) / LABEL_FADE_FRACTION, 0, 1); +} + +/** cubic-bezier(0.4, 0, 0.2, 1) — standard easing for the rAF tween, matching the CSS transitions. */ +function easeStandard(t: number) { + // Solve the bezier for x = t, then return y. Control points: p1 = (0.4, 0), p2 = (0.2, 1). + const x1 = 0.4; + const y1 = 0; + const x2 = 0.2; + const y2 = 1; + const cx = 3 * x1; + const bx = 3 * (x2 - x1) - cx; + const ax = 1 - cx - bx; + const cy = 3 * y1; + const by = 3 * (y2 - y1) - cy; + const ay = 1 - cy - by; + const sampleX = (u: number) => ((ax * u + bx) * u + cx) * u; + const sampleY = (u: number) => ((ay * u + by) * u + cy) * u; + const sampleDerivativeX = (u: number) => (3 * ax * u + 2 * bx) * u + cx; + // Newton-Raphson to invert x(u) = t. + let u = t; + for (let i = 0; i < 6; i++) { + const x = sampleX(u) - t; + const dx = sampleDerivativeX(u); + if (Math.abs(x) < 1e-4 || Math.abs(dx) < 1e-6) break; + u -= x / dx; + } + return sampleY(clamp(u, 0, 1)); +} + type SideMenuUser = Pick< UserWithDashboardPreferences, "email" | "admin" | "dashboardPreferences" @@ -155,14 +294,49 @@ export function SideMenu({ organization, organizations, }: SideMenuProps) { - const borderRef = useRef(null); - const [showHeaderDivider, setShowHeaderDivider] = useState(false); const [isCollapsed, setIsCollapsed] = useState( user.dashboardPreferences.sideMenu?.isCollapsed ?? false ); + const [isDragging, setIsDragging] = useState(false); + // True during a click/⌘B/release-snap animation. With isDragging, marks any in-flight transition + // (the gutter is only reserved once settled — see `showReservedGutter`). + const [isAnimating, setIsAnimating] = useState(false); + // Direction of an in-flight drag, for the Free-plan banner slide. A drag that started expanded is a + // close (the banner tracks it down); one that started collapsed is an open (the banner stays hidden + // until fully open, then rises). Only meaningful while `isDragging`. + const [dragStartedCollapsed, setDragStartedCollapsed] = useState(false); + + // --- Resize state (see the module constants above) --- + const rootRef = useRef(null); + const rafRef = useRef(null); + // Mirror of `isCollapsed` for the drag handlers (outside React's render cycle; no stale closures). + const isCollapsedRef = useRef(isCollapsed); + // The last-committed expanded width; animation targets and re-expansion read from here. + const expandedWidthRef = useRef( + clamp(user.dashboardPreferences.sideMenu?.width ?? DEFAULT_WIDTH, DEFAULT_WIDTH, MAX_WIDTH) + ); + // Frozen first-paint width; never changes, so React never fights the imperative width writes. + const initialWidthRef = useRef( + (user.dashboardPreferences.sideMenu?.isCollapsed ?? false) + ? COLLAPSED_WIDTH + : expandedWidthRef.current + ); + const widthRef = useRef(initialWidthRef.current); + const progressRef = useRef((user.dashboardPreferences.sideMenu?.isCollapsed ?? false) ? 1 : 0); + // Frozen initial style (incl. CSS vars) so the SSR HTML has the right collapsed/expanded visuals + // (no pre-hydration flash). Stable identity, so React never rewrites it after writeVisual. + const initialStyleRef = useRef({ + width: initialWidthRef.current, + "--sm-collapse": String(progressRef.current), + "--sm-label-opacity": String(progressToLabelOpacity(progressRef.current)), + } as CSSProperties); + // Removes an in-flight drag's window listeners (set on pointerdown; cleared on finish/unmount). + const dragCleanupRef = useRef<(() => void) | null>(null); + const preferencesFetcher = useFetcher(); const pendingPreferencesRef = useRef<{ isCollapsed?: boolean; + width?: number; sectionId?: SideMenuSectionId; sectionCollapsed?: boolean; }>({}); @@ -179,6 +353,7 @@ export function SideMenu({ const persistSideMenuPreferences = useCallback( (data: { isCollapsed?: boolean; + width?: number; sectionId?: SideMenuSectionId; sectionCollapsed?: boolean; }) => { @@ -202,6 +377,9 @@ export function SideMenu({ if (pending.isCollapsed !== undefined) { formData.append("isCollapsed", String(pending.isCollapsed)); } + if (pending.width !== undefined) { + formData.append("width", String(pending.width)); + } if (pending.sectionId !== undefined && pending.sectionCollapsed !== undefined) { formData.append("sectionId", pending.sectionId); formData.append("sectionCollapsed", String(pending.sectionCollapsed)); @@ -216,41 +394,243 @@ export function SideMenu({ [user.isImpersonating, preferencesFetcher] ); - // Flush pending preferences on unmount to avoid losing the last toggle + // Flush routine in a ref so the unmount effect can have empty deps. `useFetcher` returns a fresh + // object each render, so depending on it would fire the cleanup (flushing the debounce) every + // render — and drags re-render constantly — instead of only on unmount. + const flushPendingPreferencesRef = useRef<() => void>(); + flushPendingPreferencesRef.current = () => { + if (debounceTimeoutRef.current) { + clearTimeout(debounceTimeoutRef.current); + } + if (user.isImpersonating) return; + const pending = pendingPreferencesRef.current; + const hasPendingChanges = + pending.isCollapsed !== undefined || + pending.width !== undefined || + (pending.sectionId !== undefined && pending.sectionCollapsed !== undefined); + if (!hasPendingChanges) return; + + const formData = new FormData(); + if (pending.isCollapsed !== undefined) { + formData.append("isCollapsed", String(pending.isCollapsed)); + } + if (pending.width !== undefined) { + formData.append("width", String(pending.width)); + } + if (pending.sectionId !== undefined && pending.sectionCollapsed !== undefined) { + formData.append("sectionId", pending.sectionId); + formData.append("sectionCollapsed", String(pending.sectionCollapsed)); + } + preferencesFetcher.submit(formData, { + method: "POST", + action: "/resources/preferences/sidemenu", + }); + pendingPreferencesRef.current = {}; + }; + + // Flush pending preferences on unmount. Empty deps so cleanup runs only on a real unmount + // (see flushPendingPreferencesRef). + useEffect(() => { + return () => flushPendingPreferencesRef.current?.(); + }, []); + + // Measure the reserved scrollbar-gutter width once and expose it as `--sm-sb-gutter` (the padding + // hands off to it seamlessly; platform-dependent, so it must be measured). A probe with the same + // scrollbar classes is measured so the value matches what that styling reserves. + useEffect(() => { + const el = rootRef.current; + if (!el) return; + const probe = document.createElement("div"); + probe.className = "scrollbar-gutter-stable scrollbar-thumb-on-hover"; + probe.style.cssText = + "position:absolute;top:-9999px;left:-9999px;width:100px;height:100px;overflow-y:auto;visibility:hidden;"; + document.body.appendChild(probe); + const gutter = probe.offsetWidth - probe.clientWidth; + document.body.removeChild(probe); + if (gutter > 0) el.style.setProperty("--sm-sb-gutter", `${gutter}px`); + }, []); + + // Write width + collapse vars straight to the DOM (no re-render) so drags stay smooth; all + // width-driven visuals (labels, headers, padding, dividers) read them. + const writeVisual = useCallback((width: number, progress: number) => { + widthRef.current = width; + progressRef.current = progress; + const el = rootRef.current; + if (!el) return; + el.style.width = `${width}px`; + el.style.setProperty("--sm-collapse", String(progress)); + el.style.setProperty("--sm-label-opacity", String(progressToLabelOpacity(progress))); + }, []); + + // Animate width + progress over COLLAPSE_ANIM_MS (toggle button, ⌘B shortcut, release-snap). + const animateTo = useCallback( + (targetWidth: number, targetProgress: number) => { + if (rafRef.current !== null) cancelAnimationFrame(rafRef.current); + const startWidth = widthRef.current; + const startProgress = progressRef.current; + if (startWidth === targetWidth && startProgress === targetProgress) { + setIsAnimating(false); + return; + } + setIsAnimating(true); + const startTime = performance.now(); + const step = (now: number) => { + const t = clamp((now - startTime) / COLLAPSE_ANIM_MS, 0, 1); + const eased = easeStandard(t); + writeVisual( + startWidth + (targetWidth - startWidth) * eased, + startProgress + (targetProgress - startProgress) * eased + ); + if (t < 1) { + rafRef.current = requestAnimationFrame(step); + } else { + rafRef.current = null; + writeVisual(targetWidth, targetProgress); + setIsAnimating(false); + } + }; + rafRef.current = requestAnimationFrame(step); + }, + [writeVisual] + ); + + // Collapse/expand to a resting state and remember it. + const applyCollapsed = useCallback( + (next: boolean) => { + isCollapsedRef.current = next; + setIsCollapsed(next); + persistSideMenuPreferences({ isCollapsed: next }); + animateTo(next ? COLLAPSED_WIDTH : expandedWidthRef.current, next ? 1 : 0); + }, + [animateTo, persistSideMenuPreferences] + ); + + const handleToggleCollapsed = useCallback(() => { + applyCollapsed(!isCollapsedRef.current); + }, [applyCollapsed]); + + // Drag runs on window-level listeners so releasing anywhere finalizes it. (Pointer capture alone + // was unreliable: if the browser drops it mid-drag, the release never fires and the menu strands.) + const onHandlePointerDown = useCallback( + (e: ReactPointerEvent) => { + if (e.button !== 0) return; + e.preventDefault(); + // Capture just quiets hover states while dragging; the drag doesn't depend on it. + try { + e.currentTarget.setPointerCapture(e.pointerId); + } catch {} + if (rafRef.current !== null) { + cancelAnimationFrame(rafRef.current); + rafRef.current = null; + } + // Grabbing the handle interrupts any in-flight collapse/expand; clear the flag so a drag that + // rests via writeVisual (not animateTo) doesn't strand it true and keep the gutter hidden. + setIsAnimating(false); + // Never allow two concurrent drags. + dragCleanupRef.current?.(); + + const drag = { + startX: e.clientX, + startWidth: rootRef.current?.getBoundingClientRect().width ?? widthRef.current, + startedCollapsed: isCollapsedRef.current, + didDrag: false, + }; + + const cleanup = () => { + window.removeEventListener("pointermove", onMove); + window.removeEventListener("pointerup", onUp); + window.removeEventListener("pointercancel", onCancel); + window.removeEventListener("blur", onCancel); + document.body.style.userSelect = ""; + document.body.style.cursor = ""; + dragCleanupRef.current = null; + }; + + const onMove = (ev: PointerEvent) => { + const dx = ev.clientX - drag.startX; + if (!drag.didDrag) { + // Ignore tiny movement so a click still reads as a click (toggle), not a drag. + if (Math.abs(dx) < DRAG_CLICK_THRESHOLD) return; + drag.didDrag = true; + setIsDragging(true); + setDragStartedCollapsed(drag.startedCollapsed); + document.body.style.userSelect = "none"; + document.body.style.cursor = "col-resize"; + } + const width = clamp(drag.startWidth + dx, COLLAPSED_WIDTH, MAX_WIDTH); + writeVisual(width, widthToProgress(width)); + }; + + const onUp = () => { + cleanup(); + setIsDragging(false); + + // A press with no meaningful drag toggles the menu. + if (!drag.didDrag) { + applyCollapsed(!isCollapsedRef.current); + return; + } + + const width = widthRef.current; + // A drag that started collapsed is an opening gesture: flip the snap zone so an early + // release keeps opening. + const snapThreshold = drag.startedCollapsed + ? EXPAND_SNAP_THRESHOLD + : COLLAPSE_SNAP_THRESHOLD; + if (width >= DEFAULT_WIDTH) { + // Rest at the dragged width. + const rounded = Math.round(width); + expandedWidthRef.current = rounded; + isCollapsedRef.current = false; + setIsCollapsed(false); + persistSideMenuPreferences({ isCollapsed: false, width: rounded }); + writeVisual(rounded, 0); + } else if (widthToProgress(width) <= snapThreshold) { + // Released near the default width — spring back open. + expandedWidthRef.current = DEFAULT_WIDTH; + isCollapsedRef.current = false; + setIsCollapsed(false); + persistSideMenuPreferences({ isCollapsed: false, width: DEFAULT_WIDTH }); + animateTo(DEFAULT_WIDTH, 0); + } else { + // Released deeper in (or over-dragged past min width) — collapse the rest of the way. + isCollapsedRef.current = true; + setIsCollapsed(true); + persistSideMenuPreferences({ isCollapsed: true }); + animateTo(COLLAPSED_WIDTH, 1); + } + }; + + const onCancel = () => { + cleanup(); + setIsDragging(false); + if (!drag.didDrag) return; + // Settle back to the current resting state. + animateTo( + isCollapsedRef.current ? COLLAPSED_WIDTH : expandedWidthRef.current, + isCollapsedRef.current ? 1 : 0 + ); + }; + + window.addEventListener("pointermove", onMove); + window.addEventListener("pointerup", onUp); + window.addEventListener("pointercancel", onCancel); + window.addEventListener("blur", onCancel); + dragCleanupRef.current = cleanup; + }, + [animateTo, applyCollapsed, persistSideMenuPreferences, writeVisual] + ); + + // Keep the drag handlers' collapsed mirror in sync; tear down any in-flight animation/drag on unmount. + useEffect(() => { + isCollapsedRef.current = isCollapsed; + }, [isCollapsed]); useEffect(() => { return () => { - if (debounceTimeoutRef.current) { - clearTimeout(debounceTimeoutRef.current); - } - if (user.isImpersonating) return; - const pending = pendingPreferencesRef.current; - const hasPendingChanges = - pending.isCollapsed !== undefined || - (pending.sectionId !== undefined && pending.sectionCollapsed !== undefined); - - if (hasPendingChanges) { - const formData = new FormData(); - if (pending.isCollapsed !== undefined) { - formData.append("isCollapsed", String(pending.isCollapsed)); - } - if (pending.sectionId !== undefined && pending.sectionCollapsed !== undefined) { - formData.append("sectionId", pending.sectionId); - formData.append("sectionCollapsed", String(pending.sectionCollapsed)); - } - preferencesFetcher.submit(formData, { - method: "POST", - action: "/resources/preferences/sidemenu", - }); - pendingPreferencesRef.current = {}; - } + if (rafRef.current !== null) cancelAnimationFrame(rafRef.current); + dragCleanupRef.current?.(); }; - }, [preferencesFetcher, user.isImpersonating]); - - const handleToggleCollapsed = () => { - const newIsCollapsed = !isCollapsed; - setIsCollapsed(newIsCollapsed); - persistSideMenuPreferences({ isCollapsed: newIsCollapsed }); - }; + }, []); /** Generic handler for any collapsible section - just pass the section ID */ const handleSectionToggle = useCallback( @@ -265,95 +645,84 @@ export function SideMenu({ action: handleToggleCollapsed, }); - useEffect(() => { - const handleScroll = () => { - if (borderRef.current) { - const shouldShowHeaderDivider = borderRef.current.scrollTop > 1; - if (showHeaderDivider !== shouldShowHeaderDivider) { - setShowHeaderDivider(shouldShowHeaderDivider); - } - } - }; + // Reserve the scrollbar gutter only when fully settled open (stops the list shifting as it starts/ + // stops overflowing). Dropped mid-transition so the right padding can animate instead of a fixed + // gutter snapping away (see SIDE_MENU_SCROLL_PAD_RIGHT). + const showReservedGutter = !isCollapsed && !isDragging && !isAnimating; - borderRef.current?.addEventListener("scroll", handleScroll); - return () => borderRef.current?.removeEventListener("scroll", handleScroll); - }, [showHeaderDivider]); + // Free-plan banner slide (see FreePlanBanner). "tracking" = a close in progress, so the banner + // follows --sm-collapse down and is gone by the halfway point; "hidden" = collapsed or an open in + // progress (stays off-screen); "shown" = settled open, so it rises back up. Drag and click/⌘B share + // this: a click drives --sm-collapse through the same animation, with isAnimating standing in for + // isDragging and isCollapsed giving the direction. + const bannerPhase: "shown" | "tracking" | "hidden" = isDragging + ? dragStartedCollapsed + ? "hidden" + : "tracking" + : isAnimating + ? isCollapsed + ? "tracking" + : "hidden" + : isCollapsed + ? "hidden" + : "shown"; return (
- -
-
-
- +
+
+
+
- {isAdmin && !user.isImpersonating ? ( - - - - - - - - Admin dashboard - - - - - ) : isAdmin && user.isImpersonating ? ( - - - - ) : null} + + +
-
-
- + +
+
{environment.type === "DEVELOPMENT" && project.engine === "V2" && ( - + @@ -363,6 +732,13 @@ export function SideMenu({
- +
+
+
+
-
@@ -697,16 +1082,17 @@ export function SideMenu({ > {isFreeUser && ( - - - + )}
@@ -809,30 +1195,34 @@ function V3DeprecationContent() { ); } -function ProjectSelector({ - project, +function OrgSelector({ organization, organizations, - user, isCollapsed = false, + isDragging = false, + isAdmin, + isImpersonating, }: { - project: SideMenuProject; organization: MatchedOrganization; organizations: MatchedOrganization[]; - user: SideMenuUser; isCollapsed?: boolean; + /** True while the menu is being drag-resized; keeps the row in its expanded arrangement. */ + isDragging?: boolean; + /** Account context, only used to render the collapsed-rail "Account" submenu (see below). */ + isAdmin: boolean; + isImpersonating: boolean; }) { const currentPlan = useCurrentPlan(); const [isOrgMenuOpen, setOrgMenuOpen] = useState(false); const navigation = useNavigation(); const { isManagedCloud } = useFeatures(); + const featureFlags = useFeatureFlags(); + const showSelfServe = useShowSelfServe(); + const isUsingRbacPlugin = useIsUsingRbacPlugin(); + const isUsingSsoPlugin = useIsUsingSsoPlugin(); - let plan: string | undefined = undefined; - if (currentPlan?.v3Subscription?.isPaying === false) { - plan = "Free"; - } else if (currentPlan?.v3Subscription?.isPaying === true) { - plan = currentPlan.v3Subscription.plan?.title; - } + const isPaying = currentPlan?.v3Subscription?.isPaying === true; + const planTitle = currentPlan?.v3Subscription?.plan?.title; useEffect(() => { setOrgMenuOpen(false); @@ -844,40 +1234,37 @@ function ProjectSelector({ button={ - - - {project.name ?? "Select a project"} + + {organization.title} - + } - content={`${organization.title} / ${project.name ?? "Select a project"}`} + content={organization.title} side="right" sideOffset={8} hidden={!isCollapsed} buttonClassName="h-8!" asChild + tabbable disableHoverableContent /> -
-
- - -
- -
- -
- {organization.title} -
- {plan && ( - - {plan} plan - - )} - {simplur`${organization.membersCount} member[|s]`} -
-
-
-
- - - Settings - - {isManagedCloud && ( - - - Usage - - )} -
-
+ {!isCollapsed && }
+ + {isManagedCloud && ( + + )} + {isManagedCloud && ( + + Billing + {isPaying && planTitle ? {planTitle} : null} +
+ } + icon={CreditCardIcon} + leadingIconClassName={SIDE_MENU_POPOVER_ITEM_ICON} + className={SIDE_MENU_POPOVER_ITEM_LABEL} + /> + )} + {isManagedCloud && showSelfServe && ( + + )} + + {featureFlags.hasPrivateConnections && ( + + )} + {isUsingRbacPlugin && ( + + )} + {isUsingSsoPlugin && ( + + )} + +
+
+ {organizations.length > 1 ? ( + + ) : ( + + )} +
+ {/* Collapsed: the account button is hidden, so surface Account as a submenu here (the only + always-reachable menu on the rail). */} + {isCollapsed && ( +
+ }> + + +
+ )} + + + ); +} + +/** + * Account menu entries, shared by the standalone account popover (expanded rail) and the "Account" + * submenu in the org popover (collapsed rail). + */ +function AccountMenuItems({ + isAdmin, + isImpersonating, +}: { + isAdmin: boolean; + isImpersonating: boolean; +}) { + const submit = useSubmit(); + const stopImpersonating = () => + submit(null, { action: "/resources/impersonation", method: "delete" }); + + return ( + <> + {isAdmin && ( +
+ {isImpersonating ? ( + + Stop impersonating + +
+ } + icon={UserCrossIcon} + onClick={stopImpersonating} + leadingIconClassName={cn(SIDE_MENU_POPOVER_ITEM_ICON, IMPERSONATION_ACCENT.text)} + className={SIDE_MENU_POPOVER_ITEM_LABEL} + /> + ) : ( + + Admin dashboard + +
+ } + icon={HomeIcon} + leadingIconClassName={SIDE_MENU_POPOVER_ITEM_ICON} + className={SIDE_MENU_POPOVER_ITEM_LABEL} + /> + )} +
+ )} +
+ + + +
+
+ +
+ + ); +} + +function AccountMenu({ isAdmin, isImpersonating }: { isAdmin: boolean; isImpersonating: boolean }) { + const [isOpen, setIsOpen] = useState(false); + const navigation = useNavigation(); + + useEffect(() => { + setIsOpen(false); + }, [navigation.location?.pathname]); + + // The admin shortcut lives in so it works everywhere, not just where this menu is. + return ( + setIsOpen(open)} open={isOpen}> + + + + } + content="Account" + side="bottom" + sideOffset={8} + asChild + tabbable + disableHoverableContent + /> + + + + + + ); +} + +function ProjectSelector({ + project, + organization, + environment, + isCollapsed = false, + isDragging = false, + className, +}: { + project: SideMenuProject; + organization: MatchedOrganization; + environment: SideMenuEnvironment; + isCollapsed?: boolean; + /** True while the menu is being drag-resized; keeps the row in its expanded arrangement. */ + isDragging?: boolean; + className?: string; +}) { + const [isMenuOpen, setIsMenuOpen] = useState(false); + const navigation = useNavigation(); + + useEffect(() => { + setIsMenuOpen(false); + }, [navigation.location?.pathname]); + + return ( + setIsMenuOpen(open)} open={isMenuOpen}> + + + + + + {project.name ?? "Select a project"} + + + + + + + + } + content={project.name ?? "Select a project"} + side="right" + sideOffset={8} + hidden={!isCollapsed} + buttonClassName="h-8!" + asChild + tabbable + disableHoverableContent + /> + +
+ + +
+
{organization.projects.map((p) => { const isSelected = p.id === project.id; return ( @@ -957,46 +1609,91 @@ function ProjectSelector({ } isSelected={isSelected} icon={isSelected ? FolderOpenIcon : FolderClosedIcon} - leadingIconClassName="text-indigo-500" + leadingIconClassName="h-5 w-5 text-indigo-500" + className={SIDE_MENU_POPOVER_ITEM_LABEL} /> ); })} - -
-
- {organizations.length > 1 ? ( - - ) : ( - - )} -
-
- -
-
-
); } +/** + * Hover-expandable submenu row for side-menu popovers (Account, Switch organization, Integrations): + * a menu item with a trailing chevron that reveals `children` in a popover to the right, with a + * short close delay so the pointer can cross the gap. + */ +function SideMenuPopoverSubMenu({ + title, + icon, + leadingIconClassName, + children, +}: { + title: string; + icon: RenderIcon; + leadingIconClassName?: string; + children: ReactNode; +}) { + const navigation = useNavigation(); + const [isOpen, setIsOpen] = useState(false); + const timeoutRef = useRef(null); + + useEffect(() => { + return () => { + if (timeoutRef.current) clearTimeout(timeoutRef.current); + }; + }, []); + + // Close the submenu on navigation (the parent popover closes too). + useEffect(() => { + setIsOpen(false); + }, [navigation.location?.pathname]); + + const openNow = () => { + if (timeoutRef.current) clearTimeout(timeoutRef.current); + setIsOpen(true); + }; + const closeSoon = () => { + // Small delay before closing so the pointer can move onto the content. + timeoutRef.current = setTimeout(() => setIsOpen(false), 150); + }; + + return ( + setIsOpen(open)} open={isOpen}> +
+ + + {title} + + + + {children} + +
+ + ); +} + function SwitchOrganizations({ organizations, organization, @@ -1004,159 +1701,172 @@ function SwitchOrganizations({ organizations: MatchedOrganization[]; organization: MatchedOrganization; }) { - const navigation = useNavigation(); - const [isMenuOpen, setMenuOpen] = useState(false); - const timeoutRef = useRef(null); - - // Clear timeout on unmount - useEffect(() => { - return () => { - if (timeoutRef.current) { - clearTimeout(timeoutRef.current); - } - }; - }, []); - - useEffect(() => { - setMenuOpen(false); - }, [navigation.location?.pathname]); - - const handleMouseEnter = () => { - if (timeoutRef.current) { - clearTimeout(timeoutRef.current); - } - setMenuOpen(true); - }; - - const handleMouseLeave = () => { - // Small delay before closing to allow moving to the content - timeoutRef.current = setTimeout(() => { - setMenuOpen(false); - }, 150); - }; - return ( - setMenuOpen(open)} open={isMenuOpen}> -
- - - Switch organization - - - -
- {organizations.map((org) => ( - } - leadingIconClassName="text-text-dimmed" - isSelected={org.id === organization.id} - /> - ))} -
-
- -
- + +
+
- +
+ {organizations.map((org) => ( + } + leadingIconClassName="text-text-dimmed" + className={SIDE_MENU_POPOVER_ITEM_LABEL} + isSelected={org.id === organization.id} + /> + ))} +
+ ); } -function SelectorDivider() { +function Integrations({ organization }: { organization: MatchedOrganization }) { return ( - - - + +
+ + +
+ ); } -/** Helper component that fades out but preserves width (collapses to 0 width) */ +/** + * Fades out and collapses to 0 width via the menu's `--sm-label-opacity` variable, tracking a drag + * in real time (no CSS opacity transition — it would lag the per-frame variable writes). + */ function CollapsibleElement({ - isCollapsed, + isDragging = false, children, className, }: { - isCollapsed: boolean; + /** Only blocks clicks on the fading button mid-drag; the hiding is width+opacity below. */ + isDragging?: boolean; children: ReactNode; className?: string; }) { + // Width AND opacity follow --sm-label-opacity: opacity alone would leave the invisible button + // holding 32px of row width, pushing the primary item's clip edge into its icon ("masked" mid-drag). + // Shrinking width on the same curve hands that space back. No CSS transition (it would lag the writes). return (
{children}
); } -/** Helper component that fades out and collapses height completely */ -function CollapsibleHeight({ - isCollapsed, - children, - className, +/** + * The Free-plan banner at the foot of the menu. On close it doesn't collapse or slide on its own: its + * reserved height collapses (tracking --sm-collapse, gone by the halfway point) and, because the bottom + * section is pinned to the bottom, that pushes the whole section (Help & Feedback + this banner) down so + * the full-height banner slides off the bottom edge. On open it lags, waiting for the settled "shown" + * phase, then rises back up via translateY. Height is measured so the reclaimed space matches the banner. + */ +function FreePlanBanner({ + to, + percentage, + phase, }: { - isCollapsed: boolean; - children: ReactNode; - className?: string; + to: string; + percentage: number; + phase: "shown" | "tracking" | "hidden"; }) { + const contentRef = useRef(null); + const [height, setHeight] = useState(0); + + useEffect(() => { + const el = contentRef.current; + if (!el) return; + const measure = () => setHeight(el.offsetHeight); + measure(); + const ro = new ResizeObserver(measure); + ro.observe(el); + return () => ro.disconnect(); + }, []); + + // Close progress, doubled + clamped so the banner is fully gone by the time the menu is halfway shut. + const closeProgress = "min(var(--sm-collapse, 0) * 2, 1)"; + // Slide a little past its own height to clear the section padding + the viewport edge. + const offset = height + 24; + + const maxHeight = + phase === "shown" + ? height + ? `${height}px` + : "none" + : phase === "hidden" + ? "0px" + : `calc((1 - ${closeProgress}) * ${height}px)`; + // On close the banner no longer slides itself: its reserved height collapses (maxHeight) and, because + // the bottom section is pinned to the bottom, that drops Help & Feedback down while the full-height + // banner overflows off the bottom edge. Only the pop-up-from-hidden rise uses translateY, so "hidden" + // parks it below the edge; "shown" and "tracking" both sit at 0. + const translateY = phase === "hidden" ? `${offset}px` : "0px"; + // Fade out as it slides off (tracking --sm-collapse) and fade back in on the settled-open rise. + const opacity = phase === "shown" ? 1 : phase === "hidden" ? 0 : `calc(1 - ${closeProgress})`; + return (
-
{children}
+
+ +
); } function HelpAndAI({ isCollapsed, + isDragging, organizationId, projectId, + onToggleCollapsed, }: { isCollapsed: boolean; + isDragging: boolean; organizationId: string; projectId: string; + onToggleCollapsed: () => void; }) { return ( @@ -1172,126 +1882,77 @@ function HelpAndAI({ organizationId={organizationId} projectId={projectId} /> - +
); } -function AnimatedChevron({ - isHovering, +function CollapseMenuButton({ isCollapsed, + isDragging = false, + onToggle, }: { - isHovering: boolean; isCollapsed: boolean; + isDragging?: boolean; + onToggle: () => void; }) { - // When hovering and expanded: left chevron (pointing left to collapse) - // When hovering and collapsed: right chevron (pointing right to expand) - // When not hovering: straight vertical line - - const getRotation = () => { - if (!isHovering) return { top: 0, bottom: 0 }; - if (isCollapsed) { - // Right chevron - return { top: -17, bottom: 17 }; - } else { - // Left chevron - return { top: 17, bottom: -17 }; - } - }; - - const { top, bottom } = getRotation(); - - // Calculate horizontal offset to keep chevron centered when rotated - // Left chevron: translate left (-1.5px) - // Right chevron: translate right (+1.5px) - const getTranslateX = () => { - if (!isHovering) return 0; - return isCollapsed ? 1.5 : -1.5; - }; - - return ( - - {/* Top segment */} - - {/* Bottom segment */} - - - ); -} - -function CollapseToggle({ isCollapsed, onToggle }: { isCollapsed: boolean; onToggle: () => void }) { const [isHovering, setIsHovering] = useState(false); return ( -
- {/* Vertical line to mask the side menu border */} -
+ // Shrink-and-fade only while dragging CLOSED, where this sits beside Help & Feedback and would + // overlap it as the row narrows. Dragging OPEN it stays put: collapsed, this IS the expand + // affordance, and the 0->1 variable would make the icon grow from nothing. At rest: natural size. +
- + - + + - + {isCollapsed ? "Expand" : "Collapse"} @@ -1303,3 +1964,71 @@ function CollapseToggle({ isCollapsed, onToggle }: { isCollapsed: boolean; onTog
); } + +/** + * Resize affordance straddling the menu's right border: hover reveals an indigo line, drag resizes, + * click toggles collapsed/expanded, and the tooltip follows the pointer's Y. The strip extends 4px + * past the edge, so the menu root deliberately has no overflow-hidden (only its inner grid does). + */ +function ResizeHandle({ + isCollapsed, + isDragging, + onPointerDown, +}: { + isCollapsed: boolean; + isDragging: boolean; + onPointerDown: (e: ReactPointerEvent) => void; +}) { + // Fully controlled so open never flips controlled/uncontrolled mid-interaction; open requests + // during a drag are dropped. + const [isTooltipOpen, setTooltipOpen] = useState(false); + // Pointer Y within the strip — anchors the tooltip beside the cursor, not the strip's center. + const [anchorY, setAnchorY] = useState(0); + + return ( + + setTooltipOpen(open && !isDragging)} + > + +
{ + if (isDragging) return; + setAnchorY(Math.round(e.clientY - e.currentTarget.getBoundingClientRect().top)); + }} + className="group/resize absolute inset-y-0 -right-1 z-30 w-2 cursor-col-resize touch-none" + > +
+
+ + + Drag to resize + + {isCollapsed ? "Click to expand" : "Click to collapse"} + + + + + + + + + ); +} diff --git a/apps/webapp/app/components/navigation/SideMenuHeader.tsx b/apps/webapp/app/components/navigation/SideMenuHeader.tsx index 80143e9e0..8ccbfa7de 100644 --- a/apps/webapp/app/components/navigation/SideMenuHeader.tsx +++ b/apps/webapp/app/components/navigation/SideMenuHeader.tsx @@ -40,15 +40,8 @@ export function SideMenuHeader({

{visiblePart} {fadingPart && ( - - {fadingPart} - + // --sm-label-opacity morphs "Project" → "Proj" as the menu narrows (unset elsewhere → 1). + {fadingPart} )}

{children !== undefined ? ( diff --git a/apps/webapp/app/components/navigation/SideMenuItem.tsx b/apps/webapp/app/components/navigation/SideMenuItem.tsx index c47f949ab..f17051266 100644 --- a/apps/webapp/app/components/navigation/SideMenuItem.tsx +++ b/apps/webapp/app/components/navigation/SideMenuItem.tsx @@ -1,4 +1,9 @@ -import { type AnchorHTMLAttributes, type ReactNode } from "react"; +import { + type AnchorHTMLAttributes, + type ButtonHTMLAttributes, + forwardRef, + type ReactNode, +} from "react"; import { Link } from "@remix-run/react"; import { motion } from "framer-motion"; import { usePathName } from "~/hooks/usePathName"; @@ -14,6 +19,7 @@ export function SideMenuItem({ trailingIcon, trailingIconClassName, name, + nameClassName, to, badge, target, @@ -30,18 +36,14 @@ export function SideMenuItem({ trailingIcon?: RenderIcon; trailingIconClassName?: string; name: string; + nameClassName?: string; to: string; badge?: ReactNode; target?: AnchorHTMLAttributes["target"]; isCollapsed?: boolean; action?: ReactNode; disableIconHover?: boolean; - /** - * Visually indented variant — same item, just pushed further from - * the left edge so it reads as a child of the row above. Used for - * grouped sub-items like the Tasks > (Agents / Standard / Scheduled) - * cluster. The indent is only applied when the side menu is expanded. - */ + /** Indented variant for grouped sub-items; only applied when the menu is expanded. */ indented?: boolean; "data-action"?: string; }) { @@ -56,7 +58,7 @@ export function SideMenuItem({ target={target} data-action={dataAction} className={cn( - "group/menulink flex h-8 items-center gap-2 overflow-hidden rounded pl-1.75 pr-2", + "group/menulink flex h-8 items-center gap-2 overflow-hidden rounded pl-1.75 pr-2 focus-custom", isIndented ? "min-w-0 flex-1" : "w-full", isActive ? "bg-tertiary text-text-bright" @@ -75,32 +77,39 @@ export function SideMenuItem({ )} /> - - {name} - - {badge && !isCollapsed && ( - + - {badge} - - )} - {trailingIcon && !isCollapsed && ( - - )} + {name} + + {badge && !isCollapsed && ( +
{badge}
+ )} + {trailingIcon && !isCollapsed && ( + + )} +
); @@ -125,9 +134,11 @@ export function SideMenuItem({ buttonClassName="h-8! block w-full" hidden={!isCollapsed} asChild + tabbable disableHoverableContent /> {!isCollapsed && ( + // Fades with the labels via --sm-label-opacity (unset → fully visible).
{action}
@@ -152,7 +164,35 @@ export function SideMenuItem({ buttonClassName="h-8! block w-full" hidden={!isCollapsed} asChild + tabbable disableHoverableContent /> ); } + +/** Button styled to match {@link SideMenuItem}, for entries that open a dialog rather than navigate. */ +export const SideMenuItemButton = forwardRef< + HTMLButtonElement, + { icon: RenderIcon; name: string; trailing?: ReactNode } & ButtonHTMLAttributes +>(function SideMenuItemButton({ icon, name, trailing, className, type, ...props }, ref) { + return ( + + ); +}); diff --git a/apps/webapp/app/components/navigation/SideMenuSection.tsx b/apps/webapp/app/components/navigation/SideMenuSection.tsx index 70b5e0995..8225d9e70 100644 --- a/apps/webapp/app/components/navigation/SideMenuSection.tsx +++ b/apps/webapp/app/components/navigation/SideMenuSection.tsx @@ -1,5 +1,5 @@ import { AnimatePresence, motion } from "framer-motion"; -import React, { useCallback, useState } from "react"; +import React, { useCallback, useEffect, useRef, useState } from "react"; import { ToggleArrowIcon } from "~/assets/icons/ToggleArrowIcon"; type Props = { @@ -14,9 +14,7 @@ type Props = { headerAction?: React.ReactNode; }; -/** A collapsible section for the side menu - * The collapsed state is passed in as a prop, and there's a callback when it's toggled so we can save the state. - */ +/** A collapsible section for the side menu. Collapsed state is controlled via props + a toggle callback. */ export function SideMenuSection({ title, initialCollapsed = false, @@ -27,6 +25,7 @@ export function SideMenuSection({ headerAction, }: Props) { const [isCollapsed, setIsCollapsed] = useState(initialCollapsed); + const contentRef = useRef(null); const handleToggle = useCallback(() => { const newIsCollapsed = !isCollapsed; @@ -34,22 +33,37 @@ export function SideMenuSection({ onCollapseToggle?.(newIsCollapsed); }, [isCollapsed, onCollapseToggle]); + // Collapsed items stay in the DOM (height 0) for the animation, so `inert` removes them from the + // tab order and a11y tree (it doesn't affect layout). Set the DOM property directly — React 18's + // `inert` prop handling is unreliable. + useEffect(() => { + if (contentRef.current) { + contentRef.current.inert = isCollapsed; + } + }, [isCollapsed]); + return (
{/* Header container - stays in DOM to preserve height */}
- {/* Header - fades out when sidebar is collapsed */} -