From 1a033b665b6f095180e4dd4365239d0a4a6cffbe Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 7 Jul 2026 11:51:59 +0100 Subject: [PATCH] fix(webapp,core): retry run resume through transient database outages (#4161) ## Summary When the platform database is briefly unreachable while a run is resuming from a wait, the run no longer fails with `TASK_EXECUTION_ABORTED`. The worker now retries the resume through the outage instead of aborting on the first blip. ## Root cause Resuming a run calls the engine's `continue` worker-action endpoint. That route caught every error and returned a `422`, which the worker's HTTP client treats as non-retryable. So a transient Prisma infrastructure error (for example `P1001` "Can't reach database server") was flattened into a permanent failure: the worker gave up, force-killed the run process, and completed it with `TASK_EXECUTION_ABORTED`. ## Fix - The `continue` route now lets infrastructure errors propagate to the generic 500 handler (message scrubbed, and retryable by the worker's HTTP client), the same treatment the trigger path already gives them via `isInfrastructureError`. Genuine validation errors (snapshot mismatch, invalid state) still return `422`, so a stale retry stays non-retryable. Resuming is idempotent server-side (guarded by the snapshot id), so retrying is safe. - The worker's `continueRunExecution` calls (both the runner-to-supervisor and supervisor-to-engine hops) retry with a longer, jittered backoff so they can ride out an outage lasting tens of seconds, and the jitter keeps a fleet of resuming runs from stampeding the database the moment it recovers. Builds on #3960, which scrubbed the leaked message on these routes but left the status non-retryable. No changeset: this is a server-side behaviour fix recorded via `.server-changes`. The `@trigger.dev/core` edits are internal run-engine worker plumbing, not a public API change. --- .server-changes/resume-retry-transient-db.md | 6 ++++++ ...d.snapshots.$snapshotFriendlyId.continue.ts | 18 ++++++++++++++++-- .../src/v3/runEngineWorker/supervisor/http.ts | 15 +++++++++++++++ .../src/v3/runEngineWorker/workload/http.ts | 14 ++++++++++++++ 4 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 .server-changes/resume-retry-transient-db.md diff --git a/.server-changes/resume-retry-transient-db.md b/.server-changes/resume-retry-transient-db.md new file mode 100644 index 000000000..bdd704329 --- /dev/null +++ b/.server-changes/resume-retry-transient-db.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Runs resuming after a wait no longer fail with TASK_EXECUTION_ABORTED when the database is briefly unreachable; the resume endpoint returns a retryable response for transient infrastructure errors instead of a permanent one. diff --git a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.continue.ts b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.continue.ts index 85b323398..3c264adcd 100644 --- a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.continue.ts +++ b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.continue.ts @@ -4,7 +4,7 @@ import type { WorkerApiContinueRunExecutionRequestBody } from "@trigger.dev/core import { z } from "zod"; import { logger } from "~/services/logger.server"; import { createLoaderWorkerApiRoute } from "~/services/routeBuilders/apiBuilder.server"; -import { clientSafeErrorMessage } from "~/utils/prismaErrors"; +import { clientSafeErrorMessage, isInfrastructureError } from "~/utils/prismaErrors"; export const loader = createLoaderWorkerApiRoute( { @@ -31,7 +31,21 @@ export const loader = createLoaderWorkerApiRoute( return json(continuationResult); } catch (error) { - logger.warn("Failed to suspend run", { runFriendlyId, snapshotFriendlyId, error }); + logger.warn("Failed to continue run execution", { + runFriendlyId, + snapshotFriendlyId, + error, + }); + + // A Prisma infrastructure error (e.g. P1001 "Can't reach database + // server") means the DB was transiently unreachable while resuming. A 422 + // is non-retryable, so the worker would permanently abort a run over a + // blip. Let it propagate to the generic 500 handler, which scrubs the + // message and is retried by the worker's HTTP client. + if (isInfrastructureError(error)) { + throw error; + } + if (error instanceof Error) { throw json({ error: clientSafeErrorMessage(error) }, { status: 422 }); } diff --git a/packages/core/src/v3/runEngineWorker/supervisor/http.ts b/packages/core/src/v3/runEngineWorker/supervisor/http.ts index 7b605373d..bbb1fe227 100644 --- a/packages/core/src/v3/runEngineWorker/supervisor/http.ts +++ b/packages/core/src/v3/runEngineWorker/supervisor/http.ts @@ -245,6 +245,21 @@ export class SupervisorHttpClient { ...this.defaultHeaders, ...this.runnerIdHeader(runnerId), }, + }, + { + // This is the hop that reaches the engine, so it's where a transient + // database outage during resume surfaces (as a retryable 5xx). Resuming + // is idempotent server-side (guarded by the snapshot id), so retry + // generously to ride out the outage rather than aborting the run. + // `randomize` jitters the delay so a fleet of runs resuming at once + // doesn't stampede the DB the moment it recovers. + retry: { + minTimeoutInMs: 500, + maxTimeoutInMs: 10_000, + maxAttempts: 8, + factor: 2, + randomize: true, + }, } ); } diff --git a/packages/core/src/v3/runEngineWorker/workload/http.ts b/packages/core/src/v3/runEngineWorker/workload/http.ts index 8f4703a9e..43328b724 100644 --- a/packages/core/src/v3/runEngineWorker/workload/http.ts +++ b/packages/core/src/v3/runEngineWorker/workload/http.ts @@ -132,6 +132,20 @@ export class WorkloadHttpClient { headers: { ...this.defaultHeaders(), }, + }, + { + // This hop only reaches the supervisor's workload server, so retry + // generously with jittered backoff to ride out a transient blip + // talking to the supervisor (e.g. a restart) rather than aborting the + // run. Database outages surface one hop further in, on the + // supervisor-to-engine call, which carries its own retry for them. + retry: { + minTimeoutInMs: 500, + maxTimeoutInMs: 10_000, + maxAttempts: 8, + factor: 2, + randomize: true, + }, } ) );