From 8e675a4e935d25457c234c55ded049542a3ec0f2 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 12 May 2026 15:05:01 +0100 Subject: [PATCH] fix(core): retry TASK_PROCESS_SIGSEGV under the user's retry policy (#3552) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes [TRI-9234](https://linear.app/triggerdotdev/issue/TRI-9234/retry-task-process-sigsegv-errors-respecting-user-retry-config) ## What this changes SIGSEGV crashes (`TASK_PROCESS_SIGSEGV`) will now be **retried when an attempt fails**, in line with the task's configured retry settings (`retry.maxAttempts` etc.) — the same path SIGTERM and uncaught exceptions already use. Previously SIGSEGV was hard-classified as non-retriable and failed the run on the first segfault, ignoring the user's retry policy. Tasks without a retry policy still fail fast on the first SIGSEGV. Behaviour is unchanged for OOM kills (separate machine-bump retry path) and SIGKILL_TIMEOUT. ## Deploy **Only the webapp needs to ship.** The retry decision lives entirely in the webapp: - V2 path: `internal-packages/run-engine` (bundled into the webapp) - V1 path: `apps/webapp/app/v3/services/completeAttempt.server.ts` No supervisor, CLI, SDK, or customer-task-image changes required. Customers do not need to redeploy. The `@trigger.dev/core` changeset is just keeping the public package in sync — the published npm version isn't what makes the fix work. ## Why retry SIGSEGV in Node tasks is frequently non-deterministic across processes: - **Native addon races** (`sharp`, `canvas`, `better-sqlite3`, `node-rdkafka`, `bcrypt`, …) — libuv thread-pool work stepping on V8 handles. Different heap layout / thread schedule on a fresh process → retry often succeeds. - **JIT / GC interaction** — V8 turbofan deopt or GC during a native callback. Timing-dependent. - **Near-OOM in native code** — when RSS approaches the cgroup limit, native allocations fail and poorly-written addons dereference NULL → SIGSEGV instead of clean OOM-kill. - **Host / hardware issues** — bit flips, kernel quirks. Retry lands on a different host. The genuinely deterministic case (a user-code bug always tripping the same addon) is real, but a subset — and `maxAttempts` bounds the damage. ## Pre-existing inconsistency this resolves - `shouldRetryError` returned `false` for `TASK_PROCESS_SIGSEGV` → `fail_run`. - `shouldLookupRetrySettings` already listed `TASK_PROCESS_SIGSEGV` as retry-config-aware — but that branch was unreachable because `shouldRetryError` short-circuited first in `retrying.ts:86-90`. - We already retry `TASK_RUN_UNCAUGHT_EXCEPTION` (clearly a user-code bug) under the user's retry policy; refusing to retry SIGSEGV was the odd one out. ## Test plan - [x] `pnpm exec vitest run test/errors.test.ts` in `packages/core` — 26/26 pass (4 new) - [x] `pnpm run build --filter @trigger.dev/core` - [ ] CI green on PR 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.7 (1M context) --- .changeset/retry-sigsegv.md | 5 +++++ packages/core/src/v3/errors.ts | 2 +- packages/core/test/errors.test.ts | 36 ++++++++++++++++++++++++++++++- 3 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 .changeset/retry-sigsegv.md diff --git a/.changeset/retry-sigsegv.md b/.changeset/retry-sigsegv.md new file mode 100644 index 000000000..5a53c351e --- /dev/null +++ b/.changeset/retry-sigsegv.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +Retry `TASK_PROCESS_SIGSEGV` task crashes under the user's retry policy instead of failing the run on the first segfault. SIGSEGV in Node tasks is frequently non-deterministic (native addon races, JIT/GC interaction, near-OOM in native code, host issues), so retrying on a fresh process often succeeds. The retry is gated by the task's existing `retry` config + `maxAttempts` — same path `TASK_PROCESS_SIGTERM` and uncaught exceptions already use — so tasks without a retry policy still fail fast. diff --git a/packages/core/src/v3/errors.ts b/packages/core/src/v3/errors.ts index a538ca935..90650bbd1 100644 --- a/packages/core/src/v3/errors.ts +++ b/packages/core/src/v3/errors.ts @@ -361,7 +361,6 @@ export function shouldRetryError(error: TaskRunError): boolean { case "CONFIGURED_INCORRECTLY": case "TASK_ALREADY_RUNNING": case "TASK_PROCESS_SIGKILL_TIMEOUT": - case "TASK_PROCESS_SIGSEGV": case "TASK_PROCESS_OOM_KILLED": case "TASK_PROCESS_MAYBE_OOM_KILLED": case "TASK_RUN_CANCELLED": @@ -398,6 +397,7 @@ export function shouldRetryError(error: TaskRunError): boolean { case "TASK_RUN_UNCAUGHT_EXCEPTION": case "TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE": case "TASK_PROCESS_SIGTERM": + case "TASK_PROCESS_SIGSEGV": return true; default: diff --git a/packages/core/test/errors.test.ts b/packages/core/test/errors.test.ts index dee6509d3..9a94366d8 100644 --- a/packages/core/test/errors.test.ts +++ b/packages/core/test/errors.test.ts @@ -1,5 +1,13 @@ import { describe, it, expect } from "vitest"; -import { truncateStack, truncateMessage, parseError, sanitizeError } from "../src/v3/errors.js"; +import { + truncateStack, + truncateMessage, + parseError, + sanitizeError, + shouldRetryError, + shouldLookupRetrySettings, +} from "../src/v3/errors.js"; +import type { TaskRunError } from "../src/v3/schemas/common.js"; // Helper: build a fake stack with N frames function buildStack(messageLines: string[], frameCount: number): string { @@ -238,3 +246,29 @@ describe("truncateStack message line bounding", () => { expect(result).toContain("...[truncated]"); }); }); + +describe("shouldRetryError + shouldLookupRetrySettings", () => { + const internal = (code: string): TaskRunError => + ({ type: "INTERNAL_ERROR", code } as TaskRunError); + + it("retries SIGSEGV (changed from non-retriable) and looks up retry settings", () => { + const err = internal("TASK_PROCESS_SIGSEGV"); + expect(shouldRetryError(err)).toBe(true); + expect(shouldLookupRetrySettings(err)).toBe(true); + }); + + it("retries SIGTERM via the same path", () => { + const err = internal("TASK_PROCESS_SIGTERM"); + expect(shouldRetryError(err)).toBe(true); + expect(shouldLookupRetrySettings(err)).toBe(true); + }); + + it("still does not retry SIGKILL timeout", () => { + expect(shouldRetryError(internal("TASK_PROCESS_SIGKILL_TIMEOUT"))).toBe(false); + }); + + it("still does not retry OOM kills (handled by the separate machine-bump path)", () => { + expect(shouldRetryError(internal("TASK_PROCESS_OOM_KILLED"))).toBe(false); + expect(shouldRetryError(internal("TASK_PROCESS_MAYBE_OOM_KILLED"))).toBe(false); + }); +});