8e675a4e93
Closes [TRI-9234](https://linear.app/triggerdotdev/issue/TRI-9234/retry-task-process-sigsegv-errors-respecting-user-retry-config) ## What this changes SIGSEGV crashes (`TASK_PROCESS_SIGSEGV`) will now be **retried when an attempt fails**, in line with the task's configured retry settings (`retry.maxAttempts` etc.) — the same path SIGTERM and uncaught exceptions already use. Previously SIGSEGV was hard-classified as non-retriable and failed the run on the first segfault, ignoring the user's retry policy. Tasks without a retry policy still fail fast on the first SIGSEGV. Behaviour is unchanged for OOM kills (separate machine-bump retry path) and SIGKILL_TIMEOUT. ## Deploy **Only the webapp needs to ship.** The retry decision lives entirely in the webapp: - V2 path: `internal-packages/run-engine` (bundled into the webapp) - V1 path: `apps/webapp/app/v3/services/completeAttempt.server.ts` No supervisor, CLI, SDK, or customer-task-image changes required. Customers do not need to redeploy. The `@trigger.dev/core` changeset is just keeping the public package in sync — the published npm version isn't what makes the fix work. ## Why retry SIGSEGV in Node tasks is frequently non-deterministic across processes: - **Native addon races** (`sharp`, `canvas`, `better-sqlite3`, `node-rdkafka`, `bcrypt`, …) — libuv thread-pool work stepping on V8 handles. Different heap layout / thread schedule on a fresh process → retry often succeeds. - **JIT / GC interaction** — V8 turbofan deopt or GC during a native callback. Timing-dependent. - **Near-OOM in native code** — when RSS approaches the cgroup limit, native allocations fail and poorly-written addons dereference NULL → SIGSEGV instead of clean OOM-kill. - **Host / hardware issues** — bit flips, kernel quirks. Retry lands on a different host. The genuinely deterministic case (a user-code bug always tripping the same addon) is real, but a subset — and `maxAttempts` bounds the damage. ## Pre-existing inconsistency this resolves - `shouldRetryError` returned `false` for `TASK_PROCESS_SIGSEGV` → `fail_run`. - `shouldLookupRetrySettings` already listed `TASK_PROCESS_SIGSEGV` as retry-config-aware — but that branch was unreachable because `shouldRetryError` short-circuited first in `retrying.ts:86-90`. - We already retry `TASK_RUN_UNCAUGHT_EXCEPTION` (clearly a user-code bug) under the user's retry policy; refusing to retry SIGSEGV was the odd one out. ## Test plan - [x] `pnpm exec vitest run test/errors.test.ts` in `packages/core` — 26/26 pass (4 new) - [x] `pnpm run build --filter @trigger.dev/core` - [ ] CI green on PR 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
275 lines
9.1 KiB
TypeScript
275 lines
9.1 KiB
TypeScript
import { describe, it, expect } from "vitest";
|
|
import {
|
|
truncateStack,
|
|
truncateMessage,
|
|
parseError,
|
|
sanitizeError,
|
|
shouldRetryError,
|
|
shouldLookupRetrySettings,
|
|
} from "../src/v3/errors.js";
|
|
import type { TaskRunError } from "../src/v3/schemas/common.js";
|
|
|
|
// Helper: build a fake stack with N frames
|
|
function buildStack(messageLines: string[], frameCount: number): string {
|
|
const frames = Array.from(
|
|
{ length: frameCount },
|
|
(_, i) => ` at functionName${i} (/path/to/file${i}.ts:${i + 1}:${i + 10})`
|
|
);
|
|
return [...messageLines, ...frames].join("\n");
|
|
}
|
|
|
|
describe("truncateStack", () => {
|
|
it("returns empty string for undefined", () => {
|
|
expect(truncateStack(undefined)).toBe("");
|
|
});
|
|
|
|
it("returns empty string for empty string", () => {
|
|
expect(truncateStack("")).toBe("");
|
|
});
|
|
|
|
it("preserves a short stack unchanged", () => {
|
|
const stack = buildStack(["Error: something broke"], 10);
|
|
expect(truncateStack(stack)).toBe(stack);
|
|
});
|
|
|
|
it("preserves exactly 50 frames", () => {
|
|
const stack = buildStack(["Error: at the limit"], 50);
|
|
const result = truncateStack(stack);
|
|
expect(result).toBe(stack);
|
|
expect(result.split("\n").filter((l) => l.trimStart().startsWith("at ")).length).toBe(50);
|
|
});
|
|
|
|
it("truncates to 50 frames when exceeding the limit", () => {
|
|
const stack = buildStack(["Error: too many frames"], 200);
|
|
const result = truncateStack(stack);
|
|
const lines = result.split("\n");
|
|
|
|
// Message line + 5 top + 1 omitted notice + 45 bottom = 52 lines
|
|
expect(lines[0]).toBe("Error: too many frames");
|
|
expect(lines).toContain(" ... 150 frames omitted ...");
|
|
|
|
const frameLines = lines.filter((l) => l.trimStart().startsWith("at "));
|
|
expect(frameLines.length).toBe(50);
|
|
|
|
// First kept frame is frame 0 (top of stack)
|
|
expect(frameLines[0]).toContain("functionName0");
|
|
// Last kept frame is the last original frame
|
|
expect(frameLines[frameLines.length - 1]).toContain("functionName199");
|
|
});
|
|
|
|
it("preserves multi-line error messages before frames", () => {
|
|
const stack = buildStack(["TypeError: cannot read property", " caused by: something"], 60);
|
|
const result = truncateStack(stack);
|
|
const lines = result.split("\n");
|
|
|
|
expect(lines[0]).toBe("TypeError: cannot read property");
|
|
expect(lines[1]).toBe(" caused by: something");
|
|
expect(lines).toContain(" ... 10 frames omitted ...");
|
|
});
|
|
|
|
it("truncates individual lines longer than 1024 chars", () => {
|
|
const longFrame = ` at someFn (${"x".repeat(2000)}:1:1)`;
|
|
const stack = ["Error: long line", longFrame].join("\n");
|
|
const result = truncateStack(stack);
|
|
const frameLine = result.split("\n")[1]!;
|
|
|
|
expect(frameLine.length).toBeLessThan(1100);
|
|
expect(frameLine).toContain("...[truncated]");
|
|
});
|
|
});
|
|
|
|
describe("truncateMessage", () => {
|
|
it("returns empty string for undefined", () => {
|
|
expect(truncateMessage(undefined)).toBe("");
|
|
});
|
|
|
|
it("returns empty string for empty string", () => {
|
|
expect(truncateMessage("")).toBe("");
|
|
});
|
|
|
|
it("preserves a short message", () => {
|
|
expect(truncateMessage("hello")).toBe("hello");
|
|
});
|
|
|
|
it("truncates messages over 1000 chars", () => {
|
|
const long = "x".repeat(5000);
|
|
const result = truncateMessage(long);
|
|
expect(result.length).toBeLessThan(1100);
|
|
expect(result).toContain("...[truncated]");
|
|
});
|
|
|
|
it("preserves a message at exactly 1000 chars", () => {
|
|
const exact = "x".repeat(1000);
|
|
expect(truncateMessage(exact)).toBe(exact);
|
|
});
|
|
});
|
|
|
|
describe("parseError truncation", () => {
|
|
it("truncates large stack traces in Error objects", () => {
|
|
const error = new Error("boom");
|
|
error.stack = buildStack(["Error: boom"], 200);
|
|
const parsed = parseError(error);
|
|
|
|
expect(parsed.type).toBe("BUILT_IN_ERROR");
|
|
if (parsed.type === "BUILT_IN_ERROR") {
|
|
const frameLines = parsed.stackTrace.split("\n").filter((l) => l.trimStart().startsWith("at "));
|
|
expect(frameLines.length).toBe(50);
|
|
expect(parsed.stackTrace).toContain("frames omitted");
|
|
}
|
|
});
|
|
|
|
it("truncates large error messages", () => {
|
|
const error = new Error("x".repeat(5000));
|
|
const parsed = parseError(error);
|
|
|
|
if (parsed.type === "BUILT_IN_ERROR") {
|
|
expect(parsed.message.length).toBeLessThan(1100);
|
|
expect(parsed.message).toContain("...[truncated]");
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("sanitizeError truncation", () => {
|
|
it("truncates stack traces during sanitization", () => {
|
|
const result = sanitizeError({
|
|
type: "BUILT_IN_ERROR",
|
|
name: "Error",
|
|
message: "boom",
|
|
stackTrace: buildStack(["Error: boom"], 200),
|
|
});
|
|
|
|
if (result.type === "BUILT_IN_ERROR") {
|
|
const frameLines = result.stackTrace.split("\n").filter((l) => l.trimStart().startsWith("at "));
|
|
expect(frameLines.length).toBe(50);
|
|
}
|
|
});
|
|
|
|
it("strips null bytes and truncates", () => {
|
|
const result = sanitizeError({
|
|
type: "BUILT_IN_ERROR",
|
|
name: "Error\0",
|
|
message: "hello\0world",
|
|
stackTrace: "Error: hello\0world\n at fn (/path.ts:1:1)",
|
|
});
|
|
|
|
if (result.type === "BUILT_IN_ERROR") {
|
|
expect(result.name).toBe("Error");
|
|
expect(result.message).toBe("helloworld");
|
|
expect(result.stackTrace).not.toContain("\0");
|
|
}
|
|
});
|
|
|
|
it("truncates STRING_ERROR raw field", () => {
|
|
const result = sanitizeError({
|
|
type: "STRING_ERROR",
|
|
raw: "x".repeat(5000),
|
|
});
|
|
|
|
if (result.type === "STRING_ERROR") {
|
|
expect(result.raw.length).toBeLessThan(1100);
|
|
expect(result.raw).toContain("...[truncated]");
|
|
}
|
|
});
|
|
|
|
it("preserves small CUSTOM_ERROR raw as valid JSON", () => {
|
|
const originalJson = JSON.stringify({ foo: "bar", nested: { baz: 1 } });
|
|
const result = sanitizeError({
|
|
type: "CUSTOM_ERROR",
|
|
raw: originalJson,
|
|
});
|
|
|
|
if (result.type === "CUSTOM_ERROR") {
|
|
// Small JSON should pass through unchanged and remain parseable
|
|
expect(result.raw).toBe(originalJson);
|
|
expect(() => JSON.parse(result.raw)).not.toThrow();
|
|
}
|
|
});
|
|
|
|
it("wraps oversized CUSTOM_ERROR raw in a valid JSON envelope", () => {
|
|
const hugeJson = JSON.stringify({ data: "x".repeat(5000) });
|
|
const result = sanitizeError({
|
|
type: "CUSTOM_ERROR",
|
|
raw: hugeJson,
|
|
});
|
|
|
|
if (result.type === "CUSTOM_ERROR") {
|
|
// Must remain valid JSON (critical: createErrorTaskError calls JSON.parse on this)
|
|
expect(() => JSON.parse(result.raw)).not.toThrow();
|
|
const parsed = JSON.parse(result.raw);
|
|
expect(parsed.truncated).toBe(true);
|
|
expect(typeof parsed.preview).toBe("string");
|
|
expect(parsed.preview.length).toBeLessThanOrEqual(1000);
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("sanitizeError INTERNAL_ERROR optional fields", () => {
|
|
it("preserves undefined message (does not convert to empty string)", () => {
|
|
const result = sanitizeError({
|
|
type: "INTERNAL_ERROR",
|
|
code: "SOME_INTERNAL_CODE" as any,
|
|
// message and stackTrace intentionally undefined
|
|
});
|
|
|
|
if (result.type === "INTERNAL_ERROR") {
|
|
// Must stay undefined so `error.message ?? fallback` works downstream
|
|
expect(result.message).toBeUndefined();
|
|
expect(result.stackTrace).toBeUndefined();
|
|
}
|
|
});
|
|
|
|
it("truncates INTERNAL_ERROR message when present", () => {
|
|
const result = sanitizeError({
|
|
type: "INTERNAL_ERROR",
|
|
code: "SOME_INTERNAL_CODE" as any,
|
|
message: "x".repeat(5000),
|
|
});
|
|
|
|
if (result.type === "INTERNAL_ERROR") {
|
|
expect(result.message).toBeDefined();
|
|
expect(result.message!.length).toBeLessThan(1100);
|
|
expect(result.message).toContain("...[truncated]");
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("truncateStack message line bounding", () => {
|
|
it("truncates huge error messages embedded in the stack", () => {
|
|
// V8 format: "Error: <message>\n at ..."
|
|
// A huge message on the first line must still be bounded.
|
|
const hugeMessage = "x".repeat(100_000);
|
|
const stack = `Error: ${hugeMessage}\n at fn (/path.ts:1:1)`;
|
|
const result = truncateStack(stack);
|
|
|
|
// Total output should be bounded (not 100KB+)
|
|
expect(result.length).toBeLessThan(5_000);
|
|
expect(result).toContain("...[truncated]");
|
|
});
|
|
});
|
|
|
|
describe("shouldRetryError + shouldLookupRetrySettings", () => {
|
|
const internal = (code: string): TaskRunError =>
|
|
({ type: "INTERNAL_ERROR", code } as TaskRunError);
|
|
|
|
it("retries SIGSEGV (changed from non-retriable) and looks up retry settings", () => {
|
|
const err = internal("TASK_PROCESS_SIGSEGV");
|
|
expect(shouldRetryError(err)).toBe(true);
|
|
expect(shouldLookupRetrySettings(err)).toBe(true);
|
|
});
|
|
|
|
it("retries SIGTERM via the same path", () => {
|
|
const err = internal("TASK_PROCESS_SIGTERM");
|
|
expect(shouldRetryError(err)).toBe(true);
|
|
expect(shouldLookupRetrySettings(err)).toBe(true);
|
|
});
|
|
|
|
it("still does not retry SIGKILL timeout", () => {
|
|
expect(shouldRetryError(internal("TASK_PROCESS_SIGKILL_TIMEOUT"))).toBe(false);
|
|
});
|
|
|
|
it("still does not retry OOM kills (handled by the separate machine-bump path)", () => {
|
|
expect(shouldRetryError(internal("TASK_PROCESS_OOM_KILLED"))).toBe(false);
|
|
expect(shouldRetryError(internal("TASK_PROCESS_MAYBE_OOM_KILLED"))).toBe(false);
|
|
});
|
|
});
|