From 7246f677dbeab6668d485a2ccca88a0498d8213a Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 7 Aug 2026 13:28:52 +0100 Subject: [PATCH] fix(webapp): strip null bytes from idempotency and debounce keys at trigger (#4527) ## What A trigger request carrying a Unicode NUL (`U+0000`) in the **idempotency key** or **debounce key** reached `prisma.taskRun.create()` and failed the insert, so the caller got an opaque 500 and the run was never created. These two keys are stored in `jsonb` columns (`idempotencyKeyOptions`, `debounce`), and Postgres rejects a NUL inside a `jsonb` value with `SQLSTATE 22P05` ("unsupported Unicode escape sequence ... cannot be converted to text"). This fix strips the NUL from both keys at the single trigger-input chokepoint (`#buildEngineTriggerInput`), which every trigger path flows through (single, batch item, mollified, and drainer replay). Stripping matches the existing precedent for run errors and task events. It does not change dedup behaviour: the idempotency **dedup identity** is the hashed key (a clean 64-char digest), computed independently of the raw key we clean, so dedup keeps working exactly as before. For debounce the key is used directly, so the cleaned key also becomes the grouping key, an acceptable change for input that is already malformed. ## Why not payload / metadata / tags Those are `text` columns fed by `JSON.stringify`, which escapes a NUL to a safe escape sequence, so they do not hit this failure on the normal JSON path. (A raw NUL in a `text` column throws a different code, `22021`, and is not what triggers this issue.) The observed failures are the `jsonb` `22P05` variant, which is only reachable via the two key fields. ## Evidence Red then green (containerTest, real Postgres): with the fix reverted, triggering through the real service with a NUL in `idempotencyKeyOptions.key` / `debounce.key` fails with the exact `22P05` signature; with the fix, the run is created and the stored key has the NUL removed. Full-stack e2e (isolated stack, real HTTP): `POST /api/v1/tasks/:taskId/trigger` with a NUL inside `idempotencyKeyOptions.key` (`"acmeinc"`) and, separately, `debounce.key` (`"grp1"`): - both returned `HTTP 200` with a created run (previously `500`) - stored `idempotencyKeyOptions` = `{ "key": "acmeinc", "scope": "run" }` (7 chars, NUL removed) - stored `debounce.key` = `"grp1"` (4 chars, NUL removed) - both runs render in the dashboard Unit tests cover the helper (strip, no-op fast path, object-reference reuse, null/undefined pass-through). ## Rollout / rollback Server-only webapp change, no flag. Zero behaviour change for clean input; only affects inputs that previously 500'd. Rollback is a straight revert, no data migration. ## Known limitation A raw NUL in a plain-string idempotency key (not created via `idempotencyKeys.create()`) lands in a `text` column and throws `22021` instead. That variant is not addressed here because stripping it would change the dedup identity, so it warrants a separate decision. Not observed in practice. refs TRI-13030 --- .../strip-null-bytes-trigger-keys.md | 6 + .../triggerTask.server.nullBytes.test.ts | 110 ++++++++++++++++++ .../runEngine/services/triggerTask.server.ts | 5 +- apps/webapp/app/utils/nullBytes.test.ts | 36 ++++++ apps/webapp/app/utils/nullBytes.ts | 26 +++++ 5 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 .server-changes/strip-null-bytes-trigger-keys.md create mode 100644 apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts create mode 100644 apps/webapp/app/utils/nullBytes.test.ts create mode 100644 apps/webapp/app/utils/nullBytes.ts diff --git a/.server-changes/strip-null-bytes-trigger-keys.md b/.server-changes/strip-null-bytes-trigger-keys.md new file mode 100644 index 000000000..2ffa5b86f --- /dev/null +++ b/.server-changes/strip-null-bytes-trigger-keys.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Fixed a rare error where triggering a task could fail if the idempotency key or debounce key contained an invalid null character. The character is now removed automatically and the run is created as normal. diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts b/apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts new file mode 100644 index 000000000..9612f103e --- /dev/null +++ b/apps/webapp/app/runEngine/services/triggerTask.server.nullBytes.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, vi } from "vitest"; + +vi.mock("~/db.server", () => ({ + prisma: {}, + $replica: {}, + runOpsNewPrisma: {}, + runOpsLegacyPrisma: {}, + runOpsNewReplica: {}, + runOpsLegacyReplica: {}, +})); +vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false })); +vi.mock("~/services/platform.v3.server", async (importOriginal) => { + const actual = (await importOriginal()) as Record; + return { + ...actual, + getEntitlement: vi.fn(), + }; +}); + +import { setupAuthenticatedEnvironment } from "@internal/run-engine/tests"; +import { assertNonNullable, containerTest } from "@internal/testcontainers"; +import { trace } from "@opentelemetry/api"; +import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server"; +import { DefaultQueueManager } from "~/runEngine/concerns/queues.server"; +import { RunEngineTriggerTaskService } from "./triggerTask.server"; +import { + buildEngine, + CapturingParentRunValidator, + MockPayloadProcessor, + MockTraceEventConcern, +} from "./triggerTask.server.test.helpers"; + +vi.setConfig({ testTimeout: 60_000 }); + +const NUL = String.fromCharCode(0); + +function buildService(engine: any, prisma: any) { + return new RunEngineTriggerTaskService({ + engine, + prisma, + payloadProcessor: new MockPayloadProcessor(), + queueConcern: new DefaultQueueManager(prisma, engine), + idempotencyKeyConcern: new IdempotencyKeyConcern(prisma, engine, new MockTraceEventConcern()), + validator: new CapturingParentRunValidator(), + traceEventConcern: new MockTraceEventConcern(), + tracer: trace.getTracer("test", "0.0.0"), + metadataMaximumSize: 1024 * 1024 * 1, + }); +} + +describe("RunEngineTriggerTaskService null-byte sanitization", () => { + containerTest( + "strips a NUL from idempotencyKeyOptions.key so the jsonb insert does not 22P05", + async ({ prisma, redisOptions }) => { + const engine = buildEngine(prisma, redisOptions); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const service = buildService(engine, prisma); + + const result = await service.call({ + taskId: "nul-idem-task", + environment, + body: { + payload: { kind: "idem" }, + options: { + idempotencyKey: "a".repeat(64), + idempotencyKeyOptions: { key: `acme${NUL}inc`, scope: "run" }, + }, + }, + }); + assertNonNullable(result); + + const row = await prisma.taskRun.findUniqueOrThrow({ where: { id: result.run.id } }); + expect(row.idempotencyKeyOptions).toEqual({ key: "acmeinc", scope: "run" }); + } finally { + await engine.quit(); + } + } + ); + + containerTest( + "strips a NUL from debounce.key so the jsonb insert does not 22P05", + async ({ prisma, redisOptions }) => { + const engine = buildEngine(prisma, redisOptions); + + try { + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const service = buildService(engine, prisma); + + const result = await service.call({ + taskId: "nul-debounce-task", + environment, + body: { + payload: { kind: "debounce" }, + options: { + debounce: { key: `grp${NUL}1`, delay: "1s" }, + }, + }, + }); + assertNonNullable(result); + + const row = await prisma.taskRun.findUniqueOrThrow({ where: { id: result.run.id } }); + expect((row.debounce as { key: string }).key).toBe("grp1"); + } finally { + await engine.quit(); + } + } + ); +}); diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.ts b/apps/webapp/app/runEngine/services/triggerTask.server.ts index 34805d4c3..6d15c5543 100644 --- a/apps/webapp/app/runEngine/services/triggerTask.server.ts +++ b/apps/webapp/app/runEngine/services/triggerTask.server.ts @@ -25,6 +25,7 @@ import type { PrismaClientOrTransaction } from "@trigger.dev/database"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { parseDelay } from "~/utils/delays"; +import { removeNullBytesFromKey } from "~/utils/nullBytes"; import { handleMetadataPacket } from "~/utils/packets"; import { startSpan } from "~/v3/tracing.server"; import { resolveRunIdMintKind } from "~/v3/engineVersion.server"; @@ -906,7 +907,7 @@ export class RunEngineTriggerTaskService { environment: args.environment, idempotencyKey: args.idempotencyKey, idempotencyKeyExpiresAt: args.idempotencyKey ? args.idempotencyKeyExpiresAt : undefined, - idempotencyKeyOptions: args.body.options?.idempotencyKeyOptions, + idempotencyKeyOptions: removeNullBytesFromKey(args.body.options?.idempotencyKeyOptions), taskIdentifier: args.taskId, payload: args.payloadPacket.data ?? "", payloadType: args.payloadPacket.dataType, @@ -971,7 +972,7 @@ export class RunEngineTriggerTaskService { planType: args.planType, realtimeStreamsVersion: args.options.realtimeStreamsVersion, streamBasinName: args.environment.organization.streamBasinName, - debounce: args.body.options?.debounce, + debounce: removeNullBytesFromKey(args.body.options?.debounce), annotations: args.annotations, }; } diff --git a/apps/webapp/app/utils/nullBytes.test.ts b/apps/webapp/app/utils/nullBytes.test.ts new file mode 100644 index 000000000..98447f0cf --- /dev/null +++ b/apps/webapp/app/utils/nullBytes.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; +import { removeNullBytes, removeNullBytesFromKey } from "./nullBytes"; + +describe("removeNullBytes", () => { + it("strips every NUL from a string", () => { + expect(removeNullBytes(`a\u0000b\u0000c`)).toBe("abc"); + }); + + it("returns the same reference when there is no NUL", () => { + const clean = "acme-inc"; + expect(removeNullBytes(clean)).toBe(clean); + }); + + it("passes through undefined and null", () => { + expect(removeNullBytes(undefined)).toBeUndefined(); + expect(removeNullBytes(null)).toBeNull(); + }); +}); + +describe("removeNullBytesFromKey", () => { + it("strips a NUL from the key while preserving other fields", () => { + expect(removeNullBytesFromKey({ key: `k\u00001`, scope: "run" })).toEqual({ + key: "k1", + scope: "run", + }); + }); + + it("returns the same object reference when the key is clean", () => { + const opts = { key: "clean", scope: "run" }; + expect(removeNullBytesFromKey(opts)).toBe(opts); + }); + + it("passes through undefined", () => { + expect(removeNullBytesFromKey(undefined)).toBeUndefined(); + }); +}); diff --git a/apps/webapp/app/utils/nullBytes.ts b/apps/webapp/app/utils/nullBytes.ts new file mode 100644 index 000000000..08c0a7341 --- /dev/null +++ b/apps/webapp/app/utils/nullBytes.ts @@ -0,0 +1,26 @@ +/** + * Removes Unicode NUL (U+0000) from a string. Postgres cannot store a NUL in a + * `text` column (SQLSTATE 22021) and rejects a `\u0000` escape when a JSON value + * is stored as `jsonb` (SQLSTATE 22P05), so a caller-supplied NUL reaching + * `taskRun.create()` fails the insert. The `indexOf` guard keeps the common + * (NUL-free) case allocation-free on the trigger hot path. + */ +export function removeNullBytes(value: T): T { + if (typeof value !== "string" || value.indexOf("\u0000") === -1) { + return value; + } + return value.replace(/\u0000/g, "") as T; +} + +/** + * Returns `value` with a NUL-stripped `key`, reusing the original object when no + * NUL is present. Used for the user-supplied idempotency-key and debounce + * options, whose `key` lands in a `jsonb` column on the TaskRun row. + */ +export function removeNullBytesFromKey(value: T): T { + if (!value) { + return value; + } + const cleaned = removeNullBytes(value.key); + return cleaned === value.key ? value : { ...value, key: cleaned }; +}