From b65a04eb37e0328744b7dd7abb802487d6657298 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 1 May 2026 17:11:06 +0100 Subject: [PATCH] fix(cli,core): stop dev workers spinning at 100% CPU after parent CLI disconnect (#3491) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Orphaned `trigger-dev-run-worker` processes were pinning CPU at 100% after the dev CLI exited — stuck in an uncaughtException feedback loop where a closed IPC channel kept throwing `ERR_IPC_CHANNEL_CLOSED` back into a handler that itself called `process.send`. Fix: - `ZodIpcConnection` no-ops sends when the channel is disconnected. - Dev workers exit on `process.disconnect` instead of being re-parented to init. - All worker `uncaughtException` handlers route through a `safeSend` guard so the handler can never re-enter itself. Verified end-to-end: `kill -9` of the dev CLI now cleans up all child workers within ~2s. --- .changeset/dev-worker-disconnect-loop.md | 6 ++ .../src/entryPoints/dev-index-worker.ts | 57 +++++++++++------ .../cli-v3/src/entryPoints/dev-run-worker.ts | 64 ++++++++++++------- .../src/entryPoints/managed-index-worker.ts | 53 ++++++++------- .../src/entryPoints/managed-run-worker.ts | 57 ++++++++++------- packages/core/src/v3/zodIpc.ts | 15 ++++- 6 files changed, 160 insertions(+), 92 deletions(-) create mode 100644 .changeset/dev-worker-disconnect-loop.md diff --git a/.changeset/dev-worker-disconnect-loop.md b/.changeset/dev-worker-disconnect-loop.md new file mode 100644 index 000000000..cf5afbb21 --- /dev/null +++ b/.changeset/dev-worker-disconnect-loop.md @@ -0,0 +1,6 @@ +--- +"@trigger.dev/core": patch +"trigger.dev": patch +--- + +Fix dev workers spinning at 100% CPU after the parent CLI disconnects. Orphaned `trigger-dev-run-worker` (and indexer) processes were caught in an `uncaughtException` feedback loop: a periodic IPC send via `process.send` would throw `ERR_IPC_CHANNEL_CLOSED` once the parent closed the channel, which re-entered the same handler that itself called `process.send`, scheduled via `setImmediate` and amplified by source-map-support's `prepareStackTrace`. Fixed by (1) silently dropping packets in `ZodIpcConnection` when the channel is disconnected, (2) adding a `process.on("disconnect", ...)` handler in dev workers so they exit cleanly when the CLI closes the IPC channel, and (3) wrapping all `uncaughtException`-path `process.send` calls in a `safeSend` guard that checks `process.connected` and swallows synchronous throws. diff --git a/packages/cli-v3/src/entryPoints/dev-index-worker.ts b/packages/cli-v3/src/entryPoints/dev-index-worker.ts index 0b8ee54b0..53b95ad04 100644 --- a/packages/cli-v3/src/entryPoints/dev-index-worker.ts +++ b/packages/cli-v3/src/entryPoints/dev-index-worker.ts @@ -27,30 +27,45 @@ sourceMapSupport.install({ hookRequire: false, }); +// If the parent CLI closes the IPC channel, exit cleanly instead of being +// re-parented to init and busy-looping on `process.send` against a dead channel. +process.on("disconnect", () => { + process.exit(0); +}); + +function safeSend(message: unknown) { + if (!process.connected || !process.send) { + return; + } + try { + process.send(message); + } catch { + // swallow: a throw here would re-enter this handler and busy-loop the worker + } +} + process.on("uncaughtException", function (error, origin) { if (error instanceof Error) { - process.send && - process.send({ - type: "UNCAUGHT_EXCEPTION", - payload: { - error: { name: error.name, message: error.message, stack: error.stack }, - origin, - }, - version: "v1", - }); + safeSend({ + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { name: error.name, message: error.message, stack: error.stack }, + origin, + }, + version: "v1", + }); } else { - process.send && - process.send({ - type: "UNCAUGHT_EXCEPTION", - payload: { - error: { - name: "Error", - message: typeof error === "string" ? error : JSON.stringify(error), - }, - origin, + safeSend({ + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { + name: "Error", + message: typeof error === "string" ? error : JSON.stringify(error), }, - version: "v1", - }); + origin, + }, + version: "v1", + }); } }); @@ -183,7 +198,7 @@ await sendMessageInCatalog( importErrors, }, async (msg) => { - process.send?.(msg); + safeSend(msg); } ).catch((err) => { if (err instanceof ZodSchemaParsedError) { diff --git a/packages/cli-v3/src/entryPoints/dev-run-worker.ts b/packages/cli-v3/src/entryPoints/dev-run-worker.ts index ae9a2667f..6ffc6afd2 100644 --- a/packages/cli-v3/src/entryPoints/dev-run-worker.ts +++ b/packages/cli-v3/src/entryPoints/dev-run-worker.ts @@ -77,37 +77,53 @@ sourceMapSupport.install({ hookRequire: false, }); +// If the parent CLI closes the IPC channel (process restart, crash, lost +// handle), exit cleanly instead of being re-parented to init and busy-looping +// on `process.send` that throws against a dead channel. +process.on("disconnect", () => { + process.exit(0); +}); + +function safeSend(message: unknown) { + if (!process.connected || !process.send) { + return; + } + try { + process.send(message); + } catch { + // swallow: a throw here would re-enter this handler and busy-loop the worker + } +} + process.on("uncaughtException", function (error, origin) { logError("Uncaught exception", { error, origin }); if (error instanceof Error) { - process.send && - process.send({ - type: "EVENT", - message: { - type: "UNCAUGHT_EXCEPTION", - payload: { - error: { name: error.name, message: error.message, stack: error.stack }, - origin, - }, - version: "v1", + safeSend({ + type: "EVENT", + message: { + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { name: error.name, message: error.message, stack: error.stack }, + origin, }, - }); + version: "v1", + }, + }); } else { - process.send && - process.send({ - type: "EVENT", - message: { - type: "UNCAUGHT_EXCEPTION", - payload: { - error: { - name: "Error", - message: typeof error === "string" ? error : JSON.stringify(error), - }, - origin, + safeSend({ + type: "EVENT", + message: { + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { + name: "Error", + message: typeof error === "string" ? error : JSON.stringify(error), }, - version: "v1", + origin, }, - }); + version: "v1", + }, + }); } }); diff --git a/packages/cli-v3/src/entryPoints/managed-index-worker.ts b/packages/cli-v3/src/entryPoints/managed-index-worker.ts index efc226b99..644673537 100644 --- a/packages/cli-v3/src/entryPoints/managed-index-worker.ts +++ b/packages/cli-v3/src/entryPoints/managed-index-worker.ts @@ -27,30 +27,39 @@ sourceMapSupport.install({ hookRequire: false, }); +function safeSend(message: unknown) { + if (!process.connected || !process.send) { + return; + } + try { + process.send(message); + } catch { + // swallow: a throw here would re-enter this handler and busy-loop the worker + } +} + process.on("uncaughtException", function (error, origin) { if (error instanceof Error) { - process.send && - process.send({ - type: "UNCAUGHT_EXCEPTION", - payload: { - error: { name: error.name, message: error.message, stack: error.stack }, - origin, - }, - version: "v1", - }); + safeSend({ + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { name: error.name, message: error.message, stack: error.stack }, + origin, + }, + version: "v1", + }); } else { - process.send && - process.send({ - type: "UNCAUGHT_EXCEPTION", - payload: { - error: { - name: "Error", - message: typeof error === "string" ? error : JSON.stringify(error), - }, - origin, + safeSend({ + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { + name: "Error", + message: typeof error === "string" ? error : JSON.stringify(error), }, - version: "v1", - }); + origin, + }, + version: "v1", + }); } }); @@ -191,7 +200,7 @@ await sendMessageInCatalog( importErrors, }, async (msg) => { - process.send?.(msg); + safeSend(msg); } ).catch((err) => { if (err instanceof ZodSchemaParsedError) { @@ -200,7 +209,7 @@ await sendMessageInCatalog( "TASKS_FAILED_TO_PARSE", { zodIssues: err.error.issues, tasks }, async (msg) => { - process.send?.(msg); + safeSend(msg); } ); } else { diff --git a/packages/cli-v3/src/entryPoints/managed-run-worker.ts b/packages/cli-v3/src/entryPoints/managed-run-worker.ts index f8234aa68..1d5d2a5f0 100644 --- a/packages/cli-v3/src/entryPoints/managed-run-worker.ts +++ b/packages/cli-v3/src/entryPoints/managed-run-worker.ts @@ -77,37 +77,46 @@ sourceMapSupport.install({ hookRequire: false, }); +function safeSend(message: unknown) { + if (!process.connected || !process.send) { + return; + } + try { + process.send(message); + } catch { + // swallow: a throw here would re-enter this handler and busy-loop the worker + } +} + process.on("uncaughtException", function (error, origin) { console.error("Uncaught exception", { error, origin }); if (error instanceof Error) { - process.send && - process.send({ - type: "EVENT", - message: { - type: "UNCAUGHT_EXCEPTION", - payload: { - error: { name: error.name, message: error.message, stack: error.stack }, - origin, - }, - version: "v1", + safeSend({ + type: "EVENT", + message: { + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { name: error.name, message: error.message, stack: error.stack }, + origin, }, - }); + version: "v1", + }, + }); } else { - process.send && - process.send({ - type: "EVENT", - message: { - type: "UNCAUGHT_EXCEPTION", - payload: { - error: { - name: "Error", - message: typeof error === "string" ? error : JSON.stringify(error), - }, - origin, + safeSend({ + type: "EVENT", + message: { + type: "UNCAUGHT_EXCEPTION", + payload: { + error: { + name: "Error", + message: typeof error === "string" ? error : JSON.stringify(error), }, - version: "v1", + origin, }, - }); + version: "v1", + }, + }); } }); diff --git a/packages/core/src/v3/zodIpc.ts b/packages/core/src/v3/zodIpc.ts index 8879c7d05..f42de5a9d 100644 --- a/packages/core/src/v3/zodIpc.ts +++ b/packages/core/src/v3/zodIpc.ts @@ -143,6 +143,7 @@ interface ZodIpcConnectionOptions< process: { send?: (message: any) => any; on?: (event: "message", listener: (message: any) => void) => void; + connected?: boolean; }; handlers?: ZodIpcMessageHandlers; } @@ -257,7 +258,19 @@ export class ZodIpcConnection< } async #sendPacket(packet: Packet) { - await this.opts.process.send?.(packet); + // When the IPC channel is closed (e.g. parent process exited), there is no + // recipient — drop the packet rather than letting `process.send` throw + // ERR_IPC_CHANNEL_CLOSED, which would otherwise propagate as an + // uncaughtException and re-enter any handler that itself calls `process.send`. + if (this.opts.process.connected === false) { + return; + } + + try { + await this.opts.process.send?.(packet); + } catch { + // swallow: channel raced from open to closed between the check and the send + } } async send>(