diff --git a/.changeset/nervous-planets-sparkle.md b/.changeset/nervous-planets-sparkle.md new file mode 100644 index 000000000..24b61117b --- /dev/null +++ b/.changeset/nervous-planets-sparkle.md @@ -0,0 +1,9 @@ +--- +"trigger.dev": patch +"@trigger.dev/core": patch +--- + +- Improve non-zero exit code error messages +- Detect OOM conditions within worker child processes +- Internal errors can have optional stack traces +- Docker provider can be set to enforce machine presets \ No newline at end of file diff --git a/apps/docker-provider/src/index.ts b/apps/docker-provider/src/index.ts index 636a423c2..a37c54b46 100644 --- a/apps/docker-provider/src/index.ts +++ b/apps/docker-provider/src/index.ts @@ -109,22 +109,27 @@ class DockerTaskOperations implements TaskOperations { const containerName = this.#getRunContainerName(opts.runId); + const runArgs = [ + "run", + "--network=host", + "--detach", + `--env=TRIGGER_ENV_ID=${opts.envId}`, + `--env=TRIGGER_RUN_ID=${opts.runId}`, + `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`, + `--env=POD_NAME=${containerName}`, + `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`, + `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`, + `--name=${containerName}`, + ]; + + if (process.env.ENFORCE_MACHINE_PRESETS) { + runArgs.push(`--cpus=${opts.machine.cpu}`, `--memory=${opts.machine.memory}G`); + } + + runArgs.push(`${opts.image}`); + try { - logger.debug( - await execa("docker", [ - "run", - "--network=host", - "--detach", - `--env=TRIGGER_ENV_ID=${opts.envId}`, - `--env=TRIGGER_RUN_ID=${opts.runId}`, - `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`, - `--env=POD_NAME=${containerName}`, - `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`, - `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`, - `--name=${containerName}`, - `${opts.image}`, - ]) - ); + logger.debug(await execa("docker", runArgs)); } catch (error) { if (!isExecaChildProcess(error)) { throw error; diff --git a/apps/kubernetes-provider/src/taskMonitor.ts b/apps/kubernetes-provider/src/taskMonitor.ts index c0296788d..20fdb64f2 100644 --- a/apps/kubernetes-provider/src/taskMonitor.ts +++ b/apps/kubernetes-provider/src/taskMonitor.ts @@ -181,7 +181,8 @@ export class TaskMonitor { } break; case "OOMKilled": - reason = "Out of memory! Try increasing the memory on this task."; + reason = + "Process ran out of memory! Try choosing a machine preset with more memory for this task."; break; default: break; diff --git a/apps/webapp/app/v3/eventRepository.server.ts b/apps/webapp/app/v3/eventRepository.server.ts index 2e7545b7b..d89016401 100644 --- a/apps/webapp/app/v3/eventRepository.server.ts +++ b/apps/webapp/app/v3/eventRepository.server.ts @@ -1118,6 +1118,7 @@ export function createExceptionPropertiesFromError(error: TaskRunError): Excepti return { type: "Internal error", message: [error.code, error.message].filter(Boolean).join(": "), + stacktrace: error.stackTrace, }; } case "STRING_ERROR": { diff --git a/packages/cli-v3/src/workers/common/errors.ts b/packages/cli-v3/src/workers/common/errors.ts index 053ab8d19..5be691dfc 100644 --- a/packages/cli-v3/src/workers/common/errors.ts +++ b/packages/cli-v3/src/workers/common/errors.ts @@ -23,7 +23,11 @@ export class TaskMetadataParseError extends Error { } export class UnexpectedExitError extends Error { - constructor(public code: number) { + constructor( + public code: number, + public signal: NodeJS.Signals | null, + public stderr: string | undefined + ) { super(`Unexpected exit with code ${code}`); this.name = "UnexpectedExitError"; @@ -61,3 +65,25 @@ export class GracefulExitTimeoutError extends Error { this.name = "GracefulExitTimeoutError"; } } + +export function getFriendlyErrorMessage( + code: number, + signal: NodeJS.Signals | null, + stderr: string | undefined +) { + const message = (text: string) => { + if (signal) { + return `[${signal}] ${text}`; + } else { + return text; + } + }; + + if (code === 137 || stderr?.includes("OOMErrorHandler")) { + return message( + "Process ran out of memory! Try choosing a machine preset with more memory for this task." + ); + } + + return message(`Process exited with code ${code}.`); +} diff --git a/packages/cli-v3/src/workers/dev/backgroundWorker.ts b/packages/cli-v3/src/workers/dev/backgroundWorker.ts index 7b619f60d..250b1dd59 100644 --- a/packages/cli-v3/src/workers/dev/backgroundWorker.ts +++ b/packages/cli-v3/src/workers/dev/backgroundWorker.ts @@ -45,6 +45,7 @@ import { TaskMetadataParseError, UncaughtExceptionError, UnexpectedExitError, + getFriendlyErrorMessage, } from "../common/errors.js"; import { CliApiClient } from "../../apiClient.js"; @@ -723,6 +724,8 @@ export class BackgroundWorker { error: { type: "INTERNAL_ERROR", code: TaskRunErrorCodes.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE, + message: getFriendlyErrorMessage(e.code, e.signal, e.stderr), + stackTrace: e.stderr, }, }; } @@ -782,6 +785,7 @@ class TaskRunProcess { private _currentExecution: TaskRunExecution | undefined; private _isBeingKilled: boolean = false; private _isBeingCancelled: boolean = false; + private _stderr: Array = []; /** * @deprecated use onTaskRunHeartbeat instead */ @@ -1009,7 +1013,13 @@ class TaskRunProcess { } else if (this._isBeingKilled) { rejecter(new CleanupProcessError()); } else { - rejecter(new UnexpectedExitError(code ?? -1)); + rejecter( + new UnexpectedExitError( + code ?? -1, + signal, + this._stderr.length ? this._stderr.join("\n") : undefined + ) + ); } } } @@ -1048,9 +1058,16 @@ class TaskRunProcess { `${this._currentExecution.run.id}.${this._currentExecution.attempt.number}` ); + const errorLine = data.toString(); + logger.log( - `${chalkError("○")} ${chalkGrey(prettyPrintDate(new Date()))} ${runId} ${data.toString()}` + `${chalkError("○")} ${chalkGrey(prettyPrintDate(new Date()))} ${runId} ${errorLine}` ); + + if (this._stderr.length > 100) { + this._stderr.shift(); + } + this._stderr.push(errorLine); } #kill() { diff --git a/packages/cli-v3/src/workers/prod/backgroundWorker.ts b/packages/cli-v3/src/workers/prod/backgroundWorker.ts index e7fd521fe..88867d22d 100644 --- a/packages/cli-v3/src/workers/prod/backgroundWorker.ts +++ b/packages/cli-v3/src/workers/prod/backgroundWorker.ts @@ -29,6 +29,7 @@ import { TaskMetadataParseError, UncaughtExceptionError, UnexpectedExitError, + getFriendlyErrorMessage, } from "../common/errors"; type BackgroundWorkerParams = { @@ -462,6 +463,8 @@ export class ProdBackgroundWorker { error: { type: "INTERNAL_ERROR", code: TaskRunErrorCodes.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE, + message: getFriendlyErrorMessage(e.code, e.signal, e.stderr), + stackTrace: e.stderr, }, }; } @@ -576,6 +579,7 @@ class TaskRunProcess { private _isBeingKilled: boolean = false; private _isBeingCancelled: boolean = false; private _gracefulExitTimeoutElapsed: boolean = false; + private _stderr: Array = []; /** * @deprecated use onTaskRunHeartbeat instead @@ -854,7 +858,13 @@ class TaskRunProcess { } else if (this._isBeingKilled) { rejecter(new CleanupProcessError()); } else { - rejecter(new UnexpectedExitError(code ?? -1)); + rejecter( + new UnexpectedExitError( + code ?? -1, + signal, + this._stderr.length ? this._stderr.join("\n") : undefined + ) + ); } } } @@ -867,7 +877,13 @@ class TaskRunProcess { } #handleStdErr(data: Buffer) { - console.error(data.toString()); + const text = data.toString(); + console.error(text); + + if (this._stderr.length > 100) { + this._stderr.shift(); + } + this._stderr.push(text); } async kill(signal?: number | NodeJS.Signals, timeoutInMs?: number) { diff --git a/packages/core/src/v3/schemas/common.ts b/packages/core/src/v3/schemas/common.ts index 8d2ed2cdb..34eb42690 100644 --- a/packages/core/src/v3/schemas/common.ts +++ b/packages/core/src/v3/schemas/common.ts @@ -109,6 +109,7 @@ export const TaskRunInternalError = z.object({ "TASK_RUN_HEARTBEAT_TIMEOUT", ]), message: z.string().optional(), + stackTrace: z.string().optional(), }); export type TaskRunInternalError = z.infer; diff --git a/references/v3-catalog/src/trigger/other.ts b/references/v3-catalog/src/trigger/other.ts index dd57fc07c..273466b16 100644 --- a/references/v3-catalog/src/trigger/other.ts +++ b/references/v3-catalog/src/trigger/other.ts @@ -82,3 +82,28 @@ export const unfriendlyIdTask = task({ console.log("Hello world"); }, }); + +export const oomTask = task({ + id: "oom-task", + machine: { + preset: "micro", + }, + run: async () => { + logger.info("running out of memory below this line"); + + let a = "a"; + + try { + while (true) { + a += a; + } + } catch (error) { + logger.error(error instanceof Error ? error.message : "Unknown error", { error }); + + let b = []; + while (true) { + b.push(a.replace(/a/g, "b")); + } + } + }, +});