v3: Refactor attempt creation to be worker requested (#1077)
* WIP worker TaskRunAttempt creation
* Handling failing task runs that cannot create an attempt for whatever reason
* Move the visibility queue stuff into a graphile job
* Fixed task runs with unsanitized queue names
* “Borrow” the code from alerts PR to get self hosted deployments working
* Add an admin API endpoint to get info about the shared marqs queue
* Allow admins to view any project metrics
* start adding lazy attempts to prod
* lazy attempt creation for prod workers
* resurrect prod stack traces
* add exception event to failed run spans
* simplify dependency resumes
* fix typecheck
* fix merge
* fresh process for all attempts
* always try sigterm first
* stop heartbeat timeout on non-inplace replace message
* add missing ack on checkpoint creation service failure
* bypass dequeue for retries with running worker
* respect retry delays
* crash runs with invalid run status for execution
* remove debug logs
* fix nack message
* fix version locking
* fresh attempt processes in dev and prod
* improve handling of ipc timeouts
* consider checkpoint failures on cancellation
* add basic chaos monkey to checkpointer
* changeset
* control forced checkpoint simulation via env var
* fix merge
* kill old attempt processes before checkpointing
* detailed perf logging for checkpointing
* add coordinator otlp endpoint example
* improve prod run cancellation
* rename supports lazy attempts migration
* fix graceful exit
* fix retry mechanics
* clear paused state before retry
* remove checkpoint image after push
* crash worker on unrecoverable errors
* refactor unrecoverable error emit
* switch to do hosted busybox image
* increase wait for duration ipc timeout
* add changeset for misc fixes
* fix merge
* fix retry delay span runId
* fix dev retries
* improve prod worker logging
* log checkpoint sizes
* add lazy attempts catalog entries
* Fixed merge issue: use zodFetch, not wrapZodFetch
* Revert "Fixed merge issue: use zodFetch, not wrapZodFetch"
This reverts commit d137e4e1fe.
* importEnvVars uses wrapZodFetch now
* add backwards compat for retries without checkpoints
* handle more cases of unrecoverable runs
* don't kill the child process if it shouldn't be killed
---------
Co-authored-by: nicktrn <55853254+nicktrn@users.noreply.github.com>
Co-authored-by: Matt Aitken <matt@mattaitken.com>
This commit is contained in:
Eric Allam
@@ -0,0 +1,14 @@
|
||||
---
|
||||
"trigger.dev": patch
|
||||
"@trigger.dev/core": patch
|
||||
---
|
||||
|
||||
- Clear paused states before retry
|
||||
- Detect and handle unrecoverable worker errors
|
||||
- Remove checkpoints after successful push
|
||||
- Permanently switch to DO hosted busybox image
|
||||
- Fix IPC timeout issue, or at least handle it more gracefully
|
||||
- Handle checkpoint failures
|
||||
- Basic chaos monkey for checkpoint testing
|
||||
- Stack traces are back in the dashboard
|
||||
- Display final errors on root span
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@trigger.dev/core": patch
|
||||
---
|
||||
|
||||
Improve handling of IPC timeouts and fix checkpoint cancellation after failures
|
||||
+296
-50
@@ -1,5 +1,6 @@
|
||||
import { createServer } from "node:http";
|
||||
import { $ } from "execa";
|
||||
import fs from "node:fs/promises";
|
||||
import { $, type ExecaChildProcess } from "execa";
|
||||
import { nanoid } from "nanoid";
|
||||
import { Server } from "socket.io";
|
||||
import {
|
||||
@@ -19,6 +20,11 @@ collectDefaultMetrics();
|
||||
const HTTP_SERVER_PORT = Number(process.env.HTTP_SERVER_PORT || 8020);
|
||||
const NODE_NAME = process.env.NODE_NAME || "coordinator";
|
||||
const DEFAULT_RETRY_DELAY_THRESHOLD_IN_MS = 30_000;
|
||||
const CHAOS_MONKEY_ENABLED = !!process.env.CHAOS_MONKEY_ENABLED;
|
||||
|
||||
const FORCE_CHECKPOINT_SIMULATION = ["1", "true"].includes(
|
||||
process.env.FORCE_CHECKPOINT_SIMULATION ?? "true"
|
||||
);
|
||||
|
||||
const REGISTRY_HOST = process.env.REGISTRY_HOST || "localhost:5000";
|
||||
const CHECKPOINT_PATH = process.env.CHECKPOINT_PATH || "/checkpoints";
|
||||
@@ -32,6 +38,10 @@ const SECURE_CONNECTION = ["1", "true"].includes(process.env.SECURE_CONNECTION ?
|
||||
|
||||
const logger = new SimpleLogger(`[${NODE_NAME}]`);
|
||||
|
||||
if (CHAOS_MONKEY_ENABLED) {
|
||||
logger.log("🍌 Chaos monkey enabled");
|
||||
}
|
||||
|
||||
type CheckpointerInitializeReturn = {
|
||||
canCheckpoint: boolean;
|
||||
willSimulate: boolean;
|
||||
@@ -49,6 +59,40 @@ type CheckpointData = {
|
||||
docker: boolean;
|
||||
};
|
||||
|
||||
function isExecaChildProcess(maybeExeca: unknown): maybeExeca is Awaited<ExecaChildProcess> {
|
||||
return typeof maybeExeca === "object" && maybeExeca !== null && "escapedCommand" in maybeExeca;
|
||||
}
|
||||
|
||||
async function getFileSize(filePath: string): Promise<number> {
|
||||
try {
|
||||
const stats = await fs.stat(filePath);
|
||||
return stats.size;
|
||||
} catch (error) {
|
||||
console.error("Error getting file size:", error);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
async function getParsedFileSize(filePath: string) {
|
||||
const sizeInBytes = await getFileSize(filePath);
|
||||
|
||||
let message = `Size in bytes: ${sizeInBytes}`;
|
||||
|
||||
if (sizeInBytes > 1024 * 1024) {
|
||||
const sizeInMB = (sizeInBytes / 1024 / 1024).toFixed(2);
|
||||
message = `Size in MB (rounded): ${sizeInMB}`;
|
||||
} else if (sizeInBytes > 1024) {
|
||||
const sizeInKB = (sizeInBytes / 1024).toFixed(2);
|
||||
message = `Size in KB (rounded): ${sizeInKB}`;
|
||||
}
|
||||
|
||||
return {
|
||||
path: filePath,
|
||||
sizeInBytes,
|
||||
message,
|
||||
};
|
||||
}
|
||||
|
||||
class Checkpointer {
|
||||
#initialized = false;
|
||||
#canCheckpoint = false;
|
||||
@@ -56,6 +100,7 @@ class Checkpointer {
|
||||
|
||||
#logger = new SimpleLogger("[checkptr]");
|
||||
#abortControllers = new Map<string, AbortController>();
|
||||
#failedCheckpoints = new Map<string, unknown>();
|
||||
|
||||
constructor(private opts = { forceSimulate: false }) {}
|
||||
|
||||
@@ -150,7 +195,11 @@ class Checkpointer {
|
||||
success: !!result,
|
||||
});
|
||||
|
||||
return result;
|
||||
if (!result.success) {
|
||||
return;
|
||||
}
|
||||
|
||||
return result.checkpoint;
|
||||
}
|
||||
|
||||
isCheckpointing(runId: string) {
|
||||
@@ -158,6 +207,13 @@ class Checkpointer {
|
||||
}
|
||||
|
||||
cancelCheckpoint(runId: string): boolean {
|
||||
// If the last checkpoint failed, pretend we canceled it
|
||||
// This ensures tasks don't wait for external resume messages to continue
|
||||
if (this.#hasFailedCheckpoint(runId)) {
|
||||
this.#clearFailedCheckpoint(runId);
|
||||
return true;
|
||||
}
|
||||
|
||||
const controller = this.#abortControllers.get(runId);
|
||||
|
||||
if (!controller) {
|
||||
@@ -176,44 +232,58 @@ class Checkpointer {
|
||||
leaveRunning = true, // This mirrors kubernetes behaviour more accurately
|
||||
projectRef,
|
||||
deploymentVersion,
|
||||
}: CheckpointAndPushOptions): Promise<CheckpointData | undefined> {
|
||||
}: CheckpointAndPushOptions): Promise<
|
||||
{ success: true; checkpoint: CheckpointData } | { success: false; reason?: "CANCELED" }
|
||||
> {
|
||||
await this.initialize();
|
||||
|
||||
const options = {
|
||||
runId,
|
||||
leaveRunning,
|
||||
projectRef,
|
||||
deploymentVersion,
|
||||
};
|
||||
|
||||
if (!this.#dockerMode && !this.#canCheckpoint) {
|
||||
this.#logger.error("No checkpoint support. Simulation requires docker.");
|
||||
return;
|
||||
return { success: false };
|
||||
}
|
||||
|
||||
if (this.#abortControllers.has(runId)) {
|
||||
logger.error("Checkpoint procedure already in progress", {
|
||||
options: {
|
||||
runId,
|
||||
leaveRunning,
|
||||
projectRef,
|
||||
deploymentVersion,
|
||||
},
|
||||
});
|
||||
return;
|
||||
logger.error("Checkpoint procedure already in progress", { options });
|
||||
return { success: false };
|
||||
}
|
||||
|
||||
// This is a new checkpoint, clear any last failure for this run
|
||||
this.#clearFailedCheckpoint(runId);
|
||||
|
||||
const controller = new AbortController();
|
||||
this.#abortControllers.set(runId, controller);
|
||||
|
||||
const $$ = $({ signal: controller.signal });
|
||||
|
||||
try {
|
||||
if (CHAOS_MONKEY_ENABLED) {
|
||||
console.log("🍌 Chaos monkey wreaking havoc");
|
||||
|
||||
const random = Math.random();
|
||||
|
||||
if (random < 0.33) {
|
||||
// Fake long checkpoint duration
|
||||
await $$`sleep 300`;
|
||||
} else if (random < 0.66) {
|
||||
// Fake checkpoint error
|
||||
await $$`false`;
|
||||
} else {
|
||||
// no-op
|
||||
}
|
||||
}
|
||||
|
||||
const shortCode = nanoid(8);
|
||||
const imageRef = this.#getImageRef(projectRef, deploymentVersion, shortCode);
|
||||
const exportLocation = this.#getExportLocation(projectRef, deploymentVersion, shortCode);
|
||||
|
||||
this.#logger.log("Checkpointing:", {
|
||||
options: {
|
||||
runId,
|
||||
leaveRunning,
|
||||
projectRef,
|
||||
deploymentVersion,
|
||||
},
|
||||
});
|
||||
this.#logger.log("Checkpointing:", { options });
|
||||
|
||||
const containterName = this.#getRunContainerName(runId);
|
||||
|
||||
@@ -234,9 +304,9 @@ class Checkpointer {
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
this.#logger.error(error.stderr);
|
||||
return;
|
||||
} catch (error) {
|
||||
this.#logger.error("Failed while creating docker checkpoint", { exportLocation });
|
||||
throw error;
|
||||
}
|
||||
|
||||
this.#logger.log("checkpoint created:", {
|
||||
@@ -245,8 +315,11 @@ class Checkpointer {
|
||||
});
|
||||
|
||||
return {
|
||||
location: exportLocation,
|
||||
docker: true,
|
||||
success: true,
|
||||
checkpoint: {
|
||||
location: exportLocation,
|
||||
docker: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -266,54 +339,104 @@ class Checkpointer {
|
||||
throw new Error("could not find container id");
|
||||
}
|
||||
|
||||
const start = performance.now();
|
||||
|
||||
// Create checkpoint
|
||||
this.#logger.debug(await $$`crictl checkpoint --export=${exportLocation} ${containerId}`);
|
||||
const postCheckpoint = performance.now();
|
||||
|
||||
// Print checkpoint size
|
||||
const size = await getParsedFileSize(exportLocation);
|
||||
this.#logger.log("checkpoint archive created", { size, options });
|
||||
|
||||
// Create image from checkpoint
|
||||
const container = this.#logger.debug(await $$`buildah from scratch`);
|
||||
const postFrom = performance.now();
|
||||
|
||||
this.#logger.debug(await $$`buildah add ${container} ${exportLocation} /`);
|
||||
const postAdd = performance.now();
|
||||
|
||||
this.#logger.debug(
|
||||
await $$`buildah config --annotation=io.kubernetes.cri-o.annotations.checkpoint.name=counter ${container}`
|
||||
);
|
||||
const postConfig = performance.now();
|
||||
|
||||
this.#logger.debug(await $$`buildah commit ${container} ${imageRef}`);
|
||||
const postCommit = performance.now();
|
||||
|
||||
this.#logger.debug(await $$`buildah rm ${container}`);
|
||||
const postRm = performance.now();
|
||||
|
||||
// Push checkpoint image
|
||||
this.#logger.debug(await $$`buildah push --tls-verify=${REGISTRY_TLS_VERIFY} ${imageRef}`);
|
||||
const postPush = performance.now();
|
||||
|
||||
this.#logger.log("Checkpointed and pushed image to:", { location: imageRef });
|
||||
const perf = {
|
||||
"crictl checkpoint": postCheckpoint - start,
|
||||
"buildah from": postFrom - postCheckpoint,
|
||||
"buildah add": postAdd - postFrom,
|
||||
"buildah config": postConfig - postAdd,
|
||||
"buildah commit": postCommit - postConfig,
|
||||
"buildah rm": postRm - postCommit,
|
||||
"buildah push": postPush - postRm,
|
||||
};
|
||||
|
||||
this.#logger.log("Checkpointed and pushed image to:", { location: imageRef, perf });
|
||||
|
||||
try {
|
||||
await $$`rm ${exportLocation}`;
|
||||
this.#logger.log("Deleted checkpoint archive", { exportLocation });
|
||||
|
||||
// Disabled for now as this will increase restore time by having to pull the image again
|
||||
// await $`buildah rmi ${imageRef}`;
|
||||
// this.#logger.log("Deleted checkpoint image", { imageRef });
|
||||
await $`buildah rmi ${imageRef}`;
|
||||
this.#logger.log("Deleted checkpoint image", { imageRef });
|
||||
} catch (error) {
|
||||
this.#logger.error("Failed during checkpoint cleanup", { exportLocation });
|
||||
this.#logger.debug(error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
return {
|
||||
location: imageRef,
|
||||
docker: false,
|
||||
success: true,
|
||||
checkpoint: {
|
||||
location: imageRef,
|
||||
docker: false,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
this.#logger.error("checkpoint failed", {
|
||||
options: {
|
||||
runId,
|
||||
leaveRunning,
|
||||
projectRef,
|
||||
deploymentVersion,
|
||||
},
|
||||
error,
|
||||
});
|
||||
return;
|
||||
if (isExecaChildProcess(error)) {
|
||||
if (error.isCanceled) {
|
||||
this.#logger.error("Checkpoint canceled", { options, error });
|
||||
|
||||
return { success: false, reason: "CANCELED" };
|
||||
}
|
||||
|
||||
// Everything that's not a cancellation is a failure
|
||||
this.#failCheckpoint(runId, error);
|
||||
this.#logger.error("Checkpoint command error", { options, error });
|
||||
|
||||
return { success: false };
|
||||
}
|
||||
|
||||
this.#failCheckpoint(runId, error);
|
||||
this.#logger.error("Unhandled checkpoint error", { options, error });
|
||||
|
||||
return { success: false };
|
||||
} finally {
|
||||
this.#abortControllers.delete(runId);
|
||||
}
|
||||
}
|
||||
|
||||
#failCheckpoint(runId: string, error: unknown) {
|
||||
this.#failedCheckpoints.set(runId, error);
|
||||
}
|
||||
|
||||
#clearFailedCheckpoint(runId: string) {
|
||||
this.#failedCheckpoints.delete(runId);
|
||||
}
|
||||
|
||||
#hasFailedCheckpoint(runId: string) {
|
||||
return this.#failedCheckpoints.has(runId);
|
||||
}
|
||||
|
||||
#getRunContainerName(suffix: string) {
|
||||
return `task-run-${suffix}`;
|
||||
}
|
||||
@@ -321,7 +444,7 @@ class Checkpointer {
|
||||
|
||||
class TaskCoordinator {
|
||||
#httpServer: ReturnType<typeof createServer>;
|
||||
#checkpointer = new Checkpointer({ forceSimulate: true });
|
||||
#checkpointer = new Checkpointer({ forceSimulate: FORCE_CHECKPOINT_SIMULATION });
|
||||
|
||||
#prodWorkerNamespace: ZodNamespace<
|
||||
typeof ProdWorkerToCoordinatorMessages,
|
||||
@@ -442,6 +565,28 @@ class TaskCoordinator {
|
||||
|
||||
taskSocket.emit("REQUEST_ATTEMPT_CANCELLATION", message);
|
||||
},
|
||||
REQUEST_RUN_CANCELLATION: async (message) => {
|
||||
const taskSocket = await this.#getRunSocket(message.runId);
|
||||
|
||||
if (!taskSocket) {
|
||||
logger.log("Socket for run not found", {
|
||||
runId: message.runId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.delayInMs) {
|
||||
taskSocket.emit("REQUEST_EXIT", {
|
||||
version: "v2",
|
||||
delayInMs: message.delayInMs,
|
||||
});
|
||||
} else {
|
||||
// If there's no delay, assume the worker doesn't support non-v1 messages
|
||||
taskSocket.emit("REQUEST_EXIT", {
|
||||
version: "v1",
|
||||
});
|
||||
}
|
||||
},
|
||||
READY_FOR_RETRY: async (message) => {
|
||||
const taskSocket = await this.#getRunSocket(message.runId);
|
||||
|
||||
@@ -528,6 +673,20 @@ class TaskCoordinator {
|
||||
onConnection: async (socket, handler, sender) => {
|
||||
const logger = new SimpleLogger(`[prod-worker][${socket.id}]`);
|
||||
|
||||
const crashRun = async (error: { name: string; message: string; stack?: string }) => {
|
||||
try {
|
||||
this.#platformSocket?.send("RUN_CRASHED", {
|
||||
version: "v1",
|
||||
runId: socket.data.runId,
|
||||
error,
|
||||
});
|
||||
} finally {
|
||||
socket.emit("REQUEST_EXIT", {
|
||||
version: "v1",
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const checkpointInProgress = () => {
|
||||
return this.#checkpointableTasks.has(socket.data.runId);
|
||||
};
|
||||
@@ -596,8 +755,9 @@ class TaskCoordinator {
|
||||
if (!executionAck) {
|
||||
logger.error("no execution ack", { runId: socket.data.runId });
|
||||
|
||||
socket.emit("REQUEST_EXIT", {
|
||||
version: "v1",
|
||||
await crashRun({
|
||||
name: "ReadyForExecutionError",
|
||||
message: "No execution ack",
|
||||
});
|
||||
|
||||
return;
|
||||
@@ -606,8 +766,9 @@ class TaskCoordinator {
|
||||
if (!executionAck.success) {
|
||||
logger.error("failed to get execution payload", { runId: socket.data.runId });
|
||||
|
||||
socket.emit("REQUEST_EXIT", {
|
||||
version: "v1",
|
||||
await crashRun({
|
||||
name: "ReadyForExecutionError",
|
||||
message: "Failed to get execution payload",
|
||||
});
|
||||
|
||||
return;
|
||||
@@ -624,6 +785,46 @@ class TaskCoordinator {
|
||||
}
|
||||
});
|
||||
|
||||
socket.on("READY_FOR_LAZY_ATTEMPT", async (message) => {
|
||||
logger.log("[READY_FOR_LAZY_ATTEMPT]", message);
|
||||
|
||||
try {
|
||||
const lazyAttempt = await this.#platformSocket?.sendWithAck("READY_FOR_LAZY_ATTEMPT", {
|
||||
...message,
|
||||
envId: socket.data.envId,
|
||||
});
|
||||
|
||||
if (!lazyAttempt) {
|
||||
logger.error("no lazy attempt ack", { runId: socket.data.runId });
|
||||
|
||||
await crashRun({
|
||||
name: "ReadyForLazyAttemptError",
|
||||
message: "No lazy attempt ack",
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (!lazyAttempt.success) {
|
||||
logger.error("failed to get lazy attempt payload", { runId: socket.data.runId });
|
||||
|
||||
await crashRun({
|
||||
name: "ReadyForLazyAttemptError",
|
||||
message: "Failed to get lazy attempt payload",
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
socket.emit("EXECUTE_TASK_RUN_LAZY_ATTEMPT", {
|
||||
version: "v1",
|
||||
lazyPayload: lazyAttempt.lazyPayload,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error("Error", { error });
|
||||
}
|
||||
});
|
||||
|
||||
socket.on("READY_FOR_RESUME", async (message) => {
|
||||
logger.log("[READY_FOR_RESUME]", message);
|
||||
|
||||
@@ -714,6 +915,19 @@ class TaskCoordinator {
|
||||
}
|
||||
});
|
||||
|
||||
socket.on("TASK_RUN_FAILED_TO_RUN", async ({ completion }) => {
|
||||
logger.log("completed task", { completionId: completion.id });
|
||||
|
||||
this.#platformSocket?.send("TASK_RUN_FAILED_TO_RUN", {
|
||||
version: "v1",
|
||||
completion,
|
||||
});
|
||||
|
||||
socket.emit("REQUEST_EXIT", {
|
||||
version: "v1",
|
||||
});
|
||||
});
|
||||
|
||||
socket.on("READY_FOR_CHECKPOINT", async (message) => {
|
||||
logger.log("[READY_FOR_CHECKPOINT]", message);
|
||||
|
||||
@@ -890,7 +1104,7 @@ class TaskCoordinator {
|
||||
logger.log("[INDEX_TASKS]", message);
|
||||
|
||||
const workerAck = await this.#platformSocket?.sendWithAck("CREATE_WORKER", {
|
||||
version: "v1",
|
||||
version: "v2",
|
||||
projectRef: socket.data.projectRef,
|
||||
envId: socket.data.envId,
|
||||
deploymentId: message.deploymentId,
|
||||
@@ -899,6 +1113,7 @@ class TaskCoordinator {
|
||||
packageVersion: message.packageVersion,
|
||||
tasks: message.tasks,
|
||||
},
|
||||
supportsLazyAttempts: message.version !== "v1" && message.supportsLazyAttempts,
|
||||
});
|
||||
|
||||
if (!workerAck) {
|
||||
@@ -917,6 +1132,34 @@ class TaskCoordinator {
|
||||
error: message.error,
|
||||
});
|
||||
});
|
||||
|
||||
socket.on("CREATE_TASK_RUN_ATTEMPT", async (message, callback) => {
|
||||
logger.log("[CREATE_TASK_RUN_ATTEMPT]", message);
|
||||
|
||||
const createAttempt = await this.#platformSocket?.sendWithAck("CREATE_TASK_RUN_ATTEMPT", {
|
||||
runId: message.runId,
|
||||
envId: socket.data.envId,
|
||||
});
|
||||
|
||||
if (!createAttempt?.success) {
|
||||
logger.debug("no ack while creating attempt", message);
|
||||
callback({ success: false });
|
||||
return;
|
||||
}
|
||||
|
||||
socket.data.attemptFriendlyId = createAttempt.executionPayload.execution.attempt.id;
|
||||
|
||||
callback({
|
||||
success: true,
|
||||
executionPayload: createAttempt.executionPayload,
|
||||
});
|
||||
});
|
||||
|
||||
socket.on("UNRECOVERABLE_ERROR", async (message) => {
|
||||
logger.log("[UNRECOVERABLE_ERROR]", message);
|
||||
|
||||
await crashRun(message.error);
|
||||
});
|
||||
},
|
||||
onDisconnect: async (socket, handler, sender, logger) => {
|
||||
this.#platformSocket?.send("LOG", {
|
||||
@@ -928,13 +1171,16 @@ class TaskCoordinator {
|
||||
TASK_HEARTBEAT: async (message) => {
|
||||
this.#platformSocket?.send("TASK_HEARTBEAT", message);
|
||||
},
|
||||
TASK_RUN_HEARTBEAT: async (message) => {
|
||||
this.#platformSocket?.send("TASK_RUN_HEARTBEAT", message);
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return provider;
|
||||
}
|
||||
|
||||
#cancelCheckpoint(runId: string) {
|
||||
#cancelCheckpoint(runId: string): boolean {
|
||||
const checkpointWait = this.#checkpointableTasks.get(runId);
|
||||
|
||||
if (checkpointWait) {
|
||||
|
||||
@@ -4,6 +4,8 @@ PLATFORM_WS_PORT=3030
|
||||
PLATFORM_SECRET=provider-secret
|
||||
SECURE_CONNECTION=false
|
||||
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://0.0.0.0:3030/otel
|
||||
|
||||
# Use this if you are on macOS
|
||||
# COORDINATOR_HOST="host.docker.internal"
|
||||
# OTEL_EXPORTER_OTLP_ENDPOINT="http://host.docker.internal:4318"
|
||||
@@ -13,9 +13,14 @@ import { PostStartCauses, PreStopCauses } from "@trigger.dev/core/v3";
|
||||
const MACHINE_NAME = process.env.MACHINE_NAME || "local";
|
||||
const COORDINATOR_PORT = process.env.COORDINATOR_PORT || 8020;
|
||||
const COORDINATOR_HOST = process.env.COORDINATOR_HOST || "127.0.0.1";
|
||||
|
||||
const OTEL_EXPORTER_OTLP_ENDPOINT =
|
||||
process.env.OTEL_EXPORTER_OTLP_ENDPOINT || "http://0.0.0.0:4318";
|
||||
|
||||
const FORCE_CHECKPOINT_SIMULATION = ["1", "true"].includes(
|
||||
process.env.FORCE_CHECKPOINT_SIMULATION ?? "true"
|
||||
);
|
||||
|
||||
const logger = new SimpleLogger(`[${MACHINE_NAME}]`);
|
||||
|
||||
type InitializeReturn = {
|
||||
@@ -278,7 +283,7 @@ class DockerTaskOperations implements TaskOperations {
|
||||
}
|
||||
|
||||
const provider = new ProviderShell({
|
||||
tasks: new DockerTaskOperations({ forceSimulate: true }),
|
||||
tasks: new DockerTaskOperations({ forceSimulate: FORCE_CHECKPOINT_SIMULATION }),
|
||||
type: "docker",
|
||||
});
|
||||
|
||||
|
||||
@@ -212,7 +212,7 @@ class KubernetesTaskOperations implements TaskOperations {
|
||||
},
|
||||
{
|
||||
name: "populate-taskinfo",
|
||||
image: "docker.io/library/busybox",
|
||||
image: "registry.digitalocean.com/trigger/busybox",
|
||||
imagePullPolicy: "IfNotPresent",
|
||||
command: ["/bin/sh", "-c"],
|
||||
args: ["printenv COORDINATOR_HOST | tee /etc/taskinfo/coordinator-host"],
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import { LoaderFunctionArgs, json } from "@remix-run/server-runtime";
|
||||
import { prisma } from "~/db.server";
|
||||
import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
|
||||
import { marqs } from "~/v3/marqs/index.server";
|
||||
|
||||
export async function loader({ request, params }: LoaderFunctionArgs) {
|
||||
// Next authenticate the request
|
||||
const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
|
||||
|
||||
if (!authenticationResult) {
|
||||
return json({ error: "Invalid or Missing API key" }, { status: 401 });
|
||||
}
|
||||
|
||||
const user = await prisma.user.findUnique({
|
||||
where: {
|
||||
id: authenticationResult.userId,
|
||||
},
|
||||
});
|
||||
|
||||
if (!user) {
|
||||
return json({ error: "Invalid or Missing API key" }, { status: 401 });
|
||||
}
|
||||
|
||||
if (!user.admin) {
|
||||
return json({ error: "You must be an admin to perform this action" }, { status: 403 });
|
||||
}
|
||||
|
||||
const details = await marqs?.getSharedQueueDetails();
|
||||
|
||||
return json(details);
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import type { ActionFunctionArgs } from "@remix-run/server-runtime";
|
||||
import { json } from "@remix-run/server-runtime";
|
||||
import { z } from "zod";
|
||||
import { authenticateApiRequest } from "~/services/apiAuth.server";
|
||||
import { ServiceValidationError } from "~/v3/services/baseService.server";
|
||||
import { CreateTaskRunAttemptService } from "~/v3/services/createTaskRunAttempt.server";
|
||||
|
||||
const ParamsSchema = z.object({
|
||||
/* This is the run friendly ID */
|
||||
runParam: z.string(),
|
||||
});
|
||||
|
||||
export async function action({ request, params }: ActionFunctionArgs) {
|
||||
// Authenticate the request
|
||||
const authenticationResult = await authenticateApiRequest(request);
|
||||
|
||||
if (!authenticationResult) {
|
||||
return json({ error: "Invalid or Missing API Key" }, { status: 401 });
|
||||
}
|
||||
|
||||
const parsed = ParamsSchema.safeParse(params);
|
||||
|
||||
if (!parsed.success) {
|
||||
return json({ error: "Invalid or missing run ID" }, { status: 400 });
|
||||
}
|
||||
|
||||
const { runParam } = parsed.data;
|
||||
|
||||
const service = new CreateTaskRunAttemptService();
|
||||
|
||||
try {
|
||||
const { execution } = await service.call(runParam, authenticationResult.environment);
|
||||
|
||||
return json(execution, { status: 200 });
|
||||
} catch (error) {
|
||||
if (error instanceof ServiceValidationError) {
|
||||
return json({ error: error.message }, { status: error.status ?? 422 });
|
||||
}
|
||||
|
||||
return json(
|
||||
{ error: error instanceof Error ? error.message : "Internal Server Error" },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -44,6 +44,8 @@ import { GraphileMigrationHelperService } from "./db/graphileMigrationHelper.ser
|
||||
import { PerformBulkActionService } from "~/v3/services/bulk/performBulkAction.server";
|
||||
import { CancelTaskRunService } from "~/v3/services/cancelTaskRun.server";
|
||||
import { ReplayTaskRunService } from "~/v3/services/replayTaskRun.server";
|
||||
import { RequeueTaskRunService } from "~/v3/requeueTaskRun.server";
|
||||
import { RetryAttemptService } from "~/v3/services/retryAttempt.server";
|
||||
|
||||
const workerCatalog = {
|
||||
indexEndpoint: z.object({
|
||||
@@ -158,6 +160,12 @@ const workerCatalog = {
|
||||
"v3.performBulkActionItem": z.object({
|
||||
bulkActionItemId: z.string(),
|
||||
}),
|
||||
"v3.requeueTaskRun": z.object({
|
||||
runId: z.string(),
|
||||
}),
|
||||
"v3.retryAttempt": z.object({
|
||||
runId: z.string(),
|
||||
}),
|
||||
};
|
||||
|
||||
const executionWorkerCatalog = {
|
||||
@@ -600,6 +608,24 @@ function getWorkerQueue() {
|
||||
await service.performBulkActionItem(payload.bulkActionItemId);
|
||||
},
|
||||
},
|
||||
"v3.requeueTaskRun": {
|
||||
priority: 0,
|
||||
maxAttempts: 3,
|
||||
handler: async (payload, job) => {
|
||||
const service = new RequeueTaskRunService();
|
||||
|
||||
await service.call(payload.runId);
|
||||
},
|
||||
},
|
||||
"v3.retryAttempt": {
|
||||
priority: 0,
|
||||
maxAttempts: 3,
|
||||
handler: async (payload, job) => {
|
||||
const service = new RetryAttemptService();
|
||||
|
||||
return await service.call(payload.runId);
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -54,7 +54,10 @@ export class AuthenticatedSocketConnection {
|
||||
schema: clientWebsocketMessages,
|
||||
messages: {
|
||||
READY_FOR_TASKS: async (payload) => {
|
||||
await this._consumer.registerBackgroundWorker(payload.backgroundWorkerId);
|
||||
await this._consumer.registerBackgroundWorker(
|
||||
payload.backgroundWorkerId,
|
||||
payload.inProgressRuns ?? []
|
||||
);
|
||||
},
|
||||
BACKGROUND_WORKER_DEPRECATED: async (payload) => {
|
||||
await this._consumer.deprecateBackgroundWorker(payload.backgroundWorkerId);
|
||||
@@ -69,10 +72,22 @@ export class AuthenticatedSocketConnection {
|
||||
);
|
||||
break;
|
||||
}
|
||||
case "TASK_RUN_FAILED_TO_RUN": {
|
||||
await this._consumer.taskRunFailed(
|
||||
payload.backgroundWorkerId,
|
||||
payload.data.completion
|
||||
);
|
||||
|
||||
break;
|
||||
}
|
||||
case "TASK_HEARTBEAT": {
|
||||
await this._consumer.taskHeartbeat(payload.backgroundWorkerId, payload.data.id);
|
||||
break;
|
||||
}
|
||||
case "TASK_RUN_HEARTBEAT": {
|
||||
await this._consumer.taskRunHeartbeat(payload.backgroundWorkerId, payload.data.id);
|
||||
break;
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
SpanEvents,
|
||||
SpanMessagingEvent,
|
||||
TaskEventStyle,
|
||||
TaskRunError,
|
||||
correctErrorStackTrace,
|
||||
createPacketAttributesAsJson,
|
||||
flattenAttributes,
|
||||
@@ -117,6 +118,7 @@ export type QueriedEvent = Prisma.TaskEventGetPayload<{
|
||||
isCancelled: true;
|
||||
level: true;
|
||||
events: true;
|
||||
environmentType: true;
|
||||
};
|
||||
}>;
|
||||
|
||||
@@ -156,6 +158,7 @@ export type SpanSummary = {
|
||||
isPartial: boolean;
|
||||
isCancelled: boolean;
|
||||
level: NonNullable<CreatableEvent["level"]>;
|
||||
environmentType: CreatableEventEnvironmentType;
|
||||
};
|
||||
};
|
||||
|
||||
@@ -165,6 +168,7 @@ export type UpdateEventOptions = {
|
||||
attributes: TraceAttributes;
|
||||
endTime?: Date;
|
||||
immediate?: boolean;
|
||||
events?: SpanEvents;
|
||||
};
|
||||
|
||||
export class EventRepository {
|
||||
@@ -239,7 +243,7 @@ export class EventRepository {
|
||||
isCancelled: false,
|
||||
status: options?.attributes.isError ? "ERROR" : "OK",
|
||||
links: event.links ?? [],
|
||||
events: event.events ?? [],
|
||||
events: event.events ?? (options?.events as any) ?? [],
|
||||
duration: calculateDurationFromStart(event.startTime, options?.endTime),
|
||||
properties: event.properties as Attributes,
|
||||
metadata: event.metadata as Attributes,
|
||||
@@ -386,6 +390,7 @@ export class EventRepository {
|
||||
isCancelled: true,
|
||||
level: true,
|
||||
events: true,
|
||||
environmentType: true,
|
||||
},
|
||||
where: {
|
||||
traceId,
|
||||
@@ -421,6 +426,7 @@ export class EventRepository {
|
||||
startTime: getDateFromNanoseconds(event.startTime),
|
||||
level: event.level,
|
||||
events: event.events,
|
||||
environmentType: event.environmentType,
|
||||
},
|
||||
};
|
||||
});
|
||||
@@ -505,7 +511,11 @@ export class EventRepository {
|
||||
});
|
||||
}
|
||||
|
||||
const events = transformEvents(span.data.events, fullEvent.metadata as Attributes);
|
||||
const events = transformEvents(
|
||||
span.data.events,
|
||||
fullEvent.metadata as Attributes,
|
||||
traceSummary?.rootSpan.data.environmentType === "DEVELOPMENT"
|
||||
);
|
||||
|
||||
return {
|
||||
...fullEvent,
|
||||
@@ -877,6 +887,36 @@ export function stripAttributePrefix(attributes: Attributes, prefix: string) {
|
||||
return result;
|
||||
}
|
||||
|
||||
export function createExceptionPropertiesFromError(error: TaskRunError): ExceptionEventProperties {
|
||||
switch (error.type) {
|
||||
case "BUILT_IN_ERROR": {
|
||||
return {
|
||||
type: error.name,
|
||||
message: error.message,
|
||||
stacktrace: error.stackTrace,
|
||||
};
|
||||
}
|
||||
case "CUSTOM_ERROR": {
|
||||
return {
|
||||
type: "Error",
|
||||
message: error.raw,
|
||||
};
|
||||
}
|
||||
case "INTERNAL_ERROR": {
|
||||
return {
|
||||
type: "Internal error",
|
||||
message: [error.code, error.message].filter(Boolean).join(": "),
|
||||
};
|
||||
}
|
||||
case "STRING_ERROR": {
|
||||
return {
|
||||
type: "Error",
|
||||
message: error.raw,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Filters out partial events from a batch of creatable events, excluding those that have a corresponding full event.
|
||||
* @param batch - The batch of creatable events to filter.
|
||||
@@ -1097,16 +1137,16 @@ function removePrivateProperties(
|
||||
return result;
|
||||
}
|
||||
|
||||
function transformEvents(events: SpanEvents, properties: Attributes): SpanEvents {
|
||||
return (events ?? []).map((event) => transformEvent(event, properties));
|
||||
function transformEvents(events: SpanEvents, properties: Attributes, isDev: boolean): SpanEvents {
|
||||
return (events ?? []).map((event) => transformEvent(event, properties, isDev));
|
||||
}
|
||||
|
||||
function transformEvent(event: SpanEvent, properties: Attributes): SpanEvent {
|
||||
function transformEvent(event: SpanEvent, properties: Attributes, isDev: boolean): SpanEvent {
|
||||
if (isExceptionSpanEvent(event)) {
|
||||
return {
|
||||
...event,
|
||||
properties: {
|
||||
exception: transformException(event.properties.exception, properties),
|
||||
exception: transformException(event.properties.exception, properties, isDev),
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1116,11 +1156,12 @@ function transformEvent(event: SpanEvent, properties: Attributes): SpanEvent {
|
||||
|
||||
function transformException(
|
||||
exception: ExceptionEventProperties,
|
||||
properties: Attributes
|
||||
properties: Attributes,
|
||||
isDev: boolean
|
||||
): ExceptionEventProperties {
|
||||
const projectDirAttributeValue = properties[SemanticInternalAttributes.PROJECT_DIR];
|
||||
|
||||
if (typeof projectDirAttributeValue !== "string") {
|
||||
if (projectDirAttributeValue !== undefined && typeof projectDirAttributeValue !== "string") {
|
||||
return exception;
|
||||
}
|
||||
|
||||
@@ -1129,6 +1170,7 @@ function transformException(
|
||||
stacktrace: exception.stacktrace
|
||||
? correctErrorStackTrace(exception.stacktrace, projectDirAttributeValue, {
|
||||
removeFirstLine: true,
|
||||
isDev,
|
||||
})
|
||||
: undefined,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import { TaskRunFailedExecutionResult } from "@trigger.dev/core/v3";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { marqs } from "~/v3/marqs/index.server";
|
||||
|
||||
import { TaskRunStatus } from "@trigger.dev/database";
|
||||
import { createExceptionPropertiesFromError, eventRepository } from "./eventRepository.server";
|
||||
import { BaseService } from "./services/baseService.server";
|
||||
|
||||
const FAILABLE_TASK_RUN_STATUSES: TaskRunStatus[] = ["EXECUTING", "PENDING", "WAITING_FOR_DEPLOY"];
|
||||
|
||||
export class FailedTaskRunService extends BaseService {
|
||||
public async call(runFriendlyId: string, completion: TaskRunFailedExecutionResult) {
|
||||
const taskRun = await this._prisma.taskRun.findUnique({
|
||||
where: { friendlyId: runFriendlyId },
|
||||
});
|
||||
|
||||
if (!taskRun) {
|
||||
logger.error("[FailedTaskRunService] Task run not found", {
|
||||
runFriendlyId,
|
||||
completion,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (!FAILABLE_TASK_RUN_STATUSES.includes(taskRun.status)) {
|
||||
logger.error("[FailedTaskRunService] Task run is not in a failable state", {
|
||||
taskRun,
|
||||
completion,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// No more retries, we need to fail the task run
|
||||
logger.debug("[FailedTaskRunService] Failing task run", { taskRun, completion });
|
||||
|
||||
await marqs?.acknowledgeMessage(taskRun.id);
|
||||
|
||||
// Now we need to "complete" the task run event/span
|
||||
await eventRepository.completeEvent(taskRun.spanId, {
|
||||
endTime: new Date(),
|
||||
attributes: {
|
||||
isError: true,
|
||||
},
|
||||
events: [
|
||||
{
|
||||
name: "exception",
|
||||
time: new Date(),
|
||||
properties: {
|
||||
exception: createExceptionPropertiesFromError(completion.error),
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
await this._prisma.taskRun.update({
|
||||
where: {
|
||||
id: taskRun.id,
|
||||
},
|
||||
data: {
|
||||
status: "SYSTEM_FAILURE",
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import { DeploymentIndexFailed } from "./services/deploymentIndexFailed.server";
|
||||
import { Redis } from "ioredis";
|
||||
import { createAdapter } from "@socket.io/redis-adapter";
|
||||
import { CrashTaskRunService } from "./services/crashTaskRun.server";
|
||||
import { CreateTaskRunAttemptService } from "./services/createTaskRunAttempt.server";
|
||||
|
||||
export const socketIo = singleton("socketIo", initalizeIoServer);
|
||||
|
||||
@@ -91,6 +92,23 @@ function createCoordinatorNamespace(io: Server) {
|
||||
return { success: true, payload };
|
||||
}
|
||||
},
|
||||
READY_FOR_LAZY_ATTEMPT: async (message) => {
|
||||
try {
|
||||
const payload = await sharedQueueTasks.getLazyAttemptPayload(
|
||||
message.envId,
|
||||
message.runId
|
||||
);
|
||||
|
||||
if (!payload) {
|
||||
logger.error("Failed to retrieve lazy attempt payload", message);
|
||||
return { success: false, reason: "Failed to retrieve payload" };
|
||||
}
|
||||
|
||||
return { success: true, lazyPayload: payload };
|
||||
} catch (error) {
|
||||
return { success: false };
|
||||
}
|
||||
},
|
||||
READY_FOR_RESUME: async (message) => {
|
||||
const resumeAttempt = new ResumeAttemptService();
|
||||
await resumeAttempt.call(message);
|
||||
@@ -103,9 +121,15 @@ function createCoordinatorNamespace(io: Server) {
|
||||
checkpoint: message.checkpoint,
|
||||
});
|
||||
},
|
||||
TASK_RUN_FAILED_TO_RUN: async (message) => {
|
||||
await sharedQueueTasks.taskRunFailed(message.completion);
|
||||
},
|
||||
TASK_HEARTBEAT: async (message) => {
|
||||
await sharedQueueTasks.taskHeartbeat(message.attemptFriendlyId);
|
||||
},
|
||||
TASK_RUN_HEARTBEAT: async (message) => {
|
||||
await sharedQueueTasks.taskRunHeartbeat(message.runId);
|
||||
},
|
||||
CHECKPOINT_CREATED: async (message) => {
|
||||
const createCheckpoint = new CreateCheckpointService();
|
||||
await createCheckpoint.call(message);
|
||||
@@ -123,6 +147,7 @@ function createCoordinatorNamespace(io: Server) {
|
||||
const worker = await service.call(message.projectRef, environment, message.deploymentId, {
|
||||
localOnly: false,
|
||||
metadata: message.metadata,
|
||||
supportsLazyAttempts: message.version !== "v1" && message.supportsLazyAttempts,
|
||||
});
|
||||
|
||||
return { success: !!worker };
|
||||
@@ -131,13 +156,52 @@ function createCoordinatorNamespace(io: Server) {
|
||||
return { success: false };
|
||||
}
|
||||
},
|
||||
CREATE_TASK_RUN_ATTEMPT: async (message) => {
|
||||
try {
|
||||
const environment = await findEnvironmentById(message.envId);
|
||||
|
||||
if (!environment) {
|
||||
logger.error("Environment not found", { id: message.envId });
|
||||
return { success: false, reason: "Environment not found" };
|
||||
}
|
||||
|
||||
const service = new CreateTaskRunAttemptService();
|
||||
const { attempt } = await service.call(message.runId, environment, false);
|
||||
|
||||
const payload = await sharedQueueTasks.getExecutionPayloadFromAttempt(attempt.id, true);
|
||||
|
||||
if (!payload) {
|
||||
logger.error("Failed to retrieve payload after attempt creation", {
|
||||
id: message.envId,
|
||||
});
|
||||
return { success: false, reason: "Failed to retrieve payload" };
|
||||
}
|
||||
|
||||
return { success: true, executionPayload: payload };
|
||||
} catch (error) {
|
||||
logger.error("Error while creating attempt", { error });
|
||||
return { success: false };
|
||||
}
|
||||
},
|
||||
INDEXING_FAILED: async (message) => {
|
||||
try {
|
||||
const service = new DeploymentIndexFailed();
|
||||
|
||||
await service.call(message.deploymentId, message.error);
|
||||
} catch (e) {
|
||||
logger.error("Error while indexing", { error: e });
|
||||
logger.error("Error while processing index failure", { error: e });
|
||||
}
|
||||
},
|
||||
RUN_CRASHED: async (message) => {
|
||||
try {
|
||||
const service = new CrashTaskRunService();
|
||||
|
||||
await service.call(message.runId, {
|
||||
reason: `${message.error.name}: ${message.error.message}`,
|
||||
logs: message.error.stack,
|
||||
});
|
||||
} catch (e) {
|
||||
logger.error("Error while processing run failure", { error: e });
|
||||
}
|
||||
},
|
||||
},
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import { Context, ROOT_CONTEXT, Span, SpanKind, context, trace } from "@opentelemetry/api";
|
||||
import {
|
||||
TaskRunExecution,
|
||||
TaskRunExecutionLazyAttemptPayload,
|
||||
TaskRunExecutionPayload,
|
||||
TaskRunExecutionResult,
|
||||
TaskRunFailedExecutionResult,
|
||||
serverWebsocketMessages,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import { ZodMessageSender } from "@trigger.dev/core/v3/zodMessageHandler";
|
||||
@@ -14,16 +16,16 @@ import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { marqs, sanitizeQueueName } from "~/v3/marqs/index.server";
|
||||
import { EnvironmentVariablesRepository } from "../environmentVariables/environmentVariablesRepository.server";
|
||||
import { generateFriendlyId } from "../friendlyIdentifiers";
|
||||
import { CancelAttemptService } from "../services/cancelAttempt.server";
|
||||
import { CancelTaskRunService } from "../services/cancelTaskRun.server";
|
||||
import { CompleteAttemptService } from "../services/completeAttempt.server";
|
||||
import { CreateTaskRunAttemptService } from "../services/createTaskRunAttempt.server";
|
||||
import {
|
||||
SEMINTATTRS_FORCE_RECORDING,
|
||||
attributesFromAuthenticatedEnv,
|
||||
tracer,
|
||||
} from "../tracer.server";
|
||||
import { DevSubscriber, devPubSub } from "./devPubSub.server";
|
||||
import { FailedTaskRunService } from "../failedTaskRun.server";
|
||||
|
||||
const MessageBody = z.discriminatedUnion("type", [
|
||||
z.object({
|
||||
@@ -54,7 +56,6 @@ export class DevQueueConsumer {
|
||||
private _taskSuccesses: number = 0;
|
||||
private _currentSpan: Span | undefined;
|
||||
private _endSpanInNextIteration = false;
|
||||
private _inProgressAttempts: Map<string, string> = new Map(); // Keys are task attempt friendly IDs, values are TaskRun ids/queue message ids
|
||||
private _inProgressRuns: Map<string, string> = new Map(); // Keys are task run friendly IDs, values are TaskRun internal ids/queue message ids
|
||||
|
||||
constructor(
|
||||
@@ -78,7 +79,7 @@ export class DevQueueConsumer {
|
||||
this._backgroundWorkers.delete(id);
|
||||
}
|
||||
|
||||
public async registerBackgroundWorker(id: string) {
|
||||
public async registerBackgroundWorker(id: string, inProgressRuns: string[] = []) {
|
||||
const backgroundWorker = await prisma.backgroundWorker.findUnique({
|
||||
where: { friendlyId: id, runtimeEnvironmentId: this.env.id },
|
||||
include: {
|
||||
@@ -96,7 +97,10 @@ export class DevQueueConsumer {
|
||||
|
||||
this._backgroundWorkers.set(backgroundWorker.id, backgroundWorker);
|
||||
|
||||
logger.debug("Registered background worker", { backgroundWorker: backgroundWorker.id });
|
||||
logger.debug("Registered background worker", {
|
||||
backgroundWorker: backgroundWorker.id,
|
||||
inProgressRuns,
|
||||
});
|
||||
|
||||
const subscriber = await devPubSub.subscribe(`backgroundWorker:${backgroundWorker.id}:*`);
|
||||
|
||||
@@ -113,6 +117,10 @@ export class DevQueueConsumer {
|
||||
|
||||
this._backgroundWorkerSubscriber.set(backgroundWorker.id, subscriber);
|
||||
|
||||
for (const runId of inProgressRuns) {
|
||||
this._inProgressRuns.set(runId, runId);
|
||||
}
|
||||
|
||||
// Start reading from the queue if we haven't already
|
||||
await this.#enable();
|
||||
}
|
||||
@@ -122,15 +130,16 @@ export class DevQueueConsumer {
|
||||
completion: TaskRunExecutionResult,
|
||||
execution: TaskRunExecution
|
||||
) {
|
||||
this._inProgressAttempts.delete(execution.attempt.id);
|
||||
|
||||
if (completion.ok) {
|
||||
this._taskSuccesses++;
|
||||
} else {
|
||||
this._taskFailures++;
|
||||
}
|
||||
|
||||
logger.debug("Task run completed", { taskRunCompletion: completion, execution });
|
||||
logger.debug("[DevQueueConsumer] taskAttemptCompleted()", {
|
||||
taskRunCompletion: completion,
|
||||
execution,
|
||||
});
|
||||
|
||||
const service = new CompleteAttemptService();
|
||||
const result = await service.call({ completion, execution, env: this.env });
|
||||
@@ -140,7 +149,24 @@ export class DevQueueConsumer {
|
||||
}
|
||||
}
|
||||
|
||||
public async taskRunFailed(workerId: string, completion: TaskRunFailedExecutionResult) {
|
||||
this._taskFailures++;
|
||||
|
||||
logger.debug("[DevQueueConsumer] taskRunFailed()", { completion });
|
||||
|
||||
this._inProgressRuns.delete(completion.id);
|
||||
|
||||
const service = new FailedTaskRunService();
|
||||
|
||||
await service.call(completion.id, completion);
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated Use `taskRunHeartbeat` instead
|
||||
*/
|
||||
public async taskHeartbeat(workerId: string, id: string, seconds: number = 60) {
|
||||
logger.debug("[DevQueueConsumer] taskHeartbeat()", { id, seconds });
|
||||
|
||||
const taskRunAttempt = await prisma.taskRunAttempt.findUnique({
|
||||
where: { friendlyId: id },
|
||||
});
|
||||
@@ -152,6 +178,12 @@ export class DevQueueConsumer {
|
||||
await marqs?.heartbeatMessage(taskRunAttempt.taskRunId, seconds);
|
||||
}
|
||||
|
||||
public async taskRunHeartbeat(workerId: string, id: string, seconds: number = 60) {
|
||||
logger.debug("[DevQueueConsumer] taskRunHeartbeat()", { id, seconds });
|
||||
|
||||
await marqs?.heartbeatMessage(id, seconds);
|
||||
}
|
||||
|
||||
public async stop(reason: string = "CLI disconnected") {
|
||||
if (!this._enabled) {
|
||||
return;
|
||||
@@ -184,66 +216,23 @@ export class DevQueueConsumer {
|
||||
}
|
||||
|
||||
async #cancelInProgressRunsAndAttempts(reason: string) {
|
||||
const cancelAttemptService = new CancelAttemptService();
|
||||
const cancelTaskRunService = new CancelTaskRunService();
|
||||
|
||||
const cancelledAt = new Date();
|
||||
|
||||
const inProgressAttempts = new Map(this._inProgressAttempts);
|
||||
const inProgressRuns = new Map(this._inProgressRuns);
|
||||
|
||||
this._inProgressAttempts.clear();
|
||||
this._inProgressRuns.clear();
|
||||
|
||||
const inProgressRunsWithNoInProgressAttempts: string[] = [];
|
||||
const inProgressAttemptRunIds = new Set(inProgressAttempts.values());
|
||||
|
||||
for (const [runId, messageId] of inProgressRuns) {
|
||||
if (!inProgressAttemptRunIds.has(messageId)) {
|
||||
inProgressRunsWithNoInProgressAttempts.push(messageId);
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug("Cancelling in progress runs and attempts", {
|
||||
attempts: Array.from(inProgressAttempts.keys()),
|
||||
runs: Array.from(inProgressRuns.keys()),
|
||||
});
|
||||
|
||||
for (const [attemptId, messageId] of inProgressAttempts) {
|
||||
await this.#cancelInProgressAttempt(
|
||||
attemptId,
|
||||
messageId,
|
||||
cancelAttemptService,
|
||||
cancelledAt,
|
||||
reason
|
||||
);
|
||||
}
|
||||
|
||||
for (const runId of inProgressRunsWithNoInProgressAttempts) {
|
||||
for (const [_, runId] of inProgressRuns) {
|
||||
await this.#cancelInProgressRun(runId, cancelTaskRunService, cancelledAt, reason);
|
||||
}
|
||||
}
|
||||
|
||||
async #cancelInProgressAttempt(
|
||||
attemptId: string,
|
||||
messageId: string,
|
||||
cancelAttemptService: CancelAttemptService,
|
||||
cancelledAt: Date,
|
||||
reason: string
|
||||
) {
|
||||
logger.debug("Cancelling in progress attempt", { attemptId, messageId });
|
||||
|
||||
try {
|
||||
await cancelAttemptService.call(attemptId, messageId, cancelledAt, reason, this.env);
|
||||
} catch (e) {
|
||||
logger.error("Failed to cancel in progress attempt", {
|
||||
attemptId,
|
||||
messageId,
|
||||
error: e,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async #cancelInProgressRun(
|
||||
runId: string,
|
||||
service: CancelTaskRunService,
|
||||
@@ -252,16 +241,20 @@ export class DevQueueConsumer {
|
||||
) {
|
||||
logger.debug("Cancelling in progress run", { runId });
|
||||
|
||||
const taskRun = await prisma.taskRun.findUnique({
|
||||
where: { id: runId },
|
||||
});
|
||||
const taskRun = runId.startsWith("run_")
|
||||
? await prisma.taskRun.findUnique({
|
||||
where: { friendlyId: runId },
|
||||
})
|
||||
: await prisma.taskRun.findUnique({
|
||||
where: { id: runId },
|
||||
});
|
||||
|
||||
if (!taskRun) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await service.call(taskRun, { reason, cancelAttempts: false, cancelledAt });
|
||||
await service.call(taskRun, { reason, cancelAttempts: true, cancelledAt });
|
||||
} catch (e) {
|
||||
logger.error("Failed to cancel in progress run", {
|
||||
runId,
|
||||
@@ -474,141 +467,131 @@ export class DevQueueConsumer {
|
||||
}
|
||||
|
||||
if (!this._enabled) {
|
||||
logger.debug("Dev queue consumer is disabled", { env: this.env, queueMessage: message });
|
||||
|
||||
await marqs?.nackMessage(message.messageId);
|
||||
return;
|
||||
}
|
||||
|
||||
const taskRunAttempt = await prisma.taskRunAttempt.create({
|
||||
data: {
|
||||
number: lockedTaskRun.attempts[0] ? lockedTaskRun.attempts[0].number + 1 : 1,
|
||||
friendlyId: generateFriendlyId("attempt"),
|
||||
taskRunId: lockedTaskRun.id,
|
||||
startedAt: new Date(),
|
||||
backgroundWorkerId: backgroundTask.workerId,
|
||||
backgroundWorkerTaskId: backgroundTask.id,
|
||||
status: "EXECUTING" as const,
|
||||
queueId: queue.id,
|
||||
runtimeEnvironmentId: this.env.id,
|
||||
},
|
||||
});
|
||||
|
||||
const execution: TaskRunExecution = {
|
||||
task: {
|
||||
id: backgroundTask.slug,
|
||||
filePath: backgroundTask.filePath,
|
||||
exportName: backgroundTask.exportName,
|
||||
},
|
||||
attempt: {
|
||||
id: taskRunAttempt.friendlyId,
|
||||
number: taskRunAttempt.number,
|
||||
startedAt: taskRunAttempt.startedAt ?? taskRunAttempt.createdAt,
|
||||
backgroundWorkerId: backgroundWorker.id,
|
||||
backgroundWorkerTaskId: backgroundTask.id,
|
||||
status: "EXECUTING" as const,
|
||||
},
|
||||
run: {
|
||||
id: lockedTaskRun.friendlyId,
|
||||
payload: lockedTaskRun.payload,
|
||||
payloadType: lockedTaskRun.payloadType,
|
||||
context: lockedTaskRun.context,
|
||||
createdAt: lockedTaskRun.createdAt,
|
||||
tags: lockedTaskRun.tags.map((tag) => tag.name),
|
||||
isTest: lockedTaskRun.isTest,
|
||||
idempotencyKey: lockedTaskRun.idempotencyKey ?? undefined,
|
||||
},
|
||||
queue: {
|
||||
id: queue.friendlyId,
|
||||
name: queue.name,
|
||||
},
|
||||
environment: {
|
||||
id: this.env.id,
|
||||
slug: this.env.slug,
|
||||
type: this.env.type,
|
||||
},
|
||||
organization: {
|
||||
id: this.env.organization.id,
|
||||
slug: this.env.organization.slug,
|
||||
name: this.env.organization.title,
|
||||
},
|
||||
project: {
|
||||
id: this.env.project.id,
|
||||
ref: this.env.project.externalRef,
|
||||
slug: this.env.project.slug,
|
||||
name: this.env.project.name,
|
||||
},
|
||||
batch:
|
||||
lockedTaskRun.batchItems[0] && lockedTaskRun.batchItems[0].batchTaskRun
|
||||
? { id: lockedTaskRun.batchItems[0].batchTaskRun.friendlyId }
|
||||
: undefined,
|
||||
};
|
||||
|
||||
const environmentRepository = new EnvironmentVariablesRepository();
|
||||
const variables = await environmentRepository.getEnvironmentVariables(
|
||||
this.env.project.id,
|
||||
this.env.id
|
||||
);
|
||||
|
||||
const payload: TaskRunExecutionPayload = {
|
||||
execution,
|
||||
traceContext: lockedTaskRun.traceContext as Record<string, unknown>,
|
||||
environment: variables.reduce((acc: Record<string, string>, curr) => {
|
||||
acc[curr.key] = curr.value;
|
||||
return acc;
|
||||
}, {}),
|
||||
};
|
||||
if (backgroundWorker.supportsLazyAttempts) {
|
||||
const payload: TaskRunExecutionLazyAttemptPayload = {
|
||||
traceContext: lockedTaskRun.traceContext as Record<string, unknown>,
|
||||
environment: variables.reduce((acc: Record<string, string>, curr) => {
|
||||
acc[curr.key] = curr.value;
|
||||
return acc;
|
||||
}, {}),
|
||||
runId: lockedTaskRun.friendlyId,
|
||||
messageId: lockedTaskRun.id,
|
||||
isTest: lockedTaskRun.isTest,
|
||||
};
|
||||
|
||||
try {
|
||||
// TODO: send trace context down to the CLI
|
||||
await this._sender.send("BACKGROUND_WORKER_MESSAGE", {
|
||||
backgroundWorkerId: backgroundWorker.friendlyId,
|
||||
data: {
|
||||
type: "EXECUTE_RUNS",
|
||||
payloads: [payload],
|
||||
},
|
||||
});
|
||||
|
||||
logger.debug("Saving the in progress attempt", {
|
||||
taskRunAttempt: taskRunAttempt.id,
|
||||
messageId: message.messageId,
|
||||
});
|
||||
|
||||
this._inProgressAttempts.set(taskRunAttempt.friendlyId, message.messageId);
|
||||
this._inProgressRuns.set(lockedTaskRun.friendlyId, message.messageId);
|
||||
} catch (e) {
|
||||
if (e instanceof Error) {
|
||||
this._currentSpan?.recordException(e);
|
||||
} else {
|
||||
this._currentSpan?.recordException(new Error(String(e)));
|
||||
}
|
||||
|
||||
this._endSpanInNextIteration = true;
|
||||
|
||||
// We now need to unlock the task run and delete the task run attempt
|
||||
await prisma.$transaction([
|
||||
prisma.taskRun.update({
|
||||
where: {
|
||||
id: lockedTaskRun.id,
|
||||
},
|
||||
try {
|
||||
await this._sender.send("BACKGROUND_WORKER_MESSAGE", {
|
||||
backgroundWorkerId: backgroundWorker.friendlyId,
|
||||
data: {
|
||||
lockedAt: null,
|
||||
lockedById: null,
|
||||
status: "PENDING",
|
||||
type: "EXECUTE_RUN_LAZY_ATTEMPT",
|
||||
payload,
|
||||
},
|
||||
}),
|
||||
prisma.taskRunAttempt.delete({
|
||||
where: {
|
||||
id: taskRunAttempt.id,
|
||||
});
|
||||
|
||||
logger.debug("Executing the run", {
|
||||
messageId: message.messageId,
|
||||
});
|
||||
|
||||
this._inProgressRuns.set(lockedTaskRun.friendlyId, message.messageId);
|
||||
} catch (e) {
|
||||
if (e instanceof Error) {
|
||||
this._currentSpan?.recordException(e);
|
||||
} else {
|
||||
this._currentSpan?.recordException(new Error(String(e)));
|
||||
}
|
||||
|
||||
this._endSpanInNextIteration = true;
|
||||
|
||||
// We now need to unlock the task run and delete the task run attempt
|
||||
await prisma.$transaction([
|
||||
prisma.taskRun.update({
|
||||
where: {
|
||||
id: lockedTaskRun.id,
|
||||
},
|
||||
data: {
|
||||
lockedAt: null,
|
||||
lockedById: null,
|
||||
status: "PENDING",
|
||||
},
|
||||
}),
|
||||
]);
|
||||
|
||||
this._inProgressRuns.delete(lockedTaskRun.friendlyId);
|
||||
|
||||
// Finally we need to nack the message so it can be retried
|
||||
await marqs?.nackMessage(message.messageId);
|
||||
} finally {
|
||||
setTimeout(() => this.#doWork(), 100);
|
||||
}
|
||||
} else {
|
||||
const service = new CreateTaskRunAttemptService();
|
||||
const { execution } = await service.call(lockedTaskRun.friendlyId, this.env);
|
||||
|
||||
const payload: TaskRunExecutionPayload = {
|
||||
traceContext: lockedTaskRun.traceContext as Record<string, unknown>,
|
||||
environment: variables.reduce((acc: Record<string, string>, curr) => {
|
||||
acc[curr.key] = curr.value;
|
||||
return acc;
|
||||
}, {}),
|
||||
execution,
|
||||
};
|
||||
|
||||
try {
|
||||
await this._sender.send("BACKGROUND_WORKER_MESSAGE", {
|
||||
backgroundWorkerId: backgroundWorker.friendlyId,
|
||||
data: {
|
||||
type: "EXECUTE_RUNS",
|
||||
payloads: [payload],
|
||||
},
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
this._inProgressAttempts.delete(taskRunAttempt.friendlyId);
|
||||
this._inProgressRuns.delete(lockedTaskRun.friendlyId);
|
||||
logger.debug("Executing the run", {
|
||||
messageId: message.messageId,
|
||||
});
|
||||
|
||||
// Finally we need to nack the message so it can be retried
|
||||
await marqs?.nackMessage(message.messageId);
|
||||
} finally {
|
||||
setTimeout(() => this.#doWork(), 100);
|
||||
this._inProgressRuns.set(lockedTaskRun.friendlyId, message.messageId);
|
||||
} catch (e) {
|
||||
if (e instanceof Error) {
|
||||
this._currentSpan?.recordException(e);
|
||||
} else {
|
||||
this._currentSpan?.recordException(new Error(String(e)));
|
||||
}
|
||||
|
||||
this._endSpanInNextIteration = true;
|
||||
|
||||
// We now need to unlock the task run and delete the task run attempt
|
||||
await prisma.$transaction([
|
||||
prisma.taskRun.update({
|
||||
where: {
|
||||
id: lockedTaskRun.id,
|
||||
},
|
||||
data: {
|
||||
lockedAt: null,
|
||||
lockedById: null,
|
||||
status: "PENDING",
|
||||
},
|
||||
}),
|
||||
]);
|
||||
|
||||
this._inProgressRuns.delete(lockedTaskRun.friendlyId);
|
||||
|
||||
// Finally we need to nack the message so it can be retried
|
||||
await marqs?.nackMessage(message.messageId);
|
||||
} finally {
|
||||
setTimeout(() => this.#doWork(), 100);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
QueueCapacities,
|
||||
QueueRange,
|
||||
} from "./types";
|
||||
import { RequeueTaskRunService } from "../requeueTaskRun.server";
|
||||
|
||||
const tracer = trace.getTracer("marqs");
|
||||
|
||||
@@ -259,6 +260,11 @@ export class MarQS {
|
||||
});
|
||||
}
|
||||
|
||||
await RequeueTaskRunService.enqueue(
|
||||
messageData.messageId,
|
||||
new Date(Date.now() + this.visibilityTimeoutInMs)
|
||||
);
|
||||
|
||||
return message;
|
||||
},
|
||||
{
|
||||
@@ -272,6 +278,35 @@ export class MarQS {
|
||||
);
|
||||
}
|
||||
|
||||
public async getSharedQueueDetails() {
|
||||
const parentQueue = constants.SHARED_QUEUE;
|
||||
|
||||
const { range, selectionId } = await this.queuePriorityStrategy.nextCandidateSelection(
|
||||
parentQueue
|
||||
);
|
||||
const queues = await this.#getChildQueuesWithScores(parentQueue, range);
|
||||
|
||||
const queuesWithScores = await this.#calculateQueueScores(queues, (queue) =>
|
||||
this.#calculateMessageQueueCapacities(queue)
|
||||
);
|
||||
|
||||
// We need to priority shuffle here to ensure all workers aren't just working on the highest priority queue
|
||||
const choice = this.queuePriorityStrategy.chooseQueue(
|
||||
queuesWithScores,
|
||||
parentQueue,
|
||||
selectionId
|
||||
);
|
||||
|
||||
return {
|
||||
selectionId,
|
||||
queues,
|
||||
queuesWithScores,
|
||||
nextRange: range,
|
||||
queueCount: queues.length,
|
||||
queueChoice: choice,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Dequeue a message from the shared queue (this should be used in production environments)
|
||||
*/
|
||||
@@ -350,6 +385,8 @@ export class MarQS {
|
||||
[SemanticAttributes.PARENT_QUEUE]: message.parentQueue,
|
||||
});
|
||||
|
||||
await RequeueTaskRunService.dequeue(messageId);
|
||||
|
||||
await this.#callAcknowledgeMessage({
|
||||
parentQueue: message.parentQueue,
|
||||
messageKey: this.keys.messageKey(messageId),
|
||||
@@ -415,6 +452,8 @@ export class MarQS {
|
||||
return;
|
||||
}
|
||||
|
||||
await RequeueTaskRunService.dequeue(messageId);
|
||||
|
||||
await this.#callAcknowledgeMessage({
|
||||
parentQueue: oldMessage.parentQueue,
|
||||
messageKey: this.keys.messageKey(messageId),
|
||||
@@ -481,6 +520,8 @@ export class MarQS {
|
||||
[SemanticAttributes.PARENT_QUEUE]: message.parentQueue,
|
||||
});
|
||||
|
||||
await RequeueTaskRunService.dequeue(messageId);
|
||||
|
||||
await this.#callNackMessage({
|
||||
messageKey: this.keys.messageKey(messageId),
|
||||
messageQueue: message.queue,
|
||||
@@ -506,16 +547,19 @@ export class MarQS {
|
||||
|
||||
// This should increment by the number of seconds, but with a max value of Date.now() + visibilityTimeoutInMs
|
||||
public async heartbeatMessage(messageId: string, seconds: number = 30) {
|
||||
// We are still calling this for backwards compatibility, but we should be using the v3.requeueTaskRun job
|
||||
await this.#callHeartbeatMessage({
|
||||
visibilityQueue: constants.MESSAGE_VISIBILITY_TIMEOUT_QUEUE,
|
||||
messageId,
|
||||
milliseconds: seconds * 1000,
|
||||
maxVisibilityTimeout: Date.now() + this.visibilityTimeoutInMs,
|
||||
});
|
||||
|
||||
await RequeueTaskRunService.enqueue(messageId, new Date(Date.now() + seconds * 1000));
|
||||
}
|
||||
|
||||
get visibilityTimeoutInMs() {
|
||||
return this.options.visibilityTimeoutInMs ?? 300000;
|
||||
return this.options.visibilityTimeoutInMs ?? 300000; // 5 minutes
|
||||
}
|
||||
|
||||
async readMessage(messageId: string) {
|
||||
@@ -873,7 +917,6 @@ export class MarQS {
|
||||
const result = await this.redis.dequeueMessage(
|
||||
messageQueue,
|
||||
parentQueue,
|
||||
visibilityQueue,
|
||||
concurrencyLimitKey,
|
||||
envConcurrencyLimitKey,
|
||||
orgConcurrencyLimitKey,
|
||||
@@ -881,7 +924,6 @@ export class MarQS {
|
||||
envCurrentConcurrencyKey,
|
||||
orgCurrentConcurrencyKey,
|
||||
messageQueue,
|
||||
String(this.options.visibilityTimeoutInMs ?? 300000), // 5 minutes
|
||||
String(Date.now()),
|
||||
String(this.options.defaultEnvConcurrency),
|
||||
String(this.options.defaultOrgConcurrency)
|
||||
@@ -1007,6 +1049,9 @@ export class MarQS {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated This is being replaced by the v3.requeueTaskRun graphile worker job
|
||||
*/
|
||||
#callHeartbeatMessage({
|
||||
visibilityQueue,
|
||||
messageId,
|
||||
@@ -1145,25 +1190,23 @@ end
|
||||
});
|
||||
|
||||
this.redis.defineCommand("dequeueMessage", {
|
||||
numberOfKeys: 9,
|
||||
numberOfKeys: 8,
|
||||
lua: `
|
||||
-- Keys: childQueue, parentQueue, visibilityQueue, concurrencyLimitKey, envConcurrencyLimitKey, orgConcurrencyLimitKey, currentConcurrencyKey, envCurrentConcurrencyKey, orgCurrentConcurrencyKey
|
||||
-- Keys: childQueue, parentQueue, concurrencyLimitKey, envConcurrencyLimitKey, orgConcurrencyLimitKey, currentConcurrencyKey, envCurrentConcurrencyKey, orgCurrentConcurrencyKey
|
||||
local childQueue = KEYS[1]
|
||||
local parentQueue = KEYS[2]
|
||||
local visibilityQueue = KEYS[3]
|
||||
local concurrencyLimitKey = KEYS[4]
|
||||
local envConcurrencyLimitKey = KEYS[5]
|
||||
local orgConcurrencyLimitKey = KEYS[6]
|
||||
local currentConcurrencyKey = KEYS[7]
|
||||
local envCurrentConcurrencyKey = KEYS[8]
|
||||
local orgCurrentConcurrencyKey = KEYS[9]
|
||||
local concurrencyLimitKey = KEYS[3]
|
||||
local envConcurrencyLimitKey = KEYS[4]
|
||||
local orgConcurrencyLimitKey = KEYS[5]
|
||||
local currentConcurrencyKey = KEYS[6]
|
||||
local envCurrentConcurrencyKey = KEYS[7]
|
||||
local orgCurrentConcurrencyKey = KEYS[8]
|
||||
|
||||
-- Args: childQueueName, visibilityQueue, currentTime, defaultEnvConcurrencyLimit, defaultOrgConcurrencyLimit
|
||||
-- Args: childQueueName, currentTime, defaultEnvConcurrencyLimit, defaultOrgConcurrencyLimit
|
||||
local childQueueName = ARGV[1]
|
||||
local visibilityTimeout = tonumber(ARGV[2])
|
||||
local currentTime = tonumber(ARGV[3])
|
||||
local defaultEnvConcurrencyLimit = ARGV[4]
|
||||
local defaultOrgConcurrencyLimit = ARGV[5]
|
||||
local currentTime = tonumber(ARGV[2])
|
||||
local defaultEnvConcurrencyLimit = ARGV[3]
|
||||
local defaultOrgConcurrencyLimit = ARGV[4]
|
||||
|
||||
-- Check current org concurrency against the limit
|
||||
local orgCurrentConcurrency = tonumber(redis.call('SCARD', orgCurrentConcurrencyKey) or '0')
|
||||
@@ -1199,11 +1242,9 @@ end
|
||||
|
||||
local messageId = messages[1]
|
||||
local messageScore = tonumber(messages[2])
|
||||
local timeoutScore = currentTime + visibilityTimeout
|
||||
|
||||
-- Move message to timeout queue and update concurrency
|
||||
redis.call('ZREM', childQueue, messageId)
|
||||
redis.call('ZADD', visibilityQueue, timeoutScore, messageId)
|
||||
redis.call('SADD', currentConcurrencyKey, messageId)
|
||||
redis.call('SADD', envCurrentConcurrencyKey, messageId)
|
||||
redis.call('SADD', orgCurrentConcurrencyKey, messageId)
|
||||
@@ -1269,7 +1310,7 @@ else
|
||||
redis.call('ZADD', parentQueue, earliestMessage[2], messageQueueName)
|
||||
end
|
||||
|
||||
-- Remove the message from the timeout queue
|
||||
-- Remove the message from the timeout queue (deprecated, will eventually remove this)
|
||||
redis.call('ZREM', visibilityQueue, messageId)
|
||||
|
||||
-- Update the concurrency keys
|
||||
@@ -1297,20 +1338,18 @@ local messageId = ARGV[2]
|
||||
local currentTime = tonumber(ARGV[3])
|
||||
local messageScore = tonumber(ARGV[4])
|
||||
|
||||
-- Check to see if the message is still in the visibilityQueue
|
||||
local messageVisibility = tonumber(redis.call('ZSCORE', visibilityQueue, messageId)) or 0
|
||||
|
||||
if messageVisibility == 0 then
|
||||
return
|
||||
end
|
||||
|
||||
-- Update the concurrency keys
|
||||
redis.call('SREM', concurrencyKey, messageId)
|
||||
redis.call('SREM', envConcurrencyKey, messageId)
|
||||
redis.call('SREM', orgConcurrencyKey, messageId)
|
||||
|
||||
-- Remove the message from the timeout queue
|
||||
redis.call('ZREM', visibilityQueue, messageId)
|
||||
-- Check to see if the message is still in the visibilityQueue
|
||||
local messageVisibility = tonumber(redis.call('ZSCORE', visibilityQueue, messageId)) or 0
|
||||
|
||||
if messageVisibility > 0 then
|
||||
-- Remove the message from the timeout queue (deprecated, will eventually remove this)
|
||||
redis.call('ZREM', visibilityQueue, messageId)
|
||||
end
|
||||
|
||||
-- Enqueue the message into the queue
|
||||
redis.call('ZADD', childQueueKey, messageScore, messageId)
|
||||
@@ -1337,12 +1376,16 @@ local milliseconds = tonumber(ARGV[2])
|
||||
local maxVisibilityTimeout = tonumber(ARGV[3])
|
||||
|
||||
-- Get the current visibility timeout
|
||||
local currentVisibilityTimeout = tonumber(redis.call('ZSCORE', visibilityQueue, messageId)) or 0
|
||||
local zscoreResult = redis.call('ZSCORE', visibilityQueue, messageId)
|
||||
|
||||
if currentVisibilityTimeout == 0 then
|
||||
-- If there's no currentVisibilityTimeout, return and do not execute ZADD
|
||||
if zscoreResult == false then
|
||||
return
|
||||
end
|
||||
|
||||
local currentVisibilityTimeout = tonumber(zscoreResult)
|
||||
|
||||
|
||||
-- Calculate the new visibility timeout
|
||||
local newVisibilityTimeout = math.min(currentVisibilityTimeout + milliseconds * 1000, maxVisibilityTimeout)
|
||||
|
||||
@@ -1445,7 +1488,6 @@ declare module "ioredis" {
|
||||
dequeueMessage(
|
||||
childQueue: string,
|
||||
parentQueue: string,
|
||||
visibilityQueue: string,
|
||||
concurrencyLimitKey: string,
|
||||
envConcurrencyLimitKey: string,
|
||||
orgConcurrencyLimitKey: string,
|
||||
@@ -1453,7 +1495,6 @@ declare module "ioredis" {
|
||||
envCurrentConcurrencyKey: string,
|
||||
orgCurrentConcurrencyKey: string,
|
||||
childQueueName: string,
|
||||
visibilityTimeout: string,
|
||||
currentTime: string,
|
||||
defaultEnvConcurrencyLimit: string,
|
||||
defaultOrgConcurrencyLimit: string,
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
ProdTaskRunExecutionPayload,
|
||||
TaskRunError,
|
||||
TaskRunExecution,
|
||||
TaskRunExecutionLazyAttemptPayload,
|
||||
TaskRunExecutionResult,
|
||||
TaskRunFailedExecutionResult,
|
||||
TaskRunSuccessfulExecutionResult,
|
||||
@@ -33,6 +34,9 @@ import {
|
||||
import { RestoreCheckpointService } from "../services/restoreCheckpoint.server";
|
||||
import { SEMINTATTRS_FORCE_RECORDING, tracer } from "../tracer.server";
|
||||
import { CrashTaskRunService } from "../services/crashTaskRun.server";
|
||||
import { FailedTaskRunService } from "../failedTaskRun.server";
|
||||
import { CreateTaskRunAttemptService } from "../services/createTaskRunAttempt.server";
|
||||
import { findEnvironmentById } from "~/models/runtimeEnvironment.server";
|
||||
|
||||
const WithTraceContext = z.object({
|
||||
traceparent: z.string().optional(),
|
||||
@@ -260,6 +264,14 @@ export class SharedQueueConsumer {
|
||||
where: {
|
||||
id: message.messageId,
|
||||
},
|
||||
include: {
|
||||
lockedToVersion: {
|
||||
include: {
|
||||
deployment: true,
|
||||
tasks: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!existingTaskRun) {
|
||||
@@ -291,7 +303,7 @@ export class SharedQueueConsumer {
|
||||
(!retryingFromCheckpoint &&
|
||||
!EXECUTABLE_RUN_STATUSES.withoutCheckpoint.includes(existingTaskRun.status))
|
||||
) {
|
||||
logger.debug("Task run has invalid status for execution", {
|
||||
logger.error("Task run has invalid status for execution", {
|
||||
queueMessage: message.data,
|
||||
messageId: message.messageId,
|
||||
taskRun: existingTaskRun.id,
|
||||
@@ -299,6 +311,12 @@ export class SharedQueueConsumer {
|
||||
retryingFromCheckpoint,
|
||||
});
|
||||
|
||||
const service = new CrashTaskRunService();
|
||||
await service.call(existingTaskRun.id, {
|
||||
crashAttempts: true,
|
||||
reason: `Invalid run status for execution: ${existingTaskRun.status}`,
|
||||
});
|
||||
|
||||
await this.#ackAndDoMoreWork(message.messageId);
|
||||
return;
|
||||
}
|
||||
@@ -398,6 +416,7 @@ export class SharedQueueConsumer {
|
||||
createdAt: "desc",
|
||||
},
|
||||
},
|
||||
lockedBy: true,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -443,39 +462,12 @@ export class SharedQueueConsumer {
|
||||
return;
|
||||
}
|
||||
|
||||
const taskRunAttempt = await prisma.taskRunAttempt.create({
|
||||
data: {
|
||||
number: lockedTaskRun.attempts[0] ? lockedTaskRun.attempts[0].number + 1 : 1,
|
||||
friendlyId: generateFriendlyId("attempt"),
|
||||
taskRunId: lockedTaskRun.id,
|
||||
startedAt: new Date(),
|
||||
backgroundWorkerId: backgroundTask.workerId,
|
||||
backgroundWorkerTaskId: backgroundTask.id,
|
||||
status: "PENDING" as const,
|
||||
queueId: queue.id,
|
||||
runtimeEnvironmentId: lockedTaskRun.runtimeEnvironmentId,
|
||||
},
|
||||
include: {
|
||||
backgroundWorkerTask: true,
|
||||
},
|
||||
});
|
||||
const nextAttemptNumber = lockedTaskRun.attempts[0]
|
||||
? lockedTaskRun.attempts[0].number + 1
|
||||
: 1;
|
||||
|
||||
const isRetry = taskRunAttempt.number > 1;
|
||||
const isRetry = nextAttemptNumber > 1;
|
||||
|
||||
const { machineConfig } = taskRunAttempt.backgroundWorkerTask;
|
||||
const machine = Machine.safeParse(machineConfig ?? {});
|
||||
|
||||
if (!machine.success) {
|
||||
logger.error("Failed to parse machine config", {
|
||||
queueMessage: message.data,
|
||||
messageId: message.messageId,
|
||||
attemptId: taskRunAttempt.id,
|
||||
machineConfig,
|
||||
});
|
||||
|
||||
await this.#ackAndDoMoreWork(message.messageId);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (messageBody.data.checkpointEventId) {
|
||||
const restoreService = new RestoreCheckpointService();
|
||||
@@ -494,12 +486,35 @@ export class SharedQueueConsumer {
|
||||
await this.#ackAndDoMoreWork(message.messageId);
|
||||
return;
|
||||
}
|
||||
} else if (isRetry) {
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
if (!deployment.worker.supportsLazyAttempts) {
|
||||
const service = new CreateTaskRunAttemptService();
|
||||
await service.call(lockedTaskRun.friendlyId, undefined, false);
|
||||
}
|
||||
|
||||
if (isRetry) {
|
||||
socketIo.coordinatorNamespace.emit("READY_FOR_RETRY", {
|
||||
version: "v1",
|
||||
runId: taskRunAttempt.taskRunId,
|
||||
runId: lockedTaskRun.id,
|
||||
});
|
||||
} else {
|
||||
const machineConfig = lockedTaskRun.lockedBy?.machineConfig;
|
||||
const machine = Machine.safeParse(machineConfig ?? {});
|
||||
|
||||
if (!machine.success) {
|
||||
logger.error("Failed to parse machine config", {
|
||||
queueMessage: message.data,
|
||||
messageId: message.messageId,
|
||||
machineConfig,
|
||||
});
|
||||
|
||||
await this.#ackAndDoMoreWork(message.messageId);
|
||||
return;
|
||||
}
|
||||
|
||||
await this._sender.send("BACKGROUND_WORKER_MESSAGE", {
|
||||
backgroundWorkerId: deployment.worker.friendlyId,
|
||||
data: {
|
||||
@@ -508,12 +523,12 @@ export class SharedQueueConsumer {
|
||||
version: deployment.version,
|
||||
machine: machine.data,
|
||||
// identifiers
|
||||
id: taskRunAttempt.id,
|
||||
id: "placeholder", // TODO: Remove this completely in a future release
|
||||
envId: lockedTaskRun.runtimeEnvironment.id,
|
||||
envType: lockedTaskRun.runtimeEnvironment.type,
|
||||
orgId: lockedTaskRun.runtimeEnvironment.organizationId,
|
||||
projectId: lockedTaskRun.runtimeEnvironment.projectId,
|
||||
runId: taskRunAttempt.taskRunId,
|
||||
runId: lockedTaskRun.id,
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -535,11 +550,7 @@ export class SharedQueueConsumer {
|
||||
data: {
|
||||
lockedAt: null,
|
||||
lockedById: null,
|
||||
},
|
||||
}),
|
||||
prisma.taskRunAttempt.delete({
|
||||
where: {
|
||||
id: taskRunAttempt.id,
|
||||
status: lockedTaskRun.status,
|
||||
},
|
||||
}),
|
||||
]);
|
||||
@@ -1096,7 +1107,50 @@ class SharedQueueTasks {
|
||||
return this.getExecutionPayloadFromAttempt(latestAttempt.id, setToExecuting, isRetrying);
|
||||
}
|
||||
|
||||
async getLazyAttemptPayload(
|
||||
envId: string,
|
||||
runId: string
|
||||
): Promise<TaskRunExecutionLazyAttemptPayload | undefined> {
|
||||
const environment = await findEnvironmentById(envId);
|
||||
|
||||
if (!environment) {
|
||||
logger.error("Environment not found", { id: envId });
|
||||
return;
|
||||
}
|
||||
|
||||
const run = await prisma.taskRun.findUnique({
|
||||
where: {
|
||||
id: runId,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
},
|
||||
});
|
||||
|
||||
if (!run) {
|
||||
logger.error("Run not found", { id: runId, envId });
|
||||
return;
|
||||
}
|
||||
|
||||
const environmentRepository = new EnvironmentVariablesRepository();
|
||||
const variables = await environmentRepository.getEnvironmentVariables(
|
||||
environment.projectId,
|
||||
environment.id
|
||||
);
|
||||
|
||||
return {
|
||||
traceContext: run.traceContext as Record<string, unknown>,
|
||||
environment: variables.reduce((acc: Record<string, string>, curr) => {
|
||||
acc[curr.key] = curr.value;
|
||||
return acc;
|
||||
}, {}),
|
||||
runId: run.friendlyId,
|
||||
messageId: run.id,
|
||||
isTest: run.isTest,
|
||||
} satisfies TaskRunExecutionLazyAttemptPayload;
|
||||
}
|
||||
|
||||
async taskHeartbeat(attemptFriendlyId: string, seconds: number = 60) {
|
||||
logger.debug("[SharedQueueConsumer] taskHeartbeat()", { id: attemptFriendlyId, seconds });
|
||||
|
||||
const taskRunAttempt = await prisma.taskRunAttempt.findUnique({
|
||||
where: { friendlyId: attemptFriendlyId },
|
||||
});
|
||||
@@ -1107,6 +1161,20 @@ class SharedQueueTasks {
|
||||
|
||||
await marqs?.heartbeatMessage(taskRunAttempt.taskRunId, seconds);
|
||||
}
|
||||
|
||||
async taskRunHeartbeat(runId: string, seconds: number = 60) {
|
||||
logger.debug("[SharedQueueConsumer] taskRunHeartbeat()", { runId, seconds });
|
||||
|
||||
await marqs?.heartbeatMessage(runId, seconds);
|
||||
}
|
||||
|
||||
public async taskRunFailed(completion: TaskRunFailedExecutionResult) {
|
||||
logger.debug("[SharedQueueConsumer] taskRunFailed()", { completion });
|
||||
|
||||
const service = new FailedTaskRunService();
|
||||
|
||||
await service.call(completion.id, completion);
|
||||
}
|
||||
}
|
||||
|
||||
export const sharedQueueTasks = singleton("sharedQueueTasks", () => new SharedQueueTasks());
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { marqs } from "~/v3/marqs/index.server";
|
||||
|
||||
import assertNever from "assert-never";
|
||||
import { FailedTaskRunService } from "./failedTaskRun.server";
|
||||
import { BaseService } from "./services/baseService.server";
|
||||
import { PrismaClientOrTransaction } from "~/db.server";
|
||||
import { workerQueue } from "~/services/worker.server";
|
||||
|
||||
export class RequeueTaskRunService extends BaseService {
|
||||
public async call(runId: string) {
|
||||
const taskRun = await this._prisma.taskRun.findUnique({
|
||||
where: { id: runId },
|
||||
});
|
||||
|
||||
if (!taskRun) {
|
||||
logger.error("[RequeueTaskRunService] Task run not found", {
|
||||
runId,
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
switch (taskRun.status) {
|
||||
case "PENDING": {
|
||||
logger.debug("[RequeueTaskRunService] Requeueing task run", { taskRun });
|
||||
|
||||
await marqs?.nackMessage(taskRun.id);
|
||||
|
||||
break;
|
||||
}
|
||||
case "EXECUTING":
|
||||
case "RETRYING_AFTER_FAILURE": {
|
||||
logger.debug("[RequeueTaskRunService] Failing task run", { taskRun });
|
||||
|
||||
const service = new FailedTaskRunService();
|
||||
|
||||
await service.call(taskRun.friendlyId, {
|
||||
ok: false,
|
||||
id: taskRun.friendlyId,
|
||||
retry: undefined,
|
||||
error: {
|
||||
type: "INTERNAL_ERROR",
|
||||
code: "TASK_RUN_HEARTBEAT_TIMEOUT",
|
||||
message: "Did not receive a heartbeat from the worker in time",
|
||||
},
|
||||
});
|
||||
|
||||
break;
|
||||
}
|
||||
case "WAITING_FOR_DEPLOY": {
|
||||
logger.debug("[RequeueTaskRunService] Removing task run from queue", { taskRun });
|
||||
|
||||
await marqs?.acknowledgeMessage(taskRun.id);
|
||||
|
||||
break;
|
||||
}
|
||||
case "WAITING_TO_RESUME":
|
||||
case "PAUSED": {
|
||||
logger.debug("[RequeueTaskRunService] Requeueing task run", { taskRun });
|
||||
|
||||
await marqs?.nackMessage(taskRun.id);
|
||||
|
||||
break;
|
||||
}
|
||||
case "SYSTEM_FAILURE":
|
||||
case "INTERRUPTED":
|
||||
case "CRASHED":
|
||||
case "COMPLETED_WITH_ERRORS":
|
||||
case "COMPLETED_SUCCESSFULLY":
|
||||
case "CANCELED": {
|
||||
logger.debug("[RequeueTaskRunService] Task run is completed", { taskRun });
|
||||
|
||||
await marqs?.acknowledgeMessage(taskRun.id);
|
||||
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
assertNever(taskRun.status);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static async enqueue(runId: string, runAt?: Date, tx?: PrismaClientOrTransaction) {
|
||||
return await workerQueue.enqueue(
|
||||
"v3.requeueTaskRun",
|
||||
{ runId },
|
||||
{ runAt, jobKey: `requeueTaskRun:${runId}` }
|
||||
);
|
||||
}
|
||||
|
||||
public static async dequeue(runId: string, tx?: PrismaClientOrTransaction) {
|
||||
return await workerQueue.dequeue(`requeueTaskRun:${runId}`, { tx });
|
||||
}
|
||||
}
|
||||
@@ -34,7 +34,7 @@ export abstract class BaseService {
|
||||
}
|
||||
|
||||
export class ServiceValidationError extends Error {
|
||||
constructor(message: string) {
|
||||
constructor(message: string, public status?: number) {
|
||||
super(message);
|
||||
this.name = "ServiceValidationError";
|
||||
}
|
||||
|
||||
@@ -24,9 +24,15 @@ const CANCELLABLE_ATTEMPT_STATUSES: Array<TaskRunAttemptStatus> = [
|
||||
"PENDING",
|
||||
];
|
||||
|
||||
type ExtendedTaskRunAttempt = Prisma.TaskRunAttemptGetPayload<{
|
||||
type ExtendedTaskRun = Prisma.TaskRunGetPayload<{
|
||||
include: {
|
||||
runtimeEnvironment: true;
|
||||
lockedToVersion: true;
|
||||
};
|
||||
}>;
|
||||
|
||||
type ExtendedTaskRunAttempt = Prisma.TaskRunAttemptGetPayload<{
|
||||
include: {
|
||||
backgroundWorker: true;
|
||||
};
|
||||
}>;
|
||||
@@ -71,11 +77,10 @@ export class CancelTaskRunService extends BaseService {
|
||||
},
|
||||
include: {
|
||||
backgroundWorker: true,
|
||||
runtimeEnvironment: true,
|
||||
},
|
||||
},
|
||||
dependency: true,
|
||||
runtimeEnvironment: true,
|
||||
lockedToVersion: true,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -96,6 +101,7 @@ export class CancelTaskRunService extends BaseService {
|
||||
// Cancel any in progress attempts
|
||||
if (opts.cancelAttempts) {
|
||||
await this.#cancelPotentiallyRunningAttempts(cancelledTaskRun, cancelledTaskRun.attempts);
|
||||
await this.#cancelRemainingRunWorkers(cancelledTaskRun);
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -103,9 +109,12 @@ export class CancelTaskRunService extends BaseService {
|
||||
};
|
||||
}
|
||||
|
||||
async #cancelPotentiallyRunningAttempts(run: TaskRun, attempts: ExtendedTaskRunAttempt[]) {
|
||||
async #cancelPotentiallyRunningAttempts(
|
||||
run: ExtendedTaskRun,
|
||||
attempts: ExtendedTaskRunAttempt[]
|
||||
) {
|
||||
for (const attempt of attempts) {
|
||||
if (attempt.runtimeEnvironment.type === "DEVELOPMENT") {
|
||||
if (run.runtimeEnvironment.type === "DEVELOPMENT") {
|
||||
// Signal the task run attempt to stop
|
||||
await devPubSub.publish(
|
||||
`backgroundWorker:${attempt.backgroundWorkerId}:${attempt.id}`,
|
||||
@@ -158,4 +167,19 @@ export class CancelTaskRunService extends BaseService {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async #cancelRemainingRunWorkers(run: ExtendedTaskRun) {
|
||||
if (run.runtimeEnvironment.type === "DEVELOPMENT") {
|
||||
// Nothing to do
|
||||
return;
|
||||
}
|
||||
|
||||
// Broadcast cancel message to all coordinators
|
||||
socketIo.coordinatorNamespace.emit("REQUEST_RUN_CANCELLATION", {
|
||||
version: "v1",
|
||||
runId: run.id,
|
||||
// Give the attempts some time to exit gracefully. If the runs supports lazy attempts, it also supports exit delays.
|
||||
delayInMs: run.lockedToVersion?.supportsLazyAttempts ? 5_000 : undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ import { PrismaClientOrTransaction } from "~/db.server";
|
||||
import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { safeJsonParse } from "~/utils/json";
|
||||
import { eventRepository } from "../eventRepository.server";
|
||||
import { createExceptionPropertiesFromError, eventRepository } from "../eventRepository.server";
|
||||
import { marqs } from "~/v3/marqs/index.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { CancelAttemptService } from "./cancelAttempt.server";
|
||||
@@ -20,6 +20,7 @@ import { MAX_TASK_RUN_ATTEMPTS } from "~/consts";
|
||||
import { CreateCheckpointService } from "./createCheckpoint.server";
|
||||
import { TaskRun } from "@trigger.dev/database";
|
||||
import { PerformTaskAttemptAlertsService } from "./alerts/performTaskAttemptAlerts.server";
|
||||
import { RetryAttemptService } from "./retryAttempt.server";
|
||||
|
||||
type FoundAttempt = Awaited<ReturnType<typeof findAttempt>>;
|
||||
|
||||
@@ -57,6 +58,8 @@ export class CompleteAttemptService extends BaseService {
|
||||
},
|
||||
});
|
||||
|
||||
// No attempt, so there's no message to ACK
|
||||
|
||||
return "COMPLETED";
|
||||
}
|
||||
|
||||
@@ -143,6 +146,8 @@ export class CompleteAttemptService extends BaseService {
|
||||
env
|
||||
);
|
||||
|
||||
// The cancel service handles ACK
|
||||
|
||||
return "COMPLETED";
|
||||
}
|
||||
|
||||
@@ -173,7 +178,7 @@ export class CompleteAttemptService extends BaseService {
|
||||
properties: {
|
||||
retryAt: retryAt.toISOString(),
|
||||
},
|
||||
runId: taskRunAttempt.taskRunId,
|
||||
runId: taskRunAttempt.taskRun.friendlyId,
|
||||
style: {
|
||||
icon: "schedule-attempt",
|
||||
},
|
||||
@@ -185,7 +190,10 @@ export class CompleteAttemptService extends BaseService {
|
||||
endTime: retryAt,
|
||||
});
|
||||
|
||||
logger.debug("Retrying", { taskRun: taskRunAttempt.taskRun.friendlyId });
|
||||
logger.debug("Retrying", {
|
||||
taskRun: taskRunAttempt.taskRun.friendlyId,
|
||||
retry: completion.retry,
|
||||
});
|
||||
|
||||
await this._prisma.taskRun.update({
|
||||
where: {
|
||||
@@ -203,7 +211,12 @@ export class CompleteAttemptService extends BaseService {
|
||||
}
|
||||
|
||||
if (!checkpoint) {
|
||||
await this.#enqueueRetry(taskRunAttempt.taskRun, completion.retry.timestamp);
|
||||
await this.#retryAttempt(
|
||||
taskRunAttempt.taskRun,
|
||||
completion.retry.timestamp,
|
||||
undefined,
|
||||
taskRunAttempt.backgroundWorker.supportsLazyAttempts
|
||||
);
|
||||
return "RETRIED";
|
||||
}
|
||||
|
||||
@@ -231,10 +244,12 @@ export class CompleteAttemptService extends BaseService {
|
||||
},
|
||||
});
|
||||
|
||||
await marqs?.acknowledgeMessage(taskRunAttempt.taskRunId);
|
||||
|
||||
return "COMPLETED";
|
||||
}
|
||||
|
||||
await this.#enqueueRetry(
|
||||
await this.#retryAttempt(
|
||||
taskRunAttempt.taskRun,
|
||||
completion.retry.timestamp,
|
||||
checkpointCreateResult.event.id
|
||||
@@ -253,6 +268,15 @@ export class CompleteAttemptService extends BaseService {
|
||||
attributes: {
|
||||
isError: true,
|
||||
},
|
||||
events: [
|
||||
{
|
||||
name: "exception",
|
||||
time: new Date(),
|
||||
properties: {
|
||||
exception: createExceptionPropertiesFromError(completion.error),
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
if (
|
||||
@@ -310,17 +334,28 @@ export class CompleteAttemptService extends BaseService {
|
||||
}
|
||||
}
|
||||
|
||||
async #enqueueRetry(run: TaskRun, retryTimestamp: number, checkpointEventId?: string) {
|
||||
// We have to replace a potential RESUME with EXECUTE to correctly retry the attempt
|
||||
return await marqs?.replaceMessage(
|
||||
run.id,
|
||||
{
|
||||
type: "EXECUTE",
|
||||
taskIdentifier: run.taskIdentifier,
|
||||
checkpointEventId: checkpointEventId,
|
||||
},
|
||||
retryTimestamp
|
||||
);
|
||||
async #retryAttempt(
|
||||
run: TaskRun,
|
||||
retryTimestamp: number,
|
||||
checkpointEventId?: string,
|
||||
supportsLazyAttempts?: boolean
|
||||
) {
|
||||
if (checkpointEventId || !supportsLazyAttempts) {
|
||||
// We have to replace a potential RESUME with EXECUTE to correctly retry the attempt
|
||||
return await marqs?.replaceMessage(
|
||||
run.id,
|
||||
{
|
||||
type: "EXECUTE",
|
||||
taskIdentifier: run.taskIdentifier,
|
||||
checkpointEventId: checkpointEventId,
|
||||
},
|
||||
retryTimestamp
|
||||
);
|
||||
} else {
|
||||
// There's no checkpoint so the worker is still running and waiting for a retry message
|
||||
// It supports lazy attempts so we can bypass the queue and send the message directly to the worker
|
||||
RetryAttemptService.enqueue(run.id, this._prisma, new Date(retryTimestamp));
|
||||
}
|
||||
}
|
||||
|
||||
#generateMetadataAttributesForNextAttempt(execution: TaskRunExecution) {
|
||||
@@ -353,6 +388,7 @@ async function findAttempt(prismaClient: PrismaClientOrTransaction, friendlyId:
|
||||
include: {
|
||||
taskRun: true,
|
||||
backgroundWorkerTask: true,
|
||||
backgroundWorker: true,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -63,6 +63,7 @@ export class CreateBackgroundWorkerService extends BaseService {
|
||||
contentHash: body.metadata.contentHash,
|
||||
cliVersion: body.metadata.cliPackageVersion,
|
||||
sdkVersion: body.metadata.packageVersion,
|
||||
supportsLazyAttempts: body.supportsLazyAttempts,
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -45,6 +45,7 @@ export class CreateDeployedBackgroundWorkerService extends BaseService {
|
||||
contentHash: body.metadata.contentHash,
|
||||
cliVersion: body.metadata.cliPackageVersion,
|
||||
sdkVersion: body.metadata.packageVersion,
|
||||
supportsLazyAttempts: body.supportsLazyAttempts,
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
import { TaskRunExecution } from "@trigger.dev/core/v3";
|
||||
import { $transaction, PrismaClientOrTransaction, prisma } from "~/db.server";
|
||||
import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { generateFriendlyId } from "../friendlyIdentifiers";
|
||||
import { BaseService, ServiceValidationError } from "./baseService.server";
|
||||
import { TaskRun, TaskRunAttempt } from "@trigger.dev/database";
|
||||
|
||||
export class CreateTaskRunAttemptService extends BaseService {
|
||||
public async call(
|
||||
runId: string,
|
||||
env?: AuthenticatedEnvironment,
|
||||
setToExecuting = true
|
||||
): Promise<{
|
||||
execution: TaskRunExecution;
|
||||
run: TaskRun;
|
||||
attempt: TaskRunAttempt;
|
||||
}> {
|
||||
const environment = env ?? (await getAuthenticatedEnvironmentFromRun(runId, this._prisma));
|
||||
|
||||
if (!environment) {
|
||||
throw new ServiceValidationError("Environment not found", 404);
|
||||
}
|
||||
|
||||
const isFriendlyId = runId.startsWith("run_");
|
||||
|
||||
return await this.traceWithEnv("call()", environment, async (span) => {
|
||||
if (isFriendlyId) {
|
||||
span.setAttribute("taskRunFriendlyId", runId);
|
||||
} else {
|
||||
span.setAttribute("taskRunId", runId);
|
||||
}
|
||||
|
||||
const taskRun = await this._prisma.taskRun.findUnique({
|
||||
where: {
|
||||
id: !isFriendlyId ? runId : undefined,
|
||||
friendlyId: isFriendlyId ? runId : undefined,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
},
|
||||
include: {
|
||||
tags: true,
|
||||
attempts: {
|
||||
take: 1,
|
||||
orderBy: {
|
||||
number: "desc",
|
||||
},
|
||||
},
|
||||
lockedBy: {
|
||||
include: {
|
||||
worker: true,
|
||||
},
|
||||
},
|
||||
batchItems: {
|
||||
include: {
|
||||
batchTaskRun: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
logger.debug("Creating a task run attempt", { taskRun });
|
||||
|
||||
if (!taskRun) {
|
||||
throw new ServiceValidationError("Task run not found", 404);
|
||||
}
|
||||
|
||||
span.setAttribute("taskRunId", taskRun.id);
|
||||
span.setAttribute("taskRunFriendlyId", taskRun.friendlyId);
|
||||
|
||||
if (taskRun.status === "CANCELED") {
|
||||
throw new ServiceValidationError("Task run is cancelled", 400);
|
||||
}
|
||||
|
||||
if (!taskRun.lockedBy) {
|
||||
throw new ServiceValidationError("Task run is not locked", 400);
|
||||
}
|
||||
|
||||
const queue = await this._prisma.taskQueue.findUnique({
|
||||
where: {
|
||||
runtimeEnvironmentId_name: {
|
||||
runtimeEnvironmentId: environment.id,
|
||||
name: taskRun.queue,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!queue) {
|
||||
throw new ServiceValidationError("Queue not found", 404);
|
||||
}
|
||||
|
||||
const nextAttemptNumber = taskRun.attempts[0] ? taskRun.attempts[0].number + 1 : 1;
|
||||
|
||||
const taskRunAttempt = await $transaction(this._prisma, async (tx) => {
|
||||
const taskRunAttempt = await tx.taskRunAttempt.create({
|
||||
data: {
|
||||
number: nextAttemptNumber,
|
||||
friendlyId: generateFriendlyId("attempt"),
|
||||
taskRunId: taskRun.id,
|
||||
startedAt: new Date(),
|
||||
backgroundWorkerId: taskRun.lockedBy!.worker.id,
|
||||
backgroundWorkerTaskId: taskRun.lockedBy!.id,
|
||||
status: setToExecuting ? "EXECUTING" : "PENDING",
|
||||
queueId: queue.id,
|
||||
runtimeEnvironmentId: environment.id,
|
||||
},
|
||||
include: {
|
||||
backgroundWorker: true,
|
||||
backgroundWorkerTask: true,
|
||||
},
|
||||
});
|
||||
|
||||
if (setToExecuting) {
|
||||
await tx.taskRun.update({
|
||||
where: {
|
||||
id: taskRun.id,
|
||||
},
|
||||
data: {
|
||||
status: "EXECUTING",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return taskRunAttempt;
|
||||
});
|
||||
|
||||
if (!taskRunAttempt) {
|
||||
logger.error("Failed to create task run attempt", { runId: taskRun.id, nextAttemptNumber });
|
||||
throw new ServiceValidationError("Failed to create task run attempt", 500);
|
||||
}
|
||||
|
||||
const execution: TaskRunExecution = {
|
||||
task: {
|
||||
id: taskRun.lockedBy.slug,
|
||||
filePath: taskRun.lockedBy.filePath,
|
||||
exportName: taskRun.lockedBy.exportName,
|
||||
},
|
||||
attempt: {
|
||||
id: taskRunAttempt.friendlyId,
|
||||
number: taskRunAttempt.number,
|
||||
startedAt: taskRunAttempt.startedAt ?? taskRunAttempt.createdAt,
|
||||
backgroundWorkerId: taskRun.lockedBy.worker.id,
|
||||
backgroundWorkerTaskId: taskRun.lockedBy.id,
|
||||
status: "EXECUTING" as const,
|
||||
},
|
||||
run: {
|
||||
id: taskRun.friendlyId,
|
||||
payload: taskRun.payload,
|
||||
payloadType: taskRun.payloadType,
|
||||
context: taskRun.context,
|
||||
createdAt: taskRun.createdAt,
|
||||
tags: taskRun.tags.map((tag) => tag.name),
|
||||
isTest: taskRun.isTest,
|
||||
idempotencyKey: taskRun.idempotencyKey ?? undefined,
|
||||
},
|
||||
queue: {
|
||||
id: queue.friendlyId,
|
||||
name: queue.name,
|
||||
},
|
||||
environment: {
|
||||
id: environment.id,
|
||||
slug: environment.slug,
|
||||
type: environment.type,
|
||||
},
|
||||
organization: {
|
||||
id: environment.organization.id,
|
||||
slug: environment.organization.slug,
|
||||
name: environment.organization.title,
|
||||
},
|
||||
project: {
|
||||
id: environment.project.id,
|
||||
ref: environment.project.externalRef,
|
||||
slug: environment.project.slug,
|
||||
name: environment.project.name,
|
||||
},
|
||||
batch:
|
||||
taskRun.batchItems[0] && taskRun.batchItems[0].batchTaskRun
|
||||
? { id: taskRun.batchItems[0].batchTaskRun.friendlyId }
|
||||
: undefined,
|
||||
};
|
||||
|
||||
return {
|
||||
execution,
|
||||
run: taskRun,
|
||||
attempt: taskRunAttempt,
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function getAuthenticatedEnvironmentFromRun(
|
||||
friendlyId: string,
|
||||
prismaClient?: PrismaClientOrTransaction
|
||||
) {
|
||||
const taskRun = await (prismaClient ?? prisma).taskRun.findUnique({
|
||||
where: {
|
||||
friendlyId,
|
||||
},
|
||||
include: {
|
||||
runtimeEnvironment: {
|
||||
include: {
|
||||
organization: true,
|
||||
project: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!taskRun) {
|
||||
return;
|
||||
}
|
||||
|
||||
return taskRun?.runtimeEnvironment;
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import { BaseService } from "./baseService.server";
|
||||
import { logger } from "~/services/logger.server";
|
||||
import { socketIo } from "../handleSocketIo.server";
|
||||
import { PrismaClientOrTransaction } from "~/db.server";
|
||||
import { workerQueue } from "~/services/worker.server";
|
||||
|
||||
export class RetryAttemptService extends BaseService {
|
||||
public async call(runId: string) {
|
||||
const taskRun = await this._prisma.taskRun.findFirst({
|
||||
where: {
|
||||
id: runId,
|
||||
},
|
||||
});
|
||||
|
||||
if (!taskRun) {
|
||||
logger.error("Task run not found", { runId });
|
||||
return;
|
||||
}
|
||||
|
||||
socketIo.coordinatorNamespace.emit("READY_FOR_RETRY", {
|
||||
version: "v1",
|
||||
runId,
|
||||
});
|
||||
}
|
||||
|
||||
static async enqueue(runId: string, tx: PrismaClientOrTransaction, runAt?: Date) {
|
||||
return await workerQueue.enqueue(
|
||||
"v3.retryAttempt",
|
||||
{
|
||||
runId,
|
||||
},
|
||||
{
|
||||
tx,
|
||||
runAt,
|
||||
jobKey: `retryAttempt:${runId}`,
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -4,13 +4,12 @@ import {
|
||||
TriggerTaskRequestBody,
|
||||
packetRequiresOffloading,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import { nanoid } from "nanoid";
|
||||
import { createHash } from "node:crypto";
|
||||
import { $transaction } from "~/db.server";
|
||||
import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
|
||||
import { eventRepository } from "../eventRepository.server";
|
||||
import { generateFriendlyId } from "../friendlyIdentifiers";
|
||||
import { marqs } from "~/v3/marqs/index.server";
|
||||
import { marqs, sanitizeQueueName } from "~/v3/marqs/index.server";
|
||||
import { uploadToObjectStore } from "../r2.server";
|
||||
import { BaseService } from "./baseService.server";
|
||||
|
||||
@@ -112,7 +111,7 @@ export class TriggerTaskService extends BaseService {
|
||||
select: { lastNumber: true },
|
||||
});
|
||||
|
||||
const queueName = body.options?.queue?.name ?? `task/${taskId}`;
|
||||
const queueName = sanitizeQueueName(body.options?.queue?.name ?? `task/${taskId}`);
|
||||
|
||||
event.setAttribute("queueName", queueName);
|
||||
span.setAttribute("queueName", queueName);
|
||||
|
||||
@@ -16,7 +16,10 @@ import {
|
||||
GetProjectResponseBody,
|
||||
ImportEnvironmentVariablesRequestBody,
|
||||
EnvironmentVariableResponseBody,
|
||||
TaskRunExecution,
|
||||
APIError,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import { zodfetch } from "@trigger.dev/core/v3/zodfetch";
|
||||
|
||||
export class CliApiClient {
|
||||
private readonly apiURL: string;
|
||||
@@ -29,7 +32,7 @@ export class CliApiClient {
|
||||
}
|
||||
|
||||
async createAuthorizationCode() {
|
||||
return zodfetch(
|
||||
return wrapZodFetch(
|
||||
CreateAuthorizationCodeResponseSchema,
|
||||
`${this.apiURL}/api/v1/authorization-code`,
|
||||
{
|
||||
@@ -39,7 +42,7 @@ export class CliApiClient {
|
||||
}
|
||||
|
||||
async getPersonalAccessToken(authorizationCode: string) {
|
||||
return zodfetch(GetPersonalAccessTokenResponseSchema, `${this.apiURL}/api/v1/token`, {
|
||||
return wrapZodFetch(GetPersonalAccessTokenResponseSchema, `${this.apiURL}/api/v1/token`, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
authorizationCode,
|
||||
@@ -52,7 +55,7 @@ export class CliApiClient {
|
||||
throw new Error("whoAmI: No access token");
|
||||
}
|
||||
|
||||
return zodfetch(WhoAmIResponseSchema, `${this.apiURL}/api/v2/whoami`, {
|
||||
return wrapZodFetch(WhoAmIResponseSchema, `${this.apiURL}/api/v2/whoami`, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
@@ -65,7 +68,7 @@ export class CliApiClient {
|
||||
throw new Error("getProject: No access token");
|
||||
}
|
||||
|
||||
return zodfetch(GetProjectResponseBody, `${this.apiURL}/api/v1/projects/${projectRef}`, {
|
||||
return wrapZodFetch(GetProjectResponseBody, `${this.apiURL}/api/v1/projects/${projectRef}`, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
@@ -78,7 +81,7 @@ export class CliApiClient {
|
||||
throw new Error("getProjects: No access token");
|
||||
}
|
||||
|
||||
return zodfetch(GetProjectsResponseBody, `${this.apiURL}/api/v1/projects`, {
|
||||
return wrapZodFetch(GetProjectsResponseBody, `${this.apiURL}/api/v1/projects`, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
@@ -91,7 +94,7 @@ export class CliApiClient {
|
||||
throw new Error("createBackgroundWorker: No access token");
|
||||
}
|
||||
|
||||
return zodfetch(
|
||||
return wrapZodFetch(
|
||||
CreateBackgroundWorkerResponse,
|
||||
`${this.apiURL}/api/v1/projects/${projectRef}/background-workers`,
|
||||
{
|
||||
@@ -105,6 +108,20 @@ export class CliApiClient {
|
||||
);
|
||||
}
|
||||
|
||||
async createTaskRunAttempt(runFriendlyId: string) {
|
||||
if (!this.accessToken) {
|
||||
throw new Error("creatTaskRunAttempt: No access token");
|
||||
}
|
||||
|
||||
return wrapZodFetch(TaskRunExecution, `${this.apiURL}/api/v1/runs/${runFriendlyId}/attempts`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async getProjectEnv({
|
||||
projectRef,
|
||||
env,
|
||||
@@ -116,12 +133,16 @@ export class CliApiClient {
|
||||
throw new Error("getProjectDevEnv: No access token");
|
||||
}
|
||||
|
||||
return zodfetch(GetProjectEnvResponse, `${this.apiURL}/api/v1/projects/${projectRef}/${env}`, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
return wrapZodFetch(
|
||||
GetProjectEnvResponse,
|
||||
`${this.apiURL}/api/v1/projects/${projectRef}/${env}`,
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async getEnvironmentVariables(projectRef: string) {
|
||||
@@ -129,7 +150,7 @@ export class CliApiClient {
|
||||
throw new Error("getEnvironmentVariables: No access token");
|
||||
}
|
||||
|
||||
return zodfetch(
|
||||
return wrapZodFetch(
|
||||
GetEnvironmentVariablesResponseBody,
|
||||
`${this.apiURL}/api/v1/projects/${projectRef}/envvars`,
|
||||
{
|
||||
@@ -150,7 +171,7 @@ export class CliApiClient {
|
||||
throw new Error("importEnvVars: No access token");
|
||||
}
|
||||
|
||||
return zodfetch(
|
||||
return wrapZodFetch(
|
||||
EnvironmentVariableResponseBody,
|
||||
`${this.apiURL}/api/v1/projects/${projectRef}/envvars/${slug}/import`,
|
||||
{
|
||||
@@ -169,7 +190,7 @@ export class CliApiClient {
|
||||
throw new Error("initializeDeployment: No access token");
|
||||
}
|
||||
|
||||
return zodfetch(InitializeDeploymentResponseBody, `${this.apiURL}/api/v1/deployments`, {
|
||||
return wrapZodFetch(InitializeDeploymentResponseBody, `${this.apiURL}/api/v1/deployments`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.accessToken}`,
|
||||
@@ -184,7 +205,7 @@ export class CliApiClient {
|
||||
throw new Error("startDeploymentIndexing: No access token");
|
||||
}
|
||||
|
||||
return zodfetch(
|
||||
return wrapZodFetch(
|
||||
StartDeploymentIndexingResponseBody,
|
||||
`${this.apiURL}/api/v1/deployments/${deploymentId}/start-indexing`,
|
||||
{
|
||||
@@ -203,7 +224,7 @@ export class CliApiClient {
|
||||
throw new Error("getDeployment: No access token");
|
||||
}
|
||||
|
||||
return zodfetch(
|
||||
return wrapZodFetch(
|
||||
GetDeploymentResponseBody,
|
||||
`${this.apiURL}/api/v1/deployments/${deploymentId}`,
|
||||
{
|
||||
@@ -223,56 +244,42 @@ type ApiResult<TSuccessResult> =
|
||||
error: string;
|
||||
};
|
||||
|
||||
async function zodfetch<TResponseBody extends any>(
|
||||
schema: z.Schema<TResponseBody>,
|
||||
async function wrapZodFetch<T extends z.ZodTypeAny>(
|
||||
schema: T,
|
||||
url: string,
|
||||
requestInit?: RequestInit
|
||||
): Promise<ApiResult<TResponseBody>> {
|
||||
): Promise<ApiResult<z.infer<T>>> {
|
||||
try {
|
||||
const response = await fetch(url, requestInit);
|
||||
const response = await zodfetch(schema, url, requestInit, {
|
||||
retry: {
|
||||
minTimeoutInMs: 500,
|
||||
maxTimeoutInMs: 5000,
|
||||
maxAttempts: 3,
|
||||
factor: 2,
|
||||
randomize: false,
|
||||
},
|
||||
});
|
||||
|
||||
if ((!requestInit || requestInit.method === "GET") && response.status === 404) {
|
||||
return {
|
||||
success: false,
|
||||
error: `404: ${response.statusText}`,
|
||||
};
|
||||
}
|
||||
|
||||
if (response.status >= 400 && response.status < 500) {
|
||||
const body = await response.json();
|
||||
if (!body.error) {
|
||||
return { success: false, error: "Something went wrong" };
|
||||
}
|
||||
|
||||
return { success: false, error: body.error };
|
||||
}
|
||||
|
||||
if (response.status !== 200) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Failed to fetch ${url}, got status code ${response.status}`,
|
||||
};
|
||||
}
|
||||
|
||||
const jsonBody = await response.json();
|
||||
const parsedResult = schema.safeParse(jsonBody);
|
||||
|
||||
if (parsedResult.success) {
|
||||
return { success: true, data: parsedResult.data };
|
||||
}
|
||||
|
||||
if ("error" in jsonBody) {
|
||||
return {
|
||||
success: false,
|
||||
error: typeof jsonBody.error === "string" ? jsonBody.error : JSON.stringify(jsonBody.error),
|
||||
};
|
||||
}
|
||||
|
||||
return { success: false, error: parsedResult.error.message };
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : JSON.stringify(error),
|
||||
success: true,
|
||||
data: response,
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof APIError) {
|
||||
return {
|
||||
success: false,
|
||||
error: error.message,
|
||||
};
|
||||
} else if (error instanceof Error) {
|
||||
return {
|
||||
success: false,
|
||||
error: error.message,
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
success: false,
|
||||
error: String(error),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -199,7 +199,9 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) {
|
||||
`Failed to connect to ${authorization.auth?.apiUrl}. Are you sure it's the correct URL?`
|
||||
);
|
||||
} else {
|
||||
throw new Error("You must login first. Use `trigger.dev login` to login.");
|
||||
throw new Error(
|
||||
`You must login first. Use the \`login\` CLI command.\n\n${authorization.error}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -112,7 +112,11 @@ export async function devCommand(dir: string, options: DevCommandOptions) {
|
||||
)} Connecting to the server failed. Please check your internet connection or contact eric@trigger.dev for help.`
|
||||
);
|
||||
} else {
|
||||
logger.log(`${chalkError("X Error:")} You must login first. Use the \`login\` CLI command.`);
|
||||
logger.log(
|
||||
`${chalkError("X Error:")} You must login first. Use the \`login\` CLI command.\n\n${
|
||||
authorization.error
|
||||
}`
|
||||
);
|
||||
}
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
@@ -285,6 +289,7 @@ function useDev({
|
||||
websocket.addEventListener("close", (event) => {});
|
||||
websocket.addEventListener("error", (event) => {});
|
||||
|
||||
// This is the deprecated task heart beat that uses the friendly attempt ID
|
||||
backgroundWorkerCoordinator.onWorkerTaskHeartbeat.attach(
|
||||
async ({ worker, backgroundWorkerId, id }) => {
|
||||
await sender.send("BACKGROUND_WORKER_MESSAGE", {
|
||||
@@ -297,6 +302,19 @@ function useDev({
|
||||
}
|
||||
);
|
||||
|
||||
// "Task Run Heartbeat" id is the actual run ID that corresponds to the MarQS message ID
|
||||
backgroundWorkerCoordinator.onWorkerTaskRunHeartbeat.attach(
|
||||
async ({ worker, backgroundWorkerId, id }) => {
|
||||
await sender.send("BACKGROUND_WORKER_MESSAGE", {
|
||||
backgroundWorkerId,
|
||||
data: {
|
||||
type: "TASK_RUN_HEARTBEAT",
|
||||
id,
|
||||
},
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
backgroundWorkerCoordinator.onTaskCompleted.attach(
|
||||
async ({ backgroundWorkerId, completion, execution }) => {
|
||||
await sender.send("BACKGROUND_WORKER_MESSAGE", {
|
||||
@@ -310,6 +328,18 @@ function useDev({
|
||||
}
|
||||
);
|
||||
|
||||
backgroundWorkerCoordinator.onTaskFailedToRun.attach(
|
||||
async ({ backgroundWorkerId, completion }) => {
|
||||
await sender.send("BACKGROUND_WORKER_MESSAGE", {
|
||||
backgroundWorkerId,
|
||||
data: {
|
||||
type: "TASK_RUN_FAILED_TO_RUN",
|
||||
completion,
|
||||
},
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
backgroundWorkerCoordinator.onWorkerRegistered.attach(async ({ id, worker, record }) => {
|
||||
await sender.send("READY_FOR_TASKS", {
|
||||
backgroundWorkerId: id,
|
||||
@@ -334,6 +364,7 @@ function useDev({
|
||||
for (const worker of backgroundWorkerCoordinator.currentWorkers) {
|
||||
await sender.send("READY_FOR_TASKS", {
|
||||
backgroundWorkerId: worker.id,
|
||||
inProgressRuns: worker.worker.inProgressRuns,
|
||||
});
|
||||
}
|
||||
},
|
||||
@@ -505,21 +536,25 @@ function useDev({
|
||||
|
||||
const processEnv = await gatherProcessEnv();
|
||||
|
||||
const backgroundWorker = new BackgroundWorker(fullPath, {
|
||||
projectConfig: config,
|
||||
dependencies,
|
||||
env: {
|
||||
...processEnv,
|
||||
TRIGGER_API_URL: apiUrl,
|
||||
TRIGGER_SECRET_KEY: apiKey,
|
||||
...(environmentVariablesResponse.success
|
||||
? environmentVariablesResponse.data.variables
|
||||
: {}),
|
||||
const backgroundWorker = new BackgroundWorker(
|
||||
fullPath,
|
||||
{
|
||||
projectConfig: config,
|
||||
dependencies,
|
||||
env: {
|
||||
...processEnv,
|
||||
TRIGGER_API_URL: apiUrl,
|
||||
TRIGGER_SECRET_KEY: apiKey,
|
||||
...(environmentVariablesResponse.success
|
||||
? environmentVariablesResponse.data.variables
|
||||
: {}),
|
||||
},
|
||||
debuggerOn,
|
||||
debugOtel,
|
||||
resolveEnvVariables: createResolveEnvironmentVariablesFunction(configModule),
|
||||
},
|
||||
debuggerOn,
|
||||
debugOtel,
|
||||
resolveEnvVariables: createResolveEnvironmentVariablesFunction(configModule),
|
||||
});
|
||||
environmentClient
|
||||
);
|
||||
|
||||
try {
|
||||
await backgroundWorker.initialize();
|
||||
@@ -576,6 +611,7 @@ function useDev({
|
||||
tasks: taskResources,
|
||||
contentHash: contentHash,
|
||||
},
|
||||
supportsLazyAttempts: true,
|
||||
};
|
||||
|
||||
const backgroundWorkerRecord = await environmentClient.createBackgroundWorker(
|
||||
@@ -827,18 +863,9 @@ function createDuplicateTaskIdOutputErrorMessage(
|
||||
|
||||
async function gatherProcessEnv() {
|
||||
const env = {
|
||||
...process.env,
|
||||
NODE_ENV: process.env.NODE_ENV ?? "development",
|
||||
PATH: process.env.PATH,
|
||||
USER: process.env.USER,
|
||||
SHELL: process.env.SHELL,
|
||||
NVM_INC: process.env.NVM_INC,
|
||||
NVM_DIR: process.env.NVM_DIR,
|
||||
NVM_BIN: process.env.NVM_BIN,
|
||||
LANG: process.env.LANG,
|
||||
TERM: process.env.TERM,
|
||||
NODE_PATH: await amendNodePathWithPnpmNodeModules(process.env.NODE_PATH),
|
||||
HOME: process.env.HOME,
|
||||
BUN_INSTALL: process.env.BUN_INSTALL,
|
||||
};
|
||||
|
||||
// Filter out undefined values
|
||||
|
||||
@@ -78,7 +78,7 @@ export async function whoAmI(
|
||||
options?.profile ?? "default"
|
||||
}\` to login.`
|
||||
);
|
||||
outro("Whoami failed");
|
||||
outro(`Whoami failed: ${authentication.error}`);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -21,3 +21,43 @@ export class TaskMetadataParseError extends Error {
|
||||
this.name = "TaskMetadataParseError";
|
||||
}
|
||||
}
|
||||
|
||||
export class UnexpectedExitError extends Error {
|
||||
constructor(public code: number) {
|
||||
super(`Unexpected exit with code ${code}`);
|
||||
|
||||
this.name = "UnexpectedExitError";
|
||||
}
|
||||
}
|
||||
|
||||
export class CleanupProcessError extends Error {
|
||||
constructor() {
|
||||
super("Cancelled");
|
||||
|
||||
this.name = "CleanupProcessError";
|
||||
}
|
||||
}
|
||||
|
||||
export class CancelledProcessError extends Error {
|
||||
constructor() {
|
||||
super("Cancelled");
|
||||
|
||||
this.name = "CancelledProcessError";
|
||||
}
|
||||
}
|
||||
|
||||
export class SigKillTimeoutProcessError extends Error {
|
||||
constructor() {
|
||||
super("Process kill timeout");
|
||||
|
||||
this.name = "SigKillTimeoutProcessError";
|
||||
}
|
||||
}
|
||||
|
||||
export class GracefulExitTimeoutError extends Error {
|
||||
constructor() {
|
||||
super("Graceful exit timeout");
|
||||
|
||||
this.name = "GracefulExitTimeoutError";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,8 +9,10 @@ import {
|
||||
TaskRunError,
|
||||
TaskRunErrorCodes,
|
||||
TaskRunExecution,
|
||||
TaskRunExecutionLazyAttemptPayload,
|
||||
TaskRunExecutionPayload,
|
||||
TaskRunExecutionResult,
|
||||
TaskRunFailedExecutionResult,
|
||||
childToWorkerMessages,
|
||||
correctErrorStackTrace,
|
||||
formatDurationMilliseconds,
|
||||
@@ -36,8 +38,15 @@ import {
|
||||
import { safeDeleteFileSync } from "../../utilities/fileSystem.js";
|
||||
import { installPackages } from "../../utilities/installPackages.js";
|
||||
import { logger } from "../../utilities/logger.js";
|
||||
import { TaskMetadataParseError, UncaughtExceptionError } from "../common/errors.js";
|
||||
import { env } from "node:process";
|
||||
import {
|
||||
CancelledProcessError,
|
||||
CleanupProcessError,
|
||||
SigKillTimeoutProcessError,
|
||||
TaskMetadataParseError,
|
||||
UncaughtExceptionError,
|
||||
UnexpectedExitError,
|
||||
} from "../common/errors.js";
|
||||
import { CliApiClient } from "../../apiClient.js";
|
||||
|
||||
export type CurrentWorkers = BackgroundWorkerCoordinator["currentWorkers"];
|
||||
export class BackgroundWorkerCoordinator {
|
||||
@@ -47,37 +56,52 @@ export class BackgroundWorkerCoordinator {
|
||||
worker: BackgroundWorker;
|
||||
execution: TaskRunExecution;
|
||||
}> = new Evt();
|
||||
public onTaskFailedToRun: Evt<{
|
||||
backgroundWorkerId: string;
|
||||
worker: BackgroundWorker;
|
||||
completion: TaskRunFailedExecutionResult;
|
||||
}> = new Evt();
|
||||
public onWorkerRegistered: Evt<{
|
||||
worker: BackgroundWorker;
|
||||
id: string;
|
||||
record: CreateBackgroundWorkerResponse;
|
||||
}> = new Evt();
|
||||
|
||||
/**
|
||||
* @deprecated use onWorkerTaskRunHeartbeat instead
|
||||
*/
|
||||
public onWorkerTaskHeartbeat: Evt<{
|
||||
id: string;
|
||||
backgroundWorkerId: string;
|
||||
worker: BackgroundWorker;
|
||||
}> = new Evt();
|
||||
public onWorkerTaskRunHeartbeat: Evt<{
|
||||
id: string;
|
||||
backgroundWorkerId: string;
|
||||
worker: BackgroundWorker;
|
||||
}> = new Evt();
|
||||
public onWorkerDeprecated: Evt<{ worker: BackgroundWorker; id: string }> = new Evt();
|
||||
private _backgroundWorkers: Map<string, BackgroundWorker> = new Map();
|
||||
private _records: Map<string, CreateBackgroundWorkerResponse> = new Map();
|
||||
private _deprecatedWorkers: Set<string> = new Set();
|
||||
|
||||
constructor(private baseURL: string) {
|
||||
this.onTaskCompleted.attach(async ({ completion, execution }) => {
|
||||
this.onTaskCompleted.attach(async ({ completion }) => {
|
||||
if (!completion.ok && typeof completion.retry !== "undefined") {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.#notifyWorkersOfTaskCompletion(completion, execution);
|
||||
await this.#notifyWorkersOfTaskCompletion(completion);
|
||||
});
|
||||
|
||||
this.onTaskFailedToRun.attach(async ({ completion }) => {
|
||||
await this.#notifyWorkersOfTaskCompletion(completion);
|
||||
});
|
||||
}
|
||||
|
||||
async #notifyWorkersOfTaskCompletion(
|
||||
completion: TaskRunExecutionResult,
|
||||
execution: TaskRunExecution
|
||||
) {
|
||||
async #notifyWorkersOfTaskCompletion(completion: TaskRunExecutionResult) {
|
||||
for (const worker of this._backgroundWorkers.values()) {
|
||||
await worker.taskRunCompletedNotification(completion, execution);
|
||||
await worker.taskRunCompletedNotification(completion);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,6 +131,10 @@ export class BackgroundWorkerCoordinator {
|
||||
worker.onTaskHeartbeat.attach((id) => {
|
||||
this.onWorkerTaskHeartbeat.post({ id, backgroundWorkerId: record.id, worker });
|
||||
});
|
||||
|
||||
worker.onTaskRunHeartbeat.attach((id) => {
|
||||
this.onWorkerTaskRunHeartbeat.post({ id, backgroundWorkerId: record.id, worker });
|
||||
});
|
||||
}
|
||||
|
||||
close() {
|
||||
@@ -136,7 +164,65 @@ export class BackgroundWorkerCoordinator {
|
||||
}
|
||||
|
||||
await worker.cancelRun(message.taskRunId);
|
||||
break;
|
||||
}
|
||||
case "EXECUTE_RUN_LAZY_ATTEMPT": {
|
||||
await this.#executeTaskRunLazyAttempt(id, message.payload);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async #executeTaskRunLazyAttempt(id: string, payload: TaskRunExecutionLazyAttemptPayload) {
|
||||
const worker = this._backgroundWorkers.get(id);
|
||||
|
||||
if (!worker) {
|
||||
logger.error(`Could not find worker ${id}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const record = this._records.get(id);
|
||||
|
||||
if (!record) {
|
||||
logger.error(`Could not find worker record ${id}`);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const { completion, execution } = await worker.executeTaskRunLazyAttempt(
|
||||
payload,
|
||||
this.baseURL
|
||||
);
|
||||
|
||||
this.onTaskCompleted.post({
|
||||
completion,
|
||||
execution,
|
||||
worker,
|
||||
backgroundWorkerId: id,
|
||||
});
|
||||
} catch (error) {
|
||||
this.onTaskFailedToRun.post({
|
||||
backgroundWorkerId: id,
|
||||
worker,
|
||||
completion: {
|
||||
ok: false,
|
||||
id: payload.runId,
|
||||
retry: undefined,
|
||||
error:
|
||||
error instanceof Error
|
||||
? {
|
||||
type: "BUILT_IN_ERROR",
|
||||
name: error.name,
|
||||
message: error.message,
|
||||
stackTrace: error.stack ?? "",
|
||||
}
|
||||
: {
|
||||
type: "BUILT_IN_ERROR",
|
||||
name: "UnknownError",
|
||||
message: String(error),
|
||||
stackTrace: "",
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -155,106 +241,14 @@ export class BackgroundWorkerCoordinator {
|
||||
return;
|
||||
}
|
||||
|
||||
const { execution } = payload;
|
||||
const completion = await worker.executeTaskRun(payload, this.baseURL);
|
||||
|
||||
// ○ Mar 27 09:17:25.653 -> View logs | 20240326.20 | create-avatar | run_slufhjdfiv8ejnrkw9dsj.1
|
||||
|
||||
const logsUrl = `${this.baseURL}/runs/${execution.run.id}`;
|
||||
|
||||
const pipe = chalkGrey("|");
|
||||
const bullet = chalkGrey("○");
|
||||
const link = chalkLink(terminalLink("View logs", logsUrl));
|
||||
let timestampPrefix = chalkGrey(prettyPrintDate(payload.execution.attempt.startedAt));
|
||||
const workerPrefix = chalkWorker(record.version);
|
||||
const taskPrefix = chalkTask(execution.task.id);
|
||||
const runId = chalkRun(`${execution.run.id}.${execution.attempt.number}`);
|
||||
|
||||
logger.log(
|
||||
`${bullet} ${timestampPrefix} ${chalkGrey(
|
||||
"->"
|
||||
)} ${link} ${pipe} ${workerPrefix} ${pipe} ${taskPrefix} ${pipe} ${runId}`
|
||||
);
|
||||
|
||||
const now = performance.now();
|
||||
|
||||
const completion = await worker.executeTaskRun(payload);
|
||||
|
||||
const elapsed = performance.now() - now;
|
||||
|
||||
const retryingText = chalkGrey(
|
||||
!completion.ok && completion.skippedRetrying
|
||||
? " (retrying skipped)"
|
||||
: !completion.ok && completion.retry !== undefined
|
||||
? ` (retrying in ${completion.retry.delay}ms)`
|
||||
: ""
|
||||
);
|
||||
|
||||
const resultText = !completion.ok
|
||||
? completion.error.type === "INTERNAL_ERROR" &&
|
||||
(completion.error.code === TaskRunErrorCodes.TASK_EXECUTION_ABORTED ||
|
||||
completion.error.code === TaskRunErrorCodes.TASK_RUN_CANCELLED)
|
||||
? chalkWarning("Cancelled")
|
||||
: `${chalkError("Error")}${retryingText}`
|
||||
: chalkSuccess("Success");
|
||||
|
||||
const errorText = !completion.ok
|
||||
? this.#formatErrorLog(completion.error)
|
||||
: "retry" in completion
|
||||
? `retry in ${completion.retry}ms`
|
||||
: "";
|
||||
|
||||
const elapsedText = chalkGrey(`(${formatDurationMilliseconds(elapsed, { style: "short" })})`);
|
||||
|
||||
timestampPrefix = chalkGrey(prettyPrintDate());
|
||||
|
||||
logger.log(
|
||||
`${bullet} ${timestampPrefix} ${chalkGrey(
|
||||
"->"
|
||||
)} ${link} ${pipe} ${workerPrefix} ${pipe} ${taskPrefix} ${pipe} ${runId} ${pipe} ${resultText} ${elapsedText}${errorText}`
|
||||
);
|
||||
|
||||
this.onTaskCompleted.post({ completion, execution, worker, backgroundWorkerId: id });
|
||||
}
|
||||
|
||||
#formatErrorLog(error: TaskRunError) {
|
||||
switch (error.type) {
|
||||
case "INTERNAL_ERROR": {
|
||||
return "";
|
||||
}
|
||||
case "STRING_ERROR": {
|
||||
return `\n\n${chalkError("X Error:")} ${error.raw}\n`;
|
||||
}
|
||||
case "CUSTOM_ERROR": {
|
||||
return `\n\n${chalkError("X Error:")} ${error.raw}\n`;
|
||||
}
|
||||
case "BUILT_IN_ERROR": {
|
||||
return `\n\n${error.stackTrace.replace(/^Error: /, chalkError("X Error: "))}\n`;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class UnexpectedExitError extends Error {
|
||||
constructor(public code: number) {
|
||||
super(`Unexpected exit with code ${code}`);
|
||||
|
||||
this.name = "UnexpectedExitError";
|
||||
}
|
||||
}
|
||||
|
||||
class CleanupProcessError extends Error {
|
||||
constructor() {
|
||||
super("Cancelled");
|
||||
|
||||
this.name = "CleanupProcessError";
|
||||
}
|
||||
}
|
||||
|
||||
class CancelledProcessError extends Error {
|
||||
constructor() {
|
||||
super("Cancelled");
|
||||
|
||||
this.name = "CancelledProcessError";
|
||||
this.onTaskCompleted.post({
|
||||
completion,
|
||||
execution: payload.execution,
|
||||
worker,
|
||||
backgroundWorkerId: id,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -276,13 +270,18 @@ export class BackgroundWorker {
|
||||
schema: childToWorkerMessages,
|
||||
});
|
||||
|
||||
/**
|
||||
* @deprecated use onTaskRunHeartbeat instead
|
||||
*/
|
||||
public onTaskHeartbeat: Evt<string> = new Evt();
|
||||
public onTaskRunHeartbeat: Evt<string> = new Evt();
|
||||
private _onClose: Evt<void> = new Evt();
|
||||
|
||||
public tasks: Array<TaskMetadataWithFilePath> = [];
|
||||
public metadata: BackgroundWorkerProperties | undefined;
|
||||
|
||||
_taskRunProcesses: Map<string, TaskRunProcess> = new Map();
|
||||
private _taskRunProcessesBeingKilled: Set<number> = new Set();
|
||||
|
||||
private _closed: boolean = false;
|
||||
|
||||
@@ -290,7 +289,8 @@ export class BackgroundWorker {
|
||||
|
||||
constructor(
|
||||
public path: string,
|
||||
public params: BackgroundWorkerParams
|
||||
public params: BackgroundWorkerParams,
|
||||
private apiClient: CliApiClient
|
||||
) {}
|
||||
|
||||
close() {
|
||||
@@ -301,6 +301,7 @@ export class BackgroundWorker {
|
||||
this._closed = true;
|
||||
|
||||
this.onTaskHeartbeat.detach();
|
||||
this.onTaskRunHeartbeat.detach();
|
||||
|
||||
// We need to close all the task run processes
|
||||
for (const taskRunProcess of this._taskRunProcesses.values()) {
|
||||
@@ -314,6 +315,10 @@ export class BackgroundWorker {
|
||||
safeDeleteFileSync(`${this.path}.map`);
|
||||
}
|
||||
|
||||
get inProgressRuns(): Array<string> {
|
||||
return Array.from(this._taskRunProcesses.keys());
|
||||
}
|
||||
|
||||
async initialize() {
|
||||
if (this._initialized) {
|
||||
throw new Error("Worker already initialized");
|
||||
@@ -408,46 +413,142 @@ export class BackgroundWorker {
|
||||
|
||||
// We need to notify all the task run processes that a task run has completed,
|
||||
// in case they are waiting for it through triggerAndWait
|
||||
async taskRunCompletedNotification(
|
||||
completion: TaskRunExecutionResult,
|
||||
execution: TaskRunExecution
|
||||
) {
|
||||
async taskRunCompletedNotification(completion: TaskRunExecutionResult) {
|
||||
for (const taskRunProcess of this._taskRunProcesses.values()) {
|
||||
taskRunProcess.taskRunCompletedNotification(completion, execution);
|
||||
taskRunProcess.taskRunCompletedNotification(completion);
|
||||
}
|
||||
}
|
||||
|
||||
async #initializeTaskRunProcess(payload: TaskRunExecutionPayload): Promise<TaskRunProcess> {
|
||||
#prefixedMessage(payload: TaskRunExecutionPayload, message: string = "") {
|
||||
return `[${payload.execution.run.id}.${payload.execution.attempt.number}] ${message}`;
|
||||
}
|
||||
|
||||
async #getFreshTaskRunProcess(
|
||||
payload: TaskRunExecutionPayload,
|
||||
messageId?: string
|
||||
): Promise<TaskRunProcess> {
|
||||
logger.debug(this.#prefixedMessage(payload, "getFreshTaskRunProcess()"));
|
||||
|
||||
if (!this.metadata) {
|
||||
throw new Error("Worker not registered");
|
||||
}
|
||||
|
||||
if (!this._taskRunProcesses.has(payload.execution.run.id)) {
|
||||
const taskRunProcess = new TaskRunProcess(
|
||||
payload.execution,
|
||||
this.path,
|
||||
{
|
||||
...this._fullEnv,
|
||||
...(payload.environment ?? {}),
|
||||
},
|
||||
this.metadata,
|
||||
this.params
|
||||
);
|
||||
this._closed = false;
|
||||
|
||||
taskRunProcess.onExit.attach(() => {
|
||||
logger.debug(this.#prefixedMessage(payload, "killing current task run process before attempt"));
|
||||
|
||||
await this.#killCurrentTaskRunProcessBeforeAttempt(payload.execution.run.id);
|
||||
|
||||
logger.debug(this.#prefixedMessage(payload, "creating new task run process"));
|
||||
|
||||
const taskRunProcess = new TaskRunProcess(
|
||||
payload.execution.run.id,
|
||||
payload.execution.run.isTest,
|
||||
this.path,
|
||||
{
|
||||
...this._fullEnv,
|
||||
...(payload.environment ?? {}),
|
||||
...this.#readEnvVars(),
|
||||
},
|
||||
this.metadata,
|
||||
this.params,
|
||||
messageId
|
||||
);
|
||||
|
||||
taskRunProcess.onExit.attach(({ pid }) => {
|
||||
logger.debug(this.#prefixedMessage(payload, "onExit()"), { pid });
|
||||
|
||||
const taskRunProcess = this._taskRunProcesses.get(payload.execution.run.id);
|
||||
|
||||
// Only delete the task run process if the pid matches
|
||||
if (taskRunProcess?.pid === pid) {
|
||||
this._taskRunProcesses.delete(payload.execution.run.id);
|
||||
});
|
||||
}
|
||||
|
||||
taskRunProcess.onTaskHeartbeat.attach((id) => {
|
||||
this.onTaskHeartbeat.post(id);
|
||||
});
|
||||
if (pid) {
|
||||
this._taskRunProcessesBeingKilled.delete(pid);
|
||||
}
|
||||
});
|
||||
|
||||
await taskRunProcess.initialize();
|
||||
taskRunProcess.onIsBeingKilled.attach((pid) => {
|
||||
if (pid) {
|
||||
this._taskRunProcessesBeingKilled.add(pid);
|
||||
}
|
||||
});
|
||||
|
||||
this._taskRunProcesses.set(payload.execution.run.id, taskRunProcess);
|
||||
taskRunProcess.onTaskHeartbeat.attach((id) => {
|
||||
this.onTaskHeartbeat.post(id);
|
||||
});
|
||||
|
||||
taskRunProcess.onTaskRunHeartbeat.attach((id) => {
|
||||
this.onTaskRunHeartbeat.post(id);
|
||||
});
|
||||
|
||||
await taskRunProcess.initialize();
|
||||
|
||||
this._taskRunProcesses.set(payload.execution.run.id, taskRunProcess);
|
||||
|
||||
return taskRunProcess;
|
||||
}
|
||||
|
||||
async #killCurrentTaskRunProcessBeforeAttempt(runId: string) {
|
||||
const taskRunProcess = this._taskRunProcesses.get(runId);
|
||||
|
||||
if (!taskRunProcess) {
|
||||
logger.debug(`[${runId}] no current task process to kill`);
|
||||
return;
|
||||
}
|
||||
|
||||
return this._taskRunProcesses.get(payload.execution.run.id) as TaskRunProcess;
|
||||
logger.debug(`[${runId}] killing current task process`, {
|
||||
pid: taskRunProcess.pid,
|
||||
});
|
||||
|
||||
if (taskRunProcess.isBeingKilled) {
|
||||
if (this._taskRunProcessesBeingKilled.size > 1) {
|
||||
await this.#tryGracefulExit(taskRunProcess);
|
||||
} else {
|
||||
// If there's only one or none being killed, don't do anything so we can create a fresh one in parallel
|
||||
}
|
||||
} else {
|
||||
// It's not being killed, so kill it
|
||||
if (this._taskRunProcessesBeingKilled.size > 0) {
|
||||
await this.#tryGracefulExit(taskRunProcess);
|
||||
} else {
|
||||
// There's none being killed yet, so we can kill it without waiting. We still set a timeout to kill it forcefully just in case it sticks around.
|
||||
taskRunProcess.kill("SIGTERM", 5_000).catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async #tryGracefulExit(
|
||||
taskRunProcess: TaskRunProcess,
|
||||
kill = false,
|
||||
initialSignal: number | NodeJS.Signals = "SIGTERM"
|
||||
) {
|
||||
try {
|
||||
const initialExit = taskRunProcess.onExit.waitFor(5_000);
|
||||
|
||||
if (kill) {
|
||||
taskRunProcess.kill(initialSignal);
|
||||
}
|
||||
|
||||
await initialExit;
|
||||
} catch (error) {
|
||||
logger.error("TaskRunProcess graceful kill timeout exceeded", error);
|
||||
|
||||
this.#tryForcefulExit(taskRunProcess);
|
||||
}
|
||||
}
|
||||
|
||||
async #tryForcefulExit(taskRunProcess: TaskRunProcess) {
|
||||
try {
|
||||
const forcedKill = taskRunProcess.onExit.waitFor(5_000);
|
||||
taskRunProcess.kill("SIGKILL");
|
||||
await forcedKill;
|
||||
} catch (error) {
|
||||
logger.error("TaskRunProcess forced kill timeout exceeded", error);
|
||||
throw new SigKillTimeoutProcessError();
|
||||
}
|
||||
}
|
||||
|
||||
async cancelRun(taskRunId: string) {
|
||||
@@ -460,14 +561,113 @@ export class BackgroundWorker {
|
||||
await taskRunProcess.cancel();
|
||||
}
|
||||
|
||||
async executeTaskRunLazyAttempt(payload: TaskRunExecutionLazyAttemptPayload, baseURL: string) {
|
||||
const attemptResponse = await this.apiClient.createTaskRunAttempt(payload.runId);
|
||||
|
||||
if (!attemptResponse.success) {
|
||||
throw new Error(`Failed to create task run attempt: ${attemptResponse.error}`);
|
||||
}
|
||||
|
||||
const execution = attemptResponse.data;
|
||||
|
||||
const completion = await this.executeTaskRun(
|
||||
{ execution, traceContext: payload.traceContext, environment: payload.environment },
|
||||
baseURL,
|
||||
payload.messageId
|
||||
);
|
||||
|
||||
return { execution, completion };
|
||||
}
|
||||
|
||||
// We need to fork the process before we can execute any tasks
|
||||
async executeTaskRun(payload: TaskRunExecutionPayload): Promise<TaskRunExecutionResult> {
|
||||
async executeTaskRun(
|
||||
payload: TaskRunExecutionPayload,
|
||||
baseURL: string,
|
||||
messageId?: string
|
||||
): Promise<TaskRunExecutionResult> {
|
||||
if (this._closed) {
|
||||
throw new Error("Worker is closed");
|
||||
}
|
||||
|
||||
if (!this.metadata) {
|
||||
throw new Error("Worker not registered");
|
||||
}
|
||||
|
||||
const { execution } = payload;
|
||||
// ○ Mar 27 09:17:25.653 -> View logs | 20240326.20 | create-avatar | run_slufhjdfiv8ejnrkw9dsj.1
|
||||
|
||||
const logsUrl = `${baseURL}/runs/${execution.run.id}`;
|
||||
|
||||
const pipe = chalkGrey("|");
|
||||
const bullet = chalkGrey("○");
|
||||
const link = chalkLink(terminalLink("View logs", logsUrl));
|
||||
let timestampPrefix = chalkGrey(prettyPrintDate(payload.execution.attempt.startedAt));
|
||||
const workerPrefix = chalkWorker(this.metadata.version);
|
||||
const taskPrefix = chalkTask(execution.task.id);
|
||||
const runId = chalkRun(`${execution.run.id}.${execution.attempt.number}`);
|
||||
|
||||
logger.log(
|
||||
`${bullet} ${timestampPrefix} ${chalkGrey(
|
||||
"->"
|
||||
)} ${link} ${pipe} ${workerPrefix} ${pipe} ${taskPrefix} ${pipe} ${runId}`
|
||||
);
|
||||
|
||||
const now = performance.now();
|
||||
|
||||
const completion = await this.#doExecuteTaskRun(payload, messageId);
|
||||
|
||||
const elapsed = performance.now() - now;
|
||||
|
||||
const retryingText = chalkGrey(
|
||||
!completion.ok && completion.skippedRetrying
|
||||
? " (retrying skipped)"
|
||||
: !completion.ok && completion.retry !== undefined
|
||||
? ` (retrying in ${completion.retry.delay}ms)`
|
||||
: ""
|
||||
);
|
||||
|
||||
const resultText = !completion.ok
|
||||
? completion.error.type === "INTERNAL_ERROR" &&
|
||||
(completion.error.code === TaskRunErrorCodes.TASK_EXECUTION_ABORTED ||
|
||||
completion.error.code === TaskRunErrorCodes.TASK_RUN_CANCELLED)
|
||||
? chalkWarning("Cancelled")
|
||||
: `${chalkError("Error")}${retryingText}`
|
||||
: chalkSuccess("Success");
|
||||
|
||||
const errorText = !completion.ok
|
||||
? formatErrorLog(completion.error)
|
||||
: "retry" in completion
|
||||
? `retry in ${completion.retry}ms`
|
||||
: "";
|
||||
|
||||
const elapsedText = chalkGrey(`(${formatDurationMilliseconds(elapsed, { style: "short" })})`);
|
||||
|
||||
timestampPrefix = chalkGrey(prettyPrintDate());
|
||||
|
||||
logger.log(
|
||||
`${bullet} ${timestampPrefix} ${chalkGrey(
|
||||
"->"
|
||||
)} ${link} ${pipe} ${workerPrefix} ${pipe} ${taskPrefix} ${pipe} ${runId} ${pipe} ${resultText} ${elapsedText}${errorText}`
|
||||
);
|
||||
|
||||
return completion;
|
||||
}
|
||||
|
||||
async #doExecuteTaskRun(
|
||||
payload: TaskRunExecutionPayload,
|
||||
messageId?: string
|
||||
): Promise<TaskRunExecutionResult> {
|
||||
try {
|
||||
const taskRunProcess = await this.#initializeTaskRunProcess(payload);
|
||||
const taskRunProcess = await this.#getFreshTaskRunProcess(payload, messageId);
|
||||
|
||||
logger.debug(this.#prefixedMessage(payload, "executing task run"), {
|
||||
pid: taskRunProcess.pid,
|
||||
});
|
||||
|
||||
const result = await taskRunProcess.executeTaskRun(payload);
|
||||
|
||||
// Kill the worker if the task was successful or if it's not going to be retried);
|
||||
await taskRunProcess.cleanup(result.ok || result.retry === undefined);
|
||||
// Always kill the worker
|
||||
await taskRunProcess.cleanup(true);
|
||||
|
||||
if (result.ok) {
|
||||
return result;
|
||||
@@ -568,6 +768,7 @@ class TaskRunProcess {
|
||||
});
|
||||
private _sender: ZodMessageSender<typeof workerToChildMessages>;
|
||||
private _child: ChildProcess | undefined;
|
||||
private _childPid?: number;
|
||||
private _attemptPromises: Map<
|
||||
string,
|
||||
{ resolver: (value: TaskRunExecutionResult) => void; rejecter: (err?: any) => void }
|
||||
@@ -576,15 +777,23 @@ class TaskRunProcess {
|
||||
private _currentExecution: TaskRunExecution | undefined;
|
||||
private _isBeingKilled: boolean = false;
|
||||
private _isBeingCancelled: boolean = false;
|
||||
/**
|
||||
* @deprecated use onTaskRunHeartbeat instead
|
||||
*/
|
||||
public onTaskHeartbeat: Evt<string> = new Evt();
|
||||
public onExit: Evt<number> = new Evt();
|
||||
public onTaskRunHeartbeat: Evt<string> = new Evt();
|
||||
public onExit: Evt<{ code: number | null; signal: NodeJS.Signals | null; pid?: number }> =
|
||||
new Evt();
|
||||
public onIsBeingKilled: Evt<number | undefined> = new Evt();
|
||||
|
||||
constructor(
|
||||
private execution: TaskRunExecution,
|
||||
private runId: string,
|
||||
private isTest: boolean,
|
||||
private path: string,
|
||||
private env: NodeJS.ProcessEnv,
|
||||
private metadata: BackgroundWorkerProperties,
|
||||
private worker: BackgroundWorkerParams
|
||||
private worker: BackgroundWorkerParams,
|
||||
private messageId?: string
|
||||
) {
|
||||
this._sender = new ZodMessageSender({
|
||||
schema: workerToChildMessages,
|
||||
@@ -604,7 +813,7 @@ class TaskRunProcess {
|
||||
|
||||
async initialize() {
|
||||
const fullEnv = {
|
||||
...(this.execution.run.isTest ? { TRIGGER_LOG_LEVEL: "debug" } : {}),
|
||||
...(this.isTest ? { TRIGGER_LOG_LEVEL: "debug" } : {}),
|
||||
...this.env,
|
||||
OTEL_RESOURCE_ATTRIBUTES: JSON.stringify({
|
||||
[SemanticInternalAttributes.PROJECT_DIR]: this.worker.projectConfig.projectDir,
|
||||
@@ -615,7 +824,7 @@ class TaskRunProcess {
|
||||
|
||||
const cwd = dirname(this.path);
|
||||
|
||||
logger.debug(`[${this.execution.run.id}] initializing task run process`, {
|
||||
logger.debug(`[${this.runId}] initializing task run process`, {
|
||||
env: fullEnv,
|
||||
path: this.path,
|
||||
cwd,
|
||||
@@ -629,6 +838,7 @@ class TaskRunProcess {
|
||||
? ["--inspect-brk", "--trace-uncaught", "--no-warnings=ExperimentalWarning"]
|
||||
: ["--trace-uncaught", "--no-warnings=ExperimentalWarning"],
|
||||
});
|
||||
this._childPid = this._child?.pid;
|
||||
|
||||
this._child.on("message", this.#handleMessage.bind(this));
|
||||
this._child.on("exit", this.#handleExit.bind(this));
|
||||
@@ -641,19 +851,28 @@ class TaskRunProcess {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.debug(`[${this.execution.run.id}] cleaning up task run process`, { kill });
|
||||
if (kill) {
|
||||
this._isBeingKilled = true;
|
||||
this.onIsBeingKilled.post(this._child?.pid);
|
||||
}
|
||||
|
||||
logger.debug(`[${this.runId}] cleaning up task run process`, { kill, pid: this.pid });
|
||||
|
||||
await this._sender.send("CLEANUP", {
|
||||
flush: true,
|
||||
kill,
|
||||
});
|
||||
|
||||
this._isBeingKilled = kill;
|
||||
// FIXME: Something broke READY_TO_DISPOSE. We never receive it, so we always have to kill the process after the timeout below.
|
||||
|
||||
if (!kill) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Set a timeout to kill the child process if it hasn't been killed within 5 seconds
|
||||
setTimeout(() => {
|
||||
if (this._child && !this._child.killed) {
|
||||
logger.debug(`[${this.execution.run.id}] killing task run process after timeout`);
|
||||
logger.debug(`[${this.runId}] killing task run process after timeout`, { pid: this.pid });
|
||||
|
||||
this._child.kill();
|
||||
}
|
||||
@@ -691,24 +910,23 @@ class TaskRunProcess {
|
||||
return result;
|
||||
}
|
||||
|
||||
taskRunCompletedNotification(completion: TaskRunExecutionResult, execution: TaskRunExecution) {
|
||||
taskRunCompletedNotification(completion: TaskRunExecutionResult) {
|
||||
if (!completion.ok && typeof completion.retry !== "undefined") {
|
||||
return;
|
||||
}
|
||||
|
||||
if (execution.run.id === this.execution.run.id) {
|
||||
if (completion.id === this.runId) {
|
||||
// We don't need to notify the task run process if it's the same as the one we're running
|
||||
return;
|
||||
}
|
||||
|
||||
logger.debug(`[${this.execution.run.id}] task run completed notification`, {
|
||||
logger.debug(`[${this.runId}] task run completed notification`, {
|
||||
completion,
|
||||
execution,
|
||||
});
|
||||
|
||||
this._sender.send("TASK_RUN_COMPLETED_NOTIFICATION", {
|
||||
version: "v2",
|
||||
completion,
|
||||
execution,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -740,14 +958,18 @@ class TaskRunProcess {
|
||||
break;
|
||||
}
|
||||
case "READY_TO_DISPOSE": {
|
||||
logger.debug(`[${this.execution.run.id}] task run process is ready to dispose`);
|
||||
logger.debug(`[${this.runId}] task run process is ready to dispose`);
|
||||
|
||||
this.#kill();
|
||||
|
||||
break;
|
||||
}
|
||||
case "TASK_HEARTBEAT": {
|
||||
this.onTaskHeartbeat.post(message.payload.id);
|
||||
if (this.messageId) {
|
||||
this.onTaskRunHeartbeat.post(this.messageId);
|
||||
} else {
|
||||
this.onTaskHeartbeat.post(message.payload.id);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
@@ -757,8 +979,8 @@ class TaskRunProcess {
|
||||
}
|
||||
}
|
||||
|
||||
async #handleExit(code: number) {
|
||||
logger.debug(`[${this.execution.run.id}] task run process exiting`, { code });
|
||||
async #handleExit(code: number | null, signal: NodeJS.Signals | null) {
|
||||
logger.debug(`[${this.runId}] handle task run process exit`, { code, signal, pid: this.pid });
|
||||
|
||||
// Go through all the attempts currently pending and reject them
|
||||
for (const [id, status] of this._attemptStatuses.entries()) {
|
||||
@@ -778,12 +1000,12 @@ class TaskRunProcess {
|
||||
} else if (this._isBeingKilled) {
|
||||
rejecter(new CleanupProcessError());
|
||||
} else {
|
||||
rejecter(new UnexpectedExitError(code));
|
||||
rejecter(new UnexpectedExitError(code ?? -1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.onExit.post(code);
|
||||
this.onExit.post({ code, signal, pid: this.pid });
|
||||
}
|
||||
|
||||
#handleLog(data: Buffer) {
|
||||
@@ -823,10 +1045,54 @@ class TaskRunProcess {
|
||||
}
|
||||
|
||||
#kill() {
|
||||
if (this._child && !this._child.killed) {
|
||||
logger.debug(`[${this.execution.run.id}] killing task run process`);
|
||||
logger.debug(`[${this.runId}] #kill()`, { pid: this.pid });
|
||||
|
||||
if (this._child && !this._child.killed) {
|
||||
this._child?.kill();
|
||||
}
|
||||
}
|
||||
|
||||
async kill(signal?: number | NodeJS.Signals, timeoutInMs?: number) {
|
||||
logger.debug(`[${this.runId}] killing task run process`, {
|
||||
signal,
|
||||
timeoutInMs,
|
||||
pid: this.pid,
|
||||
});
|
||||
|
||||
this._isBeingKilled = true;
|
||||
|
||||
const killTimeout = this.onExit.waitFor(timeoutInMs);
|
||||
|
||||
this.onIsBeingKilled.post(this._child?.pid);
|
||||
this._child?.kill(signal);
|
||||
|
||||
if (timeoutInMs) {
|
||||
await killTimeout;
|
||||
}
|
||||
}
|
||||
|
||||
get isBeingKilled() {
|
||||
return this._isBeingKilled || this._child?.killed;
|
||||
}
|
||||
|
||||
get pid() {
|
||||
return this._childPid;
|
||||
}
|
||||
}
|
||||
|
||||
function formatErrorLog(error: TaskRunError) {
|
||||
switch (error.type) {
|
||||
case "INTERNAL_ERROR": {
|
||||
return "";
|
||||
}
|
||||
case "STRING_ERROR": {
|
||||
return `\n\n${chalkError("X Error:")} ${error.raw}\n`;
|
||||
}
|
||||
case "CUSTOM_ERROR": {
|
||||
return `\n\n${chalkError("X Error:")} ${error.raw}\n`;
|
||||
}
|
||||
case "BUILT_IN_ERROR": {
|
||||
return `\n\n${error.stackTrace.replace(/^Error: /, chalkError("X Error: "))}\n`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,8 +182,17 @@ const handler = new ZodMessageHandler({
|
||||
_isRunning = false;
|
||||
}
|
||||
},
|
||||
TASK_RUN_COMPLETED_NOTIFICATION: async ({ completion, execution }) => {
|
||||
devRuntimeManager.resumeTask(completion, execution);
|
||||
TASK_RUN_COMPLETED_NOTIFICATION: async (payload) => {
|
||||
switch (payload.version) {
|
||||
case "v1": {
|
||||
devRuntimeManager.resumeTask(payload.completion, payload.execution.run.id);
|
||||
break;
|
||||
}
|
||||
case "v2": {
|
||||
devRuntimeManager.resumeTask(payload.completion, payload.completion.id);
|
||||
break;
|
||||
}
|
||||
}
|
||||
},
|
||||
CLEANUP: async ({ flush, kill }) => {
|
||||
if (kill) {
|
||||
@@ -215,7 +224,7 @@ sender.send("TASKS_READY", { tasks: TASK_METADATA }).catch((err) => {
|
||||
|
||||
process.title = "trigger-dev-worker";
|
||||
|
||||
async function asyncHeartbeat(initialDelayInSeconds: number = 30, intervalInSeconds: number = 5) {
|
||||
async function asyncHeartbeat(initialDelayInSeconds: number = 30, intervalInSeconds: number = 30) {
|
||||
async function _doHeartbeat() {
|
||||
while (true) {
|
||||
if (_isRunning && _execution) {
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
TaskRunBuiltInError,
|
||||
TaskRunErrorCodes,
|
||||
TaskRunExecution,
|
||||
TaskRunExecutionLazyAttemptPayload,
|
||||
TaskRunExecutionPayload,
|
||||
TaskRunExecutionResult,
|
||||
WaitReason,
|
||||
@@ -20,31 +21,15 @@ import { ZodIpcConnection } from "@trigger.dev/core/v3/zodIpc";
|
||||
import type { InferSocketMessageSchema } from "@trigger.dev/core/v3/zodSocket";
|
||||
import { Evt } from "evt";
|
||||
import { ChildProcess, fork } from "node:child_process";
|
||||
import { TaskMetadataParseError, UncaughtExceptionError } from "../common/errors";
|
||||
|
||||
class UnexpectedExitError extends Error {
|
||||
constructor(public code: number) {
|
||||
super(`Unexpected exit with code ${code}`);
|
||||
|
||||
this.name = "UnexpectedExitError";
|
||||
}
|
||||
}
|
||||
|
||||
class CleanupProcessError extends Error {
|
||||
constructor() {
|
||||
super("Cancelled");
|
||||
|
||||
this.name = "CleanupProcessError";
|
||||
}
|
||||
}
|
||||
|
||||
class CancelledProcessError extends Error {
|
||||
constructor() {
|
||||
super("Cancelled");
|
||||
|
||||
this.name = "CancelledProcessError";
|
||||
}
|
||||
}
|
||||
import {
|
||||
CancelledProcessError,
|
||||
CleanupProcessError,
|
||||
GracefulExitTimeoutError,
|
||||
SigKillTimeoutProcessError,
|
||||
TaskMetadataParseError,
|
||||
UncaughtExceptionError,
|
||||
UnexpectedExitError,
|
||||
} from "../common/errors";
|
||||
|
||||
type BackgroundWorkerParams = {
|
||||
env: Record<string, string>;
|
||||
@@ -56,7 +41,11 @@ type BackgroundWorkerParams = {
|
||||
export class ProdBackgroundWorker {
|
||||
private _initialized: boolean = false;
|
||||
|
||||
/**
|
||||
* @deprecated use onTaskRunHeartbeat instead
|
||||
*/
|
||||
public onTaskHeartbeat: Evt<string> = new Evt();
|
||||
public onTaskRunHeartbeat: Evt<string> = new Evt();
|
||||
|
||||
public onWaitForBatch: Evt<
|
||||
InferSocketMessageSchema<typeof ProdChildToWorkerMessages, "WAIT_FOR_BATCH">
|
||||
@@ -74,11 +63,24 @@ export class ProdBackgroundWorker {
|
||||
public onReadyForCheckpoint = Evt.create<{ version?: "v1" }>();
|
||||
public onCancelCheckpoint = Evt.create<{ version?: "v1" | "v2"; reason?: WaitReason }>();
|
||||
|
||||
public onCreateTaskRunAttempt = Evt.create<{ version?: "v1"; runId: string }>();
|
||||
public attemptCreatedNotification = Evt.create<
|
||||
| {
|
||||
success: false;
|
||||
reason?: string;
|
||||
}
|
||||
| {
|
||||
success: true;
|
||||
execution: ProdTaskRunExecution;
|
||||
}
|
||||
>();
|
||||
|
||||
private _onClose: Evt<void> = new Evt();
|
||||
|
||||
public tasks: Array<TaskMetadataWithFilePath> = [];
|
||||
|
||||
_taskRunProcess: TaskRunProcess | undefined;
|
||||
private _taskRunProcessesBeingKilled: Map<number, TaskRunProcess> = new Map();
|
||||
|
||||
private _closed: boolean = false;
|
||||
|
||||
@@ -87,7 +89,9 @@ export class ProdBackgroundWorker {
|
||||
private params: BackgroundWorkerParams
|
||||
) {}
|
||||
|
||||
async close() {
|
||||
async close(gracefulExitTimeoutElapsed = false) {
|
||||
console.log("Closing worker", { gracefulExitTimeoutElapsed, closed: this._closed });
|
||||
|
||||
if (this._closed) {
|
||||
return;
|
||||
}
|
||||
@@ -95,9 +99,35 @@ export class ProdBackgroundWorker {
|
||||
this._closed = true;
|
||||
|
||||
this.onTaskHeartbeat.detach();
|
||||
this.onTaskRunHeartbeat.detach();
|
||||
|
||||
// We need to close the task run process
|
||||
await this._taskRunProcess?.cleanup(true);
|
||||
await this._taskRunProcess?.cleanup(true, gracefulExitTimeoutElapsed);
|
||||
}
|
||||
|
||||
async #killTaskRunProcess(flush = true, initialSignal: number | NodeJS.Signals = "SIGTERM") {
|
||||
console.log("Killing task run process", { flush, initialSignal, closed: this._closed });
|
||||
|
||||
if (this._closed || !this._taskRunProcess) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (flush) {
|
||||
await this.flushTelemetry();
|
||||
}
|
||||
|
||||
const currentTaskRunProcess = this._taskRunProcess;
|
||||
|
||||
// Try graceful exit but don't wait. We limit the amount of processes during creation instead.
|
||||
this.#tryGracefulExit(currentTaskRunProcess, true, initialSignal).catch((error) => {
|
||||
console.error("Error while trying graceful exit", error);
|
||||
});
|
||||
|
||||
console.log("Killed task run process, setting closed to true", {
|
||||
closed: this._closed,
|
||||
pid: currentTaskRunProcess.pid,
|
||||
});
|
||||
this._closed = true;
|
||||
}
|
||||
|
||||
async flushTelemetry() {
|
||||
@@ -193,83 +223,191 @@ export class ProdBackgroundWorker {
|
||||
|
||||
// We need to notify all the task run processes that a task run has completed,
|
||||
// in case they are waiting for it through triggerAndWait
|
||||
async taskRunCompletedNotification(
|
||||
completion: TaskRunExecutionResult,
|
||||
execution: TaskRunExecution
|
||||
) {
|
||||
this._taskRunProcess?.taskRunCompletedNotification(completion, execution);
|
||||
async taskRunCompletedNotification(completion: TaskRunExecutionResult) {
|
||||
this._taskRunProcess?.taskRunCompletedNotification(completion);
|
||||
}
|
||||
|
||||
async waitCompletedNotification() {
|
||||
this._taskRunProcess?.waitCompletedNotification();
|
||||
}
|
||||
|
||||
async #initializeTaskRunProcess(payload: ProdTaskRunExecutionPayload): Promise<TaskRunProcess> {
|
||||
async #getFreshTaskRunProcess(
|
||||
payload: ProdTaskRunExecutionPayload,
|
||||
messageId?: string
|
||||
): Promise<TaskRunProcess> {
|
||||
const metadata = this.getMetadata(
|
||||
payload.execution.worker.id,
|
||||
payload.execution.worker.version
|
||||
);
|
||||
|
||||
if (!this._taskRunProcess) {
|
||||
const taskRunProcess = new TaskRunProcess(
|
||||
payload.execution,
|
||||
this.path,
|
||||
{
|
||||
...this.params.env,
|
||||
...(payload.environment ?? {}),
|
||||
},
|
||||
metadata,
|
||||
this.params
|
||||
);
|
||||
console.log("Getting fresh task run process, setting closed to false", {
|
||||
closed: this._closed,
|
||||
});
|
||||
this._closed = false;
|
||||
|
||||
taskRunProcess.onExit.attach(() => {
|
||||
await this.#killCurrentTaskRunProcessBeforeAttempt();
|
||||
|
||||
const taskRunProcess = new TaskRunProcess(
|
||||
payload.execution.run.id,
|
||||
payload.execution.run.isTest,
|
||||
this.path,
|
||||
{
|
||||
...this.params.env,
|
||||
...(payload.environment ?? {}),
|
||||
},
|
||||
metadata,
|
||||
this.params,
|
||||
messageId
|
||||
);
|
||||
|
||||
taskRunProcess.onExit.attach(({ pid }) => {
|
||||
console.log("Task run process exited", { pid });
|
||||
|
||||
// Only delete the task run process if the pid matches
|
||||
if (this._taskRunProcess?.pid === pid) {
|
||||
this._taskRunProcess = undefined;
|
||||
});
|
||||
}
|
||||
|
||||
taskRunProcess.onTaskHeartbeat.attach((id) => {
|
||||
this.onTaskHeartbeat.post(id);
|
||||
});
|
||||
if (pid) {
|
||||
this._taskRunProcessesBeingKilled.delete(pid);
|
||||
}
|
||||
});
|
||||
|
||||
taskRunProcess.onWaitForBatch.attach((message) => {
|
||||
this.onWaitForBatch.post(message);
|
||||
});
|
||||
taskRunProcess.onIsBeingKilled.attach((taskRunProcess) => {
|
||||
if (taskRunProcess?.pid) {
|
||||
this._taskRunProcessesBeingKilled.set(taskRunProcess.pid, taskRunProcess);
|
||||
}
|
||||
});
|
||||
|
||||
taskRunProcess.onWaitForDuration.attach((message) => {
|
||||
this.onWaitForDuration.post(message);
|
||||
});
|
||||
taskRunProcess.onTaskHeartbeat.attach((id) => {
|
||||
this.onTaskHeartbeat.post(id);
|
||||
});
|
||||
|
||||
taskRunProcess.onWaitForTask.attach((message) => {
|
||||
this.onWaitForTask.post(message);
|
||||
});
|
||||
taskRunProcess.onTaskRunHeartbeat.attach((id) => {
|
||||
this.onTaskRunHeartbeat.post(id);
|
||||
});
|
||||
|
||||
taskRunProcess.onReadyForCheckpoint.attach((message) => {
|
||||
this.onReadyForCheckpoint.post(message);
|
||||
});
|
||||
taskRunProcess.onWaitForBatch.attach((message) => {
|
||||
this.onWaitForBatch.post(message);
|
||||
});
|
||||
|
||||
taskRunProcess.onCancelCheckpoint.attach((message) => {
|
||||
this.onCancelCheckpoint.post(message);
|
||||
});
|
||||
taskRunProcess.onWaitForDuration.attach((message) => {
|
||||
this.onWaitForDuration.post(message);
|
||||
});
|
||||
|
||||
// Notify down the chain
|
||||
this.preCheckpointNotification.attach((message) => {
|
||||
taskRunProcess.preCheckpointNotification.post(message);
|
||||
});
|
||||
this.checkpointCanceledNotification.attach((message) => {
|
||||
taskRunProcess.checkpointCanceledNotification.post(message);
|
||||
});
|
||||
taskRunProcess.onWaitForTask.attach((message) => {
|
||||
this.onWaitForTask.post(message);
|
||||
});
|
||||
|
||||
await taskRunProcess.initialize();
|
||||
taskRunProcess.onReadyForCheckpoint.attach((message) => {
|
||||
this.onReadyForCheckpoint.post(message);
|
||||
});
|
||||
|
||||
this._taskRunProcess = taskRunProcess;
|
||||
}
|
||||
taskRunProcess.onCancelCheckpoint.attach((message) => {
|
||||
this.onCancelCheckpoint.post(message);
|
||||
});
|
||||
|
||||
// Notify down the chain
|
||||
this.preCheckpointNotification.attach((message) => {
|
||||
taskRunProcess.preCheckpointNotification.post(message);
|
||||
});
|
||||
this.checkpointCanceledNotification.attach((message) => {
|
||||
taskRunProcess.checkpointCanceledNotification.post(message);
|
||||
});
|
||||
|
||||
await taskRunProcess.initialize();
|
||||
|
||||
this._taskRunProcess = taskRunProcess;
|
||||
|
||||
return this._taskRunProcess;
|
||||
}
|
||||
|
||||
// We need to fork the process before we can execute any tasks
|
||||
async executeTaskRun(payload: ProdTaskRunExecutionPayload): Promise<TaskRunExecutionResult> {
|
||||
async forceKillOldTaskRunProcesses() {
|
||||
for (const taskRunProcess of this._taskRunProcessesBeingKilled.values()) {
|
||||
try {
|
||||
await taskRunProcess.kill("SIGKILL");
|
||||
} catch (error) {
|
||||
console.error("Error while force killing old task run processes", error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async #killCurrentTaskRunProcessBeforeAttempt() {
|
||||
console.log("killCurrentTaskRunProcessBeforeAttempt()", {
|
||||
hasTaskRunProcess: !!this._taskRunProcess,
|
||||
});
|
||||
|
||||
if (!this._taskRunProcess) {
|
||||
return;
|
||||
}
|
||||
|
||||
const currentTaskRunProcess = this._taskRunProcess;
|
||||
|
||||
console.log("Killing current task run process", {
|
||||
isBeingKilled: currentTaskRunProcess?.isBeingKilled,
|
||||
totalBeingKilled: this._taskRunProcessesBeingKilled.size,
|
||||
});
|
||||
|
||||
if (currentTaskRunProcess.isBeingKilled) {
|
||||
if (this._taskRunProcessesBeingKilled.size > 1) {
|
||||
await this.#tryGracefulExit(currentTaskRunProcess);
|
||||
} else {
|
||||
// If there's only one or none being killed, don't do anything so we can create a fresh one in parallel
|
||||
}
|
||||
} else {
|
||||
// It's not being killed, so kill it
|
||||
if (this._taskRunProcessesBeingKilled.size > 0) {
|
||||
await this.#tryGracefulExit(currentTaskRunProcess);
|
||||
} else {
|
||||
// There's none being killed yet, so we can kill it without waiting. We still set a timeout to kill it forcefully just in case it sticks around.
|
||||
currentTaskRunProcess.kill("SIGTERM", 5_000).catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async #tryGracefulExit(
|
||||
taskRunProcess: TaskRunProcess,
|
||||
kill = false,
|
||||
initialSignal: number | NodeJS.Signals = "SIGTERM"
|
||||
) {
|
||||
try {
|
||||
const taskRunProcess = await this.#initializeTaskRunProcess(payload);
|
||||
const initialExit = taskRunProcess.onExit.waitFor(5_000);
|
||||
|
||||
if (kill) {
|
||||
taskRunProcess.kill(initialSignal);
|
||||
}
|
||||
|
||||
await initialExit;
|
||||
} catch (error) {
|
||||
console.error("TaskRunProcess graceful kill timeout exceeded", error);
|
||||
|
||||
this.#tryForcefulExit(taskRunProcess);
|
||||
}
|
||||
}
|
||||
|
||||
async #tryForcefulExit(taskRunProcess: TaskRunProcess) {
|
||||
try {
|
||||
const forcedKill = taskRunProcess.onExit.waitFor(5_000);
|
||||
taskRunProcess.kill("SIGKILL");
|
||||
await forcedKill;
|
||||
} catch (error) {
|
||||
console.error("TaskRunProcess forced kill timeout exceeded", error);
|
||||
throw new SigKillTimeoutProcessError();
|
||||
}
|
||||
}
|
||||
|
||||
// We need to fork the process before we can execute any tasks, use a fresh process for each execution
|
||||
async executeTaskRun(
|
||||
payload: ProdTaskRunExecutionPayload,
|
||||
messageId?: string
|
||||
): Promise<TaskRunExecutionResult> {
|
||||
try {
|
||||
const taskRunProcess = await this.#getFreshTaskRunProcess(payload, messageId);
|
||||
|
||||
console.log("executing task run", {
|
||||
attempt: payload.execution.attempt.id,
|
||||
taskRunPid: taskRunProcess.pid,
|
||||
});
|
||||
|
||||
const result = await taskRunProcess.executeTaskRun(payload);
|
||||
|
||||
@@ -326,6 +464,31 @@ export class ProdBackgroundWorker {
|
||||
};
|
||||
}
|
||||
|
||||
if (e instanceof SigKillTimeoutProcessError) {
|
||||
return {
|
||||
id: payload.execution.attempt.id,
|
||||
ok: false,
|
||||
retry: undefined,
|
||||
error: {
|
||||
type: "INTERNAL_ERROR",
|
||||
code: TaskRunErrorCodes.TASK_PROCESS_SIGKILL_TIMEOUT,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
if (e instanceof GracefulExitTimeoutError) {
|
||||
return {
|
||||
id: payload.execution.attempt.id,
|
||||
ok: false,
|
||||
retry: undefined,
|
||||
error: {
|
||||
type: "INTERNAL_ERROR",
|
||||
code: TaskRunErrorCodes.GRACEFUL_EXIT_TIMEOUT,
|
||||
message: "Worker process killed while attempt in progress.",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
id: payload.execution.attempt.id,
|
||||
ok: false,
|
||||
@@ -335,11 +498,52 @@ export class ProdBackgroundWorker {
|
||||
code: TaskRunErrorCodes.TASK_EXECUTION_FAILED,
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
await this.#killTaskRunProcess();
|
||||
}
|
||||
}
|
||||
|
||||
async cancelAttempt(attemptId: string) {
|
||||
await this._taskRunProcess?.cancel();
|
||||
if (!this._taskRunProcess) {
|
||||
console.error("No task run process to cancel attempt", { attemptId });
|
||||
return;
|
||||
}
|
||||
|
||||
await this._taskRunProcess.cancel();
|
||||
}
|
||||
|
||||
async executeTaskRunLazyAttempt(payload: TaskRunExecutionLazyAttemptPayload) {
|
||||
// Post to coordinator
|
||||
this.onCreateTaskRunAttempt.post({ runId: payload.runId });
|
||||
|
||||
let execution: ProdTaskRunExecution;
|
||||
|
||||
try {
|
||||
// ..and wait for response
|
||||
const attemptCreated = await this.attemptCreatedNotification.waitFor(30_000);
|
||||
|
||||
if (!attemptCreated.success) {
|
||||
throw new Error(
|
||||
`Failed to create attempt${attemptCreated.reason ? `: ${attemptCreated.reason}` : ""}`
|
||||
);
|
||||
}
|
||||
|
||||
execution = attemptCreated.execution;
|
||||
} catch (error) {
|
||||
console.error("Error while creating attempt", error);
|
||||
throw new Error(`Failed to create task run attempt: ${error}`);
|
||||
}
|
||||
|
||||
const completion = await this.executeTaskRun(
|
||||
{
|
||||
execution,
|
||||
traceContext: payload.traceContext,
|
||||
environment: payload.environment,
|
||||
},
|
||||
payload.messageId
|
||||
);
|
||||
|
||||
return { execution, completion };
|
||||
}
|
||||
|
||||
async #correctError(
|
||||
@@ -359,6 +563,7 @@ class TaskRunProcess {
|
||||
typeof ProdWorkerToChildMessages
|
||||
>;
|
||||
private _child?: ChildProcess;
|
||||
private _childPid?: number;
|
||||
|
||||
private _attemptPromises: Map<
|
||||
string,
|
||||
@@ -368,9 +573,16 @@ class TaskRunProcess {
|
||||
private _currentExecution: TaskRunExecution | undefined;
|
||||
private _isBeingKilled: boolean = false;
|
||||
private _isBeingCancelled: boolean = false;
|
||||
private _gracefulExitTimeoutElapsed: boolean = false;
|
||||
|
||||
/**
|
||||
* @deprecated use onTaskRunHeartbeat instead
|
||||
*/
|
||||
public onTaskHeartbeat: Evt<string> = new Evt();
|
||||
public onExit: Evt<number> = new Evt();
|
||||
public onTaskRunHeartbeat: Evt<string> = new Evt();
|
||||
public onExit: Evt<{ code: number | null; signal: NodeJS.Signals | null; pid?: number }> =
|
||||
new Evt();
|
||||
public onIsBeingKilled: Evt<TaskRunProcess> = new Evt();
|
||||
|
||||
public onWaitForBatch: Evt<
|
||||
InferSocketMessageSchema<typeof ProdChildToWorkerMessages, "WAIT_FOR_BATCH">
|
||||
@@ -389,18 +601,20 @@ class TaskRunProcess {
|
||||
public onCancelCheckpoint = Evt.create<{ version?: "v1" | "v2"; reason?: WaitReason }>();
|
||||
|
||||
constructor(
|
||||
private execution: ProdTaskRunExecution,
|
||||
private runId: string,
|
||||
private isTest: boolean,
|
||||
private path: string,
|
||||
private env: NodeJS.ProcessEnv,
|
||||
private metadata: BackgroundWorkerProperties,
|
||||
private worker: BackgroundWorkerParams
|
||||
private worker: BackgroundWorkerParams,
|
||||
private messageId?: string
|
||||
) {}
|
||||
|
||||
async initialize() {
|
||||
this._child = fork(this.path, {
|
||||
stdio: [/*stdin*/ "ignore", /*stdout*/ "pipe", /*stderr*/ "pipe", "ipc"],
|
||||
env: {
|
||||
...(this.execution.run.isTest ? { TRIGGER_LOG_LEVEL: "debug" } : {}),
|
||||
...(this.isTest ? { TRIGGER_LOG_LEVEL: "debug" } : {}),
|
||||
...this.env,
|
||||
OTEL_RESOURCE_ATTRIBUTES: JSON.stringify({
|
||||
[SemanticInternalAttributes.PROJECT_DIR]: this.worker.projectConfig.projectDir,
|
||||
@@ -408,6 +622,7 @@ class TaskRunProcess {
|
||||
...(this.worker.debugOtel ? { OTEL_LOG_LEVEL: "debug" } : {}),
|
||||
},
|
||||
});
|
||||
this._childPid = this._child?.pid;
|
||||
|
||||
this._ipc = new ZodIpcConnection({
|
||||
listenSchema: ProdChildToWorkerMessages,
|
||||
@@ -439,7 +654,11 @@ class TaskRunProcess {
|
||||
process.exit(0);
|
||||
},
|
||||
TASK_HEARTBEAT: async (message) => {
|
||||
this.onTaskHeartbeat.post(message.id);
|
||||
if (this.messageId) {
|
||||
this.onTaskRunHeartbeat.post(this.messageId);
|
||||
} else {
|
||||
this.onTaskHeartbeat.post(message.id);
|
||||
}
|
||||
},
|
||||
TASKS_READY: async (message) => {},
|
||||
WAIT_FOR_TASK: async (message) => {
|
||||
@@ -513,17 +732,38 @@ class TaskRunProcess {
|
||||
await this.cleanup(true);
|
||||
}
|
||||
|
||||
async cleanup(kill: boolean = false) {
|
||||
async cleanup(kill = false, gracefulExitTimeoutElapsed = false) {
|
||||
console.log("cleanup()", { kill, gracefulExitTimeoutElapsed });
|
||||
|
||||
if (kill && this._isBeingKilled) {
|
||||
return;
|
||||
}
|
||||
|
||||
this._isBeingKilled = kill;
|
||||
if (kill) {
|
||||
this._isBeingKilled = true;
|
||||
this.onIsBeingKilled.post(this);
|
||||
}
|
||||
|
||||
const killChildProcess = gracefulExitTimeoutElapsed && !!this._currentExecution;
|
||||
|
||||
// Kill parent unless graceful exit timeout has elapsed and we're in the middle of an execution
|
||||
const killParentProcess = kill && !killChildProcess;
|
||||
|
||||
console.log("Cleaning up task run process", {
|
||||
killChildProcess,
|
||||
killParentProcess,
|
||||
});
|
||||
|
||||
await this._ipc?.sendWithAck("CLEANUP", {
|
||||
flush: true,
|
||||
kill,
|
||||
kill: killParentProcess,
|
||||
});
|
||||
|
||||
if (killChildProcess) {
|
||||
this._gracefulExitTimeoutElapsed = true;
|
||||
// Kill the child process
|
||||
await this.kill("SIGKILL");
|
||||
}
|
||||
}
|
||||
|
||||
async executeTaskRun(payload: TaskRunExecutionPayload): Promise<TaskRunExecutionResult> {
|
||||
@@ -559,15 +799,15 @@ class TaskRunProcess {
|
||||
return result;
|
||||
}
|
||||
|
||||
taskRunCompletedNotification(completion: TaskRunExecutionResult, execution: TaskRunExecution) {
|
||||
taskRunCompletedNotification(completion: TaskRunExecutionResult) {
|
||||
if (!completion.ok && typeof completion.retry !== "undefined") {
|
||||
return;
|
||||
}
|
||||
|
||||
if (this._child?.connected && !this._isBeingKilled && !this._child.killed) {
|
||||
this._ipc?.send("TASK_RUN_COMPLETED_NOTIFICATION", {
|
||||
version: "v2",
|
||||
completion,
|
||||
execution,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -578,10 +818,14 @@ class TaskRunProcess {
|
||||
}
|
||||
}
|
||||
|
||||
async #handleExit(code: number) {
|
||||
async #handleExit(code: number | null, signal: NodeJS.Signals | null) {
|
||||
console.log("handling child exit", { code, signal });
|
||||
|
||||
// Go through all the attempts currently pending and reject them
|
||||
for (const [id, status] of this._attemptStatuses.entries()) {
|
||||
if (status === "PENDING") {
|
||||
console.log("found pending attempt", { id });
|
||||
|
||||
this._attemptStatuses.set(id, "REJECTED");
|
||||
|
||||
const attemptPromise = this._attemptPromises.get(id);
|
||||
@@ -594,15 +838,18 @@ class TaskRunProcess {
|
||||
|
||||
if (this._isBeingCancelled) {
|
||||
rejecter(new CancelledProcessError());
|
||||
} else if (this._gracefulExitTimeoutElapsed) {
|
||||
// Order matters, this has to be before the graceful exit timeout
|
||||
rejecter(new GracefulExitTimeoutError());
|
||||
} else if (this._isBeingKilled) {
|
||||
rejecter(new CleanupProcessError());
|
||||
} else {
|
||||
rejecter(new UnexpectedExitError(code));
|
||||
rejecter(new UnexpectedExitError(code ?? -1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.onExit.post(code);
|
||||
this.onExit.post({ code, signal, pid: this.pid });
|
||||
}
|
||||
|
||||
#handleLog(data: Buffer) {
|
||||
@@ -635,9 +882,24 @@ class TaskRunProcess {
|
||||
);
|
||||
}
|
||||
|
||||
#kill() {
|
||||
if (this._child && !this._child.killed) {
|
||||
this._child?.kill();
|
||||
async kill(signal?: number | NodeJS.Signals, timeoutInMs?: number) {
|
||||
this._isBeingKilled = true;
|
||||
|
||||
const killTimeout = this.onExit.waitFor(timeoutInMs);
|
||||
|
||||
this.onIsBeingKilled.post(this);
|
||||
this._child?.kill(signal);
|
||||
|
||||
if (timeoutInMs) {
|
||||
await killTimeout;
|
||||
}
|
||||
}
|
||||
|
||||
get isBeingKilled() {
|
||||
return this._isBeingKilled || this._child?.killed;
|
||||
}
|
||||
|
||||
get pid() {
|
||||
return this._childPid;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
PreStopCauses,
|
||||
ProdWorkerToCoordinatorMessages,
|
||||
TaskResource,
|
||||
TaskRunFailedExecutionResult,
|
||||
WaitReason,
|
||||
} from "@trigger.dev/core/v3";
|
||||
import { ZodSocketConnection } from "@trigger.dev/core/v3/zodSocket";
|
||||
@@ -60,104 +61,7 @@ class ProdWorker {
|
||||
process.on("SIGTERM", this.#handleSignal.bind(this, "SIGTERM"));
|
||||
|
||||
this.#coordinatorSocket = this.#createCoordinatorSocket(COORDINATOR_HOST);
|
||||
|
||||
this.#backgroundWorker = new ProdBackgroundWorker("worker.js", {
|
||||
projectConfig: __PROJECT_CONFIG__,
|
||||
env: {
|
||||
...gatherProcessEnv(),
|
||||
TRIGGER_API_URL: this.apiUrl,
|
||||
TRIGGER_SECRET_KEY: this.apiKey,
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT:
|
||||
process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
|
||||
},
|
||||
contentHash: this.contentHash,
|
||||
});
|
||||
|
||||
this.#backgroundWorker.onTaskHeartbeat.attach((attemptFriendlyId) => {
|
||||
// TODO: Switch to .send() once coordinator uses zod handler for all messages
|
||||
this.#coordinatorSocket.socket.emit("TASK_HEARTBEAT", { version: "v1", attemptFriendlyId });
|
||||
});
|
||||
|
||||
this.#backgroundWorker.onReadyForCheckpoint.attach(async (message) => {
|
||||
// Flush before checkpointing so we don't flush the same spans again after restore
|
||||
await this.#backgroundWorker.flushTelemetry();
|
||||
this.#coordinatorSocket.socket.emit("READY_FOR_CHECKPOINT", { version: "v1" });
|
||||
});
|
||||
|
||||
// Currently, this is only used for duration waits. Might need adjusting for other use cases.
|
||||
this.#backgroundWorker.onCancelCheckpoint.attach(async (message) => {
|
||||
logger.log("onCancelCheckpoint", { message });
|
||||
|
||||
const { checkpointCanceled } = await this.#coordinatorSocket.socket.emitWithAck(
|
||||
"CANCEL_CHECKPOINT",
|
||||
{
|
||||
version: "v2",
|
||||
reason: message.reason,
|
||||
}
|
||||
);
|
||||
|
||||
if (checkpointCanceled) {
|
||||
if (message.reason === "WAIT_FOR_DURATION") {
|
||||
// Worker will resume immediately
|
||||
this.paused = false;
|
||||
this.nextResumeAfter = undefined;
|
||||
this.waitForPostStart = false;
|
||||
}
|
||||
}
|
||||
|
||||
this.#backgroundWorker.checkpointCanceledNotification.post({ checkpointCanceled });
|
||||
});
|
||||
|
||||
this.#backgroundWorker.onWaitForDuration.attach(async (message) => {
|
||||
if (!this.attemptFriendlyId) {
|
||||
logger.error("Failed to send wait message, attempt friendly ID not set", { message });
|
||||
return;
|
||||
}
|
||||
|
||||
const { willCheckpointAndRestore } = await this.#coordinatorSocket.socket.emitWithAck(
|
||||
"WAIT_FOR_DURATION",
|
||||
{
|
||||
...message,
|
||||
attemptFriendlyId: this.attemptFriendlyId,
|
||||
}
|
||||
);
|
||||
|
||||
this.#prepareForWait("WAIT_FOR_DURATION", willCheckpointAndRestore);
|
||||
});
|
||||
|
||||
this.#backgroundWorker.onWaitForTask.attach(async (message) => {
|
||||
if (!this.attemptFriendlyId) {
|
||||
logger.error("Failed to send wait message, attempt friendly ID not set", { message });
|
||||
return;
|
||||
}
|
||||
|
||||
const { willCheckpointAndRestore } = await this.#coordinatorSocket.socket.emitWithAck(
|
||||
"WAIT_FOR_TASK",
|
||||
{
|
||||
...message,
|
||||
attemptFriendlyId: this.attemptFriendlyId,
|
||||
}
|
||||
);
|
||||
|
||||
this.#prepareForWait("WAIT_FOR_TASK", willCheckpointAndRestore);
|
||||
});
|
||||
|
||||
this.#backgroundWorker.onWaitForBatch.attach(async (message) => {
|
||||
if (!this.attemptFriendlyId) {
|
||||
logger.error("Failed to send wait message, attempt friendly ID not set", { message });
|
||||
return;
|
||||
}
|
||||
|
||||
const { willCheckpointAndRestore } = await this.#coordinatorSocket.socket.emitWithAck(
|
||||
"WAIT_FOR_BATCH",
|
||||
{
|
||||
...message,
|
||||
attemptFriendlyId: this.attemptFriendlyId,
|
||||
}
|
||||
);
|
||||
|
||||
this.#prepareForWait("WAIT_FOR_BATCH", willCheckpointAndRestore);
|
||||
});
|
||||
this.#backgroundWorker = this.#createBackgroundWorker();
|
||||
|
||||
this.#httpPort = port;
|
||||
this.#httpServer = this.#createHttpServer();
|
||||
@@ -167,6 +71,8 @@ class ProdWorker {
|
||||
logger.log("Received signal", { signal });
|
||||
|
||||
if (signal === "SIGTERM") {
|
||||
let gracefulExitTimeoutElapsed = false;
|
||||
|
||||
if (this.executing) {
|
||||
const terminationGracePeriodSeconds = 60 * 60;
|
||||
|
||||
@@ -176,21 +82,27 @@ class ProdWorker {
|
||||
|
||||
// Wait for termination grace period minus 5s to give cleanup a chance to complete
|
||||
await setTimeout(terminationGracePeriodSeconds * 1000 - 5000);
|
||||
gracefulExitTimeoutElapsed = true;
|
||||
|
||||
logger.log("Termination timeout reached, exiting gracefully.");
|
||||
} else {
|
||||
logger.log("Not executing, exiting immediately.");
|
||||
}
|
||||
|
||||
await this.#exitGracefully();
|
||||
await this.#exitGracefully(gracefulExitTimeoutElapsed);
|
||||
return;
|
||||
}
|
||||
|
||||
logger.log("Unhandled signal", { signal });
|
||||
}
|
||||
|
||||
async #exitGracefully() {
|
||||
await this.#backgroundWorker.close();
|
||||
process.exit(0);
|
||||
async #exitGracefully(gracefulExitTimeoutElapsed = false) {
|
||||
await this.#backgroundWorker.close(gracefulExitTimeoutElapsed);
|
||||
|
||||
if (!gracefulExitTimeoutElapsed) {
|
||||
// TODO: Maybe add a sensible timeout instead of a conditional to avoid zombies
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
async #reconnect(isPostStart = false, reconnectImmediately = false) {
|
||||
@@ -222,12 +134,169 @@ class ProdWorker {
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error("taskinfo read error during reconnect", { error });
|
||||
logger.error("taskinfo read error during reconnect", {
|
||||
error: error instanceof Error ? error.message : error,
|
||||
});
|
||||
} finally {
|
||||
this.#coordinatorSocket = this.#createCoordinatorSocket(coordinatorHost);
|
||||
}
|
||||
}
|
||||
|
||||
#createBackgroundWorker() {
|
||||
const backgroundWorker = new ProdBackgroundWorker("worker.js", {
|
||||
projectConfig: __PROJECT_CONFIG__,
|
||||
env: {
|
||||
...gatherProcessEnv(),
|
||||
TRIGGER_API_URL: this.apiUrl,
|
||||
TRIGGER_SECRET_KEY: this.apiKey,
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT:
|
||||
process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
|
||||
},
|
||||
contentHash: this.contentHash,
|
||||
});
|
||||
|
||||
backgroundWorker.onTaskHeartbeat.attach((attemptFriendlyId) => {
|
||||
// TODO: Switch to .send() once coordinator uses zod handler for all messages
|
||||
this.#coordinatorSocket.socket.emit("TASK_HEARTBEAT", { version: "v1", attemptFriendlyId });
|
||||
});
|
||||
|
||||
backgroundWorker.onTaskRunHeartbeat.attach((runId) => {
|
||||
this.#coordinatorSocket.socket.emit("TASK_RUN_HEARTBEAT", { version: "v1", runId });
|
||||
});
|
||||
|
||||
// Currently, this is only used for duration waits
|
||||
backgroundWorker.onReadyForCheckpoint.attach(async (message) => {
|
||||
await this.#prepareForCheckpoint();
|
||||
|
||||
this.#coordinatorSocket.socket.emit("READY_FOR_CHECKPOINT", { version: "v1" });
|
||||
});
|
||||
|
||||
// Currently, this is only used for duration waits. Might need adjusting for other use cases.
|
||||
backgroundWorker.onCancelCheckpoint.attach(async (message) => {
|
||||
logger.log("onCancelCheckpoint", { message });
|
||||
|
||||
const { checkpointCanceled } = await this.#coordinatorSocket.socket.emitWithAck(
|
||||
"CANCEL_CHECKPOINT",
|
||||
{
|
||||
version: "v2",
|
||||
reason: message.reason,
|
||||
}
|
||||
);
|
||||
|
||||
logger.log("onCancelCheckpoint coordinator response", { checkpointCanceled });
|
||||
|
||||
if (checkpointCanceled) {
|
||||
if (message.reason === "WAIT_FOR_DURATION") {
|
||||
// Worker will resume immediately
|
||||
this.paused = false;
|
||||
this.nextResumeAfter = undefined;
|
||||
this.waitForPostStart = false;
|
||||
}
|
||||
}
|
||||
|
||||
backgroundWorker.checkpointCanceledNotification.post({ checkpointCanceled });
|
||||
});
|
||||
|
||||
backgroundWorker.onCreateTaskRunAttempt.attach(async (message) => {
|
||||
logger.log("onCreateTaskRunAttempt()", { message });
|
||||
|
||||
const createAttempt = await this.#coordinatorSocket.socket.emitWithAck(
|
||||
"CREATE_TASK_RUN_ATTEMPT",
|
||||
{
|
||||
version: "v1",
|
||||
runId: message.runId,
|
||||
}
|
||||
);
|
||||
|
||||
if (!createAttempt.success) {
|
||||
backgroundWorker.attemptCreatedNotification.post({
|
||||
success: false,
|
||||
reason: createAttempt.reason,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
backgroundWorker.attemptCreatedNotification.post({
|
||||
success: true,
|
||||
execution: createAttempt.executionPayload.execution,
|
||||
});
|
||||
});
|
||||
|
||||
backgroundWorker.attemptCreatedNotification.attach((message) => {
|
||||
if (!message.success) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Workers with lazy attempt support set their friendly ID here
|
||||
this.attemptFriendlyId = message.execution.attempt.id;
|
||||
});
|
||||
|
||||
backgroundWorker.onWaitForDuration.attach(async (message) => {
|
||||
if (!this.attemptFriendlyId) {
|
||||
logger.error("Failed to send wait message, attempt friendly ID not set", { message });
|
||||
|
||||
this.#emitUnrecoverableError(
|
||||
"NoAttemptId",
|
||||
"Attempt ID not set before waiting for duration"
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const { willCheckpointAndRestore } = await this.#coordinatorSocket.socket.emitWithAck(
|
||||
"WAIT_FOR_DURATION",
|
||||
{
|
||||
...message,
|
||||
attemptFriendlyId: this.attemptFriendlyId,
|
||||
}
|
||||
);
|
||||
|
||||
this.#prepareForWait("WAIT_FOR_DURATION", willCheckpointAndRestore);
|
||||
});
|
||||
|
||||
backgroundWorker.onWaitForTask.attach(async (message) => {
|
||||
if (!this.attemptFriendlyId) {
|
||||
logger.error("Failed to send wait message, attempt friendly ID not set", { message });
|
||||
|
||||
this.#emitUnrecoverableError("NoAttemptId", "Attempt ID not set before waiting for task");
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const { willCheckpointAndRestore } = await this.#coordinatorSocket.socket.emitWithAck(
|
||||
"WAIT_FOR_TASK",
|
||||
{
|
||||
...message,
|
||||
attemptFriendlyId: this.attemptFriendlyId,
|
||||
}
|
||||
);
|
||||
|
||||
this.#prepareForWait("WAIT_FOR_TASK", willCheckpointAndRestore);
|
||||
});
|
||||
|
||||
backgroundWorker.onWaitForBatch.attach(async (message) => {
|
||||
if (!this.attemptFriendlyId) {
|
||||
logger.error("Failed to send wait message, attempt friendly ID not set", { message });
|
||||
|
||||
this.#emitUnrecoverableError("NoAttemptId", "Attempt ID not set before waiting for batch");
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const { willCheckpointAndRestore } = await this.#coordinatorSocket.socket.emitWithAck(
|
||||
"WAIT_FOR_BATCH",
|
||||
{
|
||||
...message,
|
||||
attemptFriendlyId: this.attemptFriendlyId,
|
||||
}
|
||||
);
|
||||
|
||||
this.#prepareForWait("WAIT_FOR_BATCH", willCheckpointAndRestore);
|
||||
});
|
||||
|
||||
return backgroundWorker;
|
||||
}
|
||||
|
||||
async #prepareForWait(reason: WaitReason, willCheckpointAndRestore: boolean) {
|
||||
logger.log(`prepare for ${reason}`, { willCheckpointAndRestore });
|
||||
|
||||
@@ -239,9 +308,8 @@ class ProdWorker {
|
||||
this.waitForPostStart = true;
|
||||
|
||||
if (reason === "WAIT_FOR_TASK" || reason === "WAIT_FOR_BATCH") {
|
||||
// Flush before checkpointing so we don't flush the same spans again after restore
|
||||
// Duration waits do this via the "ready for checkpoint" event instead
|
||||
await this.#backgroundWorker.flushTelemetry();
|
||||
await this.#prepareForCheckpoint();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -256,18 +324,36 @@ class ProdWorker {
|
||||
}
|
||||
|
||||
await this.#exitGracefully();
|
||||
return;
|
||||
}
|
||||
|
||||
// Clear state for next execution
|
||||
this.paused = false;
|
||||
this.waitForPostStart = false;
|
||||
this.executing = false;
|
||||
this.attemptFriendlyId = undefined;
|
||||
|
||||
if (willCheckpointAndRestore) {
|
||||
this.waitForPostStart = true;
|
||||
|
||||
// We already flush after completion, so we don't need to do it here
|
||||
this.#prepareForCheckpoint(false);
|
||||
|
||||
this.#coordinatorSocket.socket.emit("READY_FOR_CHECKPOINT", { version: "v1" });
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
async #prepareForCheckpoint(flush = true) {
|
||||
if (flush) {
|
||||
// Flush before checkpointing so we don't flush the same spans again after restore
|
||||
await this.#backgroundWorker.flushTelemetry();
|
||||
}
|
||||
|
||||
// Kill the previous worker process to prevent large checkpoints
|
||||
await this.#backgroundWorker.forceKillOldTaskRunProcesses();
|
||||
}
|
||||
|
||||
#resumeAfterDuration() {
|
||||
this.paused = false;
|
||||
this.nextResumeAfter = undefined;
|
||||
@@ -303,11 +389,8 @@ class ProdWorker {
|
||||
extraHeaders["x-trigger-attempt-friendly-id"] = this.attemptFriendlyId;
|
||||
}
|
||||
|
||||
logger.log("connecting to coordinator", {
|
||||
host,
|
||||
port: COORDINATOR_PORT,
|
||||
extraHeaders,
|
||||
});
|
||||
logger.log(`connecting to coordinator: ${host}:${COORDINATOR_PORT}`);
|
||||
logger.debug(`connecting with extra headers`, { extraHeaders });
|
||||
|
||||
const coordinatorConnection = new ZodSocketConnection({
|
||||
namespace: "prod-worker",
|
||||
@@ -317,28 +400,14 @@ class ProdWorker {
|
||||
serverMessages: CoordinatorToProdWorkerMessages,
|
||||
extraHeaders,
|
||||
handlers: {
|
||||
RESUME_AFTER_DEPENDENCY: async (message) => {
|
||||
RESUME_AFTER_DEPENDENCY: async ({ completions }) => {
|
||||
if (!this.paused) {
|
||||
logger.error("worker not paused", {
|
||||
completions: message.completions,
|
||||
executions: message.executions,
|
||||
});
|
||||
logger.error("Failed to resume after dependency: Worker not paused");
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.completions.length !== message.executions.length) {
|
||||
logger.error("did not receive the same number of completions and executions", {
|
||||
completions: message.completions,
|
||||
executions: message.executions,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (message.completions.length === 0 || message.executions.length === 0) {
|
||||
logger.error("no completions or executions", {
|
||||
completions: message.completions,
|
||||
executions: message.executions,
|
||||
});
|
||||
if (completions.length === 0) {
|
||||
logger.error("Failed to resume after dependency: No completions");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -346,17 +415,19 @@ class ProdWorker {
|
||||
this.nextResumeAfter !== "WAIT_FOR_TASK" &&
|
||||
this.nextResumeAfter !== "WAIT_FOR_BATCH"
|
||||
) {
|
||||
logger.error("not waiting to resume after dependency", {
|
||||
logger.error("Failed to resume after dependency: Invalid next resume", {
|
||||
nextResumeAfter: this.nextResumeAfter,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.nextResumeAfter === "WAIT_FOR_TASK" && message.completions.length > 1) {
|
||||
logger.error("waiting for single task but got multiple completions", {
|
||||
completions: message.completions,
|
||||
executions: message.executions,
|
||||
});
|
||||
if (this.nextResumeAfter === "WAIT_FOR_TASK" && completions.length > 1) {
|
||||
logger.error(
|
||||
"Failed to resume after dependency: Waiting for single task but got multiple completions",
|
||||
{
|
||||
completions: completions,
|
||||
}
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -364,13 +435,12 @@ class ProdWorker {
|
||||
this.nextResumeAfter = undefined;
|
||||
this.waitForPostStart = false;
|
||||
|
||||
for (let i = 0; i < message.completions.length; i++) {
|
||||
const completion = message.completions[i];
|
||||
const execution = message.executions[i];
|
||||
for (let i = 0; i < completions.length; i++) {
|
||||
const completion = completions[i];
|
||||
|
||||
if (!completion || !execution) continue;
|
||||
if (!completion) continue;
|
||||
|
||||
this.#backgroundWorker.taskRunCompletedNotification(completion, execution);
|
||||
this.#backgroundWorker.taskRunCompletedNotification(completion);
|
||||
}
|
||||
},
|
||||
RESUME_AFTER_DURATION: async (message) => {
|
||||
@@ -420,14 +490,75 @@ class ProdWorker {
|
||||
|
||||
this.#prepareForRetry(willCheckpointAndRestore, shouldExit);
|
||||
},
|
||||
REQUEST_ATTEMPT_CANCELLATION: async (message) => {
|
||||
if (!this.executing) {
|
||||
EXECUTE_TASK_RUN_LAZY_ATTEMPT: async (message) => {
|
||||
if (this.executing) {
|
||||
logger.error("dropping execute request, already executing");
|
||||
return;
|
||||
}
|
||||
|
||||
this.executing = true;
|
||||
|
||||
try {
|
||||
const { completion, execution } =
|
||||
await this.#backgroundWorker.executeTaskRunLazyAttempt(message.lazyPayload);
|
||||
|
||||
logger.log("completed", completion);
|
||||
|
||||
this.completed.add(execution.attempt.id);
|
||||
|
||||
const { willCheckpointAndRestore, shouldExit } =
|
||||
await this.#coordinatorSocket.socket.emitWithAck("TASK_RUN_COMPLETED", {
|
||||
version: "v1",
|
||||
execution,
|
||||
completion,
|
||||
});
|
||||
|
||||
logger.log("completion acknowledged", { willCheckpointAndRestore, shouldExit });
|
||||
|
||||
this.#prepareForRetry(willCheckpointAndRestore, shouldExit);
|
||||
} catch (error) {
|
||||
const completion: TaskRunFailedExecutionResult = {
|
||||
ok: false,
|
||||
id: message.lazyPayload.runId,
|
||||
retry: undefined,
|
||||
error:
|
||||
error instanceof Error
|
||||
? {
|
||||
type: "BUILT_IN_ERROR",
|
||||
name: error.name,
|
||||
message: error.message,
|
||||
stackTrace: error.stack ?? "",
|
||||
}
|
||||
: {
|
||||
type: "BUILT_IN_ERROR",
|
||||
name: "UnknownError",
|
||||
message: String(error),
|
||||
stackTrace: "",
|
||||
},
|
||||
};
|
||||
|
||||
this.#coordinatorSocket.socket.emit("TASK_RUN_FAILED_TO_RUN", {
|
||||
version: "v1",
|
||||
completion,
|
||||
});
|
||||
}
|
||||
},
|
||||
REQUEST_ATTEMPT_CANCELLATION: async (message) => {
|
||||
if (!this.executing) {
|
||||
logger.log("dropping cancel request, not executing", { status: this.#status });
|
||||
return;
|
||||
}
|
||||
|
||||
logger.log("cancelling attempt", { attemptId: message.attemptId, status: this.#status });
|
||||
|
||||
await this.#backgroundWorker.cancelAttempt(message.attemptId);
|
||||
},
|
||||
REQUEST_EXIT: async () => {
|
||||
REQUEST_EXIT: async (message) => {
|
||||
if (message.version === "v2" && message.delayInMs) {
|
||||
logger.log("exit requested with delay", { delayInMs: message.delayInMs });
|
||||
await setTimeout(message.delayInMs);
|
||||
}
|
||||
|
||||
this.#coordinatorSocket.close();
|
||||
process.exit(0);
|
||||
},
|
||||
@@ -436,7 +567,7 @@ class ProdWorker {
|
||||
return;
|
||||
}
|
||||
|
||||
this.#coordinatorSocket.socket.emit("READY_FOR_EXECUTION", {
|
||||
this.#coordinatorSocket.socket.emit("READY_FOR_LAZY_ATTEMPT", {
|
||||
version: "v1",
|
||||
runId: this.runId,
|
||||
totalCompletions: this.completed.size,
|
||||
@@ -444,6 +575,8 @@ class ProdWorker {
|
||||
},
|
||||
},
|
||||
onConnection: async (socket, handler, sender, logger) => {
|
||||
logger.log("connected to coordinator", { status: this.#status });
|
||||
|
||||
if (this.waitForPostStart) {
|
||||
logger.log("skip connection handler, waiting for post start hook");
|
||||
return;
|
||||
@@ -451,11 +584,24 @@ class ProdWorker {
|
||||
|
||||
if (this.paused) {
|
||||
if (!this.nextResumeAfter) {
|
||||
logger.error("Missing next resume reason", { status: this.#status });
|
||||
|
||||
this.#emitUnrecoverableError(
|
||||
"NoNextResume",
|
||||
"Next resume reason not set while resuming from paused state"
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (!this.attemptFriendlyId) {
|
||||
logger.error("Missing friendly ID");
|
||||
logger.error("Missing friendly ID", { status: this.#status });
|
||||
|
||||
this.#emitUnrecoverableError(
|
||||
"NoAttemptId",
|
||||
"Attempt ID not set while resuming from paused state"
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -473,9 +619,10 @@ class ProdWorker {
|
||||
const taskResources = await this.#initializeWorker();
|
||||
|
||||
const { success } = await socket.emitWithAck("INDEX_TASKS", {
|
||||
version: "v1",
|
||||
version: "v2",
|
||||
deploymentId: this.deploymentId,
|
||||
...taskResources,
|
||||
supportsLazyAttempts: true,
|
||||
});
|
||||
|
||||
if (success) {
|
||||
@@ -563,7 +710,7 @@ class ProdWorker {
|
||||
return;
|
||||
}
|
||||
|
||||
socket.emit("READY_FOR_EXECUTION", {
|
||||
socket.emit("READY_FOR_LAZY_ATTEMPT", {
|
||||
version: "v1",
|
||||
runId: this.runId,
|
||||
totalCompletions: this.completed.size,
|
||||
@@ -601,12 +748,7 @@ class ProdWorker {
|
||||
}
|
||||
|
||||
case "/status": {
|
||||
return reply.json({
|
||||
executing: this.executing,
|
||||
paused: this.paused,
|
||||
completed: this.completed.size,
|
||||
nextResumeAfter: this.nextResumeAfter,
|
||||
});
|
||||
return reply.json(this.#status);
|
||||
}
|
||||
|
||||
case "/connect": {
|
||||
@@ -768,6 +910,27 @@ class ProdWorker {
|
||||
return data?.variables ?? {};
|
||||
}
|
||||
|
||||
get #status() {
|
||||
return {
|
||||
executing: this.executing,
|
||||
paused: this.paused,
|
||||
completed: this.completed.size,
|
||||
nextResumeAfter: this.nextResumeAfter,
|
||||
waitForPostStart: this.waitForPostStart,
|
||||
attemptFriendlyId: this.attemptFriendlyId,
|
||||
};
|
||||
}
|
||||
|
||||
#emitUnrecoverableError(name: string, message: string) {
|
||||
this.#coordinatorSocket.socket.emit("UNRECOVERABLE_ERROR", {
|
||||
version: "v1",
|
||||
error: {
|
||||
name,
|
||||
message,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
start() {
|
||||
this.#httpServer.listen(this.#httpPort, this.host);
|
||||
}
|
||||
|
||||
@@ -170,8 +170,8 @@ const zodIpc = new ZodIpcConnection({
|
||||
_isRunning = false;
|
||||
}
|
||||
},
|
||||
TASK_RUN_COMPLETED_NOTIFICATION: async ({ completion, execution }) => {
|
||||
prodRuntimeManager.resumeTask(completion, execution);
|
||||
TASK_RUN_COMPLETED_NOTIFICATION: async ({ completion }) => {
|
||||
prodRuntimeManager.resumeTask(completion);
|
||||
},
|
||||
WAIT_COMPLETED_NOTIFICATION: async () => {
|
||||
prodRuntimeManager.resumeAfterDuration();
|
||||
@@ -179,23 +179,6 @@ const zodIpc = new ZodIpcConnection({
|
||||
CLEANUP: async ({ flush, kill }, sender) => {
|
||||
if (kill) {
|
||||
await tracingSDK.flush();
|
||||
|
||||
if (_execution) {
|
||||
// Fail currently executing attempt
|
||||
await sender.send("TASK_RUN_COMPLETED", {
|
||||
execution: _execution,
|
||||
result: {
|
||||
ok: false,
|
||||
id: _execution.run.id,
|
||||
error: {
|
||||
type: "INTERNAL_ERROR",
|
||||
code: TaskRunErrorCodes.GRACEFUL_EXIT_TIMEOUT,
|
||||
message: "Worker process killed while attempt in progress.",
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Now we need to exit the process
|
||||
await sender.send("READY_TO_DISPOSE", undefined);
|
||||
} else {
|
||||
@@ -228,7 +211,7 @@ zodIpc.send("TASKS_READY", { tasks: TASK_METADATA }).catch((err) => {
|
||||
|
||||
process.title = "trigger-prod-worker";
|
||||
|
||||
async function asyncHeartbeat(initialDelayInSeconds: number = 30, intervalInSeconds: number = 5) {
|
||||
async function asyncHeartbeat(initialDelayInSeconds: number = 30, intervalInSeconds: number = 20) {
|
||||
async function _doHeartbeat() {
|
||||
while (true) {
|
||||
if (_isRunning && _execution) {
|
||||
|
||||
@@ -46,7 +46,6 @@ export interface TaskOperationsCreateOptions {
|
||||
orgId: string;
|
||||
projectId: string;
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
}
|
||||
|
||||
export interface TaskOperationsRestoreOptions {
|
||||
@@ -129,7 +128,6 @@ export class ProviderShell implements Provider {
|
||||
orgId: message.data.orgId,
|
||||
projectId: message.data.projectId,
|
||||
runId: message.data.runId,
|
||||
attemptId: message.data.id,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error("create failed", error);
|
||||
|
||||
@@ -37,6 +37,14 @@
|
||||
"require": "./dist/v3/otel/index.js",
|
||||
"types": "./dist/v3/otel/index.d.ts"
|
||||
},
|
||||
"./v3/zodfetch": {
|
||||
"import": {
|
||||
"types": "./dist/v3/zodfetch.d.mts",
|
||||
"default": "./dist/v3/zodfetch.mjs"
|
||||
},
|
||||
"require": "./dist/v3/zodfetch.js",
|
||||
"types": "./dist/v3/zodfetch.d.ts"
|
||||
},
|
||||
"./v3/zodMessageHandler": {
|
||||
"import": {
|
||||
"types": "./dist/v3/zodMessageHandler.d.mts",
|
||||
|
||||
@@ -84,13 +84,13 @@ export function createJsonErrorObject(error: TaskRunError) {
|
||||
export function correctErrorStackTrace(
|
||||
stackTrace: string,
|
||||
projectDir?: string,
|
||||
options?: { removeFirstLine?: boolean }
|
||||
options?: { removeFirstLine?: boolean; isDev?: boolean }
|
||||
) {
|
||||
const [errorLine, ...traceLines] = stackTrace.split("\n");
|
||||
|
||||
return [
|
||||
options?.removeFirstLine ? undefined : errorLine,
|
||||
...traceLines.map((line) => correctStackTraceLine(line, projectDir)),
|
||||
...traceLines.map((line) => correctStackTraceLine(line, projectDir, options?.isDev)),
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
@@ -102,17 +102,21 @@ const LINES_TO_IGNORE = [
|
||||
/TaskExecutor/,
|
||||
/EXECUTE_TASK_RUN/,
|
||||
/@trigger.dev\/core/,
|
||||
/packages\/core\/src\/v3/,
|
||||
/safeJsonProcess/,
|
||||
/__entryPoint.ts/,
|
||||
/ZodIpc/,
|
||||
/startActiveSpan/,
|
||||
/processTicksAndRejections/,
|
||||
];
|
||||
|
||||
function correctStackTraceLine(line: string, projectDir?: string) {
|
||||
function correctStackTraceLine(line: string, projectDir?: string, isDev?: boolean) {
|
||||
if (LINES_TO_IGNORE.some((regex) => regex.test(line))) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check to see if the path is inside the project directory
|
||||
if (projectDir && !line.includes(projectDir)) {
|
||||
if (isDev && projectDir && !line.includes(projectDir)) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -80,18 +80,18 @@ export class DevRuntimeManager implements RuntimeManager {
|
||||
};
|
||||
}
|
||||
|
||||
resumeTask(completion: TaskRunExecutionResult, execution: TaskRunExecution): void {
|
||||
const wait = this._taskWaits.get(execution.run.id);
|
||||
resumeTask(completion: TaskRunExecutionResult, runId: string): void {
|
||||
const wait = this._taskWaits.get(runId);
|
||||
|
||||
if (!wait) {
|
||||
// We need to store the completion in case the task is awaited later
|
||||
this._pendingCompletionNotifications.set(execution.run.id, completion);
|
||||
this._pendingCompletionNotifications.set(runId, completion);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
wait.resolve(completion);
|
||||
|
||||
this._taskWaits.delete(execution.run.id);
|
||||
this._taskWaits.delete(runId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,10 +55,14 @@ export class ProdRuntimeManager implements RuntimeManager {
|
||||
this._waitForDuration = { resolve, reject };
|
||||
});
|
||||
|
||||
const { willCheckpointAndRestore } = await this.ipc.sendWithAck("WAIT_FOR_DURATION", {
|
||||
ms,
|
||||
now,
|
||||
});
|
||||
const { willCheckpointAndRestore } = await this.ipc.sendWithAck(
|
||||
"WAIT_FOR_DURATION",
|
||||
{
|
||||
ms,
|
||||
now,
|
||||
},
|
||||
31_000
|
||||
);
|
||||
|
||||
if (!willCheckpointAndRestore) {
|
||||
await internalTimeout;
|
||||
@@ -74,18 +78,24 @@ export class ProdRuntimeManager implements RuntimeManager {
|
||||
// Resets the clock to the current time
|
||||
clock.reset();
|
||||
|
||||
// The coordinator should cancel any in-progress checkpoints
|
||||
const { checkpointCanceled, version } = await this.ipc.sendWithAck(
|
||||
"CANCEL_CHECKPOINT",
|
||||
{
|
||||
version: "v2",
|
||||
reason: "WAIT_FOR_DURATION",
|
||||
},
|
||||
31_000
|
||||
);
|
||||
try {
|
||||
// The coordinator should cancel any in-progress checkpoints
|
||||
const { checkpointCanceled, version } = await this.ipc.sendWithAck(
|
||||
"CANCEL_CHECKPOINT",
|
||||
{
|
||||
version: "v2",
|
||||
reason: "WAIT_FOR_DURATION",
|
||||
},
|
||||
31_000
|
||||
);
|
||||
|
||||
if (checkpointCanceled) {
|
||||
// There won't be a checkpoint or external resume and we've already completed our internal timeout
|
||||
if (checkpointCanceled) {
|
||||
// There won't be a checkpoint or external resume and we've already completed our internal timeout
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
// If the cancellation times out, we will proceed as if the checkpoint was canceled
|
||||
logger.debug("Checkpoint cancellation timed out", { error });
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -98,19 +108,9 @@ export class ProdRuntimeManager implements RuntimeManager {
|
||||
return;
|
||||
}
|
||||
|
||||
process.stdout.write("pre");
|
||||
process.stdout.write(JSON.stringify(clock.preciseNow()));
|
||||
|
||||
console.log("pre", clock.preciseNow());
|
||||
|
||||
// Resets the clock to the current time
|
||||
clock.reset();
|
||||
|
||||
console.log("post", clock.preciseNow());
|
||||
|
||||
process.stdout.write("post");
|
||||
process.stdout.write(JSON.stringify(clock.preciseNow()));
|
||||
|
||||
this._waitForDuration.resolve("external");
|
||||
this._waitForDuration = undefined;
|
||||
}
|
||||
@@ -167,8 +167,8 @@ export class ProdRuntimeManager implements RuntimeManager {
|
||||
};
|
||||
}
|
||||
|
||||
resumeTask(completion: TaskRunExecutionResult, execution: TaskRunExecution): void {
|
||||
const wait = this._taskWaits.get(execution.run.id);
|
||||
resumeTask(completion: TaskRunExecutionResult): void {
|
||||
const wait = this._taskWaits.get(completion.id);
|
||||
|
||||
if (!wait) {
|
||||
return;
|
||||
@@ -176,7 +176,7 @@ export class ProdRuntimeManager implements RuntimeManager {
|
||||
|
||||
wait.resolve(completion);
|
||||
|
||||
this._taskWaits.delete(execution.run.id);
|
||||
this._taskWaits.delete(completion.id);
|
||||
}
|
||||
|
||||
private get waitThresholdInMs(): number {
|
||||
|
||||
@@ -41,6 +41,7 @@ export type GetProjectEnvResponse = z.infer<typeof GetProjectEnvResponse>;
|
||||
export const CreateBackgroundWorkerRequestBody = z.object({
|
||||
localOnly: z.boolean(),
|
||||
metadata: BackgroundWorkerMetadata,
|
||||
supportsLazyAttempts: z.boolean().optional(),
|
||||
});
|
||||
|
||||
export type CreateBackgroundWorkerRequestBody = z.infer<typeof CreateBackgroundWorkerRequestBody>;
|
||||
|
||||
@@ -31,6 +31,7 @@ export const TaskRunErrorCodes = {
|
||||
TASK_EXECUTION_FAILED: "TASK_EXECUTION_FAILED",
|
||||
TASK_EXECUTION_ABORTED: "TASK_EXECUTION_ABORTED",
|
||||
TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE: "TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE",
|
||||
TASK_PROCESS_SIGKILL_TIMEOUT: "TASK_PROCESS_SIGKILL_TIMEOUT",
|
||||
TASK_RUN_CANCELLED: "TASK_RUN_CANCELLED",
|
||||
TASK_OUTPUT_ERROR: "TASK_OUTPUT_ERROR",
|
||||
HANDLE_ERROR_ERROR: "HANDLE_ERROR_ERROR",
|
||||
@@ -47,10 +48,12 @@ export const TaskRunInternalError = z.object({
|
||||
"TASK_EXECUTION_FAILED",
|
||||
"TASK_EXECUTION_ABORTED",
|
||||
"TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE",
|
||||
"TASK_PROCESS_SIGKILL_TIMEOUT",
|
||||
"TASK_RUN_CANCELLED",
|
||||
"TASK_OUTPUT_ERROR",
|
||||
"HANDLE_ERROR_ERROR",
|
||||
"GRACEFUL_EXIT_TIMEOUT",
|
||||
"TASK_RUN_HEARTBEAT_TIMEOUT",
|
||||
]),
|
||||
message: z.string().optional(),
|
||||
});
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
import { z } from "zod";
|
||||
import { TaskRunExecution, TaskRunExecutionResult } from "./common";
|
||||
import { TaskRunExecution, TaskRunExecutionResult, TaskRunFailedExecutionResult } from "./common";
|
||||
|
||||
import {
|
||||
EnvironmentType,
|
||||
Machine,
|
||||
ProdTaskRunExecution,
|
||||
ProdTaskRunExecutionPayload,
|
||||
TaskMetadataWithFilePath,
|
||||
TaskRunExecutionLazyAttemptPayload,
|
||||
TaskRunExecutionPayload,
|
||||
WaitReason,
|
||||
} from "./schemas";
|
||||
@@ -27,13 +29,17 @@ export const BackgroundWorkerServerMessages = z.discriminatedUnion("type", [
|
||||
version: z.string(),
|
||||
machine: Machine,
|
||||
// identifiers
|
||||
id: z.string(), // attempt
|
||||
id: z.string().optional(), // TODO: Remove this completely in a future release
|
||||
envId: z.string(),
|
||||
envType: EnvironmentType,
|
||||
orgId: z.string(),
|
||||
projectId: z.string(),
|
||||
runId: z.string(),
|
||||
}),
|
||||
z.object({
|
||||
type: z.literal("EXECUTE_RUN_LAZY_ATTEMPT"),
|
||||
payload: TaskRunExecutionLazyAttemptPayload,
|
||||
}),
|
||||
]);
|
||||
|
||||
export type BackgroundWorkerServerMessages = z.infer<typeof BackgroundWorkerServerMessages>;
|
||||
@@ -57,11 +63,21 @@ export const BackgroundWorkerClientMessages = z.discriminatedUnion("type", [
|
||||
completion: TaskRunExecutionResult,
|
||||
execution: TaskRunExecution,
|
||||
}),
|
||||
z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
type: z.literal("TASK_RUN_FAILED_TO_RUN"),
|
||||
completion: TaskRunFailedExecutionResult,
|
||||
}),
|
||||
z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
type: z.literal("TASK_HEARTBEAT"),
|
||||
id: z.string(),
|
||||
}),
|
||||
z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
type: z.literal("TASK_RUN_HEARTBEAT"),
|
||||
id: z.string(),
|
||||
}),
|
||||
]);
|
||||
|
||||
export type BackgroundWorkerClientMessages = z.infer<typeof BackgroundWorkerClientMessages>;
|
||||
@@ -78,6 +94,7 @@ export const clientWebsocketMessages = {
|
||||
READY_FOR_TASKS: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
backgroundWorkerId: z.string(),
|
||||
inProgressRuns: z.string().array().optional(),
|
||||
}),
|
||||
BACKGROUND_WORKER_DEPRECATED: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
@@ -97,11 +114,17 @@ export const workerToChildMessages = {
|
||||
traceContext: z.record(z.unknown()),
|
||||
metadata: BackgroundWorkerProperties,
|
||||
}),
|
||||
TASK_RUN_COMPLETED_NOTIFICATION: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
completion: TaskRunExecutionResult,
|
||||
execution: TaskRunExecution,
|
||||
}),
|
||||
TASK_RUN_COMPLETED_NOTIFICATION: z.discriminatedUnion("version", [
|
||||
z.object({
|
||||
version: z.literal("v1"),
|
||||
completion: TaskRunExecutionResult,
|
||||
execution: TaskRunExecution,
|
||||
}),
|
||||
z.object({
|
||||
version: z.literal("v2"),
|
||||
completion: TaskRunExecutionResult,
|
||||
}),
|
||||
]),
|
||||
CLEANUP: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
flush: z.boolean().default(false),
|
||||
@@ -142,6 +165,10 @@ export const childToWorkerMessages = {
|
||||
version: z.literal("v1").default("v1"),
|
||||
id: z.string(),
|
||||
}),
|
||||
TASK_RUN_HEARTBEAT: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
id: z.string(),
|
||||
}),
|
||||
READY_TO_DISPOSE: z.undefined(),
|
||||
WAIT_FOR_DURATION: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
@@ -182,6 +209,12 @@ export const ProdChildToWorkerMessages = {
|
||||
id: z.string(),
|
||||
}),
|
||||
},
|
||||
TASK_RUN_HEARTBEAT: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
id: z.string(),
|
||||
}),
|
||||
},
|
||||
READY_TO_DISPOSE: {
|
||||
message: z.undefined(),
|
||||
},
|
||||
@@ -247,11 +280,17 @@ export const ProdWorkerToChildMessages = {
|
||||
}),
|
||||
},
|
||||
TASK_RUN_COMPLETED_NOTIFICATION: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
completion: TaskRunExecutionResult,
|
||||
execution: TaskRunExecution,
|
||||
}),
|
||||
message: z.discriminatedUnion("version", [
|
||||
z.object({
|
||||
version: z.literal("v1"),
|
||||
completion: TaskRunExecutionResult,
|
||||
execution: TaskRunExecution,
|
||||
}),
|
||||
z.object({
|
||||
version: z.literal("v2"),
|
||||
completion: TaskRunExecutionResult,
|
||||
}),
|
||||
]),
|
||||
},
|
||||
CLEANUP: {
|
||||
message: z.object({
|
||||
@@ -379,6 +418,18 @@ export const PlatformToProviderMessages = {
|
||||
},
|
||||
};
|
||||
|
||||
const CreateWorkerMessage = z.object({
|
||||
projectRef: z.string(),
|
||||
envId: z.string(),
|
||||
deploymentId: z.string(),
|
||||
metadata: z.object({
|
||||
cliPackageVersion: z.string().optional(),
|
||||
contentHash: z.string(),
|
||||
packageVersion: z.string(),
|
||||
tasks: TaskResource.array(),
|
||||
}),
|
||||
});
|
||||
|
||||
export const CoordinatorToPlatformMessages = {
|
||||
LOG: {
|
||||
message: z.object({
|
||||
@@ -388,18 +439,15 @@ export const CoordinatorToPlatformMessages = {
|
||||
}),
|
||||
},
|
||||
CREATE_WORKER: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
projectRef: z.string(),
|
||||
envId: z.string(),
|
||||
deploymentId: z.string(),
|
||||
metadata: z.object({
|
||||
cliPackageVersion: z.string().optional(),
|
||||
contentHash: z.string(),
|
||||
packageVersion: z.string(),
|
||||
tasks: TaskResource.array(),
|
||||
message: z.discriminatedUnion("version", [
|
||||
CreateWorkerMessage.extend({
|
||||
version: z.literal("v1"),
|
||||
}),
|
||||
}),
|
||||
CreateWorkerMessage.extend({
|
||||
version: z.literal("v2"),
|
||||
supportsLazyAttempts: z.boolean(),
|
||||
}),
|
||||
]),
|
||||
callback: z.discriminatedUnion("success", [
|
||||
z.object({
|
||||
success: z.literal(false),
|
||||
@@ -409,6 +457,23 @@ export const CoordinatorToPlatformMessages = {
|
||||
}),
|
||||
]),
|
||||
},
|
||||
CREATE_TASK_RUN_ATTEMPT: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
runId: z.string(),
|
||||
envId: z.string(),
|
||||
}),
|
||||
callback: z.discriminatedUnion("success", [
|
||||
z.object({
|
||||
success: z.literal(false),
|
||||
reason: z.string().optional(),
|
||||
}),
|
||||
z.object({
|
||||
success: z.literal(true),
|
||||
executionPayload: ProdTaskRunExecutionPayload,
|
||||
}),
|
||||
]),
|
||||
},
|
||||
READY_FOR_EXECUTION: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
@@ -425,6 +490,24 @@ export const CoordinatorToPlatformMessages = {
|
||||
}),
|
||||
]),
|
||||
},
|
||||
READY_FOR_LAZY_ATTEMPT: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
runId: z.string(),
|
||||
envId: z.string(),
|
||||
totalCompletions: z.number(),
|
||||
}),
|
||||
callback: z.discriminatedUnion("success", [
|
||||
z.object({
|
||||
success: z.literal(false),
|
||||
reason: z.string().optional(),
|
||||
}),
|
||||
z.object({
|
||||
success: z.literal(true),
|
||||
lazyPayload: TaskRunExecutionLazyAttemptPayload,
|
||||
}),
|
||||
]),
|
||||
},
|
||||
READY_FOR_RESUME: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
@@ -445,12 +528,24 @@ export const CoordinatorToPlatformMessages = {
|
||||
.optional(),
|
||||
}),
|
||||
},
|
||||
TASK_RUN_FAILED_TO_RUN: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
completion: TaskRunFailedExecutionResult,
|
||||
}),
|
||||
},
|
||||
TASK_HEARTBEAT: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
attemptFriendlyId: z.string(),
|
||||
}),
|
||||
},
|
||||
TASK_RUN_HEARTBEAT: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
runId: z.string(),
|
||||
}),
|
||||
},
|
||||
CHECKPOINT_CREATED: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
@@ -490,6 +585,17 @@ export const CoordinatorToPlatformMessages = {
|
||||
}),
|
||||
}),
|
||||
},
|
||||
RUN_CRASHED: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
runId: z.string(),
|
||||
error: z.object({
|
||||
name: z.string(),
|
||||
message: z.string(),
|
||||
stack: z.string().optional(),
|
||||
}),
|
||||
}),
|
||||
},
|
||||
};
|
||||
|
||||
export const PlatformToCoordinatorMessages = {
|
||||
@@ -517,6 +623,13 @@ export const PlatformToCoordinatorMessages = {
|
||||
attemptFriendlyId: z.string(),
|
||||
}),
|
||||
},
|
||||
REQUEST_RUN_CANCELLATION: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
runId: z.string(),
|
||||
delayInMs: z.number().optional(),
|
||||
}),
|
||||
},
|
||||
READY_FOR_RETRY: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
@@ -563,6 +676,13 @@ export const SharedQueueToClientMessages = {
|
||||
},
|
||||
};
|
||||
|
||||
const IndexTasksMessage = z.object({
|
||||
version: z.literal("v1"),
|
||||
deploymentId: z.string(),
|
||||
tasks: TaskResource.array(),
|
||||
packageVersion: z.string(),
|
||||
});
|
||||
|
||||
export const ProdWorkerToCoordinatorMessages = {
|
||||
LOG: {
|
||||
message: z.object({
|
||||
@@ -572,12 +692,15 @@ export const ProdWorkerToCoordinatorMessages = {
|
||||
callback: z.void(),
|
||||
},
|
||||
INDEX_TASKS: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
deploymentId: z.string(),
|
||||
tasks: TaskResource.array(),
|
||||
packageVersion: z.string(),
|
||||
}),
|
||||
message: z.discriminatedUnion("version", [
|
||||
IndexTasksMessage.extend({
|
||||
version: z.literal("v1"),
|
||||
}),
|
||||
IndexTasksMessage.extend({
|
||||
version: z.literal("v2"),
|
||||
supportsLazyAttempts: z.boolean(),
|
||||
}),
|
||||
]),
|
||||
callback: z.discriminatedUnion("success", [
|
||||
z.object({
|
||||
success: z.literal(false),
|
||||
@@ -594,6 +717,13 @@ export const ProdWorkerToCoordinatorMessages = {
|
||||
totalCompletions: z.number(),
|
||||
}),
|
||||
},
|
||||
READY_FOR_LAZY_ATTEMPT: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
runId: z.string(),
|
||||
totalCompletions: z.number(),
|
||||
}),
|
||||
},
|
||||
READY_FOR_RESUME: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
@@ -630,6 +760,12 @@ export const ProdWorkerToCoordinatorMessages = {
|
||||
attemptFriendlyId: z.string(),
|
||||
}),
|
||||
},
|
||||
TASK_RUN_HEARTBEAT: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
runId: z.string(),
|
||||
}),
|
||||
},
|
||||
TASK_RUN_COMPLETED: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
@@ -641,6 +777,12 @@ export const ProdWorkerToCoordinatorMessages = {
|
||||
shouldExit: z.boolean(),
|
||||
}),
|
||||
},
|
||||
TASK_RUN_FAILED_TO_RUN: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
completion: TaskRunFailedExecutionResult,
|
||||
}),
|
||||
},
|
||||
WAIT_FOR_DURATION: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
@@ -686,8 +828,35 @@ export const ProdWorkerToCoordinatorMessages = {
|
||||
}),
|
||||
}),
|
||||
},
|
||||
CREATE_TASK_RUN_ATTEMPT: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
runId: z.string(),
|
||||
}),
|
||||
callback: z.discriminatedUnion("success", [
|
||||
z.object({
|
||||
success: z.literal(false),
|
||||
reason: z.string().optional(),
|
||||
}),
|
||||
z.object({
|
||||
success: z.literal(true),
|
||||
executionPayload: ProdTaskRunExecutionPayload,
|
||||
}),
|
||||
]),
|
||||
},
|
||||
UNRECOVERABLE_ERROR: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
error: z.object({
|
||||
name: z.string(),
|
||||
message: z.string(),
|
||||
stack: z.string().optional(),
|
||||
}),
|
||||
}),
|
||||
},
|
||||
};
|
||||
|
||||
// TODO: The coordinator can only safely use v1 worker messages, higher versions will need a new flag, e.g. SUPPORTS_VERSIONED_MESSAGES
|
||||
export const CoordinatorToProdWorkerMessages = {
|
||||
RESUME_AFTER_DEPENDENCY: {
|
||||
message: z.object({
|
||||
@@ -709,6 +878,12 @@ export const CoordinatorToProdWorkerMessages = {
|
||||
executionPayload: ProdTaskRunExecutionPayload,
|
||||
}),
|
||||
},
|
||||
EXECUTE_TASK_RUN_LAZY_ATTEMPT: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
lazyPayload: TaskRunExecutionLazyAttemptPayload,
|
||||
}),
|
||||
},
|
||||
REQUEST_ATTEMPT_CANCELLATION: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
@@ -716,9 +891,15 @@ export const CoordinatorToProdWorkerMessages = {
|
||||
}),
|
||||
},
|
||||
REQUEST_EXIT: {
|
||||
message: z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
}),
|
||||
message: z.discriminatedUnion("version", [
|
||||
z.object({
|
||||
version: z.literal("v1"),
|
||||
}),
|
||||
z.object({
|
||||
version: z.literal("v2"),
|
||||
delayInMs: z.number().optional(),
|
||||
}),
|
||||
]),
|
||||
},
|
||||
READY_FOR_RETRY: {
|
||||
message: z.object({
|
||||
|
||||
@@ -224,3 +224,13 @@ export type ResolvedConfig = RequireKeys<
|
||||
export const WaitReason = z.enum(["WAIT_FOR_DURATION", "WAIT_FOR_TASK", "WAIT_FOR_BATCH"]);
|
||||
|
||||
export type WaitReason = z.infer<typeof WaitReason>;
|
||||
|
||||
export const TaskRunExecutionLazyAttemptPayload = z.object({
|
||||
runId: z.string(),
|
||||
messageId: z.string(),
|
||||
isTest: z.boolean(),
|
||||
traceContext: z.record(z.unknown()),
|
||||
environment: z.record(z.string()).optional(),
|
||||
});
|
||||
|
||||
export type TaskRunExecutionLazyAttemptPayload = z.infer<typeof TaskRunExecutionLazyAttemptPayload>;
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import type { Socket } from "socket.io-client";
|
||||
import { io } from "socket.io-client";
|
||||
import { z } from "zod";
|
||||
import { ZodError, z } from "zod";
|
||||
import { EventEmitterLike, ZodMessageValueSchema } from "./zodMessageHandler";
|
||||
import { LogLevel, SimpleStructuredLogger, StructuredLogger } from "./utils/structuredLogger";
|
||||
import { fromZodError } from "zod-validation-error";
|
||||
|
||||
export interface ZodSocketMessageCatalogSchema {
|
||||
[key: string]:
|
||||
@@ -81,7 +82,7 @@ export type MessagesFromSocketCatalog<TMessageCatalog extends ZodSocketMessageCa
|
||||
}[keyof TMessageCatalog];
|
||||
|
||||
const messageSchema = z.object({
|
||||
version: z.literal("v1").default("v1"),
|
||||
version: z.string(),
|
||||
type: z.string(),
|
||||
payload: z.unknown(),
|
||||
});
|
||||
@@ -127,10 +128,22 @@ export class ZodSocketMessageHandler<TRPCCatalog extends ZodSocketMessageCatalog
|
||||
throw new Error(`Unknown message type: ${parsedMessage.data.type}`);
|
||||
}
|
||||
|
||||
const parsedPayload = schema.safeParse(parsedMessage.data.payload);
|
||||
const messageWithVersion = {
|
||||
version: parsedMessage.data.version,
|
||||
...(typeof parsedMessage.data.payload === "object" ? parsedMessage.data.payload : {}),
|
||||
};
|
||||
|
||||
const parsedPayload = schema.safeParse(messageWithVersion);
|
||||
|
||||
if (!parsedPayload.success) {
|
||||
throw new Error(`Failed to parse message payload: ${JSON.stringify(parsedPayload.error)}`);
|
||||
console.error("Failed to parse message payload", {
|
||||
message,
|
||||
payload: messageWithVersion,
|
||||
});
|
||||
|
||||
throw parsedPayload.error instanceof ZodError
|
||||
? fromZodError(parsedPayload.error)
|
||||
: parsedPayload.error;
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -166,7 +179,15 @@ export class ZodSocketMessageHandler<TRPCCatalog extends ZodSocketMessageCatalog
|
||||
ack = await this.handleMessage({ type: eventName, version, payload });
|
||||
}
|
||||
} catch (error) {
|
||||
log.error("Error while handling message", { error });
|
||||
log.error("Error while handling message", {
|
||||
error:
|
||||
error instanceof Error
|
||||
? {
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
}
|
||||
: error,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ export default defineConfig({
|
||||
"./src/v3/dev/index.ts",
|
||||
"./src/v3/prod/index.ts",
|
||||
"./src/v3/workers/index.ts",
|
||||
"./src/v3/zodfetch.ts",
|
||||
],
|
||||
external: ["node:stream"],
|
||||
});
|
||||
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
-- AlterTable
|
||||
ALTER TABLE "BackgroundWorker" ADD COLUMN "supportsLazyAttempts" BOOLEAN NOT NULL DEFAULT false;
|
||||
@@ -1563,6 +1563,8 @@ model BackgroundWorker {
|
||||
|
||||
deployment WorkerDeployment?
|
||||
|
||||
supportsLazyAttempts Boolean @default(false)
|
||||
|
||||
@@unique([projectId, runtimeEnvironmentId, version])
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,394 @@
|
||||
import { logger, task, wait } from "@trigger.dev/sdk/v3";
|
||||
|
||||
export const lazyImmediate = task({
|
||||
id: "lazy-immediate",
|
||||
run: async (payload: { forceError?: boolean }) => {
|
||||
logger.info("Log something", { payload });
|
||||
logger.info("Log something else", { payload });
|
||||
|
||||
if (payload.forceError) {
|
||||
throw new Error("Forced error");
|
||||
}
|
||||
|
||||
return {
|
||||
message: "This is a message",
|
||||
payload,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const lazyWait = task({
|
||||
id: "lazy-wait",
|
||||
run: async (payload: { forceError?: boolean; delayInSeconds?: number }) => {
|
||||
logger.info("Log something", { payload });
|
||||
|
||||
await wait.for({ seconds: payload.delayInSeconds ?? 1 });
|
||||
|
||||
logger.info("Log something else", { payload });
|
||||
|
||||
if (payload.forceError) {
|
||||
throw new Error("Forced error");
|
||||
}
|
||||
|
||||
return {
|
||||
message: "This is a message",
|
||||
payload,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const lazySingleDependency = task({
|
||||
id: "lazy-single-dependency",
|
||||
run: async (payload: {
|
||||
forceError?: boolean;
|
||||
forceChildError?: boolean;
|
||||
delayInSeconds?: number;
|
||||
}) => {
|
||||
logger.info("Log something", { payload });
|
||||
|
||||
const result = await lazyWait.triggerAndWait({
|
||||
delayInSeconds: payload.delayInSeconds,
|
||||
forceError: payload.forceChildError,
|
||||
});
|
||||
logger.info("Single result", { result });
|
||||
|
||||
logger.info("Log something else", { payload });
|
||||
|
||||
if (payload.forceError) {
|
||||
throw new Error("Forced error");
|
||||
}
|
||||
|
||||
return {
|
||||
message: "This is a message",
|
||||
payload,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const lazyBatchDependency = task({
|
||||
id: "lazy-batch-dependency",
|
||||
run: async (payload: {
|
||||
forceError?: boolean;
|
||||
forceChildError?: boolean;
|
||||
delayInSeconds?: number;
|
||||
}) => {
|
||||
logger.info("Log something", { payload });
|
||||
|
||||
const results = await lazyWait.batchTriggerAndWait([
|
||||
{ payload: { delayInSeconds: payload.delayInSeconds, forceError: payload.forceChildError } },
|
||||
{ payload: { delayInSeconds: payload.delayInSeconds, forceError: payload.forceChildError } },
|
||||
]);
|
||||
logger.info("Batch results", { results });
|
||||
|
||||
logger.info("Log something else", { payload });
|
||||
|
||||
if (payload.forceError) {
|
||||
throw new Error("Forced error");
|
||||
}
|
||||
|
||||
return {
|
||||
message: "This is a message",
|
||||
payload,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const lazyConsecutiveWaits = task({
|
||||
id: "lazy-consecutive-waits",
|
||||
run: async (payload: {
|
||||
forceError?: boolean;
|
||||
forceChildError?: boolean;
|
||||
delayInSeconds?: number;
|
||||
}) => {
|
||||
logger.info("Log something", { payload });
|
||||
|
||||
await wait.for({ seconds: payload.delayInSeconds ?? 1 });
|
||||
|
||||
logger.info("Log something else", { payload });
|
||||
|
||||
await wait.for({ seconds: payload.delayInSeconds ?? 1 });
|
||||
|
||||
logger.info("Log something else again", { payload });
|
||||
|
||||
if (payload.forceError) {
|
||||
throw new Error("Forced error");
|
||||
}
|
||||
|
||||
return {
|
||||
message: "This is a message",
|
||||
payload,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const lazyConsecutiveDependencies = task({
|
||||
id: "lazy-consecutive-dependencies",
|
||||
run: async (payload: {
|
||||
forceError?: boolean;
|
||||
forceChildError?: boolean;
|
||||
delayInSeconds?: number;
|
||||
}) => {
|
||||
logger.info("Log something", { payload });
|
||||
|
||||
const result = await lazyWait.triggerAndWait({
|
||||
delayInSeconds: payload.delayInSeconds,
|
||||
forceError: payload.forceChildError,
|
||||
});
|
||||
logger.info("Single result #1", { result });
|
||||
|
||||
logger.info("Log something else", { payload });
|
||||
|
||||
const result2 = await lazyWait.triggerAndWait({
|
||||
delayInSeconds: payload.delayInSeconds,
|
||||
forceError: payload.forceChildError,
|
||||
});
|
||||
logger.info("Single result #2", { result2 });
|
||||
|
||||
logger.info("Log something else again", { payload });
|
||||
|
||||
if (payload.forceError) {
|
||||
throw new Error("Forced error");
|
||||
}
|
||||
|
||||
return {
|
||||
message: "This is a message",
|
||||
payload,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const lazyConsecutiveBatchDependencies = task({
|
||||
id: "lazy-consecutive-batch-dependencies",
|
||||
run: async (payload: {
|
||||
forceError?: boolean;
|
||||
forceChildError?: boolean;
|
||||
delayInSeconds?: number;
|
||||
}) => {
|
||||
logger.info("Log something", { payload });
|
||||
|
||||
const results = await lazyWait.batchTriggerAndWait([
|
||||
{ payload: { delayInSeconds: payload.delayInSeconds, forceError: payload.forceChildError } },
|
||||
{ payload: { delayInSeconds: payload.delayInSeconds, forceError: payload.forceChildError } },
|
||||
]);
|
||||
logger.info("Batch results #1", { results });
|
||||
|
||||
logger.info("Log something else", { payload });
|
||||
|
||||
const results2 = await lazyWait.batchTriggerAndWait([
|
||||
{ payload: { delayInSeconds: payload.delayInSeconds, forceError: payload.forceChildError } },
|
||||
{ payload: { delayInSeconds: payload.delayInSeconds, forceError: payload.forceChildError } },
|
||||
]);
|
||||
logger.info("Batch results #2", { results2 });
|
||||
|
||||
logger.info("Log something else again", { payload });
|
||||
|
||||
if (payload.forceError) {
|
||||
throw new Error("Forced error");
|
||||
}
|
||||
|
||||
return {
|
||||
message: "This is a message",
|
||||
payload,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const lazyWaitThenSingleDependency = task({
|
||||
id: "lazy-wait-then-single-dependency",
|
||||
run: async (payload: {
|
||||
forceError?: boolean;
|
||||
forceChildError?: boolean;
|
||||
delayInSeconds?: number;
|
||||
}) => {
|
||||
logger.info("Log something", { payload });
|
||||
|
||||
await wait.for({ seconds: payload.delayInSeconds ?? 1 });
|
||||
|
||||
logger.info("Log something else", { payload });
|
||||
|
||||
const result = await lazyWait.triggerAndWait({
|
||||
delayInSeconds: payload.delayInSeconds,
|
||||
forceError: payload.forceChildError,
|
||||
});
|
||||
logger.info("Single result", { result });
|
||||
|
||||
logger.info("Log something else again", { payload });
|
||||
|
||||
if (payload.forceError) {
|
||||
throw new Error("Forced error");
|
||||
}
|
||||
|
||||
return {
|
||||
message: "This is a message",
|
||||
payload,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const lazyWaitThenBatchDependency = task({
|
||||
id: "lazy-wait-then-batch-dependency",
|
||||
run: async (payload: {
|
||||
forceError?: boolean;
|
||||
forceChildError?: boolean;
|
||||
delayInSeconds?: number;
|
||||
}) => {
|
||||
logger.info("Log something", { payload });
|
||||
|
||||
await wait.for({ seconds: payload.delayInSeconds ?? 1 });
|
||||
|
||||
logger.info("Log something else", { payload });
|
||||
|
||||
const results = await lazyWait.batchTriggerAndWait([
|
||||
{ payload: { delayInSeconds: payload.delayInSeconds, forceError: payload.forceChildError } },
|
||||
{ payload: { delayInSeconds: payload.delayInSeconds, forceError: payload.forceChildError } },
|
||||
]);
|
||||
logger.info("Batch results", { results });
|
||||
|
||||
logger.info("Log something else again", { payload });
|
||||
|
||||
if (payload.forceError) {
|
||||
throw new Error("Forced error");
|
||||
}
|
||||
|
||||
return {
|
||||
message: "This is a message",
|
||||
payload,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const lazySingleDependencyThenWait = task({
|
||||
id: "lazy-single-dependency-then-wait",
|
||||
run: async (payload: {
|
||||
forceError?: boolean;
|
||||
forceChildError?: boolean;
|
||||
delayInSeconds?: number;
|
||||
}) => {
|
||||
logger.info("Log something", { payload });
|
||||
|
||||
const result = await lazyWait.triggerAndWait({
|
||||
delayInSeconds: payload.delayInSeconds,
|
||||
forceError: payload.forceChildError,
|
||||
});
|
||||
logger.info("Single result", { result });
|
||||
|
||||
logger.info("Log something else", { payload });
|
||||
|
||||
await wait.for({ seconds: payload.delayInSeconds ?? 1 });
|
||||
|
||||
logger.info("Log something else again", { payload });
|
||||
|
||||
if (payload.forceError) {
|
||||
throw new Error("Forced error");
|
||||
}
|
||||
|
||||
return {
|
||||
message: "This is a message",
|
||||
payload,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const lazySingleDependencyThenBatch = task({
|
||||
id: "lazy-single-dependency-then-batch",
|
||||
run: async (payload: {
|
||||
forceError?: boolean;
|
||||
forceChildError?: boolean;
|
||||
delayInSeconds?: number;
|
||||
}) => {
|
||||
logger.info("Log something", { payload });
|
||||
|
||||
const result = await lazyWait.triggerAndWait({
|
||||
delayInSeconds: payload.delayInSeconds,
|
||||
forceError: payload.forceChildError,
|
||||
});
|
||||
logger.info("Single result", { result });
|
||||
|
||||
logger.info("Log something else", { payload });
|
||||
|
||||
const results = await lazyWait.batchTriggerAndWait([
|
||||
{ payload: { delayInSeconds: payload.delayInSeconds, forceError: payload.forceChildError } },
|
||||
{ payload: { delayInSeconds: payload.delayInSeconds, forceError: payload.forceChildError } },
|
||||
]);
|
||||
logger.info("Batch results", { results });
|
||||
|
||||
logger.info("Log something else again", { payload });
|
||||
|
||||
if (payload.forceError) {
|
||||
throw new Error("Forced error");
|
||||
}
|
||||
|
||||
return {
|
||||
message: "This is a message",
|
||||
payload,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const lazyBatchDependencyThenWait = task({
|
||||
id: "lazy-batch-dependency-then-wait",
|
||||
run: async (payload: {
|
||||
forceError?: boolean;
|
||||
forceChildError?: boolean;
|
||||
delayInSeconds?: number;
|
||||
}) => {
|
||||
logger.info("Log something", { payload });
|
||||
|
||||
const results = await lazyWait.batchTriggerAndWait([
|
||||
{ payload: { delayInSeconds: payload.delayInSeconds, forceError: payload.forceChildError } },
|
||||
{ payload: { delayInSeconds: payload.delayInSeconds, forceError: payload.forceChildError } },
|
||||
]);
|
||||
logger.info("Batch results", { results });
|
||||
|
||||
logger.info("Log something else", { payload });
|
||||
|
||||
await wait.for({ seconds: payload.delayInSeconds ?? 1 });
|
||||
|
||||
logger.info("Log something else again", { payload });
|
||||
|
||||
if (payload.forceError) {
|
||||
throw new Error("Forced error");
|
||||
}
|
||||
|
||||
return {
|
||||
message: "This is a message",
|
||||
payload,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const lazyBatchDependencyThenSingle = task({
|
||||
id: "lazy-batch-dependency-then-single",
|
||||
run: async (payload: {
|
||||
forceError?: boolean;
|
||||
forceChildError?: boolean;
|
||||
delayInSeconds?: number;
|
||||
}) => {
|
||||
logger.info("Log something", { payload });
|
||||
|
||||
const results = await lazyWait.batchTriggerAndWait([
|
||||
{ payload: { delayInSeconds: payload.delayInSeconds, forceError: payload.forceChildError } },
|
||||
{ payload: { delayInSeconds: payload.delayInSeconds, forceError: payload.forceChildError } },
|
||||
]);
|
||||
logger.info("Batch results", { results });
|
||||
|
||||
logger.info("Log something else", { payload });
|
||||
|
||||
const result = await lazyWait.triggerAndWait({
|
||||
delayInSeconds: payload.delayInSeconds,
|
||||
forceError: payload.forceChildError,
|
||||
});
|
||||
logger.info("Single result", { result });
|
||||
|
||||
logger.info("Log something else again", { payload });
|
||||
|
||||
if (payload.forceError) {
|
||||
throw new Error("Forced error");
|
||||
}
|
||||
|
||||
return {
|
||||
message: "This is a message",
|
||||
payload,
|
||||
};
|
||||
},
|
||||
});
|
||||
@@ -3,7 +3,7 @@ import { logger, task } from "@trigger.dev/sdk/v3";
|
||||
export const longRunning = task({
|
||||
id: "long-running",
|
||||
run: async (payload: { message: string }) => {
|
||||
logger.info("Long running payloadd", { payload });
|
||||
logger.info("Long running payloadddd", { payload });
|
||||
|
||||
// Wait for 3 minutes
|
||||
await new Promise((resolve) => setTimeout(resolve, 3 * 60 * 1000));
|
||||
@@ -19,7 +19,22 @@ export const longRunningParent = task({
|
||||
run: async (payload: { message: string }) => {
|
||||
logger.info("Long running parent", { payload });
|
||||
|
||||
await longRunning.triggerAndWait({ message: "child" });
|
||||
const result = await longRunning.triggerAndWait({ message: "child" });
|
||||
|
||||
return {
|
||||
finished: new Date().toISOString(),
|
||||
result,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const longRunningWithDotInName = task({
|
||||
id: "long.running.with.dot",
|
||||
run: async (payload: { message: string }) => {
|
||||
logger.info("Long running payloadd", { payload });
|
||||
|
||||
// Wait for 3 minutes
|
||||
await new Promise((resolve) => setTimeout(resolve, 3 * 60 * 1000));
|
||||
|
||||
return {
|
||||
finished: new Date().toISOString(),
|
||||
|
||||
@@ -37,7 +37,7 @@ export const config: TriggerConfig = {
|
||||
retries: {
|
||||
enabledInDev: true,
|
||||
default: {
|
||||
maxAttempts: 3,
|
||||
maxAttempts: 4,
|
||||
minTimeoutInMs: 1000,
|
||||
maxTimeoutInMs: 10000,
|
||||
factor: 2,
|
||||
|
||||
Reference in New Issue
Block a user