e69ffd314a
* WIP worker TaskRunAttempt creation
* Handling failing task runs that cannot create an attempt for whatever reason
* Move the visibility queue stuff into a graphile job
* Fixed task runs with unsanitized queue names
* “Borrow” the code from alerts PR to get self hosted deployments working
* Add an admin API endpoint to get info about the shared marqs queue
* Allow admins to view any project metrics
* start adding lazy attempts to prod
* lazy attempt creation for prod workers
* resurrect prod stack traces
* add exception event to failed run spans
* simplify dependency resumes
* fix typecheck
* fix merge
* fresh process for all attempts
* always try sigterm first
* stop heartbeat timeout on non-inplace replace message
* add missing ack on checkpoint creation service failure
* bypass dequeue for retries with running worker
* respect retry delays
* crash runs with invalid run status for execution
* remove debug logs
* fix nack message
* fix version locking
* fresh attempt processes in dev and prod
* improve handling of ipc timeouts
* consider checkpoint failures on cancellation
* add basic chaos monkey to checkpointer
* changeset
* control forced checkpoint simulation via env var
* fix merge
* kill old attempt processes before checkpointing
* detailed perf logging for checkpointing
* add coordinator otlp endpoint example
* improve prod run cancellation
* rename supports lazy attempts migration
* fix graceful exit
* fix retry mechanics
* clear paused state before retry
* remove checkpoint image after push
* crash worker on unrecoverable errors
* refactor unrecoverable error emit
* switch to do hosted busybox image
* increase wait for duration ipc timeout
* add changeset for misc fixes
* fix merge
* fix retry delay span runId
* fix dev retries
* improve prod worker logging
* log checkpoint sizes
* add lazy attempts catalog entries
* Fixed merge issue: use zodFetch, not wrapZodFetch
* Revert "Fixed merge issue: use zodFetch, not wrapZodFetch"
This reverts commit d137e4e1fe.
* importEnvVars uses wrapZodFetch now
* add backwards compat for retries without checkpoints
* handle more cases of unrecoverable runs
* don't kill the child process if it shouldn't be killed
---------
Co-authored-by: nicktrn <55853254+nicktrn@users.noreply.github.com>
Co-authored-by: Matt Aitken <matt@mattaitken.com>
96 lines
2.6 KiB
TypeScript
96 lines
2.6 KiB
TypeScript
import { logger } from "~/services/logger.server";
|
|
import { marqs } from "~/v3/marqs/index.server";
|
|
|
|
import assertNever from "assert-never";
|
|
import { FailedTaskRunService } from "./failedTaskRun.server";
|
|
import { BaseService } from "./services/baseService.server";
|
|
import { PrismaClientOrTransaction } from "~/db.server";
|
|
import { workerQueue } from "~/services/worker.server";
|
|
|
|
export class RequeueTaskRunService extends BaseService {
|
|
public async call(runId: string) {
|
|
const taskRun = await this._prisma.taskRun.findUnique({
|
|
where: { id: runId },
|
|
});
|
|
|
|
if (!taskRun) {
|
|
logger.error("[RequeueTaskRunService] Task run not found", {
|
|
runId,
|
|
});
|
|
|
|
return;
|
|
}
|
|
|
|
switch (taskRun.status) {
|
|
case "PENDING": {
|
|
logger.debug("[RequeueTaskRunService] Requeueing task run", { taskRun });
|
|
|
|
await marqs?.nackMessage(taskRun.id);
|
|
|
|
break;
|
|
}
|
|
case "EXECUTING":
|
|
case "RETRYING_AFTER_FAILURE": {
|
|
logger.debug("[RequeueTaskRunService] Failing task run", { taskRun });
|
|
|
|
const service = new FailedTaskRunService();
|
|
|
|
await service.call(taskRun.friendlyId, {
|
|
ok: false,
|
|
id: taskRun.friendlyId,
|
|
retry: undefined,
|
|
error: {
|
|
type: "INTERNAL_ERROR",
|
|
code: "TASK_RUN_HEARTBEAT_TIMEOUT",
|
|
message: "Did not receive a heartbeat from the worker in time",
|
|
},
|
|
});
|
|
|
|
break;
|
|
}
|
|
case "WAITING_FOR_DEPLOY": {
|
|
logger.debug("[RequeueTaskRunService] Removing task run from queue", { taskRun });
|
|
|
|
await marqs?.acknowledgeMessage(taskRun.id);
|
|
|
|
break;
|
|
}
|
|
case "WAITING_TO_RESUME":
|
|
case "PAUSED": {
|
|
logger.debug("[RequeueTaskRunService] Requeueing task run", { taskRun });
|
|
|
|
await marqs?.nackMessage(taskRun.id);
|
|
|
|
break;
|
|
}
|
|
case "SYSTEM_FAILURE":
|
|
case "INTERRUPTED":
|
|
case "CRASHED":
|
|
case "COMPLETED_WITH_ERRORS":
|
|
case "COMPLETED_SUCCESSFULLY":
|
|
case "CANCELED": {
|
|
logger.debug("[RequeueTaskRunService] Task run is completed", { taskRun });
|
|
|
|
await marqs?.acknowledgeMessage(taskRun.id);
|
|
|
|
break;
|
|
}
|
|
default: {
|
|
assertNever(taskRun.status);
|
|
}
|
|
}
|
|
}
|
|
|
|
public static async enqueue(runId: string, runAt?: Date, tx?: PrismaClientOrTransaction) {
|
|
return await workerQueue.enqueue(
|
|
"v3.requeueTaskRun",
|
|
{ runId },
|
|
{ runAt, jobKey: `requeueTaskRun:${runId}` }
|
|
);
|
|
}
|
|
|
|
public static async dequeue(runId: string, tx?: PrismaClientOrTransaction) {
|
|
return await workerQueue.dequeue(`requeueTaskRun:${runId}`, { tx });
|
|
}
|
|
}
|