Files
triggerdotdev--trigger.dev/apps/webapp/app/v3/requeueTaskRun.server.ts
Eric Allam e69ffd314a v3: Refactor attempt creation to be worker requested (#1077)
* WIP worker TaskRunAttempt creation

* Handling failing task runs that cannot create an attempt for whatever reason

* Move the visibility queue stuff into a graphile job

* Fixed task runs with unsanitized queue names

* “Borrow” the code from alerts PR to get self hosted deployments working

* Add an admin API endpoint to get info about the shared marqs queue

* Allow admins to view any project metrics

* start adding lazy attempts to prod

* lazy attempt creation for prod workers

* resurrect prod stack traces

* add exception event to failed run spans

* simplify dependency resumes

* fix typecheck

* fix merge

* fresh process for all attempts

* always try sigterm first

* stop heartbeat timeout on non-inplace replace message

* add missing ack on checkpoint creation service failure

* bypass dequeue for retries with running worker

* respect retry delays

* crash runs with invalid run status for execution

* remove debug logs

* fix nack message

* fix version locking

* fresh attempt processes in dev and prod

* improve handling of ipc timeouts

* consider checkpoint failures on cancellation

* add basic chaos monkey to checkpointer

* changeset

* control forced checkpoint simulation via env var

* fix merge

* kill old attempt processes before checkpointing

* detailed perf logging for checkpointing

* add coordinator otlp endpoint example

* improve prod run cancellation

* rename supports lazy attempts migration

* fix graceful exit

* fix retry mechanics

* clear paused state before retry

* remove checkpoint image after push

* crash worker on unrecoverable errors

* refactor unrecoverable error emit

* switch to do hosted busybox image

* increase wait for duration ipc timeout

* add changeset for misc fixes

* fix merge

* fix retry delay span runId

* fix dev retries

* improve prod worker logging

* log checkpoint sizes

* add lazy attempts catalog entries

* Fixed merge issue: use zodFetch, not wrapZodFetch

* Revert "Fixed merge issue: use zodFetch, not wrapZodFetch"

This reverts commit d137e4e1fe.

* importEnvVars uses wrapZodFetch now

* add backwards compat for retries without checkpoints

* handle more cases of unrecoverable runs

* don't kill the child process if it shouldn't be killed

---------

Co-authored-by: nicktrn <55853254+nicktrn@users.noreply.github.com>
Co-authored-by: Matt Aitken <matt@mattaitken.com>
2024-05-30 11:05:52 +01:00

96 lines
2.6 KiB
TypeScript

import { logger } from "~/services/logger.server";
import { marqs } from "~/v3/marqs/index.server";
import assertNever from "assert-never";
import { FailedTaskRunService } from "./failedTaskRun.server";
import { BaseService } from "./services/baseService.server";
import { PrismaClientOrTransaction } from "~/db.server";
import { workerQueue } from "~/services/worker.server";
export class RequeueTaskRunService extends BaseService {
public async call(runId: string) {
const taskRun = await this._prisma.taskRun.findUnique({
where: { id: runId },
});
if (!taskRun) {
logger.error("[RequeueTaskRunService] Task run not found", {
runId,
});
return;
}
switch (taskRun.status) {
case "PENDING": {
logger.debug("[RequeueTaskRunService] Requeueing task run", { taskRun });
await marqs?.nackMessage(taskRun.id);
break;
}
case "EXECUTING":
case "RETRYING_AFTER_FAILURE": {
logger.debug("[RequeueTaskRunService] Failing task run", { taskRun });
const service = new FailedTaskRunService();
await service.call(taskRun.friendlyId, {
ok: false,
id: taskRun.friendlyId,
retry: undefined,
error: {
type: "INTERNAL_ERROR",
code: "TASK_RUN_HEARTBEAT_TIMEOUT",
message: "Did not receive a heartbeat from the worker in time",
},
});
break;
}
case "WAITING_FOR_DEPLOY": {
logger.debug("[RequeueTaskRunService] Removing task run from queue", { taskRun });
await marqs?.acknowledgeMessage(taskRun.id);
break;
}
case "WAITING_TO_RESUME":
case "PAUSED": {
logger.debug("[RequeueTaskRunService] Requeueing task run", { taskRun });
await marqs?.nackMessage(taskRun.id);
break;
}
case "SYSTEM_FAILURE":
case "INTERRUPTED":
case "CRASHED":
case "COMPLETED_WITH_ERRORS":
case "COMPLETED_SUCCESSFULLY":
case "CANCELED": {
logger.debug("[RequeueTaskRunService] Task run is completed", { taskRun });
await marqs?.acknowledgeMessage(taskRun.id);
break;
}
default: {
assertNever(taskRun.status);
}
}
}
public static async enqueue(runId: string, runAt?: Date, tx?: PrismaClientOrTransaction) {
return await workerQueue.enqueue(
"v3.requeueTaskRun",
{ runId },
{ runAt, jobKey: `requeueTaskRun:${runId}` }
);
}
public static async dequeue(runId: string, tx?: PrismaClientOrTransaction) {
return await workerQueue.dequeue(`requeueTaskRun:${runId}`, { tx });
}
}