Files
triggerdotdev--trigger.dev/apps/webapp/app/v3/failedTaskRun.server.ts
Eric Allam e69ffd314a v3: Refactor attempt creation to be worker requested (#1077)
* WIP worker TaskRunAttempt creation

* Handling failing task runs that cannot create an attempt for whatever reason

* Move the visibility queue stuff into a graphile job

* Fixed task runs with unsanitized queue names

* “Borrow” the code from alerts PR to get self hosted deployments working

* Add an admin API endpoint to get info about the shared marqs queue

* Allow admins to view any project metrics

* start adding lazy attempts to prod

* lazy attempt creation for prod workers

* resurrect prod stack traces

* add exception event to failed run spans

* simplify dependency resumes

* fix typecheck

* fix merge

* fresh process for all attempts

* always try sigterm first

* stop heartbeat timeout on non-inplace replace message

* add missing ack on checkpoint creation service failure

* bypass dequeue for retries with running worker

* respect retry delays

* crash runs with invalid run status for execution

* remove debug logs

* fix nack message

* fix version locking

* fresh attempt processes in dev and prod

* improve handling of ipc timeouts

* consider checkpoint failures on cancellation

* add basic chaos monkey to checkpointer

* changeset

* control forced checkpoint simulation via env var

* fix merge

* kill old attempt processes before checkpointing

* detailed perf logging for checkpointing

* add coordinator otlp endpoint example

* improve prod run cancellation

* rename supports lazy attempts migration

* fix graceful exit

* fix retry mechanics

* clear paused state before retry

* remove checkpoint image after push

* crash worker on unrecoverable errors

* refactor unrecoverable error emit

* switch to do hosted busybox image

* increase wait for duration ipc timeout

* add changeset for misc fixes

* fix merge

* fix retry delay span runId

* fix dev retries

* improve prod worker logging

* log checkpoint sizes

* add lazy attempts catalog entries

* Fixed merge issue: use zodFetch, not wrapZodFetch

* Revert "Fixed merge issue: use zodFetch, not wrapZodFetch"

This reverts commit d137e4e1fe.

* importEnvVars uses wrapZodFetch now

* add backwards compat for retries without checkpoints

* handle more cases of unrecoverable runs

* don't kill the child process if it shouldn't be killed

---------

Co-authored-by: nicktrn <55853254+nicktrn@users.noreply.github.com>
Co-authored-by: Matt Aitken <matt@mattaitken.com>
2024-05-30 11:05:52 +01:00

67 lines
1.8 KiB
TypeScript

import { TaskRunFailedExecutionResult } from "@trigger.dev/core/v3";
import { logger } from "~/services/logger.server";
import { marqs } from "~/v3/marqs/index.server";
import { TaskRunStatus } from "@trigger.dev/database";
import { createExceptionPropertiesFromError, eventRepository } from "./eventRepository.server";
import { BaseService } from "./services/baseService.server";
const FAILABLE_TASK_RUN_STATUSES: TaskRunStatus[] = ["EXECUTING", "PENDING", "WAITING_FOR_DEPLOY"];
export class FailedTaskRunService extends BaseService {
public async call(runFriendlyId: string, completion: TaskRunFailedExecutionResult) {
const taskRun = await this._prisma.taskRun.findUnique({
where: { friendlyId: runFriendlyId },
});
if (!taskRun) {
logger.error("[FailedTaskRunService] Task run not found", {
runFriendlyId,
completion,
});
return;
}
if (!FAILABLE_TASK_RUN_STATUSES.includes(taskRun.status)) {
logger.error("[FailedTaskRunService] Task run is not in a failable state", {
taskRun,
completion,
});
return;
}
// No more retries, we need to fail the task run
logger.debug("[FailedTaskRunService] Failing task run", { taskRun, completion });
await marqs?.acknowledgeMessage(taskRun.id);
// Now we need to "complete" the task run event/span
await eventRepository.completeEvent(taskRun.spanId, {
endTime: new Date(),
attributes: {
isError: true,
},
events: [
{
name: "exception",
time: new Date(),
properties: {
exception: createExceptionPropertiesFromError(completion.error),
},
},
],
});
await this._prisma.taskRun.update({
where: {
id: taskRun.id,
},
data: {
status: "SYSTEM_FAILURE",
},
});
}
}