e69ffd314a
* WIP worker TaskRunAttempt creation
* Handling failing task runs that cannot create an attempt for whatever reason
* Move the visibility queue stuff into a graphile job
* Fixed task runs with unsanitized queue names
* “Borrow” the code from alerts PR to get self hosted deployments working
* Add an admin API endpoint to get info about the shared marqs queue
* Allow admins to view any project metrics
* start adding lazy attempts to prod
* lazy attempt creation for prod workers
* resurrect prod stack traces
* add exception event to failed run spans
* simplify dependency resumes
* fix typecheck
* fix merge
* fresh process for all attempts
* always try sigterm first
* stop heartbeat timeout on non-inplace replace message
* add missing ack on checkpoint creation service failure
* bypass dequeue for retries with running worker
* respect retry delays
* crash runs with invalid run status for execution
* remove debug logs
* fix nack message
* fix version locking
* fresh attempt processes in dev and prod
* improve handling of ipc timeouts
* consider checkpoint failures on cancellation
* add basic chaos monkey to checkpointer
* changeset
* control forced checkpoint simulation via env var
* fix merge
* kill old attempt processes before checkpointing
* detailed perf logging for checkpointing
* add coordinator otlp endpoint example
* improve prod run cancellation
* rename supports lazy attempts migration
* fix graceful exit
* fix retry mechanics
* clear paused state before retry
* remove checkpoint image after push
* crash worker on unrecoverable errors
* refactor unrecoverable error emit
* switch to do hosted busybox image
* increase wait for duration ipc timeout
* add changeset for misc fixes
* fix merge
* fix retry delay span runId
* fix dev retries
* improve prod worker logging
* log checkpoint sizes
* add lazy attempts catalog entries
* Fixed merge issue: use zodFetch, not wrapZodFetch
* Revert "Fixed merge issue: use zodFetch, not wrapZodFetch"
This reverts commit d137e4e1fe.
* importEnvVars uses wrapZodFetch now
* add backwards compat for retries without checkpoints
* handle more cases of unrecoverable runs
* don't kill the child process if it shouldn't be killed
---------
Co-authored-by: nicktrn <55853254+nicktrn@users.noreply.github.com>
Co-authored-by: Matt Aitken <matt@mattaitken.com>
67 lines
1.8 KiB
TypeScript
67 lines
1.8 KiB
TypeScript
import { TaskRunFailedExecutionResult } from "@trigger.dev/core/v3";
|
|
import { logger } from "~/services/logger.server";
|
|
import { marqs } from "~/v3/marqs/index.server";
|
|
|
|
import { TaskRunStatus } from "@trigger.dev/database";
|
|
import { createExceptionPropertiesFromError, eventRepository } from "./eventRepository.server";
|
|
import { BaseService } from "./services/baseService.server";
|
|
|
|
const FAILABLE_TASK_RUN_STATUSES: TaskRunStatus[] = ["EXECUTING", "PENDING", "WAITING_FOR_DEPLOY"];
|
|
|
|
export class FailedTaskRunService extends BaseService {
|
|
public async call(runFriendlyId: string, completion: TaskRunFailedExecutionResult) {
|
|
const taskRun = await this._prisma.taskRun.findUnique({
|
|
where: { friendlyId: runFriendlyId },
|
|
});
|
|
|
|
if (!taskRun) {
|
|
logger.error("[FailedTaskRunService] Task run not found", {
|
|
runFriendlyId,
|
|
completion,
|
|
});
|
|
|
|
return;
|
|
}
|
|
|
|
if (!FAILABLE_TASK_RUN_STATUSES.includes(taskRun.status)) {
|
|
logger.error("[FailedTaskRunService] Task run is not in a failable state", {
|
|
taskRun,
|
|
completion,
|
|
});
|
|
|
|
return;
|
|
}
|
|
|
|
// No more retries, we need to fail the task run
|
|
logger.debug("[FailedTaskRunService] Failing task run", { taskRun, completion });
|
|
|
|
await marqs?.acknowledgeMessage(taskRun.id);
|
|
|
|
// Now we need to "complete" the task run event/span
|
|
await eventRepository.completeEvent(taskRun.spanId, {
|
|
endTime: new Date(),
|
|
attributes: {
|
|
isError: true,
|
|
},
|
|
events: [
|
|
{
|
|
name: "exception",
|
|
time: new Date(),
|
|
properties: {
|
|
exception: createExceptionPropertiesFromError(completion.error),
|
|
},
|
|
},
|
|
],
|
|
});
|
|
|
|
await this._prisma.taskRun.update({
|
|
where: {
|
|
id: taskRun.id,
|
|
},
|
|
data: {
|
|
status: "SYSTEM_FAILURE",
|
|
},
|
|
});
|
|
}
|
|
}
|