perf(run-engine,run-store): one execution snapshot per triggered run (#4419)

A non-delayed run used to get two execution snapshots the moment it was
triggered: `RUN_CREATED` nested in the run-create transaction,
immediately followed by `QUEUED` from its own `BEGIN`/`INSERT`/`COMMIT`.
It now gets a single `QUEUED` snapshot written inside the create, and
the trigger path only publishes to the queue. One fewer row per run on
`TaskRunExecutionSnapshot`, and one fewer round trip on the trigger hot
path.

`EnqueueSystem` gains a `publishRun` seam that enqueues without writing
a snapshot. Every re-enqueue path (waitpoint resume, checkpoint restore,
delayed enqueue, pending version, retry requeue) still calls
`enqueueRun` and writes its own `QUEUED`, so only the first enqueue
changes. The `QUEUED` snapshot still commits before the queue message,
so a dequeue sees a dequeueable status exactly as before.

Two things for reviewers. Nesting the write skips
`createExecutionSnapshot`, which is what emits
`executionSnapshotCreated` and therefore the run timeline's `[engine]
QUEUED` entry, so the trigger path now emits it directly, the same way
the dequeue and attempt-start paths already do for their nested creates.
And `RUN_CREATED` is still written when a dequeued run has no background
worker yet, so the status and both `statuses.ts` helpers stay live and
existing rows keep reading correctly.

Delayed runs are untouched: `DELAYED` then `QUEUED` are two genuinely
different moments and stay two snapshots.

Rollback is a revert. Create-and-enqueue happen in one request in one
process, so no in-flight run needs both code paths to agree during a
rollout.


One note for whoever debugs this path later. The `QUEUED` snapshot now
commits before the queue publish, so a failed publish leaves the run
recorded as `QUEUED` with no queue message. That state was already
reachable, since the publish was never part of the snapshot transaction,
but it used to be recorded as `RUN_CREATED`, which was distinctive
because it never otherwise persisted. `QUEUED` with no message is
indistinguishable from a run waiting on a concurrency slot, so
trigger-time publish failure is now one more cause of an apparently
stuck queued run.
This commit is contained in:
Eric Allam
2026-07-31 16:12:07 +01:00
committed by GitHub
parent a91c08c731
commit f10bc23785
11 changed files with 522 additions and 55 deletions
@@ -0,0 +1,6 @@
---
area: webapp
type: improvement
---
Triggering a task now does one fewer database write, so runs reach the queue slightly faster.
@@ -1 +1,11 @@
export const MAX_TASK_RUN_ATTEMPTS = 250;
/**
* The status and description a run's default entry into the queue is written with. Shared because
* the trigger path writes this snapshot nested in the run-create transaction and then emits its own
* `executionSnapshotCreated`, while every re-enqueue writes it through `enqueueRun`. Three places
* have to agree, or the persisted row and the run timeline's `[engine]` entry drift apart.
* Re-enqueues that describe why they requeued pass their own description instead.
*/
export const QUEUED_SNAPSHOT_STATUS = "QUEUED" as const;
export const QUEUED_SNAPSHOT_DESCRIPTION = "Run was QUEUED";
@@ -23,6 +23,7 @@ import {
} from "@trigger.dev/core/v3";
import type { TaskRunError } from "@trigger.dev/core/v3/schemas";
import {
generateInternalId,
parseNaturalLanguageDurationInMs,
RunId,
WaitpointId,
@@ -53,6 +54,7 @@ import { RunQueue } from "../run-queue/index.js";
import { RunQueueFullKeyProducer } from "../run-queue/keyProducer.js";
import type { AuthenticatedEnvironment, MinimalAuthenticatedEnvironment } from "../shared/index.js";
import { BillingCache } from "./billingCache.js";
import { QUEUED_SNAPSHOT_DESCRIPTION, QUEUED_SNAPSHOT_STATUS } from "./consts.js";
import {
ExecutionSnapshotNotFoundError,
NotImplementedError,
@@ -950,6 +952,7 @@ export class RunEngine {
let taskRun: TaskRun & { associatedWaitpoint: Waitpoint | null };
const taskRunId = RunId.fromFriendlyId(friendlyId);
const initialSnapshotId = generateInternalId();
// App-level replacement for the dropped TaskRun env/project Cascade FKs.
await this.controlPlaneResolver.assertEnvExists(environment.id);
@@ -1035,9 +1038,10 @@ export class RunEngine {
annotations,
},
snapshot: {
id: initialSnapshotId,
engine: "V2",
executionStatus: delayUntil ? "DELAYED" : "RUN_CREATED",
description: delayUntil ? "Run is delayed" : "Run was created",
executionStatus: delayUntil ? "DELAYED" : QUEUED_SNAPSHOT_STATUS,
description: delayUntil ? "Run is delayed" : QUEUED_SNAPSHOT_DESCRIPTION,
runStatus: status,
environmentId: environment.id,
environmentType: environment.type,
@@ -1164,13 +1168,29 @@ export class RunEngine {
await this.ttlSystem.scheduleExpireRun({ runId: taskRun.id, ttl: taskRun.ttl });
}
await this.enqueueSystem.enqueueRun({
this.eventBus.emit("executionSnapshotCreated", {
time: new Date(),
run: {
id: taskRun.id,
},
snapshot: {
id: initialSnapshotId,
executionStatus: QUEUED_SNAPSHOT_STATUS,
description: QUEUED_SNAPSHOT_DESCRIPTION,
runStatus: taskRun.status,
attemptNumber: taskRun.attemptNumber ?? null,
checkpointId: null,
workerId: workerId ?? null,
runnerId: runnerId ?? null,
isValid: true,
error: null,
completedWaitpointIds: [],
},
});
await this.enqueueSystem.publishRun({
run: taskRun,
env: environment,
workerId,
runnerId,
tx: prisma,
skipRunLock: true,
includeTtl: true,
anchorEligibilityAtQueuePosition: true,
enableFastPath,
@@ -44,27 +44,37 @@ function createEngineOptions(redisOptions: any, prisma: any, store?: PostgresRun
}
/**
* A real PostgresRunStore subclass that counts the snapshot create method that enqueueRun's
* snapshot write routes through (via executionSnapshotSystem.createExecutionSnapshot). super.*
* runs the genuine store implementation, so the routing is observed over real containers without
* ever mocking prisma or the store.
* A real PostgresRunStore subclass that counts the store methods a run's QUEUED snapshot can be
* written through: nested in `createRun` on the trigger path, or standalone via
* `createExecutionSnapshot` on every re-enqueue. super.* runs the genuine store implementation, so
* the routing is observed over real containers without ever mocking prisma or the store.
*/
class CountingPostgresRunStore extends PostgresRunStore {
public snapshotCreates = 0;
/** Snapshots written nested in a run create: the trigger path's QUEUED write. */
public nestedSnapshotCreates = 0;
/** Snapshots written standalone through `createExecutionSnapshot`: every re-enqueue. */
public standaloneSnapshotCreates = 0;
override async createExecutionSnapshot(
input: any,
tx?: any
): ReturnType<PostgresRunStore["createExecutionSnapshot"]> {
this.snapshotCreates++;
this.standaloneSnapshotCreates++;
return super.createExecutionSnapshot(input, tx);
}
override async createRun(
params: Parameters<PostgresRunStore["createRun"]>[0],
tx?: any
): ReturnType<PostgresRunStore["createRun"]> {
this.nestedSnapshotCreates++;
return super.createRun(params, tx);
}
}
describe("RunEngine enqueueRun store routing", () => {
// The QUEUED snapshot written while enqueuing a run routes through the injected store.
containerTest(
"enqueueRun snapshot routes through the store",
"the QUEUED snapshot routes through the store as a single nested write",
async ({ prisma, redisOptions }) => {
const countingStore = new CountingPostgresRunStore({ prisma, readOnlyPrisma: prisma });
const engine = new RunEngine(createEngineOptions(redisOptions, prisma, countingStore));
@@ -74,7 +84,8 @@ describe("RunEngine enqueueRun store routing", () => {
const taskIdentifier = "test-task";
await setupBackgroundWorker(engine, environment, taskIdentifier);
const before = countingStore.snapshotCreates;
const nestedBefore = countingStore.nestedSnapshotCreates;
const standaloneBefore = countingStore.standaloneSnapshotCreates;
const run = await engine.trigger(
{
@@ -96,7 +107,8 @@ describe("RunEngine enqueueRun store routing", () => {
prisma
);
expect(countingStore.snapshotCreates).toBeGreaterThan(before);
expect(countingStore.nestedSnapshotCreates).toBe(nestedBefore + 1);
expect(countingStore.standaloneSnapshotCreates).toBe(standaloneBefore);
const latest = await getLatestExecutionSnapshot(prisma, run.id);
assertNonNullable(latest);
@@ -7,6 +7,7 @@ import type {
import type { RunStore } from "@internal/run-store";
import { parseNaturalLanguageDuration } from "@trigger.dev/core/v3/isomorphic";
import type { MinimalAuthenticatedEnvironment } from "../../shared/index.js";
import { QUEUED_SNAPSHOT_DESCRIPTION, QUEUED_SNAPSHOT_STATUS } from "../consts.js";
import type { ExecutionSnapshotSystem } from "./executionSnapshotSystem.js";
import type { SystemResources } from "./systems.js";
@@ -95,8 +96,8 @@ export class EnqueueSystem {
{
run: run,
snapshot: {
executionStatus: snapshot?.status ?? "QUEUED",
description: snapshot?.description ?? "Run was QUEUED",
executionStatus: snapshot?.status ?? QUEUED_SNAPSHOT_STATUS,
description: snapshot?.description ?? QUEUED_SNAPSHOT_DESCRIPTION,
metadata: snapshot?.metadata ?? undefined,
},
previousSnapshotId,
@@ -113,42 +114,74 @@ export class EnqueueSystem {
store
);
// Force development runs to use the environment id as the worker queue.
const workerQueue = env.type === "DEVELOPMENT" ? env.id : run.workerQueue;
const queuePositionMs = (run.queueTimestamp ?? run.createdAt).getTime();
const timestamp = queuePositionMs - run.priorityMs;
const eligibleAtMs = anchorEligibilityAtQueuePosition ? queuePositionMs : Date.now();
let ttlExpiresAt: number | undefined;
if (includeTtl && run.ttl) {
const expireAt = parseNaturalLanguageDuration(run.ttl);
if (expireAt) {
ttlExpiresAt = expireAt.getTime();
}
}
await this.$.runQueue.enqueueMessage({
await this.publishRun({
run,
env,
workerQueue,
includeTtl,
anchorEligibilityAtQueuePosition,
enableFastPath,
message: {
runId: run.id,
taskIdentifier: run.taskIdentifier,
orgId: env.organization.id,
projectId: env.project.id,
environmentId: env.id,
environmentType: env.type,
queue: run.queue,
concurrencyKey: run.concurrencyKey ?? undefined,
timestamp,
eligibleAtMs,
attempt: 0,
ttlExpiresAt,
},
});
return newSnapshot;
});
}
/**
* Publishes the run to the RunQueue without writing an execution snapshot. Callers that already
* hold a `QUEUED` snapshot (the trigger path writes one inside the run-create transaction) use
* this so the run does not pay for a second snapshot write.
*/
public async publishRun({
run,
env,
includeTtl = false,
anchorEligibilityAtQueuePosition = false,
enableFastPath = false,
}: {
run: TaskRun;
env: MinimalAuthenticatedEnvironment;
/** See `enqueueRun`. */
includeTtl?: boolean;
/** See `enqueueRun`. */
anchorEligibilityAtQueuePosition?: boolean;
/** When true, allow the queue to push directly to worker queue if concurrency is available. */
enableFastPath?: boolean;
}) {
// Force development runs to use the environment id as the worker queue.
const workerQueue = env.type === "DEVELOPMENT" ? env.id : run.workerQueue;
const queuePositionMs = (run.queueTimestamp ?? run.createdAt).getTime();
const timestamp = queuePositionMs - run.priorityMs;
const eligibleAtMs = anchorEligibilityAtQueuePosition ? queuePositionMs : Date.now();
// Include TTL only when explicitly requested (first enqueue from trigger).
// Re-enqueues (waitpoint, checkpoint, delayed, pending version) must not add TTL.
let ttlExpiresAt: number | undefined;
if (includeTtl && run.ttl) {
const expireAt = parseNaturalLanguageDuration(run.ttl);
if (expireAt) {
ttlExpiresAt = expireAt.getTime();
}
}
await this.$.runQueue.enqueueMessage({
env,
workerQueue,
enableFastPath,
message: {
runId: run.id,
taskIdentifier: run.taskIdentifier,
orgId: env.organization.id,
projectId: env.project.id,
environmentId: env.id,
environmentType: env.type,
queue: run.queue,
concurrencyKey: run.concurrencyKey ?? undefined,
timestamp,
eligibleAtMs,
attempt: 0,
ttlExpiresAt,
},
});
}
}
@@ -65,6 +65,14 @@ class CountingPostgresRunStore extends PostgresRunStore {
return super.createExecutionSnapshot(input, tx);
}
override async createRun(
params: Parameters<PostgresRunStore["createRun"]>[0],
tx?: any
): ReturnType<PostgresRunStore["createRun"]> {
this.creates++;
return super.createRun(params, tx);
}
override async findLatestExecutionSnapshot(
runId: string,
client?: any
@@ -7,7 +7,7 @@ import type { PrismaClient } from "@trigger.dev/database";
import { RunEngine } from "../index.js";
import { getExecutionSnapshotsSince } from "../systems/executionSnapshotSystem.js";
import { copySnapshotsToReplica, createTestMetricsMeter } from "./helpers/replicaTestHelpers.js";
import { setupTestScenario } from "./helpers/snapshotTestHelpers.js";
import { createTestSnapshot, setupTestScenario } from "./helpers/snapshotTestHelpers.js";
import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js";
vi.setConfig({ testTimeout: 120_000 });
@@ -1154,6 +1154,15 @@ describe("RunEngine getSnapshotsSince", () => {
workerQueue: "main",
});
await createTestSnapshot(prisma, {
runId: run.id,
status: "EXECUTING",
environmentId: authenticatedEnvironment.id,
environmentType: authenticatedEnvironment.type,
projectId: authenticatedEnvironment.project.id,
organizationId: authenticatedEnvironment.organization.id,
});
const allSnapshots = await prisma.taskRunExecutionSnapshot.findMany({
where: { runId: run.id, isValid: true },
orderBy: { createdAt: "asc" },
@@ -137,9 +137,6 @@ const cancelledSnapshot = (friendlyId: string, environment: any) => ({
});
describe("RunEngine trigger/create routing", () => {
// trigger create routes through runStore.createRun with the structured
// DTO, and the persisted run + its nested first RUN_CREATED snapshot land via
// the single create call.
containerTest(
"trigger routes createRun and lands run + first snapshot",
async ({ prisma, redisOptions }) => {
@@ -169,7 +166,7 @@ describe("RunEngine trigger/create routing", () => {
orderBy: { createdAt: "asc" },
});
expect(snapshot).not.toBeNull();
expect(snapshot!.executionStatus).toBe("RUN_CREATED");
expect(snapshot!.executionStatus).toBe("QUEUED");
} finally {
await engine.quit();
}
@@ -0,0 +1,370 @@
import { assertNonNullable, containerTest } from "@internal/testcontainers";
import { trace } from "@internal/tracing";
import type { PrismaClient, TaskRunExecutionStatus } from "@trigger.dev/database";
import { setTimeout } from "node:timers/promises";
import { expect } from "vitest";
import { RunEngine } from "../index.js";
import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js";
vi.setConfig({ testTimeout: 60_000 });
async function snapshotStatuses(
prisma: PrismaClient,
runId: string
): Promise<TaskRunExecutionStatus[]> {
const snapshots = await prisma.taskRunExecutionSnapshot.findMany({
where: { runId },
orderBy: { createdAt: "asc" },
select: { executionStatus: true },
});
return snapshots.map((snapshot) => snapshot.executionStatus);
}
describe("RunEngine trigger() execution snapshots", () => {
containerTest(
"a non-delayed run is created with a single QUEUED snapshot",
async ({ prisma, redisOptions }) => {
const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
const engine = new RunEngine({
prisma,
worker: {
redis: redisOptions,
workers: 1,
tasksPerWorker: 10,
pollIntervalMs: 100,
},
queue: {
redis: redisOptions,
masterQueueConsumersDisabled: true,
processWorkerQueueDebounceMs: 50,
},
runLock: {
redis: redisOptions,
},
machines: {
defaultMachine: "small-1x",
machines: {
"small-1x": {
name: "small-1x" as const,
cpu: 0.5,
memory: 0.5,
centsPerMs: 0.0001,
},
},
baseCostInCents: 0.0001,
},
tracer: trace.getTracer("test", "0.0.0"),
});
try {
const taskIdentifier = "test-task";
await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier);
const run = await engine.trigger(
{
number: 1,
friendlyId: "run_1234",
environment: authenticatedEnvironment,
taskIdentifier,
payload: "{}",
payloadType: "application/json",
context: {},
traceContext: {},
traceId: "t_collapse_1",
spanId: "s_collapse_1",
workerQueue: "main",
queue: "task/test-task",
isTest: false,
tags: [],
},
prisma
);
expect(await snapshotStatuses(prisma, run.id)).toEqual(["QUEUED"]);
const queueLength = await engine.runQueue.lengthOfQueue(
authenticatedEnvironment,
run.queue
);
expect(queueLength).toBe(1);
const executionData = await engine.getRunExecutionData({ runId: run.id });
assertNonNullable(executionData);
expect(executionData.snapshot.executionStatus).toBe("QUEUED");
} finally {
await engine.quit();
}
}
);
containerTest(
"the QUEUED snapshot event is stamped at write time, not at an overridden run createdAt",
async ({ prisma, redisOptions }) => {
const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
const engine = new RunEngine({
prisma,
worker: {
redis: redisOptions,
workers: 1,
tasksPerWorker: 10,
pollIntervalMs: 100,
},
queue: {
redis: redisOptions,
masterQueueConsumersDisabled: true,
processWorkerQueueDebounceMs: 50,
},
runLock: {
redis: redisOptions,
},
machines: {
defaultMachine: "small-1x",
machines: {
"small-1x": {
name: "small-1x" as const,
cpu: 0.5,
memory: 0.5,
centsPerMs: 0.0001,
},
},
baseCostInCents: 0.0001,
},
tracer: trace.getTracer("test", "0.0.0"),
});
try {
const taskIdentifier = "test-task";
await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier);
const stampedTimes: Date[] = [];
engine.eventBus.on("executionSnapshotCreated", ({ time }) => {
stampedTimes.push(time);
});
const backdatedCreatedAt = new Date(Date.now() - 60 * 60 * 1000);
const triggeredAt = Date.now();
const run = await engine.trigger(
{
number: 1,
friendlyId: "run_1236",
environment: authenticatedEnvironment,
taskIdentifier,
payload: "{}",
payloadType: "application/json",
context: {},
traceContext: {},
traceId: "t_collapse_3",
spanId: "s_collapse_3",
workerQueue: "main",
queue: "task/test-task",
isTest: false,
tags: [],
createdAt: backdatedCreatedAt,
},
prisma
);
const storedRun = await prisma.taskRun.findUnique({ where: { id: run.id } });
expect(storedRun?.createdAt.getTime()).toBe(backdatedCreatedAt.getTime());
expect(stampedTimes.length).toBe(1);
expect(stampedTimes[0].getTime()).toBeGreaterThanOrEqual(triggeredAt);
} finally {
await engine.quit();
}
}
);
containerTest(
"a delayed run keeps DELAYED and QUEUED as separate snapshots",
async ({ prisma, redisOptions }) => {
const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
const engine = new RunEngine({
prisma,
worker: {
redis: redisOptions,
workers: 1,
tasksPerWorker: 10,
pollIntervalMs: 100,
},
queue: {
redis: redisOptions,
masterQueueConsumersDisabled: true,
processWorkerQueueDebounceMs: 50,
},
runLock: {
redis: redisOptions,
},
machines: {
defaultMachine: "small-1x",
machines: {
"small-1x": {
name: "small-1x" as const,
cpu: 0.5,
memory: 0.5,
centsPerMs: 0.0001,
},
},
baseCostInCents: 0.0001,
},
tracer: trace.getTracer("test", "0.0.0"),
});
try {
const taskIdentifier = "test-task";
await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier);
const run = await engine.trigger(
{
number: 1,
friendlyId: "run_1235",
environment: authenticatedEnvironment,
taskIdentifier,
payload: "{}",
payloadType: "application/json",
context: {},
traceContext: {},
traceId: "t_collapse_2",
spanId: "s_collapse_2",
workerQueue: "main",
queue: "task/test-task",
isTest: false,
tags: [],
delayUntil: new Date(Date.now() + 500),
},
prisma
);
expect(await snapshotStatuses(prisma, run.id)).toEqual(["DELAYED"]);
await setTimeout(1_500);
expect(await snapshotStatuses(prisma, run.id)).toEqual(["DELAYED", "QUEUED"]);
} finally {
await engine.quit();
}
}
);
containerTest(
"a resumed run still writes its own QUEUED snapshot",
async ({ prisma, redisOptions }) => {
const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
const engine = new RunEngine({
prisma,
worker: {
redis: redisOptions,
workers: 1,
tasksPerWorker: 10,
pollIntervalMs: 100,
},
queue: {
redis: redisOptions,
masterQueueConsumersDisabled: true,
processWorkerQueueDebounceMs: 50,
},
runLock: {
redis: redisOptions,
},
machines: {
defaultMachine: "small-1x",
machines: {
"small-1x": {
name: "small-1x" as const,
cpu: 0.5,
memory: 0.5,
centsPerMs: 0.0001,
},
},
baseCostInCents: 0.0001,
},
tracer: trace.getTracer("test", "0.0.0"),
});
try {
const taskIdentifier = "test-task";
await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier);
const run = await engine.trigger(
{
number: 1,
friendlyId: "run_1237",
environment: authenticatedEnvironment,
taskIdentifier,
payload: "{}",
payloadType: "application/json",
context: {},
traceContext: {},
traceId: "t_collapse_4",
spanId: "s_collapse_4",
workerQueue: "main",
queue: "task/test-task",
isTest: false,
tags: [],
},
prisma
);
await setTimeout(500);
const dequeued = await engine.dequeueFromWorkerQueue({
consumerId: "test_collapse_4",
workerQueue: "main",
});
assertNonNullable(dequeued[0]);
await engine.startRunAttempt({
runId: dequeued[0].run.id,
snapshotId: dequeued[0].snapshot.id,
});
const waitpointResult = await engine.createManualWaitpoint({
environmentId: authenticatedEnvironment.id,
projectId: authenticatedEnvironment.projectId,
});
const blockedResult = await engine.blockRunWithWaitpoint({
runId: run.id,
waitpoints: waitpointResult.waitpoint.id,
projectId: authenticatedEnvironment.projectId,
organizationId: authenticatedEnvironment.organizationId,
});
const checkpointResult = await engine.createCheckpoint({
runId: run.id,
snapshotId: blockedResult.id,
checkpoint: {
type: "DOCKER",
reason: "TEST_CHECKPOINT",
location: "test-location",
imageRef: "test-image-ref",
},
});
expect(checkpointResult.ok).toBe(true);
await engine.completeWaitpoint({ id: waitpointResult.waitpoint.id });
await setTimeout(500);
expect(await snapshotStatuses(prisma, run.id)).toEqual([
"QUEUED",
"PENDING_EXECUTING",
"EXECUTING",
"EXECUTING_WITH_WAITPOINTS",
"SUSPENDED",
"QUEUED",
]);
} finally {
await engine.quit();
}
}
);
});
@@ -680,6 +680,7 @@ export class PostgresRunStore implements RunStore {
const client = tx ?? this.prisma;
const snapshotCreate = {
id: params.snapshot.id,
engine: params.snapshot.engine,
executionStatus: params.snapshot.executionStatus,
description: params.snapshot.description,
+1
View File
@@ -29,6 +29,7 @@ export type IdempotencyKeyRunMatch = {
};
export type CreateRunSnapshotInput = {
id?: string;
engine: "V2";
executionStatus: TaskRunExecutionStatus;
description: string;