perf(run-engine,run-store): one execution snapshot per triggered run (#4419)
A non-delayed run used to get two execution snapshots the moment it was triggered: `RUN_CREATED` nested in the run-create transaction, immediately followed by `QUEUED` from its own `BEGIN`/`INSERT`/`COMMIT`. It now gets a single `QUEUED` snapshot written inside the create, and the trigger path only publishes to the queue. One fewer row per run on `TaskRunExecutionSnapshot`, and one fewer round trip on the trigger hot path. `EnqueueSystem` gains a `publishRun` seam that enqueues without writing a snapshot. Every re-enqueue path (waitpoint resume, checkpoint restore, delayed enqueue, pending version, retry requeue) still calls `enqueueRun` and writes its own `QUEUED`, so only the first enqueue changes. The `QUEUED` snapshot still commits before the queue message, so a dequeue sees a dequeueable status exactly as before. Two things for reviewers. Nesting the write skips `createExecutionSnapshot`, which is what emits `executionSnapshotCreated` and therefore the run timeline's `[engine] QUEUED` entry, so the trigger path now emits it directly, the same way the dequeue and attempt-start paths already do for their nested creates. And `RUN_CREATED` is still written when a dequeued run has no background worker yet, so the status and both `statuses.ts` helpers stay live and existing rows keep reading correctly. Delayed runs are untouched: `DELAYED` then `QUEUED` are two genuinely different moments and stay two snapshots. Rollback is a revert. Create-and-enqueue happen in one request in one process, so no in-flight run needs both code paths to agree during a rollout. One note for whoever debugs this path later. The `QUEUED` snapshot now commits before the queue publish, so a failed publish leaves the run recorded as `QUEUED` with no queue message. That state was already reachable, since the publish was never part of the snapshot transaction, but it used to be recorded as `RUN_CREATED`, which was distinctive because it never otherwise persisted. `QUEUED` with no message is indistinguishable from a run waiting on a concurrency slot, so trigger-time publish failure is now one more cause of an apparently stuck queued run.
This commit is contained in:
@@ -0,0 +1,6 @@
|
||||
---
|
||||
area: webapp
|
||||
type: improvement
|
||||
---
|
||||
|
||||
Triggering a task now does one fewer database write, so runs reach the queue slightly faster.
|
||||
@@ -1 +1,11 @@
|
||||
export const MAX_TASK_RUN_ATTEMPTS = 250;
|
||||
|
||||
/**
|
||||
* The status and description a run's default entry into the queue is written with. Shared because
|
||||
* the trigger path writes this snapshot nested in the run-create transaction and then emits its own
|
||||
* `executionSnapshotCreated`, while every re-enqueue writes it through `enqueueRun`. Three places
|
||||
* have to agree, or the persisted row and the run timeline's `[engine]` entry drift apart.
|
||||
* Re-enqueues that describe why they requeued pass their own description instead.
|
||||
*/
|
||||
export const QUEUED_SNAPSHOT_STATUS = "QUEUED" as const;
|
||||
export const QUEUED_SNAPSHOT_DESCRIPTION = "Run was QUEUED";
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
} from "@trigger.dev/core/v3";
|
||||
import type { TaskRunError } from "@trigger.dev/core/v3/schemas";
|
||||
import {
|
||||
generateInternalId,
|
||||
parseNaturalLanguageDurationInMs,
|
||||
RunId,
|
||||
WaitpointId,
|
||||
@@ -53,6 +54,7 @@ import { RunQueue } from "../run-queue/index.js";
|
||||
import { RunQueueFullKeyProducer } from "../run-queue/keyProducer.js";
|
||||
import type { AuthenticatedEnvironment, MinimalAuthenticatedEnvironment } from "../shared/index.js";
|
||||
import { BillingCache } from "./billingCache.js";
|
||||
import { QUEUED_SNAPSHOT_DESCRIPTION, QUEUED_SNAPSHOT_STATUS } from "./consts.js";
|
||||
import {
|
||||
ExecutionSnapshotNotFoundError,
|
||||
NotImplementedError,
|
||||
@@ -950,6 +952,7 @@ export class RunEngine {
|
||||
|
||||
let taskRun: TaskRun & { associatedWaitpoint: Waitpoint | null };
|
||||
const taskRunId = RunId.fromFriendlyId(friendlyId);
|
||||
const initialSnapshotId = generateInternalId();
|
||||
|
||||
// App-level replacement for the dropped TaskRun env/project Cascade FKs.
|
||||
await this.controlPlaneResolver.assertEnvExists(environment.id);
|
||||
@@ -1035,9 +1038,10 @@ export class RunEngine {
|
||||
annotations,
|
||||
},
|
||||
snapshot: {
|
||||
id: initialSnapshotId,
|
||||
engine: "V2",
|
||||
executionStatus: delayUntil ? "DELAYED" : "RUN_CREATED",
|
||||
description: delayUntil ? "Run is delayed" : "Run was created",
|
||||
executionStatus: delayUntil ? "DELAYED" : QUEUED_SNAPSHOT_STATUS,
|
||||
description: delayUntil ? "Run is delayed" : QUEUED_SNAPSHOT_DESCRIPTION,
|
||||
runStatus: status,
|
||||
environmentId: environment.id,
|
||||
environmentType: environment.type,
|
||||
@@ -1164,13 +1168,29 @@ export class RunEngine {
|
||||
await this.ttlSystem.scheduleExpireRun({ runId: taskRun.id, ttl: taskRun.ttl });
|
||||
}
|
||||
|
||||
await this.enqueueSystem.enqueueRun({
|
||||
this.eventBus.emit("executionSnapshotCreated", {
|
||||
time: new Date(),
|
||||
run: {
|
||||
id: taskRun.id,
|
||||
},
|
||||
snapshot: {
|
||||
id: initialSnapshotId,
|
||||
executionStatus: QUEUED_SNAPSHOT_STATUS,
|
||||
description: QUEUED_SNAPSHOT_DESCRIPTION,
|
||||
runStatus: taskRun.status,
|
||||
attemptNumber: taskRun.attemptNumber ?? null,
|
||||
checkpointId: null,
|
||||
workerId: workerId ?? null,
|
||||
runnerId: runnerId ?? null,
|
||||
isValid: true,
|
||||
error: null,
|
||||
completedWaitpointIds: [],
|
||||
},
|
||||
});
|
||||
|
||||
await this.enqueueSystem.publishRun({
|
||||
run: taskRun,
|
||||
env: environment,
|
||||
workerId,
|
||||
runnerId,
|
||||
tx: prisma,
|
||||
skipRunLock: true,
|
||||
includeTtl: true,
|
||||
anchorEligibilityAtQueuePosition: true,
|
||||
enableFastPath,
|
||||
|
||||
@@ -44,27 +44,37 @@ function createEngineOptions(redisOptions: any, prisma: any, store?: PostgresRun
|
||||
}
|
||||
|
||||
/**
|
||||
* A real PostgresRunStore subclass that counts the snapshot create method that enqueueRun's
|
||||
* snapshot write routes through (via executionSnapshotSystem.createExecutionSnapshot). super.*
|
||||
* runs the genuine store implementation, so the routing is observed over real containers without
|
||||
* ever mocking prisma or the store.
|
||||
* A real PostgresRunStore subclass that counts the store methods a run's QUEUED snapshot can be
|
||||
* written through: nested in `createRun` on the trigger path, or standalone via
|
||||
* `createExecutionSnapshot` on every re-enqueue. super.* runs the genuine store implementation, so
|
||||
* the routing is observed over real containers without ever mocking prisma or the store.
|
||||
*/
|
||||
class CountingPostgresRunStore extends PostgresRunStore {
|
||||
public snapshotCreates = 0;
|
||||
/** Snapshots written nested in a run create: the trigger path's QUEUED write. */
|
||||
public nestedSnapshotCreates = 0;
|
||||
/** Snapshots written standalone through `createExecutionSnapshot`: every re-enqueue. */
|
||||
public standaloneSnapshotCreates = 0;
|
||||
|
||||
override async createExecutionSnapshot(
|
||||
input: any,
|
||||
tx?: any
|
||||
): ReturnType<PostgresRunStore["createExecutionSnapshot"]> {
|
||||
this.snapshotCreates++;
|
||||
this.standaloneSnapshotCreates++;
|
||||
return super.createExecutionSnapshot(input, tx);
|
||||
}
|
||||
|
||||
override async createRun(
|
||||
params: Parameters<PostgresRunStore["createRun"]>[0],
|
||||
tx?: any
|
||||
): ReturnType<PostgresRunStore["createRun"]> {
|
||||
this.nestedSnapshotCreates++;
|
||||
return super.createRun(params, tx);
|
||||
}
|
||||
}
|
||||
|
||||
describe("RunEngine enqueueRun store routing", () => {
|
||||
// The QUEUED snapshot written while enqueuing a run routes through the injected store.
|
||||
containerTest(
|
||||
"enqueueRun snapshot routes through the store",
|
||||
"the QUEUED snapshot routes through the store as a single nested write",
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const countingStore = new CountingPostgresRunStore({ prisma, readOnlyPrisma: prisma });
|
||||
const engine = new RunEngine(createEngineOptions(redisOptions, prisma, countingStore));
|
||||
@@ -74,7 +84,8 @@ describe("RunEngine enqueueRun store routing", () => {
|
||||
const taskIdentifier = "test-task";
|
||||
await setupBackgroundWorker(engine, environment, taskIdentifier);
|
||||
|
||||
const before = countingStore.snapshotCreates;
|
||||
const nestedBefore = countingStore.nestedSnapshotCreates;
|
||||
const standaloneBefore = countingStore.standaloneSnapshotCreates;
|
||||
|
||||
const run = await engine.trigger(
|
||||
{
|
||||
@@ -96,7 +107,8 @@ describe("RunEngine enqueueRun store routing", () => {
|
||||
prisma
|
||||
);
|
||||
|
||||
expect(countingStore.snapshotCreates).toBeGreaterThan(before);
|
||||
expect(countingStore.nestedSnapshotCreates).toBe(nestedBefore + 1);
|
||||
expect(countingStore.standaloneSnapshotCreates).toBe(standaloneBefore);
|
||||
|
||||
const latest = await getLatestExecutionSnapshot(prisma, run.id);
|
||||
assertNonNullable(latest);
|
||||
|
||||
@@ -7,6 +7,7 @@ import type {
|
||||
import type { RunStore } from "@internal/run-store";
|
||||
import { parseNaturalLanguageDuration } from "@trigger.dev/core/v3/isomorphic";
|
||||
import type { MinimalAuthenticatedEnvironment } from "../../shared/index.js";
|
||||
import { QUEUED_SNAPSHOT_DESCRIPTION, QUEUED_SNAPSHOT_STATUS } from "../consts.js";
|
||||
import type { ExecutionSnapshotSystem } from "./executionSnapshotSystem.js";
|
||||
import type { SystemResources } from "./systems.js";
|
||||
|
||||
@@ -95,8 +96,8 @@ export class EnqueueSystem {
|
||||
{
|
||||
run: run,
|
||||
snapshot: {
|
||||
executionStatus: snapshot?.status ?? "QUEUED",
|
||||
description: snapshot?.description ?? "Run was QUEUED",
|
||||
executionStatus: snapshot?.status ?? QUEUED_SNAPSHOT_STATUS,
|
||||
description: snapshot?.description ?? QUEUED_SNAPSHOT_DESCRIPTION,
|
||||
metadata: snapshot?.metadata ?? undefined,
|
||||
},
|
||||
previousSnapshotId,
|
||||
@@ -113,42 +114,74 @@ export class EnqueueSystem {
|
||||
store
|
||||
);
|
||||
|
||||
// Force development runs to use the environment id as the worker queue.
|
||||
const workerQueue = env.type === "DEVELOPMENT" ? env.id : run.workerQueue;
|
||||
|
||||
const queuePositionMs = (run.queueTimestamp ?? run.createdAt).getTime();
|
||||
const timestamp = queuePositionMs - run.priorityMs;
|
||||
const eligibleAtMs = anchorEligibilityAtQueuePosition ? queuePositionMs : Date.now();
|
||||
|
||||
let ttlExpiresAt: number | undefined;
|
||||
if (includeTtl && run.ttl) {
|
||||
const expireAt = parseNaturalLanguageDuration(run.ttl);
|
||||
if (expireAt) {
|
||||
ttlExpiresAt = expireAt.getTime();
|
||||
}
|
||||
}
|
||||
|
||||
await this.$.runQueue.enqueueMessage({
|
||||
await this.publishRun({
|
||||
run,
|
||||
env,
|
||||
workerQueue,
|
||||
includeTtl,
|
||||
anchorEligibilityAtQueuePosition,
|
||||
enableFastPath,
|
||||
message: {
|
||||
runId: run.id,
|
||||
taskIdentifier: run.taskIdentifier,
|
||||
orgId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
environmentId: env.id,
|
||||
environmentType: env.type,
|
||||
queue: run.queue,
|
||||
concurrencyKey: run.concurrencyKey ?? undefined,
|
||||
timestamp,
|
||||
eligibleAtMs,
|
||||
attempt: 0,
|
||||
ttlExpiresAt,
|
||||
},
|
||||
});
|
||||
|
||||
return newSnapshot;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Publishes the run to the RunQueue without writing an execution snapshot. Callers that already
|
||||
* hold a `QUEUED` snapshot (the trigger path writes one inside the run-create transaction) use
|
||||
* this so the run does not pay for a second snapshot write.
|
||||
*/
|
||||
public async publishRun({
|
||||
run,
|
||||
env,
|
||||
includeTtl = false,
|
||||
anchorEligibilityAtQueuePosition = false,
|
||||
enableFastPath = false,
|
||||
}: {
|
||||
run: TaskRun;
|
||||
env: MinimalAuthenticatedEnvironment;
|
||||
/** See `enqueueRun`. */
|
||||
includeTtl?: boolean;
|
||||
/** See `enqueueRun`. */
|
||||
anchorEligibilityAtQueuePosition?: boolean;
|
||||
/** When true, allow the queue to push directly to worker queue if concurrency is available. */
|
||||
enableFastPath?: boolean;
|
||||
}) {
|
||||
// Force development runs to use the environment id as the worker queue.
|
||||
const workerQueue = env.type === "DEVELOPMENT" ? env.id : run.workerQueue;
|
||||
|
||||
const queuePositionMs = (run.queueTimestamp ?? run.createdAt).getTime();
|
||||
const timestamp = queuePositionMs - run.priorityMs;
|
||||
const eligibleAtMs = anchorEligibilityAtQueuePosition ? queuePositionMs : Date.now();
|
||||
|
||||
// Include TTL only when explicitly requested (first enqueue from trigger).
|
||||
// Re-enqueues (waitpoint, checkpoint, delayed, pending version) must not add TTL.
|
||||
let ttlExpiresAt: number | undefined;
|
||||
if (includeTtl && run.ttl) {
|
||||
const expireAt = parseNaturalLanguageDuration(run.ttl);
|
||||
if (expireAt) {
|
||||
ttlExpiresAt = expireAt.getTime();
|
||||
}
|
||||
}
|
||||
|
||||
await this.$.runQueue.enqueueMessage({
|
||||
env,
|
||||
workerQueue,
|
||||
enableFastPath,
|
||||
message: {
|
||||
runId: run.id,
|
||||
taskIdentifier: run.taskIdentifier,
|
||||
orgId: env.organization.id,
|
||||
projectId: env.project.id,
|
||||
environmentId: env.id,
|
||||
environmentType: env.type,
|
||||
queue: run.queue,
|
||||
concurrencyKey: run.concurrencyKey ?? undefined,
|
||||
timestamp,
|
||||
eligibleAtMs,
|
||||
attempt: 0,
|
||||
ttlExpiresAt,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,6 +65,14 @@ class CountingPostgresRunStore extends PostgresRunStore {
|
||||
return super.createExecutionSnapshot(input, tx);
|
||||
}
|
||||
|
||||
override async createRun(
|
||||
params: Parameters<PostgresRunStore["createRun"]>[0],
|
||||
tx?: any
|
||||
): ReturnType<PostgresRunStore["createRun"]> {
|
||||
this.creates++;
|
||||
return super.createRun(params, tx);
|
||||
}
|
||||
|
||||
override async findLatestExecutionSnapshot(
|
||||
runId: string,
|
||||
client?: any
|
||||
|
||||
@@ -7,7 +7,7 @@ import type { PrismaClient } from "@trigger.dev/database";
|
||||
import { RunEngine } from "../index.js";
|
||||
import { getExecutionSnapshotsSince } from "../systems/executionSnapshotSystem.js";
|
||||
import { copySnapshotsToReplica, createTestMetricsMeter } from "./helpers/replicaTestHelpers.js";
|
||||
import { setupTestScenario } from "./helpers/snapshotTestHelpers.js";
|
||||
import { createTestSnapshot, setupTestScenario } from "./helpers/snapshotTestHelpers.js";
|
||||
import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js";
|
||||
|
||||
vi.setConfig({ testTimeout: 120_000 });
|
||||
@@ -1154,6 +1154,15 @@ describe("RunEngine getSnapshotsSince", () => {
|
||||
workerQueue: "main",
|
||||
});
|
||||
|
||||
await createTestSnapshot(prisma, {
|
||||
runId: run.id,
|
||||
status: "EXECUTING",
|
||||
environmentId: authenticatedEnvironment.id,
|
||||
environmentType: authenticatedEnvironment.type,
|
||||
projectId: authenticatedEnvironment.project.id,
|
||||
organizationId: authenticatedEnvironment.organization.id,
|
||||
});
|
||||
|
||||
const allSnapshots = await prisma.taskRunExecutionSnapshot.findMany({
|
||||
where: { runId: run.id, isValid: true },
|
||||
orderBy: { createdAt: "asc" },
|
||||
|
||||
@@ -137,9 +137,6 @@ const cancelledSnapshot = (friendlyId: string, environment: any) => ({
|
||||
});
|
||||
|
||||
describe("RunEngine trigger/create routing", () => {
|
||||
// trigger create routes through runStore.createRun with the structured
|
||||
// DTO, and the persisted run + its nested first RUN_CREATED snapshot land via
|
||||
// the single create call.
|
||||
containerTest(
|
||||
"trigger routes createRun and lands run + first snapshot",
|
||||
async ({ prisma, redisOptions }) => {
|
||||
@@ -169,7 +166,7 @@ describe("RunEngine trigger/create routing", () => {
|
||||
orderBy: { createdAt: "asc" },
|
||||
});
|
||||
expect(snapshot).not.toBeNull();
|
||||
expect(snapshot!.executionStatus).toBe("RUN_CREATED");
|
||||
expect(snapshot!.executionStatus).toBe("QUEUED");
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,370 @@
|
||||
import { assertNonNullable, containerTest } from "@internal/testcontainers";
|
||||
import { trace } from "@internal/tracing";
|
||||
import type { PrismaClient, TaskRunExecutionStatus } from "@trigger.dev/database";
|
||||
import { setTimeout } from "node:timers/promises";
|
||||
import { expect } from "vitest";
|
||||
import { RunEngine } from "../index.js";
|
||||
import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "./setup.js";
|
||||
|
||||
vi.setConfig({ testTimeout: 60_000 });
|
||||
|
||||
async function snapshotStatuses(
|
||||
prisma: PrismaClient,
|
||||
runId: string
|
||||
): Promise<TaskRunExecutionStatus[]> {
|
||||
const snapshots = await prisma.taskRunExecutionSnapshot.findMany({
|
||||
where: { runId },
|
||||
orderBy: { createdAt: "asc" },
|
||||
select: { executionStatus: true },
|
||||
});
|
||||
|
||||
return snapshots.map((snapshot) => snapshot.executionStatus);
|
||||
}
|
||||
|
||||
describe("RunEngine trigger() execution snapshots", () => {
|
||||
containerTest(
|
||||
"a non-delayed run is created with a single QUEUED snapshot",
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
|
||||
const engine = new RunEngine({
|
||||
prisma,
|
||||
worker: {
|
||||
redis: redisOptions,
|
||||
workers: 1,
|
||||
tasksPerWorker: 10,
|
||||
pollIntervalMs: 100,
|
||||
},
|
||||
queue: {
|
||||
redis: redisOptions,
|
||||
masterQueueConsumersDisabled: true,
|
||||
processWorkerQueueDebounceMs: 50,
|
||||
},
|
||||
runLock: {
|
||||
redis: redisOptions,
|
||||
},
|
||||
machines: {
|
||||
defaultMachine: "small-1x",
|
||||
machines: {
|
||||
"small-1x": {
|
||||
name: "small-1x" as const,
|
||||
cpu: 0.5,
|
||||
memory: 0.5,
|
||||
centsPerMs: 0.0001,
|
||||
},
|
||||
},
|
||||
baseCostInCents: 0.0001,
|
||||
},
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
});
|
||||
|
||||
try {
|
||||
const taskIdentifier = "test-task";
|
||||
|
||||
await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier);
|
||||
|
||||
const run = await engine.trigger(
|
||||
{
|
||||
number: 1,
|
||||
friendlyId: "run_1234",
|
||||
environment: authenticatedEnvironment,
|
||||
taskIdentifier,
|
||||
payload: "{}",
|
||||
payloadType: "application/json",
|
||||
context: {},
|
||||
traceContext: {},
|
||||
traceId: "t_collapse_1",
|
||||
spanId: "s_collapse_1",
|
||||
workerQueue: "main",
|
||||
queue: "task/test-task",
|
||||
isTest: false,
|
||||
tags: [],
|
||||
},
|
||||
prisma
|
||||
);
|
||||
|
||||
expect(await snapshotStatuses(prisma, run.id)).toEqual(["QUEUED"]);
|
||||
|
||||
const queueLength = await engine.runQueue.lengthOfQueue(
|
||||
authenticatedEnvironment,
|
||||
run.queue
|
||||
);
|
||||
expect(queueLength).toBe(1);
|
||||
|
||||
const executionData = await engine.getRunExecutionData({ runId: run.id });
|
||||
assertNonNullable(executionData);
|
||||
expect(executionData.snapshot.executionStatus).toBe("QUEUED");
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
containerTest(
|
||||
"the QUEUED snapshot event is stamped at write time, not at an overridden run createdAt",
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
|
||||
const engine = new RunEngine({
|
||||
prisma,
|
||||
worker: {
|
||||
redis: redisOptions,
|
||||
workers: 1,
|
||||
tasksPerWorker: 10,
|
||||
pollIntervalMs: 100,
|
||||
},
|
||||
queue: {
|
||||
redis: redisOptions,
|
||||
masterQueueConsumersDisabled: true,
|
||||
processWorkerQueueDebounceMs: 50,
|
||||
},
|
||||
runLock: {
|
||||
redis: redisOptions,
|
||||
},
|
||||
machines: {
|
||||
defaultMachine: "small-1x",
|
||||
machines: {
|
||||
"small-1x": {
|
||||
name: "small-1x" as const,
|
||||
cpu: 0.5,
|
||||
memory: 0.5,
|
||||
centsPerMs: 0.0001,
|
||||
},
|
||||
},
|
||||
baseCostInCents: 0.0001,
|
||||
},
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
});
|
||||
|
||||
try {
|
||||
const taskIdentifier = "test-task";
|
||||
|
||||
await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier);
|
||||
|
||||
const stampedTimes: Date[] = [];
|
||||
engine.eventBus.on("executionSnapshotCreated", ({ time }) => {
|
||||
stampedTimes.push(time);
|
||||
});
|
||||
|
||||
const backdatedCreatedAt = new Date(Date.now() - 60 * 60 * 1000);
|
||||
const triggeredAt = Date.now();
|
||||
|
||||
const run = await engine.trigger(
|
||||
{
|
||||
number: 1,
|
||||
friendlyId: "run_1236",
|
||||
environment: authenticatedEnvironment,
|
||||
taskIdentifier,
|
||||
payload: "{}",
|
||||
payloadType: "application/json",
|
||||
context: {},
|
||||
traceContext: {},
|
||||
traceId: "t_collapse_3",
|
||||
spanId: "s_collapse_3",
|
||||
workerQueue: "main",
|
||||
queue: "task/test-task",
|
||||
isTest: false,
|
||||
tags: [],
|
||||
createdAt: backdatedCreatedAt,
|
||||
},
|
||||
prisma
|
||||
);
|
||||
|
||||
const storedRun = await prisma.taskRun.findUnique({ where: { id: run.id } });
|
||||
expect(storedRun?.createdAt.getTime()).toBe(backdatedCreatedAt.getTime());
|
||||
|
||||
expect(stampedTimes.length).toBe(1);
|
||||
expect(stampedTimes[0].getTime()).toBeGreaterThanOrEqual(triggeredAt);
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
containerTest(
|
||||
"a delayed run keeps DELAYED and QUEUED as separate snapshots",
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
|
||||
const engine = new RunEngine({
|
||||
prisma,
|
||||
worker: {
|
||||
redis: redisOptions,
|
||||
workers: 1,
|
||||
tasksPerWorker: 10,
|
||||
pollIntervalMs: 100,
|
||||
},
|
||||
queue: {
|
||||
redis: redisOptions,
|
||||
masterQueueConsumersDisabled: true,
|
||||
processWorkerQueueDebounceMs: 50,
|
||||
},
|
||||
runLock: {
|
||||
redis: redisOptions,
|
||||
},
|
||||
machines: {
|
||||
defaultMachine: "small-1x",
|
||||
machines: {
|
||||
"small-1x": {
|
||||
name: "small-1x" as const,
|
||||
cpu: 0.5,
|
||||
memory: 0.5,
|
||||
centsPerMs: 0.0001,
|
||||
},
|
||||
},
|
||||
baseCostInCents: 0.0001,
|
||||
},
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
});
|
||||
|
||||
try {
|
||||
const taskIdentifier = "test-task";
|
||||
|
||||
await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier);
|
||||
|
||||
const run = await engine.trigger(
|
||||
{
|
||||
number: 1,
|
||||
friendlyId: "run_1235",
|
||||
environment: authenticatedEnvironment,
|
||||
taskIdentifier,
|
||||
payload: "{}",
|
||||
payloadType: "application/json",
|
||||
context: {},
|
||||
traceContext: {},
|
||||
traceId: "t_collapse_2",
|
||||
spanId: "s_collapse_2",
|
||||
workerQueue: "main",
|
||||
queue: "task/test-task",
|
||||
isTest: false,
|
||||
tags: [],
|
||||
delayUntil: new Date(Date.now() + 500),
|
||||
},
|
||||
prisma
|
||||
);
|
||||
|
||||
expect(await snapshotStatuses(prisma, run.id)).toEqual(["DELAYED"]);
|
||||
|
||||
await setTimeout(1_500);
|
||||
|
||||
expect(await snapshotStatuses(prisma, run.id)).toEqual(["DELAYED", "QUEUED"]);
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
containerTest(
|
||||
"a resumed run still writes its own QUEUED snapshot",
|
||||
async ({ prisma, redisOptions }) => {
|
||||
const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
|
||||
|
||||
const engine = new RunEngine({
|
||||
prisma,
|
||||
worker: {
|
||||
redis: redisOptions,
|
||||
workers: 1,
|
||||
tasksPerWorker: 10,
|
||||
pollIntervalMs: 100,
|
||||
},
|
||||
queue: {
|
||||
redis: redisOptions,
|
||||
masterQueueConsumersDisabled: true,
|
||||
processWorkerQueueDebounceMs: 50,
|
||||
},
|
||||
runLock: {
|
||||
redis: redisOptions,
|
||||
},
|
||||
machines: {
|
||||
defaultMachine: "small-1x",
|
||||
machines: {
|
||||
"small-1x": {
|
||||
name: "small-1x" as const,
|
||||
cpu: 0.5,
|
||||
memory: 0.5,
|
||||
centsPerMs: 0.0001,
|
||||
},
|
||||
},
|
||||
baseCostInCents: 0.0001,
|
||||
},
|
||||
tracer: trace.getTracer("test", "0.0.0"),
|
||||
});
|
||||
|
||||
try {
|
||||
const taskIdentifier = "test-task";
|
||||
|
||||
await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier);
|
||||
|
||||
const run = await engine.trigger(
|
||||
{
|
||||
number: 1,
|
||||
friendlyId: "run_1237",
|
||||
environment: authenticatedEnvironment,
|
||||
taskIdentifier,
|
||||
payload: "{}",
|
||||
payloadType: "application/json",
|
||||
context: {},
|
||||
traceContext: {},
|
||||
traceId: "t_collapse_4",
|
||||
spanId: "s_collapse_4",
|
||||
workerQueue: "main",
|
||||
queue: "task/test-task",
|
||||
isTest: false,
|
||||
tags: [],
|
||||
},
|
||||
prisma
|
||||
);
|
||||
|
||||
await setTimeout(500);
|
||||
const dequeued = await engine.dequeueFromWorkerQueue({
|
||||
consumerId: "test_collapse_4",
|
||||
workerQueue: "main",
|
||||
});
|
||||
assertNonNullable(dequeued[0]);
|
||||
|
||||
await engine.startRunAttempt({
|
||||
runId: dequeued[0].run.id,
|
||||
snapshotId: dequeued[0].snapshot.id,
|
||||
});
|
||||
|
||||
const waitpointResult = await engine.createManualWaitpoint({
|
||||
environmentId: authenticatedEnvironment.id,
|
||||
projectId: authenticatedEnvironment.projectId,
|
||||
});
|
||||
|
||||
const blockedResult = await engine.blockRunWithWaitpoint({
|
||||
runId: run.id,
|
||||
waitpoints: waitpointResult.waitpoint.id,
|
||||
projectId: authenticatedEnvironment.projectId,
|
||||
organizationId: authenticatedEnvironment.organizationId,
|
||||
});
|
||||
|
||||
const checkpointResult = await engine.createCheckpoint({
|
||||
runId: run.id,
|
||||
snapshotId: blockedResult.id,
|
||||
checkpoint: {
|
||||
type: "DOCKER",
|
||||
reason: "TEST_CHECKPOINT",
|
||||
location: "test-location",
|
||||
imageRef: "test-image-ref",
|
||||
},
|
||||
});
|
||||
expect(checkpointResult.ok).toBe(true);
|
||||
|
||||
await engine.completeWaitpoint({ id: waitpointResult.waitpoint.id });
|
||||
await setTimeout(500);
|
||||
|
||||
expect(await snapshotStatuses(prisma, run.id)).toEqual([
|
||||
"QUEUED",
|
||||
"PENDING_EXECUTING",
|
||||
"EXECUTING",
|
||||
"EXECUTING_WITH_WAITPOINTS",
|
||||
"SUSPENDED",
|
||||
"QUEUED",
|
||||
]);
|
||||
} finally {
|
||||
await engine.quit();
|
||||
}
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -680,6 +680,7 @@ export class PostgresRunStore implements RunStore {
|
||||
const client = tx ?? this.prisma;
|
||||
|
||||
const snapshotCreate = {
|
||||
id: params.snapshot.id,
|
||||
engine: params.snapshot.engine,
|
||||
executionStatus: params.snapshot.executionStatus,
|
||||
description: params.snapshot.description,
|
||||
|
||||
@@ -29,6 +29,7 @@ export type IdempotencyKeyRunMatch = {
|
||||
};
|
||||
|
||||
export type CreateRunSnapshotInput = {
|
||||
id?: string;
|
||||
engine: "V2";
|
||||
executionStatus: TaskRunExecutionStatus;
|
||||
description: string;
|
||||
|
||||
Reference in New Issue
Block a user