19908436b8
## Summary Speeds up webapp test jobs by balancing measured work across runners, reducing repeated container setup, and ensuring test workers release shutdown resources promptly. Unit tests run across 24 duration-aware shards, while E2E tests run across two balanced shards. ## Design `RunEngine` shutdown now closes processing resources before support resources, continues cleanup if one close fails, and reuses one shutdown promise for concurrent callers. Redis workers clear completed shutdown deadlines so finished tests no longer wait on idle timers. Container-heavy suites are split only where it improves parallelism, and repeated replication and engine fixtures are consolidated where one end-to-end case provides coverage. Timing weights are refreshed for all affected files. Dependency installation overlaps container pulls, and both workflows use WarpBuild's Node setup action.
118 lines
3.8 KiB
TypeScript
118 lines
3.8 KiB
TypeScript
import { RunEngine } from "@internal/run-engine";
|
|
import { trace } from "@opentelemetry/api";
|
|
import type { PrismaClient } from "@trigger.dev/database";
|
|
import type { RedisOptions } from "ioredis";
|
|
import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server";
|
|
import { DefaultQueueManager } from "~/runEngine/concerns/queues.server";
|
|
import type {
|
|
ExternalDeploymentCache,
|
|
ExternalDeploymentCacheEntry,
|
|
} from "~/services/externalDeploymentCache.server";
|
|
import { RunEngineTriggerTaskService } from "../../app/runEngine/services/triggerTask.server";
|
|
import {
|
|
MockPayloadProcessor,
|
|
MockTraceEventConcern,
|
|
MockTriggerTaskValidator,
|
|
} from "./triggerTaskTestHelpers";
|
|
|
|
export class RecordingExternalDeploymentCache implements ExternalDeploymentCache {
|
|
readonly gets: Array<{ environmentId: string; externalId: string }> = [];
|
|
readonly writes: Array<{
|
|
environmentId: string;
|
|
externalId: string;
|
|
entry: ExternalDeploymentCacheEntry;
|
|
}> = [];
|
|
readonly missing: Array<{ environmentId: string; externalId: string }> = [];
|
|
private readonly entries = new Map<string, ExternalDeploymentCacheEntry>();
|
|
|
|
constructor(
|
|
entries: Array<{
|
|
environmentId: string;
|
|
externalId: string;
|
|
entry: ExternalDeploymentCacheEntry;
|
|
}> = []
|
|
) {
|
|
for (const { environmentId, externalId, entry } of entries) {
|
|
this.entries.set(this.key(environmentId, externalId), entry);
|
|
}
|
|
}
|
|
|
|
async get(environmentId: string, externalId: string) {
|
|
this.gets.push({ environmentId, externalId });
|
|
|
|
const entry = this.entries.get(this.key(environmentId, externalId));
|
|
|
|
if (entry) {
|
|
return { outcome: "deployed" as const, entry };
|
|
}
|
|
|
|
return this.missing.some(
|
|
(missing) => missing.environmentId === environmentId && missing.externalId === externalId
|
|
)
|
|
? { outcome: "missing" as const }
|
|
: null;
|
|
}
|
|
|
|
async setIfNewer(environmentId: string, externalId: string, entry: ExternalDeploymentCacheEntry) {
|
|
this.writes.push({ environmentId, externalId, entry });
|
|
this.entries.set(this.key(environmentId, externalId), entry);
|
|
}
|
|
|
|
async setMissing(environmentId: string, externalId: string) {
|
|
this.missing.push({ environmentId, externalId });
|
|
}
|
|
|
|
private key(environmentId: string, externalId: string) {
|
|
return JSON.stringify([environmentId, externalId]);
|
|
}
|
|
}
|
|
|
|
export function createEngine(prisma: PrismaClient, redisOptions: RedisOptions) {
|
|
return new RunEngine({
|
|
prisma,
|
|
worker: { redis: redisOptions, disabled: true },
|
|
queue: {
|
|
redis: redisOptions,
|
|
masterQueueConsumersDisabled: true,
|
|
ttlSystem: { disabled: true },
|
|
},
|
|
batchQueue: { redis: redisOptions, consumerEnabled: false },
|
|
runLock: { redis: redisOptions },
|
|
machines: {
|
|
defaultMachine: "small-1x",
|
|
machines: {
|
|
"small-1x": { name: "small-1x", cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 },
|
|
},
|
|
baseCostInCents: 0.0005,
|
|
},
|
|
tracer: trace.getTracer("test", "0.0.0"),
|
|
});
|
|
}
|
|
|
|
export function createService(
|
|
prisma: PrismaClient,
|
|
engine: RunEngine,
|
|
externalDeploymentCache: ExternalDeploymentCache
|
|
) {
|
|
return new RunEngineTriggerTaskService({
|
|
engine,
|
|
prisma,
|
|
payloadProcessor: new MockPayloadProcessor(),
|
|
queueConcern: new DefaultQueueManager(prisma, engine),
|
|
idempotencyKeyConcern: new IdempotencyKeyConcern(prisma, engine, new MockTraceEventConcern()),
|
|
validator: new MockTriggerTaskValidator(),
|
|
traceEventConcern: new MockTraceEventConcern(),
|
|
tracer: trace.getTracer("test", "0.0.0"),
|
|
metadataMaximumSize: 1024 * 1024,
|
|
externalDeploymentCache,
|
|
});
|
|
}
|
|
|
|
export async function nameDeploymentWithExternalId(
|
|
prisma: PrismaClient,
|
|
workerId: string,
|
|
externalId: string
|
|
) {
|
|
await prisma.workerDeployment.update({ where: { workerId }, data: { externalId } });
|
|
}
|