Files
triggerdotdev--trigger.dev/apps/webapp/test/engine/triggerTask.debounce.test.ts
Matt Aitken 04f9c4e1a5 fix(webapp,run-engine,core): drop the hidden debounce ceiling, fail fast on an unusable maxDelay (#4521)
Debouncing with a `delay` longer than an hour did nothing at all.

The engine applied a server-side ceiling on how long a debounced run
could be pushed back, measured from the run's `createdAt` and defaulting
to one hour. A run is only pushed back while its new execution time
stays inside that ceiling, so a `delay` at or above it could never push
anything: the waiting run was released, the trigger started its own run,
and the next trigger repeated it. A `delay: "12h"` produced one run per
trigger, each correctly delayed by 12h, with no error raised and nothing
on the run to show the debounce key had been ignored.

The ceiling is now unset by default. A debounce key with no `maxDelay`
keeps collapsing triggers for as long as they keep arriving, which is
what the docs have always described. Self-hosters who want a bound can
still set `RUN_ENGINE_MAXIMUM_DEBOUNCE_DURATION_MS`.

That has a consequence worth stating plainly, so the docs now carry a
warning for it: with no `maxDelay`, a continuously triggered key never
executes. Set `maxDelay` when the work has to happen eventually.

**Failing fast on an unusable `maxDelay`.** A caller who sets `maxDelay`
no longer than their `delay` hits exactly the dead end described above,
so that pair is now rejected at trigger time instead of silently
behaving as if no debounce were set:

```
debounce.maxDelay (1h) must be longer than debounce.delay (12h). A debounced run is only
pushed back while it stays inside maxDelay, so with these values every trigger would create
its own run.
```

An unparseable `maxDelay` is rejected too, rather than quietly falling
back to no bound at all, and so is a `delay` given as a date rather than
a duration, which could never work because the value is re-applied on
every push.

The same check runs against a configured server ceiling, so a
self-hosted deployment that sets
`RUN_ENGINE_MAXIMUM_DEBOUNCE_DURATION_MS` gets the error rather than the
silent failure this PR is about. With no `maxDelay` and no configured
ceiling, which is the default, there is nothing to conflict with and
nothing is rejected.

The docs, the `TriggerOptions` JSDoc and the engine option all now state
that the room available to push is the gap between `delay` and
`maxDelay`. The run engine suite gains the case that motivated this:
four triggers on one key with a 12h delay now collapse to a single run.
2026-08-07 07:55:35 +00:00

610 lines
19 KiB
TypeScript

import { describe, expect, onTestFinished, vi } from "vitest";
// db.server + splitMode are mocked so the idempotency dedup client resolves to
// the container prisma passed into the concern (split stays off).
vi.mock("~/db.server", () => ({
prisma: {},
$replica: {},
runOpsNewPrisma: {},
runOpsLegacyPrisma: {},
}));
vi.mock("~/v3/runOpsMigration/splitMode.server", () => ({ isSplitEnabled: async () => false }));
vi.mock("~/services/platform.v3.server", async (importOriginal) => {
const actual = (await importOriginal()) as Record<string, unknown>;
return {
...actual,
getEntitlement: vi.fn(),
};
});
import { RunEngine } from "@internal/run-engine";
import { setupAuthenticatedEnvironment, setupBackgroundWorker } from "@internal/run-engine/tests";
import { containerTest } from "@internal/testcontainers";
import { trace } from "@opentelemetry/api";
import type { IOPacket } from "@trigger.dev/core/v3";
import { IdempotencyKeyConcern } from "~/runEngine/concerns/idempotencyKeys.server";
import { DefaultQueueManager } from "~/runEngine/concerns/queues.server";
import type { PayloadProcessor, TriggerTaskRequest } from "~/runEngine/types";
import { RunEngineTriggerTaskService } from "../../app/runEngine/services/triggerTask.server";
import { setTimeout } from "node:timers/promises";
import {
MockPayloadProcessor,
MockTraceEventConcern,
MockTriggerRacepointSystem,
MockTriggerTaskValidator,
} from "./triggerTaskTestHelpers";
vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 });
describe("RunEngineTriggerTaskService", () => {
containerTest(
"should preserve runFriendlyId across retries when RunDuplicateIdempotencyKeyError is thrown",
async ({ prisma, redisOptions }) => {
const engine = new RunEngine({
prisma,
worker: {
redis: redisOptions,
workers: 1,
tasksPerWorker: 10,
pollIntervalMs: 100,
},
queue: {
redis: redisOptions,
},
runLock: {
redis: redisOptions,
},
machines: {
defaultMachine: "small-1x",
machines: {
"small-1x": {
name: "small-1x" as const,
cpu: 0.5,
memory: 0.5,
centsPerMs: 0.0001,
},
},
baseCostInCents: 0.0005,
},
tracer: trace.getTracer("test", "0.0.0"),
logLevel: "debug",
});
onTestFinished(() => engine.quit());
const parentTask = "parent-task";
const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
const taskIdentifier = "test-task";
// Create background worker
await setupBackgroundWorker(engine, authenticatedEnvironment, [parentTask, taskIdentifier]);
// Create parent runs and start their attempts (required for resumeParentOnCompletion)
const parentRun1 = await engine.trigger(
{
number: 1,
friendlyId: "run_cmqxvncxq0000kaulzpafkicv",
environment: authenticatedEnvironment,
taskIdentifier: parentTask,
payload: "{}",
payloadType: "application/json",
context: {},
traceContext: {},
traceId: "t12345",
spanId: "s12345",
queue: `task/${parentTask}`,
isTest: false,
tags: [],
workerQueue: "main",
},
prisma
);
await setTimeout(500);
const dequeued = await engine.dequeueFromWorkerQueue({
consumerId: "test_12345",
workerQueue: "main",
});
await engine.startRunAttempt({
runId: parentRun1.id,
snapshotId: dequeued[0].snapshot.id,
});
const parentRun2 = await engine.trigger(
{
number: 2,
friendlyId: "run_cmqxvncxr0001kauldv9mqa9z",
environment: authenticatedEnvironment,
taskIdentifier: parentTask,
payload: "{}",
payloadType: "application/json",
context: {},
traceContext: {},
traceId: "t12346",
spanId: "s12346",
queue: `task/${parentTask}`,
isTest: false,
tags: [],
workerQueue: "main",
},
prisma
);
await setTimeout(500);
const dequeued2 = await engine.dequeueFromWorkerQueue({
consumerId: "test_12345",
workerQueue: "main",
});
await engine.startRunAttempt({
runId: parentRun2.id,
snapshotId: dequeued2[0].snapshot.id,
});
const queuesManager = new DefaultQueueManager(prisma, engine);
const idempotencyKeyConcern = new IdempotencyKeyConcern(
prisma,
engine,
new MockTraceEventConcern()
);
const triggerRacepointSystem = new MockTriggerRacepointSystem();
// Track all friendlyIds passed to the payload processor
const processedFriendlyIds: string[] = [];
class TrackingPayloadProcessor implements PayloadProcessor {
async process(request: TriggerTaskRequest): Promise<IOPacket> {
processedFriendlyIds.push(request.friendlyId);
return {
data: JSON.stringify(request.body.payload),
dataType: "application/json",
};
}
}
const triggerTaskService = new RunEngineTriggerTaskService({
engine,
prisma,
payloadProcessor: new TrackingPayloadProcessor(),
queueConcern: queuesManager,
idempotencyKeyConcern,
validator: new MockTriggerTaskValidator(),
traceEventConcern: new MockTraceEventConcern(),
tracer: trace.getTracer("test", "0.0.0"),
metadataMaximumSize: 1024 * 1024 * 1, // 1MB
triggerRacepointSystem,
});
const idempotencyKey = "test-preserve-friendly-id";
const racepoint = triggerRacepointSystem.registerRacepoint("idempotencyKey", idempotencyKey);
// Trigger two concurrent requests with same idempotency key
// One will succeed, one will fail with RunDuplicateIdempotencyKeyError and retry
const childTriggerPromise1 = triggerTaskService.call({
taskId: taskIdentifier,
environment: authenticatedEnvironment,
body: {
payload: { test: "test1" },
options: {
idempotencyKey,
parentRunId: parentRun1.friendlyId,
resumeParentOnCompletion: true,
},
},
});
const childTriggerPromise2 = triggerTaskService.call({
taskId: taskIdentifier,
environment: authenticatedEnvironment,
body: {
payload: { test: "test2" },
options: {
idempotencyKey,
parentRunId: parentRun2.friendlyId,
resumeParentOnCompletion: true,
},
},
});
await setTimeout(500);
// Resolve the racepoint to allow both requests to proceed
racepoint.resolve();
const result1 = await childTriggerPromise1;
const result2 = await childTriggerPromise2;
// Both should return the same run (one created, one cached)
expect(result1).toBeDefined();
expect(result2).toBeDefined();
expect(result1?.run.friendlyId).toBe(result2?.run.friendlyId);
// The key assertion: When a retry happens due to RunDuplicateIdempotencyKeyError,
// the same friendlyId should be used. We expect exactly 2 calls to payloadProcessor
// (one for each concurrent request), not 3 (which would indicate a new friendlyId on retry)
// Since the retry returns early from the idempotency cache, payloadProcessor is not called again.
expect(processedFriendlyIds.length).toBe(2);
// Verify that we have exactly 2 unique friendlyIds (one per original request)
const uniqueFriendlyIds = new Set(processedFriendlyIds);
expect(uniqueFriendlyIds.size).toBe(2);
}
);
containerTest(
"should reject invalid debounce.delay when no explicit delay is provided",
async ({ prisma, redisOptions }) => {
const engine = new RunEngine({
prisma,
worker: {
redis: redisOptions,
workers: 1,
tasksPerWorker: 10,
pollIntervalMs: 100,
},
queue: {
redis: redisOptions,
},
runLock: {
redis: redisOptions,
},
machines: {
defaultMachine: "small-1x",
machines: {
"small-1x": {
name: "small-1x" as const,
cpu: 0.5,
memory: 0.5,
centsPerMs: 0.0001,
},
},
baseCostInCents: 0.0005,
},
tracer: trace.getTracer("test", "0.0.0"),
});
onTestFinished(() => engine.quit());
const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
const taskIdentifier = "test-task";
await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier);
const queuesManager = new DefaultQueueManager(prisma, engine);
const idempotencyKeyConcern = new IdempotencyKeyConcern(
prisma,
engine,
new MockTraceEventConcern()
);
const triggerTaskService = new RunEngineTriggerTaskService({
engine,
prisma,
payloadProcessor: new MockPayloadProcessor(),
queueConcern: queuesManager,
idempotencyKeyConcern,
validator: new MockTriggerTaskValidator(),
traceEventConcern: new MockTraceEventConcern(),
tracer: trace.getTracer("test", "0.0.0"),
metadataMaximumSize: 1024 * 1024 * 1,
});
// Invalid debounce.delay format (ms not supported)
await expect(
triggerTaskService.call({
taskId: taskIdentifier,
environment: authenticatedEnvironment,
body: {
payload: { test: "test" },
options: {
debounce: {
key: "test-key",
delay: "300ms", // Invalid - ms not supported
},
},
},
})
).rejects.toThrow("Debounce requires a valid delay duration");
}
);
containerTest(
"should reject invalid debounce.delay even when explicit delay is valid",
async ({ prisma, redisOptions }) => {
const engine = new RunEngine({
prisma,
worker: {
redis: redisOptions,
workers: 1,
tasksPerWorker: 10,
pollIntervalMs: 100,
},
queue: {
redis: redisOptions,
},
runLock: {
redis: redisOptions,
},
machines: {
defaultMachine: "small-1x",
machines: {
"small-1x": {
name: "small-1x" as const,
cpu: 0.5,
memory: 0.5,
centsPerMs: 0.0001,
},
},
baseCostInCents: 0.0005,
},
tracer: trace.getTracer("test", "0.0.0"),
});
onTestFinished(() => engine.quit());
const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
const taskIdentifier = "test-task";
await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier);
const queuesManager = new DefaultQueueManager(prisma, engine);
const idempotencyKeyConcern = new IdempotencyKeyConcern(
prisma,
engine,
new MockTraceEventConcern()
);
const triggerTaskService = new RunEngineTriggerTaskService({
engine,
prisma,
payloadProcessor: new MockPayloadProcessor(),
queueConcern: queuesManager,
idempotencyKeyConcern,
validator: new MockTriggerTaskValidator(),
traceEventConcern: new MockTraceEventConcern(),
tracer: trace.getTracer("test", "0.0.0"),
metadataMaximumSize: 1024 * 1024 * 1,
});
// Valid explicit delay but invalid debounce.delay
// This is the bug case: the explicit delay passes validation,
// but debounce.delay would fail later when rescheduling
await expect(
triggerTaskService.call({
taskId: taskIdentifier,
environment: authenticatedEnvironment,
body: {
payload: { test: "test" },
options: {
delay: "5m", // Valid explicit delay
debounce: {
key: "test-key",
delay: "invalid-delay", // Invalid debounce delay
},
},
},
})
).rejects.toThrow("Invalid debounce delay");
}
);
containerTest("should accept valid debounce.delay formats", async ({ prisma, redisOptions }) => {
const engine = new RunEngine({
prisma,
worker: {
redis: redisOptions,
workers: 1,
tasksPerWorker: 10,
pollIntervalMs: 100,
},
queue: {
redis: redisOptions,
},
runLock: {
redis: redisOptions,
},
machines: {
defaultMachine: "small-1x",
machines: {
"small-1x": {
name: "small-1x" as const,
cpu: 0.5,
memory: 0.5,
centsPerMs: 0.0001,
},
},
baseCostInCents: 0.0005,
},
tracer: trace.getTracer("test", "0.0.0"),
});
onTestFinished(() => engine.quit());
const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
const taskIdentifier = "test-task";
await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier);
const queuesManager = new DefaultQueueManager(prisma, engine);
const idempotencyKeyConcern = new IdempotencyKeyConcern(
prisma,
engine,
new MockTraceEventConcern()
);
const triggerTaskService = new RunEngineTriggerTaskService({
engine,
prisma,
payloadProcessor: new MockPayloadProcessor(),
queueConcern: queuesManager,
idempotencyKeyConcern,
validator: new MockTriggerTaskValidator(),
traceEventConcern: new MockTraceEventConcern(),
tracer: trace.getTracer("test", "0.0.0"),
metadataMaximumSize: 1024 * 1024 * 1,
});
// Valid debounce.delay format
const result = await triggerTaskService.call({
taskId: taskIdentifier,
environment: authenticatedEnvironment,
body: {
payload: { test: "test" },
options: {
debounce: {
key: "test-key",
delay: "5s", // Valid format
},
},
},
});
expect(result).toBeDefined();
expect(result?.run.friendlyId).toBeDefined();
});
containerTest(
"should reject a debounce maxDelay that leaves no room to push the run back",
async ({ prisma, redisOptions }) => {
const engine = new RunEngine({
prisma,
worker: {
redis: redisOptions,
workers: 1,
tasksPerWorker: 10,
pollIntervalMs: 100,
},
queue: {
redis: redisOptions,
},
runLock: {
redis: redisOptions,
},
machines: {
defaultMachine: "small-1x",
machines: {
"small-1x": {
name: "small-1x" as const,
cpu: 0.5,
memory: 0.5,
centsPerMs: 0.0001,
},
},
baseCostInCents: 0.0005,
},
tracer: trace.getTracer("test", "0.0.0"),
});
onTestFinished(() => engine.quit());
const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
const taskIdentifier = "test-task";
await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier);
const queuesManager = new DefaultQueueManager(prisma, engine);
const idempotencyKeyConcern = new IdempotencyKeyConcern(
prisma,
engine,
new MockTraceEventConcern()
);
const triggerTaskService = new RunEngineTriggerTaskService({
engine,
prisma,
payloadProcessor: new MockPayloadProcessor(),
queueConcern: queuesManager,
idempotencyKeyConcern,
validator: new MockTriggerTaskValidator(),
traceEventConcern: new MockTraceEventConcern(),
tracer: trace.getTracer("test", "0.0.0"),
metadataMaximumSize: 1024 * 1024 * 1,
});
const triggerWithDebounce = (debounce: { key: string; delay: string; maxDelay?: string }) =>
triggerTaskService.call({
taskId: taskIdentifier,
environment: authenticatedEnvironment,
body: { payload: { test: "test" }, options: { debounce } },
});
await expect(
triggerWithDebounce({ key: "equal", delay: "12h", maxDelay: "12h" })
).rejects.toThrow(/must be longer than debounce.delay/);
await expect(
triggerWithDebounce({ key: "shorter", delay: "12h", maxDelay: "1h" })
).rejects.toThrow(/must be longer than debounce.delay/);
await expect(
triggerWithDebounce({ key: "unparseable", delay: "10s", maxDelay: "soon" })
).rejects.toThrow(/Invalid debounce maxDelay/);
await expect(
triggerWithDebounce({ key: "empty", delay: "10s", maxDelay: "" })
).rejects.toThrow(/Invalid debounce maxDelay/);
const compound = await triggerWithDebounce({
key: "compound",
delay: "2h30m",
maxDelay: "1d",
});
expect(compound?.run.friendlyId).toBeDefined();
const withRoom = await triggerWithDebounce({
key: "with-room",
delay: "10s",
maxDelay: "5m",
});
expect(withRoom?.run.friendlyId).toBeDefined();
const noMaxDelay = await triggerWithDebounce({ key: "no-max-delay", delay: "12h" });
expect(noMaxDelay?.run.friendlyId).toBeDefined();
await expect(
triggerWithDebounce({ key: "date-delay", delay: "2027-01-01T00:00:00.000Z" })
).rejects.toThrow(/must be a duration, not a date/);
const withServerCeiling = new RunEngineTriggerTaskService({
engine,
prisma,
payloadProcessor: new MockPayloadProcessor(),
queueConcern: queuesManager,
idempotencyKeyConcern,
validator: new MockTriggerTaskValidator(),
traceEventConcern: new MockTraceEventConcern(),
tracer: trace.getTracer("test", "0.0.0"),
metadataMaximumSize: 1024 * 1024 * 1,
maximumDebounceDurationMs: 60 * 60 * 1000,
});
await expect(
withServerCeiling.call({
taskId: taskIdentifier,
environment: authenticatedEnvironment,
body: {
payload: { test: "test" },
options: { debounce: { key: "over-server-ceiling", delay: "12h" } },
},
})
).rejects.toThrow(/at or above this server's maximum debounce duration of 1h/);
const underServerCeiling = await withServerCeiling.call({
taskId: taskIdentifier,
environment: authenticatedEnvironment,
body: {
payload: { test: "test" },
options: { debounce: { key: "under-server-ceiling", delay: "10s" } },
},
});
expect(underServerCeiling?.run.friendlyId).toBeDefined();
const overCeilingWithMaxDelay = await withServerCeiling.call({
taskId: taskIdentifier,
environment: authenticatedEnvironment,
body: {
payload: { test: "test" },
options: { debounce: { key: "override", delay: "12h", maxDelay: "24h" } },
},
});
expect(overCeilingWithMaxDelay?.run.friendlyId).toBeDefined();
}
);
});