092b9ef07a
## Problem
The run-ops split mints NEW-store run ids as **27-char base62 KSUIDs**.
The supervisor writes the run id into the Kubernetes pod name
(`runner-<id>`), and pod names must be DNS-1123 labels (lowercase
`[a-z0-9-]`) — so uppercase base62 ids make k8s reject the pod (422) and
**those runs never launch** (they loop in `PENDING_EXECUTING` until the
heartbeat-stall handler nacks them, forever). `.toLowerCase()` can't fix
it: base62 has both `A`(10) and `a`(36) as distinct symbols, so folding
collides distinct ids and destroys sort order.
## Fix: change the encoding, not the structure
Mint a **26-char lowercase base32hex** run id:
```
run_<24-char base32hex core><region char><version char>
[ 6-byte ms timestamp ][ 9 CSPRNG bytes ]
```
- **base32hex** (RFC 4648 §7, alphabet `0-9a-v`): lowercase,
order-preserving, DNS-safe; 15 bytes → exactly 24 chars, no padding.
Hand-rolled encode/decode (no new dependency).
- **48-bit ms timestamp** in the leading bytes → plain string sort ==
creation order at millisecond resolution.
- **72 bits CSPRNG** entropy; PK unique constraint is the backstop (no
retry loop).
- **region / version** are raw positional chars (read via one `charAt`
before decoding/routing), version = `"1"`.
DNS-safe from birth and hyphen-free, so **firekeeper is unchanged** —
`runner-<id>-attempt-N` → strip `runner-`, cut at first hyphen still
recovers the exact id incl. region+version.
## Residency discriminator: length → version char
`classifyKind`/`classifyResidency` (`runOpsResidency.ts`) previously
distinguished NEW vs LEGACY by **id length**. That gets ambiguous with a
third format. It now discriminates on the **version char at a fixed
position** (`isRunOpsIdBody`: 26 chars, `[25] === "1"`, base32hex
alphabet) → NEW; everything else → LEGACY. Total, never throws. The
`Residency` (NEW/LEGACY) contract the routing store consumes is
unchanged; the `"ksuid"` `ResidencyKind` label is retained only because
it's the persisted `runOpsMintKsuid` feature-flag value.
## Scope / verification
- Generator + discriminator in `@trigger.dev/core` isomorphic; mint path
+ all id-shape call sites swept (~40 webapp files); changeset added
(`@trigger.dev/core` patch).
- Core unit tests (encode/decode round-trip + property, generator shape,
ms sort-order incl. intra-second, parse partitioned-vs-legacy,
firekeeper round-trip): **24 pass**. `@trigger.dev/core` builds; webapp
typechecks; format/lint clean.
## Open decisions (flagged, not silently chosen)
1. **Backward-compat**: existing 27-char base62 KSUID runs now classify
LEGACY. On test cloud these are the broken/looping runs that never
completed, so this is acceptable — but worth a conscious call before
prod. No transitional length-recognition added (keeps the discriminator
clean).
2. **Storage collation**: the sort guarantee is byte-order — if the
run-ops id column is `TEXT` with default locale collation it's silently
not honored. Confirm whether `COLLATE "C"` / `BYTEA` is needed on the
run-ops schema.
3. **Region sourcing** wiring — see `regionCharForRegion` /
`REGION_CODES`.
---
## ⚠️ Required migration — deploy in lockstep
This PR renames a persisted feature-flag key/value and an env var. These
are **not** changed by the code alone and must be migrated when this
deploys, or affected orgs silently fall back to `cuid` minting (no crash
— `defaultValue: "cuid"`):
1. **Env var** (terraform): `RUN_OPS_MINT_KSUID_ENABLED` →
`RUN_OPS_MINT_ENABLED` (carry the value over).
2. **DB** `organization.featureFlags`: migrate both the key and value
together:
- key `runOpsMintKsuid` → `runOpsMintKind`
- value `"ksuid"` → `"runOpsId"`
Until an org's flag row is migrated, its `runOpsMintKind` lookup misses
and it mints `cuid` (legacy) — so no NEW-store ids for that org until
the data lands.
---------
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
250 lines
7.9 KiB
TypeScript
250 lines
7.9 KiB
TypeScript
// Real PG14 (legacy) + PG17 (new) proof for the dev-session-cancel TaskRun read.
|
|
// The DB is never mocked: reads hit the two real containers. Only the pure
|
|
// splitEnabled boundary and recording client wrappers are injected.
|
|
import { heteroPostgresTest, postgresTest } from "@internal/testcontainers";
|
|
import type { PrismaClient } from "@trigger.dev/database";
|
|
import { generateRunOpsId } from "@trigger.dev/core/v3/isomorphic";
|
|
import { describe, expect, vi } from "vitest";
|
|
import type { PrismaReplicaClient } from "~/db.server";
|
|
import { CancelDevSessionRunsService } from "~/v3/services/cancelDevSessionRuns.server";
|
|
|
|
vi.setConfig({ testTimeout: 60_000 });
|
|
|
|
// 25-char cuid body (no v1 version marker) → LEGACY residency.
|
|
function generateLegacyCuid() {
|
|
const suffix = Array.from(
|
|
{ length: 24 },
|
|
() => "0123456789abcdefghijklmnopqrstuvwxyz"[Math.floor(Math.random() * 36)]
|
|
).join("");
|
|
return `c${suffix}`;
|
|
}
|
|
|
|
async function seedOrgProjectEnv(prisma: PrismaClient, suffix: string) {
|
|
const organization = await prisma.organization.create({
|
|
data: { title: `test-${suffix}`, slug: `test-${suffix}` },
|
|
});
|
|
const project = await prisma.project.create({
|
|
data: {
|
|
name: `test-${suffix}`,
|
|
slug: `test-${suffix}`,
|
|
organizationId: organization.id,
|
|
externalRef: `test-${suffix}`,
|
|
},
|
|
});
|
|
const runtimeEnvironment = await prisma.runtimeEnvironment.create({
|
|
data: {
|
|
slug: `test-${suffix}`,
|
|
type: "DEVELOPMENT",
|
|
projectId: project.id,
|
|
organizationId: organization.id,
|
|
apiKey: `test-${suffix}`,
|
|
pkApiKey: `test-${suffix}`,
|
|
shortcode: `test-${suffix}`,
|
|
},
|
|
});
|
|
return { organization, project, runtimeEnvironment };
|
|
}
|
|
|
|
async function seedRun(
|
|
prisma: PrismaClient,
|
|
ids: { id: string; friendlyId: string },
|
|
env: { runtimeEnvironmentId: string; projectId: string; organizationId: string }
|
|
) {
|
|
return prisma.taskRun.create({
|
|
data: {
|
|
id: ids.id,
|
|
friendlyId: ids.friendlyId,
|
|
taskIdentifier: "my-task",
|
|
payload: JSON.stringify({ foo: "bar" }),
|
|
payloadType: "application/json",
|
|
traceId: "1234",
|
|
spanId: "1234",
|
|
queue: "test",
|
|
runtimeEnvironmentId: env.runtimeEnvironmentId,
|
|
projectId: env.projectId,
|
|
organizationId: env.organizationId,
|
|
environmentType: "DEVELOPMENT",
|
|
// V1 so the (best-effort, error-swallowed) cancel does not require the V2 engine;
|
|
// the unit under test is the READ resolution, not the cancel side effect.
|
|
engine: "V1",
|
|
status: "EXECUTING",
|
|
},
|
|
});
|
|
}
|
|
|
|
// A read client whose taskRun.findFirst is recorded; throws if used after being marked
|
|
// forbidden, so we can prove a store was NEVER read.
|
|
function recording(client: PrismaClient, opts: { forbidden?: boolean } = {}) {
|
|
const calls: unknown[] = [];
|
|
const taskRun = {
|
|
findFirst: (args: unknown) => {
|
|
calls.push(args);
|
|
if (opts.forbidden) {
|
|
throw new Error("this store must never be read");
|
|
}
|
|
return (client as unknown as PrismaReplicaClient).taskRun.findFirst(args as never);
|
|
},
|
|
};
|
|
return { handle: { ...client, taskRun } as unknown as PrismaReplicaClient, calls };
|
|
}
|
|
|
|
describe("CancelDevSessionRunsService store routing (hetero)", () => {
|
|
heteroPostgresTest(
|
|
"a NEW run (run-ops id) resolves on the new store via read-through, by friendlyId and by id",
|
|
async ({ prisma17, prisma14 }) => {
|
|
const id = generateRunOpsId();
|
|
expect(id.length).toBe(26);
|
|
const friendlyId = `run_${id}`;
|
|
|
|
const { project, organization, runtimeEnvironment } = await seedOrgProjectEnv(
|
|
prisma17,
|
|
"new"
|
|
);
|
|
await seedRun(
|
|
prisma17,
|
|
{ id, friendlyId },
|
|
{
|
|
runtimeEnvironmentId: runtimeEnvironment.id,
|
|
projectId: project.id,
|
|
organizationId: organization.id,
|
|
}
|
|
);
|
|
|
|
// by friendlyId
|
|
{
|
|
const newClient = recording(prisma17);
|
|
const legacy = recording(prisma14, { forbidden: true });
|
|
const service = new CancelDevSessionRunsService({
|
|
prisma: prisma17,
|
|
readThroughDeps: {
|
|
splitEnabled: true,
|
|
newClient: newClient.handle,
|
|
legacyReplica: legacy.handle,
|
|
},
|
|
});
|
|
await service.call({
|
|
runIds: [friendlyId],
|
|
cancelledAt: new Date(),
|
|
reason: "test",
|
|
});
|
|
// run-ops id → NEW: new store served the read, legacy never touched.
|
|
expect(newClient.calls.length).toBe(1);
|
|
expect(legacy.calls.length).toBe(0);
|
|
}
|
|
|
|
// by internal id
|
|
{
|
|
const newClient = recording(prisma17);
|
|
const legacy = recording(prisma14, { forbidden: true });
|
|
const service = new CancelDevSessionRunsService({
|
|
prisma: prisma17,
|
|
readThroughDeps: {
|
|
splitEnabled: true,
|
|
newClient: newClient.handle,
|
|
legacyReplica: legacy.handle,
|
|
},
|
|
});
|
|
await service.call({
|
|
runIds: [id],
|
|
cancelledAt: new Date(),
|
|
reason: "test",
|
|
});
|
|
expect(newClient.calls.length).toBe(1);
|
|
expect(legacy.calls.length).toBe(0);
|
|
}
|
|
}
|
|
);
|
|
|
|
heteroPostgresTest(
|
|
"an OLD in-retention run (cuid) resolves off the LEGACY replica, never a legacy primary",
|
|
async ({ prisma17, prisma14 }) => {
|
|
const id = generateLegacyCuid();
|
|
expect(id.length).toBe(25);
|
|
const friendlyId = `run_${id}`;
|
|
|
|
const { project, organization, runtimeEnvironment } = await seedOrgProjectEnv(
|
|
prisma14,
|
|
"legacy"
|
|
);
|
|
await seedRun(
|
|
prisma14,
|
|
{ id, friendlyId },
|
|
{
|
|
runtimeEnvironmentId: runtimeEnvironment.id,
|
|
projectId: project.id,
|
|
organizationId: organization.id,
|
|
}
|
|
);
|
|
|
|
const newClient = recording(prisma17);
|
|
const legacy = recording(prisma14);
|
|
const service = new CancelDevSessionRunsService({
|
|
prisma: prisma14,
|
|
readThroughDeps: {
|
|
splitEnabled: true,
|
|
newClient: newClient.handle,
|
|
legacyReplica: legacy.handle,
|
|
},
|
|
});
|
|
|
|
await service.call({
|
|
runIds: [id],
|
|
cancelledAt: new Date(),
|
|
reason: "test",
|
|
});
|
|
|
|
// NEW first (miss) → resolved off the LEGACY REPLICA handle (no primary handle exists).
|
|
expect(newClient.calls.length).toBe(1);
|
|
expect(legacy.calls.length).toBe(1);
|
|
}
|
|
);
|
|
});
|
|
|
|
describe("CancelDevSessionRunsService passthrough (single-DB)", () => {
|
|
postgresTest(
|
|
"with no read-through deps, the run is read from the single DB and session reads stay on it",
|
|
async ({ prisma }) => {
|
|
const id = generateRunOpsId();
|
|
const friendlyId = `run_${id}`;
|
|
|
|
const { project, organization, runtimeEnvironment } = await seedOrgProjectEnv(prisma, "pt");
|
|
await seedRun(
|
|
prisma,
|
|
{ id, friendlyId },
|
|
{
|
|
runtimeEnvironmentId: runtimeEnvironment.id,
|
|
projectId: project.id,
|
|
organizationId: organization.id,
|
|
}
|
|
);
|
|
|
|
const session = await prisma.runtimeEnvironmentSession.create({
|
|
data: { environmentId: runtimeEnvironment.id, ipAddress: "127.0.0.1" },
|
|
});
|
|
|
|
// splitEnabled=false → single plain read against the one client; the session
|
|
// control-plane read runs on the same prisma.
|
|
const service = new CancelDevSessionRunsService({
|
|
prisma,
|
|
replica: prisma,
|
|
readThroughDeps: {
|
|
splitEnabled: false,
|
|
newClient: prisma as unknown as PrismaReplicaClient,
|
|
},
|
|
});
|
|
|
|
await service.call({
|
|
runIds: [id],
|
|
cancelledAt: new Date(),
|
|
reason: "test",
|
|
cancelledSessionId: session.id,
|
|
});
|
|
|
|
// Run found + handed to cancel against the single DB; confirm the row is present.
|
|
const row = await prisma.taskRun.findFirst({ where: { id } });
|
|
expect(row).not.toBeNull();
|
|
expect(row?.friendlyId).toBe(friendlyId);
|
|
}
|
|
);
|
|
});
|