092b9ef07a
## Problem
The run-ops split mints NEW-store run ids as **27-char base62 KSUIDs**.
The supervisor writes the run id into the Kubernetes pod name
(`runner-<id>`), and pod names must be DNS-1123 labels (lowercase
`[a-z0-9-]`) — so uppercase base62 ids make k8s reject the pod (422) and
**those runs never launch** (they loop in `PENDING_EXECUTING` until the
heartbeat-stall handler nacks them, forever). `.toLowerCase()` can't fix
it: base62 has both `A`(10) and `a`(36) as distinct symbols, so folding
collides distinct ids and destroys sort order.
## Fix: change the encoding, not the structure
Mint a **26-char lowercase base32hex** run id:
```
run_<24-char base32hex core><region char><version char>
[ 6-byte ms timestamp ][ 9 CSPRNG bytes ]
```
- **base32hex** (RFC 4648 §7, alphabet `0-9a-v`): lowercase,
order-preserving, DNS-safe; 15 bytes → exactly 24 chars, no padding.
Hand-rolled encode/decode (no new dependency).
- **48-bit ms timestamp** in the leading bytes → plain string sort ==
creation order at millisecond resolution.
- **72 bits CSPRNG** entropy; PK unique constraint is the backstop (no
retry loop).
- **region / version** are raw positional chars (read via one `charAt`
before decoding/routing), version = `"1"`.
DNS-safe from birth and hyphen-free, so **firekeeper is unchanged** —
`runner-<id>-attempt-N` → strip `runner-`, cut at first hyphen still
recovers the exact id incl. region+version.
## Residency discriminator: length → version char
`classifyKind`/`classifyResidency` (`runOpsResidency.ts`) previously
distinguished NEW vs LEGACY by **id length**. That gets ambiguous with a
third format. It now discriminates on the **version char at a fixed
position** (`isRunOpsIdBody`: 26 chars, `[25] === "1"`, base32hex
alphabet) → NEW; everything else → LEGACY. Total, never throws. The
`Residency` (NEW/LEGACY) contract the routing store consumes is
unchanged; the `"ksuid"` `ResidencyKind` label is retained only because
it's the persisted `runOpsMintKsuid` feature-flag value.
## Scope / verification
- Generator + discriminator in `@trigger.dev/core` isomorphic; mint path
+ all id-shape call sites swept (~40 webapp files); changeset added
(`@trigger.dev/core` patch).
- Core unit tests (encode/decode round-trip + property, generator shape,
ms sort-order incl. intra-second, parse partitioned-vs-legacy,
firekeeper round-trip): **24 pass**. `@trigger.dev/core` builds; webapp
typechecks; format/lint clean.
## Open decisions (flagged, not silently chosen)
1. **Backward-compat**: existing 27-char base62 KSUID runs now classify
LEGACY. On test cloud these are the broken/looping runs that never
completed, so this is acceptable — but worth a conscious call before
prod. No transitional length-recognition added (keeps the discriminator
clean).
2. **Storage collation**: the sort guarantee is byte-order — if the
run-ops id column is `TEXT` with default locale collation it's silently
not honored. Confirm whether `COLLATE "C"` / `BYTEA` is needed on the
run-ops schema.
3. **Region sourcing** wiring — see `regionCharForRegion` /
`REGION_CODES`.
---
## ⚠️ Required migration — deploy in lockstep
This PR renames a persisted feature-flag key/value and an env var. These
are **not** changed by the code alone and must be migrated when this
deploys, or affected orgs silently fall back to `cuid` minting (no crash
— `defaultValue: "cuid"`):
1. **Env var** (terraform): `RUN_OPS_MINT_KSUID_ENABLED` →
`RUN_OPS_MINT_ENABLED` (carry the value over).
2. **DB** `organization.featureFlags`: migrate both the key and value
together:
- key `runOpsMintKsuid` → `runOpsMintKind`
- value `"ksuid"` → `"runOpsId"`
Until an org's flag row is migrated, its `runOpsMintKind` lookup misses
and it mints `cuid` (legacy) — so no NEW-store ids for that org until
the data lands.
---------
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
268 lines
8.7 KiB
TypeScript
268 lines
8.7 KiB
TypeScript
import { heteroPostgresTest, heteroRunOpsPostgresTest } from "@internal/testcontainers";
|
|
import { PostgresRunStore, RoutingRunStore } from "@internal/run-store";
|
|
import type { PrismaClient } from "@trigger.dev/database";
|
|
import { describe, expect, vi } from "vitest";
|
|
import { buildRunStore } from "./runStore.server";
|
|
|
|
vi.setConfig({ testTimeout: 60_000 });
|
|
|
|
// 25-char internal id -> cuid -> LEGACY; v1 body (version "1" at index 25) -> NEW.
|
|
const CUID_25 = "c".repeat(25);
|
|
const NEW_ID_26 = "k".repeat(24) + "01";
|
|
|
|
async function seedEnvironment(prisma: PrismaClient, slugSuffix: string) {
|
|
const organization = await prisma.organization.create({
|
|
data: { title: `Org ${slugSuffix}`, slug: `org-${slugSuffix}` },
|
|
});
|
|
const project = await prisma.project.create({
|
|
data: {
|
|
name: `Project ${slugSuffix}`,
|
|
slug: `project-${slugSuffix}`,
|
|
externalRef: `proj_${slugSuffix}`,
|
|
organizationId: organization.id,
|
|
},
|
|
});
|
|
const environment = await prisma.runtimeEnvironment.create({
|
|
data: {
|
|
type: "DEVELOPMENT",
|
|
slug: "dev",
|
|
projectId: project.id,
|
|
organizationId: organization.id,
|
|
apiKey: `tr_dev_${slugSuffix}`,
|
|
pkApiKey: `pk_dev_${slugSuffix}`,
|
|
shortcode: `short_${slugSuffix}`,
|
|
},
|
|
});
|
|
return { organization, project, environment };
|
|
}
|
|
|
|
function createRunInput(params: {
|
|
runId: string;
|
|
friendlyId: string;
|
|
organizationId: string;
|
|
projectId: string;
|
|
runtimeEnvironmentId: string;
|
|
}) {
|
|
return {
|
|
data: {
|
|
id: params.runId,
|
|
engine: "V2" as const,
|
|
status: "PENDING" as const,
|
|
friendlyId: params.friendlyId,
|
|
runtimeEnvironmentId: params.runtimeEnvironmentId,
|
|
environmentType: "DEVELOPMENT" as const,
|
|
organizationId: params.organizationId,
|
|
projectId: params.projectId,
|
|
taskIdentifier: "my-task",
|
|
payload: '{"hello":"world"}',
|
|
payloadType: "application/json",
|
|
context: { foo: "bar" },
|
|
traceContext: { trace: "ctx" },
|
|
traceId: "trace_1",
|
|
spanId: "span_1",
|
|
runTags: ["alpha"],
|
|
queue: "task/my-task",
|
|
isTest: false,
|
|
taskEventStore: "taskEvent",
|
|
depth: 0,
|
|
createdAt: new Date("2024-01-01T00:00:00.000Z"),
|
|
},
|
|
snapshot: {
|
|
engine: "V2" as const,
|
|
executionStatus: "RUN_CREATED" as const,
|
|
description: "Run was created",
|
|
runStatus: "PENDING" as const,
|
|
environmentId: params.runtimeEnvironmentId,
|
|
environmentType: "DEVELOPMENT" as const,
|
|
projectId: params.projectId,
|
|
organizationId: params.organizationId,
|
|
},
|
|
};
|
|
}
|
|
|
|
describe("T24 — findRun resolves run-ops run on dedicated DB", () => {
|
|
heteroRunOpsPostgresTest(
|
|
"split ON: findRun({friendlyId, runtimeEnvironmentId}, {select}) finds a run-ops run on the new store",
|
|
async ({ prisma14, prisma17 }) => {
|
|
const ENV_ID = "env_t24_runops_probe";
|
|
const WORKER_ID = "worker_t24_lock";
|
|
await prisma17.taskRun.create({
|
|
data: {
|
|
id: NEW_ID_26,
|
|
engine: "V2",
|
|
status: "EXECUTING",
|
|
friendlyId: `run_${NEW_ID_26}`,
|
|
runtimeEnvironmentId: ENV_ID,
|
|
environmentType: "DEVELOPMENT",
|
|
organizationId: "org_t24",
|
|
projectId: "proj_t24",
|
|
taskIdentifier: "my-task",
|
|
payload: "{}",
|
|
payloadType: "application/json",
|
|
traceId: "trace_t24",
|
|
spanId: "span_t24",
|
|
queue: "task/my-task",
|
|
lockedToVersionId: WORKER_ID,
|
|
createdAt: new Date("2024-01-01T00:00:00.000Z"),
|
|
},
|
|
});
|
|
|
|
const store = buildRunStore({
|
|
splitEnabled: true,
|
|
newWriter: prisma17,
|
|
newReplica: prisma17,
|
|
legacyWriter: prisma14,
|
|
legacyReplica: prisma14,
|
|
singleWriter: prisma14,
|
|
singleReplica: prisma14,
|
|
});
|
|
|
|
const run = await store.findRun(
|
|
{ friendlyId: `run_${NEW_ID_26}`, runtimeEnvironmentId: ENV_ID },
|
|
{ select: { lockedToVersionId: true } }
|
|
);
|
|
|
|
expect(run).not.toBeNull();
|
|
expect(run?.lockedToVersionId).toBe(WORKER_ID);
|
|
expect(await prisma14.taskRun.findUnique({ where: { id: NEW_ID_26 } })).toBeNull();
|
|
}
|
|
);
|
|
});
|
|
|
|
describe("buildRunStore", () => {
|
|
heteroPostgresTest(
|
|
"split OFF returns a passthrough PostgresRunStore that writes only to the single DB",
|
|
async ({ prisma14, prisma17 }) => {
|
|
// Single-DB: every handle is prisma14. prisma17 must stay untouched.
|
|
const store = buildRunStore({
|
|
splitEnabled: false,
|
|
newWriter: prisma14,
|
|
newReplica: prisma14,
|
|
legacyWriter: prisma14,
|
|
legacyReplica: prisma14,
|
|
singleWriter: prisma14,
|
|
singleReplica: prisma14,
|
|
});
|
|
|
|
expect(store).toBeInstanceOf(PostgresRunStore);
|
|
|
|
const seed = await seedEnvironment(prisma14, "off");
|
|
// A run-ops id (would route to NEW under split) must still land on the single DB.
|
|
const runId = NEW_ID_26;
|
|
await store.createRun(
|
|
createRunInput({
|
|
runId,
|
|
friendlyId: "run_off",
|
|
organizationId: seed.organization.id,
|
|
projectId: seed.project.id,
|
|
runtimeEnvironmentId: seed.environment.id,
|
|
})
|
|
);
|
|
|
|
expect(await prisma14.taskRun.findUnique({ where: { id: runId } })).not.toBeNull();
|
|
expect(await prisma17.taskRun.findUnique({ where: { id: runId } })).toBeNull();
|
|
}
|
|
);
|
|
|
|
heteroPostgresTest(
|
|
"split ON routes a NEW-classified create to the new store and a LEGACY-classified create to the legacy store",
|
|
async ({ prisma14, prisma17 }) => {
|
|
// legacy = PG14, new = PG17.
|
|
const store = buildRunStore({
|
|
splitEnabled: true,
|
|
newWriter: prisma17,
|
|
newReplica: prisma17,
|
|
legacyWriter: prisma14,
|
|
legacyReplica: prisma14,
|
|
singleWriter: prisma14,
|
|
singleReplica: prisma14,
|
|
});
|
|
|
|
expect(store).toBeInstanceOf(RoutingRunStore);
|
|
|
|
const seedNew = await seedEnvironment(prisma17, "on_new");
|
|
const seedLegacy = await seedEnvironment(prisma14, "on_legacy");
|
|
|
|
// run-ops id -> NEW (PG17)
|
|
await store.createRun(
|
|
createRunInput({
|
|
runId: NEW_ID_26,
|
|
friendlyId: "run_new",
|
|
organizationId: seedNew.organization.id,
|
|
projectId: seedNew.project.id,
|
|
runtimeEnvironmentId: seedNew.environment.id,
|
|
})
|
|
);
|
|
expect(await prisma17.taskRun.findUnique({ where: { id: NEW_ID_26 } })).not.toBeNull();
|
|
expect(await prisma14.taskRun.findUnique({ where: { id: NEW_ID_26 } })).toBeNull();
|
|
|
|
// cuid -> LEGACY (PG14)
|
|
await store.createRun(
|
|
createRunInput({
|
|
runId: CUID_25,
|
|
friendlyId: "run_legacy",
|
|
organizationId: seedLegacy.organization.id,
|
|
projectId: seedLegacy.project.id,
|
|
runtimeEnvironmentId: seedLegacy.environment.id,
|
|
})
|
|
);
|
|
expect(await prisma14.taskRun.findUnique({ where: { id: CUID_25 } })).not.toBeNull();
|
|
expect(await prisma17.taskRun.findUnique({ where: { id: CUID_25 } })).toBeNull();
|
|
}
|
|
);
|
|
|
|
heteroPostgresTest(
|
|
"split ON keeps a write on a LEGACY-classified id on the legacy store",
|
|
async ({ prisma14, prisma17 }) => {
|
|
// Routing is pure id-shape, so a cuid write stays LEGACY.
|
|
const store = buildRunStore({
|
|
splitEnabled: true,
|
|
newWriter: prisma17,
|
|
newReplica: prisma17,
|
|
legacyWriter: prisma14,
|
|
legacyReplica: prisma14,
|
|
singleWriter: prisma14,
|
|
singleReplica: prisma14,
|
|
});
|
|
|
|
const seedLegacy = await seedEnvironment(prisma14, "no_marker_legacy");
|
|
// The run lives on LEGACY (PG14); seed it directly.
|
|
await prisma14.taskRun.create({
|
|
data: {
|
|
id: CUID_25,
|
|
engine: "V2",
|
|
status: "EXECUTING",
|
|
friendlyId: "run_no_marker",
|
|
runtimeEnvironmentId: seedLegacy.environment.id,
|
|
environmentType: "DEVELOPMENT",
|
|
organizationId: seedLegacy.organization.id,
|
|
projectId: seedLegacy.project.id,
|
|
taskIdentifier: "my-task",
|
|
payload: "{}",
|
|
payloadType: "application/json",
|
|
traceId: "t",
|
|
spanId: "s",
|
|
queue: "task/my-task",
|
|
createdAt: new Date("2024-01-01T00:00:00.000Z"),
|
|
},
|
|
});
|
|
|
|
const updated = await store.updateMetadata(
|
|
CUID_25,
|
|
{
|
|
metadata: '{"k":"v"}',
|
|
metadataType: "application/json",
|
|
metadataVersion: { increment: 1 },
|
|
updatedAt: new Date("2024-01-02T00:00:00.000Z"),
|
|
},
|
|
{}
|
|
);
|
|
expect(updated.count).toBe(1);
|
|
|
|
const onLegacy = await prisma14.taskRun.findUnique({ where: { id: CUID_25 } });
|
|
expect(onLegacy?.metadata).toBe('{"k":"v"}');
|
|
expect(await prisma17.taskRun.findUnique({ where: { id: CUID_25 } })).toBeNull();
|
|
}
|
|
);
|
|
});
|