chore(webapp): admin endpoint to backfill Vercel deployment external ids

Skew protection resolves a run's worker by (environmentId, externalId,
status=DEPLOYED). A miss parks the run and then expires it, so deployments
predating the feature — which already carry the same value in commitSHA — need
externalId populated to stay reachable. Vercel instant-rollback is the sharpest
case, which is why the scope is the current promotion plus a recent window
rather than current alone.

Follows the existing backfill shape: admin PAT, keyset cursor over environments,
per-environment action results, pMap, dryRun defaulting to true. Reuses
normalizeExternalDeploymentId so a backfilled id is byte-identical to what a
build writes, and the update re-checks externalId IS NULL so a deploy landing
mid-backfill keeps its own id.

Refs TRI-13464.
This commit is contained in:
Oskar Otwinowski
2026-08-24 11:51:57 +02:00
parent b082e44389
commit a11838e28c
3 changed files with 634 additions and 0 deletions
@@ -0,0 +1,73 @@
import type { ActionFunctionArgs } from "@remix-run/server-runtime";
import { json } from "@remix-run/server-runtime";
import { tryCatch } from "@trigger.dev/core/v3";
import { z } from "zod";
import { $replica, prisma } from "~/db.server";
import { logger } from "~/services/logger.server";
import { requireAdminApiRequest } from "~/services/personalAccessToken.server";
import { backfillVercelExternalIds } from "~/v3/services/vercelExternalIdBackfill.server";
const BodySchema = z.object({
cursor: z.string().optional(),
limit: z.number().int().min(1).max(500).default(50),
recentPerEnvironment: z.number().int().min(0).max(200).default(10),
parallelism: z.number().int().min(1).max(20).default(5),
dryRun: z.boolean().default(true),
});
export async function action({ request }: ActionFunctionArgs) {
await requireAdminApiRequest(request);
if (request.method.toUpperCase() !== "POST") {
return json({ error: "Method Not Allowed" }, { status: 405 });
}
const [bodyError, body] = await tryCatch(request.json());
if (bodyError) {
return json({ error: bodyError.message }, { status: 400 });
}
const parsedBody = BodySchema.safeParse(body);
if (!parsedBody.success) {
return json({ error: parsedBody.error.message }, { status: 400 });
}
const { cursor, limit, recentPerEnvironment, parallelism, dryRun } = parsedBody.data;
logger.info("Vercel external id backfill starting", {
cursor,
limit,
recentPerEnvironment,
parallelism,
dryRun,
});
const [error, result] = await tryCatch(
backfillVercelExternalIds({
prisma,
replica: $replica,
cursor,
limit,
recentPerEnvironment,
parallelism,
dryRun,
})
);
if (error) {
logger.error("Vercel external id backfill failed", { cursor, error });
return json({ error: error.message }, { status: 500 });
}
logger.info("Vercel external id backfill batch complete", {
dryRun,
cursor,
environmentCount: result.environments.length,
summary: result.summary,
deployments: result.deployments,
next: result.next,
done: result.done,
});
return json(result);
}
@@ -0,0 +1,217 @@
import type { PrismaClientOrTransaction } from "@trigger.dev/database";
import { normalizeExternalDeploymentId, tryCatch } from "@trigger.dev/core/v3";
import { CURRENT_DEPLOYMENT_LABEL } from "@trigger.dev/core/v3/isomorphic";
import pMap from "p-map";
import { logger } from "~/services/logger.server";
type BackfillEnvironmentResult = {
id: string;
action: "updated" | "would_update" | "skipped_nothing_eligible" | "error";
eligible?: number;
written?: number;
error?: string;
};
export type BackfillResult = {
environments: BackfillEnvironmentResult[];
summary: Record<string, number>;
deployments: { eligible: number; written: number };
next?: string;
done?: boolean;
};
export type BackfillOptions = {
prisma: PrismaClientOrTransaction;
replica: PrismaClientOrTransaction;
cursor?: string;
limit: number;
recentPerEnvironment: number;
parallelism: number;
dryRun: boolean;
};
type Candidate = { id: string; externalId: string };
/**
* Copy `commitSHA` into `externalId` for Vercel deployments that predate skew
* protection, one keyset page of environments at a time.
*
* Resolution reads (environmentId, externalId, status=DEPLOYED) and a miss parks
* the run rather than falling back, so a deployment that stores a commit SHA but
* no external id is unreachable to an app that sends one.
*/
export async function backfillVercelExternalIds(options: BackfillOptions): Promise<BackfillResult> {
const { replica, cursor, limit, parallelism } = options;
const environments = await replica.runtimeEnvironment.findMany({
where: {
type: { not: "DEVELOPMENT" },
id: cursor ? { gt: cursor } : undefined,
project: {
organizationProjectIntegration: {
some: {
deletedAt: null,
organizationIntegration: { service: "VERCEL", deletedAt: null },
},
},
},
},
select: { id: true },
orderBy: { id: "asc" },
take: limit,
});
if (environments.length === 0) {
return {
environments: [],
summary: {},
deployments: { eligible: 0, written: 0 },
done: true,
};
}
const results = await pMap(
environments,
(environment) => backfillEnvironment(environment.id, options),
{ concurrency: parallelism, stopOnError: false }
);
const summary = results.reduce<Record<string, number>>((acc, result) => {
acc[result.action] = (acc[result.action] ?? 0) + 1;
return acc;
}, {});
const deployments = results.reduce(
(acc, result) => ({
eligible: acc.eligible + (result.eligible ?? 0),
written: acc.written + (result.written ?? 0),
}),
{ eligible: 0, written: 0 }
);
return {
environments: results,
summary,
deployments,
next: environments[environments.length - 1]?.id,
};
}
async function backfillEnvironment(
environmentId: string,
options: BackfillOptions
): Promise<BackfillEnvironmentResult> {
const [readError, candidates] = await tryCatch(findCandidates(environmentId, options));
if (readError) {
logger.error("Vercel external id backfill could not read deployments", {
environmentId,
error: readError,
});
return { id: environmentId, action: "error", error: readError.message };
}
if (candidates.length === 0) {
return { id: environmentId, action: "skipped_nothing_eligible", eligible: 0 };
}
if (options.dryRun) {
return { id: environmentId, action: "would_update", eligible: candidates.length };
}
let written = 0;
for (const candidate of candidates) {
const [writeError, result] = await tryCatch(
options.prisma.workerDeployment.updateMany({
// Re-checking externalId lets a deploy landing mid-backfill keep the id it set.
where: { id: candidate.id, externalId: null },
data: { externalId: candidate.externalId },
})
);
if (writeError) {
logger.error("Vercel external id backfill could not write a deployment", {
environmentId,
deploymentId: candidate.id,
error: writeError,
});
return {
id: environmentId,
action: "error",
eligible: candidates.length,
written,
error: writeError.message,
};
}
written += result.count;
}
return { id: environmentId, action: "updated", eligible: candidates.length, written };
}
/**
* The deployment holding the `current` promotion, plus the most recent DEPLOYED
* ones. Only DEPLOYED deployments are ever resolved, and `current` plus a recent
* window is what can still receive traffic. The window is there for Vercel
* instant-rollback, where the live app is an older commit than `current`.
*/
async function findCandidates(
environmentId: string,
{ replica, recentPerEnvironment }: BackfillOptions
): Promise<Candidate[]> {
const select = {
id: true,
externalId: true,
commitSHA: true,
workerId: true,
status: true,
} as const;
const [promotion, recent] = await Promise.all([
replica.workerDeploymentPromotion.findFirst({
where: { environmentId, label: CURRENT_DEPLOYMENT_LABEL },
select: { deployment: { select } },
}),
recentPerEnvironment > 0
? replica.workerDeployment.findMany({
where: { environmentId, status: "DEPLOYED" },
select,
orderBy: [{ createdAt: "desc" }, { id: "desc" }],
take: recentPerEnvironment,
})
: Promise.resolve([]),
]);
const byId = new Map<string, (typeof recent)[number]>();
for (const deployment of recent) {
byId.set(deployment.id, deployment);
}
if (promotion?.deployment) {
byId.set(promotion.deployment.id, promotion.deployment);
}
const candidates: Candidate[] = [];
for (const deployment of byId.values()) {
if (
deployment.externalId !== null ||
deployment.workerId === null ||
deployment.status !== "DEPLOYED"
) {
continue;
}
// Reusing the live normalizer keeps a backfilled id byte-identical to what a
// build would have written.
const externalId = normalizeExternalDeploymentId(deployment.commitSHA ?? undefined);
if (!externalId) {
continue;
}
candidates.push({ id: deployment.id, externalId });
}
return candidates;
}
@@ -0,0 +1,344 @@
import { postgresTest } from "@internal/testcontainers";
import type { PrismaClient } from "@trigger.dev/database";
import { describe, expect } from "vitest";
import { backfillVercelExternalIds } from "~/v3/services/vercelExternalIdBackfill.server";
let seedCounter = 0;
const SHA_A = "a".repeat(40);
const SHA_B = "b".repeat(40);
type SeedOptions = {
vercelConnected?: boolean;
environmentType?: "PRODUCTION" | "STAGING" | "PREVIEW" | "DEVELOPMENT";
};
async function seedEnv(prisma: PrismaClient, slug: string, options: SeedOptions = {}) {
const { vercelConnected = true, environmentType = "PRODUCTION" } = options;
const n = seedCounter++;
const organization = await prisma.organization.create({
data: { title: `Org ${slug}`, slug: `org-${slug}-${n}` },
});
const project = await prisma.project.create({
data: {
name: `Proj ${slug}`,
slug: `proj-${slug}-${n}`,
organizationId: organization.id,
externalRef: `ext-${slug}-${n}`,
},
});
const environment = await prisma.runtimeEnvironment.create({
data: {
slug: `env-${slug}-${n}`,
type: environmentType,
projectId: project.id,
organizationId: organization.id,
apiKey: `api-${slug}-${n}`,
pkApiKey: `pk-${slug}-${n}`,
shortcode: `sc-${slug}-${n}`,
},
});
if (vercelConnected) {
const tokenReference = await prisma.secretReference.create({
data: { key: `secret-${slug}-${n}` },
});
const organizationIntegration = await prisma.organizationIntegration.create({
data: {
friendlyId: `oi-${slug}-${n}`,
service: "VERCEL",
integrationData: {},
tokenReferenceId: tokenReference.id,
organizationId: organization.id,
},
});
await prisma.organizationProjectIntegration.create({
data: {
organizationIntegrationId: organizationIntegration.id,
projectId: project.id,
externalEntityId: `vercel-project-${n}`,
integrationData: {},
},
});
}
return { organization, project, environment };
}
type SeedCtx = Awaited<ReturnType<typeof seedEnv>>;
type DeploymentOptions = {
version: string;
commitSHA?: string | null;
externalId?: string | null;
status?: "DEPLOYED" | "FAILED" | "BUILDING";
withWorker?: boolean;
createdAt?: Date;
};
async function seedDeployment(prisma: PrismaClient, ctx: SeedCtx, options: DeploymentOptions) {
const {
version,
commitSHA = SHA_A,
externalId = null,
status = "DEPLOYED",
withWorker = true,
createdAt,
} = options;
const n = seedCounter++;
let workerId: string | undefined;
if (withWorker) {
const worker = await prisma.backgroundWorker.create({
data: {
friendlyId: `worker-${n}`,
contentHash: `hash-${n}`,
projectId: ctx.project.id,
runtimeEnvironmentId: ctx.environment.id,
version,
metadata: {},
},
});
workerId = worker.id;
}
return prisma.workerDeployment.create({
data: {
contentHash: `hash-${n}`,
friendlyId: `deployment-${n}`,
shortCode: `short-${n}`,
version,
status,
projectId: ctx.project.id,
environmentId: ctx.environment.id,
commitSHA,
externalId,
workerId,
...(createdAt ? { createdAt } : {}),
},
});
}
function run(
prisma: PrismaClient,
overrides: Partial<Parameters<typeof backfillVercelExternalIds>[0]> = {}
) {
return backfillVercelExternalIds({
prisma,
replica: prisma,
limit: 100,
recentPerEnvironment: 10,
parallelism: 5,
dryRun: false,
...overrides,
});
}
describe("backfillVercelExternalIds", () => {
postgresTest("copies commitSHA into externalId for a Vercel deployment", async ({ prisma }) => {
const ctx = await seedEnv(prisma, "copy");
const deployment = await seedDeployment(prisma, ctx, { version: "20260101.1" });
const result = await run(prisma);
expect(result.deployments.written).toBe(1);
const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } });
expect(after?.externalId).toBe(SHA_A);
});
postgresTest("a dry run reports the work and writes nothing", async ({ prisma }) => {
const ctx = await seedEnv(prisma, "dry");
const deployment = await seedDeployment(prisma, ctx, { version: "20260101.1" });
const result = await run(prisma, { dryRun: true });
expect(result.summary.would_update).toBe(1);
expect(result.deployments.eligible).toBe(1);
expect(result.deployments.written).toBe(0);
const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } });
expect(after?.externalId).toBeNull();
});
postgresTest("never overwrites an existing externalId", async ({ prisma }) => {
const ctx = await seedEnv(prisma, "keep");
const deployment = await seedDeployment(prisma, ctx, {
version: "20260101.1",
commitSHA: SHA_A,
externalId: SHA_B,
});
const result = await run(prisma);
expect(result.deployments.written).toBe(0);
const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } });
expect(after?.externalId).toBe(SHA_B);
});
postgresTest("skips projects with no Vercel integration", async ({ prisma }) => {
const ctx = await seedEnv(prisma, "novercel", { vercelConnected: false });
const deployment = await seedDeployment(prisma, ctx, { version: "20260101.1" });
const result = await run(prisma);
expect(result.environments.find((e) => e.id === ctx.environment.id)).toBeUndefined();
const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } });
expect(after?.externalId).toBeNull();
});
postgresTest("skips development environments", async ({ prisma }) => {
const ctx = await seedEnv(prisma, "dev", { environmentType: "DEVELOPMENT" });
const deployment = await seedDeployment(prisma, ctx, { version: "20260101.1" });
await run(prisma);
const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } });
expect(after?.externalId).toBeNull();
});
postgresTest("skips deployments that are not DEPLOYED", async ({ prisma }) => {
const ctx = await seedEnv(prisma, "failed");
const deployment = await seedDeployment(prisma, ctx, {
version: "20260101.1",
status: "FAILED",
});
await run(prisma);
const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } });
expect(after?.externalId).toBeNull();
});
postgresTest("skips deployments with no usable commitSHA", async ({ prisma }) => {
const ctx = await seedEnv(prisma, "nosha");
const missing = await seedDeployment(prisma, ctx, { version: "20260101.1", commitSHA: null });
const blank = await seedDeployment(prisma, ctx, { version: "20260101.2", commitSHA: " " });
const tooLong = await seedDeployment(prisma, ctx, {
version: "20260101.3",
commitSHA: "c".repeat(129),
});
const result = await run(prisma);
expect(result.deployments.written).toBe(0);
for (const deployment of [missing, blank, tooLong]) {
const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } });
expect(after?.externalId).toBeNull();
}
});
postgresTest("skips deployments with no worker", async ({ prisma }) => {
const ctx = await seedEnv(prisma, "noworker");
const deployment = await seedDeployment(prisma, ctx, {
version: "20260101.1",
withWorker: false,
});
await run(prisma);
const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } });
expect(after?.externalId).toBeNull();
});
postgresTest(
"recentPerEnvironment bounds the window, and current is always included",
async ({ prisma }) => {
const ctx = await seedEnv(prisma, "window");
const oldest = await seedDeployment(prisma, ctx, {
version: "20260101.1",
createdAt: new Date("2026-01-01T00:00:00Z"),
});
const newest = await seedDeployment(prisma, ctx, {
version: "20260101.2",
createdAt: new Date("2026-06-01T00:00:00Z"),
});
// Promote the oldest, so it can only be reached via the promotion arm.
await prisma.workerDeploymentPromotion.create({
data: {
label: "current",
deploymentId: oldest.id,
environmentId: ctx.environment.id,
},
});
const result = await run(prisma, { recentPerEnvironment: 1 });
expect(result.deployments.written).toBe(2);
const afterOldest = await prisma.workerDeployment.findFirst({ where: { id: oldest.id } });
const afterNewest = await prisma.workerDeployment.findFirst({ where: { id: newest.id } });
expect(afterOldest?.externalId).toBe(SHA_A);
expect(afterNewest?.externalId).toBe(SHA_A);
}
);
postgresTest(
"recentPerEnvironment 0 backfills only the current promotion",
async ({ prisma }) => {
const ctx = await seedEnv(prisma, "currentonly");
const promoted = await seedDeployment(prisma, ctx, { version: "20260101.1" });
const other = await seedDeployment(prisma, ctx, { version: "20260101.2" });
await prisma.workerDeploymentPromotion.create({
data: {
label: "current",
deploymentId: promoted.id,
environmentId: ctx.environment.id,
},
});
const result = await run(prisma, { recentPerEnvironment: 0 });
expect(result.deployments.written).toBe(1);
const afterPromoted = await prisma.workerDeployment.findFirst({ where: { id: promoted.id } });
const afterOther = await prisma.workerDeployment.findFirst({ where: { id: other.id } });
expect(afterPromoted?.externalId).toBe(SHA_A);
expect(afterOther?.externalId).toBeNull();
}
);
postgresTest("is idempotent across a second run", async ({ prisma }) => {
const ctx = await seedEnv(prisma, "idem");
await seedDeployment(prisma, ctx, { version: "20260101.1" });
const first = await run(prisma);
const second = await run(prisma);
expect(first.deployments.written).toBe(1);
expect(second.deployments.written).toBe(0);
});
postgresTest("paginates by environment id and reports done at the end", async ({ prisma }) => {
const first = await seedEnv(prisma, "page-a");
const second = await seedEnv(prisma, "page-b");
await seedDeployment(prisma, first, { version: "20260101.1" });
await seedDeployment(prisma, second, { version: "20260101.1" });
const ordered = [first.environment.id, second.environment.id].sort();
const page = await run(prisma, { limit: 1, dryRun: true });
expect(page.environments).toHaveLength(1);
expect(page.environments[0]?.id).toBe(ordered[0]);
expect(page.next).toBe(ordered[0]);
const lastId = await prisma.runtimeEnvironment.findFirst({
orderBy: { id: "desc" },
select: { id: true },
});
const exhausted = await run(prisma, { cursor: lastId?.id, dryRun: true });
expect(exhausted.done).toBe(true);
expect(exhausted.environments).toHaveLength(0);
});
});