diff --git a/apps/webapp/app/routes/admin.api.v1.vercel-external-ids.backfill.ts b/apps/webapp/app/routes/admin.api.v1.vercel-external-ids.backfill.ts new file mode 100644 index 000000000..babf77b1e --- /dev/null +++ b/apps/webapp/app/routes/admin.api.v1.vercel-external-ids.backfill.ts @@ -0,0 +1,73 @@ +import type { ActionFunctionArgs } from "@remix-run/server-runtime"; +import { json } from "@remix-run/server-runtime"; +import { tryCatch } from "@trigger.dev/core/v3"; +import { z } from "zod"; +import { $replica, prisma } from "~/db.server"; +import { logger } from "~/services/logger.server"; +import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; +import { backfillVercelExternalIds } from "~/v3/services/vercelExternalIdBackfill.server"; + +const BodySchema = z.object({ + cursor: z.string().optional(), + limit: z.number().int().min(1).max(500).default(50), + recentPerEnvironment: z.number().int().min(0).max(200).default(10), + parallelism: z.number().int().min(1).max(20).default(5), + dryRun: z.boolean().default(true), +}); + +export async function action({ request }: ActionFunctionArgs) { + await requireAdminApiRequest(request); + + if (request.method.toUpperCase() !== "POST") { + return json({ error: "Method Not Allowed" }, { status: 405 }); + } + + const [bodyError, body] = await tryCatch(request.json()); + if (bodyError) { + return json({ error: bodyError.message }, { status: 400 }); + } + + const parsedBody = BodySchema.safeParse(body); + if (!parsedBody.success) { + return json({ error: parsedBody.error.message }, { status: 400 }); + } + + const { cursor, limit, recentPerEnvironment, parallelism, dryRun } = parsedBody.data; + + logger.info("Vercel external id backfill starting", { + cursor, + limit, + recentPerEnvironment, + parallelism, + dryRun, + }); + + const [error, result] = await tryCatch( + backfillVercelExternalIds({ + prisma, + replica: $replica, + cursor, + limit, + recentPerEnvironment, + parallelism, + dryRun, + }) + ); + + if (error) { + logger.error("Vercel external id backfill failed", { cursor, error }); + return json({ error: error.message }, { status: 500 }); + } + + logger.info("Vercel external id backfill batch complete", { + dryRun, + cursor, + environmentCount: result.environments.length, + summary: result.summary, + deployments: result.deployments, + next: result.next, + done: result.done, + }); + + return json(result); +} diff --git a/apps/webapp/app/v3/services/vercelExternalIdBackfill.server.ts b/apps/webapp/app/v3/services/vercelExternalIdBackfill.server.ts new file mode 100644 index 000000000..99f7d706d --- /dev/null +++ b/apps/webapp/app/v3/services/vercelExternalIdBackfill.server.ts @@ -0,0 +1,217 @@ +import type { PrismaClientOrTransaction } from "@trigger.dev/database"; +import { normalizeExternalDeploymentId, tryCatch } from "@trigger.dev/core/v3"; +import { CURRENT_DEPLOYMENT_LABEL } from "@trigger.dev/core/v3/isomorphic"; +import pMap from "p-map"; +import { logger } from "~/services/logger.server"; + +type BackfillEnvironmentResult = { + id: string; + action: "updated" | "would_update" | "skipped_nothing_eligible" | "error"; + eligible?: number; + written?: number; + error?: string; +}; + +export type BackfillResult = { + environments: BackfillEnvironmentResult[]; + summary: Record; + deployments: { eligible: number; written: number }; + next?: string; + done?: boolean; +}; + +export type BackfillOptions = { + prisma: PrismaClientOrTransaction; + replica: PrismaClientOrTransaction; + cursor?: string; + limit: number; + recentPerEnvironment: number; + parallelism: number; + dryRun: boolean; +}; + +type Candidate = { id: string; externalId: string }; + +/** + * Copy `commitSHA` into `externalId` for Vercel deployments that predate skew + * protection, one keyset page of environments at a time. + * + * Resolution reads (environmentId, externalId, status=DEPLOYED) and a miss parks + * the run rather than falling back, so a deployment that stores a commit SHA but + * no external id is unreachable to an app that sends one. + */ +export async function backfillVercelExternalIds(options: BackfillOptions): Promise { + const { replica, cursor, limit, parallelism } = options; + + const environments = await replica.runtimeEnvironment.findMany({ + where: { + type: { not: "DEVELOPMENT" }, + id: cursor ? { gt: cursor } : undefined, + project: { + organizationProjectIntegration: { + some: { + deletedAt: null, + organizationIntegration: { service: "VERCEL", deletedAt: null }, + }, + }, + }, + }, + select: { id: true }, + orderBy: { id: "asc" }, + take: limit, + }); + + if (environments.length === 0) { + return { + environments: [], + summary: {}, + deployments: { eligible: 0, written: 0 }, + done: true, + }; + } + + const results = await pMap( + environments, + (environment) => backfillEnvironment(environment.id, options), + { concurrency: parallelism, stopOnError: false } + ); + + const summary = results.reduce>((acc, result) => { + acc[result.action] = (acc[result.action] ?? 0) + 1; + return acc; + }, {}); + + const deployments = results.reduce( + (acc, result) => ({ + eligible: acc.eligible + (result.eligible ?? 0), + written: acc.written + (result.written ?? 0), + }), + { eligible: 0, written: 0 } + ); + + return { + environments: results, + summary, + deployments, + next: environments[environments.length - 1]?.id, + }; +} + +async function backfillEnvironment( + environmentId: string, + options: BackfillOptions +): Promise { + const [readError, candidates] = await tryCatch(findCandidates(environmentId, options)); + + if (readError) { + logger.error("Vercel external id backfill could not read deployments", { + environmentId, + error: readError, + }); + return { id: environmentId, action: "error", error: readError.message }; + } + + if (candidates.length === 0) { + return { id: environmentId, action: "skipped_nothing_eligible", eligible: 0 }; + } + + if (options.dryRun) { + return { id: environmentId, action: "would_update", eligible: candidates.length }; + } + + let written = 0; + + for (const candidate of candidates) { + const [writeError, result] = await tryCatch( + options.prisma.workerDeployment.updateMany({ + // Re-checking externalId lets a deploy landing mid-backfill keep the id it set. + where: { id: candidate.id, externalId: null }, + data: { externalId: candidate.externalId }, + }) + ); + + if (writeError) { + logger.error("Vercel external id backfill could not write a deployment", { + environmentId, + deploymentId: candidate.id, + error: writeError, + }); + return { + id: environmentId, + action: "error", + eligible: candidates.length, + written, + error: writeError.message, + }; + } + + written += result.count; + } + + return { id: environmentId, action: "updated", eligible: candidates.length, written }; +} + +/** + * The deployment holding the `current` promotion, plus the most recent DEPLOYED + * ones. Only DEPLOYED deployments are ever resolved, and `current` plus a recent + * window is what can still receive traffic. The window is there for Vercel + * instant-rollback, where the live app is an older commit than `current`. + */ +async function findCandidates( + environmentId: string, + { replica, recentPerEnvironment }: BackfillOptions +): Promise { + const select = { + id: true, + externalId: true, + commitSHA: true, + workerId: true, + status: true, + } as const; + + const [promotion, recent] = await Promise.all([ + replica.workerDeploymentPromotion.findFirst({ + where: { environmentId, label: CURRENT_DEPLOYMENT_LABEL }, + select: { deployment: { select } }, + }), + recentPerEnvironment > 0 + ? replica.workerDeployment.findMany({ + where: { environmentId, status: "DEPLOYED" }, + select, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], + take: recentPerEnvironment, + }) + : Promise.resolve([]), + ]); + + const byId = new Map(); + for (const deployment of recent) { + byId.set(deployment.id, deployment); + } + if (promotion?.deployment) { + byId.set(promotion.deployment.id, promotion.deployment); + } + + const candidates: Candidate[] = []; + + for (const deployment of byId.values()) { + if ( + deployment.externalId !== null || + deployment.workerId === null || + deployment.status !== "DEPLOYED" + ) { + continue; + } + + // Reusing the live normalizer keeps a backfilled id byte-identical to what a + // build would have written. + const externalId = normalizeExternalDeploymentId(deployment.commitSHA ?? undefined); + if (!externalId) { + continue; + } + + candidates.push({ id: deployment.id, externalId }); + } + + return candidates; +} diff --git a/apps/webapp/test/vercelExternalIdBackfill.test.ts b/apps/webapp/test/vercelExternalIdBackfill.test.ts new file mode 100644 index 000000000..e146b8395 --- /dev/null +++ b/apps/webapp/test/vercelExternalIdBackfill.test.ts @@ -0,0 +1,344 @@ +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; +import { backfillVercelExternalIds } from "~/v3/services/vercelExternalIdBackfill.server"; + +let seedCounter = 0; + +const SHA_A = "a".repeat(40); +const SHA_B = "b".repeat(40); + +type SeedOptions = { + vercelConnected?: boolean; + environmentType?: "PRODUCTION" | "STAGING" | "PREVIEW" | "DEVELOPMENT"; +}; + +async function seedEnv(prisma: PrismaClient, slug: string, options: SeedOptions = {}) { + const { vercelConnected = true, environmentType = "PRODUCTION" } = options; + const n = seedCounter++; + + const organization = await prisma.organization.create({ + data: { title: `Org ${slug}`, slug: `org-${slug}-${n}` }, + }); + + const project = await prisma.project.create({ + data: { + name: `Proj ${slug}`, + slug: `proj-${slug}-${n}`, + organizationId: organization.id, + externalRef: `ext-${slug}-${n}`, + }, + }); + + const environment = await prisma.runtimeEnvironment.create({ + data: { + slug: `env-${slug}-${n}`, + type: environmentType, + projectId: project.id, + organizationId: organization.id, + apiKey: `api-${slug}-${n}`, + pkApiKey: `pk-${slug}-${n}`, + shortcode: `sc-${slug}-${n}`, + }, + }); + + if (vercelConnected) { + const tokenReference = await prisma.secretReference.create({ + data: { key: `secret-${slug}-${n}` }, + }); + + const organizationIntegration = await prisma.organizationIntegration.create({ + data: { + friendlyId: `oi-${slug}-${n}`, + service: "VERCEL", + integrationData: {}, + tokenReferenceId: tokenReference.id, + organizationId: organization.id, + }, + }); + + await prisma.organizationProjectIntegration.create({ + data: { + organizationIntegrationId: organizationIntegration.id, + projectId: project.id, + externalEntityId: `vercel-project-${n}`, + integrationData: {}, + }, + }); + } + + return { organization, project, environment }; +} + +type SeedCtx = Awaited>; + +type DeploymentOptions = { + version: string; + commitSHA?: string | null; + externalId?: string | null; + status?: "DEPLOYED" | "FAILED" | "BUILDING"; + withWorker?: boolean; + createdAt?: Date; +}; + +async function seedDeployment(prisma: PrismaClient, ctx: SeedCtx, options: DeploymentOptions) { + const { + version, + commitSHA = SHA_A, + externalId = null, + status = "DEPLOYED", + withWorker = true, + createdAt, + } = options; + const n = seedCounter++; + + let workerId: string | undefined; + if (withWorker) { + const worker = await prisma.backgroundWorker.create({ + data: { + friendlyId: `worker-${n}`, + contentHash: `hash-${n}`, + projectId: ctx.project.id, + runtimeEnvironmentId: ctx.environment.id, + version, + metadata: {}, + }, + }); + workerId = worker.id; + } + + return prisma.workerDeployment.create({ + data: { + contentHash: `hash-${n}`, + friendlyId: `deployment-${n}`, + shortCode: `short-${n}`, + version, + status, + projectId: ctx.project.id, + environmentId: ctx.environment.id, + commitSHA, + externalId, + workerId, + ...(createdAt ? { createdAt } : {}), + }, + }); +} + +function run( + prisma: PrismaClient, + overrides: Partial[0]> = {} +) { + return backfillVercelExternalIds({ + prisma, + replica: prisma, + limit: 100, + recentPerEnvironment: 10, + parallelism: 5, + dryRun: false, + ...overrides, + }); +} + +describe("backfillVercelExternalIds", () => { + postgresTest("copies commitSHA into externalId for a Vercel deployment", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "copy"); + const deployment = await seedDeployment(prisma, ctx, { version: "20260101.1" }); + + const result = await run(prisma); + + expect(result.deployments.written).toBe(1); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBe(SHA_A); + }); + + postgresTest("a dry run reports the work and writes nothing", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "dry"); + const deployment = await seedDeployment(prisma, ctx, { version: "20260101.1" }); + + const result = await run(prisma, { dryRun: true }); + + expect(result.summary.would_update).toBe(1); + expect(result.deployments.eligible).toBe(1); + expect(result.deployments.written).toBe(0); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBeNull(); + }); + + postgresTest("never overwrites an existing externalId", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "keep"); + const deployment = await seedDeployment(prisma, ctx, { + version: "20260101.1", + commitSHA: SHA_A, + externalId: SHA_B, + }); + + const result = await run(prisma); + + expect(result.deployments.written).toBe(0); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBe(SHA_B); + }); + + postgresTest("skips projects with no Vercel integration", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "novercel", { vercelConnected: false }); + const deployment = await seedDeployment(prisma, ctx, { version: "20260101.1" }); + + const result = await run(prisma); + + expect(result.environments.find((e) => e.id === ctx.environment.id)).toBeUndefined(); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBeNull(); + }); + + postgresTest("skips development environments", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "dev", { environmentType: "DEVELOPMENT" }); + const deployment = await seedDeployment(prisma, ctx, { version: "20260101.1" }); + + await run(prisma); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBeNull(); + }); + + postgresTest("skips deployments that are not DEPLOYED", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "failed"); + const deployment = await seedDeployment(prisma, ctx, { + version: "20260101.1", + status: "FAILED", + }); + + await run(prisma); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBeNull(); + }); + + postgresTest("skips deployments with no usable commitSHA", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "nosha"); + const missing = await seedDeployment(prisma, ctx, { version: "20260101.1", commitSHA: null }); + const blank = await seedDeployment(prisma, ctx, { version: "20260101.2", commitSHA: " " }); + const tooLong = await seedDeployment(prisma, ctx, { + version: "20260101.3", + commitSHA: "c".repeat(129), + }); + + const result = await run(prisma); + + expect(result.deployments.written).toBe(0); + for (const deployment of [missing, blank, tooLong]) { + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBeNull(); + } + }); + + postgresTest("skips deployments with no worker", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "noworker"); + const deployment = await seedDeployment(prisma, ctx, { + version: "20260101.1", + withWorker: false, + }); + + await run(prisma); + + const after = await prisma.workerDeployment.findFirst({ where: { id: deployment.id } }); + expect(after?.externalId).toBeNull(); + }); + + postgresTest( + "recentPerEnvironment bounds the window, and current is always included", + async ({ prisma }) => { + const ctx = await seedEnv(prisma, "window"); + + const oldest = await seedDeployment(prisma, ctx, { + version: "20260101.1", + createdAt: new Date("2026-01-01T00:00:00Z"), + }); + const newest = await seedDeployment(prisma, ctx, { + version: "20260101.2", + createdAt: new Date("2026-06-01T00:00:00Z"), + }); + + // Promote the oldest, so it can only be reached via the promotion arm. + await prisma.workerDeploymentPromotion.create({ + data: { + label: "current", + deploymentId: oldest.id, + environmentId: ctx.environment.id, + }, + }); + + const result = await run(prisma, { recentPerEnvironment: 1 }); + + expect(result.deployments.written).toBe(2); + + const afterOldest = await prisma.workerDeployment.findFirst({ where: { id: oldest.id } }); + const afterNewest = await prisma.workerDeployment.findFirst({ where: { id: newest.id } }); + expect(afterOldest?.externalId).toBe(SHA_A); + expect(afterNewest?.externalId).toBe(SHA_A); + } + ); + + postgresTest( + "recentPerEnvironment 0 backfills only the current promotion", + async ({ prisma }) => { + const ctx = await seedEnv(prisma, "currentonly"); + + const promoted = await seedDeployment(prisma, ctx, { version: "20260101.1" }); + const other = await seedDeployment(prisma, ctx, { version: "20260101.2" }); + + await prisma.workerDeploymentPromotion.create({ + data: { + label: "current", + deploymentId: promoted.id, + environmentId: ctx.environment.id, + }, + }); + + const result = await run(prisma, { recentPerEnvironment: 0 }); + + expect(result.deployments.written).toBe(1); + + const afterPromoted = await prisma.workerDeployment.findFirst({ where: { id: promoted.id } }); + const afterOther = await prisma.workerDeployment.findFirst({ where: { id: other.id } }); + expect(afterPromoted?.externalId).toBe(SHA_A); + expect(afterOther?.externalId).toBeNull(); + } + ); + + postgresTest("is idempotent across a second run", async ({ prisma }) => { + const ctx = await seedEnv(prisma, "idem"); + await seedDeployment(prisma, ctx, { version: "20260101.1" }); + + const first = await run(prisma); + const second = await run(prisma); + + expect(first.deployments.written).toBe(1); + expect(second.deployments.written).toBe(0); + }); + + postgresTest("paginates by environment id and reports done at the end", async ({ prisma }) => { + const first = await seedEnv(prisma, "page-a"); + const second = await seedEnv(prisma, "page-b"); + await seedDeployment(prisma, first, { version: "20260101.1" }); + await seedDeployment(prisma, second, { version: "20260101.1" }); + + const ordered = [first.environment.id, second.environment.id].sort(); + + const page = await run(prisma, { limit: 1, dryRun: true }); + expect(page.environments).toHaveLength(1); + expect(page.environments[0]?.id).toBe(ordered[0]); + expect(page.next).toBe(ordered[0]); + + const lastId = await prisma.runtimeEnvironment.findFirst({ + orderBy: { id: "desc" }, + select: { id: true }, + }); + const exhausted = await run(prisma, { cursor: lastId?.id, dryRun: true }); + expect(exhausted.done).toBe(true); + expect(exhausted.environments).toHaveLength(0); + }); +});