From 38e78f8c7e7c09fdcfd6cd97fd016baa2debc444 Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Wed, 26 Aug 2026 12:57:46 +0200 Subject: [PATCH] feat(webapp): deployment lifecycle telemetry events (#4778) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deployments currently leave little analytical trace. This PR makes every deployment emit two analytics events to enable useful queries. It also enables comparing deployments across build paths, CLI versions, runtimes, and orgs. ### Where the events come from ``` trigger deploy │ ▼ initialize ─────────────────────────────▶ ✨ deployment.initialized │ createdAt ▼ PENDING waiting for a build slot ┐ │ startedAt │ queue time ▼ ┘ INSTALLING build server installs deps ┐ │ installedAt (native paths only) │ install time ▼ ┘ BUILDING the image is built ┐ │ builtAt │ building time ▼ ┘ DEPLOYING indexing + registry push ┐ │ deployedAt / failedAt / canceledAt │ deploying time ▼ ┘ DEPLOYED · FAILED · TIMED_OUT · CANCELED │ └───────────────────────────────────▶ ✨ deployment.finished ``` `deployment.finished` fires exactly once, whichever way the deployment ends, and is backdated to cover the deployment's real lifetime. Not every path visits every state (Depot deploys skip PENDING/INSTALLING, for example) — a phase duration is simply omitted when its state was never entered. ### What each event carries - **Which path built it**: `depot`, `native`, or `native_local_bundle` - **How it ended**: status, plus an error class and message when it failed - **How long each phase took**: queue, install, building, deploying, and total — derived from the timestamps above - **Who and with what**: org, project, environment, runtime, CLI version, and how the deploy was triggered (CLI, GitHub, Vercel) With that, one query gives failure rate per build path, duration percentiles per phase, adoption per CLI version, or a per-org health table. ### Fixes that ride along - The old `deployment.outcome` span was silently dropped ~95% of the time (it was subject to trace sampling). The new events opt out of sampling explicitly, so every deployment is counted. - The fail/timeout/finalize transitions were racy: a late timeout could overwrite a successful deployment. They now use guarded writes, so exactly one caller wins the terminal transition — and exactly one event is emitted. - Canceled deployments previously recorded nothing; they do now. - The deployment's CLI version is now stored at initialization (new nullable column), so even deploys that fail early are attributable to a CLI release. - Telemetry is flushed on shutdown (the last batch used to be lost on every webapp deploy), and an optional second exporter can mirror just these events into a dedicated dataset. --- .changeset/violet-buses-tease.md | 5 + apps/webapp/app/env.server.ts | 4 + apps/webapp/app/routes/api.v1.deployments.ts | 12 +- apps/webapp/app/v3/deploymentTelemetry.ts | 116 +++++++++++ ...eateDeploymentBackgroundWorkerV4.server.ts | 30 +-- .../app/v3/services/deployment.server.ts | 63 ++++-- .../app/v3/services/failDeployment.server.ts | 48 ++++- .../v3/services/finalizeDeployment.server.ts | 45 +++-- .../services/initializeDeployment.server.ts | 16 +- .../recordDeploymentFinished.server.ts | 183 ++++++++++++++++++ .../recordDeploymentOutcome.server.ts | 50 ----- .../v3/services/timeoutDeployment.server.ts | 44 ++++- apps/webapp/app/v3/tracer.server.ts | 71 ++++++- apps/webapp/test/deploymentTelemetry.test.ts | 83 ++++++++ .../migration.sql | 1 + .../database/prisma/schema.prisma | 2 + packages/cli-v3/src/apiClient.ts | 1 + 17 files changed, 661 insertions(+), 113 deletions(-) create mode 100644 .changeset/violet-buses-tease.md create mode 100644 apps/webapp/app/v3/deploymentTelemetry.ts create mode 100644 apps/webapp/app/v3/services/recordDeploymentFinished.server.ts delete mode 100644 apps/webapp/app/v3/services/recordDeploymentOutcome.server.ts create mode 100644 apps/webapp/test/deploymentTelemetry.test.ts create mode 100644 internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql diff --git a/.changeset/violet-buses-tease.md b/.changeset/violet-buses-tease.md new file mode 100644 index 000000000..191cff776 --- /dev/null +++ b/.changeset/violet-buses-tease.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +Send the CLI version header on all API requests so deployments are attributable to a CLI version diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index c1155d377..2b1fba869 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -940,6 +940,10 @@ const EnvironmentSchema = z DISABLE_HTTP_INSTRUMENTATION: BoolEnv.default(false), INTERNAL_OTEL_LOG_EXPORTER_URL: z.string().optional(), + + // Second trace exporter receiving only `deployment.*` spans; they still flow to the main one + INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL: z.string().optional(), + INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_AUTH_HEADERS: z.string().optional(), INTERNAL_OTEL_METRIC_EXPORTER_URL: z.string().optional(), INTERNAL_OTEL_METRIC_EXPORTER_AUTH_HEADERS: z.string().optional(), INTERNAL_OTEL_METRIC_EXPORTER_ENABLED: z.string().default("0"), diff --git a/apps/webapp/app/routes/api.v1.deployments.ts b/apps/webapp/app/routes/api.v1.deployments.ts index 5be291bae..184fa996d 100644 --- a/apps/webapp/app/routes/api.v1.deployments.ts +++ b/apps/webapp/app/routes/api.v1.deployments.ts @@ -42,7 +42,9 @@ export async function action({ request, params }: ActionFunctionArgs) { const service = new InitializeDeploymentService(); try { - const result = await service.call(authenticatedEnv, body.data); + const result = await service.call(authenticatedEnv, body.data, { + cliVersion: parseCliVersionHeader(request), + }); const { deployment, imageRef } = result; const responseBody: InitializeDeploymentResponseBody = { @@ -75,6 +77,14 @@ export async function action({ request, params }: ActionFunctionArgs) { } } +// Client-controlled and persisted, so cap what we accept +const CLI_VERSION_MAX_LENGTH = 128; + +function parseCliVersionHeader(request: Request): string | undefined { + const value = request.headers.get("x-trigger-cli-version"); + return value && value.length <= CLI_VERSION_MAX_LENGTH ? value : undefined; +} + export const loader = createLoaderApiRoute( { searchParams: ApiDeploymentListSearchParams, diff --git a/apps/webapp/app/v3/deploymentTelemetry.ts b/apps/webapp/app/v3/deploymentTelemetry.ts new file mode 100644 index 000000000..2dd0725f8 --- /dev/null +++ b/apps/webapp/app/v3/deploymentTelemetry.ts @@ -0,0 +1,116 @@ +import { BuildServerMetadata } from "@trigger.dev/core/v3"; + +/** + * Attribute names for the `deployment.finished` and `deployment.initialized` + * telemetry events (emitted by services/recordDeploymentFinished.server.ts). + * This module is the single owner of these names — external queries, + * dashboards, and monitors reference them, so treat renames as breaking. + * + * Query gotchas: dedup with `arg_max(_time, *) by deployment.id` (job retries + * can double-emit); the span's `_time` is the deployment's createdAt, so a + * TIMED_OUT event lands backdated by up to the full deploy timeout — monitor + * windows must exceed it; phase durations are omitted (not zero) when a + * boundary timestamp is missing, and `total_ms` excludes local-bundle's + * pre-init client work (esbuild + upload) until the CLI reports timings. + */ +export const DeploymentTelemetryAttributes = { + ORG_ID: "$trigger.org.id", + PROJECT_ID: "$trigger.project.id", + // Project external ref ("proj_…") + PROJECT_REF: "$trigger.project.ref", + ENV_ID: "$trigger.env.id", + // PRODUCTION / STAGING / PREVIEW / DEVELOPMENT + ENV_TYPE: "$trigger.env.type", + // Deployment friendly id — the dedup key + DEPLOYMENT_ID: "deployment.id", + VERSION: "deployment.version", + // finished: terminal status; initialized: initial status (PENDING/BUILDING) + STATUS: "deployment.status", + // status === DEPLOYED; CANCELED is excluded from failure rates + SUCCESS: "deployment.success", + // depot / native / native_local_bundle (see deriveBuildPath) + BUILD_PATH: "deployment.build_path", + // V1 / MANAGED (run engine) + WORKER_TYPE: "deployment.worker_type", + RUNTIME: "deployment.runtime", + // Set at indexing; null for pre-index failures + RUNTIME_VERSION: "deployment.runtime_version", + // From x-trigger-cli-version at init; null for pre-column history + CLI_VERSION: "deployment.cli_version", + TRIGGERED_VIA: "deployment.triggered_via", + COMMIT_SHA: "deployment.commit_sha", + // error.* only on FAILED/TIMED_OUT; CANCELED uses canceled_reason + ERROR_NAME: "deployment.error.name", + ERROR_MESSAGE: "deployment.error.message", + CANCELED_REASON: "deployment.canceled_reason", + // createdAt → terminal (also the span's own duration) + DURATION_TOTAL_MS: "deployment.duration.total_ms", + // createdAt → startedAt; ≈0 when created directly in BUILDING (depot) + DURATION_QUEUE_MS: "deployment.duration.queue_ms", + // startedAt → installedAt; build-server paths only (depot never sets it) + DURATION_INSTALL_MS: "deployment.duration.install_ms", + // (installedAt ?? startedAt) → builtAt + DURATION_BUILDING_MS: "deployment.duration.building_ms", + // builtAt → terminal; for depot dominated by the server-side registry push + DURATION_DEPLOYING_MS: "deployment.duration.deploying_ms", +} as const; + +export type DeploymentBuildPath = "native_local_bundle" | "native" | "depot"; + +/** + * Everything that is not a native-build-server deployment falls into the depot + * bucket, including rare `--local-build` deploys (their flag is not persisted). + * `externalBuildData` is NOT a usable depot signal: init writes a placeholder + * for every path. + */ +export function deriveBuildPath(buildServerMetadata: unknown): DeploymentBuildPath { + const metadata = BuildServerMetadata.safeParse(buildServerMetadata); + + if (metadata.success && metadata.data.isNativeBuild) { + return metadata.data.fromBundle ? "native_local_bundle" : "native"; + } + + return "depot"; +} + +export type DeploymentTimestamps = { + createdAt: Date; + startedAt?: Date | null; + installedAt?: Date | null; + builtAt?: Date | null; +}; + +export type DeploymentDurations = { + totalMs: number; + queueMs?: number; + installMs?: number; + buildingMs?: number; + deployingMs?: number; +}; + +/** + * Timestamp chains are path-shaped (e.g. depot never sets installedAt), so + * each phase is derived only when both of its boundary timestamps exist and + * are ordered. + */ +export function deriveDeploymentDurations( + timestamps: DeploymentTimestamps, + terminalAt: Date +): DeploymentDurations { + const { createdAt, startedAt, installedAt, builtAt } = timestamps; + const buildingFrom = installedAt ?? startedAt; + + return { + totalMs: Math.max(terminalAt.getTime() - createdAt.getTime(), 0), + queueMs: msBetween(createdAt, startedAt), + installMs: msBetween(startedAt, installedAt), + buildingMs: msBetween(buildingFrom, builtAt), + deployingMs: msBetween(builtAt, terminalAt), + }; +} + +function msBetween(from?: Date | null, to?: Date | null): number | undefined { + if (!from || !to) return undefined; + const ms = to.getTime() - from.getTime(); + return ms >= 0 ? ms : undefined; +} diff --git a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts index d09707a0e..7215c09c3 100644 --- a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts +++ b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts @@ -18,7 +18,7 @@ import { } from "./createBackgroundWorker.server"; import { findOrCreateBackgroundWorker } from "./createDeploymentBackgroundWorkerV4/findOrCreateBackgroundWorker.server"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; -import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; import { env } from "~/env.server"; import { webhookPrisma } from "~/db.server"; @@ -298,6 +298,12 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { error: Error, environment: AuthenticatedEnvironment ) { + const failedAt = new Date(); + const errorData = { + name: error.name, + message: error.message, + }; + // Guarded BUILDING → FAILED transition, symmetric with the BUILDING → DEPLOYING // transition in `call()`. With idempotent retries, two attempts can run side-by-side; // without the predicate, one attempt's failure could downgrade the deployment after @@ -309,11 +315,8 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { }, data: { status: "FAILED", - failedAt: new Date(), - errorData: { - name: error.name, - message: error.message, - }, + failedAt, + errorData, buildEnvVars: Prisma.DbNull, }, }); @@ -332,13 +335,16 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { // BUILDING → DEPLOYING transition. await TimeoutDeploymentService.dequeue(deployment.id, this._prisma); - recordDeploymentOutcome({ + recordDeploymentFinished({ status: "FAILED", - deploymentFriendlyId: deployment.friendlyId, - organizationId: environment.organizationId, - projectId: environment.projectId, - environmentId: environment.id, - environmentType: environment.type, + deployment: { ...deployment, status: "FAILED", failedAt, errorData }, + environment: { + organizationId: environment.organizationId, + projectId: environment.projectId, + projectRef: environment.project.externalRef, + environmentId: environment.id, + environmentType: environment.type, + }, reason: error.message, }); } diff --git a/apps/webapp/app/v3/services/deployment.server.ts b/apps/webapp/app/v3/services/deployment.server.ts index 7a891ae4f..e8ac53cc4 100644 --- a/apps/webapp/app/v3/services/deployment.server.ts +++ b/apps/webapp/app/v3/services/deployment.server.ts @@ -9,6 +9,7 @@ import { type DeploymentEvent, } from "@trigger.dev/core/v3"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; import { env } from "~/env.server"; import { createRemoteImageBuild } from "../remoteImageBuilder.server"; import { FINAL_DEPLOYMENT_STATUSES } from "./failDeployment.server"; @@ -195,10 +196,8 @@ export class DeploymentService extends BaseService { friendlyId: string, data?: Partial> ) { - const validateDeployment = ( - deployment: Pick & { - environment: { project: { externalRef: string } }; - } + const validateDeployment = >( + deployment: T ) => { if (FINAL_DEPLOYMENT_STATUSES.includes(deployment.status)) { logger.warn("Attempted cancelling deployment in a final state", { @@ -210,11 +209,7 @@ export class DeploymentService extends BaseService { return okAsync(deployment); }; - const cancelDeployment = ( - deployment: Pick & { - environment: { project: { externalRef: string } }; - } - ) => + const cancelDeployment = >(deployment: T) => fromPromise( this._prisma.workerDeployment.updateMany({ where: { @@ -250,7 +245,25 @@ export class DeploymentService extends BaseService { return this.getDeployment(authenticatedEnv.id, friendlyId) .andThen(validateDeployment) .andThen(cancelDeployment) - .andThen(({ deployment }) => + .andTee(({ deployment }) => + recordDeploymentFinished({ + status: "CANCELED", + deployment: { + ...deployment, + status: "CANCELED", + canceledAt: new Date(), + canceledReason: data?.canceledReason ?? null, + }, + environment: { + organizationId: deployment.environment.project.organizationId, + projectId: deployment.environment.project.id, + projectRef: deployment.environment.project.externalRef, + environmentId: deployment.environment.id, + environmentType: deployment.environment.type, + }, + }) + ) + .andTee(({ deployment }) => this.appendToEventLog(deployment.environment.project, deployment, [ { type: "finalized", @@ -259,14 +272,11 @@ export class DeploymentService extends BaseService { message: data?.canceledReason ?? undefined, }, }, - ]) - .orElse((error) => { - logger.error("Failed to append event to deployment event log", { error }); - return okAsync(deployment); - }) - .map(() => deployment) + ]).orTee((error) => { + logger.error("Failed to append event to deployment event log", { error }); + }) ) - .andThen(deleteTimeout) + .andThen(({ deployment }) => deleteTimeout(deployment)) .map(() => undefined); } @@ -484,6 +494,23 @@ export class DeploymentService extends BaseService { select: { status: true, id: true, + friendlyId: true, + version: true, + type: true, + createdAt: true, + startedAt: true, + installedAt: true, + builtAt: true, + deployedAt: true, + failedAt: true, + canceledAt: true, + canceledReason: true, + errorData: true, + runtime: true, + runtimeVersion: true, + cliVersion: true, + triggeredVia: true, + commitSHA: true, buildServerMetadata: true, imageReference: true, shortCode: true, @@ -491,6 +518,8 @@ export class DeploymentService extends BaseService { include: { project: { select: { + id: true, + organizationId: true, externalRef: true, }, }, diff --git a/apps/webapp/app/v3/services/failDeployment.server.ts b/apps/webapp/app/v3/services/failDeployment.server.ts index cb5c622b7..534158308 100644 --- a/apps/webapp/app/v3/services/failDeployment.server.ts +++ b/apps/webapp/app/v3/services/failDeployment.server.ts @@ -1,11 +1,11 @@ import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server"; import { BaseService } from "./baseService.server"; import { logger } from "~/services/logger.server"; -import { Prisma, type WorkerDeploymentStatus } from "@trigger.dev/database"; +import { boundedIn, Prisma, type WorkerDeploymentStatus } from "@trigger.dev/database"; import { type FailDeploymentRequestBody } from "@trigger.dev/core/v3/schemas"; import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { DeploymentService } from "./deployment.server"; -import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; export const FINAL_DEPLOYMENT_STATUSES: WorkerDeploymentStatus[] = [ "CANCELED", @@ -41,25 +41,53 @@ export class FailDeploymentService extends BaseService { return; } - const failedDeployment = await this._prisma.workerDeployment.update({ + const failedAt = new Date(); + + // Guarded: a concurrent terminal transition can win after the check above + const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({ where: { id: deployment.id, + status: { notIn: boundedIn(FINAL_DEPLOYMENT_STATUSES) }, }, data: { status: "FAILED", - failedAt: new Date(), + failedAt, errorData: params.error, buildEnvVars: Prisma.DbNull, }, }); - recordDeploymentOutcome({ + if (updatedCount === 0) { + logger.warn("Worker deployment reached a final state concurrently, skipping fail", { + id: deployment.id, + friendlyId, + }); + return; + } + + // Re-read: the row can gain phase timestamps between the read and the guarded write + const failedDeployment = await this._prisma.workerDeployment.findFirst({ + where: { id: deployment.id }, + }); + + if (!failedDeployment) { + logger.error("Worker deployment disappeared after fail transition", { + id: deployment.id, + friendlyId, + }); + return; + } + + recordDeploymentFinished({ status: "FAILED", - deploymentFriendlyId: friendlyId, - organizationId: authenticatedEnv.organizationId, - projectId: authenticatedEnv.projectId, - environmentId: authenticatedEnv.id, - environmentType: authenticatedEnv.type, + deployment: failedDeployment, + environment: { + organizationId: authenticatedEnv.organizationId, + projectId: authenticatedEnv.projectId, + projectRef: authenticatedEnv.project.externalRef, + environmentId: authenticatedEnv.id, + environmentType: authenticatedEnv.type, + }, reason: params.error.message, }); diff --git a/apps/webapp/app/v3/services/finalizeDeployment.server.ts b/apps/webapp/app/v3/services/finalizeDeployment.server.ts index 51f5b1e37..3ee7a1beb 100644 --- a/apps/webapp/app/v3/services/finalizeDeployment.server.ts +++ b/apps/webapp/app/v3/services/finalizeDeployment.server.ts @@ -10,7 +10,7 @@ import { projectPubSub } from "./projectPubSub.server"; import { FailDeploymentService } from "./failDeployment.server"; import { TimeoutDeploymentService } from "./timeoutDeployment.server"; import { DeploymentService } from "./deployment.server"; -import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; import { engine } from "../runEngine.server"; import { tryCatch } from "@trigger.dev/core"; import { externalDeploymentCacheInstance } from "~/services/externalDeploymentCacheInstance.server"; @@ -66,28 +66,51 @@ export class FinalizeDeploymentService extends BaseService { } const imageDigest = validatedImageDigest(body.imageDigest); + const deployedAt = new Date(); + const imageReference = imageDigest + ? `${deployment.imageReference}@${imageDigest}` + : deployment.imageReference; - // Link the deployment with the background worker - const finalizedDeployment = await this._prisma.workerDeployment.update({ + // Guarded: stops a concurrent transition (e.g. a late timeout) from double-committing + const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({ where: { id: deployment.id, + status: "DEPLOYING", }, data: { status: "DEPLOYED", - deployedAt: new Date(), + deployedAt, // Only add the digest, if any - imageReference: imageDigest ? `${deployment.imageReference}@${imageDigest}` : undefined, + imageReference: imageDigest ? imageReference : undefined, buildEnvVars: Prisma.DbNull, }, }); - recordDeploymentOutcome({ + if (updatedCount === 0) { + logger.warn("Worker deployment left DEPLOYING concurrently, skipping finalize", { + id: deployment.id, + }); + throw new ServiceValidationError("Worker deployment is not in DEPLOYING status"); + } + + const finalizedDeployment = { + ...deployment, + status: "DEPLOYED" as const, + deployedAt, + imageReference, + buildEnvVars: null, + }; + + recordDeploymentFinished({ status: "DEPLOYED", - deploymentFriendlyId: deployment.friendlyId, - organizationId: authenticatedEnv.organizationId, - projectId: authenticatedEnv.projectId, - environmentId: authenticatedEnv.id, - environmentType: authenticatedEnv.type, + deployment: finalizedDeployment, + environment: { + organizationId: authenticatedEnv.organizationId, + projectId: authenticatedEnv.projectId, + projectRef: authenticatedEnv.project.externalRef, + environmentId: authenticatedEnv.id, + environmentType: authenticatedEnv.type, + }, }); const deploymentService = new DeploymentService(); diff --git a/apps/webapp/app/v3/services/initializeDeployment.server.ts b/apps/webapp/app/v3/services/initializeDeployment.server.ts index ee55d8bd8..56cf50e68 100644 --- a/apps/webapp/app/v3/services/initializeDeployment.server.ts +++ b/apps/webapp/app/v3/services/initializeDeployment.server.ts @@ -16,6 +16,7 @@ import { getDeploymentImageRef } from "../getDeploymentImageRef.server"; import { tryCatch } from "@trigger.dev/core"; import { getRegistryConfig } from "../registryConfig.server"; import { DeploymentService } from "./deployment.server"; +import { recordDeploymentInitialized } from "./recordDeploymentFinished.server"; import { createDeploymentWithNextVersion } from "./initializeDeployment/createDeploymentWithNextVersion.server"; import { cancelSupersededDeployments, @@ -56,7 +57,8 @@ export type InitializeDeploymentResult = export class InitializeDeploymentService extends BaseService { public async call( environment: AuthenticatedEnvironment, - payload: InitializeDeploymentRequestBody + payload: InitializeDeploymentRequestBody, + options?: { cliVersion?: string } ): Promise { return this.traceWithEnv("call", environment, async (span) => { if (payload.externalId) { @@ -386,12 +388,24 @@ export class InitializeDeploymentService extends BaseService { commitSHA: payload.gitMeta?.commitSha ?? undefined, externalId: payload.externalId, runtime: payload.runtime ?? environment.project.defaultRuntime ?? undefined, + cliVersion: options?.cliVersion, triggeredVia: payload.triggeredVia ?? undefined, startedAt: initialStatus === "BUILDING" ? new Date() : undefined, }; } ); + recordDeploymentInitialized({ + deployment, + environment: { + organizationId: environment.organizationId, + projectId: environment.projectId, + projectRef: environment.project.externalRef, + environmentId: environment.id, + environmentType: environment.type, + }, + }); + const timeoutMs = deployment.status === "PENDING" ? env.DEPLOY_QUEUE_TIMEOUT_MS : env.DEPLOY_TIMEOUT_MS; diff --git a/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts b/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts new file mode 100644 index 000000000..8d6727cac --- /dev/null +++ b/apps/webapp/app/v3/services/recordDeploymentFinished.server.ts @@ -0,0 +1,183 @@ +import { ROOT_CONTEXT, SpanStatusCode } from "@opentelemetry/api"; +import { type WorkerDeployment, type WorkerDeploymentStatus } from "@trigger.dev/database"; +import { logger } from "~/services/logger.server"; +import { SEMINTATTRS_FORCE_RECORDING, tracer } from "~/v3/tracer.server"; +import { + DeploymentTelemetryAttributes as ATTRS, + deriveBuildPath, + deriveDeploymentDurations, +} from "~/v3/deploymentTelemetry"; + +type TerminalDeploymentStatus = Extract< + WorkerDeploymentStatus, + "DEPLOYED" | "FAILED" | "TIMED_OUT" | "CANCELED" +>; + +type FinishedDeployment = Pick< + WorkerDeployment, + | "friendlyId" + | "version" + | "type" + | "status" + | "createdAt" + | "startedAt" + | "installedAt" + | "builtAt" + | "deployedAt" + | "failedAt" + | "canceledAt" + | "canceledReason" + | "errorData" + | "runtime" + | "runtimeVersion" + | "cliVersion" + | "triggeredVia" + | "commitSHA" +> & { buildServerMetadata: unknown }; + +type EnvironmentInfo = { + organizationId?: string; + projectId?: string; + projectRef?: string; + environmentId?: string; + environmentType?: string; +}; + +/** + * Records a deployment's terminal transition as a single wide + * `deployment.finished` span, backdated createdAt → terminal (attribute + * contract in ../deploymentTelemetry.ts). Call exactly once, only after a + * guarded status write confirmed this caller won the transition. Emitted on + * ROOT_CONTEXT with forceRecording so the sampler can never drop it; never + * throws. + */ +export function recordDeploymentFinished(params: { + status: TerminalDeploymentStatus; + deployment: FinishedDeployment; + environment: EnvironmentInfo; + reason?: string; +}): void { + try { + const { status, deployment, environment, reason } = params; + + const isFailure = status === "FAILED" || status === "TIMED_OUT"; + const terminalAt = + deployment.deployedAt ?? deployment.failedAt ?? deployment.canceledAt ?? new Date(); + const durations = deriveDeploymentDurations(deployment, terminalAt); + const errorData = parseErrorData(deployment.errorData); + + const span = tracer.startSpan( + "deployment.finished", + { + startTime: deployment.createdAt, + attributes: { + [SEMINTATTRS_FORCE_RECORDING]: true, + [ATTRS.ORG_ID]: environment.organizationId, + [ATTRS.PROJECT_ID]: environment.projectId, + [ATTRS.PROJECT_REF]: environment.projectRef, + [ATTRS.ENV_ID]: environment.environmentId, + [ATTRS.ENV_TYPE]: environment.environmentType, + [ATTRS.DEPLOYMENT_ID]: deployment.friendlyId, + [ATTRS.VERSION]: deployment.version, + [ATTRS.STATUS]: status, + [ATTRS.SUCCESS]: status === "DEPLOYED", + [ATTRS.BUILD_PATH]: deriveBuildPath(deployment.buildServerMetadata), + [ATTRS.WORKER_TYPE]: deployment.type, + [ATTRS.RUNTIME]: deployment.runtime ?? undefined, + [ATTRS.RUNTIME_VERSION]: deployment.runtimeVersion ?? undefined, + [ATTRS.CLI_VERSION]: deployment.cliVersion ?? undefined, + [ATTRS.TRIGGERED_VIA]: deployment.triggeredVia ?? undefined, + [ATTRS.COMMIT_SHA]: deployment.commitSHA ?? undefined, + [ATTRS.ERROR_NAME]: isFailure ? errorData?.name : undefined, + [ATTRS.ERROR_MESSAGE]: isFailure ? (reason ?? errorData?.message) : undefined, + [ATTRS.CANCELED_REASON]: deployment.canceledReason ?? undefined, + [ATTRS.DURATION_TOTAL_MS]: durations.totalMs, + [ATTRS.DURATION_QUEUE_MS]: durations.queueMs, + [ATTRS.DURATION_INSTALL_MS]: durations.installMs, + [ATTRS.DURATION_BUILDING_MS]: durations.buildingMs, + [ATTRS.DURATION_DEPLOYING_MS]: durations.deployingMs, + }, + }, + ROOT_CONTEXT + ); + + // CANCELED is deliberately not an error: it stays out of failure rates + if (isFailure) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: reason ?? errorData?.message, + }); + } + + span.end(terminalAt); + } catch (error) { + logger.debug("recordDeploymentFinished failed", { + deploymentFriendlyId: params.deployment.friendlyId, + error: error instanceof Error ? error.message : String(error), + }); + } +} + +/** + * Records a deployment's creation as a zero-duration `deployment.initialized` + * event — the funnel counterpart to `deployment.finished` for detecting + * stuck deployments. Never throws. + */ +export function recordDeploymentInitialized(params: { + deployment: Pick< + WorkerDeployment, + | "friendlyId" + | "version" + | "type" + | "status" + | "createdAt" + | "runtime" + | "cliVersion" + | "triggeredVia" + > & { buildServerMetadata: unknown }; + environment: EnvironmentInfo; +}): void { + try { + const { deployment, environment } = params; + + const span = tracer.startSpan( + "deployment.initialized", + { + startTime: deployment.createdAt, + attributes: { + [SEMINTATTRS_FORCE_RECORDING]: true, + [ATTRS.ORG_ID]: environment.organizationId, + [ATTRS.PROJECT_ID]: environment.projectId, + [ATTRS.PROJECT_REF]: environment.projectRef, + [ATTRS.ENV_ID]: environment.environmentId, + [ATTRS.ENV_TYPE]: environment.environmentType, + [ATTRS.DEPLOYMENT_ID]: deployment.friendlyId, + [ATTRS.VERSION]: deployment.version, + [ATTRS.STATUS]: deployment.status, + [ATTRS.BUILD_PATH]: deriveBuildPath(deployment.buildServerMetadata), + [ATTRS.WORKER_TYPE]: deployment.type, + [ATTRS.RUNTIME]: deployment.runtime ?? undefined, + [ATTRS.CLI_VERSION]: deployment.cliVersion ?? undefined, + [ATTRS.TRIGGERED_VIA]: deployment.triggeredVia ?? undefined, + }, + }, + ROOT_CONTEXT + ); + + span.end(deployment.createdAt); + } catch (error) { + logger.debug("recordDeploymentInitialized failed", { + deploymentFriendlyId: params.deployment.friendlyId, + error: error instanceof Error ? error.message : String(error), + }); + } +} + +function parseErrorData(errorData: unknown): { name?: string; message?: string } | undefined { + if (!errorData || typeof errorData !== "object") return undefined; + const record = errorData as Record; + return { + name: typeof record.name === "string" ? record.name : undefined, + message: typeof record.message === "string" ? record.message : undefined, + }; +} diff --git a/apps/webapp/app/v3/services/recordDeploymentOutcome.server.ts b/apps/webapp/app/v3/services/recordDeploymentOutcome.server.ts deleted file mode 100644 index e66a7a6a9..000000000 --- a/apps/webapp/app/v3/services/recordDeploymentOutcome.server.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { SpanStatusCode } from "@opentelemetry/api"; -import { type WorkerDeploymentStatus } from "@trigger.dev/database"; -import { logger } from "~/services/logger.server"; -import { tracer } from "~/v3/tracer.server"; - -type TerminalDeploymentStatus = Extract< - WorkerDeploymentStatus, - "DEPLOYED" | "FAILED" | "TIMED_OUT" ->; - -/** - * Records a deployment's terminal status as a `deployment.outcome` span so - * deploy success/failure is queryable from traces (no DB read). Call after each - * terminal-status write. Org/project/env are best-effort; never throws. - */ -export function recordDeploymentOutcome(params: { - status: TerminalDeploymentStatus; - deploymentFriendlyId: string; - organizationId?: string; - projectId?: string; - environmentId?: string; - environmentType?: string; - reason?: string; -}): void { - try { - const span = tracer.startSpan("deployment.outcome", { - attributes: { - "$trigger.org.id": params.organizationId, - "$trigger.project.id": params.projectId, - "$trigger.env.id": params.environmentId, - "$trigger.env.type": params.environmentType, - "deployment.outcome.status": params.status, - "deployment.outcome.success": params.status === "DEPLOYED", - "deployment.outcome.deployment_id": params.deploymentFriendlyId, - "deployment.outcome.reason": params.reason, - }, - }); - - if (params.status !== "DEPLOYED") { - span.setStatus({ code: SpanStatusCode.ERROR, message: params.reason }); - } - - span.end(); - } catch (error) { - logger.debug("recordDeploymentOutcome failed", { - deploymentFriendlyId: params.deploymentFriendlyId, - error: error instanceof Error ? error.message : String(error), - }); - } -} diff --git a/apps/webapp/app/v3/services/timeoutDeployment.server.ts b/apps/webapp/app/v3/services/timeoutDeployment.server.ts index 5e417a786..63d81ba19 100644 --- a/apps/webapp/app/v3/services/timeoutDeployment.server.ts +++ b/apps/webapp/app/v3/services/timeoutDeployment.server.ts @@ -5,7 +5,7 @@ import { commonWorker } from "../commonWorker.server"; import { PerformDeploymentAlertsService } from "./alerts/performDeploymentAlerts.server"; import { type PrismaClientOrTransaction } from "~/db.server"; import { DeploymentService } from "./deployment.server"; -import { recordDeploymentOutcome } from "./recordDeploymentOutcome.server"; +import { recordDeploymentFinished } from "./recordDeploymentFinished.server"; export class TimeoutDeploymentService extends BaseService { public async call(id: string, fromStatus: string, errorMessage: string) { @@ -38,25 +38,49 @@ export class TimeoutDeploymentService extends BaseService { return; } - const timedOutDeployment = await this._prisma.workerDeployment.update({ + const failedAt = new Date(); + const errorData = { message: errorMessage, name: "TimeoutError" }; + + // Guarded: keeps the fromStatus check atomic with the write + const { count: updatedCount } = await this._prisma.workerDeployment.updateMany({ where: { id: deployment.id, + status: deployment.status, }, data: { status: "TIMED_OUT", - failedAt: new Date(), - errorData: { message: errorMessage, name: "TimeoutError" }, + failedAt, + errorData, buildEnvVars: Prisma.DbNull, }, }); - recordDeploymentOutcome({ + if (updatedCount === 0) { + logger.warn("Deployment moved out of the expected state concurrently, skipping timeout", { + id: deployment.id, + fromStatus, + }); + return; + } + + const timedOutDeployment = { + ...deployment, + status: "TIMED_OUT" as const, + failedAt, + errorData, + buildEnvVars: null, + }; + + recordDeploymentFinished({ status: "TIMED_OUT", - deploymentFriendlyId: deployment.friendlyId, - organizationId: deployment.environment.project.organizationId, - projectId: deployment.environment.projectId, - environmentId: deployment.environmentId, - environmentType: deployment.environment.type, + deployment: timedOutDeployment, + environment: { + organizationId: deployment.environment.project.organizationId, + projectId: deployment.environment.projectId, + projectRef: deployment.environment.project.externalRef, + environmentId: deployment.environmentId, + environmentType: deployment.environment.type, + }, reason: errorMessage, }); diff --git a/apps/webapp/app/v3/tracer.server.ts b/apps/webapp/app/v3/tracer.server.ts index cbf9a937c..7047d65d2 100644 --- a/apps/webapp/app/v3/tracer.server.ts +++ b/apps/webapp/app/v3/tracer.server.ts @@ -36,7 +36,9 @@ import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-proto"; import { BatchSpanProcessor, ParentBasedSampler, + type ReadableSpan, type Sampler, + type Span as SdkTraceSpan, SamplingDecision, type SamplingResult, SimpleSpanProcessor, @@ -69,7 +71,7 @@ import { metricsRegister } from "~/metrics.server"; import { collectDatabaseClientMetrics } from "~/utils/databaseMetrics.server"; import { performance } from "node:perf_hooks"; -const SEMINTATTRS_FORCE_RECORDING = "forceRecording"; +export const SEMINTATTRS_FORCE_RECORDING = "forceRecording"; export const DATASOURCE_CONTEXT_KEY = createContextKey("trigger.db.datasource"); @@ -89,6 +91,29 @@ class DatasourceAttributeSpanProcessor implements SpanProcessor { } } +// Mirrors name-prefixed spans into a second exporter; they still flow to the main one +class SpanNamePrefixMirrorProcessor implements SpanProcessor { + constructor( + private readonly _inner: SpanProcessor, + private readonly _prefix: string + ) {} + + onStart(span: SdkTraceSpan, parentContext: Context): void { + this._inner.onStart(span, parentContext); + } + onEnd(span: ReadableSpan): void { + if (span.name.startsWith(this._prefix)) { + this._inner.onEnd(span); + } + } + shutdown(): Promise { + return this._inner.shutdown(); + } + forceFlush(): Promise { + return this._inner.forceFlush(); + } +} + class CustomWebappSampler implements Sampler { constructor(private readonly _baseSampler: Sampler) {} @@ -270,6 +295,30 @@ function setupTelemetry() { } } + if (env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL) { + const deploymentEventExporter = new OTLPTraceExporter({ + url: env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL, + timeoutMillis: 15_000, + headers: parseInternalDeploymentEventHeaders() ?? {}, + }); + + spanProcessors.push( + new SpanNamePrefixMirrorProcessor( + new BatchSpanProcessor(deploymentEventExporter, { + maxExportBatchSize: 64, + scheduledDelayMillis: 1000, + exportTimeoutMillis: 30000, + maxQueueSize: 2048, + }), + "deployment." + ) + ); + + console.log( + `🔦 Tracer: deployment-event exporter enabled to ${env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_URL}` + ); + } + const ratioSampler = new TraceIdRatioBasedSampler(samplingRate); const provider = new NodeTracerProvider({ @@ -341,6 +390,13 @@ function setupTelemetry() { instrumentations, }); + // Without this flush every shutdown drops the last batch of spans + const flushOnShutdown = () => { + provider.forceFlush().catch(() => {}); + }; + process.once("SIGTERM", flushOnShutdown); + process.once("SIGINT", flushOnShutdown); + return { tracer: provider.getTracer("trigger.dev", "3.3.12"), logger: logs.getLogger("trigger.dev", "3.3.12"), @@ -874,6 +930,19 @@ function parseInternalTraceHeaders(): Record | undefined { } } +function parseInternalDeploymentEventHeaders(): Record | undefined { + try { + return env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_AUTH_HEADERS + ? (JSON.parse(env.INTERNAL_OTEL_DEPLOYMENT_EVENT_EXPORTER_AUTH_HEADERS) as Record< + string, + string + >) + : undefined; + } catch { + return; + } +} + function parseInternalMetricsHeaders(): Record | undefined { try { return env.INTERNAL_OTEL_METRIC_EXPORTER_AUTH_HEADERS diff --git a/apps/webapp/test/deploymentTelemetry.test.ts b/apps/webapp/test/deploymentTelemetry.test.ts new file mode 100644 index 000000000..9020f0c71 --- /dev/null +++ b/apps/webapp/test/deploymentTelemetry.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from "vitest"; +import { deriveBuildPath, deriveDeploymentDurations } from "~/v3/deploymentTelemetry"; + +describe("deriveBuildPath", () => { + it("classifies fromBundle native builds as native_local_bundle", () => { + expect(deriveBuildPath({ isNativeBuild: true, fromBundle: true })).toBe("native_local_bundle"); + }); + + it("classifies native builds without fromBundle as native", () => { + expect(deriveBuildPath({ isNativeBuild: true })).toBe("native"); + expect(deriveBuildPath({ isNativeBuild: true, fromBundle: false })).toBe("native"); + }); + + it("classifies everything else as depot", () => { + expect(deriveBuildPath(null)).toBe("depot"); + expect(deriveBuildPath(undefined)).toBe("depot"); + expect(deriveBuildPath({})).toBe("depot"); + expect(deriveBuildPath({ buildId: "depot-build-id" })).toBe("depot"); + expect(deriveBuildPath({ isNativeBuild: false })).toBe("depot"); + // fromBundle alone (skewed writer) must not count as native_local_bundle + expect(deriveBuildPath({ fromBundle: true })).toBe("depot"); + expect(deriveBuildPath("garbage")).toBe("depot"); + }); +}); + +describe("deriveDeploymentDurations", () => { + const t = (seconds: number) => new Date(1_700_000_000_000 + seconds * 1000); + + it("derives all phases for the full build-server chain", () => { + const durations = deriveDeploymentDurations( + { createdAt: t(0), startedAt: t(10), installedAt: t(40), builtAt: t(100) }, + t(130) + ); + + expect(durations).toEqual({ + totalMs: 130_000, + queueMs: 10_000, + installMs: 30_000, + buildingMs: 60_000, + deployingMs: 30_000, + }); + }); + + it("omits install and measures building from startedAt when installedAt is missing (depot)", () => { + const durations = deriveDeploymentDurations( + { createdAt: t(0), startedAt: t(0), installedAt: null, builtAt: t(90) }, + t(120) + ); + + expect(durations).toEqual({ + totalMs: 120_000, + queueMs: 0, + installMs: undefined, + buildingMs: 90_000, + deployingMs: 30_000, + }); + }); + + it("omits phases whose boundaries are missing (failed before building)", () => { + const durations = deriveDeploymentDurations( + { createdAt: t(0), startedAt: t(5), installedAt: null, builtAt: null }, + t(20) + ); + + expect(durations).toEqual({ + totalMs: 20_000, + queueMs: 5_000, + installMs: undefined, + buildingMs: undefined, + deployingMs: undefined, + }); + }); + + it("never returns negative durations on clock skew", () => { + const durations = deriveDeploymentDurations( + { createdAt: t(10), startedAt: t(5), installedAt: null, builtAt: null }, + t(3) + ); + + expect(durations.totalMs).toBe(0); + expect(durations.queueMs).toBeUndefined(); + }); +}); diff --git a/internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql b/internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql new file mode 100644 index 000000000..931ea9479 --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260825120000_add_worker_deployment_cli_version/migration.sql @@ -0,0 +1 @@ +ALTER TABLE "public"."WorkerDeployment" ADD COLUMN IF NOT EXISTS "cliVersion" TEXT; diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index a77890930..a1423f340 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -2265,6 +2265,8 @@ model WorkerDeployment { runtime String? runtimeVersion String? + /// CLI version that initiated the deploy, stamped at initialization + cliVersion String? imageReference String? imagePlatform String @default("linux/amd64") diff --git a/packages/cli-v3/src/apiClient.ts b/packages/cli-v3/src/apiClient.ts index 8b9fd56eb..a7a07cf40 100644 --- a/packages/cli-v3/src/apiClient.ts +++ b/packages/cli-v3/src/apiClient.ts @@ -1016,6 +1016,7 @@ export class CliApiClient { Authorization: `Bearer ${this.accessToken}`, "Content-Type": "application/json", "x-trigger-source": this.source, + "x-trigger-cli-version": VERSION, ...this.getBranchHeader(), }; }